Replace build-submodules with Python based package builder

Support building individual VyOS packages by this modules. Call
scripts/build-packages -l to list all available packages which will be
build when invoking scripts/build-packages.

5 files changed, 554 insertions, 541 deletions

diff --git a/Jenkinsfile b/Jenkinsfile
index 4bc64b26..169a52f3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,22 +1,76 @@
 #!/usr/bin/env groovy
 
+// Copyright (C) 2018 VyOS maintainers and contributors
+//
+// This program is free software; you can redistribute it and/or modify
+// in order to easy exprort images built to "external" world
+// it under the terms of the GNU General Public License version 2 or later as
+// published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 @NonCPS
+
+def getGitBranchName() {
+    return scm.branches[0].name
+}
+
+def getGitRepoURL() {
+    return scm.userRemoteConfigs[0].url
+}
+
+// Returns true if this is a custom build launched on any project fork,
+// returns false if this is build from git@github.com:vyos/vyos-build.git
+def isCustomBuild() {
+    def gitURI = "git@github.com:vyos/vyos-build.git"
+    def httpURI = "https://github.com/vyos/vyos-build.git"
+
+    return ! ((getGitRepoURL() == gitURI) || (getGitRepoURL() == httpURI))
+}
+
 def setDescription() {
     def item = Jenkins.instance.getItemByFullName(env.JOB_NAME)
-    item.setDescription("VyOS image build using a\nPipeline build inside Docker container.")
+
+    // build up the main description text
+    def description = ""
+    description += "<h2>Build VyOS ISO image</h2>"
+    description += "All required Vyatta/VyOS packages are build from source prior to assembling the ISO."
+
+    if (isCustomBuild()) {
+        description += "<p style='border: 3px dashed red; width: 50%;'>"
+        description += "<b>Build not started from official Git repository!</b><br>"
+        description += "<br>"
+        description += "Repository: <font face = 'courier'>" + getGitRepoURL() + "</font><br>"
+        description += "Branch: <font face = 'courier'>" + getGitBranchName() + "</font><br>"
+        description += "</p>"
+    } else {
+        description += "Sources taken from Git branch: <font face = 'courier'>" + getGitBranchName() + "</font><br>"
+    }
+
+    item.setDescription(description)
     item.save()
 }
 
-setDescription()
-
 /* Only keep the 10 most recent builds. */
 def projectProperties = [
-    [$class: 'BuildDiscarderProperty',strategy: [$class: 'LogRotator', numToKeepStr: '5']],
+    [$class: 'BuildDiscarderProperty',strategy: [$class: 'LogRotator', numToKeepStr: '1']],
 ]
 
 properties(projectProperties)
+setDescription()
 
 pipeline {
+    options {
+        disableConcurrentBuilds()
+        timeout(time: 4, unit: 'HOURS')
+        parallelsAlwaysFailFast()
+    }
     agent {
         dockerfile {
             filename 'Dockerfile'
@@ -27,23 +81,71 @@ pipeline {
     }
 
     stages {
-        stage('Submodule Init') {
+        stage('VyOS Packages') {
             steps {
-                sh '''
-                    git submodule update --init --recursive --remote
-                '''
+                script {
+                    def build = [:]
+                    // get a list of available package from scripts/build-packages
+                    packageList = sh(
+                        script: "scripts/build-packages -l | grep '*' | sed -e 's/ \\* //'",
+                        returnStdout: true
+                    ).split("\r?\n")
+
+                    packageList.each { pkg ->
+                        skipList = ['vyos-kernel', 'vyos-wireguard', 'vyos-accel-ppp']
+                        if (pkg in skipList) {
+                            return
+                        }
+
+                        // add each object from 'packageList' to the 'build' array
+                        build[pkg] = {
+                            // we're already in the script{} block, so do our advanced stuff here
+                            sh(
+                                script: "scripts/build-packages -vvv -b ${pkg}",
+                                returnStdout: true
+                            )
+                        }
+                    }
+                    // Still within the 'Script' block, run the parallel array object
+                    parallel build
+                }
             }
         }
-        stage('Build Packages') {
+
+        stage('Kernel') {
             steps {
-                sh '''
-                    #!/bin/sh
-                    scripts/build-submodules --verbose
-                '''
+                sh "scripts/build-packages -vvv -b vyos-kernel"
+            }
+        }
+
+        stage('Kernel Modules') {
+            steps {
+                script {
+                    def build = [:]
+                    kernelModules = ['vyos-wireguard', 'vyos-accel-ppp']
+                    kernelModules.each { pkg ->
+                        // add each object from 'packageList' to the 'build' array
+                        build[pkg] = {
+                            // we're already in the script{} block, so do our advanced stuff here
+                            sh(
+                                script: "scripts/build-packages -vvv -b ${pkg}",
+                                returnStdout: true
+                            )
+                        }
+                    }
+                    // Still within the 'Script' block, run the parallel array object
+                    parallel build
+                }
+            }
+        }
+
+        stage('Intel Drivers') {
+            steps {
+                sh "KSRC=\$(pwd)/packages/vyos-kernel scripts/build-intel-drivers"
             }
         }
 
-        stage('Build ISO') {
+        stage('ISO Image') {
             steps {
                 sh '''
                     #!/bin/sh
diff --git a/README.md b/README.md
index d4cd6c97..655994c8 100644
--- a/README.md
+++ b/README.md
@@ -106,47 +106,18 @@ Run following command after building the QEMU image.
 $ make vmware
 ```
 
-## Building the ISO image inside a docker container
+# Building ISO or individual packages via Docker
 
 Using our [Dockerfile](docker/Dockerfile) you create your own Docker container
 that is used to build a VyOS ISO image or other required VyOS packages. The
 [Dockerfile](docker/Dockerfile) contains some of the most used packages needed
-to build a VyOS ISO, a qemu image, and several of the submodules. Please note
-that this is not complete and only gives you a brief overview!
+to build a VyOS ISO, a qemu image, and several of the submodules.
 
-```
-squashfs-tools           # Required for squashfs file system
-git                      # Required, for cloning the source
-autoconf                 # Required, for generating build scripts
-dpkg-dev                 # Required, used in build scripts
-live-build               # Required, for ISO build
-syslinux                 # Required, for ISO build
-genisoimage              # Required, for ISO build
-make                     # Required, for ISO build
-lsb-release              # Required, used by configure script
-fakechroot               # Required, for ISO build
-devscripts               # Optional, for building submodules (kernel etc)
-kernel-package           # Optional, for building the kernel
-libtool                  # Optional, for building certain packages (vyatta-op-vpn)
-libglib2.0-dev           # Optional, for building vyatta-cfg
-libboost-filesystem-dev  # Optional, for building vyatta-cfg
-libapt-pkg-dev           # Optional, for building vyatta-cfg
-flex                     # Optional, for building vyatta-cfg
-bison                    # Optional, for building vyatta-cfg
-libperl-dev              # Optional, for building vyatta-cfg
-libnfnetlink-dev         # Optional, for building vyatta-cfg-vpn
-vim                      # Optional, vim, vi, nano or other text editor
-jq                       # Optional, for qemu build
-qemu-system-x86          # Optional, for qemu build
-qemu-utils               # Optional, for qemu build
-packer                   # Optional, for qemu build
-quilt                    # Optional, for building vyos-1x
-python3-lxml             # Optional, for building vyos-1x
-python3-setuptools       # Optional, for building vyos-1x
-python3-nose             # Optional, for building vyos-1x
-python3-coverage         # Optional, for building vyos-1x
-...
-```
+If you are interested which individual packages are required please check our
+[Dockerfile](docker/Dockerfile) which holds the most complete documentation
+of required packages. Listing individual packages here tend to be always
+outdated. We try to list required packages in groups through their inheritance
+in the [Dockerfile](docker/Dockerfile).
 
 To build the docker image ensure you have a working [Docker](https://www.docker.com)
 environment and then run the following commands:
@@ -155,7 +126,7 @@ environment and then run the following commands:
 $ docker build -t vyos-builder docker
 ```
 
-Run the newly built container:
+Run newly built container:
 ```bash
 $ docker run --rm -it --privileged -v $(pwd):/vyos -w /vyos vyos-builder bash
 ```
@@ -183,32 +154,27 @@ order to build the VyOS ISO image.
 
 ## Building subpackages inside Docker
 
-Prior to building packages you need to checkout and update the submodules you want to compile
-
-```bash
-$ git submodule update --init packages/PACKAGENAME
-$ cd packages/PACKAGENAME
-$ git checkout BRANCH
-```
-
-`PACKAGENAME` is the name of the package you want to compile
-`BRANCH` is `crux` for VyOS 1.2.x, latest rolling releases use `current`
-
-Fetching all submodules at once and update them to the recent remote branches
-`HEAD` is done by calling:
+[scripts/build-packages](scripts/build-packages) provides an easy interface
+for automate the process of building all VyOS related packages that are not part
+of the upstream Debian version. Execute it in the root of the `vyos-build`
+directory to start compilation.
 
 ```bash
-$ git submodule update --init --recursive
-$ git submodule update --remote
+$ scripts/build-packages -h
+usage: build-packages [-h] [-v] [-c] [-l] [-b BUILD [BUILD ...]] [-f]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -v, --verbose         Increase logging verbosity for each occurance
+  -c, --clean           Re-clone required Git repositories
+  -l, --list-packages   List all packages to build
+  -b BUILD [BUILD ...], --build BUILD [BUILD ...]
+                        Whitespace separated list of packages to build
+  -f, --fetch           Fetch sources only, no build
 ```
 
-### Building packages
-
-The [scripts/build-submodules](scripts/build-submodules) script is used to
-automate the process of building (in the future) all VyOS related packages that
-are not part of the upstream Debian version. Execute it in the root of the
-`vyos-build` directory to start compilation on all supported packages that are
-checked out.
+Git repositoriers are automatically fetched and build on demand. If you wan't to
+work offline you can fetch all source code first with the `-f` option.
 
 The easiest way to compile is with the above mentioned [Docker](docker/Dockerfile)
 container, it includes all dependencies for compiling supported packages.
@@ -217,7 +183,7 @@ container, it includes all dependencies for compiling supported packages.
 $ docker run --rm -it -v $(pwd):/vyos -w /vyos \
              --sysctl net.ipv6.conf.lo.disable_ipv6=0 \
              vyos-builder \
-             ./scripts/build-submodules
+             ./scripts/build-packages
 ```
 
 **NOTE:** `--sysctl net.ipv6.conf.lo.disable_ipv6=0` is required to build the
@@ -237,29 +203,15 @@ Executed from the root of `vyos-build`
 $ docker run --rm -it -v $(pwd):/vyos -w /vyos/packages/PACKAGENAME \
              --sysctl net.ipv6.conf.lo.disable_ipv6=0 \
              vyos-builder \
-             dpkg-buildpackage -uc -us -tc -b
+             ./scripts/build-packages -b <package>
 ```
 
 **NOTE:** `--sysctl net.ipv6.conf.lo.disable_ipv6=0` is only needed when
 building `vyos-strongswan` and can be ignored on other packages.
 
-**NOTE:** Prior to executing this you need to checkout and update the submodules
-you want to recompile!
-
 **NOTE:** `vyos-strongswan` will only compile on a Linux system, running on macOS
 or Windows might result in a unittest deadlock (it never exits).
 
-Packages that are known to not build using this procedure (as of now):
-
-```
-vyatta-util     - Not needed anymore
-vyatta-quagga   - Not needed anymore
-vyos-1x         - Unmet build dependencies: whois libvyosconfig0
-vyos-frr        - A lot of requirements, scary stuff...
-vyos-kernel     - Need special build instructions
-vyos-wireguard  - Needs special build instructions
-```
-
 # Development process
 
 ## Git branches
diff --git a/scripts/build-intel-drivers b/scripts/build-intel-drivers
new file mode 100755
index 00000000..07c71f88
--- /dev/null
+++ b/scripts/build-intel-drivers
@@ -0,0 +1,79 @@
+#!/bin/bash -x
+
+basedir=$(pwd)
+if [ -z "$KSRC" ]; then
+    echo "Kernel source KSRC= not specified"
+    exit 1
+fi
+
+VERSION=$(grep "^VERSION" $KSRC/Makefile | grep -Eo '[0-9]{1,4}')
+PATCHLEVEL=$(grep "^PATCHLEVEL" $KSRC/Makefile | grep -Eo '[0-9]{1,4}')
+SUBLEVEL=$(grep "^SUBLEVEL" $KSRC/Makefile | grep -Eo '[0-9]{1,4}')
+KERNEL_VER="$VERSION.$PATCHLEVEL.$SUBLEVEL-amd64-vyos"
+
+pkgdir="$basedir/packages"
+if [ ! -d "$basedir/packages" ]; then
+    mkdir $basedir/packages
+fi
+
+URLS=" \
+    https://downloadmirror.intel.com/14687/eng/ixgbe-5.5.5.tar.gz \
+    https://downloadmirror.intel.com/13663/eng/igb-5.3.5.22.tar.gz \
+    https://downloadmirror.intel.com/15817/eng/e1000e-3.4.2.1.tar.gz \
+    https://downloadmirror.intel.com/24411/eng/i40e-2.7.29.tar.gz \
+    https://downloadmirror.intel.com/18700/eng/ixgbevf-4.5.2.tar.gz \
+    https://downloadmirror.intel.com/24693/eng/i40evf-3.6.15.tar.gz \
+"
+
+# The intel IGBVF driver can't be compiled with recent Kernel versions
+# due to interlanl API changes. Driver is from 10.09.2015
+# Source code is here: https://downloadmirror.intel.com/18298/eng/igbvf-2.3.9.6.tar.gz
+for URL in $URLS
+do
+    cd $pkgdir
+    filename=${URL##*/}
+    dirname_full=$(echo $filename | awk -F".tar.gz" '{print $1}')
+    dirname=$(echo $dirname_full | awk -F- '{print $1}')
+    version="$(echo $dirname_full | awk -F- '{print $2}')-0"
+    deb_pkg=${dirname}_${version}_amd64
+    deb_pkg_dir=$pkgdir/vyos-intel-${deb_pkg}
+
+    if [ -d "$dirname_full" ]; then
+        rm -rf $dirname_full
+    fi
+    if [ -d "$deb_pkg_dir" ]; then
+        rm -rf $deb_pkg_dir
+    fi
+    if [ ! -f "$filename" ]; then
+        wget ${URL}
+        ret=$?
+        if [ "$ret" != "0" ]; then
+            echo "Download of ${URL} failed!"
+            exit $ret
+        fi
+    fi
+
+    tar xf $filename
+    cd $dirname_full/src
+
+    KSRC=$KSRC INSTALL_MOD_PATH=$deb_pkg_dir \
+        make -j $(cat /proc/cpuinfo | grep processor | wc -l) install
+
+    mkdir -p $deb_pkg_dir/DEBIAN
+    echo "Package: vyos-intel-$dirname" >$deb_pkg_dir/DEBIAN/control
+    echo "Version: $version" >>$deb_pkg_dir/DEBIAN/control
+    echo "Section: kernel" >>$deb_pkg_dir/DEBIAN/control
+    echo "Priority: extra" >>$deb_pkg_dir/DEBIAN/control
+    echo "Architecture: amd64" >>$deb_pkg_dir/DEBIAN/control
+    echo "Maintainer: VyOS Package Maintainers <maintainers@vyos.net>" >>$deb_pkg_dir/DEBIAN/control
+    echo "Installed-Size: 9" >>$deb_pkg_dir/DEBIAN/control
+    echo "Depends: linux-image" >>$deb_pkg_dir/DEBIAN/control
+    echo "Description: Intel Vendor driver for $dirname" >>$deb_pkg_dir/DEBIAN/control
+    echo "  Replacement for the in Kernel drivers" >>$deb_pkg_dir/DEBIAN/control
+
+    # Cleanup files which might also be present in linux-image-4.19.20-amd64-vyos
+    rm -rf $deb_pkg_dir/usr $deb_pkg_dir/lib/modules/$KERNEL_VER/modules.*
+
+    cd $pkgdir
+    dpkg-deb --build $(basename $deb_pkg_dir)
+done
diff --git a/scripts/build-packages b/scripts/build-packages
new file mode 100755
index 00000000..e800b80a
--- /dev/null
+++ b/scripts/build-packages
@@ -0,0 +1,332 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+import sys
+import shutil
+import argparse
+import logging
+
+current_working_directory = os.getcwd()
+repo_root = subprocess.check_output('git rev-parse --show-toplevel', shell=True, universal_newlines=True).rstrip('\n')
+repo_sha  = subprocess.check_output('git rev-parse --short=12 HEAD', shell=True, universal_newlines=True).rstrip('\n')
+
+def add_package(name, url=None, commit='HEAD', branch='current', tag=None, custombuild_cmd=None):
+    """
+    Build up source package with URL and build commands executed during the later
+    called build_package step.
+
+    If no additional information is passed we will use the latest commit from current
+    branch
+
+    If no URL is passed we assume it's a regular VyOS package from the VyOS Github
+    namespace at https://github.com/vyos
+    """
+
+    if not url:
+        url = 'https://github.com/vyos/' + name + '.git'
+
+    package = {
+        'name': name,
+        'url': url,
+        'commit': commit,
+        'tag': tag,
+        'branch': branch,
+        'path': repo_root + '/packages/' + name,
+        'custombuild_cmd': custombuild_cmd
+    }
+
+    return package
+
+
+
+def call(bashCommand, log, **kwargs):
+    """
+    Run command with arguments. Wait for command to complete.
+
+    Send output to logging module passed as 'log'.
+    """
+
+    from subprocess import Popen, PIPE, STDOUT, check_output, CalledProcessError
+    from tempfile import TemporaryFile
+    from time import sleep
+
+    log.debug("Executing '{}'".format(bashCommand))
+
+    # code borrowsed from:
+    # https://stackoverflow.com/questions/38374063/python-can-we-use-tempfile-with-subprocess-to-get-non-buffering-live-output-in-p
+    # the temp file will be automatically cleaned up
+    output = TemporaryFile()
+    error = TemporaryFile()
+
+    kwargs['stdout'] = output
+    kwargs['stderr'] = error
+    kwargs['shell'] = True
+    kwargs['universal_newlines'] = True
+
+    sub = Popen(bashCommand, **kwargs)
+    while sub.poll() is None:
+        where = output.tell()
+        lines = output.readline()
+        if not lines:
+            sleep(0.3)
+            output.seek(where)
+        else:
+            log.debug(lines.decode().rstrip('\n'))
+
+        where = error.tell()
+        lines = error.readline()
+        if not lines:
+            sleep(0.3)
+            error.seek(where)
+        else:
+           log.info(lines.decode().rstrip('\n'))
+
+    error.close()
+    output.close()
+    return sub.returncode
+
+def clone_package(pkg, log):
+    """
+    Clone Git repository from URL embedded in pkg to local disk
+
+    First cleanup any possible leftovers from previous builds
+    """
+
+    if args.clean:
+        # delete repository from disk
+        if os.path.isdir(pkg['path']):
+            log.debug("Cleaning '{}'".format(pkg['path']))
+            shutil.rmtree(pkg['path'])
+    else:
+        if os.path.isdir(pkg['path']):
+            # Change current directory into Git repo for this package
+            os.chdir(pkg['path'])
+
+            bashCommand = 'git clean -d -x --force && git reset --hard ' + pkg['commit']
+            return call(bashCommand, log)
+
+    # resolve given tag to commit id to use shallow clone
+    bashCommand = 'git clone ' + pkg['url']
+    if pkg['tag']:
+        bashCommand += ' --branch ' + pkg['tag']
+    elif pkg['branch']:
+        bashCommand += ' --depth 1 --branch ' + pkg['branch']
+
+    bashCommand += ' ' + pkg['path']
+    return call(bashCommand, log)
+
+
+def build_package(pkg, log=None):
+    """
+    Generate Debian package from passed 'pkg'
+    """
+
+    # Change current directory into Git repo for this package
+    os.chdir(pkg['path'])
+
+    # Overwrite custom build command if required, e.g. libyang
+    bashCommand = ''
+    if pkg['custombuild_cmd']:
+        bashCommand = pkg['custombuild_cmd']
+    else:
+        # Build package
+        bashCommand = 'dpkg-buildpackage -uc -us -tc -b'
+        if args.parallel:
+            bashCommand += ' -j' + str(os.cpu_count())
+
+    return call(bashCommand, log)
+
+# a List of all Vyatta/VyOS based packages
+vyos_packages = ['vyatta-bash',
+                 'vyatta-cfg',
+                 'vyatta-op',
+                 'vyatta-cfg-system',
+                 'vyatta-cfg-firewall',
+                 'vyatta-op-firewall',
+                 'vyatta-cfg-vpn',
+                 'vyatta-op-vpn',
+                 'vyatta-cfg-qos',
+                 'vyatta-op-qos',
+                 'vyatta-cfg-op-pppoe',
+                 'vyatta-openvpn',
+                 'vyatta-conntrack',
+                 'vyatta-conntrack-sync',
+                 'vyatta-nat',
+                 'vyatta-config-mgmt',
+                 'vyatta-config-migrate',
+                 'vyatta-zone',
+                 'vyatta-cluster',
+                 'vyatta-eventwatch',
+                 'vyatta-webproxy',
+                 'vyatta-cfg-quagga',
+                 'vyatta-op-quagga',
+                 'vyatta-op-dhcp-server',
+                 'vyatta-wireless',
+                 'vyatta-wirelessmodem',
+                 'vyatta-wanloadbalance',
+                 'vyatta-netflow',
+                 'vyatta-lldp',
+                 'vyatta-ipv6-rtradv',
+                 'vyatta-ravpn',
+                 'vyos-nhrp',
+                 'vyos-world',
+                 'vyos-1x',
+                 'vyatta-iproute',
+                 'vyatta-quagga',
+                 'vyos-strongswan',
+                 'vyos-vmwaretools-scripts',
+                 'vyos-netplug',
+                 'vyos-xe-guest-utilities',
+                 'vyatta-biosdevname',
+                 'vyos-opennhrp',
+                 'vyos-salt-minion',
+                 'xl2tpd',
+                 'mdns-repeater',
+                 'udp-broadcast-relay',
+                 'pmacct',
+                 'ddclient',
+                 'igmpproxy',
+                 'eventwatchd',
+                 'conntrack-tools']
+
+# Special packages mean packages which are located no in the VyOS namespace
+# or require fancy build instructions
+pkg_special = []
+
+# libvyosconfig/ipaddrcheck uses a different default branch
+libvyosconfig_build_cmd = "eval $(opam env --root=/opt/opam --set-root) && " \
+                          "dpkg-buildpackage -b -us -uc -tc"
+pkg_special.append( add_package('libvyosconfig', branch='master', custombuild_cmd=libvyosconfig_build_cmd))
+pkg_special.append( add_package('ipaddrcheck', branch='master'))
+
+# Packages where we directly build the upstream source
+pkg_special.append( add_package('hvinfo', url='https://github.com/dmbaturin/hvinfo.git', branch='master') )
+pkg_special.append( add_package('lldpd',  url='https://github.com/vincentbernat/lldpd.git', branch='master', tag='1.0.3') )
+
+#
+# FreeRangeRouting (FRR) packages
+#
+pkg_special.append( add_package('rtrlib', url='https://github.com/rtrlib/rtrlib.git', branch='master', tag='v0.6.3') )
+
+frr_build_cmd  = './tools/tarsource.sh -V && dpkg-buildpackage -us -uc -Ppkg.frr.rtrlib -d'
+pkg_special.append( add_package('frr', url='https://github.com/FRRouting/frr.git', branch='master', tag='frr-7.0', custombuild_cmd=frr_build_cmd) )
+#libyang_build_cmd = 'mkdir build && cd build && cmake .. && make build-deb && mv debs/* ' + repo_root + '/packages'
+#pkg_special.append( add_package('libyang', url='https://github.com/opensourcerouting/libyang.git', commit='179da47', branch='master', custombuild_cmd=libyang_build_cmd) )
+
+
+#
+# Linux (VyOS) Kernel
+#
+kernel_build_cmd = "export VERSION=$(grep '^VERSION' Makefile | grep -Eo '[0-9]{1,4}') && " \
+                   "export PATCHLEVEL=$(grep '^PATCHLEVEL' Makefile | grep -Eo '[0-9]{1,4}') && " \
+                   "export SUBLEVEL=$(grep '^SUBLEVEL' Makefile | grep -Eo '[0-9]{1,4}') && " \
+                   "echo ${VERSION}.${PATCHLEVEL}.${SUBLEVEL}-amd64-vyos > " + repo_root + "/data/kernel_version && " \
+                   "make x86_64_vyos_defconfig && " \
+                   "LOCALVERSION='' make-kpkg --rootcmd fakeroot --initrd --append_to_version -amd64-vyos " \
+                   "  --revision=${VERSION}.${PATCHLEVEL}.${SUBLEVEL}-0 kernel_headers kernel_image -j" + str(os.cpu_count())
+pkg_special.append( add_package('vyos-kernel', branch='linux-vyos-4.19.y', custombuild_cmd=kernel_build_cmd) )
+
+
+
+#
+# WireGuard Kernel Module
+#
+wireguard_build_cmd = "echo 'src/wireguard.ko /lib/modules/'$(cat " + repo_root + "/data/kernel_version) > debian/wireguard-modules.install && " \
+                      "KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count())
+pkg_special.append( add_package('vyos-wireguard', custombuild_cmd=wireguard_build_cmd) )
+
+
+
+#
+# Accell-PPP Package and Kernel Module
+#
+accel_ppp_build_cmd = "echo 'lib/modules/'$(cat " + repo_root + "/data/kernel_version)'/extra/*.ko' > debian/vyos-accel-ppp-ipoe-kmod.install && " \
+                      "sed -i 's#[0-9].[0-9][0-9].[0-9]*-amd64-vyos#'$(cat " + repo_root + "/data/kernel_version)'#g' debian/rules && " \
+                      "KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count())
+pkg_special.append( add_package('vyos-accel-ppp', custombuild_cmd=accel_ppp_build_cmd) )
+
+
+# A list of all packages we will build in the end
+pkg_build = []
+
+
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-v', '--verbose', action='count', default=0, help='Increase logging verbosity for each occurance')
+    parser.add_argument('-c', '--clean', action='store_true', help='Re-clone required Git repositories')
+    parser.add_argument('-l', '--list-packages', action='store_true', help='List all packages to build')
+    parser.add_argument('-b', '--build', nargs='+', help='Whitespace separated list of packages to build')
+    parser.add_argument('-f', '--fetch', action='store_true', help='Fetch sources only, no build')
+    parser.add_argument('-p', '--parallel', action='store_true', help='Build on all CPUs')
+
+
+    args = parser.parse_args()
+
+    levels = [ logging.INFO, logging.WARNING, logging.DEBUG ]
+    level = levels[min(len(levels)-1,args.verbose)]  # capped to number of levels
+    logging.basicConfig(level=level, format="%(asctime)s %(name)s %(message)s")
+
+    print("Using vyos-build repository ('{}') commit '{}'\n".format(repo_root, repo_sha))
+
+    if args.list_packages:
+        print("Individual packages available for build:")
+        for pkg in vyos_packages:
+            print(' * ' + pkg)
+        for pkg in pkg_special:
+            print(' * ' + pkg['name'])
+
+        sys.exit(0)
+
+    #
+    # Only add selective packages to the build list
+    #
+    if args.build:
+        # NOTE: remove double added packages from list
+        for target in args.build:
+            if target in vyos_packages:
+                pkg_build.append(add_package( target ))
+                continue
+
+            found = False
+            for pkg in pkg_special:
+                if target == pkg['name']:
+                    found = True
+                    # package already formed
+                    pkg_build.append( pkg )
+                    break
+
+            if not found:
+                print("Invalid choice '" + target + "', for -b/--build use --list-packages for complete list!")
+                sys.exit(1)
+
+    else:
+        # Add all VyOS packages to the package list
+        for pkg in vyos_packages:
+            pkg_build.append(add_package( pkg ))
+
+        # We also wan't to build all of our special packages
+        for pkg in pkg_special:
+            pkg_build.append( pkg )
+
+    # Build all VyOS packages (packages found on https://github.com/vyos
+    # and referenced in vyos_packages)
+    for pkg in pkg_build:
+        # Create a logging instance per package
+        log = logging.getLogger(pkg['name'])
+
+        ret = clone_package(pkg, log)
+        if ret:
+            log.error("ERROR cloning source")
+            sys.exit(1)
+        else:
+            # only build packages if fetch flag is not set
+            if not args.fetch:
+                ret = build_package(pkg, log)
+                if ret:
+                    log.error("ERROR building source")
+                    sys.exit(1)
+
+    sys.exit(0)
diff --git a/scripts/build-submodules b/scripts/build-submodules
deleted file mode 100755
index fc8fdf83..00000000
--- a/scripts/build-submodules
+++ /dev/null
@@ -1,452 +0,0 @@
-#!/bin/bash
-
-if [ ! -d "packages" ]; then
-  echo "This script needs to be executed inside the top root of vyos-build"
-  exit 1
-fi
-
-print_help() {
-  echo "Script for building all subpackages to vyos"
-  echo "Execute this sctipt from the root of the vyos-build directory"
-  echo ""
-  echo "This script could be executed from a Debian Jessie installation with all dependencies"
-  echo "or from the vyos-builder docker container."
-  echo ""
-  echo "Build the container:"
-  echo "  docker build -t vyos-builder ."
-  echo "Compile packages:"
-  echo "  docker run --rm -it -v $(pwd):/vyos -w /vyos --sysctl net.ipv6.conf.lo.disable_ipv6=0 vyos-builder scripts/build-submodules"
-  echo ""
-  echo "Parameters:"
-  echo "  --init-packages      - Initiate all subpackages before building"
-  echo "  --verbose            - Enable verbose output"
-  echo "  --build-[packagename] - build only selected packages"
-}
-
-BUILDLIST=""
-VERBOSE=0
-
-while test $# -gt 0
-do
-  case "$1" in
-    -h|-?|--help)
-      print_help
-      exit 0
-      ;;
-    --init-packages)
-      INIT_PACKAGES=1
-      ;;
-    --verbose)
-      VERBOSE=1
-      ;;
-    --build-*)
-      VAL=$(sed s/^--build-//g <<< $1)
-      BUILDLIST="$BUILDLIST $VAL"
-      ;;
-    *)
-      (>&2 echo "Error: Argument $1 is not valid")
-      echo ""
-      print_help
-      exit 1
-      ;;
-  esac
-  shift
-done
-
-status_start() {
-    echo -ne "[    ] $1"
-}
-status_ok() {
-    echo -ne "\r[\e[32m OK \e[39m]\n"
-}
-
-status_fail() {
-    echo -ne "\r[\e[31mFAIL\e[39m]\n"
-}
-
-status_skip() {
-    echo -ne "\r[SKIP] $1\033[K\n"
-}
-
-error_msg() {
-    echo -ne "      $1\n"
-}
-
-verbose_msg() {
-    if [ $VERBOSE -ne 0 ]; then
-        echo "Current Environment:"
-        env
-
-        if [ ! -z "$1" ]; then
-            echo "Logfile:"
-            cat $1
-        fi
-    fi
-}
-
-ROOTDIR="$(pwd)"
-PKGDIR="$ROOTDIR/packages"
-SCRIPTDIR="$ROOTDIR/scripts"
-
-# Source OPAM environment if not already set
-if [ -z "$OPAMROOT" ]; then
-    if [ -x "$(command -v opam)" ]; then
-        eval $(opam env --root=/opt/opam --set-root)
-    else
-        echo "WARNING: 'opam' not installed, can't build VyConf and libvyosconfig"
-    fi
-fi
-
-package_in_buildlist() {
-  # Return true if buildlist is not set
-  if [ -z "$BUILDLIST" ]; then
-    return 0
-  fi
-
-  if [[ $BUILDLIST =~ (^|[[:space:]])$1($|[[:space:]]) ]]; then
-    return 0
-  fi
-
-  return 1
-}
-
-build_package() {
-  PKG=$1
-  COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD)
-  if ! package_in_buildlist $1; then
-    return 0
-  fi
-  status_start "Building package: $PKG     Commit id: $COMMITID"
-  if [ ! -d "$PKGDIR/$PKG/debian" ]; then
-    status_skip "No source for: $PKG"
-    return 1
-  fi
-  ( set -e; set -x
-    cd $PKGDIR/$PKG
-    dpkg-buildpackage -uc -us -tc -b -j$(cat /proc/cpuinfo | grep processor | wc -l)
-  ) >>$PKGDIR/$PKG.buildlog 2>&1
-  if [ $? -ne 0 ]; then
-    status_fail
-    verbose_msg "$PKGDIR/$PKG.buildlog"
-    error_msg "Failed to build package $PKG, look in $PKG.buildlog to examine the fault\n"
-    return 2
-  fi
-  status_ok
-}
-
-echo "Cleaning up buildfiles..."
-rm -rf $PKGDIR/*.deb
-rm -rf $PKGDIR/*.changes
-rm -rf $PKGDIR/*.buildlog
-COMMITID=$(cd $PKGDIR; git rev-parse --short=10 HEAD)
-echo "-----------------------------------------------------"
-echo "Starting build process for all packages"
-echo "vyos-build Commit ID: $COMMITID"
-
-initialize_packages() {
-  status_start "Initializing packages"
-  (
-    set -x
-    git submodule update --init --recursive
-    git submodule update --remote
-  ) >>$PKGDIR/init-packages.buildlog 2>&1
-  if [ $? -ne 0 ]; then
-      status_fail
-      verbose_msg "$PKGDIR/init-packages.buildlog"
-      error_msg "Failed to update all package, look in init-packages.buildlog to examine the fault\n"
-      return 1
-  fi
-  status_ok
-}
-if [ $INIT_PACKAGES ]; then
-  initialize_packages
-fi
-
-build_libyang() {
-  PKG=libyang
-  COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD)
-  if ! package_in_buildlist $1; then
-    return 0
-  fi
-  status_start "Building package: $PKG Commit id: $COMMITID"
-  if [ ! -f "$PKGDIR/$PKG/README.md" ]; then
-    status_skip "No source for: $PKG"
-    return 1
-  fi
-
-  ( set -e; set -x
-    cd $PKGDIR/$PKG
-    git checkout 179da47f2e8de
-
-    git clean -dxf
-    git reset --hard
-
-    mkdir build
-    cd build
-
-    cmake ..
-    make build-deb
-
-    cp debs/* $PKGDIR
-
-  ) >>$PKGDIR/$PKG.buildlog 2>&1
-
-  if [ $? -ne 0 ]; then
-    status_fail
-    error_msg "Failed to build package $PKG, look in $PKG.buildlog to examine the fault\n"
-    return 2
-  fi
-  status_ok
-}
-build_libyang
-
-build_frr() {
-  PKG=frr
-  COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD)
-  if ! package_in_buildlist $1; then
-    return 0
-  fi
-  status_start "Building package: $PKG Commit id: $COMMITID"
-  if [ ! -f "$PKGDIR/$PKG/README.md" ]; then
-    status_skip "No source for: $PKG"
-    return 1
-  fi
-
-  ( set -e; set -x
-    cd $PKGDIR/$PKG
-    git clean -dxf
-    git reset --hard
-
-    # Run bootstrap.sh and make a dist tarball
-    # http://docs.frrouting.org/projects/dev-guide/en/latest/packaging-debian.html
-    ./bootstrap.sh
-    ./configure --with-pkg-extra-version=-vyos
-    make dist
-
-    # Create backports debian sources
-    mv debianpkg debian
-    make -f debian/rules backports
-
-    # Create a new directory to build the package and populate with package source.
-    mkdir frrpkg
-    cd frrpkg
-    tar xf ../frr_*.orig.tar.gz
-    cd frr*
-    source /etc/os-release
-    tar xf ../../frr_*${ID}${VERSION_ID}*.debian.tar.xz
-
-    # Build Debian Package
-    debuild --no-lintian --set-envvar=WANT_SNMP=1 --set-envvar=WANT_RPKI=1 \
-        --set-envvar=WANT_CUMULUS_MODE=0 -b -uc -us \
-        -j$(cat /proc/cpuinfo | grep processor | wc -l)
-
-    mv ../frr_*.deb $PKGDIR
-  ) >>$PKGDIR/$PKG.buildlog 2>&1
-  if [ $? -ne 0 ]; then
-    status_fail
-    error_msg "Failed to build package $PKG, look in $PKG.buildlog to examine the fault\n"
-    return 2
-  fi
-  status_ok
-}
-build_frr
-
-for PKG in mdns-repeater \
-           pmacct \
-           udp-broadcast-relay \
-           conntrack-tools \
-           eventwatchd \
-           ddclient \
-           rtrlib \
-           hvinfo \
-           igmpproxy \
-           ipaddrcheck \
-           lldpd \
-           libvyosconfig \
-           vyatta-bash \
-           vyatta-biosdevname \
-           vyatta-cfg \
-           vyatta-cfg-firewall \
-           vyatta-cfg-op-pppoe \
-           vyatta-cfg-qos \
-           vyatta-cfg-quagga \
-           vyatta-cfg-system \
-           vyatta-cfg-vpn \
-           vyatta-cluster \
-           vyatta-config-mgmt \
-           vyatta-config-migrate \
-           vyatta-conntrack \
-           vyatta-conntrack-sync \
-           vyatta-eventwatch \
-           vyatta-iproute \
-           vyatta-ipv6-rtradv \
-           vyatta-lldp \
-           vyatta-nat \
-           vyatta-netflow \
-           vyatta-op \
-           vyatta-op-dhcp-server \
-           vyatta-op-firewall \
-           vyatta-op-qos \
-           vyatta-op-quagga \
-           vyatta-op-vpn \
-           vyatta-openvpn \
-           vyatta-ravpn \
-           vyatta-wanloadbalance \
-           vyatta-webgui \
-           vyatta-webproxy \
-           vyatta-wireless \
-           vyatta-wirelessmodem \
-           vyatta-zone \
-           vyos-xe-guest-utilities \
-           vyos-keepalived \
-           vyos-netplug \
-           vyos-nhrp \
-           vyos-opennhrp \
-           vyos-salt-minion \
-           vyos-strongswan \
-           vyos-vmwaretools-scripts \
-           vyos-world \
-           vyos-1x \
-           ; do
-  build_package "$PKG"
-  ERRCODE=$?
-  if [ "$ERRCODE" -ne "0" ]; then
-      exit $ERRCODE
-  fi
-done
-
-# KERNEL
-build_kernel() {
-  PKG="vyos-kernel"
-  if ! package_in_buildlist "vyos-kernel"; then
-    return 0
-  fi
-  COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD)
-  status_start "Building-package: vyos-kernel   Commit ID:$COMMITID"
-  if [ ! -f "$PKGDIR/vyos-kernel/Makefile" ]; then
-    status_skip "No source for: vyos-kernel"
-    return 0
-  fi
-
-  ( set -e; set -x
-    cd $PKGDIR/vyos-kernel > /dev/null
-    bash -c "$SCRIPTDIR/build-kernel"
-  ) >>$PKGDIR/vyos-kernel.buildlog 2>&1
-  if [ $? -ne 0 ]; then
-    status_fail
-    verbose_msg "$PKGDIR/vyos-kernel.buildlog"
-    error_msg "Failed to build package vyos-kernel, look in vyos-kernel.buildlog to examine the fault\n"
-    return 1
-  fi
-
-  VERSION=$(grep "^VERSION" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
-  PATCHLEVEL=$(grep "^PATCHLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
-  SUBLEVEL=$(grep "^SUBLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
-  ARCH=$(dpkg --print-architecture)
-  status_ok
-}
-build_kernel
-ERRCODE=$?
-if [ "$ERRCODE" -ne "0" ]; then
-    exit $ERRCODE
-fi
-
-
-# WIREGUARD
-build_wireguard() {
-  PKG="vyos-wireguard"
-  if ! package_in_buildlist "vyos-wireguard"; then
-    return 0
-  fi
-  COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD)
-  status_start "Building package: vyos-wireguard    Commit ID:$COMMITID"
-
-  if [ ! -d "$PKGDIR/vyos-wireguard/debian" ]; then
-    status_skip "No source for: vyos-wireguard"
-    return 0
-  fi
-
-  if [ ! -f "$PKGDIR/vyos-kernel/Makefile" ]; then
-    status_fail
-    error_msg "No Makefile found in kernel package"
-    return 1
-  fi
-
-  if  ! grep -q "KBUILD_OUTPUT" $PKGDIR/vyos-kernel/Makefile; then
-    status_fail
-    error_msg "Failed to build package vyos-wireguard, no kernel source found\n"
-    return 1
-  fi
-
-  VERSION=$(grep "^VERSION" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
-  PATCHLEVEL=$(grep "^PATCHLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
-  SUBLEVEL=$(grep "^SUBLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
-  ARCH=$(dpkg --print-architecture)
-  # Collect kernel information
-  ( set -e; set -x
-    cd $PKGDIR/vyos-wireguard
-    echo "src/wireguard.ko /lib/modules/$VERSION.$PATCHLEVEL.$SUBLEVEL-$ARCH-vyos/extra" | tee debian/wireguard-modules.install
-    KERNELDIR=$PKGDIR/vyos-kernel dpkg-buildpackage -uc -us -tc -b -j$(cat /proc/cpuinfo | grep processor | wc -l)
-  ) >>$PKGDIR/vyos-wireguard.buildlog 2>&1
-  if [ $? -ne 0 ]; then
-    status_fail
-    verbose_msg "$PKGDIR/vyos-wireguard.buildlog"
-    error_msg "Failed to build package vyos-wireguard, look in vyos-wireguard.buildlog to examine the fault\n"
-    return 2
-  fi
-  status_ok
-}
-build_wireguard
-ERRCODE=$?
-if [ "$ERRCODE" -ne "0" ]; then
-    exit $ERRCODE
-fi
-
-# ACCEL-PPP
-build_accel-ppp() {
-  PKG="vyos-accel-ppp"
-  if ! package_in_buildlist "accel-ppp"; then
-    return 0
-  fi
-  COMMITID=$(cd $PKGDIR/$PKG; git rev-parse --short=10 HEAD)
-  status_start "Building package: vyos-accel-ppp   Commit ID: $COMMITID"
-  if [ ! -d "$PKGDIR/vyos-accel-ppp/debian" ]; then
-    status_skip "No source for: vyos-accel-ppp"
-    return 0
-  fi
-
-  if [ ! -f "$PKGDIR/vyos-kernel/Makefile" ]; then
-    status_fail
-    error_msg "No Makefile found in kernel package"
-    return 1
-  fi
-
-  if ! grep -q "KBUILD_OUTPUT" $PKGDIR/vyos-kernel/Makefile; then
-    status_fail
-    error_msg "Failed to build package vyos-accel-ppp, no kernel source found\n"
-  fi
-  # Collect kernel information
-  VERSION=$(grep "^VERSION" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
-  PATCHLEVEL=$(grep "^PATCHLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
-  SUBLEVEL=$(grep "^SUBLEVEL" $PKGDIR/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
-  ARCH=$(dpkg --print-architecture)
-
-  ( set -e; set -x
-    cd $PKGDIR/vyos-accel-ppp
-    echo "lib/modules/$VERSION.$PATCHLEVEL.$SUBLEVEL-$ARCH-vyos/extra/*.ko" | tee debian/vyos-accel-ppp-ipoe-kmod.install
-    sed -i "s#[0-9].[0-9][0-9].[0-9]-amd64-vyos#$VERSION.$PATCHLEVEL.$SUBLEVEL-$ARCH-vyos#g" debian/rules
-    KERNELDIR=$PKGDIR/vyos-kernel dpkg-buildpackage -uc -us -tc -b -j$(cat /proc/cpuinfo | grep processor | wc -l)
-  ) >>$PKGDIR/vyos-accel-ppp.buildlog 2>&1
-  if [ $? -ne 0 ]; then
-    status_fail
-    verbose_msg "$PKGDIR/vyos-accel-ppp.buildlog"
-    error_msg "Failed to build package vyos-accel-ppp, look in vyos-accel-ppp.buildlog to examine the fault\n"
-    return 1
-  fi
-  status_ok
-}
-build_accel-ppp
-ERRCODE=$?
-if [ "$ERRCODE" -ne "0" ]; then
-    exit $ERRCODE
-fi