Merge pull request #32 from runborg/autobuild

Script and Instructions for building all subpackages via docker

3 files changed, 259 insertions, 9 deletions

diff --git a/Dockerfile b/Dockerfile
index bf639707..b8ec8149 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -67,6 +67,55 @@ RUN apt-get install -y -t jessie-backports \
       libxml2-dev \
       pkg-config
 
+# Package needed for mdns-repeater
+RUN apt-get install -y -t jessie-backports \
+      dh-systemd
+
+# Packages needed for vyatta-bash
+RUN apt-get install -y \
+      libncurses5-dev \
+      locales
+
+# Packages needed for vyatta-cfg
+RUN apt-get install -y \
+      libboost-filesystem-dev
+
+# Packages needed for vyatta-iproute
+RUN apt-get install -y \
+      libatm1-dev \
+      libdb-dev
+
+# Packages needed for vyatta-webgui
+RUN apt-get install -y \
+      libexpat1-dev \
+      subversion
+
+# Packages needed for pmacct
+RUN apt-get install -y \
+      libpcap-dev \
+      libpq-dev \
+      libmysqlclient-dev \
+      libgeoip-dev \
+      librabbitmq-dev \
+      libjansson-dev \
+      librdkafka-dev \
+      libnetfilter-log-dev
+
+# Packages needed for vyos-keepalived
+RUN apt-get install -y \
+      libnl-3-dev \
+      libnl-genl-3-dev \
+      libpopt-dev \
+      libsnmp-dev
+
+# Pavkages needed for wireguard
+RUN apt-get install -y \
+      libmnl-dev
+
+# Packages needed for kernel
+RuN apt-get install -y \
+      libelf-dev
+
 # Update live-build
 RUN echo 'deb http://ftp.debian.org/debian stretch main' | tee -a /etc/apt/sources.list.d/stretch.list &&\
     apt-get update &&\
diff --git a/README.md b/README.md
index 8388015b..c6460aff 100644
--- a/README.md
+++ b/README.md
@@ -154,19 +154,61 @@ inside the container and follow up the bellow instructions in order to build the
 VyOS ISO image
 
 ## Building subpackages inside Docker
-### Strongswan
+Prior to building packages you need to checkout and update the submodules you want to compile
+```bash
+git submodule update --init packages/PACKAGENAME
+cd packages/PACKAGENAME
+git checkout BRANCH
+```
+`PACKAGENAME` is the name of the package you want to compile
+`BRANCH` is for Crux(1.2) `crux`, for latest rolling use `current`
+
+
+### Pulling all packages
+Use this with caution, only run this on a unmodified newly cloned repository
+```bash
+for dir in packages/*; do
+  git submodule update --init $dir
+  pushd $dir
+  git checkout current
+  popd
+done
+```
+### Building packages
+The script `./scripts/build-submodules` is created to automate the process of building packages, execute it in the root of `vyos-build` to start compilation on all supported packages that are checked out. 
 
-Prior to executing this you need to checkout and update the packages/vyos-strongswan submodule
-Building the strongswan package is for now only doable on a Linux system because tests fail when running on windows and OSX systems
-`/HOST_PATH/` is the path to your vyos_build directory. if youre in the vyos-build directory it can me replaced with `$(pwd)`
+The easiest way to compile is with the `vyos-builder` docker container, it includes all dependencies for compiling supported packages.
 
-`--sysctl net.ipv6.conf.lo.disable_ipv6=0` is needed to enable ipv6 inside the container. tests will fail if you don't have it.
+```bash
+$ docker run --rm -it -v $(pwd):/vyos -w /vyos \
+             --sysctl net.ipv6.conf.lo.disable_ipv6=0 \
+             vyos-builder \
+             ./scripts/build-submodules
+```
+
+NOTE: Prior to executing this script you need to create/build the `vyos-builder` container and checkout all packages you want to compile. 
 
+### Building one package
+the script above runs all package build inside a docker container, this is also possible to do by hand using: 
+Executed from the root directory of vyos-build
 ```bash
-$ docker run -it -v /HOST_PATH/:/vyos  --sysctl net.ipv6.conf.lo.disable_ipv6=0 vyos-builder \
-    bash -c '\
-      cd /vyos/packages/vyos-strongswan &&\
-      dpkg-buildpackage -uc -us -tc -b'
+$ docker run --rm -it -v $(pwd):/vyos -w /vyos/packages/PACKAGENAME \
+             --sysctl net.ipv6.conf.lo.disable_ipv6=0 \
+             vyos-builder \
+             dpkg-buildpackage -uc -us -tc -b
+```
+NOTE: `--sysctl net.ipv6.conf.lo.disable_ipv6=0` is only needed when building vyos-strongswan and can be ignored on other packages
+NOTE: Prior to executing this you need to checkout and update the submodules you want to recompile
+NOTE: vyos-strongswan will only compile on a linux system, running on osx or windows migth result in a unittest lockup. (it never exits)
+
+Packages that are known to not build using this procedure:
+```
+vyatta-util     - Not needed anymore
+vyatta-quagga   - Not needed anymore 
+vyos-1x         - Unmet build dependencies: whois libvyosconfig0
+vyos-frr        - Alott of requirements, scary stuff...
+vyos-kernel     - Need special build instructions
+vyos-wireguard  - Needs special build instructions
 ```
 
 
diff --git a/scripts/build-submodules b/scripts/build-submodules
new file mode 100755
index 00000000..1981759f
--- /dev/null
+++ b/scripts/build-submodules
@@ -0,0 +1,159 @@
+#!/bin/bash 
+#set -x
+if [ ! -d "packages" ]; then
+  echo "This script needs to be executed inside the top root of vyos-build"
+  exit 1
+fi
+
+if [ "$1" == "-h" ] || [ "$1" == "--help" ]; then
+  echo "Script for building all subpackages to vyos"
+  echo "Execute this sctipt from the root of the vyos-build directory"
+  echo ""
+  echo "This script could be executed from a Debian Jessie installation with all dependencies"
+  echo "or from the vyos-builder docker container"
+  echo "docker instructions"
+  echo "Build the container:"
+  echo "  docker build -t vyos-builder ." 
+  echo "Compile packages:"
+  echo "  docker run --rm -it -v $(pwd):/vyos -w /vyos --sysctl net.ipv6.conf.lo.disable_ipv6=0 vyos-builder scripts/build-docker-subpaclages"
+fi
+
+status_start() {
+echo -ne "[    ] $1"
+}
+status_ok() {
+echo -ne "\r[\e[32m OK \e[39m]\n"
+}
+
+status_fail() {
+echo -ne "\r[\e[31mFAIL\e[39m]\n"
+}
+
+status_skip() {
+echo -ne "\r[SKIP] $1\n"
+}
+
+error_msg() {
+echo -ne "      $1\n"
+}
+ROOTDIR="$(pwd)"
+PKGDIR="$(pwd)/packages"
+
+echo "Cleaning up buildfiles..."
+rm -rf $PKGDIR/*.deb
+rm -rf $PKGDIR/*.changes
+rm -rf $PKGDIR/*.buildlog
+echo "-----------------------------------------------------"
+echo "Starting build process for all packages"
+echo ""
+for PKG in mdns-repeater \
+           pmacct \
+           udp-broadcast-relay \
+           vyatta-bash \
+           vyatta-cfg \
+           vyatta-cfg-firewall \
+           vyatta-cfg-op-pppoe \
+           vyatta-cfg-qos \
+           vyatta-cfg-quagga \
+           vyatta-cfg-system \
+           vyatta-cfg-vpn \
+           vyatta-cluster \
+           vyatta-config-mgmt \
+           vyatta-config-migrate \
+           vyatta-conntrack \
+           vyatta-conntrack-sync \
+           vyatta-eventwatch \
+           vyatta-iproute \
+           vyatta-ipv6-rtradv \
+           vyatta-lldp \
+           vyatta-nat \
+           vyatta-netflow \
+           vyatta-op \
+           vyatta-op-dhcp-server \
+           vyatta-op-firewall \
+           vyatta-op-qos \
+           vyatta-op-quagga \
+           vyatta-op-vpn \
+           vyatta-openvpn \
+           vyatta-ravpn \
+           vyatta-vrrp \
+           vyatta-wanloadbalance \
+           vyatta-webgui \
+           vyatta-webproxy \
+           vyatta-wireless \
+           vyatta-wirelessmodem \
+           vyatta-zone \
+           vyos-keepalived \
+           vyos-nhrp \
+           vyos-pppoe-server \
+           vyos-strongswan \
+           vyos-world \
+           ; do
+  if [ -d "packages/$PKG/debian" ]; then
+    status_start "Building package: $PKG"
+    pushd $PKGDIR/$PKG > /dev/null
+    dpkg-buildpackage -uc -us -tc -b >$PKGDIR/$PKG.buildlog 2>&1
+    if [ $? -ne 0 ]; then
+      status_fail
+      error_msg "Failed to build package $PKG, look in $PKG.buildlog to examine the fault\n"
+    else
+      status_ok
+    fi
+    popd > /dev/null
+  else
+    status_skip "No source for: $PKG"
+  fi
+done
+
+# KERNEL
+if [ -f "packages/vyos-kernel/Makefile" ]; then
+  status_start "Building-package: vyos-kernel"
+  pushd packages/vyos-kernel > /dev/null
+  bash -c '../../scripts/build-kernel' >$PKGDIR/vyos-kernel.buildlog 2>&1
+  if [ $? -ne 0 ]; then
+    status_fail
+    error_msg "Failed to build package vyos-kernel, look in vyos-kernel.buildlog to examine the fault\n"
+  else
+    VERSION=$(grep "^VERSION" Makefile | grep -Eo '[0-9]{1,4}')
+    PATCHLEVEL=$(grep "^PATCHLEVEL" Makefile | grep -Eo '[0-9]{1,4}')
+    SUBLEVEL=$(grep "^SUBLEVEL" Makefile | grep -Eo '[0-9]{1,4}')
+    ARCH=$(dpkg --print-architecture)
+    sed -i.bak "s/--linux-packages .* \\\/--linux-packages linux-image-$VERSION\.$PATCHLEVEL\.$SUBLEVEL \\\/" $ROOTDIR/scripts/live-build-config 
+    status_ok
+  fi
+  popd > /dev/null
+else
+  status_skip "No source for: vyos-kernel"
+fi
+
+
+
+# WIREGUARD
+if [ -d "packages/vyos-wireguard/debian" ]; then
+  if [ -f "packages/vyos-kernel/Makefile" ]; then
+    status_start "Building package: vyos-wireguard"
+    if  grep -q "KBUILD_OUTPUT" packages/vyos-kernel/Makefile; then
+      VERSION=$(grep "^VERSION" packages/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
+      PATCHLEVEL=$(grep "^PATCHLEVEL" packages/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
+      SUBLEVEL=$(grep "^SUBLEVEL" packages/vyos-kernel/Makefile | grep -Eo '[0-9]{1,4}')
+      ARCH=$(dpkg --print-architecture)
+      pushd packages/vyos-wireguard > /dev/null
+      echo "src/wireguard.ko /lib/modules/$VERSION.$PATCHLEVEL.$SUBLEVEL-$ARCH-vyos/extra" > debian/wireguard-modules.install
+      bash -c 'KERNELDIR=/vyos/packages/vyos-kernel dpkg-buildpackage -uc -us -tc -b' >$PKGDIR/vyos-wireguard.buildlog 2>&1
+      if [ $? -ne 0 ]; then
+        status_fail
+        error_msg "Failed to build package vyos-wireguard, look in vyos-wireguard.buildlog to examine the fault\n"
+      else
+        status_ok
+      fi
+      popd > /dev/null
+    else
+      status_fail
+      error_msg "Failed to build package vyos-wireguard, no kernel source found\n"
+    fi
+  else
+    seeor_msg "Something wrong with the kernel module?"
+  fi
+else 
+    status_skip "No source for: vyos-wireguard"
+fi