From dfbe78ffbaa225b7e11d8e1e0e0b7d65fc9e022a Mon Sep 17 00:00:00 2001 From: runborg Date: Mon, 17 Dec 2018 23:35:18 +0100 Subject: T1070 - docker build of vyos-strongswan (#31) * T1070 - Added packages and readme notes about building strongswan strongswan is easely built using a docker image, but needs special options to pass tests. Added notes in the readme about building it and building instructions * T1070 - Restructured build example for wyos-strongswan --- Dockerfile | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'Dockerfile') diff --git a/Dockerfile b/Dockerfile index 2cfc826d..17864402 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,6 +41,30 @@ RUN echo 'deb http://ftp.debian.org/debian jessie-backports main' | tee -a /etc/ python3-nose \ python3-coverage +# Packages needed for building vyos-strongswan +RUN apt-get install -y -t jessie-backports \ + debhelper &&\ + apt-get install -y \ + dh-apparmor \ + gperf \ + iptables-dev \ + libcap-dev \ + libgcrypt20-dev \ + libgmp3-dev \ + libldap2-dev \ + libpam0g-dev \ + libsystemd-dev \ + libgmp-dev \ + iptables \ + xl2tpd \ + libcurl4-openssl-dev \ + libcurl4-openssl-dev \ + libkrb5-dev \ + libsqlite3-dev \ + libssl-dev \ + libxml2-dev \ + pkg-config + # Update live-build RUN echo 'deb http://ftp.debian.org/debian stretch main' | tee -a /etc/apt/sources.list.d/stretch.list &&\ apt-get update &&\ -- cgit v1.2.3 From 9473364834deae2574873af9087aa8907f775b93 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sun, 9 Dec 2018 23:44:12 +0100 Subject: Added Jenkinsfile --- Dockerfile | 27 +++++++++++++++++++++------ Jenkinsfile | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 Jenkinsfile (limited to 'Dockerfile') diff --git a/Dockerfile b/Dockerfile index 17864402..bf639707 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,8 @@ RUN echo 'deb http://ftp.debian.org/debian jessie-backports main' | tee -a /etc/ vim \ git \ make \ + sudo \ + locales \ live-build \ pbuilder \ devscripts \ @@ -73,11 +75,24 @@ RUN echo 'deb http://ftp.debian.org/debian stretch main' | tee -a /etc/apt/sourc apt-get update &&\ rm -rf /var/lib/apt/lists/* -#install packer +# Standard shell should be bash not dash +RUN echo "dash dash/sh boolean false" | debconf-set-selections && \ + DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash + +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen +ENV LANG en_US.utf8 + +# Install packer RUN export LATEST="$(curl -s https://checkpoint-api.hashicorp.com/v1/check/packer | \ - jq -r -M '.current_version')"; \ - echo "url https://releases.hashicorp.com/packer/"$LATEST"/packer_"$LATEST"_linux_amd64.zip" |\ - curl -K- | gzip -d > /usr/bin/packer -RUN chmod +x /usr/bin/packer + jq -r -M '.current_version')"; \ + echo "url https://releases.hashicorp.com/packer/"$LATEST"/packer_"$LATEST"_linux_amd64.zip" |\ + curl -K- | gzip -d > /usr/bin/packer && \ + chmod +x /usr/bin/packer + +# Create vyos_bld user account and enable sudo +RUN useradd -ms /bin/bash -u 1006 --gid users vyos_bld && \ + usermod -aG sudo vyos_bld && \ + echo "%sudo ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers -WORKDIR ~ +USER vyos_bld +WORKDIR /home/vyos_bld diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..d46fdcbf --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,50 @@ +#!/usr/bin/env groovy + +@NonCPS +def setDescription() { + def item = Jenkins.instance.getItemByFullName(env.JOB_NAME) + item.setDescription("VyOS image build using a\nPipeline build inside Docker container.") + item.save() +} + +setDescription() + +/* Only keep the 10 most recent builds. */ +def projectProperties = [ + [$class: 'BuildDiscarderProperty',strategy: [$class: 'LogRotator', numToKeepStr: '5']], +] + +properties(projectProperties) + +pipeline { + agent { + dockerfile { + filename 'Dockerfile' + label 'jessie-amd64' + args '--privileged' + } + } + + stages { + stage('Configure') { + steps { + sh './configure --build-by="autobuild@vyos.net" --debian-mirror="http://ftp.us.debian.org/debian/"' + } + } + stage('Build ISO') { + steps { + sh 'sudo make iso' + } + } + } + + post { + always { + echo 'One way or another, I have finished' + // change build dir file permissions so wen can cleanup as regular + // user (jenkins) afterwards + sh 'sudo chmod -R 777 .' + deleteDir() /* cleanup our workspace */ + } + } +} -- cgit v1.2.3 From de19301f236df04e461efcdf6158ada020eeb1e3 Mon Sep 17 00:00:00 2001 From: Runar Borge Date: Mon, 24 Dec 2018 00:06:08 +0100 Subject: Added script and instructions for building most packages from source. Using the vyos-builder docker container. Also added examples on doing builds. --- Dockerfile | 23 ++++++++++++++ README.md | 52 +++++++++++++++++++++++++------ scripts/build-docker-subpackages | 66 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 9 deletions(-) create mode 100755 scripts/build-docker-subpackages (limited to 'Dockerfile') diff --git a/Dockerfile b/Dockerfile index 17864402..b862cda4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,6 +65,29 @@ RUN apt-get install -y -t jessie-backports \ libxml2-dev \ pkg-config +# Package needed for mdns-repeater +RUN apt-get install -y -t jessie-backports \ + dh-systemd + +# Packages needed for vyatta-bash +RUN apt-get install -y \ + libncurses5-dev \ + locales + +# Packages needed for vyatta-cfg +RUN apt-get install -y \ + libboost-filesystem-dev + +# Packages needed for vyatta-iproute +RUN apt-get install -y \ + libatm1-dev \ + libdb-dev + +# Packages needed for vyatta-webgui +RUN apt-get install -y \ + libexpat1-dev \ + subversion + # Update live-build RUN echo 'deb http://ftp.debian.org/debian stretch main' | tee -a /etc/apt/sources.list.d/stretch.list &&\ apt-get update &&\ diff --git a/README.md b/README.md index 8388015b..405b1b2a 100644 --- a/README.md +++ b/README.md @@ -154,19 +154,53 @@ inside the container and follow up the bellow instructions in order to build the VyOS ISO image ## Building subpackages inside Docker -### Strongswan +Prior to building packages you need to checkout and update the submodules you want to compile +```bash +git submodule update --init packages/PACKAGENAME +cd packages/PACKAGENAME +git checkout BRANCH +``` +`PACKAGENAME` is the name of the package you want to compile +`BRANCH` is for Crux(1.2) `crux`, for latest rolling use `current` -Prior to executing this you need to checkout and update the packages/vyos-strongswan submodule -Building the strongswan package is for now only doable on a Linux system because tests fail when running on windows and OSX systems -`/HOST_PATH/` is the path to your vyos_build directory. if youre in the vyos-build directory it can me replaced with `$(pwd)` -`--sysctl net.ipv6.conf.lo.disable_ipv6=0` is needed to enable ipv6 inside the container. tests will fail if you don't have it. +### Pulling all packages +Use this with caution, only run this on a unmodified newly cloned repository +```bash +for dir in packages/*; do + git submodule update --init $dir + pushd $dir + git checkout current + popd +done +``` +### Building packages +Most packages can be built by using the vyos-builder docker container with the same parameters, the vyos-builder container should include all dependencies for compiling supported packages. +The script `./scripts/build-docker-subpackages` is created to automate the process of building packages, just execute it in the root of vyos-build to start compilation on all supported packages that are checked out. + +NOTE: Prior to executing this script you need to create/build the `vyos-builder` container and checkout all packages you want to compile. +### Building one package +the script above runs a docker container for every build it does. this is also possible to do by hand using: +Ecevuted from the root directory of vyos-build ```bash -$ docker run -it -v /HOST_PATH/:/vyos --sysctl net.ipv6.conf.lo.disable_ipv6=0 vyos-builder \ - bash -c '\ - cd /vyos/packages/vyos-strongswan &&\ - dpkg-buildpackage -uc -us -tc -b' +$ docker run --rm -it -v $(pwd):/vyos -w /vyos/packages/PACKAGENAME --sysctl net.ipv6.conf.lo.disable_ipv6=0 vyos-builder dpkg-buildpackage -uc -us -tc -b +``` +NOTE: `--sysctl net.ipv6.conf.lo.disable_ipv6=0` is only needed when building vyos-strongswan and can be ignored on other packages +NOTE: Prior to executing this you need to checkout and update the submodules you want to recompile +NOTE: vyos-strongswan will only compile on a linux system, running on osx or windows migth result in a unittest lockup. (it never exits) + +Packages that are known to not build using this procedure: +``` +pmacct - Unmet build dependencies: libpcap-dev libpq-dev libmysqlclient-dev libgeoip-dev librabbitmq-dev libjansson-dev librdkafka-dev libnetfilter-log-dev +vyatta-util - dh_clean: mv -Tf debian/.debhelper/bucket/files/47da33933b3825049bbc04871747a9598ce90fd45a438b6a8a58b74bf6d73a4d.tmp config/config.guess returned exit code 1 +vyos-keepalived - Unmet build dependencies: libnl-3-dev libnl-genl-3-dev libpopt-dev libsnmp-dev + +vyatta-quagga - Not needed anymore +vyos-1x - Unmet build dependencies: whois libvyosconfig0 +vyos-frr - Alott of requirements, scary stuff... +vyos-kernel - Need special build instructions +vyos-wireguard - Needs special build instructions ``` diff --git a/scripts/build-docker-subpackages b/scripts/build-docker-subpackages new file mode 100755 index 00000000..7798b5fc --- /dev/null +++ b/scripts/build-docker-subpackages @@ -0,0 +1,66 @@ +#!/bin/bash +#set -x +if [ ! -d "packages" ]; then + echo "This script needs to be executed inside the top root of vyos-build" + exit 1 +fi + +echo "Cleaning up buildfiles..." +rm -rf packages/*.deb +rm -rf packages/*.changes +echo "-----------------------------------------------------" + +for PKG in mdns-repeater \ + pmacct \ + udp-broadcast-relay \ + vyatta-bash \ + vyatta-cfg \ + vyatta-cfg-firewall \ + vyatta-cfg-op-pppoe \ + vyatta-cfg-qos \ + vyatta-cfg-quagga \ + vyatta-cfg-system \ + vyatta-cfg-vpn \ + vyatta-cluster \ + vyatta-config-mgmt \ + vyatta-config-migrate \ + vyatta-conntrack \ + vyatta-conntrack-sync \ + vyatta-eventwatch \ + vyatta-iproute \ + vyatta-ipv6-rtradv \ + vyatta-lldp \ + vyatta-nat \ + vyatta-netflow \ + vyatta-op \ + vyatta-op-dhcp-server \ + vyatta-op-firewall \ + vyatta-op-qos \ + vyatta-op-quagga \ + vyatta-op-vpn \ + vyatta-openvpn \ + vyatta-ravpn \ + vyatta-util \ + vyatta-vrrp \ + vyatta-wanloadbalance \ + vyatta-webgui \ + vyatta-webproxy \ + vyatta-wireless \ + vyatta-wirelessmodem \ + vyatta-zone \ + vyos-keepalived \ + vyos-nhrp \ + vyos-pppoe-server \ + vyos-strongswan \ + vyos-world \ + ; do + if [ -d "packages/$PKG/debian" ]; then + echo "Building package: $PKG" + docker run --rm -it -v $(pwd):/vyos -w /vyos/packages/$PKG --sysctl net.ipv6.conf.lo.disable_ipv6=0 vyos-builder dpkg-buildpackage -uc -us -tc -b >packages/$PKG.buildlog 2>&1 + if [ $? -ne 0 ]; then + echo "FAILED to build package $PKG, look in $PKG.buildlog to examine the fault" + fi + else + echo "Did not find source for: $PKG" + fi +done -- cgit v1.2.3 From 22454da58c03860b15dcb1ee0b496c612eefe0df Mon Sep 17 00:00:00 2001 From: Runar Borge Date: Mon, 24 Dec 2018 23:50:30 +0100 Subject: Added dependenies for compiling pmacct and vyos-keepalived to Dockerfile --- Dockerfile | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'Dockerfile') diff --git a/Dockerfile b/Dockerfile index b862cda4..4c7dcd90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -88,6 +88,24 @@ RUN apt-get install -y \ libexpat1-dev \ subversion +# Packages needed for pmacct +RUN apt-get install -y \ + libpcap-dev \ + libpq-dev \ + libmysqlclient-dev \ + libgeoip-dev \ + librabbitmq-dev \ + libjansson-dev \ + librdkafka-dev \ + libnetfilter-log-dev + +# Packages needed for vyos-keepalived +RUN apt-get install -y \ + libnl-3-dev \ + libnl-genl-3-dev \ + libpopt-dev \ + libsnmp-dev + # Update live-build RUN echo 'deb http://ftp.debian.org/debian stretch main' | tee -a /etc/apt/sources.list.d/stretch.list &&\ apt-get update &&\ -- cgit v1.2.3 From 90ec7b2e2558dce37fecdc115bc3aa52236c132b Mon Sep 17 00:00:00 2001 From: Runar Borge Date: Wed, 26 Dec 2018 00:19:18 +0100 Subject: Added compilation of the kernel and wireguard submodule. Also refactored some UI code for easyer reading --- Dockerfile | 8 +++++ scripts/build-docker-subpackages | 76 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 79 insertions(+), 5 deletions(-) (limited to 'Dockerfile') diff --git a/Dockerfile b/Dockerfile index 4c7dcd90..988bcc36 100644 --- a/Dockerfile +++ b/Dockerfile @@ -106,6 +106,14 @@ RUN apt-get install -y \ libpopt-dev \ libsnmp-dev +# Pavkages needed for wireguard +RUN apt-get install -y \ + libmnl-dev + +# Packages needed for kernel +RuN apt-get install -y \ + libelf-dev + # Update live-build RUN echo 'deb http://ftp.debian.org/debian stretch main' | tee -a /etc/apt/sources.list.d/stretch.list &&\ apt-get update &&\ diff --git a/scripts/build-docker-subpackages b/scripts/build-docker-subpackages index 148e5a56..f3aa2e6d 100755 --- a/scripts/build-docker-subpackages +++ b/scripts/build-docker-subpackages @@ -5,11 +5,31 @@ if [ ! -d "packages" ]; then exit 1 fi +status_start() { +echo -ne "[ ] $1" +} +status_ok() { +echo -ne "\r[\e[32m OK \e[39m]\n" +} + +status_fail() { +echo -ne "\r[\e[31mFAIL\e[39m]\n" +} + +status_skip() { +echo -ne "\r[SKIP] $1\n" +} + +error_msg() { +echo -ne " $1\n" +} + echo "Cleaning up buildfiles..." rm -rf packages/*.deb rm -rf packages/*.changes echo "-----------------------------------------------------" - +echo "Starting build process for all packages" +echo "" for PKG in mdns-repeater \ pmacct \ udp-broadcast-relay \ @@ -53,13 +73,59 @@ for PKG in mdns-repeater \ vyos-strongswan \ vyos-world \ ; do + break; if [ -d "packages/$PKG/debian" ]; then - echo "Building package: $PKG" - docker run --rm -it -v $(pwd):/vyos -w /vyos/packages/$PKG --sysctl net.ipv6.conf.lo.disable_ipv6=0 vyos-builder dpkg-buildpackage -uc -us -tc -b >packages/$PKG.buildlog 2>&1 + status_start "Building package: $PKG" + docker run --rm -it -v $(pwd):/vyos -w /vyos/packages/$PKG \ + --sysctl net.ipv6.conf.lo.disable_ipv6=0 \ + vyos-builder \ + dpkg-buildpackage -uc -us -tc -b >packages/$PKG.buildlog 2>&1 if [ $? -ne 0 ]; then - echo "FAILED to build package $PKG, look in $PKG.buildlog to examine the fault" + status_fail + error_msg "Failed to build package $PKG, look in $PKG.buildlog to examine the fault\n" + else + status_ok fi else - echo "Did not find source for: $PKG" + status_skip "No source for: $PKG" fi done + +# KERNEL +if [ -f "packages/vyos-kernel/Makefile" ]; then + status_start "Building-package: vyos-kernel" + docker run --rm -it -v $(pwd):/vyos -w /vyos/packages/vyos-kernel \ + --sysctl net.ipv6.conf.lo.disable_ipv6=0 \ + vyos-builder \ + bash -c '../../scripts/build-kernel' >packages/vyos-kernel.buildlog 2>&1 + if [ $? -ne 0 ]; then + status_fail + error_msg "Failed to build package vyos-kernel, look in vyos-kernel.buildlog to examine the fault\n" + else + status_ok + fi +else + status_skip "No source for: vyos-kernel" +fi + +# WIREGUARD +if [ -d "packages/vyos-wireguard/debian" ]; then + status_start "Building package: vyos-wireguard" + if [ -f "packages/vyos-kernel/Makefile" ]; then + docker run --rm -it -v $(pwd):/vyos -w /vyos/packages/vyos-wireguard \ + --sysctl net.ipv6.conf.lo.disable_ipv6=0 \ + vyos-builder \ + bash -c 'KERNELDIR=/vyos/packages/vyos-kernel dpkg-buildpackage -uc -us -tc -b' >packages/vyos-wireguard.buildlog 2>&1 + if [ $? -ne 0 ]; then + status_fail + error_msg "Failed to build package vyos-wireguard, look in vyos-wireguard.buildlog to examine the fault\n" + else + status_ok + fi + else + status_fail + error_msg "Failed to build package vyos-wireguard, no kernel source found\n" + fi +else + status_skip "No source for: vyos-wireguard" +fi -- cgit v1.2.3