diff options
Diffstat (limited to 'scripts/build-packages')
| -rwxr-xr-x | scripts/build-packages | 332 |
1 files changed, 332 insertions, 0 deletions
diff --git a/scripts/build-packages b/scripts/build-packages new file mode 100755 index 00000000..e800b80a --- /dev/null +++ b/scripts/build-packages @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import sys +import shutil +import argparse +import logging + +current_working_directory = os.getcwd() +repo_root = subprocess.check_output('git rev-parse --show-toplevel', shell=True, universal_newlines=True).rstrip('\n') +repo_sha = subprocess.check_output('git rev-parse --short=12 HEAD', shell=True, universal_newlines=True).rstrip('\n') + +def add_package(name, url=None, commit='HEAD', branch='current', tag=None, custombuild_cmd=None): + """ + Build up source package with URL and build commands executed during the later + called build_package step. + + If no additional information is passed we will use the latest commit from current + branch + + If no URL is passed we assume it's a regular VyOS package from the VyOS Github + namespace at https://github.com/vyos + """ + + if not url: + url = 'https://github.com/vyos/' + name + '.git' + + package = { + 'name': name, + 'url': url, + 'commit': commit, + 'tag': tag, + 'branch': branch, + 'path': repo_root + '/packages/' + name, + 'custombuild_cmd': custombuild_cmd + } + + return package + + + +def call(bashCommand, log, **kwargs): + """ + Run command with arguments. Wait for command to complete. + + Send output to logging module passed as 'log'. + """ + + from subprocess import Popen, PIPE, STDOUT, check_output, CalledProcessError + from tempfile import TemporaryFile + from time import sleep + + log.debug("Executing '{}'".format(bashCommand)) + + # code borrowsed from: + # https://stackoverflow.com/questions/38374063/python-can-we-use-tempfile-with-subprocess-to-get-non-buffering-live-output-in-p + # the temp file will be automatically cleaned up + output = TemporaryFile() + error = TemporaryFile() + + kwargs['stdout'] = output + kwargs['stderr'] = error + kwargs['shell'] = True + kwargs['universal_newlines'] = True + + sub = Popen(bashCommand, **kwargs) + while sub.poll() is None: + where = output.tell() + lines = output.readline() + if not lines: + sleep(0.3) + output.seek(where) + else: + log.debug(lines.decode().rstrip('\n')) + + where = error.tell() + lines = error.readline() + if not lines: + sleep(0.3) + error.seek(where) + else: + log.info(lines.decode().rstrip('\n')) + + error.close() + output.close() + return sub.returncode + +def clone_package(pkg, log): + """ + Clone Git repository from URL embedded in pkg to local disk + + First cleanup any possible leftovers from previous builds + """ + + if args.clean: + # delete repository from disk + if os.path.isdir(pkg['path']): + log.debug("Cleaning '{}'".format(pkg['path'])) + shutil.rmtree(pkg['path']) + else: + if os.path.isdir(pkg['path']): + # Change current directory into Git repo for this package + os.chdir(pkg['path']) + + bashCommand = 'git clean -d -x --force && git reset --hard ' + pkg['commit'] + return call(bashCommand, log) + + # resolve given tag to commit id to use shallow clone + bashCommand = 'git clone ' + pkg['url'] + if pkg['tag']: + bashCommand += ' --branch ' + pkg['tag'] + elif pkg['branch']: + bashCommand += ' --depth 1 --branch ' + pkg['branch'] + + bashCommand += ' ' + pkg['path'] + return call(bashCommand, log) + + +def build_package(pkg, log=None): + """ + Generate Debian package from passed 'pkg' + """ + + # Change current directory into Git repo for this package + os.chdir(pkg['path']) + + # Overwrite custom build command if required, e.g. libyang + bashCommand = '' + if pkg['custombuild_cmd']: + bashCommand = pkg['custombuild_cmd'] + else: + # Build package + bashCommand = 'dpkg-buildpackage -uc -us -tc -b' + if args.parallel: + bashCommand += ' -j' + str(os.cpu_count()) + + return call(bashCommand, log) + +# a List of all Vyatta/VyOS based packages +vyos_packages = ['vyatta-bash', + 'vyatta-cfg', + 'vyatta-op', + 'vyatta-cfg-system', + 'vyatta-cfg-firewall', + 'vyatta-op-firewall', + 'vyatta-cfg-vpn', + 'vyatta-op-vpn', + 'vyatta-cfg-qos', + 'vyatta-op-qos', + 'vyatta-cfg-op-pppoe', + 'vyatta-openvpn', + 'vyatta-conntrack', + 'vyatta-conntrack-sync', + 'vyatta-nat', + 'vyatta-config-mgmt', + 'vyatta-config-migrate', + 'vyatta-zone', + 'vyatta-cluster', + 'vyatta-eventwatch', + 'vyatta-webproxy', + 'vyatta-cfg-quagga', + 'vyatta-op-quagga', + 'vyatta-op-dhcp-server', + 'vyatta-wireless', + 'vyatta-wirelessmodem', + 'vyatta-wanloadbalance', + 'vyatta-netflow', + 'vyatta-lldp', + 'vyatta-ipv6-rtradv', + 'vyatta-ravpn', + 'vyos-nhrp', + 'vyos-world', + 'vyos-1x', + 'vyatta-iproute', + 'vyatta-quagga', + 'vyos-strongswan', + 'vyos-vmwaretools-scripts', + 'vyos-netplug', + 'vyos-xe-guest-utilities', + 'vyatta-biosdevname', + 'vyos-opennhrp', + 'vyos-salt-minion', + 'xl2tpd', + 'mdns-repeater', + 'udp-broadcast-relay', + 'pmacct', + 'ddclient', + 'igmpproxy', + 'eventwatchd', + 'conntrack-tools'] + +# Special packages mean packages which are located no in the VyOS namespace +# or require fancy build instructions +pkg_special = [] + +# libvyosconfig/ipaddrcheck uses a different default branch +libvyosconfig_build_cmd = "eval $(opam env --root=/opt/opam --set-root) && " \ + "dpkg-buildpackage -b -us -uc -tc" +pkg_special.append( add_package('libvyosconfig', branch='master', custombuild_cmd=libvyosconfig_build_cmd)) +pkg_special.append( add_package('ipaddrcheck', branch='master')) + +# Packages where we directly build the upstream source +pkg_special.append( add_package('hvinfo', url='https://github.com/dmbaturin/hvinfo.git', branch='master') ) +pkg_special.append( add_package('lldpd', url='https://github.com/vincentbernat/lldpd.git', branch='master', tag='1.0.3') ) + +# +# FreeRangeRouting (FRR) packages +# +pkg_special.append( add_package('rtrlib', url='https://github.com/rtrlib/rtrlib.git', branch='master', tag='v0.6.3') ) + +frr_build_cmd = './tools/tarsource.sh -V && dpkg-buildpackage -us -uc -Ppkg.frr.rtrlib -d' +pkg_special.append( add_package('frr', url='https://github.com/FRRouting/frr.git', branch='master', tag='frr-7.0', custombuild_cmd=frr_build_cmd) ) +#libyang_build_cmd = 'mkdir build && cd build && cmake .. && make build-deb && mv debs/* ' + repo_root + '/packages' +#pkg_special.append( add_package('libyang', url='https://github.com/opensourcerouting/libyang.git', commit='179da47', branch='master', custombuild_cmd=libyang_build_cmd) ) + + +# +# Linux (VyOS) Kernel +# +kernel_build_cmd = "export VERSION=$(grep '^VERSION' Makefile | grep -Eo '[0-9]{1,4}') && " \ + "export PATCHLEVEL=$(grep '^PATCHLEVEL' Makefile | grep -Eo '[0-9]{1,4}') && " \ + "export SUBLEVEL=$(grep '^SUBLEVEL' Makefile | grep -Eo '[0-9]{1,4}') && " \ + "echo ${VERSION}.${PATCHLEVEL}.${SUBLEVEL}-amd64-vyos > " + repo_root + "/data/kernel_version && " \ + "make x86_64_vyos_defconfig && " \ + "LOCALVERSION='' make-kpkg --rootcmd fakeroot --initrd --append_to_version -amd64-vyos " \ + " --revision=${VERSION}.${PATCHLEVEL}.${SUBLEVEL}-0 kernel_headers kernel_image -j" + str(os.cpu_count()) +pkg_special.append( add_package('vyos-kernel', branch='linux-vyos-4.19.y', custombuild_cmd=kernel_build_cmd) ) + + + +# +# WireGuard Kernel Module +# +wireguard_build_cmd = "echo 'src/wireguard.ko /lib/modules/'$(cat " + repo_root + "/data/kernel_version) > debian/wireguard-modules.install && " \ + "KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count()) +pkg_special.append( add_package('vyos-wireguard', custombuild_cmd=wireguard_build_cmd) ) + + + +# +# Accell-PPP Package and Kernel Module +# +accel_ppp_build_cmd = "echo 'lib/modules/'$(cat " + repo_root + "/data/kernel_version)'/extra/*.ko' > debian/vyos-accel-ppp-ipoe-kmod.install && " \ + "sed -i 's#[0-9].[0-9][0-9].[0-9]*-amd64-vyos#'$(cat " + repo_root + "/data/kernel_version)'#g' debian/rules && " \ + "KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count()) +pkg_special.append( add_package('vyos-accel-ppp', custombuild_cmd=accel_ppp_build_cmd) ) + + +# A list of all packages we will build in the end +pkg_build = [] + + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-v', '--verbose', action='count', default=0, help='Increase logging verbosity for each occurance') + parser.add_argument('-c', '--clean', action='store_true', help='Re-clone required Git repositories') + parser.add_argument('-l', '--list-packages', action='store_true', help='List all packages to build') + parser.add_argument('-b', '--build', nargs='+', help='Whitespace separated list of packages to build') + parser.add_argument('-f', '--fetch', action='store_true', help='Fetch sources only, no build') + parser.add_argument('-p', '--parallel', action='store_true', help='Build on all CPUs') + + + args = parser.parse_args() + + levels = [ logging.INFO, logging.WARNING, logging.DEBUG ] + level = levels[min(len(levels)-1,args.verbose)] # capped to number of levels + logging.basicConfig(level=level, format="%(asctime)s %(name)s %(message)s") + + print("Using vyos-build repository ('{}') commit '{}'\n".format(repo_root, repo_sha)) + + if args.list_packages: + print("Individual packages available for build:") + for pkg in vyos_packages: + print(' * ' + pkg) + for pkg in pkg_special: + print(' * ' + pkg['name']) + + sys.exit(0) + + # + # Only add selective packages to the build list + # + if args.build: + # NOTE: remove double added packages from list + for target in args.build: + if target in vyos_packages: + pkg_build.append(add_package( target )) + continue + + found = False + for pkg in pkg_special: + if target == pkg['name']: + found = True + # package already formed + pkg_build.append( pkg ) + break + + if not found: + print("Invalid choice '" + target + "', for -b/--build use --list-packages for complete list!") + sys.exit(1) + + else: + # Add all VyOS packages to the package list + for pkg in vyos_packages: + pkg_build.append(add_package( pkg )) + + # We also wan't to build all of our special packages + for pkg in pkg_special: + pkg_build.append( pkg ) + + # Build all VyOS packages (packages found on https://github.com/vyos + # and referenced in vyos_packages) + for pkg in pkg_build: + # Create a logging instance per package + log = logging.getLogger(pkg['name']) + + ret = clone_package(pkg, log) + if ret: + log.error("ERROR cloning source") + sys.exit(1) + else: + # only build packages if fetch flag is not set + if not args.fetch: + ret = build_package(pkg, log) + if ret: + log.error("ERROR building source") + sys.exit(1) + + sys.exit(0) |