#!/usr/bin/env python3 import os import subprocess import sys import shutil import argparse import logging current_working_directory = os.getcwd() repo_root = subprocess.check_output('git rev-parse --show-toplevel', shell=True, universal_newlines=True).rstrip('\n') repo_sha = subprocess.check_output('git rev-parse --short=12 HEAD', shell=True, universal_newlines=True).rstrip('\n') def add_package(name, url=None, commit='HEAD', branch='current', tag=None, custombuild_cmd=None): """ Build up source package with URL and build commands executed during the later called build_package step. If no additional information is passed we will use the latest commit from current branch If no URL is passed we assume it's a regular VyOS package from the VyOS Github namespace at https://github.com/vyos """ if not url: url = 'https://github.com/vyos/' + name + '.git' package = { 'name': name, 'url': url, 'commit': commit, 'tag': tag, 'branch': branch, 'path': repo_root + '/packages/' + name, 'custombuild_cmd': custombuild_cmd } return package def call(bashCommand, log, **kwargs): """ Run command with arguments. Wait for command to complete. Send output to logging module passed as 'log'. """ from subprocess import Popen, PIPE, STDOUT, check_output, CalledProcessError from tempfile import TemporaryFile from time import sleep log.debug("Executing '{}'".format(bashCommand)) # code borrowsed from: # https://stackoverflow.com/questions/38374063/python-can-we-use-tempfile-with-subprocess-to-get-non-buffering-live-output-in-p # the temp file will be automatically cleaned up output = TemporaryFile() error = TemporaryFile() kwargs['stdout'] = output kwargs['stderr'] = error kwargs['shell'] = True kwargs['universal_newlines'] = True sub = Popen(bashCommand, **kwargs) while sub.poll() is None: where = output.tell() lines = output.readline() if not lines: sleep(0.3) output.seek(where) else: log.debug(lines.decode().rstrip('\n')) where = error.tell() lines = error.readline() if not lines: sleep(0.3) error.seek(where) else: log.info(lines.decode().rstrip('\n')) # A last write needed after subprocess ends log.debug(output.read().decode().rstrip('\n')) log.info(error.read().decode().rstrip('\n')) error.close() output.close() return sub.returncode def clone_package(pkg, log): """ Clone Git repository from URL embedded in pkg to local disk First cleanup any possible leftovers from previous builds """ if args.keep: log.debug("Keep possibly modified package '{}'".format(pkg['path'])) return False elif args.clean: # delete repository from disk if os.path.isdir(pkg['path']): log.debug("Cleaning '{}'".format(pkg['path'])) shutil.rmtree(pkg['path']) else: if os.path.isdir(pkg['path']): # Change current directory into Git repo for this package os.chdir(pkg['path']) bashCommand = 'git clean -d -x --force && git reset --hard ' + pkg['commit'] return call(bashCommand, log) # resolve given tag to commit id to use shallow clone bashCommand = 'git clone ' + pkg['url'] if pkg['tag']: bashCommand += ' --branch ' + pkg['tag'] elif pkg['branch']: bashCommand += ' --depth 1 --branch ' + pkg['branch'] bashCommand += ' ' + pkg['path'] return call(bashCommand, log) def build_package(pkg, log=None): """ Generate Debian package from passed 'pkg' """ # Change current directory into Git repo for this package os.chdir(pkg['path']) # Overwrite custom build command if required, e.g. libyang bashCommand = '' if pkg['custombuild_cmd']: bashCommand = pkg['custombuild_cmd'] else: # Build package bashCommand = 'dpkg-buildpackage -uc -us -tc -b' if args.parallel: bashCommand += ' -j' + str(os.cpu_count()) return call(bashCommand, log) # a List of all Vyatta/VyOS based packages vyos_packages = ['vyatta-bash', 'vyatta-cfg', 'vyatta-op', 'vyatta-cfg-system', 'vyatta-cfg-firewall', 'vyatta-op-firewall', 'vyatta-cfg-vpn', 'vyatta-op-vpn', 'vyatta-cfg-qos', 'vyatta-op-qos', 'vyatta-cfg-op-pppoe', 'vyatta-conntrack', 'vyatta-conntrack-sync', 'vyatta-nat', 'vyatta-config-mgmt', 'vyatta-config-migrate', 'vyatta-zone', 'vyatta-cluster', 'vyatta-eventwatch', 'vyatta-webproxy', 'vyatta-cfg-quagga', 'vyatta-op-quagga', 'vyatta-wireless', 'vyatta-wirelessmodem', 'vyatta-wanloadbalance', 'vyatta-netflow', 'vyatta-lldp', 'vyatta-ipv6-rtradv', 'vyatta-ravpn', 'vyos-replace', 'vyos-nhrp', 'vyos-world', 'vyos-1x', 'vyatta-iproute', 'vyos-vmwaretools-scripts', 'vyos-netplug', 'vyos-xe-guest-utilities', 'vyatta-biosdevname', 'vyos-opennhrp', 'vyos-salt-minion', 'xl2tpd', 'mdns-repeater', 'udp-broadcast-relay', 'pmacct', 'ddclient', 'igmpproxy', 'eventwatchd', 'live-boot', 'conntrack-tools'] # Special packages mean packages which are located no in the VyOS namespace # or require fancy build instructions pkg_special = [] # libvyosconfig/ipaddrcheck uses a different default branch libvyosconfig_build_cmd = "eval $(opam env --root=/opt/opam --set-root) && " \ "dpkg-buildpackage -b -us -uc -tc" pkg_special.append( add_package('libvyosconfig', branch='master', custombuild_cmd=libvyosconfig_build_cmd)) pkg_special.append( add_package('ipaddrcheck', branch='master')) # Packages where we directly build the upstream source pkg_special.append( add_package('hvinfo', url='https://github.com/dmbaturin/hvinfo.git', branch='master') ) pkg_special.append( add_package('lldpd', url='https://github.com/vincentbernat/lldpd.git', branch='master', tag='1.0.3') ) pkg_special.append( add_package('ethtool', url='https://salsa.debian.org/kernel-team/ethtool.git', branch='master', tag='debian/1%4.19-1') ) pkg_special.append( add_package('bgpq3', url='https://salsa.debian.org/debian/bgpq3.git', branch='master', tag='debian/0.1.33-1') ) # VyOS strongswan ships additional python3-vici packages required by vyos-1x and this is not build by default vyos_strongswan_build_cmd = "dpkg-buildpackage -b -us -uc -tc && " \ "autoreconf -i && ./configure --enable-python-eggs && " \ "cd src/libcharon/plugins/vici/python && make && " \ "python3 setup.py --command-packages=stdeb.command bdist_deb && " \ "mv ./deb_dist/*.deb " + repo_root + "/packages" pkg_special.append( add_package('vyos-strongswan', custombuild_cmd=vyos_strongswan_build_cmd) ) # # FreeRangeRouting (FRR) packages # pkg_special.append( add_package('rtrlib', url='https://github.com/rtrlib/rtrlib.git', branch='master', tag='v0.6.3') ) frr_build_cmd = './tools/tarsource.sh -V && dpkg-buildpackage -us -uc -Ppkg.frr.rtrlib -d' pkg_special.append( add_package('frr', url='https://github.com/FRRouting/frr.git', branch='master', tag='frr-7.0', custombuild_cmd=frr_build_cmd) ) #libyang_build_cmd = 'mkdir build && cd build && cmake .. && make build-deb && mv debs/* ' + repo_root + '/packages' #pkg_special.append( add_package('libyang', url='https://github.com/opensourcerouting/libyang.git', commit='179da47', branch='master', custombuild_cmd=libyang_build_cmd) ) # # We use keepalived from Debian Buster # keepalived_build_cmd = "sed -i 's/debhelper (>= 11)/debhelper (>= 9)/' debian/control && " \ "echo 9 > debian/compat && " \ "dpkg-buildpackage -b -us -uc -tc -j$(getconf _NPROCESSORS_ONLN)" pkg_special.append( add_package('keepalived', url='https://salsa.debian.org/ipvs-team/pkg-keepalived.git', branch='master', commit='eae91c81', custombuild_cmd=keepalived_build_cmd) ) # # Net-SNMP # # net-snmp conflicts with already installed libsnmp-dev thus we need to build with -d net_snmp_build_cmd = "dpkg-buildpackage -b -us -uc -tc -d -j$(getconf _NPROCESSORS_ONLN)" pkg_special.append( add_package('net-snmp', custombuild_cmd=net_snmp_build_cmd) ) # # Linux (VyOS) Kernel # kernel_build_cmd = "make x86_64_vyos_defconfig && " \ "sed -i 's/\"kernel_version\": \"[0-9].[0-9][0-9].[0-9]*\"/\"kernel_version\": \"'$(make kernelversion)'\"/' " + repo_root + "/data/defaults.json && " \ "make bindeb-pkg LOCALVERSION='-amd64-vyos' KDEB_PKGVERSION=$(make kernelversion)-1 -j $(getconf _NPROCESSORS_ONLN)" pkg_special.append( add_package('vyos-kernel', branch='linux-vyos-4.19.y', custombuild_cmd=kernel_build_cmd) ) # # WireGuard Kernel Module # wireguard_build_cmd = "echo 'src/wireguard.ko /lib/modules/'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\")-amd64-vyos/extra > debian/wireguard-modules.install && " \ "KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count()) pkg_special.append( add_package('vyos-wireguard', custombuild_cmd=wireguard_build_cmd) ) # # Accell-PPP Package and Kernel Module # accel_ppp_build_cmd = "echo 'lib/modules/'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\")-amd64-vyos/extra/*.ko > debian/vyos-accel-ppp-ipoe-kmod.install && " \ "sed -i 's#[0-9].[0-9][0-9].[0-9]*-amd64-vyos#'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\")'-amd64-vyos#g' debian/rules && " \ "KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count()) pkg_special.append( add_package('vyos-accel-ppp', custombuild_cmd=accel_ppp_build_cmd) ) # A list of all packages we will build in the end pkg_build = [] if __name__ == '__main__': parser = argparse.ArgumentParser() exclusive = parser.add_mutually_exclusive_group(required=False) exclusive.add_argument('-c', '--clean', action='store_true', help='Re-clone required Git repositories') exclusive.add_argument('-k', '--keep', action='store_true', help='Keep modified Git repositories') exclusive.add_argument('-f', '--fetch', action='store_true', help='Fetch sources only, no build') parser.add_argument('-v', '--verbose', action='count', default=0, help='Increase logging verbosity for each occurance') parser.add_argument('-l', '--list-packages', action='store_true', help='List all packages to build') parser.add_argument('-b', '--build', nargs='+', help='Whitespace separated list of packages to build') parser.add_argument('-p', '--parallel', action='store_true', help='Build on all CPUs') parser.add_argument('--blacklist', nargs='+', help='Do not build/report packages when calling --list') args = parser.parse_args() levels = [ logging.INFO, logging.WARNING, logging.DEBUG ] level = levels[min(len(levels)-1,args.verbose)] # capped to number of levels logging.basicConfig(level=level, format="%(asctime)s %(name)s %(message)s") print("Using vyos-build repository ('{}') commit '{}'\n".format(repo_root, repo_sha)) # # Exclude packages from build process, # also do not list them on --list # if args.blacklist: for exclude in args.blacklist: if exclude in vyos_packages: vyos_packages.remove(exclude) continue found = False for pkg in pkg_special: if exclude == pkg['name']: found = True # package already formed pkg_special.remove(pkg) break if not found: print("Invalid choice '" + exclude + "', --list-packages for complete list!") sys.exit(1) # # List all available (to be build) packages # if args.list_packages: print("Individual packages available for build:") for pkg in vyos_packages: print(' * ' + pkg) for pkg in pkg_special: print(' * ' + pkg['name']) sys.exit(0) # # Only add selective packages to the build list # if args.build: # NOTE: remove double added packages from list for target in args.build: if target in vyos_packages: pkg_build.append(add_package( target )) continue found = False for pkg in pkg_special: if target == pkg['name']: found = True # package already formed pkg_build.append( pkg ) break if not found: print("Invalid choice '" + target + "', for -b/--build use --list-packages for complete list!") sys.exit(1) else: # Add all VyOS packages to the package list for pkg in vyos_packages: pkg_build.append(add_package( pkg )) # We also wan't to build all of our special packages for pkg in pkg_special: pkg_build.append( pkg ) # Build all VyOS packages (packages found on https://github.com/vyos # and referenced in vyos_packages) for pkg in pkg_build: # Create a logging instance per package log = logging.getLogger(pkg['name']) ret = clone_package(pkg, log) if ret: log.error("ERROR cloning source") sys.exit(1) else: # only build packages if fetch flag is not set if not args.fetch: ret = build_package(pkg, log) if ret: log.error("ERROR building source") sys.exit(1) sys.exit(0)