diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/build-packages | 370 |
1 files changed, 0 insertions, 370 deletions
diff --git a/scripts/build-packages b/scripts/build-packages deleted file mode 100755 index df7b8ed1..00000000 --- a/scripts/build-packages +++ /dev/null @@ -1,370 +0,0 @@ -#!/usr/bin/env python3 - -import os -import subprocess -import sys -import shutil -import argparse -import logging - -current_working_directory = os.getcwd() -repo_root = subprocess.check_output('git rev-parse --show-toplevel', shell=True, universal_newlines=True).rstrip('\n') -repo_sha = subprocess.check_output('git rev-parse --short=12 HEAD', shell=True, universal_newlines=True).rstrip('\n') - -def add_package(name, url=None, commit=None, branch='current', tag=None, shallow=True, custombuild_cmd=None): - """ - Build up source package with URL and build commands executed during the later - called build_package step. - - If no additional information is passed we will use the latest commit from current - branch - - If no URL is passed we assume it's a regular VyOS package from the VyOS Github - namespace at https://github.com/vyos - """ - - if not url: - url = 'https://github.com/vyos/' + name + '.git' - - package = { - 'name': name, - 'url': url, - 'commit': commit, - 'tag': tag, - 'branch': branch, - 'shallow': shallow, - 'path': repo_root + '/packages/' + name, - 'custombuild_cmd': custombuild_cmd - } - - return package - - - -def call(bashCommand, log, **kwargs): - """ - Run command with arguments. Wait for command to complete. - - Send output to logging module passed as 'log'. - """ - - from subprocess import Popen, PIPE, STDOUT, check_output, CalledProcessError - from tempfile import TemporaryFile - from time import sleep - - log.debug("Executing '{}'".format(bashCommand)) - - # code borrowsed from: - # https://stackoverflow.com/questions/38374063/python-can-we-use-tempfile-with-subprocess-to-get-non-buffering-live-output-in-p - # the temp file will be automatically cleaned up - output = TemporaryFile() - error = TemporaryFile() - - kwargs['stdout'] = output - kwargs['stderr'] = error - kwargs['shell'] = True - kwargs['universal_newlines'] = True - - sub = Popen(bashCommand, **kwargs) - while sub.poll() is None: - where = output.tell() - lines = output.readline() - if not lines: - sleep(0.3) - output.seek(where) - else: - log.debug(lines.decode().rstrip('\n')) - - where = error.tell() - lines = error.readline() - if not lines: - sleep(0.3) - error.seek(where) - else: - log.info(lines.decode().rstrip('\n')) - - - # A last write needed after subprocess ends - log.debug(output.read().decode().rstrip('\n')) - log.info(error.read().decode().rstrip('\n')) - - error.close() - output.close() - return sub.returncode - -def clone_package(pkg, log): - """ - Clone Git repository from URL embedded in pkg to local disk - - First cleanup any possible leftovers from previous builds - """ - - if args.keep and os.path.isdir(pkg['path']): - log.debug("Keep possibly modified package '{}'".format(pkg['path'])) - return False - elif args.clean: - # delete repository from disk - if os.path.isdir(pkg['path']): - log.debug("Cleaning '{}'".format(pkg['path'])) - shutil.rmtree(pkg['path']) - else: - if os.path.isdir(pkg['path']): - # Change current directory into Git repo for this package - os.chdir(pkg['path']) - - bashCommand = 'git clean -d -x --force && git reset --hard ' + pkg['commit'] - return call(bashCommand, log) - - if pkg['commit'] and pkg['shallow']: - log.debug("Package '{}' has both commit and shallow set, unsetting shallow to do a full clone.".format(pkg['name'])) - pkg['shallow']=False - - bashCommand = 'git clone ' + pkg['url'] - - if pkg['shallow']: - bashCommand += ' --depth 1' - - if pkg['tag']: - bashCommand += ' --branch ' + pkg['tag'] - elif pkg['branch']: - bashCommand += ' --branch ' + pkg['branch'] - - bashCommand += ' ' + pkg['path'] - - if pkg['commit']: - bashCommand += ' && cd ' + pkg['path'] + ' && git reset --hard ' + pkg['commit'] - - return call(bashCommand, log) - - -def build_package(pkg, log=None): - """ - Generate Debian package from passed 'pkg' - """ - - # Change current directory into Git repo for this package - os.chdir(pkg['path']) - - # Overwrite custom build command if required, e.g. libyang - bashCommand = '' - if pkg['custombuild_cmd']: - bashCommand = pkg['custombuild_cmd'] - else: - # Build package - bashCommand = 'dpkg-buildpackage -uc -us -tc -b' - if args.parallel: - bashCommand += ' -j' + str(os.cpu_count()) - - return call(bashCommand, log) - -# a List of all Vyatta/VyOS based packages -vyos_packages = ['vyatta-bash', - 'vyatta-cfg', - 'vyatta-op', - 'vyatta-cfg-system', - 'vyatta-cfg-firewall', - 'vyatta-op-firewall', - 'vyatta-cfg-vpn', - 'vyatta-op-vpn', - 'vyatta-cfg-qos', - 'vyatta-op-qos', - 'vyatta-cfg-op-pppoe', - 'vyatta-nat', - 'vyatta-config-mgmt', - 'vyatta-config-migrate', - 'vyatta-zone', - 'vyatta-cluster', - 'vyatta-eventwatch', - 'vyatta-webproxy', - 'vyatta-cfg-quagga', - 'vyatta-op-quagga', - 'vyatta-wanloadbalance', - 'vyatta-ravpn', - 'vyos-nhrp', - 'vyos-world', - 'vyatta-iproute', - 'vyatta-biosdevname', - 'vyos-opennhrp', - 'udp-broadcast-relay', - 'vyos-1x', - 'vyatta-conntrack', - 'vyatta-conntrack-sync', - 'vyos-xe-guest-utilities', - 'vyos-netplug', - 'ddclient', - 'live-boot'] - -# Special packages mean packages which are located no in the VyOS namespace -# or require fancy build instructions -pkg_special = [] - -# libvyosconfig/ipaddrcheck uses a different default branch -libvyosconfig_build_cmd = "eval $(opam env --root=/opt/opam --set-root) && " \ - "dpkg-buildpackage -b -us -uc -tc" -pkg_special.append( add_package('libvyosconfig', branch='master', custombuild_cmd=libvyosconfig_build_cmd)) -pkg_special.append( add_package('ipaddrcheck', branch='master')) - -# Packages where we directly build the upstream source -pkg_special.append( add_package('hvinfo', url='https://github.com/dmbaturin/hvinfo.git', branch='master') ) - -# VyOS strongswan ships additional python3-vici packages required by vyos-1x and this is not build by default -vyos_strongswan_build_cmd = "dpkg-buildpackage -b -us -uc -tc && " \ - "autoreconf -i && ./configure --enable-python-eggs && " \ - "cd src/libcharon/plugins/vici/python && make && " \ - "python3 setup.py --command-packages=stdeb.command bdist_deb && " \ - "mv ./deb_dist/*.deb " + repo_root + "/packages" -pkg_special.append( add_package('vyos-strongswan', custombuild_cmd=vyos_strongswan_build_cmd) ) - -# -# FreeRangeRouting (FRR) packages -# -pkg_special.append( add_package('rtrlib', url='https://github.com/rtrlib/rtrlib.git', branch='master', tag='v0.6.3') ) - -frr_build_cmd = './tools/tarsource.sh -V && dpkg-buildpackage -us -uc -Ppkg.frr.rtrlib -d' -pkg_special.append( add_package('frr', url='https://github.com/FRRouting/frr.git', branch='master', tag='frr-7.0', custombuild_cmd=frr_build_cmd) ) -#libyang_build_cmd = 'mkdir build && cd build && cmake .. && make build-deb && mv debs/* ' + repo_root + '/packages' -#pkg_special.append( add_package('libyang', url='https://github.com/opensourcerouting/libyang.git', commit='179da47', branch='master', custombuild_cmd=libyang_build_cmd) ) - - -# -# We use keepalived from Debian Buster -# -pkg_special.append( add_package('keepalived', url='https://salsa.debian.org/ipvs-team/pkg-keepalived.git', branch='master', commit='eae91c81')) - -# -# Linux (VyOS) Kernel -# -kernel_build_cmd = "make x86_64_vyos_defconfig && " \ - "sed -i 's/\"kernel_version\": \"[0-9].[0-9][0-9].[0-9]*\"/\"kernel_version\": \"'$(make kernelversion)'\"/' " + repo_root + "/data/defaults.json && " \ - "make bindeb-pkg LOCALVERSION='-amd64-vyos' KDEB_PKGVERSION=$(make kernelversion)-1 -j $(getconf _NPROCESSORS_ONLN)" -pkg_special.append( add_package('vyos-kernel', branch='linux-vyos-4.19.y', custombuild_cmd=kernel_build_cmd) ) - - - -# -# Accell-PPP Package and Kernel Module -# -accel_ppp_build_cmd = "echo 'lib/modules/'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\")-amd64-vyos/extra/*.ko > debian/vyos-accel-ppp-ipoe-kmod.install && " \ - "sed -i 's#[0-9].[0-9][0-9].[0-9]*-amd64-vyos#'$(cat " + repo_root + "/data/defaults.json | jq '.kernel_version' | tr -d \\\")'-amd64-vyos#g' debian/rules && " \ - "KERNELDIR=" + repo_root + "/packages/vyos-kernel dpkg-buildpackage -b -us -uc -tc -j" + str(os.cpu_count()) -pkg_special.append( add_package('vyos-accel-ppp', custombuild_cmd=accel_ppp_build_cmd) ) - -# -# mdns-repeater branch for current is master -# - -pkg_special.append( add_package('mdns-repeater', branch='master') ) - - -# A list of all packages we will build in the end -pkg_build = [] - - - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - exclusive = parser.add_mutually_exclusive_group(required=False) - exclusive.add_argument('-c', '--clean', action='store_true', help='Re-clone required Git repositories') - exclusive.add_argument('-k', '--keep', action='store_true', help='Keep modified Git repositories') - exclusive.add_argument('-f', '--fetch', action='store_true', help='Fetch sources only, no build') - - parser.add_argument('-v', '--verbose', action='count', default=0, help='Increase logging verbosity for each occurance') - parser.add_argument('-l', '--list-packages', action='store_true', help='List all packages to build') - parser.add_argument('-b', '--build', nargs='+', help='Whitespace separated list of packages to build') - parser.add_argument('-p', '--parallel', action='store_true', help='Build on all CPUs') - parser.add_argument('--blacklist', nargs='+', help='Do not build/report packages when calling --list') - - - args = parser.parse_args() - - levels = [ logging.INFO, logging.WARNING, logging.DEBUG ] - level = levels[min(len(levels)-1,args.verbose)] # capped to number of levels - logging.basicConfig(level=level, format="%(asctime)s %(name)s %(message)s") - - print("Using vyos-build repository ('{}') commit '{}'\n".format(repo_root, repo_sha)) - - # - # Exclude packages from build process, - # also do not list them on --list - # - if args.blacklist: - for exclude in args.blacklist: - if exclude in vyos_packages: - vyos_packages.remove(exclude) - continue - - found = False - for pkg in pkg_special: - if exclude == pkg['name']: - found = True - # package already formed - pkg_special.remove(pkg) - break - - if not found: - print("Invalid choice '" + exclude + "', --list-packages for complete list!") - sys.exit(1) - - # - # List all available (to be build) packages - # - if args.list_packages: - print("Individual packages available for build:") - for pkg in vyos_packages: - print(' * ' + pkg) - for pkg in pkg_special: - print(' * ' + pkg['name']) - - sys.exit(0) - - # - # Only add selective packages to the build list - # - if args.build: - # NOTE: remove double added packages from list - for target in args.build: - if target in vyos_packages: - pkg_build.append(add_package( target )) - continue - - found = False - for pkg in pkg_special: - if target == pkg['name']: - found = True - # package already formed - pkg_build.append( pkg ) - break - - if not found: - print("Invalid choice '" + target + "', for -b/--build use --list-packages for complete list!") - sys.exit(1) - - else: - # Add all VyOS packages to the package list - for pkg in vyos_packages: - pkg_build.append(add_package( pkg )) - - # We also wan't to build all of our special packages - for pkg in pkg_special: - pkg_build.append( pkg ) - - # Build all VyOS packages (packages found on https://github.com/vyos - # and referenced in vyos_packages) - for pkg in pkg_build: - # Create a logging instance per package - log = logging.getLogger(pkg['name']) - - ret = clone_package(pkg, log) - if ret: - log.error("ERROR cloning source") - sys.exit(1) - else: - # only build packages if fetch flag is not set - if not args.fetch: - ret = build_package(pkg, log) - if ret: - log.error("ERROR building source") - sys.exit(1) - - sys.exit(0) |