diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/build-packages | 16 | ||||
| -rwxr-xr-x | scripts/check-qemu-install | 364 | ||||
| -rwxr-xr-x | scripts/live-build-config | 15 |
3 files changed, 376 insertions, 19 deletions
diff --git a/scripts/build-packages b/scripts/build-packages index e8508599..8d1447dd 100755 --- a/scripts/build-packages +++ b/scripts/build-packages @@ -11,7 +11,7 @@ current_working_directory = os.getcwd() repo_root = subprocess.check_output('git rev-parse --show-toplevel', shell=True, universal_newlines=True).rstrip('\n') repo_sha = subprocess.check_output('git rev-parse --short=12 HEAD', shell=True, universal_newlines=True).rstrip('\n') -def add_package(name, url=None, commit='HEAD', branch='current', tag=None, custombuild_cmd=None): +def add_package(name, url=None, commit='HEAD', branch='equuleus', tag=None, custombuild_cmd=None): """ Build up source package with URL and build commands executed during the later called build_package step. @@ -157,8 +157,6 @@ vyos_packages = ['vyatta-bash', 'vyatta-cfg-qos', 'vyatta-op-qos', 'vyatta-cfg-op-pppoe', - 'vyatta-conntrack', - 'vyatta-conntrack-sync', 'vyatta-nat', 'vyatta-config-mgmt', 'vyatta-config-migrate', @@ -178,17 +176,18 @@ vyos_packages = ['vyatta-bash', 'vyos-replace', 'vyos-nhrp', 'vyos-world', - 'vyos-1x', 'vyatta-iproute', 'vyos-vmwaretools-scripts', - 'vyos-netplug', - 'vyos-xe-guest-utilities', 'vyatta-biosdevname', 'vyos-opennhrp', 'vyos-salt-minion', - 'xl2tpd', 'mdns-repeater', 'udp-broadcast-relay', + 'vyos-1x', + 'vyatta-conntrack', + 'vyatta-conntrack-sync', + 'vyos-xe-guest-utilities', + 'vyos-netplug', 'pmacct', 'ddclient', 'igmpproxy', @@ -208,9 +207,6 @@ pkg_special.append( add_package('ipaddrcheck', branch='master')) # Packages where we directly build the upstream source pkg_special.append( add_package('hvinfo', url='https://github.com/dmbaturin/hvinfo.git', branch='master') ) -pkg_special.append( add_package('lldpd', url='https://github.com/vincentbernat/lldpd.git', branch='master', tag='1.0.3') ) -pkg_special.append( add_package('ethtool', url='https://salsa.debian.org/kernel-team/ethtool.git', branch='master', tag='debian/1%4.19-1') ) -pkg_special.append( add_package('bgpq3', url='https://salsa.debian.org/debian/bgpq3.git', branch='master', tag='debian/0.1.33-1') ) # VyOS strongswan ships additional python3-vici packages required by vyos-1x and this is not build by default vyos_strongswan_build_cmd = "dpkg-buildpackage -b -us -uc -tc && " \ diff --git a/scripts/check-qemu-install b/scripts/check-qemu-install new file mode 100755 index 00000000..a7c36c78 --- /dev/null +++ b/scripts/check-qemu-install @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019, VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# File: check-qemu-install +# Purpose: +# This script installs a system on a emulated qemu host to verify +# that the iso produced is installable and boots. +# after the iso is booted from disk it also tries to execute the +# vyos-smoketest script to verify checks there. +# +# For now it will not fail on failed smoketest but will fail on +# install and boot errors. +# Arguments: +# iso iso image to install +# [disk] disk filename to use, if none is provided it +# is autogenerated +# [--keep] Keep the disk image after completion +# [--logfile] name of logfile to save, defaulting to stdout +# [--silent] only print on errors +# [--debug] print all communication with the device + + +import pexpect +import sys +import os +import time +import argparse +import subprocess +import random +import traceback +import logging +import re +from io import BytesIO, StringIO +from datetime import datetime + +EXCEPTION = 0 +now = datetime.now() + + +parser = argparse.ArgumentParser(description='Install and start a test VyOS vm.') +parser.add_argument('iso', help='ISO file to install') +parser.add_argument('disk', help='name of disk image file', + nargs='?', + default='testinstall-{}-{}.img'.format(now.strftime('%Y%m%d-%H%M%S'), + "%04x" % random.randint(0,65535))) +parser.add_argument('--keep', help='Do not remove disk-image after installation', + action='store_true', + default=False) +parser.add_argument('--silent', help='Do not show output on stdout unless an error has occured', + action='store_true', + default=False) +parser.add_argument('--debug', help='Send all debug output to stdout', + action='store_true', + default=False) +parser.add_argument('--logfile', help='Log to file') +parser.add_argument('--no-kvm', help='Disable use of kvm', + action='store_true', + default=False) + +args = parser.parse_args() + +class StreamToLogger(object): + """ + Fake file-like stream object that redirects writes to a logger instance. + """ + def __init__(self, logger, log_level=logging.INFO): + self.logger = logger + self.log_level = log_level + self.linebuf = b'' + self.ansi_escape = re.compile(r'\x1B[@-_][0-?]*[ -/]*[@-~]') + + def write(self, buf): + self.linebuf += buf + #print('.') + while b'\n' in self.linebuf: + f = self.linebuf.split(b'\n', 1) + if len(f) == 2: + self.logger.debug(self.ansi_escape.sub('', f[0].decode(errors="replace").rstrip())) + self.linebuf = f[1] + #print(f) + + + def flush(self): + pass + + +# Setting up logger +log = logging.getLogger() +log.setLevel(logging.DEBUG) + +stl = StreamToLogger(log) +formatter = logging.Formatter('%(asctime)s + %(relativeCreated)-7d - %(levelname)5s - %(message)s') + +handler = logging.StreamHandler(sys.stdout) +if args.silent: + handler.setLevel(logging.ERROR) +elif args.debug: + handler.setLevel(logging.DEBUG) +else: + handler.setLevel(logging.INFO) + +handler.setFormatter(formatter) +log.addHandler(handler) + + +if args.logfile: + filehandler = logging.FileHandler(args.logfile) + filehandler.setLevel(logging.DEBUG) + filehandler.setFormatter(formatter) + log.addHandler(filehandler) + + + +if args.silent: + output = BytesIO() +else: + output = sys.stdout.buffer + +if not os.path.isfile(args.iso): + log.error("Unable to find iso image to install") + sys.exit(1) + +if args.no_kvm: + log.error("KVM forced off by command line") + kvm=False +elif not os.path.exists("/dev/kvm"): + log.error("KVM is not enabled on host, proceeding with software emulation") + kvm=False +else: + kvm=True + +# Creating diskimage!! + +if not os.path.isfile(args.disk): + log.info("Creating Disk image {}".format(args.disk)) + c = subprocess.check_output(["qemu-img", "create", args.disk, "2G"]) + log.debug(c.decode()) +else: + log.info("Diskimage already exists, using the existing one") + + + +try: + ################################################# + # Installing image to disk + ################################################# + log.info("Installing system") + + cmd = """qemu-system-x86_64 \ + -name "TESTVM" \ + -m 1G \ + -nic user,model=virtio,mac=52:54:99:12:34:56,hostfwd=tcp::2299-:22 \ + -machine accel=kvm \ + {CPU} \ + -smp 2 \ + -vnc 0.0.0.0:99 \ + -nographic \ + -boot d -cdrom {CD} \ + {KVM} \ + -drive format=raw,file={DISK} + """.format(CD=args.iso, DISK=args.disk, KVM="" if kvm else "--no-kvm", CPU="-cpu host" if kvm else "") + log.debug("Executing command: {}".format(cmd)) + c = pexpect.spawn(cmd, logfile=stl) + + ################################################# + # Logging into VyOS system + ################################################# + try: + c.expect('Automatic boot in', timeout=10) + c.sendline('') + except pexpect.TIMEOUT: + log.warning("Did not find grub countdown window, ignoring") + + log.info('Waiting for login prompt') + c.expect('[Ll]ogin:', timeout=300) + c.sendline('vyos') + c.expect('[Pp]assword:', timeout=10) + c.sendline('vyos') + c.expect(r'vyos@vyos:~\$') + log.info('Logged in!') + + + ################################################# + # Installing into VyOS system + ################################################# + log.info("Starting installer") + c.sendline('install image') + c.expect('\nWould you like to continue?.*:') + c.sendline('yes') + log.info("Partitioning disk") + c.expect('\nPartition.*:') + c.sendline('') + c.expect('\nInstall the image on.*:') + c.sendline('') + c.expect(r'\nContinue\?.*:') + c.sendline('Yes') + c.expect('\nHow big of a root partition should I create?.*:') + c.sendline('') + log.info('Disk partitioned, installing') + c.expect('\nWhat would you like to name this image?.*:') + c.sendline('') + log.info('Copying files') + c.expect('\nWhich one should I copy to.*:', timeout=300) + c.sendline('') + log.info('Files Copied!') + c.expect('\nEnter password for user.*:') + c.sendline('vyos') + c.expect('\nRetype password for user.*:') + c.sendline('vyos') + c.expect('\nWhich drive should GRUB modify the boot partition on.*:') + c.sendline('') + c.expect(r'\nvyos@vyos:~\$') + log.info('system installed, shutting down') + + ################################################# + # Powering down installer + ################################################# + log.info("Shutting down installation system") + c.sendline('poweroff') + c.expect(r'\nAre you sure you want to poweroff this system.*\]') + c.sendline('Y') + for i in range(30): + log.info("Waiting for shutdown...") + if not c.isalive(): + log.info("VM is shut down!") + break + time.sleep(10) + else: + log.error("VM Did not shut down after 300sec, killing") + c.close() + + + ################################################# + # Booting installed system + ################################################# + log.info("Booting installed system") + + cmd = """qemu-system-x86_64 \ + -name "TESTVM" \ + -m 1G \ + -nic user,model=virtio,mac=52:54:99:12:34:56,hostfwd=tcp::2299-:22 + -machine accel=kvm \ + {CPU} \ + -smp 2 \ + -vnc 0.0.0.0:99 \ + -nographic \ + {KVM} \ + -drive format=raw,file={DISK} + """.format(DISK=args.disk, KVM="" if kvm else "--no-kvm", CPU="-cpu host" if kvm else "") + + log.debug('Executing command: {}'.format(cmd)) + c = pexpect.spawn(cmd, logfile=stl) + + ################################################# + # Logging into VyOS system + ################################################# + try: + c.expect('The highlighted entry will be executed automatically in', timeout=10) + c.sendline('') + except pexpect.TIMEOUT: + log.warning("Did not find grub countdown window, ignoring") + + log.info('Waiting for login prompt') + c.expect('[Ll]ogin:', timeout=300) + c.sendline('vyos') + c.expect('[Pp]assword:', timeout=10) + c.sendline('vyos') + c.expect(r'vyos@vyos:~\$') + log.info('Logged in!') + + + + ################################################# + # Executing test-suite + ################################################# + log.info("Executing test-suite ") + + def cr(child, command): + child.sendline(command) + i = child.expect(['\n +Invalid command:', + '\n +Set failed', + 'No such file or directory', + r'\n\S+@\S+[$#]']) + + if i==0: + raise Exception('Invalid command detected') + elif i==1: + raise Exception('Set syntax failed :/') + elif i==2: + log.error("Did not find VyOS-smoketest, this should be an exception") + #raise Exception("WTF? did not find VyOS-smoketest, this should be an exception") + cr(c, '/usr/bin/vyos-smoketest') + + log.info("Smoke test status") + #data = c.before.decode() + + ################################################# + # Powering off system + ################################################# + log.info("Powering off system ") + c.sendline('poweroff') + c.expect(r'\nAre you sure you want to poweroff this system.*\]') + c.sendline('Y') + log.info("Shutting down virtual machine") + for i in range(30): + log.info("Waiting for shutdown...") + if not c.isalive(): + log.info("VM is shut down!") + break + time.sleep(10) + else: + log.error("VM Did not shut down after 300sec") + raise Exception("VM Did not shut down after 300sec") + c.close() + +except pexpect.exceptions.TIMEOUT: + log.error("Timeout waiting for VyOS system") + log.error(traceback.format_exc()) + EXCEPTION = 1 + +except pexpect.exceptions.ExceptionPexpect: + log.error("Exeption while executing QEMU") + log.error("Is qemu working on this system?") + log.error(traceback.format_exc()) + EXCEPTION = 1 + +except Exception: + log.error("An unknown error occured when installing the VyOS system") + traceback.print_exc() + EXCEPTION = 1 + + + +################################################# +# Cleaning up +################################################# +log.info("Cleaning up") + +if not args.keep: + log.info("Removing disk file: {}".format(args.disk)) + try: + os.remove(args.disk) + except Exception: + log.error("Exception while removing diskimage") + log.error(traceback.format_exc()) + EXCEPTION = 1 + +if EXCEPTION: + log.error("Hmm... System got an exception while processing") + log.error("The ISO is not considered usable") + sys.exit(1) diff --git a/scripts/live-build-config b/scripts/live-build-config index 1b3b4541..171b827d 100755 --- a/scripts/live-build-config +++ b/scripts/live-build-config @@ -35,8 +35,8 @@ util.check_build_config() lb_config_tmpl = """ lb config noauto \ --architectures {{architecture}} \ - --bootappend-live "boot=live components hostname=vyos username=live nopersistence noautologin nonetworking union=overlay console=ttyS0,115200 console=tty0" \ - --bootappend-live-failsafe "live components memtest noapic noapm nodma nomce nolapic nomodeset nosmp nosplash vga=normal console=ttyS0,115200 console=tty0" \ + --bootappend-live "boot=live components hostname=vyos username=live nopersistence noautologin nonetworking union=overlay console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \ + --bootappend-live-failsafe "live components memtest noapic noapm nodma nomce nolapic nomodeset nosmp nosplash vga=normal console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \ --linux-flavours {{kernel_flavor}} \ --linux-packages linux-image-{{kernel_version}} \ --bootloader syslinux,grub-efi \ @@ -46,7 +46,7 @@ lb config noauto \ --iso-application "VyOS" \ --iso-publisher "{{build_by}}" \ --iso-volume "VyOS" \ - --debootstrap-options "--variant=minbase --exclude=isc-dhcp-client,isc-dhcp-common,ifupdown --include=apt-transport-https" \ + --debootstrap-options "--variant=minbase --exclude=isc-dhcp-client,isc-dhcp-common,ifupdown --include=apt-transport-https,gnupg2" \ --mirror-bootstrap {{debian_mirror}} \ --mirror-chroot {{debian_mirror}} \ --mirror-chroot-security {{debian_security_mirror}} \ @@ -55,9 +55,9 @@ lb config noauto \ --archive-areas "main contrib non-free" \ --firmware-chroot false \ --firmware-binary false \ - --updates false \ + --updates true \ --security true \ - --apt-options "--yes -oAcquire::Check-Valid-Until=false" \ + --apt-options "--yes -oAPT::Default-Release="equuleus" -oAPT::Get::allow-downgrades=true" \ --apt-indices false "${@}" """ @@ -69,19 +69,16 @@ debug = build_config['debug'] # Add the additional repositories to package lists print("Setting up additional APT entries") -vyos_repo_entry = "deb {0}/vyos {1} main\n".format(build_config['vyos_mirror'], build_config['vyos_branch']) -vyos_debian_repo_entry = "deb {0}/debian {1} main\n".format(build_config['vyos_mirror'], build_config['vyos_branch']) +vyos_repo_entry = "deb {0} {1} main\n".format(build_config['vyos_mirror'], build_config['vyos_branch']) apt_file = os.path.join(build_config['build_dir'], defaults.VYOS_REPO_FILE) if debug: print("Adding these entries to {0}:".format(apt_file)) print("\t", vyos_repo_entry) - print("\t", vyos_debian_repo_entry) with open(apt_file, 'w') as f: f.write(vyos_repo_entry) - f.write(vyos_debian_repo_entry) # Add custom APT entries if build_config['custom_apt_entry']: |