diff options
| -rw-r--r-- | Makefile | 2 | ||||
| -rw-r--r-- | README.md | 4 | ||||
| -rw-r--r-- | data/defaults.json | 2 | ||||
| -rwxr-xr-x | data/live-build-config/hooks/live/18-enable-disable_services.chroot | 1 | ||||
| -rwxr-xr-x | data/live-build-config/hooks/live/80-delete-docs.chroot | 4 | ||||
| -rwxr-xr-x | data/live-build-config/includes.chroot/etc/vmware-tools/scripts/resume-vm-default.d/ether-resume.py | 60 | ||||
| -rw-r--r-- | data/live-build-config/package-lists/vyos-utils.list.chroot | 4 | ||||
| -rw-r--r-- | data/package-lists/vyos-x86.list.chroot | 2 | ||||
| -rw-r--r-- | docker/Dockerfile | 19 | ||||
| -rwxr-xr-x | scripts/check-qemu-install | 59 | ||||
| -rwxr-xr-x | scripts/list-build-dependencies | 120 | ||||
| -rwxr-xr-x | scripts/list-required-firmware | 75 | ||||
| -rwxr-xr-x | scripts/live-build-config | 1 | ||||
| -rw-r--r-- | tools/cloud-init/AWS/cloud-init.list.chroot | 2 |
14 files changed, 259 insertions, 96 deletions
@@ -126,7 +126,7 @@ AWS: clean prepare @echo "It's not like I'm building this specially for you or anything!" mkdir -p build/config/includes.chroot/etc/cloud/cloud.cfg.d cp tools/cloud-init/AWS/90_dpkg.cfg build/config/includes.chroot/etc/cloud/cloud.cfg.d/ - cp tools/cloud-init/cloud-init.list.chroot build/config/package-lists/ + cp tools/cloud-init/AWS/cloud-init.list.chroot build/config/package-lists/ cp -f tools/cloud-init/AWS/config.boot.default build/config/includes.chroot/opt/vyatta/etc/ cd $(build_dir) lb build 2>&1 | tee build.log @@ -1,8 +1,8 @@ VyOS toplevel build =================== -**For the most up-to-date documentation, please read -[the online build docs at docs.vyos.io](https://docs.vyos.io/en/latest/contributing/build-vyos.html)** +For the most up-to-date documentation, please read the online build guide at +[docs.vyos.io](https://docs.vyos.io/en/latest/contributing/build-vyos.html). # What is VyOS diff --git a/data/defaults.json b/data/defaults.json index b13089a4..49dcddb8 100644 --- a/data/defaults.json +++ b/data/defaults.json @@ -5,7 +5,7 @@ "debian_distribution": "buster", "vyos_mirror": "http://dev.packages.vyos.net/repositories/current", "vyos_branch": "current", - "kernel_version": "4.19.91", + "kernel_version": "4.19.101", "kernel_flavor": "amd64-vyos", "additional_repositories": [], "custom_packages": [] diff --git a/data/live-build-config/hooks/live/18-enable-disable_services.chroot b/data/live-build-config/hooks/live/18-enable-disable_services.chroot index 50325aeb..8685e9d0 100755 --- a/data/live-build-config/hooks/live/18-enable-disable_services.chroot +++ b/data/live-build-config/hooks/live/18-enable-disable_services.chroot @@ -53,6 +53,7 @@ systemctl disable man-db.timer systemctl disable ntp.service systemctl disable igmpproxy.service systemctl disable cryptsetup.service +systemctl disable live-tools.service systemctl enable vyos-hostsd.service systemctl enable acpid.service diff --git a/data/live-build-config/hooks/live/80-delete-docs.chroot b/data/live-build-config/hooks/live/80-delete-docs.chroot new file mode 100755 index 00000000..d08b8edd --- /dev/null +++ b/data/live-build-config/hooks/live/80-delete-docs.chroot @@ -0,0 +1,4 @@ +#!/bin/bash + +# We do not need any documentation on the system. This frees 43MB. +rm -rf /usr/share/doc /usr/share/doc-base /usr/share/docutils diff --git a/data/live-build-config/includes.chroot/etc/vmware-tools/scripts/resume-vm-default.d/ether-resume.py b/data/live-build-config/includes.chroot/etc/vmware-tools/scripts/resume-vm-default.d/ether-resume.py deleted file mode 100755 index 72da317f..00000000 --- a/data/live-build-config/includes.chroot/etc/vmware-tools/scripts/resume-vm-default.d/ether-resume.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2018-2020 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -import sys -import subprocess -import syslog as sl - -from vyos.config import Config -from vyos.util import vyos - -def get_config(): - c = Config() - interfaces = dict() - for intf in c.list_effective_nodes('interfaces ethernet'): - # skip interfaces that are disabled or is configured for dhcp - check_disable = "interfaces ethernet {} disable".format(intf) - check_dhcp = "interfaces ethernet {} address dhcp".format(intf) - if c.exists_effective(check_disable) or c.exists_effective(check_dhcp): - continue - - # get addresses configured on the interface - intf_addresses = c.return_effective_values( - "interfaces ethernet {} address".format(intf) - ) - interfaces[intf] = [addr.strip("'") for addr in intf_addresses] - return interfaces - -def apply(config): - for intf, addresses in config.items(): - # bring the interface up - cmd = ["ip", "link", "set", "dev", intf, "up"] - sl.syslog(sl.LOG_NOTICE, " ".join(cmd)) - subprocess.call(cmd) - - # add configured addresses to interface - for addr in addresses: - cmd = ["ip", "address", "add", addr, "dev", intf] - sl.syslog(sl.LOG_NOTICE, " ".join(cmd)) - subprocess.call(cmd) - -if __name__ == '__main__': - try: - config = get_config() - apply(config) - except vyos.ConfigError as e: - print(e) - sys.exit(1) diff --git a/data/live-build-config/package-lists/vyos-utils.list.chroot b/data/live-build-config/package-lists/vyos-utils.list.chroot index 5de95816..3c162b66 100644 --- a/data/live-build-config/package-lists/vyos-utils.list.chroot +++ b/data/live-build-config/package-lists/vyos-utils.list.chroot @@ -6,14 +6,11 @@ mtr-tiny whois netcat-openbsd socat -telnet nano vim-tiny screen minicom wakeonlan -iperf -iperf3 iftop lsof openssh-client @@ -28,3 +25,4 @@ libnss-myhostname ssl-cert nginx-light ndisc6 +systemd-sysv diff --git a/data/package-lists/vyos-x86.list.chroot b/data/package-lists/vyos-x86.list.chroot index b6caa914..fa66a5d3 100644 --- a/data/package-lists/vyos-x86.list.chroot +++ b/data/package-lists/vyos-x86.list.chroot @@ -1,8 +1,8 @@ grub2 grub-pc -open-vm-tools qemu-guest-agent hyperv-daemons vyos-netplug vyos-smoketest vyos-xe-guest-utilities +vyos-1x-vmware diff --git a/docker/Dockerfile b/docker/Dockerfile index a00422f2..452b1499 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -297,6 +297,25 @@ RUN if dpkg-architecture -ii386 || dpkg-architecture -iamd64; then \ qemu-kvm; \ fi +# Packages needed for vyos-cloud-init +RUN apt-get update && apt-get install -y \ + pep8 \ + pyflakes \ + python3-configobj \ + python3-httpretty \ + python3-jsonpatch \ + python3-mock \ + python3-oauthlib \ + python3-pep8 \ + python3-pyflakes \ + python3-serial \ + python3-unittest2 \ + python3-yaml + +# Packages needed for libnss-mapuser +RUN apt-get update && apt-get install -y \ + libaudit-dev + # Install packer RUN if dpkg-architecture -ii386 || dpkg-architecture -iamd64; then \ export LATEST="$(curl -s https://checkpoint-api.hashicorp.com/v1/check/packer | \ diff --git a/scripts/check-qemu-install b/scripts/check-qemu-install index 88477afd..0c443f18 100755 --- a/scripts/check-qemu-install +++ b/scripts/check-qemu-install @@ -94,6 +94,34 @@ class StreamToLogger(object): def flush(self): pass +def get_qemu_cmd(name, enable_kvm, disk_img, iso_img=None): + kvm = "" + cpu = "-cpu host" + if not enable_kvm: + kvm = "--no-kvm" + cpu = "" + + cdrom = "" + if iso_img: + cdrom = "-boot d -cdrom {}".format(iso_img) + + cmd = 'qemu-system-x86_64 \ + -name "{NAME}" \ + -m 1G \ + -nic user,model=virtio,mac=52:54:99:12:34:56 \ + -nic user,model=virtio,mac=52:54:99:12:34:57 \ + -nic user,model=virtio,mac=52:54:99:12:34:58 \ + -nic user,model=virtio,mac=52:54:99:12:34:59 \ + -machine accel=kvm \ + {CPU} \ + -smp 2 \ + -nographic \ + {CD} \ + {KVM} \ + -drive format=raw,file={DISK}'.format(NAME=name, CD=cdrom, DISK=disk_img, KVM=kvm, CPU=cpu) + + return cmd + # Setting up logger log = logging.getLogger() @@ -150,20 +178,7 @@ try: # Installing image to disk ################################################# log.info("Installing system") - - cmd = """qemu-system-x86_64 \ - -name "TESTVM" \ - -m 1G \ - -nic user,model=virtio,mac=52:54:99:12:34:56,hostfwd=tcp::2299-:22 \ - -machine accel=kvm \ - {CPU} \ - -smp 2 \ - -vnc 0.0.0.0:99 \ - -nographic \ - -boot d -cdrom {CD} \ - {KVM} \ - -drive format=raw,file={DISK} - """.format(CD=args.iso, DISK=args.disk, KVM="" if kvm else "--no-kvm", CPU="-cpu host" if kvm else "") + cmd = get_qemu_cmd("TESTVM", kvm, args.disk, args.iso) log.debug("Executing command: {}".format(cmd)) c = pexpect.spawn(cmd, logfile=stl) @@ -237,19 +252,7 @@ try: # Booting installed system ################################################# log.info("Booting installed system") - - cmd = """qemu-system-x86_64 \ - -name "TESTVM" \ - -m 1G \ - -nic user,model=virtio,mac=52:54:99:12:34:56,hostfwd=tcp::2299-:22 - -machine accel=kvm \ - {CPU} \ - -smp 2 \ - -nographic \ - {KVM} \ - -drive format=raw,file={DISK} - """.format(DISK=args.disk, KVM="" if kvm else "--no-kvm", CPU="-cpu host" if kvm else "") - + cmd = get_qemu_cmd("TESTVM", kvm, args.disk) log.debug('Executing command: {}'.format(cmd)) c = pexpect.spawn(cmd, logfile=stl) @@ -290,7 +293,7 @@ try: i = child.expect(['\n +Invalid command:', '\n +Set failed', 'No such file or directory', - r'\n\S+@\S+[$#]']) + r'\n\S+@\S+[$#]'], timeout=1800) if i==0: raise Exception('Invalid command detected') diff --git a/scripts/list-build-dependencies b/scripts/list-build-dependencies new file mode 100755 index 00000000..e13651f3 --- /dev/null +++ b/scripts/list-build-dependencies @@ -0,0 +1,120 @@ +#!/bin/bash +# +# Copyright (C) 2020 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# in order to easy exprort images built to "external" world +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# Parse debian/control like content and extract packages required by the +# Build-Depends statement. Return a string with all required packages. +# +# Code below copied from https://stackoverflow.com/a/47707412 - Thank You! +get_build_depends () { + echo $(awk ' + /^Build-Depends:/ || /^ / && deps { + sub(/^[^ ]+: /, "") + deps = 1 + dep_str = dep_str ", " $0 + next + } + { deps=0 } + END { + split(dep_str, dep_array, /, */) + for (d in dep_array) { + dep = dep_array[d] + gsub(/[^a-z0-9_.-].*$/, "", dep) + if (dep && !seen[dep]++) print dep + } + }' $1) +} + +get_runtime_depends () { + echo $(awk ' + /^Depends:/ || /^ / && deps { + sub(/^[^ ]+: /, "") + deps = 1 + dep_str = dep_str ", " $0 + next + } + { deps=0 } + END { + split(dep_str, dep_array, /, */) + for (d in dep_array) { + dep = dep_array[d] + gsub(/[^a-z0-9_.-].*$/, "", dep) + if (dep && !seen[dep]++) print dep + } + }' $1) +} + +# Some packages are required prior to running this script +BOOTSTRAP_PACKAGES="devscripts curl equivs" +for pkg in $BOOTSTRAP_PACKAGES +do + dpkg -s $pkg >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "Required package \"$pkg\" not installed" + exit 1 + fi +done + +echo "" +echo "Below you can find a list of packages that are required as build time" +echo "dependency for the individual package" +echo "" +echo "The generated content can be used to populate a file to provision" +echo "e.g. a native build host or a Docker container" +echo "" +echo "" + +GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + +# First we need to get vyos-world so we know all individual packages for VyOS +curl -L https://github.com/vyos/vyos-world/raw/$GIT_BRANCH/debian/control \ + --output /tmp/vyos-world.control --retry 100 --retry-delay 1 --silent + +VYOS_PACKAGES=$(get_runtime_depends /tmp/vyos-world.control) +rm -f /tmp/vyos-world.control +for pkg in $VYOS_PACKAGES +do + # Check if repo exists + res=$(curl -o /dev/null --silent -Iw '%{http_code}' https://github.com/vyos/$pkg) + if [[ $res -ne 200 ]]; then + continue + fi + + CTRLFILE=/tmp/$pkg.control + curl -L https://github.com/vyos/$pkg/raw/$GIT_BRANCH/debian/control \ + --output $CTRLFILE --retry 100 --retry-delay 1 --silent + + declare -a array + declare -i length cnt + + array=($(get_build_depends $CTRLFILE)) + length=${#array[@]} + cnt=0 + + echo "# Packages needed to build '$pkg' from https://github.com/vyos/$pkg" + echo "apt-get install -y \\" + for name in "${array[@]}"; do + cnt=$((cnt + 1)) + if [[ "$cnt" -eq "$length" ]]; then + echo " $name" + else + echo " $name \\" + fi + done + rm -f $CTRLFILE + echo "" +done + diff --git a/scripts/list-required-firmware b/scripts/list-required-firmware new file mode 100755 index 00000000..64280e03 --- /dev/null +++ b/scripts/list-required-firmware @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# Copyright (C) 2020 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +import re +import os +import sys +import glob + + +SRC_DIR = sys.argv[1] +KERNEL_CONFIG = sys.argv[2] + +def load_config(path): + with open(KERNEL_CONFIG, 'r') as f: + config = f.read() + targets = re.findall(r'(.*)=(?:y|m)', config) + return targets + +def find_subdirs(config, path): + try: + with open(os.path.join(path, 'Makefile'), 'r') as f: + makefile = f.read() + except OSError: + # No Makefile + return [] + + dir_stmts = re.findall(r'obj-\$\((.*)\)\s+\+=\s+(.*)(?:\n|$)', makefile) + subdirs = [] + for ds in dir_stmts: + print("Processing make targets from {0} ({1})".format(ds[1], ds[0]), file=sys.stderr) + if ds[0] in config: + dirname = os.path.dirname(ds[1]) + if dirname: + subdirs.append(dirname) + else: + print("{0} is disabled in the config, ignoring {1}".format(ds[0], ds[1]), file=sys.stderr) + + return subdirs + +def find_firmware(file): + with open(file, 'r') as f: + source = f.read() + fws = re.findall(r'MODULE_FIRMWARE\((.*)\)', source) + return fws + +def walk_dir(config, path): + subdirs = find_subdirs(config, path) + + print("Looking for C files in {0}".format(path), file=sys.stderr) + c_files = glob.glob("{0}/*.c".format(path)) + for cf in c_files: + fws = find_firmware(cf) + if fws: + print("Referenced firmware: {0}".format(fws)) + + for d in subdirs: + d = os.path.join(path, d) + walk_dir(config, d) + +if __name__ == '__main__': + config = load_config(KERNEL_CONFIG) + walk_dir(config, SRC_DIR) diff --git a/scripts/live-build-config b/scripts/live-build-config index 61e71c0d..1b31e4fd 100755 --- a/scripts/live-build-config +++ b/scripts/live-build-config @@ -57,6 +57,7 @@ lb config noauto \ --firmware-binary false \ --updates true \ --security true \ + --apt-recommends false \ --apt-options "--yes -oAPT::Default-Release="current" -oAPT::Get::allow-downgrades=true" \ --apt-indices false "${@}" diff --git a/tools/cloud-init/AWS/cloud-init.list.chroot b/tools/cloud-init/AWS/cloud-init.list.chroot new file mode 100644 index 00000000..0858effd --- /dev/null +++ b/tools/cloud-init/AWS/cloud-init.list.chroot @@ -0,0 +1,2 @@ +cloud-init +awscli |