diff options
20 files changed, 192 insertions, 229 deletions
diff --git a/.github/workflows/mergifyio_backport.yml b/.github/workflows/mergifyio_backport.yml new file mode 100644 index 00000000..80cd4d4c --- /dev/null +++ b/.github/workflows/mergifyio_backport.yml @@ -0,0 +1,22 @@ +name: Mergifyio backport + +on: [issue_comment] + +jobs: + mergifyio_backport: + if: github.repository == 'vyos/vyos-build' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions-ecosystem/action-regex-match@v2 + id: regex-match + with: + text: ${{ github.event.comment.body }} + regex: '[Mm]ergifyio backport ' + + - uses: actions-ecosystem/action-add-labels@v1 + if: ${{ steps.regex-match.outputs.match != '' }} + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + labels: backport diff --git a/.github/workflows/pull-request-labels.yml b/.github/workflows/pull-request-labels.yml new file mode 100644 index 00000000..778daae3 --- /dev/null +++ b/.github/workflows/pull-request-labels.yml @@ -0,0 +1,20 @@ +--- +name: Add pull request labels + +on: + pull_request_target: + branches: + - current + - crux + - equuleus + - sagitta + +jobs: + add-pr-label: + name: Add PR Labels + runs-on: ubuntu-20.04 + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/labeler@v5.0.0-alpha.1 diff --git a/Jenkinsfile b/Jenkinsfile index a89bac1c..f59b0d47 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -53,9 +53,6 @@ pipeline { reuseNode true } } - triggers { - cron('H 3 * * *') - } parameters { string(name: 'BUILD_BY', defaultValue: 'autobuild@vyos.net', description: 'Builder identifier (e.g. jrandomhacker@example.net)') string(name: 'BUILD_VERSION', defaultValue: env.BASE_VERSION + 'ISO8601-TIMESTAMP', description: 'Version number (release builds only)') @@ -32,12 +32,12 @@ checkiso: .PHONY: test .ONESHELL: test: checkiso - scripts/check-qemu-install --debug --uefi build/live-image-amd64.hybrid.iso + scripts/check-qemu-install --debug --match="$(MATCH)" --uefi build/live-image-amd64.hybrid.iso .PHONY: test-no-interfaces .ONESHELL: test-no-interfaces: checkiso - scripts/check-qemu-install --debug --no-interfaces build/live-image-amd64.hybrid.iso + scripts/check-qemu-install --debug --match="$(MATCH)" --no-interfaces build/live-image-amd64.hybrid.iso .PHONY: testd .ONESHELL: diff --git a/data/architectures/amd64.toml b/data/architectures/amd64.toml index 8ea3e91f..640a7f49 100644 --- a/data/architectures/amd64.toml +++ b/data/architectures/amd64.toml @@ -1,5 +1,5 @@ additional_repositories = [ - "deb [arch=amd64] https://repo.saltproject.io/py3/debian/11/amd64/3004 bullseye main" + "deb [arch=amd64] https://repo.saltproject.io/py3/debian/11/amd64/3005 bullseye main" ] kernel_flavor = "amd64-vyos" diff --git a/data/architectures/arm64.toml b/data/architectures/arm64.toml index 3f1e4c1d..ff95d0cf 100644 --- a/data/architectures/arm64.toml +++ b/data/architectures/arm64.toml @@ -1,5 +1,5 @@ additional_repositories = [ - "deb [arch=arm64] https://repo.saltproject.io/py3/debian/11/arm64/3004 bullseye main" + "deb [arch=arm64] https://repo.saltproject.io/py3/debian/11/arm64/3005 bullseye main" ] kernel_flavor = "arm64-vyos" diff --git a/data/defaults.toml b/data/defaults.toml index a64cbb88..997eed0c 100644 --- a/data/defaults.toml +++ b/data/defaults.toml @@ -14,7 +14,7 @@ vyos_mirror = "https://rolling-packages.vyos.net/current" vyos_branch = "current" release_train = "current" -kernel_version = "6.1.52" +kernel_version = "6.1.53" bootloaders = "syslinux,grub-efi" squashfs_compression_type = "xz -Xbcj x86 -b 256k -always-use-fragments -no-recovery" diff --git a/data/live-build-config/hooks/live/01-live-serial.binary b/data/live-build-config/hooks/live/01-live-serial.binary new file mode 100755 index 00000000..e138b20d --- /dev/null +++ b/data/live-build-config/hooks/live/01-live-serial.binary @@ -0,0 +1,31 @@ +#!/bin/sh + +GRUB_PATH=boot/grub/grub.cfg +ISOLINUX_PATH=isolinux/live.cfg + +KVM_CONSOLE="console=ttyS0,115200 console=tty0" +SERIAL_CONSOLE="console=tty0 console=ttyS0,115200" + +# Grub.cfg Update +GRUB_MENUENTRY=$(sed -e '/menuentry.*hotkey.*/,/^}/!d' -e 's/--hotkey=l//g' $GRUB_PATH) + +# Update KVM menuentry name +sed -i 's/"Live system \((.*-vyos)\)"/"Live system \1 - KVM console"/' $GRUB_PATH + +# Insert serial menuentry +echo "$GRUB_MENUENTRY" | sed \ + -e 's/"Live system \((.*-vyos)\)"/"Live system \1 - Serial console"/' \ + -e "s/$KVM_CONSOLE/$SERIAL_CONSOLE/g" >> $GRUB_PATH + +# Live.cfg Update +ISOLINUX_MENUENTRY=$(sed -e '/label live-\(.*\)-vyos$/,/^\tappend.*/!d' $ISOLINUX_PATH) + +# Update KVM menuentry name +sed -i 's/Live system \((.*-vyos)\)/Live system \1 - KVM console/' $ISOLINUX_PATH + +# Insert serial menuentry +echo "\n$ISOLINUX_MENUENTRY" | sed \ + -e 's/live-\(.*\)-vyos/live-\1-vyos-serial/' \ + -e '/^\tmenu default/d' \ + -e 's/Live system \((.*-vyos)\)/Live system \1 - Serial console/' \ + -e "s/$KVM_CONSOLE/$SERIAL_CONSOLE/g" >> $ISOLINUX_PATH diff --git a/data/live-build-config/hooks/live/20-rm_ddclient_hook.chroot b/data/live-build-config/hooks/live/20-rm_ddclient_hook.chroot deleted file mode 100755 index 350843c7..00000000 --- a/data/live-build-config/hooks/live/20-rm_ddclient_hook.chroot +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -if [ -f /etc/dhcp/dhclient-exit-hooks.d/ddclient ]; then - rm -f /etc/dhcp/dhclient-exit-hooks.d/ddclient -fi - -if [ -f /etc/ddclient.conf ]; then - rm -f /etc/ddclient.conf -fi diff --git a/data/live-build-config/hooks/live/22-rm_cron_atop.chroot b/data/live-build-config/hooks/live/22-rm_cron_atop.chroot deleted file mode 100755 index 7f77e1f7..00000000 --- a/data/live-build-config/hooks/live/22-rm_cron_atop.chroot +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -if [ -f /etc/cron.d/atop ]; then - rm -f /etc/cron.d/atop -fi - diff --git a/data/live-build-config/hooks/live/30-frr-configs.chroot b/data/live-build-config/hooks/live/30-frr-configs.chroot deleted file mode 100755 index 03b1af6c..00000000 --- a/data/live-build-config/hooks/live/30-frr-configs.chroot +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 - -# For FRR to work in VyOS as expected we need a few fixups -# -# 1. Enable daemons we use in /etc/frr/daemons -# 2. Set the VRF backend of Zebra to netns (-n option) in /etc/frr/daemons.conf -# Otherwise multiple routing tables for PBR won't work -# 3. Create empty configs for daemons with use -# That is to make them possible to start on boot before config is loaded -# - -import os - -daemons = """ -zebra=yes -bgpd=yes -ospfd=yes -ospf6d=yes -ripd=yes -ripngd=yes -isisd=yes -pimd=no -pim6d=yes -ldpd=yes -nhrpd=no -eigrpd=yes -babeld=yes -sharpd=no -pbrd=no -bfdd=yes -staticd=yes - -vtysh_enable=yes -zebra_options="-s 90000000 --daemon -A 127.0.0.1 -M snmp" -bgpd_options="--daemon -A 127.0.0.1 -M snmp -M rpki -M bmp" -ospfd_options="--daemon -A 127.0.0.1 -M snmp" -ospf6d_options="--daemon -A ::1 -M snmp" -ripd_options="--daemon -A 127.0.0.1 -M snmp" -ripngd_options="--daemon -A ::1" -isisd_options="--daemon -A 127.0.0.1 -M snmp" -pimd_options="--daemon -A 127.0.0.1" -pim6d_options=""--daemon -A ::1" -ldpd_options="--daemon -A 127.0.0.1" -nhrpd_options="--daemon -A 127.0.0.1" -mgmtd_options=" --daemon -A 127.0.0.1" -eigrpd_options="--daemon -A 127.0.0.1" -babeld_options="--daemon -A 127.0.0.1" -sharpd_options="--daemon -A 127.0.0.1" -pbrd_options="--daemon -A 127.0.0.1" -staticd_options="--daemon -A 127.0.0.1" -bfdd_options="--daemon -A 127.0.0.1" - -watchfrr_enable=no -valgrind_enable=no -""" - -frr_conf = """ -log syslog -log facility local7 -""" - -frr_log = '' - -with open("/etc/frr/daemons", "w") as f: - f.write(daemons) - -with open("/etc/frr/frr.conf", "w") as f: - f.write(frr_conf) - -# Prevent writing logs to /var/log/frr/frr.log. T2061 -with open("/etc/rsyslog.d/45-frr.conf", "w") as f: - f.write(frr_log) diff --git a/data/live-build-config/hooks/live/80-delete-docs.chroot b/data/live-build-config/hooks/live/80-delete-docs.chroot deleted file mode 100755 index 1f50a9ec..00000000 --- a/data/live-build-config/hooks/live/80-delete-docs.chroot +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# Delete various unused files and directories in order free some space and shrink imagesize. - -# We do not need any documentation on the system. -# Copyright/licenses files are ignored for deletion. -shopt -s extglob -rm -rf /usr/share/doc/*/!(copyright*|README*) /usr/share/doc-base - -# We do not need any manpages on the system since man-binary is missing. -rm -rf /usr/local/man -rm -rf /usr/local/share/man -rm -rf /usr/share/man - -# We do not need any games on the system. -rm -rf /usr/games -rm -rf /usr/local/games - -# We do not need any caches on the system (will be recreated when needed). -rm -rf /var/cache/* - -# We do not need any log-files on the system (will be recreated when needed). -rm -rf /var/log/alternatives.log -rm -rf /var/log/bootstrap.log -rm -rf /var/log/dpkg.log -rm -rf /var/log/apt/history.log -rm -rf /var/log/apt/term.log -rm -rf /var/log/nginx/access.log -rm -rf /var/log/nginx/error.log -rm -rf /var/log/squidguard/squidGuard.log -rm -rf /var/log/stunnel4/stunnel.log - -# We do not need any backup-files on the system. -rm -rf /etc/sudoers.bak -rm -rf /etc/xml/catalog.old -rm -rf /etc/xml/polkitd.xml.old -rm -rf /etc/xml/xml-core.xml.old -rm -rf /root/.gnupg/pubring.kbx~ -rm -rf /var/lib/dpkg/diversions-old -rm -rf /var/lib/dpkg/status-old -rm -rf /var/lib/sgml-base/supercatalog.old - diff --git a/data/live-build-config/hooks/live/81-cleanup-etc-defaults.chroot b/data/live-build-config/hooks/live/81-cleanup-etc-defaults.chroot deleted file mode 100755 index c93deee1..00000000 --- a/data/live-build-config/hooks/live/81-cleanup-etc-defaults.chroot +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -# we use systemd to control ISC daemons from within vyos-1x -FILES="/etc/default/isc-dhcp-server /etc/default/isc-dhcp-relay" - -for FILE in ${FILES} -do - if [ -f ${FILE} ]; then - rm -f ${FILE} - fi -done diff --git a/data/live-build-config/hooks/live/82-cleanup-udev-rules.chroot b/data/live-build-config/hooks/live/82-cleanup-udev-rules.chroot deleted file mode 100755 index a0173e49..00000000 --- a/data/live-build-config/hooks/live/82-cleanup-udev-rules.chroot +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -# 99-default.link rule always calls link_config that trying to set -# autonegotiation and duplex even for PPP interfaces. -# Need to delete this rule to prevent overhead on interface creation stage - -rm /lib/systemd/network/99-default.link diff --git a/data/live-build-config/hooks/live/83-cleanup-etc-motd-d.chroot b/data/live-build-config/hooks/live/83-cleanup-etc-motd-d.chroot deleted file mode 100755 index 9d8dc97e..00000000 --- a/data/live-build-config/hooks/live/83-cleanup-etc-motd-d.chroot +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -if [ -f /etc/update-motd.d/10-uname ]; then - rm -f /etc/update-motd.d/10-uname -fi diff --git a/data/live-build-config/rootfs/excludes b/data/live-build-config/rootfs/excludes new file mode 100644 index 00000000..2af69d7b --- /dev/null +++ b/data/live-build-config/rootfs/excludes @@ -0,0 +1,59 @@ +# Exclude various unused files and directories in order to free some space and shrink imagesize. +# +# For information on how to use wildcards properly (Anchored and Non-anchored excludes): +# +# https://github.com/plougher/squashfs-tools/blob/master/RELEASE-READMEs/README-3.3 +# +# Note: +# +# - root starts without leading '/'. +# + +# Txxx: Drop isc-dhcp helper files from /etc/default. +# We use systemd to control ISC daemons from within vyos-1x. +etc/default/isc-dhcp-server +etc/default/isc-dhcp-relay + +# T2185: Clean leftover files (ddclient) from base package. +etc/dhcp/dhclient-exit-hooks.d/ddclient +etc/ddclient.conf + +# T3242: Add hook to prevent link_config redundancy call in systemd-udev. +# 99-default.link rule always calls link_config thats trying to set autonegotiation and duplex even for PPP interfaces. +# Need to delete this rule to prevent overhead on interface creation stage. +lib/systemd/network/99-default.link + +# T3774: Disabled atop services. +etc/cron.d/atop + +# T3912: Remove superfluous motd.d kernel version shell script. +etc/update-motd.d/10-uname + +# T4415: We do not need any documentation on the system. +# Copyright/licenses files are ignored for deletion. +usr/share/doc/*/!(copyright*|README*) +usr/share/doc-base + +# T5468: We do not need any manpages on the system since man-binary is missing. +usr/local/man/* +usr/local/share/man/* +usr/share/man/* + +# T5511: We do not need any games on the system. +usr/games/* +usr/local/games/* + +# T5511: We do not need any caches on the system (will be recreated when needed). +var/cache/* + +# T5511: We do not need any log-files on the system (will be recreated when needed). +var/log/*.log +var/log/*/*.log +var/log/*/*.log.xz + +# T5511: We do not need any backup-files on the system (will be recreated when needed). +... *.bak +... *.old +... *.kbx~ +var/lib/dpkg/*-old + diff --git a/packages/aws-gateway-load-balancer-tunnel-handler/Jenkinsfile b/packages/aws-gateway-load-balancer-tunnel-handler/Jenkinsfile index e8ee142f..cbf5a407 100644 --- a/packages/aws-gateway-load-balancer-tunnel-handler/Jenkinsfile +++ b/packages/aws-gateway-load-balancer-tunnel-handler/Jenkinsfile @@ -19,14 +19,14 @@ // @Library annotation is not an import statement! @Library('vyos-build@current')_ +def timestamp = new Date().format('yyyyMMddhhmmss') +def commit_id = 'f78058a' + def pkgList = [ - [ - 'name': 'aws-gateway-load-balancer-tunnel-handler', - 'scmCommit': 'f78058a', - 'scmUrl': 'https://github.com/aws-samples/aws-gateway-load-balancer-tunnel-handler', - 'buildCmd': "cd ..; ./build.py", - 'architecture': ['amd64', 'arm64'], - ], + ['name': "aws-gwlbtun-${timestamp}-${commit_id}", + 'scmCommit': commit_id, + 'scmUrl': 'https://github.com/aws-samples/aws-gateway-load-balancer-tunnel-handler', + 'buildCmd': "../build.py --package aws-gwlbtun --version ${timestamp}-${commit_id}"], ] // Start package build using library function from https://github.com/vyos/vyos-build diff --git a/packages/aws-gateway-load-balancer-tunnel-handler/build.py b/packages/aws-gateway-load-balancer-tunnel-handler/build.py index 46778059..a8c75aab 100755 --- a/packages/aws-gateway-load-balancer-tunnel-handler/build.py +++ b/packages/aws-gateway-load-balancer-tunnel-handler/build.py @@ -1,72 +1,57 @@ #!/usr/bin/env python3 -# -# Copyright (C) 2023 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -import os -import subprocess -import toml +from argparse import ArgumentParser +from pathlib import Path +from subprocess import run -def build_package(arch, version, source_dir): - package_dir = f"aws-gwlbtun-{arch}" - deb_name = f"aws-gwlbtun_{version}_{arch}.deb" - # Navigate to the repository directory - os.chdir(source_dir) +def prepare_package() -> None: + """Prepare a package + """ + install_file = Path('./debian/install') + install_data = 'obj-*-linux-gnu/gwlbtun usr/sbin' + install_file.touch() + install_file.write_text(install_data) - # Build the binary - subprocess.run(["cmake", f"-DARCH={arch}"]) - subprocess.run(["make"]) - # Create the Debian package directory structure - os.makedirs(f"{package_dir}/DEBIAN", exist_ok=True) - os.makedirs(f"{package_dir}/usr/bin", exist_ok=True) +def build_package(package_name: str, package_ver: str) -> bool: + """Build a package using commands from external file - # Move the binary to the package directory - subprocess.run(["cp", "gwlbtun", f"{package_dir}/usr/bin"]) + Args: + package_name (str): package name + package_ver (str): package version - # Create the control file - control_file = f"""Package: aws-gwlbtun -Version: {version} -Architecture: {arch} -Maintainer: VyOS Maintainers <autobuild@vyos.net> -Description: AWS Gateway Load Balancer Tunnel Handler -""" - with open(f"{package_dir}/DEBIAN/control", "w") as f: - f.write(control_file) + Returns: + bool: build status + """ + # prepare sources + debmake_cmd = [ + 'debmake', '-e', 'support@vyos.io', '-f', 'VyOS Support', '-p', + package_name, '-u', package_ver, '-t' + ] + run(debmake_cmd) - # Build the Debian package - subprocess.run(["dpkg-deb", "--build", package_dir]) + prepare_package() - # Move the generated package to the original working directory with the correct name - subprocess.run(["mv", f"{package_dir}.deb", f"../{deb_name}"]) + # build a package + run('debuild') - # Clean up - subprocess.run(["make", "clean"]) + return True - # Go back to the initial directory - os.chdir("..") -def main(): - # Load configuration from TOML file - config = toml.load("build_config.toml") - version = config["version"] - architectures = config["architectures"] - source_dir = config.get("sourceDir", "aws-gateway-load-balancer-tunnel-handler") +# build a package +if __name__ == '__main__': + # prepare argument parser + arg_parser = ArgumentParser() + arg_parser.add_argument('--package', + required=True, + help='Package name to build') + arg_parser.add_argument('--version', + required=True, + help='Version for the package') + args = arg_parser.parse_args() - for arch in architectures: - build_package(arch, version, source_dir) + if not build_package(args.package, args.version): + exit(1) -if __name__ == "__main__": - main() + exit() diff --git a/packages/aws-gateway-load-balancer-tunnel-handler/build_config.toml b/packages/aws-gateway-load-balancer-tunnel-handler/build_config.toml deleted file mode 100644 index 08921d41..00000000 --- a/packages/aws-gateway-load-balancer-tunnel-handler/build_config.toml +++ /dev/null @@ -1,6 +0,0 @@ - -architectures = ["amd64", "arm64"] -packageName = "aws-gwlbtun" -scmUrl = "https://github.com/aws-samples/aws-gateway-load-balancer-tunnel-handler" -scmCommit = "f78058a" -version = "1-f78058a" diff --git a/scripts/check-qemu-install b/scripts/check-qemu-install index fabe8fbb..82c20ed3 100755 --- a/scripts/check-qemu-install +++ b/scripts/check-qemu-install @@ -63,6 +63,7 @@ parser.add_argument('--silent', help='Do not show output on stdout unless an err parser.add_argument('--debug', help='Send all debug output to stdout', action='store_true', default=False) parser.add_argument('--logfile', help='Log to file') +parser.add_argument('--match', help='Smoketests to run') parser.add_argument('--uefi', help='Boot using UEFI', action='store_true', default=False) parser.add_argument('--raid', help='Perform a RAID-1 install', action='store_true', default=False) parser.add_argument('--no-kvm', help='Disable use of kvm', action='store_true', default=False) @@ -486,6 +487,11 @@ try: elif not args.configtest: # run default smoketest suite + if args.match: + # Remove tests that we don't want to run + match_str = '-o '.join([f'-name "test_*{name}*.py" ' for name in args.match.split("|")]).strip() + c.sendline(f'sudo find /usr/libexec/vyos/tests/smoke/cli/test_* -type f ! \( {match_str} \) -delete') + c.expect(op_mode_prompt) if args.no_interfaces: # remove interface tests as they consume a lot of time c.sendline('sudo rm -f /usr/libexec/vyos/tests/smoke/cli/test_interfaces_*') |