From 125aab4b7e2eabf5ecf67271c71199f501fa11a6 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Fri, 3 Nov 2023 17:14:58 +0000 Subject: build script: T5711: copy version.json to the ISO image in addition to the SquashFS image (cherry picked from commit f31701f1b48b696ed4176b8ae413aa956ff15c5c) --- scripts/build-vyos-image | 12 ++++-------- scripts/vyos_build_defaults.py | 1 + 2 files changed, 5 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/build-vyos-image b/scripts/build-vyos-image index 141da5e2..5ee0b3ed 100755 --- a/scripts/build-vyos-image +++ b/scripts/build-vyos-image @@ -348,18 +348,14 @@ BUG_REPORT_URL="{build_defaults['bugtracker_url']}" DOCUMENTATION_URL="{build_config['documentation_url']}" """ - # Switch to the build directory, this is crucial for the live-build work - # because the efective build config files etc. are there. - # - # All directory paths from this point must be relative to BUILD_DIR, - # not to the vyos-build repository root. - os.chdir(defaults.BUILD_DIR) - - chroot_includes_dir = defaults.CHROOT_INCLUDES_DIR + chroot_includes_dir = os.path.join(defaults.BUILD_DIR, defaults.CHROOT_INCLUDES_DIR) + binary_includes_dir = os.path.join(defaults.BUILD_DIR, defaults.BINARY_INCLUDES_DIR) vyos_data_dir = os.path.join(chroot_includes_dir, "usr/share/vyos") os.makedirs(vyos_data_dir, exist_ok=True) with open(os.path.join(vyos_data_dir, 'version.json'), 'w') as f: json.dump(version_data, f) + with open(os.path.join(binary_includes_dir, 'version.json'), 'w') as f: + json.dump(version_data, f) # For backwards compatibility with 'add system image' script from older versions # we need a file in the old format so that script can find out the version of the image diff --git a/scripts/vyos_build_defaults.py b/scripts/vyos_build_defaults.py index 975fcb71..1e153f4b 100644 --- a/scripts/vyos_build_defaults.py +++ b/scripts/vyos_build_defaults.py @@ -37,6 +37,7 @@ PBUILDER_DIR = 'pbuilder' LB_CONFIG_DIR = 'config' CHROOT_INCLUDES_DIR = 'config/includes.chroot' +BINARY_INCLUDES_DIR = 'config/includes.binary' ARCHIVES_DIR = 'config/archives/' VYOS_REPO_FILE = 'config/archives/vyos.list.chroot' -- cgit v1.2.3 From 64d4406cb22d61bda1e826075f3052462b59289e Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Wed, 20 Mar 2024 17:50:51 +0000 Subject: chore: T671: remove unused scripts and references to them (cherry picked from commit 5dcb6b771106df130b4884867dc28bd685b64e40) --- Makefile | 7 --- packages/linux-kernel/build-linux-firmware.sh | 4 -- scripts/check-build-env | 54 ----------------- scripts/import-local-packages | 13 ----- scripts/list-package-arch | 7 --- scripts/list-required-firmware | 75 ------------------------ scripts/pbuilder-config | 54 ----------------- scripts/pbuilder-setup | 52 ----------------- scripts/pbuilder/hooks/C10shell | 6 -- tools/get_latest_iso.py | 31 ---------- tools/gpl-header-template | 17 ------ tools/submod-mk | 84 --------------------------- tools/switch-remote | 35 ----------- 13 files changed, 439 deletions(-) delete mode 100755 scripts/check-build-env delete mode 100755 scripts/import-local-packages delete mode 100755 scripts/list-package-arch delete mode 100755 scripts/list-required-firmware delete mode 100755 scripts/pbuilder-config delete mode 100755 scripts/pbuilder-setup delete mode 100755 scripts/pbuilder/hooks/C10shell delete mode 100755 tools/get_latest_iso.py delete mode 100644 tools/gpl-header-template delete mode 100644 tools/submod-mk delete mode 100755 tools/switch-remote (limited to 'scripts') diff --git a/Makefile b/Makefile index ebb37075..4d4317fc 100644 --- a/Makefile +++ b/Makefile @@ -14,13 +14,6 @@ iso: clean @./build-vyos-image iso exit 0 -.PHONY: prepare-package-env -.ONESHELL: -prepare-package-env: - @set -e - @scripts/pbuilder-config - @scripts/pbuilder-setup - .PHONY: checkiso .ONESHELL: checkiso: diff --git a/packages/linux-kernel/build-linux-firmware.sh b/packages/linux-kernel/build-linux-firmware.sh index 39cb243b..2b1fa7b7 100755 --- a/packages/linux-kernel/build-linux-firmware.sh +++ b/packages/linux-kernel/build-linux-firmware.sh @@ -1,9 +1,5 @@ #!/bin/bash -# This script will use "list-required-firmware" to scan the kernel source repository -# in combination with its configuration file which drivers are compiled. Some of those -# drivers require proprietary firmware. -# # All selected drivers are then precomfiled "make drivers/foo/bar.i" and we grep for # the magic word "UNIQUE_ID_firmware" which identifies firmware files. diff --git a/scripts/check-build-env b/scripts/check-build-env deleted file mode 100755 index 7377be64..00000000 --- a/scripts/check-build-env +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2015 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# File: check-build-env -# Purpose: -# Checks if packages required for package and ISO image build -# are installed. - - -import os -import sys - -import util - -deps = { - 'packages': [ - 'sudo', - 'make', - 'live-build', - 'pbuilder', - 'devscripts', - 'python3-pystache', - 'python3-git' - ], - 'binaries': [] -} - -print("Checking if packages required for VyOS image build are installed") - -checker = util.DependencyChecker(deps) - -missing = checker.get_missing_dependencies() -if not missing: - print("All dependencies are installed") - sys.exit(0) -else: - checker.print_missing_deps() - sys.exit(1) - -sys.exit(0) - diff --git a/scripts/import-local-packages b/scripts/import-local-packages deleted file mode 100755 index 70b4c365..00000000 --- a/scripts/import-local-packages +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -LOCAL_PKG_DIR=build/config/packages.chroot - -mkdir -p $LOCAL_PKG_DIR - -FILES=packages/*.deb -for f in $FILES -do - if [ -e "$f" ]; then - cp $f $LOCAL_PKG_DIR - fi -done diff --git a/scripts/list-package-arch b/scripts/list-package-arch deleted file mode 100755 index c9a88654..00000000 --- a/scripts/list-package-arch +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# Execute this script from the vyos-build top directory -# Will generate a list of architectures in each repository -for a in $(echo vyos-build; ./scripts/build-packages -l | egrep -e '^ \* ' | sed 's/^ \* //'); do - n=$(curl https://raw.githubusercontent.com/vyos/${a}/sagitta/debian/control 2>/dev/null | grep "Architecture" | tr '\n' ',') - printf "%-24s %s \n" "${a}" "${n}" -done diff --git a/scripts/list-required-firmware b/scripts/list-required-firmware deleted file mode 100755 index 64280e03..00000000 --- a/scripts/list-required-firmware +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2020 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -import re -import os -import sys -import glob - - -SRC_DIR = sys.argv[1] -KERNEL_CONFIG = sys.argv[2] - -def load_config(path): - with open(KERNEL_CONFIG, 'r') as f: - config = f.read() - targets = re.findall(r'(.*)=(?:y|m)', config) - return targets - -def find_subdirs(config, path): - try: - with open(os.path.join(path, 'Makefile'), 'r') as f: - makefile = f.read() - except OSError: - # No Makefile - return [] - - dir_stmts = re.findall(r'obj-\$\((.*)\)\s+\+=\s+(.*)(?:\n|$)', makefile) - subdirs = [] - for ds in dir_stmts: - print("Processing make targets from {0} ({1})".format(ds[1], ds[0]), file=sys.stderr) - if ds[0] in config: - dirname = os.path.dirname(ds[1]) - if dirname: - subdirs.append(dirname) - else: - print("{0} is disabled in the config, ignoring {1}".format(ds[0], ds[1]), file=sys.stderr) - - return subdirs - -def find_firmware(file): - with open(file, 'r') as f: - source = f.read() - fws = re.findall(r'MODULE_FIRMWARE\((.*)\)', source) - return fws - -def walk_dir(config, path): - subdirs = find_subdirs(config, path) - - print("Looking for C files in {0}".format(path), file=sys.stderr) - c_files = glob.glob("{0}/*.c".format(path)) - for cf in c_files: - fws = find_firmware(cf) - if fws: - print("Referenced firmware: {0}".format(fws)) - - for d in subdirs: - d = os.path.join(path, d) - walk_dir(config, d) - -if __name__ == '__main__': - config = load_config(KERNEL_CONFIG) - walk_dir(config, SRC_DIR) diff --git a/scripts/pbuilder-config b/scripts/pbuilder-config deleted file mode 100755 index 06e14cbf..00000000 --- a/scripts/pbuilder-config +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2015 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# File: pbuilder-config -# Purpose: -# Generates a pbuilderrc file for use by package build scripts. - - -import sys -import os -import json - -import pystache - -import defaults -import util - -util.check_build_config() - -pbuilder_config_tmpl = """ - -BASETGZ={{build_dir}}/base.tgz -BUILDPLACE={{build_dir}}/pbuilder/ -MIRRORSITE={{pbuilder_debian_mirror}} -BUILDRESULT={{build_dir}}/pbuilder/result/ - -DISTRIBUTION={{distribution}} - -ARCHITECTURE={{architecture}} - -""" - -with open(defaults.BUILD_CONFIG, 'r') as f: - build_config = json.load(f) - -pbuilder_config = pystache.render(pbuilder_config_tmpl, build_config) - -print("Configuring pbuilder") - -with open(defaults.PBUILDER_CONFIG, 'w+') as f: - f.write(pbuilder_config) diff --git a/scripts/pbuilder-setup b/scripts/pbuilder-setup deleted file mode 100755 index a89348b8..00000000 --- a/scripts/pbuilder-setup +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2015 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# File: pbuilder-setup -# Purpose: -# Bootstraps a Debian environment for use by pbuilder. - - -import sys -import os -import json -import distutils.dir_util - -import pystache - -import defaults -import util - -util.check_build_config() - -pbuilder_create_cmd_tmpl= """ - sudo pbuilder --create \ - --configfile {{pbuilder_config}} -""" - -with open(defaults.BUILD_CONFIG, 'r') as f: - build_config = json.load(f) - -pbuilder_create_command = pystache.render(pbuilder_create_cmd_tmpl, build_config) - -print("Creating a pbuilder environment") -#os.chdir(defaults.BUILD_DIR) - -distutils.dir_util.mkpath(defaults.PBUILDER_DIR) - -result = os.system(pbuilder_create_command) -if result > 0: - print("pbuilder environment bootstrap failed") - sys.exit(1) diff --git a/scripts/pbuilder/hooks/C10shell b/scripts/pbuilder/hooks/C10shell deleted file mode 100755 index f56f9f7f..00000000 --- a/scripts/pbuilder/hooks/C10shell +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -# invoke shell if build fails. - -apt-get install -y --force-yes vim nano less bash -cd /tmp/buildd/*/debian/.. -/bin/bash < /dev/tty > /dev/tty 2> /dev/tty diff --git a/tools/get_latest_iso.py b/tools/get_latest_iso.py deleted file mode 100755 index f8228ff2..00000000 --- a/tools/get_latest_iso.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys -from lxml import html -from urllib.parse import unquote -import requests - -BASE_URL = 'https://downloads.vyos.io/' -PAGE_URL = BASE_URL+'?dir=rolling/sagitta/amd64' - - -def download(): - page = requests.get(PAGE_URL) - tree = html.fromstring(page.content) - path = '//*[@id="directory-listing"]/li/a[1]/@href' - isos = [x for x in tree.xpath(path) if os.path.splitext(x)[1] == '.iso'] - latest_iso_url = os.path.join(BASE_URL, isos[-1]) - filename = unquote(os.path.basename(latest_iso_url)) - print(filename) - if os.path.exists(filename): - print("{} already exists".format(filename)) - sys.exit(0) - r = requests.get(latest_iso_url) - with open(filename, 'wb') as fd: - for chunk in r.iter_content(chunk_size=128): - fd.write(chunk) - - -if __name__ == '__main__': - download() diff --git a/tools/gpl-header-template b/tools/gpl-header-template deleted file mode 100644 index 63be9d9a..00000000 --- a/tools/gpl-header-template +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (C) 2016 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# File: -# Purpose: - diff --git a/tools/submod-mk b/tools/submod-mk deleted file mode 100644 index eb61da18..00000000 --- a/tools/submod-mk +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -# -# **** License **** -# -# Copyright (C) 2013 Vyatta, Inc. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# **** End License **** - -progname=${0##*/} -shopt -s nullglob -shopt -s extglob -cd packages - -info=echo -trace= -noclean="-nc" -build="debuild -i -b -uc -us" - -declare -a submodule -declare -a debs -for debian in !(installer|linux-kernel-di-i386-2.6)/debian ; do - smod=${debian%/*} - debs=( ${smod}_*.deb ) - if [ ${#debs[@]} -eq 0 ] ; then - submodule+=( $smod ) - fi -done - -while [ $# -gt 0 ] ; do - case "$1" in - -h | --help ) - cat <<-EOF -Usage: $progname [Options] [ SUBMODULE... ] -Options: - -n | --do-nothing Don´t actually remove or build anything, - just show what would be done - -q | --quiet Quiet, don't print progress info - -c | --clean Clean build - -b | --binary Skip source package build (default) - -s | --source Build binary and source packages - -S | --signed-source Build and sign packages - -If no SUBMODULE(s) given, build all checked-out submodules w/o debs. -EOF - exit 0;; - -n | --do-nothing ) - trace=echo - shift;; - -q | --quiet ) - info='#' - shift;; - -c | --clean ) - noclean= - shift;; - -b | --binary ) - shift ;; # default - -s | --source ) - build="git buildpackage -uc -us" - shift;; - -S | --signed-source ) - build="git buildpackage" - shift;; - * ) - submodule=( $@ ) - break;; - esac -done - -for (( i=0; i<${#submodule[@]}; i++)) ; do - eval $info P: ${submodule[i]} - ( cd ${submodule[i]} && eval $trace $build $noclean ) || exit $? -done diff --git a/tools/switch-remote b/tools/switch-remote deleted file mode 100755 index 328b735c..00000000 --- a/tools/switch-remote +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh - -HTTPS_BASE_URL="https://github.com/vyos" -SSH_BASE_URL="git@github.com:vyos" - -REMOTE=`git config remote.origin.url` - -# extract vyatta-foo.git -BASENAME=`echo $REMOTE | sed -re 's!.*/(.*)$!\1!'` -echo "Changing remote for $BASENAME" - -# Print usage if no arguments given -if [ -z "$1" ]; then - echo "Switches remote URL to SSH or HTTPS" - echo "Use \"$0 ssh\" to switch to SSH" - echo "Use \"$0 https\" to switch to HTTPS" -fi - -case $1 in - ssh) - echo "New remote: $SSH_BASE_URL/$BASENAME" - git config remote.origin.url $SSH_BASE_URL/$BASENAME - ;; - https) - echo "New remote: $HTTPS_BASE_URL/$BASENAME" - git config remote.origin.url $HTTPS_BASE_URL/$BASENAME - ;; - *) - echo "Wrong option, use \"ssh\" or \"https\"" - ;; -esac - - - - -- cgit v1.2.3 From 75c8f627a896455952abd8944f77f8fc443a3211 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Thu, 21 Mar 2024 13:58:48 +0000 Subject: build script: T3664: fix handling of missing or malformed flavor files (cherry picked from commit 65b0321a8b92a4fcbb211caeec31aaa82a3a490c) --- scripts/build-vyos-image | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'scripts') diff --git a/scripts/build-vyos-image b/scripts/build-vyos-image index 5ee0b3ed..aa0a6aad 100755 --- a/scripts/build-vyos-image +++ b/scripts/build-vyos-image @@ -191,6 +191,25 @@ if __name__ == "__main__": print("\n".join(build_flavors)) sys.exit(1) + ## Try to get correct architecture and build type from build flavor and CLI arguments + pre_build_config = merge_dicts({}, build_defaults) + + flavor_config = {} + build_flavor = args["build_flavor"] + try: + with open(make_toml_path(defaults.BUILD_FLAVORS_DIR, args["build_flavor"]), 'rb') as f: + flavor_config = tomli.load(f) + pre_build_config = merge_dicts(flavor_config, pre_build_config) + except FileNotFoundError: + print(f"E: Flavor '{build_flavor}' does not exist") + sys.exit(1) + except tomli.TOMLDecodeError as e: + print(f"E: Failed to parse TOML file for flavor '{build_flavor}': {e}") + sys.exit(1) + + ## Combine configs args > flavor > defaults + pre_build_config = merge_dicts(args, pre_build_config, skip_none=True) + # Some fixup for mirror settings. # The idea is: if --debian-mirror is specified but --pbuilder-debian-mirror is not, # use the --debian-mirror value for both lb and pbuilder bootstrap -- cgit v1.2.3 From df058a250f3845e9126ec0d684fafc4d302b0afe Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Sat, 23 Mar 2024 20:50:36 +0000 Subject: build scripts: T3664: move image build scripts to a dedicated directory to avoid a mix of image build scripts and ancilliary scripts in the same directory (cherry picked from commit 750819bfec5335566dfc48de1ab6dbbc869068a3) --- build-vyos-image | 2 +- scripts/build-vyos-image | 541 ---------------------------------- scripts/image-build/build-vyos-image | 542 +++++++++++++++++++++++++++++++++++ scripts/image-build/defaults.py | 48 ++++ scripts/image-build/utils.py | 78 +++++ scripts/vyos_build_defaults.py | 48 ---- scripts/vyos_build_utils.py | 77 ----- 7 files changed, 669 insertions(+), 667 deletions(-) delete mode 100755 scripts/build-vyos-image create mode 100755 scripts/image-build/build-vyos-image create mode 100644 scripts/image-build/defaults.py create mode 100644 scripts/image-build/utils.py delete mode 100644 scripts/vyos_build_defaults.py delete mode 100644 scripts/vyos_build_utils.py (limited to 'scripts') diff --git a/build-vyos-image b/build-vyos-image index 360be935..0c61a9d5 120000 --- a/build-vyos-image +++ b/build-vyos-image @@ -1 +1 @@ -scripts/build-vyos-image \ No newline at end of file +scripts/image-build/build-vyos-image \ No newline at end of file diff --git a/scripts/build-vyos-image b/scripts/build-vyos-image deleted file mode 100755 index aa0a6aad..00000000 --- a/scripts/build-vyos-image +++ /dev/null @@ -1,541 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2022-2024 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# File: build-vyos-image -# Purpose: builds VyOS images using a fork of Debian's live-build tool - -import re -import os -import sys -import uuid -import glob -import shutil -import getpass -import platform -import argparse -import datetime -import functools -import string - -import json - -try: - import tomli - import jinja2 - import git -except ModuleNotFoundError as e: - print(f"Cannot load a required library: {e}") - print("Please make sure the following Python3 modules are installed: tomli jinja2 git") - -import vyos_build_utils as utils -import vyos_build_defaults as defaults - -# argparse converts hyphens to underscores, -# so for lookups in the original options hash we have to convert them back -def field_to_option(s): - return re.sub(r'_', '-', s) - -def get_default_build_by(): - return "{user}@{host}".format(user= getpass.getuser(), host=platform.node()) - -def get_validator(optdict, name): - try: - return optdict[name][2] - except KeyError: - return None - -def merge_dicts(source, destination): - """ Merge two dictionaries and return a new dict which has the merged key/value pairs. - Merging logic is as follows: - Sub-dicts are merged. - List values are combined. - Scalar values are set to those from the source dict. - """ - from copy import deepcopy - tmp = deepcopy(destination) - - for key, value in source.items(): - if key not in tmp: - tmp[key] = value - elif isinstance(source[key], dict): - tmp[key] = merge_dicts(source[key], tmp[key]) - elif isinstance(source[key], list): - tmp[key] = source[key] + tmp[key] - else: - tmp[key] = source[key] - - return tmp - -def has_nonempty_key(config, key): - if key in config: - if config[key]: - return True - return False - -def make_toml_path(dir, file_basename): - return os.path.join(dir, file_basename + ".toml") - - -if __name__ == "__main__": - ## Check if the script is running wirh root permissions - ## Since live-build requires privileged calls such as chroot(), - ## there's no real way around it. - if os.getuid() != 0: - print("E: this script requires root privileges") - sys.exit(1) - - ## Check if there are missing build dependencies - deps = { - 'packages': [ - 'sudo', - 'make', - 'live-build', - 'pbuilder', - 'devscripts', - 'python3-pystache', - 'python3-git', - 'qemu-utils' - ], - 'binaries': [] - } - - print("I: Checking if packages required for VyOS image build are installed") - try: - checker = utils.check_system_dependencies(deps) - except OSError as e: - print(e) - sys.exit(1) - - ## Load the file with default build configuration options - try: - with open(defaults.DEFAULTS_FILE, 'rb') as f: - build_defaults = tomli.load(f) - except Exception as e: - print("Failed to open the defaults file {0}: {1}".format(defaults.DEFAULTS_FILE, e)) - sys.exit(1) - - ## Get a list of available build flavors - build_flavors = list(map(lambda f: os.path.splitext(f)[0], os.listdir(defaults.BUILD_FLAVORS_DIR))) - - ## Set up the option parser - ## XXX: It uses values from the default configuration file for its option defaults, - ## which is why it's defined after loading the defaults.toml file data. - - # Options dict format: - # '$option_name_without_leading_dashes': { ('$help_string', $default_value_generator_thunk, $value_checker_thunk) } - options = { - 'architecture': ('Image target architecture (amd64 or arm64)', - lambda: build_defaults['architecture'], lambda x: x in ['amd64', 'arm64']), - 'build-by': ('Builder identifier (e.g. jrandomhacker@example.net)', get_default_build_by, None), - 'debian-mirror': ('Debian repository mirror', lambda: build_defaults['debian_mirror'], None), - 'debian-security-mirror': ('Debian security updates mirror', lambda: build_defaults['debian_security_mirror'], None), - 'pbuilder-debian-mirror': ('Debian repository mirror for pbuilder env bootstrap', lambda: build_defaults['debian_mirror'], None), - 'vyos-mirror': ('VyOS package mirror', lambda: build_defaults["vyos_mirror"], None), - 'build-type': ('Build type, release or development', lambda: 'development', lambda x: x in ['release', 'development']), - 'version': ('Version number (release builds only)', None, None), - 'build-comment': ('Optional build comment', lambda: '', None) - } - - # Create the option parser - parser = argparse.ArgumentParser() - for k, v in options.items(): - help_string, default_value_thunk = v[0], v[1] - if default_value_thunk is None: - parser.add_argument('--' + k, type=str, help=help_string) - else: - parser.add_argument('--' + k, type=str, help=help_string, default=default_value_thunk()) - - # The debug option is a bit special since it different type is different - parser.add_argument('--debug', help='Enable debug output', action='store_true') - - parser.add_argument('--dry-run', help='Check build configuration and exit', action='store_true') - - # Custom APT entry and APT key options can be used multiple times - parser.add_argument('--custom-apt-entry', help="Custom APT entry", action='append', default=[]) - parser.add_argument('--custom-apt-key', help="Custom APT key file", action='append', default=[]) - parser.add_argument('--custom-package', help="Custom package to install from repositories", action='append', default=[]) - - # Build flavor is a positional argument - parser.add_argument('build_flavor', help='Build flavor', nargs='?', action='store') - - args = vars(parser.parse_args()) - - debug = args["debug"] - - # Validate options - for k, v in args.items(): - key = field_to_option(k) - func = get_validator(options, k) - if func is not None: - if not func(v): - print("{v} is not a valid value for --{o} option".format(o=key, v=v)) - sys.exit(1) - - if not args["build_flavor"]: - print("E: Build flavor is not specified!") - print("E: For example, to build the generic ISO, run {} iso".format(sys.argv[0])) - print("Available build flavors:\n") - print("\n".join(build_flavors)) - sys.exit(1) - - ## Try to get correct architecture and build type from build flavor and CLI arguments - pre_build_config = merge_dicts({}, build_defaults) - - flavor_config = {} - build_flavor = args["build_flavor"] - try: - with open(make_toml_path(defaults.BUILD_FLAVORS_DIR, args["build_flavor"]), 'rb') as f: - flavor_config = tomli.load(f) - pre_build_config = merge_dicts(flavor_config, pre_build_config) - except FileNotFoundError: - print(f"E: Flavor '{build_flavor}' does not exist") - sys.exit(1) - except tomli.TOMLDecodeError as e: - print(f"E: Failed to parse TOML file for flavor '{build_flavor}': {e}") - sys.exit(1) - - ## Combine configs args > flavor > defaults - pre_build_config = merge_dicts(args, pre_build_config, skip_none=True) - - # Some fixup for mirror settings. - # The idea is: if --debian-mirror is specified but --pbuilder-debian-mirror is not, - # use the --debian-mirror value for both lb and pbuilder bootstrap - if (args['debian_mirror'] != build_defaults["debian_mirror"]) and \ - (args['pbuilder_debian_mirror'] == build_defaults["debian_mirror"]): - args['pbuilder_debian_mirror'] = args['debian_mirror'] - - # Version can only be set for release builds, - # for dev builds it hardly makes any sense - if args['build_type'] == 'development': - if args['version'] is not None: - print("Version can only be set for release builds") - print("Use --build-type=release option if you want to set version number") - sys.exit(1) - - # Validate characters in version name - if 'version' in args and args['version'] != None: - allowed = string.ascii_letters + string.digits + '.' + '-' + '+' - if not set(args['version']) <= set(allowed): - print(f'Version contained illegal character(s), allowed: {allowed}') - sys.exit(1) - - ## Inject some useful hardcoded options - args['build_dir'] = defaults.BUILD_DIR - args['pbuilder_config'] = os.path.join(defaults.BUILD_DIR, defaults.PBUILDER_CONFIG) - - ## Combine the arguments with non-configurable defaults - build_config = merge_dicts(args, build_defaults) - - ## Load the flavor file and mix-ins - with open(make_toml_path(defaults.BUILD_TYPES_DIR, build_config["build_type"]), 'rb') as f: - build_type_config = tomli.load(f) - build_config = merge_dicts(build_type_config, build_config) - - with open(make_toml_path(defaults.BUILD_ARCHES_DIR, build_config["architecture"]), 'rb') as f: - build_arch_config = tomli.load(f) - build_config = merge_dicts(build_arch_config, build_config) - - with open(make_toml_path(defaults.BUILD_FLAVORS_DIR, build_config["build_flavor"]), 'rb') as f: - flavor_config = tomli.load(f) - build_config = merge_dicts(flavor_config, build_config) - - ## Rename and merge some fields for simplicity - ## E.g. --custom-packages is for the user, but internally - ## it's added to the same package list as everything else - if has_nonempty_key(build_config, "custom_package"): - build_config["packages"] += build_config["custom_package"] - del build_config["custom_package"] - - ## Add architecture-dependent packages from the flavor - if has_nonempty_key(build_config, "architectures"): - arch = build_config["architecture"] - if arch in build_config["architectures"]: - build_config["packages"] += build_config["architectures"][arch]["packages"] - - ## Dump the complete config if the user enabled debug mode - if debug: - import json - print("D: Effective build config:\n") - print(json.dumps(build_config, indent=4)) - - ## Clean up the old build config and set up a fresh copy - lb_config_dir = os.path.join(defaults.BUILD_DIR, defaults.LB_CONFIG_DIR) - print(lb_config_dir) - shutil.rmtree(lb_config_dir, ignore_errors=True) - shutil.copytree("data/live-build-config/", lb_config_dir) - os.makedirs(lb_config_dir, exist_ok=True) - - ## Create the version file - - # Create a build timestamp - now = datetime.datetime.today() - build_timestamp = now.strftime("%Y%m%d%H%M") - - # FIXME: use aware rather than naive object - build_date = now.strftime("%a %d %b %Y %H:%M UTC") - - # Assign a (hopefully) unique identifier to the build (UUID) - build_uuid = str(uuid.uuid4()) - - # Initialize Git object from our repository - try: - repo = git.Repo('.') - - # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient - build_git = repo.head.object.hexsha[:14] - # If somone played around with the source tree and the build is "dirty", mark it - if repo.is_dirty(): - build_git += "-dirty" - - # Retrieve git branch name or current tag - # Building a tagged release might leave us checking out a git tag that is not the tip of a named branch (detached HEAD) - # Check if the current HEAD is associated with a tag and use its name instead of an unavailable branch name. - git_branch = next((tag.name for tag in repo.tags if tag.commit == repo.head.commit), None) - if git_branch is None: - git_branch = repo.active_branch.name - except Exception as e: - exit(f'Could not retrieve information from git: {e}') - build_git = "" - git_branch = "" - - # Create the build version string - if build_config['build_type'] == 'development': - try: - if not git_branch: - raise ValueError("git branch could not be determined") - - # Load the branch to version mapping file - with open('data/versions') as f: - version_mapping = json.load(f) - - branch_version = version_mapping[git_branch] - - version = "{0}-rolling-{1}".format(branch_version, build_timestamp) - except Exception as e: - print("Could not build a version string specific to git branch, falling back to default: {0}".format(str(e))) - version = "999.{0}".format(build_timestamp) - else: - # Release build, use the version from ./configure arguments - version = build_config['version'] - - if build_config['build_type'] == 'development': - lts_build = False - else: - lts_build = True - - version_data = { - 'version': version, - 'built_by': build_config['build_by'], - 'built_on': build_date, - 'build_uuid': build_uuid, - 'build_git': build_git, - 'build_branch': git_branch, - 'release_train': build_config['release_train'], - 'lts_build': lts_build, - 'build_comment': build_config['build_comment'], - 'bugtracker_url': build_config['bugtracker_url'], - 'documentation_url': build_config['documentation_url'], - 'project_news_url': build_config['project_news_url'], - } - - # Multi line strings needs to be un-indented to not have leading - # whitespaces in the resulting file - os_release = f""" -PRETTY_NAME="VyOS {version} ({build_config['release_train']})" -NAME="VyOS" -VERSION_ID="{version}" -VERSION="{version} ({build_config['release_train']})" -VERSION_CODENAME={build_defaults['debian_distribution']} -ID=vyos -BUILD_ID="{build_git}" -HOME_URL="{build_defaults['website_url']}" -SUPPORT_URL="{build_defaults['support_url']}" -BUG_REPORT_URL="{build_defaults['bugtracker_url']}" -DOCUMENTATION_URL="{build_config['documentation_url']}" - """ - - chroot_includes_dir = os.path.join(defaults.BUILD_DIR, defaults.CHROOT_INCLUDES_DIR) - binary_includes_dir = os.path.join(defaults.BUILD_DIR, defaults.BINARY_INCLUDES_DIR) - vyos_data_dir = os.path.join(chroot_includes_dir, "usr/share/vyos") - os.makedirs(vyos_data_dir, exist_ok=True) - with open(os.path.join(vyos_data_dir, 'version.json'), 'w') as f: - json.dump(version_data, f) - with open(os.path.join(binary_includes_dir, 'version.json'), 'w') as f: - json.dump(version_data, f) - - # For backwards compatibility with 'add system image' script from older versions - # we need a file in the old format so that script can find out the version of the image - # for upgrade - os.makedirs(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/'), exist_ok=True) - with open(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/version'), 'w') as f: - print("Version: {0}".format(version), file=f) - - - # Define variables that influence to welcome message on boot - os.makedirs(os.path.join(chroot_includes_dir, 'usr/lib/'), exist_ok=True) - with open(os.path.join(chroot_includes_dir, 'usr/lib/os-release'), 'w') as f: - print(os_release, file=f) - - - ## Clean up earlier build state and artifacts - print("I: Cleaning the build workspace") - os.system("lb clean") - #iter(lambda p: shutil.rmtree(p, ignore_errors=True), - # ['config/binary', 'config/bootstrap', 'config/chroot', 'config/common', 'config/source']) - artifacts = functools.reduce( - lambda x, y: x + y, - map(glob.glob, ['*.iso', '*.raw', '*.img', '*.xz', '*.ova', '*.ovf'])) - iter(os.remove, artifacts) - - ## Create live-build configuration files - - # Add the additional repositories to package lists - print("I: Setting up additional APT entries") - vyos_repo_entry = "deb {vyos_mirror} {vyos_branch} main\n".format(**build_config) - - apt_file = defaults.VYOS_REPO_FILE - - if debug: - print(f"D: Adding these entries to {apt_file}:") - print("\t", vyos_repo_entry) - - with open(apt_file, 'w') as f: - f.write(vyos_repo_entry) - - # Add custom APT entries - if build_config.get('additional_repositories', False): - build_config['custom_apt_entry'] += build_config['additional_repositories'] - - if build_config.get('custom_apt_entry', False): - custom_apt_file = defaults.CUSTOM_REPO_FILE - entries = "\n".join(build_config['custom_apt_entry']) - if debug: - print("D: Adding custom APT entries:") - print(entries) - with open(custom_apt_file, 'w') as f: - f.write(entries) - f.write("\n") - - # Add custom APT keys - if has_nonempty_key(build_config, 'custom_apt_key'): - key_dir = defaults.ARCHIVES_DIR - for k in build_config['custom_apt_key']: - dst_name = '{0}.key.chroot'.format(os.path.basename(k)) - shutil.copy(k, os.path.join(key_dir, dst_name)) - - # Add custom packages - if has_nonempty_key(build_config, 'packages'): - package_list_file = defaults.PACKAGE_LIST_FILE - packages = "\n".join(build_config['packages']) - with open (package_list_file, 'w') as f: - f.write(packages) - - ## Create includes - if has_nonempty_key(build_config, "includes_chroot"): - for i in build_config["includes_chroot"]: - file_path = os.path.join(chroot_includes_dir, i["path"]) - os.makedirs(os.path.dirname(file_path), exist_ok=True) - with open(file_path, 'w') as f: - f.write(i["data"]) - - ## Create the default config - ## Technically it's just another includes.chroot entry, - ## but it's special enough to warrant making it easier for flavor writers - if has_nonempty_key(build_config, "default_config"): - file_path = os.path.join(chroot_includes_dir, "opt/vyatta/etc/config.boot.default") - os.makedirs(os.path.dirname(file_path), exist_ok=True) - with open(file_path, 'w') as f: - f.write(build_config["default_config"]) - - ## Configure live-build - lb_config_tmpl = jinja2.Template(""" - lb config noauto \ - --apt-indices false \ - --apt-options "--yes -oAPT::Get::allow-downgrades=true" \ - --apt-recommends false \ - --architecture {{architecture}} \ - --archive-areas {{debian_archive_areas}} \ - --backports true \ - --binary-image iso-hybrid \ - --bootappend-live "boot=live components hostname=vyos username=live nopersistence noautologin nonetworking union=overlay console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \ - --bootappend-live-failsafe "live components memtest noapic noapm nodma nomce nolapic nomodeset nosmp nosplash vga=normal console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \ - --bootloaders {{bootloaders}} \ - --checksums 'sha256 md5' \ - --chroot-squashfs-compression-type "{{squashfs_compression_type}}" \ - --debian-installer none \ - --debootstrap-options "--variant=minbase --exclude=isc-dhcp-client,isc-dhcp-common,ifupdown --include=apt-utils,ca-certificates,gnupg2" \ - --distribution {{debian_distribution}} \ - --firmware-binary false \ - --firmware-chroot false \ - --iso-application "VyOS" \ - --iso-publisher "{{build_by}}" \ - --iso-volume "VyOS" \ - --linux-flavours {{kernel_flavor}} \ - --linux-packages linux-image-{{kernel_version}} \ - --mirror-binary {{debian_mirror}} \ - --mirror-binary-security {{debian_security_mirror}} \ - --mirror-bootstrap {{debian_mirror}} \ - --mirror-chroot {{debian_mirror}} \ - --mirror-chroot-security {{debian_security_mirror}} \ - --security true \ - --updates true - "${@}" - """) - - lb_config_command = lb_config_tmpl.render(build_config) - - ## Pin release for VyOS packages - apt_pin = f"""Package: * -Pin: release n={build_config['release_train']} -Pin-Priority: 600 -""" - - with open(defaults.VYOS_PIN_FILE, 'w') as f: - f.write(apt_pin) - - print("I: Configuring live-build") - - if debug: - print("D: live-build configuration command") - print(lb_config_command) - - result = os.system(lb_config_command) - if result > 0: - print("E: live-build config failed") - sys.exit(1) - - ## In dry-run mode, exit at this point - if build_config["dry_run"]: - print("I: dry-run, not starting image build") - sys.exit(0) - - ## Add local packages - local_packages = glob.glob('../packages/*.deb') - if local_packages: - for f in local_packages: - shutil.copy(f, os.path.join(defaults.LOCAL_PACKAGES_PATH, os.path.basename(f))) - - ## Build the image - print("I: Starting image build") - if debug: - print("D: It's not like I'm building this specially for you or anything!") - res = os.system("lb build 2>&1") - if res > 0: - sys.exit(res) - - # Copy the image - shutil.copy("live-image-{0}.hybrid.iso".format(build_config["architecture"]), - "vyos-{0}-{1}.iso".format(version_data["version"], build_config["architecture"])) diff --git a/scripts/image-build/build-vyos-image b/scripts/image-build/build-vyos-image new file mode 100755 index 00000000..59ea0338 --- /dev/null +++ b/scripts/image-build/build-vyos-image @@ -0,0 +1,542 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022-2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# File: build-vyos-image +# Purpose: builds VyOS images using a fork of Debian's live-build tool + +import re +import os +import sys +import uuid +import glob +import shutil +import getpass +import platform +import argparse +import datetime +import functools +import string + +import json + +try: + import tomli + import jinja2 + import git +except ModuleNotFoundError as e: + print(f"Cannot load a required library: {e}") + print("Please make sure the following Python3 modules are installed: tomli jinja2 git") + +# Local modules +import utils +import defaults + +# argparse converts hyphens to underscores, +# so for lookups in the original options hash we have to convert them back +def field_to_option(s): + return re.sub(r'_', '-', s) + +def get_default_build_by(): + return "{user}@{host}".format(user= getpass.getuser(), host=platform.node()) + +def get_validator(optdict, name): + try: + return optdict[name][2] + except KeyError: + return None + +def merge_dicts(source, destination): + """ Merge two dictionaries and return a new dict which has the merged key/value pairs. + Merging logic is as follows: + Sub-dicts are merged. + List values are combined. + Scalar values are set to those from the source dict. + """ + from copy import deepcopy + tmp = deepcopy(destination) + + for key, value in source.items(): + if key not in tmp: + tmp[key] = value + elif isinstance(source[key], dict): + tmp[key] = merge_dicts(source[key], tmp[key]) + elif isinstance(source[key], list): + tmp[key] = source[key] + tmp[key] + else: + tmp[key] = source[key] + + return tmp + +def has_nonempty_key(config, key): + if key in config: + if config[key]: + return True + return False + +def make_toml_path(dir, file_basename): + return os.path.join(dir, file_basename + ".toml") + + +if __name__ == "__main__": + ## Check if the script is running wirh root permissions + ## Since live-build requires privileged calls such as chroot(), + ## there's no real way around it. + if os.getuid() != 0: + print("E: this script requires root privileges") + sys.exit(1) + + ## Check if there are missing build dependencies + deps = { + 'packages': [ + 'sudo', + 'make', + 'live-build', + 'pbuilder', + 'devscripts', + 'python3-pystache', + 'python3-git', + 'qemu-utils' + ], + 'binaries': [] + } + + print("I: Checking if packages required for VyOS image build are installed") + try: + checker = utils.check_system_dependencies(deps) + except OSError as e: + print(e) + sys.exit(1) + + ## Load the file with default build configuration options + try: + with open(defaults.DEFAULTS_FILE, 'rb') as f: + build_defaults = tomli.load(f) + except Exception as e: + print("Failed to open the defaults file {0}: {1}".format(defaults.DEFAULTS_FILE, e)) + sys.exit(1) + + ## Get a list of available build flavors + build_flavors = list(map(lambda f: os.path.splitext(f)[0], os.listdir(defaults.BUILD_FLAVORS_DIR))) + + ## Set up the option parser + ## XXX: It uses values from the default configuration file for its option defaults, + ## which is why it's defined after loading the defaults.toml file data. + + # Options dict format: + # '$option_name_without_leading_dashes': { ('$help_string', $default_value_generator_thunk, $value_checker_thunk) } + options = { + 'architecture': ('Image target architecture (amd64 or arm64)', + lambda: build_defaults['architecture'], lambda x: x in ['amd64', 'arm64']), + 'build-by': ('Builder identifier (e.g. jrandomhacker@example.net)', get_default_build_by, None), + 'debian-mirror': ('Debian repository mirror', lambda: build_defaults['debian_mirror'], None), + 'debian-security-mirror': ('Debian security updates mirror', lambda: build_defaults['debian_security_mirror'], None), + 'pbuilder-debian-mirror': ('Debian repository mirror for pbuilder env bootstrap', lambda: build_defaults['debian_mirror'], None), + 'vyos-mirror': ('VyOS package mirror', lambda: build_defaults["vyos_mirror"], None), + 'build-type': ('Build type, release or development', lambda: 'development', lambda x: x in ['release', 'development']), + 'version': ('Version number (release builds only)', None, None), + 'build-comment': ('Optional build comment', lambda: '', None) + } + + # Create the option parser + parser = argparse.ArgumentParser() + for k, v in options.items(): + help_string, default_value_thunk = v[0], v[1] + if default_value_thunk is None: + parser.add_argument('--' + k, type=str, help=help_string) + else: + parser.add_argument('--' + k, type=str, help=help_string, default=default_value_thunk()) + + # The debug option is a bit special since it different type is different + parser.add_argument('--debug', help='Enable debug output', action='store_true') + + parser.add_argument('--dry-run', help='Check build configuration and exit', action='store_true') + + # Custom APT entry and APT key options can be used multiple times + parser.add_argument('--custom-apt-entry', help="Custom APT entry", action='append', default=[]) + parser.add_argument('--custom-apt-key', help="Custom APT key file", action='append', default=[]) + parser.add_argument('--custom-package', help="Custom package to install from repositories", action='append', default=[]) + + # Build flavor is a positional argument + parser.add_argument('build_flavor', help='Build flavor', nargs='?', action='store') + + args = vars(parser.parse_args()) + + debug = args["debug"] + + # Validate options + for k, v in args.items(): + key = field_to_option(k) + func = get_validator(options, k) + if func is not None: + if not func(v): + print("{v} is not a valid value for --{o} option".format(o=key, v=v)) + sys.exit(1) + + if not args["build_flavor"]: + print("E: Build flavor is not specified!") + print("E: For example, to build the generic ISO, run {} iso".format(sys.argv[0])) + print("Available build flavors:\n") + print("\n".join(build_flavors)) + sys.exit(1) + + ## Try to get correct architecture and build type from build flavor and CLI arguments + pre_build_config = merge_dicts({}, build_defaults) + + flavor_config = {} + build_flavor = args["build_flavor"] + try: + with open(make_toml_path(defaults.BUILD_FLAVORS_DIR, args["build_flavor"]), 'rb') as f: + flavor_config = tomli.load(f) + pre_build_config = merge_dicts(flavor_config, pre_build_config) + except FileNotFoundError: + print(f"E: Flavor '{build_flavor}' does not exist") + sys.exit(1) + except tomli.TOMLDecodeError as e: + print(f"E: Failed to parse TOML file for flavor '{build_flavor}': {e}") + sys.exit(1) + + ## Combine configs args > flavor > defaults + pre_build_config = merge_dicts(args, pre_build_config, skip_none=True) + + # Some fixup for mirror settings. + # The idea is: if --debian-mirror is specified but --pbuilder-debian-mirror is not, + # use the --debian-mirror value for both lb and pbuilder bootstrap + if (args['debian_mirror'] != build_defaults["debian_mirror"]) and \ + (args['pbuilder_debian_mirror'] == build_defaults["debian_mirror"]): + args['pbuilder_debian_mirror'] = args['debian_mirror'] + + # Version can only be set for release builds, + # for dev builds it hardly makes any sense + if args['build_type'] == 'development': + if args['version'] is not None: + print("Version can only be set for release builds") + print("Use --build-type=release option if you want to set version number") + sys.exit(1) + + # Validate characters in version name + if 'version' in args and args['version'] != None: + allowed = string.ascii_letters + string.digits + '.' + '-' + '+' + if not set(args['version']) <= set(allowed): + print(f'Version contained illegal character(s), allowed: {allowed}') + sys.exit(1) + + ## Inject some useful hardcoded options + args['build_dir'] = defaults.BUILD_DIR + args['pbuilder_config'] = os.path.join(defaults.BUILD_DIR, defaults.PBUILDER_CONFIG) + + ## Combine the arguments with non-configurable defaults + build_config = merge_dicts(args, build_defaults) + + ## Load the flavor file and mix-ins + with open(make_toml_path(defaults.BUILD_TYPES_DIR, build_config["build_type"]), 'rb') as f: + build_type_config = tomli.load(f) + build_config = merge_dicts(build_type_config, build_config) + + with open(make_toml_path(defaults.BUILD_ARCHES_DIR, build_config["architecture"]), 'rb') as f: + build_arch_config = tomli.load(f) + build_config = merge_dicts(build_arch_config, build_config) + + with open(make_toml_path(defaults.BUILD_FLAVORS_DIR, build_config["build_flavor"]), 'rb') as f: + flavor_config = tomli.load(f) + build_config = merge_dicts(flavor_config, build_config) + + ## Rename and merge some fields for simplicity + ## E.g. --custom-packages is for the user, but internally + ## it's added to the same package list as everything else + if has_nonempty_key(build_config, "custom_package"): + build_config["packages"] += build_config["custom_package"] + del build_config["custom_package"] + + ## Add architecture-dependent packages from the flavor + if has_nonempty_key(build_config, "architectures"): + arch = build_config["architecture"] + if arch in build_config["architectures"]: + build_config["packages"] += build_config["architectures"][arch]["packages"] + + ## Dump the complete config if the user enabled debug mode + if debug: + import json + print("D: Effective build config:\n") + print(json.dumps(build_config, indent=4)) + + ## Clean up the old build config and set up a fresh copy + lb_config_dir = os.path.join(defaults.BUILD_DIR, defaults.LB_CONFIG_DIR) + print(lb_config_dir) + shutil.rmtree(lb_config_dir, ignore_errors=True) + shutil.copytree("data/live-build-config/", lb_config_dir) + os.makedirs(lb_config_dir, exist_ok=True) + + ## Create the version file + + # Create a build timestamp + now = datetime.datetime.today() + build_timestamp = now.strftime("%Y%m%d%H%M") + + # FIXME: use aware rather than naive object + build_date = now.strftime("%a %d %b %Y %H:%M UTC") + + # Assign a (hopefully) unique identifier to the build (UUID) + build_uuid = str(uuid.uuid4()) + + # Initialize Git object from our repository + try: + repo = git.Repo('.') + + # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient + build_git = repo.head.object.hexsha[:14] + # If somone played around with the source tree and the build is "dirty", mark it + if repo.is_dirty(): + build_git += "-dirty" + + # Retrieve git branch name or current tag + # Building a tagged release might leave us checking out a git tag that is not the tip of a named branch (detached HEAD) + # Check if the current HEAD is associated with a tag and use its name instead of an unavailable branch name. + git_branch = next((tag.name for tag in repo.tags if tag.commit == repo.head.commit), None) + if git_branch is None: + git_branch = repo.active_branch.name + except Exception as e: + exit(f'Could not retrieve information from git: {e}') + build_git = "" + git_branch = "" + + # Create the build version string + if build_config['build_type'] == 'development': + try: + if not git_branch: + raise ValueError("git branch could not be determined") + + # Load the branch to version mapping file + with open('data/versions') as f: + version_mapping = json.load(f) + + branch_version = version_mapping[git_branch] + + version = "{0}-rolling-{1}".format(branch_version, build_timestamp) + except Exception as e: + print("Could not build a version string specific to git branch, falling back to default: {0}".format(str(e))) + version = "999.{0}".format(build_timestamp) + else: + # Release build, use the version from ./configure arguments + version = build_config['version'] + + if build_config['build_type'] == 'development': + lts_build = False + else: + lts_build = True + + version_data = { + 'version': version, + 'built_by': build_config['build_by'], + 'built_on': build_date, + 'build_uuid': build_uuid, + 'build_git': build_git, + 'build_branch': git_branch, + 'release_train': build_config['release_train'], + 'lts_build': lts_build, + 'build_comment': build_config['build_comment'], + 'bugtracker_url': build_config['bugtracker_url'], + 'documentation_url': build_config['documentation_url'], + 'project_news_url': build_config['project_news_url'], + } + + # Multi line strings needs to be un-indented to not have leading + # whitespaces in the resulting file + os_release = f""" +PRETTY_NAME="VyOS {version} ({build_config['release_train']})" +NAME="VyOS" +VERSION_ID="{version}" +VERSION="{version} ({build_config['release_train']})" +VERSION_CODENAME={build_defaults['debian_distribution']} +ID=vyos +BUILD_ID="{build_git}" +HOME_URL="{build_defaults['website_url']}" +SUPPORT_URL="{build_defaults['support_url']}" +BUG_REPORT_URL="{build_defaults['bugtracker_url']}" +DOCUMENTATION_URL="{build_config['documentation_url']}" + """ + + chroot_includes_dir = os.path.join(defaults.BUILD_DIR, defaults.CHROOT_INCLUDES_DIR) + binary_includes_dir = os.path.join(defaults.BUILD_DIR, defaults.BINARY_INCLUDES_DIR) + vyos_data_dir = os.path.join(chroot_includes_dir, "usr/share/vyos") + os.makedirs(vyos_data_dir, exist_ok=True) + with open(os.path.join(vyos_data_dir, 'version.json'), 'w') as f: + json.dump(version_data, f) + with open(os.path.join(binary_includes_dir, 'version.json'), 'w') as f: + json.dump(version_data, f) + + # For backwards compatibility with 'add system image' script from older versions + # we need a file in the old format so that script can find out the version of the image + # for upgrade + os.makedirs(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/'), exist_ok=True) + with open(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/version'), 'w') as f: + print("Version: {0}".format(version), file=f) + + + # Define variables that influence to welcome message on boot + os.makedirs(os.path.join(chroot_includes_dir, 'usr/lib/'), exist_ok=True) + with open(os.path.join(chroot_includes_dir, 'usr/lib/os-release'), 'w') as f: + print(os_release, file=f) + + + ## Clean up earlier build state and artifacts + print("I: Cleaning the build workspace") + os.system("lb clean") + #iter(lambda p: shutil.rmtree(p, ignore_errors=True), + # ['config/binary', 'config/bootstrap', 'config/chroot', 'config/common', 'config/source']) + artifacts = functools.reduce( + lambda x, y: x + y, + map(glob.glob, ['*.iso', '*.raw', '*.img', '*.xz', '*.ova', '*.ovf'])) + iter(os.remove, artifacts) + + ## Create live-build configuration files + + # Add the additional repositories to package lists + print("I: Setting up additional APT entries") + vyos_repo_entry = "deb {vyos_mirror} {vyos_branch} main\n".format(**build_config) + + apt_file = defaults.VYOS_REPO_FILE + + if debug: + print(f"D: Adding these entries to {apt_file}:") + print("\t", vyos_repo_entry) + + with open(apt_file, 'w') as f: + f.write(vyos_repo_entry) + + # Add custom APT entries + if build_config.get('additional_repositories', False): + build_config['custom_apt_entry'] += build_config['additional_repositories'] + + if build_config.get('custom_apt_entry', False): + custom_apt_file = defaults.CUSTOM_REPO_FILE + entries = "\n".join(build_config['custom_apt_entry']) + if debug: + print("D: Adding custom APT entries:") + print(entries) + with open(custom_apt_file, 'w') as f: + f.write(entries) + f.write("\n") + + # Add custom APT keys + if has_nonempty_key(build_config, 'custom_apt_key'): + key_dir = defaults.ARCHIVES_DIR + for k in build_config['custom_apt_key']: + dst_name = '{0}.key.chroot'.format(os.path.basename(k)) + shutil.copy(k, os.path.join(key_dir, dst_name)) + + # Add custom packages + if has_nonempty_key(build_config, 'packages'): + package_list_file = defaults.PACKAGE_LIST_FILE + packages = "\n".join(build_config['packages']) + with open (package_list_file, 'w') as f: + f.write(packages) + + ## Create includes + if has_nonempty_key(build_config, "includes_chroot"): + for i in build_config["includes_chroot"]: + file_path = os.path.join(chroot_includes_dir, i["path"]) + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, 'w') as f: + f.write(i["data"]) + + ## Create the default config + ## Technically it's just another includes.chroot entry, + ## but it's special enough to warrant making it easier for flavor writers + if has_nonempty_key(build_config, "default_config"): + file_path = os.path.join(chroot_includes_dir, "opt/vyatta/etc/config.boot.default") + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, 'w') as f: + f.write(build_config["default_config"]) + + ## Configure live-build + lb_config_tmpl = jinja2.Template(""" + lb config noauto \ + --apt-indices false \ + --apt-options "--yes -oAPT::Get::allow-downgrades=true" \ + --apt-recommends false \ + --architecture {{architecture}} \ + --archive-areas {{debian_archive_areas}} \ + --backports true \ + --binary-image iso-hybrid \ + --bootappend-live "boot=live components hostname=vyos username=live nopersistence noautologin nonetworking union=overlay console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \ + --bootappend-live-failsafe "live components memtest noapic noapm nodma nomce nolapic nomodeset nosmp nosplash vga=normal console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \ + --bootloaders {{bootloaders}} \ + --checksums 'sha256 md5' \ + --chroot-squashfs-compression-type "{{squashfs_compression_type}}" \ + --debian-installer none \ + --debootstrap-options "--variant=minbase --exclude=isc-dhcp-client,isc-dhcp-common,ifupdown --include=apt-utils,ca-certificates,gnupg2" \ + --distribution {{debian_distribution}} \ + --firmware-binary false \ + --firmware-chroot false \ + --iso-application "VyOS" \ + --iso-publisher "{{build_by}}" \ + --iso-volume "VyOS" \ + --linux-flavours {{kernel_flavor}} \ + --linux-packages linux-image-{{kernel_version}} \ + --mirror-binary {{debian_mirror}} \ + --mirror-binary-security {{debian_security_mirror}} \ + --mirror-bootstrap {{debian_mirror}} \ + --mirror-chroot {{debian_mirror}} \ + --mirror-chroot-security {{debian_security_mirror}} \ + --security true \ + --updates true + "${@}" + """) + + lb_config_command = lb_config_tmpl.render(build_config) + + ## Pin release for VyOS packages + apt_pin = f"""Package: * +Pin: release n={build_config['release_train']} +Pin-Priority: 600 +""" + + with open(defaults.VYOS_PIN_FILE, 'w') as f: + f.write(apt_pin) + + print("I: Configuring live-build") + + if debug: + print("D: live-build configuration command") + print(lb_config_command) + + result = os.system(lb_config_command) + if result > 0: + print("E: live-build config failed") + sys.exit(1) + + ## In dry-run mode, exit at this point + if build_config["dry_run"]: + print("I: dry-run, not starting image build") + sys.exit(0) + + ## Add local packages + local_packages = glob.glob('../packages/*.deb') + if local_packages: + for f in local_packages: + shutil.copy(f, os.path.join(defaults.LOCAL_PACKAGES_PATH, os.path.basename(f))) + + ## Build the image + print("I: Starting image build") + if debug: + print("D: It's not like I'm building this specially for you or anything!") + res = os.system("lb build 2>&1") + if res > 0: + sys.exit(res) + + # Copy the image + shutil.copy("live-image-{0}.hybrid.iso".format(build_config["architecture"]), + "vyos-{0}-{1}.iso".format(version_data["version"], build_config["architecture"])) diff --git a/scripts/image-build/defaults.py b/scripts/image-build/defaults.py new file mode 100644 index 00000000..1d7141ea --- /dev/null +++ b/scripts/image-build/defaults.py @@ -0,0 +1,48 @@ +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# File: defaults.py +# Purpose: Various default values for use in build scripts. + + +import os + +# Relative to the repository directory + +BUILD_DIR = 'build' +BUILD_CONFIG = os.path.join(BUILD_DIR, 'build-config.toml') + +DEFAULTS_FILE = 'data/defaults.toml' + +BUILD_TYPES_DIR = 'data/build-types' +BUILD_ARCHES_DIR = 'data/architectures' +BUILD_FLAVORS_DIR = 'data/build-flavors' + +# Relative to the build directory + +PBUILDER_CONFIG = 'pbuilderrc' +PBUILDER_DIR = 'pbuilder' + +LB_CONFIG_DIR = 'config' + +CHROOT_INCLUDES_DIR = 'config/includes.chroot' +BINARY_INCLUDES_DIR = 'config/includes.binary' +ARCHIVES_DIR = 'config/archives/' + +VYOS_REPO_FILE = 'config/archives/vyos.list.chroot' +VYOS_PIN_FILE = 'config/archives/release.pref.chroot' +CUSTOM_REPO_FILE = 'config/archives/custom.list.chroot' +PACKAGE_LIST_FILE = 'config/package-lists/custom.list.chroot' + +LOCAL_PACKAGES_PATH = 'config/packages.chroot/' diff --git a/scripts/image-build/utils.py b/scripts/image-build/utils.py new file mode 100644 index 00000000..6906c52d --- /dev/null +++ b/scripts/image-build/utils.py @@ -0,0 +1,78 @@ +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# File: util.py +# Purpose: +# Various common functions for use in build scripts. + + +import sys +import os +from distutils.spawn import find_executable + +# Local modules +import defaults + +def check_build_config(): + if not os.path.exists(defaults.BUILD_CONFIG): + print("Build config file ({file}) does not exist".format(file=defaults.BUILD_CONFIG)) + print("If you are running this script by hand, you should better not. Run 'make iso' instead.") + sys.exit(1) + + +class DependencyChecker(object): + def __init__(self, spec): + missing_packages = self._get_missing_packages(spec['packages']) + missing_binaries = self._get_missing_binaries(spec['binaries']) + self.__missing = {'packages': missing_packages, 'binaries': missing_binaries} + + + def _package_installed(self, name): + result = os.system("dpkg-query -W --showformat='${{Status}}\n' {name} 2>&1 | grep 'install ok installed' >/dev/null".format(name=name)) + return True if result == 0 else False + + def _get_missing_packages(self, packages): + missing_packages = [] + for p in packages: + if not self._package_installed(p): + missing_packages.append(p) + return missing_packages + + def _get_missing_binaries(self, binaries): + missing_binaries = [] + for b in binaries: + if not find_executable(b): + missing_binaries.append(b) + return missing_binaries + + def get_missing_dependencies(self): + if self.__missing['packages'] or self.__missing['binaries']: + return self.__missing + return None + + def format_missing_dependencies(self): + msg = "E: There are missing system dependencies!\n" + if self.__missing['packages']: + msg += "E: Missing packages: " + " ".join(self.__missing['packages']) + if self.__missing['binaries']: + msg += "E: Missing binaries: " + " ".join(self.__missing['binaries']) + return msg + +def check_system_dependencies(deps): + checker = DependencyChecker(deps) + missing = checker.get_missing_dependencies() + if missing: + raise OSError(checker.format_missing_dependencies()) + else: + pass diff --git a/scripts/vyos_build_defaults.py b/scripts/vyos_build_defaults.py deleted file mode 100644 index 1e153f4b..00000000 --- a/scripts/vyos_build_defaults.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (C) 2018 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# File: defaults.py -# Purpose: Various default values for use in build scripts. - - -import os - -# Relative to the repository directory - -BUILD_DIR = 'build' -BUILD_CONFIG = os.path.join(BUILD_DIR, 'build-config.toml') - -DEFAULTS_FILE = 'data/defaults.toml' - -BUILD_TYPES_DIR = 'data/build-types' -BUILD_ARCHES_DIR = 'data/architectures' -BUILD_FLAVORS_DIR = 'data/build-flavors' - -# Relative to the build directory - -PBUILDER_CONFIG = 'pbuilderrc' -PBUILDER_DIR = 'pbuilder' - -LB_CONFIG_DIR = 'config' - -CHROOT_INCLUDES_DIR = 'config/includes.chroot' -BINARY_INCLUDES_DIR = 'config/includes.binary' -ARCHIVES_DIR = 'config/archives/' - -VYOS_REPO_FILE = 'config/archives/vyos.list.chroot' -VYOS_PIN_FILE = 'config/archives/release.pref.chroot' -CUSTOM_REPO_FILE = 'config/archives/custom.list.chroot' -PACKAGE_LIST_FILE = 'config/package-lists/custom.list.chroot' - -LOCAL_PACKAGES_PATH = 'config/packages.chroot/' diff --git a/scripts/vyos_build_utils.py b/scripts/vyos_build_utils.py deleted file mode 100644 index ed358848..00000000 --- a/scripts/vyos_build_utils.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (C) 2015 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# File: util.py -# Purpose: -# Various common functions for use in build scripts. - - -import sys -import os -from distutils.spawn import find_executable - -import vyos_build_defaults as defaults - -def check_build_config(): - if not os.path.exists(defaults.BUILD_CONFIG): - print("Build config file ({file}) does not exist".format(file=defaults.BUILD_CONFIG)) - print("If you are running this script by hand, you should better not. Run 'make iso' instead.") - sys.exit(1) - - -class DependencyChecker(object): - def __init__(self, spec): - missing_packages = self._get_missing_packages(spec['packages']) - missing_binaries = self._get_missing_binaries(spec['binaries']) - self.__missing = {'packages': missing_packages, 'binaries': missing_binaries} - - - def _package_installed(self, name): - result = os.system("dpkg-query -W --showformat='${{Status}}\n' {name} 2>&1 | grep 'install ok installed' >/dev/null".format(name=name)) - return True if result == 0 else False - - def _get_missing_packages(self, packages): - missing_packages = [] - for p in packages: - if not self._package_installed(p): - missing_packages.append(p) - return missing_packages - - def _get_missing_binaries(self, binaries): - missing_binaries = [] - for b in binaries: - if not find_executable(b): - missing_binaries.append(b) - return missing_binaries - - def get_missing_dependencies(self): - if self.__missing['packages'] or self.__missing['binaries']: - return self.__missing - return None - - def format_missing_dependencies(self): - msg = "E: There are missing system dependencies!\n" - if self.__missing['packages']: - msg += "E: Missing packages: " + " ".join(self.__missing['packages']) - if self.__missing['binaries']: - msg += "E: Missing binaries: " + " ".join(self.__missing['binaries']) - return msg - -def check_system_dependencies(deps): - checker = DependencyChecker(deps) - missing = checker.get_missing_dependencies() - if missing: - raise OSError(checker.format_missing_dependencies()) - else: - pass -- cgit v1.2.3 From 249a595f1d76e955465dafc1a757483cb75705d5 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Sun, 7 Apr 2024 16:15:50 +0000 Subject: build: T3664: add support for building non-ISO flavors (cherry picked from commit a896176ad8a1e1c7ef440a31c5afcfad358ed309) --- scripts/image-build/build-vyos-image | 631 +++++++++++++++++++---------------- scripts/image-build/defaults.py | 9 + scripts/image-build/raw_image.py | 215 ++++++++++++ 3 files changed, 569 insertions(+), 286 deletions(-) create mode 100644 scripts/image-build/raw_image.py (limited to 'scripts') diff --git a/scripts/image-build/build-vyos-image b/scripts/image-build/build-vyos-image index 59ea0338..aa397843 100755 --- a/scripts/image-build/build-vyos-image +++ b/scripts/image-build/build-vyos-image @@ -17,32 +17,43 @@ # File: build-vyos-image # Purpose: builds VyOS images using a fork of Debian's live-build tool +# Import Python's standard library modules import re import os import sys import uuid import glob +import json import shutil import getpass import platform import argparse import datetime import functools -import string -import json +# Add the vyos-1x submodule directory to the Python path +# so that we can import modules from it. +VYOS1X_DIR = os.path.join(os.getcwd(), 'vyos-1x/python') +if not os.path.exists(VYOS1X_DIR): + print("E: vyos-1x subdirectory does not exist, did you initialize submodules?") +else: + sys.path.append(VYOS1X_DIR) +# Import third-party modules try: import tomli import jinja2 import git + import psutil except ModuleNotFoundError as e: - print(f"Cannot load a required library: {e}") - print("Please make sure the following Python3 modules are installed: tomli jinja2 git") + print(f"E: Cannot load required library {e}") + print("E: Please make sure the following Python3 modules are installed: tomli jinja2 git psutil") -# Local modules +# Import local modules from scripts/image-build +# They rely on modules from vyos-1x import utils import defaults +import raw_image # argparse converts hyphens to underscores, # so for lookups in the original options hash we have to convert them back @@ -58,7 +69,7 @@ def get_validator(optdict, name): except KeyError: return None -def merge_dicts(source, destination): +def merge_dicts(source, destination, skip_none=False): """ Merge two dictionaries and return a new dict which has the merged key/value pairs. Merging logic is as follows: Sub-dicts are merged. @@ -75,7 +86,7 @@ def merge_dicts(source, destination): tmp[key] = merge_dicts(source[key], tmp[key]) elif isinstance(source[key], list): tmp[key] = source[key] + tmp[key] - else: + elif not skip_none or source[key] is not None: tmp[key] = source[key] return tmp @@ -108,7 +119,10 @@ if __name__ == "__main__": 'devscripts', 'python3-pystache', 'python3-git', - 'qemu-utils' + 'qemu-utils', + 'gdisk', + 'kpartx', + 'dosfstools' ], 'binaries': [] } @@ -117,7 +131,7 @@ if __name__ == "__main__": try: checker = utils.check_system_dependencies(deps) except OSError as e: - print(e) + print(f"E: {e}") sys.exit(1) ## Load the file with default build configuration options @@ -125,11 +139,18 @@ if __name__ == "__main__": with open(defaults.DEFAULTS_FILE, 'rb') as f: build_defaults = tomli.load(f) except Exception as e: - print("Failed to open the defaults file {0}: {1}".format(defaults.DEFAULTS_FILE, e)) + print("E: Failed to open the defaults file {0}: {1}".format(defaults.DEFAULTS_FILE, e)) sys.exit(1) ## Get a list of available build flavors - build_flavors = list(map(lambda f: os.path.splitext(f)[0], os.listdir(defaults.BUILD_FLAVORS_DIR))) + flavor_dir_env = os.getenv("VYOS_BUILD_FLAVORS_DIR") + if flavor_dir_env: + flavor_dir = flavor_dir_env + else: + flavor_dir = defaults.BUILD_FLAVORS_DIR + + print(f"I: using build flavors directory {flavor_dir}") + build_flavors = list(map(lambda f: os.path.splitext(f)[0], os.listdir(flavor_dir))) ## Set up the option parser ## XXX: It uses values from the default configuration file for its option defaults, @@ -138,14 +159,13 @@ if __name__ == "__main__": # Options dict format: # '$option_name_without_leading_dashes': { ('$help_string', $default_value_generator_thunk, $value_checker_thunk) } options = { - 'architecture': ('Image target architecture (amd64 or arm64)', - lambda: build_defaults['architecture'], lambda x: x in ['amd64', 'arm64']), + 'architecture': ('Image target architecture (amd64 or arm64)', None, lambda x: x in ['amd64', 'arm64', None]), 'build-by': ('Builder identifier (e.g. jrandomhacker@example.net)', get_default_build_by, None), - 'debian-mirror': ('Debian repository mirror', lambda: build_defaults['debian_mirror'], None), - 'debian-security-mirror': ('Debian security updates mirror', lambda: build_defaults['debian_security_mirror'], None), - 'pbuilder-debian-mirror': ('Debian repository mirror for pbuilder env bootstrap', lambda: build_defaults['debian_mirror'], None), - 'vyos-mirror': ('VyOS package mirror', lambda: build_defaults["vyos_mirror"], None), - 'build-type': ('Build type, release or development', lambda: 'development', lambda x: x in ['release', 'development']), + 'debian-mirror': ('Debian repository mirror', None, None), + 'debian-security-mirror': ('Debian security updates mirror', None, None), + 'pbuilder-debian-mirror': ('Debian repository mirror for pbuilder env bootstrap', None, None), + 'vyos-mirror': ('VyOS package mirror', None, None), + 'build-type': ('Build type, release or development', None, lambda x: x in ['release', 'development']), 'version': ('Version number (release builds only)', None, None), 'build-comment': ('Optional build comment', lambda: '', None) } @@ -159,9 +179,8 @@ if __name__ == "__main__": else: parser.add_argument('--' + k, type=str, help=help_string, default=default_value_thunk()) - # The debug option is a bit special since it different type is different + # Debug options parser.add_argument('--debug', help='Enable debug output', action='store_true') - parser.add_argument('--dry-run', help='Check build configuration and exit', action='store_true') # Custom APT entry and APT key options can be used multiple times @@ -169,6 +188,10 @@ if __name__ == "__main__": parser.add_argument('--custom-apt-key', help="Custom APT key file", action='append', default=[]) parser.add_argument('--custom-package', help="Custom package to install from repositories", action='append', default=[]) + # Options relevant for non-ISO format flavors + parser.add_argument('--reuse-iso', help='Use an existing ISO file to build additional image formats', type=str, action='store', default=None) + parser.add_argument('--disk-size', help='Disk size for non-ISO image formats', type=int, action='store', default=10) + # Build flavor is a positional argument parser.add_argument('build_flavor', help='Build flavor', nargs='?', action='store') @@ -182,7 +205,7 @@ if __name__ == "__main__": func = get_validator(options, k) if func is not None: if not func(v): - print("{v} is not a valid value for --{o} option".format(o=key, v=v)) + print("E: {v} is not a valid value for --{o} option".format(o=key, v=v)) sys.exit(1) if not args["build_flavor"]: @@ -198,7 +221,8 @@ if __name__ == "__main__": flavor_config = {} build_flavor = args["build_flavor"] try: - with open(make_toml_path(defaults.BUILD_FLAVORS_DIR, args["build_flavor"]), 'rb') as f: + toml_flavor_file = make_toml_path(flavor_dir, args["build_flavor"]) + with open(toml_flavor_file, 'rb') as f: flavor_config = tomli.load(f) pre_build_config = merge_dicts(flavor_config, pre_build_config) except FileNotFoundError: @@ -214,44 +238,40 @@ if __name__ == "__main__": # Some fixup for mirror settings. # The idea is: if --debian-mirror is specified but --pbuilder-debian-mirror is not, # use the --debian-mirror value for both lb and pbuilder bootstrap - if (args['debian_mirror'] != build_defaults["debian_mirror"]) and \ - (args['pbuilder_debian_mirror'] == build_defaults["debian_mirror"]): - args['pbuilder_debian_mirror'] = args['debian_mirror'] + if pre_build_config['debian_mirror'] is None or pre_build_config['debian_security_mirror'] is None: + print("E: debian_mirror and debian_security_mirror cannot be empty") + sys.exit(1) + + if pre_build_config['pbuilder_debian_mirror'] is None: + args['pbuilder_debian_mirror'] = pre_build_config['pbuilder_debian_mirror'] = pre_build_config['debian_mirror'] # Version can only be set for release builds, # for dev builds it hardly makes any sense - if args['build_type'] == 'development': + if pre_build_config['build_type'] == 'development': if args['version'] is not None: - print("Version can only be set for release builds") + print("E: Version can only be set for release builds") print("Use --build-type=release option if you want to set version number") sys.exit(1) - # Validate characters in version name - if 'version' in args and args['version'] != None: - allowed = string.ascii_letters + string.digits + '.' + '-' + '+' - if not set(args['version']) <= set(allowed): - print(f'Version contained illegal character(s), allowed: {allowed}') - sys.exit(1) - ## Inject some useful hardcoded options args['build_dir'] = defaults.BUILD_DIR args['pbuilder_config'] = os.path.join(defaults.BUILD_DIR, defaults.PBUILDER_CONFIG) ## Combine the arguments with non-configurable defaults - build_config = merge_dicts(args, build_defaults) + build_config = merge_dicts({}, build_defaults) - ## Load the flavor file and mix-ins - with open(make_toml_path(defaults.BUILD_TYPES_DIR, build_config["build_type"]), 'rb') as f: + ## Load correct mix-ins + with open(make_toml_path(defaults.BUILD_TYPES_DIR, pre_build_config["build_type"]), 'rb') as f: build_type_config = tomli.load(f) build_config = merge_dicts(build_type_config, build_config) - with open(make_toml_path(defaults.BUILD_ARCHES_DIR, build_config["architecture"]), 'rb') as f: + with open(make_toml_path(defaults.BUILD_ARCHES_DIR, pre_build_config["architecture"]), 'rb') as f: build_arch_config = tomli.load(f) build_config = merge_dicts(build_arch_config, build_config) - with open(make_toml_path(defaults.BUILD_FLAVORS_DIR, build_config["build_flavor"]), 'rb') as f: - flavor_config = tomli.load(f) - build_config = merge_dicts(flavor_config, build_config) + ## Override with flavor and then CLI arguments + build_config = merge_dicts(flavor_config, build_config) + build_config = merge_dicts(args, build_config, skip_none=True) ## Rename and merge some fields for simplicity ## E.g. --custom-packages is for the user, but internally @@ -264,7 +284,14 @@ if __name__ == "__main__": if has_nonempty_key(build_config, "architectures"): arch = build_config["architecture"] if arch in build_config["architectures"]: - build_config["packages"] += build_config["architectures"][arch]["packages"] + if has_nonempty_key(build_config["architectures"], "packages"): + build_config["packages"] += build_config["architectures"][arch]["packages"] + + ## Add default boot settings if needed + if "boot_settings" not in build_config: + build_config["boot_settings"] = defaults.boot_settings + else: + build_config["boot_settings"] = merge_dicts(defaults.default_consolede, build_config["boot_settings"]) ## Dump the complete config if the user enabled debug mode if debug: @@ -274,269 +301,301 @@ if __name__ == "__main__": ## Clean up the old build config and set up a fresh copy lb_config_dir = os.path.join(defaults.BUILD_DIR, defaults.LB_CONFIG_DIR) - print(lb_config_dir) shutil.rmtree(lb_config_dir, ignore_errors=True) shutil.copytree("data/live-build-config/", lb_config_dir) os.makedirs(lb_config_dir, exist_ok=True) - ## Create the version file - - # Create a build timestamp - now = datetime.datetime.today() - build_timestamp = now.strftime("%Y%m%d%H%M") - - # FIXME: use aware rather than naive object - build_date = now.strftime("%a %d %b %Y %H:%M UTC") + # Switch to the build directory, this is crucial for the live-build work + # because the efective build config files etc. are there. + # + # All directory paths from this point must be relative to BUILD_DIR, + # not to the vyos-build repository root. + os.chdir(defaults.BUILD_DIR) - # Assign a (hopefully) unique identifier to the build (UUID) - build_uuid = str(uuid.uuid4()) + iso_file = None - # Initialize Git object from our repository - try: - repo = git.Repo('.') - - # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient - build_git = repo.head.object.hexsha[:14] - # If somone played around with the source tree and the build is "dirty", mark it - if repo.is_dirty(): - build_git += "-dirty" - - # Retrieve git branch name or current tag - # Building a tagged release might leave us checking out a git tag that is not the tip of a named branch (detached HEAD) - # Check if the current HEAD is associated with a tag and use its name instead of an unavailable branch name. - git_branch = next((tag.name for tag in repo.tags if tag.commit == repo.head.commit), None) - if git_branch is None: - git_branch = repo.active_branch.name - except Exception as e: - exit(f'Could not retrieve information from git: {e}') - build_git = "" - git_branch = "" + if build_config["reuse_iso"]: + iso_file = build_config["reuse_iso"] + else: + ## Create the version file - # Create the build version string - if build_config['build_type'] == 'development': - try: - if not git_branch: - raise ValueError("git branch could not be determined") + # Create a build timestamp + now = datetime.datetime.today() + build_timestamp = now.strftime("%Y%m%d%H%M") - # Load the branch to version mapping file - with open('data/versions') as f: - version_mapping = json.load(f) + # FIXME: use aware rather than naive object + build_date = now.strftime("%a %d %b %Y %H:%M UTC") - branch_version = version_mapping[git_branch] + # Assign a (hopefully) unique identifier to the build (UUID) + build_uuid = str(uuid.uuid4()) - version = "{0}-rolling-{1}".format(branch_version, build_timestamp) + # Initialize Git object from our repository + try: + repo = git.Repo('.') + + # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient + build_git = repo.head.object.hexsha[:14] + # If somone played around with the source tree and the build is "dirty", mark it + if repo.is_dirty(): + build_git += "-dirty" + + # Retrieve git branch name or current tag + # Building a tagged release might leave us checking out a git tag that is not the tip of a named branch (detached HEAD) + # Check if the current HEAD is associated with a tag and use its name instead of an unavailable branch name. + git_branch = next((tag.name for tag in repo.tags if tag.commit == repo.head.commit), None) + if git_branch is None: + git_branch = repo.active_branch.name except Exception as e: - print("Could not build a version string specific to git branch, falling back to default: {0}".format(str(e))) - version = "999.{0}".format(build_timestamp) - else: - # Release build, use the version from ./configure arguments - version = build_config['version'] + exit(f'Could not retrieve information from git: {e}') + build_git = "" + git_branch = "" + + # Create the build version string + if build_config['build_type'] == 'development': + try: + if not git_branch: + raise ValueError("git branch could not be determined") + + # Load the branch to version mapping file + with open('data/versions') as f: + version_mapping = json.load(f) + + branch_version = version_mapping[git_branch] + + version = "{0}-rolling-{1}".format(branch_version, build_timestamp) + except Exception as e: + print("W: Could not build a version string specific to git branch, falling back to default: {0}".format(str(e))) + version = "999.{0}".format(build_timestamp) + else: + # Release build, use the version from ./configure arguments + version = build_config['version'] - if build_config['build_type'] == 'development': - lts_build = False - else: - lts_build = True - - version_data = { - 'version': version, - 'built_by': build_config['build_by'], - 'built_on': build_date, - 'build_uuid': build_uuid, - 'build_git': build_git, - 'build_branch': git_branch, - 'release_train': build_config['release_train'], - 'lts_build': lts_build, - 'build_comment': build_config['build_comment'], - 'bugtracker_url': build_config['bugtracker_url'], - 'documentation_url': build_config['documentation_url'], - 'project_news_url': build_config['project_news_url'], - } + if build_config['build_type'] == 'development': + lts_build = False + else: + lts_build = True + + version_data = { + 'version': version, + 'built_by': build_config['build_by'], + 'built_on': build_date, + 'build_uuid': build_uuid, + 'build_git': build_git, + 'build_branch': git_branch, + 'release_train': build_config['release_train'], + 'lts_build': lts_build, + 'build_comment': build_config['build_comment'], + 'bugtracker_url': build_config['bugtracker_url'], + 'documentation_url': build_config['documentation_url'], + 'project_news_url': build_config['project_news_url'], + } + + # Multi line strings needs to be un-indented to not have leading + # whitespaces in the resulting file + os_release = f""" + PRETTY_NAME="VyOS {version} ({build_config['release_train']})" + NAME="VyOS" + VERSION_ID="{version}" + VERSION="{version} ({build_config['release_train']})" + VERSION_CODENAME={build_defaults['debian_distribution']} + ID=vyos + BUILD_ID="{build_git}" + HOME_URL="{build_defaults['website_url']}" + SUPPORT_URL="{build_defaults['support_url']}" + BUG_REPORT_URL="{build_defaults['bugtracker_url']}" + DOCUMENTATION_URL="{build_config['documentation_url']}" + """ + + # Reminder: all paths relative to the build dir, not to the repository root + chroot_includes_dir = defaults.CHROOT_INCLUDES_DIR + binary_includes_dir = defaults.BINARY_INCLUDES_DIR + vyos_data_dir = os.path.join(chroot_includes_dir, "usr/share/vyos") + os.makedirs(vyos_data_dir, exist_ok=True) + with open(os.path.join(vyos_data_dir, 'version.json'), 'w') as f: + json.dump(version_data, f) + with open(os.path.join(binary_includes_dir, 'version.json'), 'w') as f: + json.dump(version_data, f) + + # For backwards compatibility with 'add system image' script from older versions + # we need a file in the old format so that script can find out the version of the image + # for upgrade + os.makedirs(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/'), exist_ok=True) + with open(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/version'), 'w') as f: + print("Version: {0}".format(version), file=f) + + # Define variables that influence to welcome message on boot + os.makedirs(os.path.join(chroot_includes_dir, 'usr/lib/'), exist_ok=True) + with open(os.path.join(chroot_includes_dir, 'usr/lib/os-release'), 'w') as f: + print(os_release, file=f) + + ## Clean up earlier build state and artifacts + print("I: Cleaning the build workspace") + os.system("lb clean") + #iter(lambda p: shutil.rmtree(p, ignore_errors=True), + # ['config/binary', 'config/bootstrap', 'config/chroot', 'config/common', 'config/source']) + artifacts = functools.reduce( + lambda x, y: x + y, + map(glob.glob, ['*.iso', '*.raw', '*.img', '*.xz', '*.ova', '*.ovf'])) + iter(os.remove, artifacts) + + ## Create the target ISO file path + iso_file = "vyos-{0}-{1}.iso".format(version_data["version"], build_config["architecture"]) + + ## Create live-build configuration files + + # Add the additional repositories to package lists + print("I: Setting up additional APT entries") + vyos_repo_entry = "deb {vyos_mirror} {vyos_branch} main\n".format(**build_config) + + apt_file = defaults.VYOS_REPO_FILE - # Multi line strings needs to be un-indented to not have leading - # whitespaces in the resulting file - os_release = f""" -PRETTY_NAME="VyOS {version} ({build_config['release_train']})" -NAME="VyOS" -VERSION_ID="{version}" -VERSION="{version} ({build_config['release_train']})" -VERSION_CODENAME={build_defaults['debian_distribution']} -ID=vyos -BUILD_ID="{build_git}" -HOME_URL="{build_defaults['website_url']}" -SUPPORT_URL="{build_defaults['support_url']}" -BUG_REPORT_URL="{build_defaults['bugtracker_url']}" -DOCUMENTATION_URL="{build_config['documentation_url']}" + if debug: + print(f"D: Adding these entries to {apt_file}:") + print("\t", vyos_repo_entry) + + with open(apt_file, 'w') as f: + f.write(vyos_repo_entry) + + # Add custom APT entries + if build_config.get('additional_repositories', False): + build_config['custom_apt_entry'] += build_config['additional_repositories'] + + if build_config.get('custom_apt_entry', False): + custom_apt_file = defaults.CUSTOM_REPO_FILE + entries = "\n".join(build_config['custom_apt_entry']) + if debug: + print("D: Adding custom APT entries:") + print(entries) + with open(custom_apt_file, 'w') as f: + f.write(entries) + f.write("\n") + + # Add custom APT keys + if has_nonempty_key(build_config, 'custom_apt_key'): + key_dir = defaults.ARCHIVES_DIR + for k in build_config['custom_apt_key']: + dst_name = '{0}.key.chroot'.format(os.path.basename(k)) + shutil.copy(k, os.path.join(key_dir, dst_name)) + + # Add custom packages + if has_nonempty_key(build_config, 'packages'): + package_list_file = defaults.PACKAGE_LIST_FILE + packages = "\n".join(build_config['packages']) + with open (package_list_file, 'w') as f: + f.write(packages) + + ## Create includes + if has_nonempty_key(build_config, "includes_chroot"): + for i in build_config["includes_chroot"]: + file_path = os.path.join(chroot_includes_dir, i["path"]) + if debug: + print(f"D: Creating chroot include file: {file_path}") + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, 'w') as f: + f.write(i["data"]) + + ## Create the default config + ## Technically it's just another includes.chroot entry, + ## but it's special enough to warrant making it easier for flavor writers + if has_nonempty_key(build_config, "default_config"): + file_path = os.path.join(chroot_includes_dir, "opt/vyatta/etc/config.boot.default") + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, 'w') as f: + f.write(build_config["default_config"]) + + ## Configure live-build + lb_config_tmpl = jinja2.Template(""" + lb config noauto \ + --apt-indices false \ + --apt-options "--yes -oAPT::Get::allow-downgrades=true" \ + --apt-recommends false \ + --architecture {{architecture}} \ + --archive-areas {{debian_archive_areas}} \ + --backports true \ + --binary-image iso-hybrid \ + --bootappend-live "boot=live components hostname=vyos username=live nopersistence noautologin nonetworking union=overlay console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \ + --bootappend-live-failsafe "live components memtest noapic noapm nodma nomce nolapic nomodeset nosmp nosplash vga=normal console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \ + --bootloaders {{bootloaders}} \ + --checksums 'sha256 md5' \ + --chroot-squashfs-compression-type "{{squashfs_compression_type}}" \ + --debian-installer none \ + --debootstrap-options "--variant=minbase --exclude=isc-dhcp-client,isc-dhcp-common,ifupdown --include=apt-utils,ca-certificates,gnupg2" \ + --distribution {{debian_distribution}} \ + --firmware-binary false \ + --firmware-chroot false \ + --iso-application "VyOS" \ + --iso-publisher "{{build_by}}" \ + --iso-volume "VyOS" \ + --linux-flavours {{kernel_flavor}} \ + --linux-packages linux-image-{{kernel_version}} \ + --mirror-binary {{debian_mirror}} \ + --mirror-binary-security {{debian_security_mirror}} \ + --mirror-bootstrap {{debian_mirror}} \ + --mirror-chroot {{debian_mirror}} \ + --mirror-chroot-security {{debian_security_mirror}} \ + --security true \ + --updates true + "${@}" + """) + + lb_config_command = lb_config_tmpl.render(build_config) + + ## Pin release for VyOS packages + apt_pin = f"""Package: * + Pin: release n={build_config['release_train']} + Pin-Priority: 600 """ - chroot_includes_dir = os.path.join(defaults.BUILD_DIR, defaults.CHROOT_INCLUDES_DIR) - binary_includes_dir = os.path.join(defaults.BUILD_DIR, defaults.BINARY_INCLUDES_DIR) - vyos_data_dir = os.path.join(chroot_includes_dir, "usr/share/vyos") - os.makedirs(vyos_data_dir, exist_ok=True) - with open(os.path.join(vyos_data_dir, 'version.json'), 'w') as f: - json.dump(version_data, f) - with open(os.path.join(binary_includes_dir, 'version.json'), 'w') as f: - json.dump(version_data, f) - - # For backwards compatibility with 'add system image' script from older versions - # we need a file in the old format so that script can find out the version of the image - # for upgrade - os.makedirs(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/'), exist_ok=True) - with open(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/version'), 'w') as f: - print("Version: {0}".format(version), file=f) - + with open(defaults.VYOS_PIN_FILE, 'w') as f: + f.write(apt_pin) - # Define variables that influence to welcome message on boot - os.makedirs(os.path.join(chroot_includes_dir, 'usr/lib/'), exist_ok=True) - with open(os.path.join(chroot_includes_dir, 'usr/lib/os-release'), 'w') as f: - print(os_release, file=f) + print("I: Configuring live-build") - - ## Clean up earlier build state and artifacts - print("I: Cleaning the build workspace") - os.system("lb clean") - #iter(lambda p: shutil.rmtree(p, ignore_errors=True), - # ['config/binary', 'config/bootstrap', 'config/chroot', 'config/common', 'config/source']) - artifacts = functools.reduce( - lambda x, y: x + y, - map(glob.glob, ['*.iso', '*.raw', '*.img', '*.xz', '*.ova', '*.ovf'])) - iter(os.remove, artifacts) - - ## Create live-build configuration files - - # Add the additional repositories to package lists - print("I: Setting up additional APT entries") - vyos_repo_entry = "deb {vyos_mirror} {vyos_branch} main\n".format(**build_config) - - apt_file = defaults.VYOS_REPO_FILE - - if debug: - print(f"D: Adding these entries to {apt_file}:") - print("\t", vyos_repo_entry) - - with open(apt_file, 'w') as f: - f.write(vyos_repo_entry) - - # Add custom APT entries - if build_config.get('additional_repositories', False): - build_config['custom_apt_entry'] += build_config['additional_repositories'] - - if build_config.get('custom_apt_entry', False): - custom_apt_file = defaults.CUSTOM_REPO_FILE - entries = "\n".join(build_config['custom_apt_entry']) if debug: - print("D: Adding custom APT entries:") - print(entries) - with open(custom_apt_file, 'w') as f: - f.write(entries) - f.write("\n") - - # Add custom APT keys - if has_nonempty_key(build_config, 'custom_apt_key'): - key_dir = defaults.ARCHIVES_DIR - for k in build_config['custom_apt_key']: - dst_name = '{0}.key.chroot'.format(os.path.basename(k)) - shutil.copy(k, os.path.join(key_dir, dst_name)) - - # Add custom packages - if has_nonempty_key(build_config, 'packages'): - package_list_file = defaults.PACKAGE_LIST_FILE - packages = "\n".join(build_config['packages']) - with open (package_list_file, 'w') as f: - f.write(packages) - - ## Create includes - if has_nonempty_key(build_config, "includes_chroot"): - for i in build_config["includes_chroot"]: - file_path = os.path.join(chroot_includes_dir, i["path"]) - os.makedirs(os.path.dirname(file_path), exist_ok=True) - with open(file_path, 'w') as f: - f.write(i["data"]) - - ## Create the default config - ## Technically it's just another includes.chroot entry, - ## but it's special enough to warrant making it easier for flavor writers - if has_nonempty_key(build_config, "default_config"): - file_path = os.path.join(chroot_includes_dir, "opt/vyatta/etc/config.boot.default") - os.makedirs(os.path.dirname(file_path), exist_ok=True) - with open(file_path, 'w') as f: - f.write(build_config["default_config"]) - - ## Configure live-build - lb_config_tmpl = jinja2.Template(""" - lb config noauto \ - --apt-indices false \ - --apt-options "--yes -oAPT::Get::allow-downgrades=true" \ - --apt-recommends false \ - --architecture {{architecture}} \ - --archive-areas {{debian_archive_areas}} \ - --backports true \ - --binary-image iso-hybrid \ - --bootappend-live "boot=live components hostname=vyos username=live nopersistence noautologin nonetworking union=overlay console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \ - --bootappend-live-failsafe "live components memtest noapic noapm nodma nomce nolapic nomodeset nosmp nosplash vga=normal console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \ - --bootloaders {{bootloaders}} \ - --checksums 'sha256 md5' \ - --chroot-squashfs-compression-type "{{squashfs_compression_type}}" \ - --debian-installer none \ - --debootstrap-options "--variant=minbase --exclude=isc-dhcp-client,isc-dhcp-common,ifupdown --include=apt-utils,ca-certificates,gnupg2" \ - --distribution {{debian_distribution}} \ - --firmware-binary false \ - --firmware-chroot false \ - --iso-application "VyOS" \ - --iso-publisher "{{build_by}}" \ - --iso-volume "VyOS" \ - --linux-flavours {{kernel_flavor}} \ - --linux-packages linux-image-{{kernel_version}} \ - --mirror-binary {{debian_mirror}} \ - --mirror-binary-security {{debian_security_mirror}} \ - --mirror-bootstrap {{debian_mirror}} \ - --mirror-chroot {{debian_mirror}} \ - --mirror-chroot-security {{debian_security_mirror}} \ - --security true \ - --updates true - "${@}" - """) - - lb_config_command = lb_config_tmpl.render(build_config) - - ## Pin release for VyOS packages - apt_pin = f"""Package: * -Pin: release n={build_config['release_train']} -Pin-Priority: 600 -""" - - with open(defaults.VYOS_PIN_FILE, 'w') as f: - f.write(apt_pin) - - print("I: Configuring live-build") + print("D: live-build configuration command") + print(lb_config_command) - if debug: - print("D: live-build configuration command") - print(lb_config_command) - - result = os.system(lb_config_command) - if result > 0: - print("E: live-build config failed") - sys.exit(1) + result = os.system(lb_config_command) + if result > 0: + print("E: live-build config failed") + sys.exit(1) - ## In dry-run mode, exit at this point - if build_config["dry_run"]: - print("I: dry-run, not starting image build") - sys.exit(0) + ## In dry-run mode, exit at this point + if build_config["dry_run"]: + print("I: dry-run, not starting image build") + sys.exit(0) - ## Add local packages - local_packages = glob.glob('../packages/*.deb') - if local_packages: - for f in local_packages: - shutil.copy(f, os.path.join(defaults.LOCAL_PACKAGES_PATH, os.path.basename(f))) + ## Add local packages + local_packages = glob.glob('../packages/*.deb') + if local_packages: + for f in local_packages: + shutil.copy(f, os.path.join(defaults.LOCAL_PACKAGES_PATH, os.path.basename(f))) - ## Build the image - print("I: Starting image build") - if debug: - print("D: It's not like I'm building this specially for you or anything!") - res = os.system("lb build 2>&1") - if res > 0: - sys.exit(res) - - # Copy the image - shutil.copy("live-image-{0}.hybrid.iso".format(build_config["architecture"]), - "vyos-{0}-{1}.iso".format(version_data["version"], build_config["architecture"])) + ## Build the image + print("I: Starting image build") + if debug: + print("D: It's not like I'm building this specially for you or anything!") + res = os.system("lb build 2>&1") + if res > 0: + sys.exit(res) + + # Copy the image + shutil.copy("live-image-{0}.hybrid.iso".format(build_config["architecture"]), iso_file) + + # Build additional flavors from the ISO, + # if the flavor calls for them + if build_config["image_format"] != ["iso"]: + raw_image = raw_image.create_raw_image(build_config, iso_file, "tmp/") + + other_formats = filter(lambda x: x not in ["iso", "raw"], build_config["image_format"]) + for f in other_formats: + target = f"{os.path.splitext(raw_image)[0]}.{f}" + print(f"I: building {f} file {target}") + os.system(f"qemu-img convert -f raw -O {f} {raw_image} {target}") + + # Some flavors require special procedures that aren't covered by qemu-img + # (most notable, the VMware OVA that requires a custom tool to make and sign the image). + # Such procedures are executed as post-build hooks. + if has_nonempty_key(build_config, "post_build_hook"): + hook_path = build_config["post_build_hook"] + os.system(f"{hook_path} {raw_image}") diff --git a/scripts/image-build/defaults.py b/scripts/image-build/defaults.py index 1d7141ea..9fd5eeed 100644 --- a/scripts/image-build/defaults.py +++ b/scripts/image-build/defaults.py @@ -18,6 +18,15 @@ import os +# Default boot settings +boot_settings: dict[str, str] = { + 'timeout': '5', + 'console_type': 'tty', + 'console_num': '0', + 'console_speed': '115200', + 'bootmode': 'normal' +} + # Relative to the repository directory BUILD_DIR = 'build' diff --git a/scripts/image-build/raw_image.py b/scripts/image-build/raw_image.py new file mode 100644 index 00000000..67039a24 --- /dev/null +++ b/scripts/image-build/raw_image.py @@ -0,0 +1,215 @@ +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# File: raw_image.py +# Purpose: Helper functions for building raw images. + +import os +import sys +import shutil +import traceback + +import vyos.utils.process + +SQUASHFS_FILE = 'live/filesystem.squashfs' +VERSION_FILE = 'version.json' + + +def cmd(command): + res = vyos.utils.process.call(command, shell=True) + if res > 0: + raise OSError(f"Command '{command}' failed") + +def mkdir(path): + os.makedirs(path, exist_ok=True) + + +class BuildContext: + def __init__(self, iso_path, work_dir, debug=False): + self.work_dir = work_dir + self.iso_path = iso_path + self.debug = debug + self.loop_device = None + + def __enter__(self): + print(f"I: Setting up a raw image build directory in {self.work_dir}") + + self.iso_dir = os.path.join(self.work_dir, "iso") + self.squash_dir = os.path.join(self.work_dir, "squash") + self.raw_dir = os.path.join(self.work_dir, "raw") + self.efi_dir = os.path.join(self.work_dir, "efi") + + # Create mount point directories + mkdir(self.iso_dir) + mkdir(self.squash_dir) + mkdir(self.raw_dir) + mkdir(self.efi_dir) + + # Mount the ISO image + cmd(f"""mount -t iso9660 -o ro,loop {self.iso_path} {self.iso_dir}""") + + # Mount the SquashFS image + cmd(f"""mount -t squashfs -o ro,loop {self.iso_dir}/{SQUASHFS_FILE} {self.squash_dir}""") + + return self + + def __exit__(self, exc_type, exc_value, exc_tb): + print(f"I: Tearing down the raw image build environment in {self.work_dir}") + cmd(f"""umount {self.squash_dir}/dev/""") + cmd(f"""umount {self.squash_dir}/proc/""") + cmd(f"""umount {self.squash_dir}/sys/""") + + cmd(f"umount {self.squash_dir}/boot/efi") + cmd(f"umount {self.squash_dir}/boot") + + cmd(f"""umount {self.squash_dir}""") + cmd(f"""umount {self.iso_dir}""") + cmd(f"""umount {self.raw_dir}""") + cmd(f"""umount {self.efi_dir}""") + + if self.loop_device: + cmd(f"""losetup -d {self.loop_device}""") + +def create_disk(path, size): + cmd(f"""qemu-img create -f raw "{path}" {size}G""") + +def read_version_data(iso_dir): + from json import load + with open(os.path.join(iso_dir, VERSION_FILE), 'r') as f: + data = load(f) + return data + +def setup_loop_device(con, raw_file): + from subprocess import Popen, PIPE, STDOUT + from re import match + command = f'losetup --show -f {raw_file}' + p = Popen(command, stderr=PIPE, stdout=PIPE, stdin=PIPE, shell=True) + (stdout, stderr) = p.communicate() + + if p.returncode > 0: + raise OSError(f"Could not set up a loop device: {stderr.decode()}") + + con.loop_device = stdout.decode().strip() + if con.debug: + print(f"I: Using loop device {con.loop_device}") + +def mount_image(con): + import vyos.system.disk + + from subprocess import Popen, PIPE, STDOUT + from re import match + + vyos.system.disk.filesystem_create(con.disk_details.partition['efi'], 'efi') + vyos.system.disk.filesystem_create(con.disk_details.partition['root'], 'ext4') + + cmd(f"mount -t ext4 {con.disk_details.partition['root']} {con.raw_dir}") + cmd(f"mount -t vfat {con.disk_details.partition['efi']} {con.efi_dir}") + +def install_image(con, version): + from glob import glob + + vyos_dir = os.path.join(con.raw_dir, f'boot/{version}/') + mkdir(vyos_dir) + mkdir(os.path.join(vyos_dir, 'work/work')) + mkdir(os.path.join(vyos_dir, 'rw')) + + shutil.copy(f"{con.iso_dir}/{SQUASHFS_FILE}", f"{vyos_dir}/{version}.squashfs") + + boot_files = glob(f'{con.squash_dir}/boot/*') + boot_files = [f for f in boot_files if os.path.isfile(f)] + + for f in boot_files: + print(f"I: Copying file {f}") + shutil.copy(f, vyos_dir) + + with open(f"{con.raw_dir}/persistence.conf", 'w') as f: + f.write("/ union\n") + +def setup_grub_configuration(build_config, root_dir) -> None: + """Install GRUB configurations + + Args: + root_dir (str): a path to the root of target filesystem + """ + from vyos.template import render + from vyos.system import grub + + print('I: Installing GRUB configuration files') + grub_cfg_main = f'{root_dir}/{grub.GRUB_DIR_MAIN}/grub.cfg' + grub_cfg_vars = f'{root_dir}/{grub.CFG_VYOS_VARS}' + grub_cfg_modules = f'{root_dir}/{grub.CFG_VYOS_MODULES}' + grub_cfg_menu = f'{root_dir}/{grub.CFG_VYOS_MENU}' + grub_cfg_options = f'{root_dir}/{grub.CFG_VYOS_OPTIONS}' + + # create new files + render(grub_cfg_main, grub.TMPL_GRUB_MAIN, {}) + grub.common_write(root_dir) + grub.vars_write(grub_cfg_vars, build_config["boot_settings"]) + grub.modules_write(grub_cfg_modules, []) + grub.write_cfg_ver(1, root_dir) + render(grub_cfg_menu, grub.TMPL_GRUB_MENU, {}) + render(grub_cfg_options, grub.TMPL_GRUB_OPTS, {}) + +def install_grub(con, version): + from re import match + from vyos.system import disk, grub + + # Mount the required virtual filesystems + os.makedirs(f"{con.raw_dir}/boot/efi", exist_ok=True) + cmd(f"mount --bind /dev {con.squash_dir}/dev") + cmd(f"mount --bind /proc {con.squash_dir}/proc") + cmd(f"mount --bind /sys {con.squash_dir}/sys") + + cmd(f"mount --bind {con.raw_dir}/boot {con.squash_dir}/boot") + cmd(f"mount --bind {con.efi_dir} {con.squash_dir}/boot/efi") + + DIR_DST_ROOT = con.raw_dir + + setup_grub_configuration(con.build_config, DIR_DST_ROOT) + # add information about version + grub.create_structure(DIR_DST_ROOT) + grub.version_add(version, DIR_DST_ROOT) + grub.set_default(version, DIR_DST_ROOT) + grub.set_console_type(con.build_config["boot_settings"]["console_type"], DIR_DST_ROOT) + + print('I: Installing GRUB to the disk image') + grub.install(con.loop_device, f'/boot/', f'/boot/efi', chroot=con.squash_dir) + + # sort inodes (to make GRUB read config files in alphabetical order) + grub.sort_inodes(f'{DIR_DST_ROOT}/{grub.GRUB_DIR_VYOS}') + grub.sort_inodes(f'{DIR_DST_ROOT}/{grub.GRUB_DIR_VYOS_VERS}') + + +def create_raw_image(build_config, iso_file, work_dir): + from vyos.system.disk import parttable_create + + if not os.path.exists(iso_file): + print(f"E: ISO file {iso_file} does not exist in the build directory") + sys.exit(1) + + with BuildContext(iso_file, work_dir, debug=True) as con: + con.build_config = build_config + version_data = read_version_data(con.iso_dir) + version = version_data['version'] + raw_file = f"vyos-{version}.raw" + print(f"I: Building raw file {raw_file}") + create_disk(raw_file, build_config["disk_size"]) + setup_loop_device(con, raw_file) + disk_details = parttable_create(con.loop_device, (int(build_config["disk_size"]) - 1) * 1024 * 1024) + con.disk_details = disk_details + mount_image(con) + install_image(con, version) + install_grub(con, version) + + return raw_file -- cgit v1.2.3 From 7c88ff5800d63b40187e1368e08c95b8d53a74a4 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 10 Apr 2024 10:26:24 -0500 Subject: Account for the working dir below the root in git invocation Signed-off-by: Daniil Baturin (cherry picked from commit 7fa66c77f19694e2cfd275785344f53518b7fb2f) --- scripts/image-build/build-vyos-image | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/image-build/build-vyos-image b/scripts/image-build/build-vyos-image index aa397843..229bc2bc 100755 --- a/scripts/image-build/build-vyos-image +++ b/scripts/image-build/build-vyos-image @@ -331,7 +331,7 @@ if __name__ == "__main__": # Initialize Git object from our repository try: - repo = git.Repo('.') + repo = git.Repo('.', search_parent_directories=True) # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient build_git = repo.head.object.hexsha[:14] @@ -346,7 +346,7 @@ if __name__ == "__main__": if git_branch is None: git_branch = repo.active_branch.name except Exception as e: - exit(f'Could not retrieve information from git: {e}') + exit(f'Could not retrieve information from git: {repr(e)}') build_git = "" git_branch = "" -- cgit v1.2.3 From f257039bdfdd7258e964ee49ee296055c716e98c Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Sat, 13 Apr 2024 19:49:59 +0000 Subject: build: T3664: filter out non-TOML files from the build flavor dir to avoid mistakenly listing auixilliary files like README as flavors (cherry picked from commit c07268987cb4f55a35240972e672e7d809f7ef4a) --- scripts/image-build/build-vyos-image | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/image-build/build-vyos-image b/scripts/image-build/build-vyos-image index 229bc2bc..f213767c 100755 --- a/scripts/image-build/build-vyos-image +++ b/scripts/image-build/build-vyos-image @@ -150,7 +150,7 @@ if __name__ == "__main__": flavor_dir = defaults.BUILD_FLAVORS_DIR print(f"I: using build flavors directory {flavor_dir}") - build_flavors = list(map(lambda f: os.path.splitext(f)[0], os.listdir(flavor_dir))) + build_flavors = [f[0] for f in map(os.path.splitext, os.listdir(flavor_dir)) if (f[1] == ".toml")] ## Set up the option parser ## XXX: It uses values from the default configuration file for its option defaults, @@ -346,9 +346,9 @@ if __name__ == "__main__": if git_branch is None: git_branch = repo.active_branch.name except Exception as e: - exit(f'Could not retrieve information from git: {repr(e)}') - build_git = "" - git_branch = "" + print(f'W: Could not retrieve information from git: {repr(e)}') + build_git = "" + git_branch = "" # Create the build version string if build_config['build_type'] == 'development': -- cgit v1.2.3 From 0d3031ab9942005c74e61c5a742fda8e2796ea40 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Sat, 13 Apr 2024 20:06:53 +0000 Subject: build: T3664: add a sanity check for image_format and normalize it to a list if only one format is specified (cherry picked from commit 72c7c7ff99b86bb3c65b44f69e05cf40cb8367c0) --- scripts/image-build/build-vyos-image | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'scripts') diff --git a/scripts/image-build/build-vyos-image b/scripts/image-build/build-vyos-image index f213767c..c852694d 100755 --- a/scripts/image-build/build-vyos-image +++ b/scripts/image-build/build-vyos-image @@ -287,12 +287,23 @@ if __name__ == "__main__": if has_nonempty_key(build_config["architectures"], "packages"): build_config["packages"] += build_config["architectures"][arch]["packages"] + ## Check if image format is specified, + ## else we have no idea what we are actually supposed to build. + if not has_nonempty_key(build_config, "image_format"): + print("E: image format is not specified in the build flavor file") + sys.exit(1) + ## Add default boot settings if needed if "boot_settings" not in build_config: build_config["boot_settings"] = defaults.boot_settings else: build_config["boot_settings"] = merge_dicts(defaults.default_consolede, build_config["boot_settings"]) + ## Convert the image_format field to a single-item list if it's a scalar + ## (like `image_format = "iso"`) + if type(build_config["image_format"]) != list: + build_config["image_format"] = [ build_config["image_format"] ] + ## Dump the complete config if the user enabled debug mode if debug: import json -- cgit v1.2.3 From 535908b02ebc6d3f74d68823cfd930c448bcb368 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 10 Apr 2024 14:48:26 -0500 Subject: fix path to versions file Signed-off-by: Daniil Baturin (cherry picked from commit ed7bcbd0dbe496cee725136a86ac19f4370d482f) --- scripts/image-build/build-vyos-image | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/image-build/build-vyos-image b/scripts/image-build/build-vyos-image index c852694d..68b1f743 100755 --- a/scripts/image-build/build-vyos-image +++ b/scripts/image-build/build-vyos-image @@ -368,7 +368,7 @@ if __name__ == "__main__": raise ValueError("git branch could not be determined") # Load the branch to version mapping file - with open('data/versions') as f: + with open('../data/versions') as f: version_mapping = json.load(f) branch_version = version_mapping[git_branch] -- cgit v1.2.3 From 43155713c100f5679ddfe1e4196d950b5ec185ef Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Mon, 22 Apr 2024 15:43:03 +0000 Subject: build: T3664: adjust the vyos-1x submodule path in scripts (cherry picked from commit ec42af75e0ab468e062add3852c80254d153c021) --- Makefile | 2 +- scripts/image-build/build-vyos-image | 67 ++++++++++++++++++++---------------- 2 files changed, 39 insertions(+), 30 deletions(-) (limited to 'scripts') diff --git a/Makefile b/Makefile index e0a22888..62587ced 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ all: @echo "The most common target is 'iso'" %: - VYOS_TEMPLATE_DIR=`pwd`/vyos-1x/data/templates/ ./build-vyos-image $* + VYOS_TEMPLATE_DIR=`pwd`/packages/vyos-1x/data/templates/ ./build-vyos-image $* .PHONY: checkiso .ONESHELL: diff --git a/scripts/image-build/build-vyos-image b/scripts/image-build/build-vyos-image index 68b1f743..f9544054 100755 --- a/scripts/image-build/build-vyos-image +++ b/scripts/image-build/build-vyos-image @@ -30,14 +30,7 @@ import platform import argparse import datetime import functools - -# Add the vyos-1x submodule directory to the Python path -# so that we can import modules from it. -VYOS1X_DIR = os.path.join(os.getcwd(), 'vyos-1x/python') -if not os.path.exists(VYOS1X_DIR): - print("E: vyos-1x subdirectory does not exist, did you initialize submodules?") -else: - sys.path.append(VYOS1X_DIR) +import string # Import third-party modules try: @@ -49,6 +42,36 @@ except ModuleNotFoundError as e: print(f"E: Cannot load required library {e}") print("E: Please make sure the following Python3 modules are installed: tomli jinja2 git psutil") +# Initialize Git object from our repository +try: + repo = git.Repo('.', search_parent_directories=True) + repo.git.submodule('update', '--init') + + # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient + build_git = repo.head.object.hexsha[:14] + # If somone played around with the source tree and the build is "dirty", mark it + if repo.is_dirty(): + build_git += "-dirty" + + # Retrieve git branch name or current tag + # Building a tagged release might leave us checking out a git tag that is not the tip of a named branch (detached HEAD) + # Check if the current HEAD is associated with a tag and use its name instead of an unavailable branch name. + git_branch = next((tag.name for tag in repo.tags if tag.commit == repo.head.commit), None) + if git_branch is None: + git_branch = repo.active_branch.name +except Exception as e: + print(f'W: Could not retrieve information from git: {repr(e)}') + build_git = "" + git_branch = "" + +# Add the vyos-1x submodule directory to the Python path +# so that we can import modules from it. +VYOS1X_DIR = os.path.join(os.getcwd(), 'packages/vyos-1x/python') +if not os.path.exists(VYOS1X_DIR): + print("E: vyos-1x subdirectory does not exist, did git submodules fail to initialize?") +else: + sys.path.append(VYOS1X_DIR) + # Import local modules from scripts/image-build # They rely on modules from vyos-1x import utils @@ -253,6 +276,13 @@ if __name__ == "__main__": print("Use --build-type=release option if you want to set version number") sys.exit(1) + # Validate characters in version name + if 'version' in args and args['version'] != None: + allowed = string.ascii_letters + string.digits + '.' + '-' + '+' + if not set(args['version']) <= set(allowed): + print(f'Version contained illegal character(s), allowed: {allowed}') + sys.exit(1) + ## Inject some useful hardcoded options args['build_dir'] = defaults.BUILD_DIR args['pbuilder_config'] = os.path.join(defaults.BUILD_DIR, defaults.PBUILDER_CONFIG) @@ -340,27 +370,6 @@ if __name__ == "__main__": # Assign a (hopefully) unique identifier to the build (UUID) build_uuid = str(uuid.uuid4()) - # Initialize Git object from our repository - try: - repo = git.Repo('.', search_parent_directories=True) - - # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient - build_git = repo.head.object.hexsha[:14] - # If somone played around with the source tree and the build is "dirty", mark it - if repo.is_dirty(): - build_git += "-dirty" - - # Retrieve git branch name or current tag - # Building a tagged release might leave us checking out a git tag that is not the tip of a named branch (detached HEAD) - # Check if the current HEAD is associated with a tag and use its name instead of an unavailable branch name. - git_branch = next((tag.name for tag in repo.tags if tag.commit == repo.head.commit), None) - if git_branch is None: - git_branch = repo.active_branch.name - except Exception as e: - print(f'W: Could not retrieve information from git: {repr(e)}') - build_git = "" - git_branch = "" - # Create the build version string if build_config['build_type'] == 'development': try: -- cgit v1.2.3 From 78819c07645a41fc5328b6636755d887282ad109 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Thu, 25 Apr 2024 13:17:52 +0000 Subject: build: T3664: typo fixes and small refactoring (cherry picked from commit f6b0809f47691a8c21718c4256d99b40c73c1564) --- scripts/image-build/build-vyos-image | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) (limited to 'scripts') diff --git a/scripts/image-build/build-vyos-image b/scripts/image-build/build-vyos-image index f9544054..2295df1f 100755 --- a/scripts/image-build/build-vyos-image +++ b/scripts/image-build/build-vyos-image @@ -68,7 +68,7 @@ except Exception as e: # so that we can import modules from it. VYOS1X_DIR = os.path.join(os.getcwd(), 'packages/vyos-1x/python') if not os.path.exists(VYOS1X_DIR): - print("E: vyos-1x subdirectory does not exist, did git submodules fail to initialize?") + print("E: packages/vyos-1x subdirectory does not exist, did git submodules fail to initialize?") else: sys.path.append(VYOS1X_DIR) @@ -327,7 +327,7 @@ if __name__ == "__main__": if "boot_settings" not in build_config: build_config["boot_settings"] = defaults.boot_settings else: - build_config["boot_settings"] = merge_dicts(defaults.default_consolede, build_config["boot_settings"]) + build_config["boot_settings"] = merge_dicts(defaults.boot_settings, build_config["boot_settings"]) ## Convert the image_format field to a single-item list if it's a scalar ## (like `image_format = "iso"`) @@ -607,15 +607,17 @@ if __name__ == "__main__": if build_config["image_format"] != ["iso"]: raw_image = raw_image.create_raw_image(build_config, iso_file, "tmp/") - other_formats = filter(lambda x: x not in ["iso", "raw"], build_config["image_format"]) - for f in other_formats: - target = f"{os.path.splitext(raw_image)[0]}.{f}" - print(f"I: building {f} file {target}") - os.system(f"qemu-img convert -f raw -O {f} {raw_image} {target}") - - # Some flavors require special procedures that aren't covered by qemu-img - # (most notable, the VMware OVA that requires a custom tool to make and sign the image). - # Such procedures are executed as post-build hooks. if has_nonempty_key(build_config, "post_build_hook"): + # Some flavors require special procedures that aren't covered by qemu-img + # (most notably, the VMware OVA that requires a custom tool to make and sign the image). + # For those cases, we support running a post-build hook on the raw image. + # The image_format field should be 'raw' if a post-build hook is used. hook_path = build_config["post_build_hook"] os.system(f"{hook_path} {raw_image}") + else: + # Most other formats, thankfully, can be produced with just `qemu-img convert` + other_formats = filter(lambda x: x not in ["iso", "raw"], build_config["image_format"]) + for f in other_formats: + target = f"{os.path.splitext(raw_image)[0]}.{f}" + print(f"I: Building {f} file {target}") + os.system(f"qemu-img convert -f raw -O {f} {raw_image} {target}") -- cgit v1.2.3 From d3696d878f9ed6eed46723550dff8c1b914b62da Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Mon, 29 Apr 2024 14:33:25 +0000 Subject: build: T3664: fail the build on external command errors (cherry picked from commit 7dfd9232da787a7befbc4338d4eb21fee4325174) --- scripts/image-build/build-vyos-image | 17 +++++++---------- scripts/image-build/raw_image.py | 6 +----- scripts/image-build/utils.py | 6 ++++++ 3 files changed, 14 insertions(+), 15 deletions(-) (limited to 'scripts') diff --git a/scripts/image-build/build-vyos-image b/scripts/image-build/build-vyos-image index 2295df1f..c6e76208 100755 --- a/scripts/image-build/build-vyos-image +++ b/scripts/image-build/build-vyos-image @@ -78,6 +78,8 @@ import utils import defaults import raw_image +from utils import cmd + # argparse converts hyphens to underscores, # so for lookups in the original options hash we have to convert them back def field_to_option(s): @@ -450,7 +452,7 @@ if __name__ == "__main__": ## Clean up earlier build state and artifacts print("I: Cleaning the build workspace") - os.system("lb clean") + cmd("lb clean") #iter(lambda p: shutil.rmtree(p, ignore_errors=True), # ['config/binary', 'config/bootstrap', 'config/chroot', 'config/common', 'config/source']) artifacts = functools.reduce( @@ -575,10 +577,7 @@ if __name__ == "__main__": print("D: live-build configuration command") print(lb_config_command) - result = os.system(lb_config_command) - if result > 0: - print("E: live-build config failed") - sys.exit(1) + cmd(lb_config_command) ## In dry-run mode, exit at this point if build_config["dry_run"]: @@ -595,9 +594,7 @@ if __name__ == "__main__": print("I: Starting image build") if debug: print("D: It's not like I'm building this specially for you or anything!") - res = os.system("lb build 2>&1") - if res > 0: - sys.exit(res) + cmd("lb build 2>&1") # Copy the image shutil.copy("live-image-{0}.hybrid.iso".format(build_config["architecture"]), iso_file) @@ -613,11 +610,11 @@ if __name__ == "__main__": # For those cases, we support running a post-build hook on the raw image. # The image_format field should be 'raw' if a post-build hook is used. hook_path = build_config["post_build_hook"] - os.system(f"{hook_path} {raw_image}") + cmd(f"{hook_path} {raw_image}") else: # Most other formats, thankfully, can be produced with just `qemu-img convert` other_formats = filter(lambda x: x not in ["iso", "raw"], build_config["image_format"]) for f in other_formats: target = f"{os.path.splitext(raw_image)[0]}.{f}" print(f"I: Building {f} file {target}") - os.system(f"qemu-img convert -f raw -O {f} {raw_image} {target}") + cmd(f"qemu-img convert -f raw -O {f} {raw_image} {target}") diff --git a/scripts/image-build/raw_image.py b/scripts/image-build/raw_image.py index 67039a24..ae061990 100644 --- a/scripts/image-build/raw_image.py +++ b/scripts/image-build/raw_image.py @@ -25,11 +25,7 @@ import vyos.utils.process SQUASHFS_FILE = 'live/filesystem.squashfs' VERSION_FILE = 'version.json' - -def cmd(command): - res = vyos.utils.process.call(command, shell=True) - if res > 0: - raise OSError(f"Command '{command}' failed") +from utils import cmd def mkdir(path): os.makedirs(path, exist_ok=True) diff --git a/scripts/image-build/utils.py b/scripts/image-build/utils.py index 6906c52d..8c3ccbab 100644 --- a/scripts/image-build/utils.py +++ b/scripts/image-build/utils.py @@ -23,6 +23,7 @@ from distutils.spawn import find_executable # Local modules import defaults +import vyos def check_build_config(): if not os.path.exists(defaults.BUILD_CONFIG): @@ -76,3 +77,8 @@ def check_system_dependencies(deps): raise OSError(checker.format_missing_dependencies()) else: pass + +def cmd(command): + res = vyos.utils.process.call(command, shell=True) + if res > 0: + raise OSError(f"Command '{command}' failed") -- cgit v1.2.3 From 3e3f2181ab5334b590c7a97d9c4b984842d712ed Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 1 May 2024 11:07:03 -0500 Subject: build: T3664: clone vyos-1x under build dir instead of as submodule (cherry picked from commit a90809e213bb10aa17223687fe8a965050959869) --- .gitmodules | 3 - packages/vyos-1x | 1 - scripts/image-build/build-vyos-image | 142 +++++++++++++++++++---------------- 3 files changed, 78 insertions(+), 68 deletions(-) delete mode 100644 .gitmodules delete mode 160000 packages/vyos-1x (limited to 'scripts') diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 7d26010c..00000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "packages/vyos-1x"] - path = packages/vyos-1x - url = https://github.com/vyos/vyos-1x diff --git a/packages/vyos-1x b/packages/vyos-1x deleted file mode 160000 index b5d3d36d..00000000 --- a/packages/vyos-1x +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b5d3d36d1f70e53ef6a8a6634ab863d94d791bf2 diff --git a/scripts/image-build/build-vyos-image b/scripts/image-build/build-vyos-image index c6e76208..5b315484 100755 --- a/scripts/image-build/build-vyos-image +++ b/scripts/image-build/build-vyos-image @@ -32,6 +32,13 @@ import datetime import functools import string +## Check if the script is running wirh root permissions +## Since live-build requires privileged calls such as chroot(), +## there's no real way around it. +if os.getuid() != 0: + print("E: this script requires root privileges") + sys.exit(1) + # Import third-party modules try: import tomli @@ -41,45 +48,72 @@ try: except ModuleNotFoundError as e: print(f"E: Cannot load required library {e}") print("E: Please make sure the following Python3 modules are installed: tomli jinja2 git psutil") + sys.exit(1) + +# Import local defaults +import defaults + +## Load the file with default build configuration options +try: + with open(defaults.DEFAULTS_FILE, 'rb') as f: + build_defaults = tomli.load(f) +except Exception as e: + print("E: Failed to open the defaults file {0}: {1}".format(defaults.DEFAULTS_FILE, e)) + sys.exit(1) -# Initialize Git object from our repository +# Checkout vyos-1x under build directory try: - repo = git.Repo('.', search_parent_directories=True) - repo.git.submodule('update', '--init') - - # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient - build_git = repo.head.object.hexsha[:14] - # If somone played around with the source tree and the build is "dirty", mark it - if repo.is_dirty(): - build_git += "-dirty" - - # Retrieve git branch name or current tag - # Building a tagged release might leave us checking out a git tag that is not the tip of a named branch (detached HEAD) - # Check if the current HEAD is associated with a tag and use its name instead of an unavailable branch name. - git_branch = next((tag.name for tag in repo.tags if tag.commit == repo.head.commit), None) - if git_branch is None: - git_branch = repo.active_branch.name + branch_name = build_defaults['vyos_branch'] + url_vyos_1x = 'https://github.com/vyos/vyos-1x' + path_vyos_1x = os.path.join(defaults.BUILD_DIR, 'vyos-1x') + repo_vyos_1x = git.Repo.clone_from(url_vyos_1x, path_vyos_1x, no_checkout=True) + # alternatively, pass commit hash or tag as arg: + repo_vyos_1x.git.checkout(branch_name) except Exception as e: - print(f'W: Could not retrieve information from git: {repr(e)}') - build_git = "" - git_branch = "" + print(f'E: Could not retrieve vyos-1x from branch {branch_name}: {repr(e)}') + sys.exit(1) -# Add the vyos-1x submodule directory to the Python path -# so that we can import modules from it. -VYOS1X_DIR = os.path.join(os.getcwd(), 'packages/vyos-1x/python') +# Add the vyos-1x directory to the Python path so that +# we can import modules from it. +VYOS1X_DIR = os.path.join(os.getcwd(), defaults.BUILD_DIR, 'vyos-1x/python') if not os.path.exists(VYOS1X_DIR): - print("E: packages/vyos-1x subdirectory does not exist, did git submodules fail to initialize?") + print("E: vyos-1x subdirectory does not exist, did git checkout fail?") + sys.exit(1) else: sys.path.append(VYOS1X_DIR) # Import local modules from scripts/image-build # They rely on modules from vyos-1x import utils -import defaults import raw_image from utils import cmd +## Check if there are missing build dependencies +deps = { + 'packages': [ + 'sudo', + 'make', + 'live-build', + 'pbuilder', + 'devscripts', + 'python3-pystache', + 'python3-git', + 'qemu-utils', + 'gdisk', + 'kpartx', + 'dosfstools' + ], + 'binaries': [] +} + +print("I: Checking if packages required for VyOS image build are installed") +try: + utils.check_system_dependencies(deps) +except OSError as e: + print(f"E: {e}") + sys.exit(1) + # argparse converts hyphens to underscores, # so for lookups in the original options hash we have to convert them back def field_to_option(s): @@ -127,46 +161,6 @@ def make_toml_path(dir, file_basename): if __name__ == "__main__": - ## Check if the script is running wirh root permissions - ## Since live-build requires privileged calls such as chroot(), - ## there's no real way around it. - if os.getuid() != 0: - print("E: this script requires root privileges") - sys.exit(1) - - ## Check if there are missing build dependencies - deps = { - 'packages': [ - 'sudo', - 'make', - 'live-build', - 'pbuilder', - 'devscripts', - 'python3-pystache', - 'python3-git', - 'qemu-utils', - 'gdisk', - 'kpartx', - 'dosfstools' - ], - 'binaries': [] - } - - print("I: Checking if packages required for VyOS image build are installed") - try: - checker = utils.check_system_dependencies(deps) - except OSError as e: - print(f"E: {e}") - sys.exit(1) - - ## Load the file with default build configuration options - try: - with open(defaults.DEFAULTS_FILE, 'rb') as f: - build_defaults = tomli.load(f) - except Exception as e: - print("E: Failed to open the defaults file {0}: {1}".format(defaults.DEFAULTS_FILE, e)) - sys.exit(1) - ## Get a list of available build flavors flavor_dir_env = os.getenv("VYOS_BUILD_FLAVORS_DIR") if flavor_dir_env: @@ -372,6 +366,26 @@ if __name__ == "__main__": # Assign a (hopefully) unique identifier to the build (UUID) build_uuid = str(uuid.uuid4()) + # Initialize Git object from our repository + try: + repo = git.Repo('.') + # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient + build_git = repo.head.object.hexsha[:14] + # If somone played around with the source tree and the build is "dirty", mark it + if repo.is_dirty(): + build_git += "-dirty" + + # Retrieve git branch name or current tag + # Building a tagged release might leave us checking out a git tag that is not the tip of a named branch (detached HEAD) + # Check if the current HEAD is associated with a tag and use its name instead of an unavailable branch name. + git_branch = next((tag.name for tag in repo.tags if tag.commit == repo.head.commit), None) + if git_branch is None: + git_branch = repo.active_branch.name + except Exception as e: + print(f'W: Could not retrieve information from git: {repr(e)}') + build_git = "" + git_branch = "" + # Create the build version string if build_config['build_type'] == 'development': try: -- cgit v1.2.3 From 255bf960995ec072ba7b72fe2d926cfd986adc2f Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 3 May 2024 15:06:16 -0500 Subject: build: T3664: fix regression and bug in clone vyos-1x repo (cherry picked from commit 02c340d9d76ee89f47d7d6d245c5dea955a5f683) --- scripts/image-build/build-vyos-image | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/image-build/build-vyos-image b/scripts/image-build/build-vyos-image index 5b315484..c2d39c82 100755 --- a/scripts/image-build/build-vyos-image +++ b/scripts/image-build/build-vyos-image @@ -66,7 +66,17 @@ try: branch_name = build_defaults['vyos_branch'] url_vyos_1x = 'https://github.com/vyos/vyos-1x' path_vyos_1x = os.path.join(defaults.BUILD_DIR, 'vyos-1x') - repo_vyos_1x = git.Repo.clone_from(url_vyos_1x, path_vyos_1x, no_checkout=True) + try: + repo_vyos_1x = git.Repo.clone_from(url_vyos_1x, path_vyos_1x, no_checkout=True) + except git.GitCommandError: + if os.path.exists(path_vyos_1x): + try: + repo_vyos_1x = git.Repo(path_vyos_1x) + except git.GitError: + print(f'E: Corrupted vyos-1x git repo: {path_vyos_1x}; remove') + sys.exit(1) + else: + raise # alternatively, pass commit hash or tag as arg: repo_vyos_1x.git.checkout(branch_name) except Exception as e: @@ -368,7 +378,7 @@ if __name__ == "__main__": # Initialize Git object from our repository try: - repo = git.Repo('.') + repo = git.Repo('.', search_parent_directories=True) # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient build_git = repo.head.object.hexsha[:14] # If somone played around with the source tree and the build is "dirty", mark it -- cgit v1.2.3