diff options
Diffstat (limited to 'scripts/image-build')
| -rwxr-xr-x | scripts/image-build/build-vyos-image | 198 | ||||
| -rw-r--r-- | scripts/image-build/raw_image.py | 11 |
2 files changed, 121 insertions, 88 deletions
diff --git a/scripts/image-build/build-vyos-image b/scripts/image-build/build-vyos-image index c6e76208..5bf5803d 100755 --- a/scripts/image-build/build-vyos-image +++ b/scripts/image-build/build-vyos-image @@ -21,6 +21,7 @@ import re import os import sys +import copy import uuid import glob import json @@ -32,6 +33,13 @@ import datetime import functools import string +## Check if the script is running wirh root permissions +## Since live-build requires privileged calls such as chroot(), +## there's no real way around it. +if os.getuid() != 0: + print("E: this script requires root privileges") + sys.exit(1) + # Import third-party modules try: import tomli @@ -41,45 +49,82 @@ try: except ModuleNotFoundError as e: print(f"E: Cannot load required library {e}") print("E: Please make sure the following Python3 modules are installed: tomli jinja2 git psutil") + sys.exit(1) + +# Import local defaults +import defaults -# Initialize Git object from our repository +## Load the file with default build configuration options try: - repo = git.Repo('.', search_parent_directories=True) - repo.git.submodule('update', '--init') - - # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient - build_git = repo.head.object.hexsha[:14] - # If somone played around with the source tree and the build is "dirty", mark it - if repo.is_dirty(): - build_git += "-dirty" - - # Retrieve git branch name or current tag - # Building a tagged release might leave us checking out a git tag that is not the tip of a named branch (detached HEAD) - # Check if the current HEAD is associated with a tag and use its name instead of an unavailable branch name. - git_branch = next((tag.name for tag in repo.tags if tag.commit == repo.head.commit), None) - if git_branch is None: - git_branch = repo.active_branch.name + with open(defaults.DEFAULTS_FILE, 'rb') as f: + build_defaults = tomli.load(f) except Exception as e: - print(f'W: Could not retrieve information from git: {repr(e)}') - build_git = "" - git_branch = "" + print("E: Failed to open the defaults file {0}: {1}".format(defaults.DEFAULTS_FILE, e)) + sys.exit(1) -# Add the vyos-1x submodule directory to the Python path -# so that we can import modules from it. -VYOS1X_DIR = os.path.join(os.getcwd(), 'packages/vyos-1x/python') +# Checkout vyos-1x under build directory +try: + branch_name = build_defaults['vyos_branch'] + url_vyos_1x = 'https://github.com/vyos/vyos-1x' + path_vyos_1x = os.path.join(defaults.BUILD_DIR, 'vyos-1x') + try: + repo_vyos_1x = git.Repo.clone_from(url_vyos_1x, path_vyos_1x, no_checkout=True) + except git.GitCommandError: + if os.path.exists(path_vyos_1x): + try: + repo_vyos_1x = git.Repo(path_vyos_1x) + except git.GitError: + print(f'E: Corrupted vyos-1x git repo: {path_vyos_1x}; remove') + sys.exit(1) + else: + raise + # alternatively, pass commit hash or tag as arg: + repo_vyos_1x.git.checkout(branch_name) +except Exception as e: + print(f'E: Could not retrieve vyos-1x from branch {branch_name}: {repr(e)}') + sys.exit(1) + +# Add the vyos-1x directory to the Python path so that +# we can import modules from it. +VYOS1X_DIR = os.path.join(os.getcwd(), defaults.BUILD_DIR, 'vyos-1x/python') if not os.path.exists(VYOS1X_DIR): - print("E: packages/vyos-1x subdirectory does not exist, did git submodules fail to initialize?") + print("E: vyos-1x subdirectory does not exist, did git checkout fail?") + sys.exit(1) else: sys.path.append(VYOS1X_DIR) # Import local modules from scripts/image-build # They rely on modules from vyos-1x import utils -import defaults import raw_image from utils import cmd +## Check if there are missing build dependencies +deps = { + 'packages': [ + 'sudo', + 'make', + 'live-build', + 'pbuilder', + 'devscripts', + 'python3-pystache', + 'python3-git', + 'qemu-utils', + 'gdisk', + 'kpartx', + 'dosfstools' + ], + 'binaries': [] +} + +print("I: Checking if packages required for VyOS image build are installed") +try: + utils.check_system_dependencies(deps) +except OSError as e: + print(f"E: {e}") + sys.exit(1) + # argparse converts hyphens to underscores, # so for lookups in the original options hash we have to convert them back def field_to_option(s): @@ -94,21 +139,22 @@ def get_validator(optdict, name): except KeyError: return None -def merge_dicts(source, destination, skip_none=False): - """ Merge two dictionaries and return a new dict which has the merged key/value pairs. +def merge_defaults(source, defaults={}, skip_none=False): + """ Merge a dict with values from a defaults dict. Merging logic is as follows: Sub-dicts are merged. List values are combined. - Scalar values are set to those from the source dict. + Scalar values are set to those from the source dict, + if they exist there. """ from copy import deepcopy - tmp = deepcopy(destination) + tmp = deepcopy(defaults) for key, value in source.items(): if key not in tmp: tmp[key] = value elif isinstance(source[key], dict): - tmp[key] = merge_dicts(source[key], tmp[key]) + tmp[key] = merge_defaults(source[key], tmp[key]) elif isinstance(source[key], list): tmp[key] = source[key] + tmp[key] elif not skip_none or source[key] is not None: @@ -127,46 +173,6 @@ def make_toml_path(dir, file_basename): if __name__ == "__main__": - ## Check if the script is running wirh root permissions - ## Since live-build requires privileged calls such as chroot(), - ## there's no real way around it. - if os.getuid() != 0: - print("E: this script requires root privileges") - sys.exit(1) - - ## Check if there are missing build dependencies - deps = { - 'packages': [ - 'sudo', - 'make', - 'live-build', - 'pbuilder', - 'devscripts', - 'python3-pystache', - 'python3-git', - 'qemu-utils', - 'gdisk', - 'kpartx', - 'dosfstools' - ], - 'binaries': [] - } - - print("I: Checking if packages required for VyOS image build are installed") - try: - checker = utils.check_system_dependencies(deps) - except OSError as e: - print(f"E: {e}") - sys.exit(1) - - ## Load the file with default build configuration options - try: - with open(defaults.DEFAULTS_FILE, 'rb') as f: - build_defaults = tomli.load(f) - except Exception as e: - print("E: Failed to open the defaults file {0}: {1}".format(defaults.DEFAULTS_FILE, e)) - sys.exit(1) - ## Get a list of available build flavors flavor_dir_env = os.getenv("VYOS_BUILD_FLAVORS_DIR") if flavor_dir_env: @@ -241,7 +247,7 @@ if __name__ == "__main__": sys.exit(1) ## Try to get correct architecture and build type from build flavor and CLI arguments - pre_build_config = merge_dicts({}, build_defaults) + pre_build_config = copy.deepcopy(build_defaults) flavor_config = {} build_flavor = args["build_flavor"] @@ -249,7 +255,7 @@ if __name__ == "__main__": toml_flavor_file = make_toml_path(flavor_dir, args["build_flavor"]) with open(toml_flavor_file, 'rb') as f: flavor_config = tomli.load(f) - pre_build_config = merge_dicts(flavor_config, pre_build_config) + pre_build_config = merge_defaults(flavor_config, defaults=pre_build_config) except FileNotFoundError: print(f"E: Flavor '{build_flavor}' does not exist") sys.exit(1) @@ -258,7 +264,7 @@ if __name__ == "__main__": sys.exit(1) ## Combine configs args > flavor > defaults - pre_build_config = merge_dicts(args, pre_build_config, skip_none=True) + pre_build_config = merge_defaults(args, defaults=pre_build_config, skip_none=True) # Some fixup for mirror settings. # The idea is: if --debian-mirror is specified but --pbuilder-debian-mirror is not, @@ -290,20 +296,20 @@ if __name__ == "__main__": args['pbuilder_config'] = os.path.join(defaults.BUILD_DIR, defaults.PBUILDER_CONFIG) ## Combine the arguments with non-configurable defaults - build_config = merge_dicts({}, build_defaults) + build_config = copy.deepcopy(build_defaults) ## Load correct mix-ins with open(make_toml_path(defaults.BUILD_TYPES_DIR, pre_build_config["build_type"]), 'rb') as f: build_type_config = tomli.load(f) - build_config = merge_dicts(build_type_config, build_config) + build_config = merge_defaults(build_type_config, defaults=build_config) with open(make_toml_path(defaults.BUILD_ARCHES_DIR, pre_build_config["architecture"]), 'rb') as f: build_arch_config = tomli.load(f) - build_config = merge_dicts(build_arch_config, build_config) + build_config = merge_defaults(build_arch_config, defaults=build_config) ## Override with flavor and then CLI arguments - build_config = merge_dicts(flavor_config, build_config) - build_config = merge_dicts(args, build_config, skip_none=True) + build_config = merge_defaults(flavor_config, defaults=build_config) + build_config = merge_defaults(args, defaults=build_config, skip_none=True) ## Rename and merge some fields for simplicity ## E.g. --custom-packages is for the user, but internally @@ -316,7 +322,7 @@ if __name__ == "__main__": if has_nonempty_key(build_config, "architectures"): arch = build_config["architecture"] if arch in build_config["architectures"]: - if has_nonempty_key(build_config["architectures"], "packages"): + if has_nonempty_key(build_config["architectures"][arch], "packages"): build_config["packages"] += build_config["architectures"][arch]["packages"] ## Check if image format is specified, @@ -329,7 +335,7 @@ if __name__ == "__main__": if "boot_settings" not in build_config: build_config["boot_settings"] = defaults.boot_settings else: - build_config["boot_settings"] = merge_dicts(defaults.boot_settings, build_config["boot_settings"]) + build_config["boot_settings"] = merge_defaults(build_config["boot_settings"], defaults=defaults.boot_settings) ## Convert the image_format field to a single-item list if it's a scalar ## (like `image_format = "iso"`) @@ -372,6 +378,26 @@ if __name__ == "__main__": # Assign a (hopefully) unique identifier to the build (UUID) build_uuid = str(uuid.uuid4()) + # Initialize Git object from our repository + try: + repo = git.Repo('.', search_parent_directories=True) + # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient + build_git = repo.head.object.hexsha[:14] + # If somone played around with the source tree and the build is "dirty", mark it + if repo.is_dirty(): + build_git += "-dirty" + + # Retrieve git branch name or current tag + # Building a tagged release might leave us checking out a git tag that is not the tip of a named branch (detached HEAD) + # Check if the current HEAD is associated with a tag and use its name instead of an unavailable branch name. + git_branch = next((tag.name for tag in repo.tags if tag.commit == repo.head.commit), None) + if git_branch is None: + git_branch = repo.active_branch.name + except Exception as e: + print(f'W: Could not retrieve information from git: {repr(e)}') + build_git = "" + git_branch = "" + # Create the build version string if build_config['build_type'] == 'development': try: @@ -399,12 +425,14 @@ if __name__ == "__main__": version_data = { 'version': version, + 'flavor': build_config["build_flavor"], 'built_by': build_config['build_by'], 'built_on': build_date, 'build_uuid': build_uuid, 'build_git': build_git, 'build_branch': git_branch, 'release_train': build_config['release_train'], + 'architecture': build_config['architecture'], 'lts_build': lts_build, 'build_comment': build_config['build_comment'], 'bugtracker_url': build_config['bugtracker_url'], @@ -564,9 +592,9 @@ if __name__ == "__main__": ## Pin release for VyOS packages apt_pin = f"""Package: * - Pin: release n={build_config['release_train']} - Pin-Priority: 600 - """ +Pin: release n={build_config['release_train']} +Pin-Priority: 600 +""" with open(defaults.VYOS_PIN_FILE, 'w') as f: f.write(apt_pin) @@ -615,6 +643,8 @@ if __name__ == "__main__": # Most other formats, thankfully, can be produced with just `qemu-img convert` other_formats = filter(lambda x: x not in ["iso", "raw"], build_config["image_format"]) for f in other_formats: - target = f"{os.path.splitext(raw_image)[0]}.{f}" + image_ext = build_config.get("image_ext", f) + image_opts = build_config.get("image_opts", "") + target = f"{os.path.splitext(raw_image)[0]}-{build_config['build_flavor']}.{image_ext}" print(f"I: Building {f} file {target}") - cmd(f"qemu-img convert -f raw -O {f} {raw_image} {target}") + cmd(f"qemu-img convert -f raw -O {f} {image_opts} {raw_image} {target}") diff --git a/scripts/image-build/raw_image.py b/scripts/image-build/raw_image.py index ae061990..27fb27ab 100644 --- a/scripts/image-build/raw_image.py +++ b/scripts/image-build/raw_image.py @@ -22,6 +22,10 @@ import traceback import vyos.utils.process +import vyos.template + +vyos.template.DEFAULT_TEMPLATE_DIR = os.path.join(os.getcwd(), 'build/vyos-1x/data/templates') + SQUASHFS_FILE = 'live/filesystem.squashfs' VERSION_FILE = 'version.json' @@ -138,7 +142,6 @@ def setup_grub_configuration(build_config, root_dir) -> None: Args: root_dir (str): a path to the root of target filesystem """ - from vyos.template import render from vyos.system import grub print('I: Installing GRUB configuration files') @@ -149,13 +152,13 @@ def setup_grub_configuration(build_config, root_dir) -> None: grub_cfg_options = f'{root_dir}/{grub.CFG_VYOS_OPTIONS}' # create new files - render(grub_cfg_main, grub.TMPL_GRUB_MAIN, {}) + vyos.template.render(grub_cfg_main, grub.TMPL_GRUB_MAIN, {}) grub.common_write(root_dir) grub.vars_write(grub_cfg_vars, build_config["boot_settings"]) grub.modules_write(grub_cfg_modules, []) grub.write_cfg_ver(1, root_dir) - render(grub_cfg_menu, grub.TMPL_GRUB_MENU, {}) - render(grub_cfg_options, grub.TMPL_GRUB_OPTS, {}) + vyos.template.render(grub_cfg_menu, grub.TMPL_GRUB_MENU, {}) + vyos.template.render(grub_cfg_options, grub.TMPL_GRUB_OPTS, {}) def install_grub(con, version): from re import match |