From 249a595f1d76e955465dafc1a757483cb75705d5 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Sun, 7 Apr 2024 16:15:50 +0000 Subject: build: T3664: add support for building non-ISO flavors (cherry picked from commit a896176ad8a1e1c7ef440a31c5afcfad358ed309) --- scripts/image-build/raw_image.py | 215 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 scripts/image-build/raw_image.py (limited to 'scripts/image-build/raw_image.py') diff --git a/scripts/image-build/raw_image.py b/scripts/image-build/raw_image.py new file mode 100644 index 00000000..67039a24 --- /dev/null +++ b/scripts/image-build/raw_image.py @@ -0,0 +1,215 @@ +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# File: raw_image.py +# Purpose: Helper functions for building raw images. + +import os +import sys +import shutil +import traceback + +import vyos.utils.process + +SQUASHFS_FILE = 'live/filesystem.squashfs' +VERSION_FILE = 'version.json' + + +def cmd(command): + res = vyos.utils.process.call(command, shell=True) + if res > 0: + raise OSError(f"Command '{command}' failed") + +def mkdir(path): + os.makedirs(path, exist_ok=True) + + +class BuildContext: + def __init__(self, iso_path, work_dir, debug=False): + self.work_dir = work_dir + self.iso_path = iso_path + self.debug = debug + self.loop_device = None + + def __enter__(self): + print(f"I: Setting up a raw image build directory in {self.work_dir}") + + self.iso_dir = os.path.join(self.work_dir, "iso") + self.squash_dir = os.path.join(self.work_dir, "squash") + self.raw_dir = os.path.join(self.work_dir, "raw") + self.efi_dir = os.path.join(self.work_dir, "efi") + + # Create mount point directories + mkdir(self.iso_dir) + mkdir(self.squash_dir) + mkdir(self.raw_dir) + mkdir(self.efi_dir) + + # Mount the ISO image + cmd(f"""mount -t iso9660 -o ro,loop {self.iso_path} {self.iso_dir}""") + + # Mount the SquashFS image + cmd(f"""mount -t squashfs -o ro,loop {self.iso_dir}/{SQUASHFS_FILE} {self.squash_dir}""") + + return self + + def __exit__(self, exc_type, exc_value, exc_tb): + print(f"I: Tearing down the raw image build environment in {self.work_dir}") + cmd(f"""umount {self.squash_dir}/dev/""") + cmd(f"""umount {self.squash_dir}/proc/""") + cmd(f"""umount {self.squash_dir}/sys/""") + + cmd(f"umount {self.squash_dir}/boot/efi") + cmd(f"umount {self.squash_dir}/boot") + + cmd(f"""umount {self.squash_dir}""") + cmd(f"""umount {self.iso_dir}""") + cmd(f"""umount {self.raw_dir}""") + cmd(f"""umount {self.efi_dir}""") + + if self.loop_device: + cmd(f"""losetup -d {self.loop_device}""") + +def create_disk(path, size): + cmd(f"""qemu-img create -f raw "{path}" {size}G""") + +def read_version_data(iso_dir): + from json import load + with open(os.path.join(iso_dir, VERSION_FILE), 'r') as f: + data = load(f) + return data + +def setup_loop_device(con, raw_file): + from subprocess import Popen, PIPE, STDOUT + from re import match + command = f'losetup --show -f {raw_file}' + p = Popen(command, stderr=PIPE, stdout=PIPE, stdin=PIPE, shell=True) + (stdout, stderr) = p.communicate() + + if p.returncode > 0: + raise OSError(f"Could not set up a loop device: {stderr.decode()}") + + con.loop_device = stdout.decode().strip() + if con.debug: + print(f"I: Using loop device {con.loop_device}") + +def mount_image(con): + import vyos.system.disk + + from subprocess import Popen, PIPE, STDOUT + from re import match + + vyos.system.disk.filesystem_create(con.disk_details.partition['efi'], 'efi') + vyos.system.disk.filesystem_create(con.disk_details.partition['root'], 'ext4') + + cmd(f"mount -t ext4 {con.disk_details.partition['root']} {con.raw_dir}") + cmd(f"mount -t vfat {con.disk_details.partition['efi']} {con.efi_dir}") + +def install_image(con, version): + from glob import glob + + vyos_dir = os.path.join(con.raw_dir, f'boot/{version}/') + mkdir(vyos_dir) + mkdir(os.path.join(vyos_dir, 'work/work')) + mkdir(os.path.join(vyos_dir, 'rw')) + + shutil.copy(f"{con.iso_dir}/{SQUASHFS_FILE}", f"{vyos_dir}/{version}.squashfs") + + boot_files = glob(f'{con.squash_dir}/boot/*') + boot_files = [f for f in boot_files if os.path.isfile(f)] + + for f in boot_files: + print(f"I: Copying file {f}") + shutil.copy(f, vyos_dir) + + with open(f"{con.raw_dir}/persistence.conf", 'w') as f: + f.write("/ union\n") + +def setup_grub_configuration(build_config, root_dir) -> None: + """Install GRUB configurations + + Args: + root_dir (str): a path to the root of target filesystem + """ + from vyos.template import render + from vyos.system import grub + + print('I: Installing GRUB configuration files') + grub_cfg_main = f'{root_dir}/{grub.GRUB_DIR_MAIN}/grub.cfg' + grub_cfg_vars = f'{root_dir}/{grub.CFG_VYOS_VARS}' + grub_cfg_modules = f'{root_dir}/{grub.CFG_VYOS_MODULES}' + grub_cfg_menu = f'{root_dir}/{grub.CFG_VYOS_MENU}' + grub_cfg_options = f'{root_dir}/{grub.CFG_VYOS_OPTIONS}' + + # create new files + render(grub_cfg_main, grub.TMPL_GRUB_MAIN, {}) + grub.common_write(root_dir) + grub.vars_write(grub_cfg_vars, build_config["boot_settings"]) + grub.modules_write(grub_cfg_modules, []) + grub.write_cfg_ver(1, root_dir) + render(grub_cfg_menu, grub.TMPL_GRUB_MENU, {}) + render(grub_cfg_options, grub.TMPL_GRUB_OPTS, {}) + +def install_grub(con, version): + from re import match + from vyos.system import disk, grub + + # Mount the required virtual filesystems + os.makedirs(f"{con.raw_dir}/boot/efi", exist_ok=True) + cmd(f"mount --bind /dev {con.squash_dir}/dev") + cmd(f"mount --bind /proc {con.squash_dir}/proc") + cmd(f"mount --bind /sys {con.squash_dir}/sys") + + cmd(f"mount --bind {con.raw_dir}/boot {con.squash_dir}/boot") + cmd(f"mount --bind {con.efi_dir} {con.squash_dir}/boot/efi") + + DIR_DST_ROOT = con.raw_dir + + setup_grub_configuration(con.build_config, DIR_DST_ROOT) + # add information about version + grub.create_structure(DIR_DST_ROOT) + grub.version_add(version, DIR_DST_ROOT) + grub.set_default(version, DIR_DST_ROOT) + grub.set_console_type(con.build_config["boot_settings"]["console_type"], DIR_DST_ROOT) + + print('I: Installing GRUB to the disk image') + grub.install(con.loop_device, f'/boot/', f'/boot/efi', chroot=con.squash_dir) + + # sort inodes (to make GRUB read config files in alphabetical order) + grub.sort_inodes(f'{DIR_DST_ROOT}/{grub.GRUB_DIR_VYOS}') + grub.sort_inodes(f'{DIR_DST_ROOT}/{grub.GRUB_DIR_VYOS_VERS}') + + +def create_raw_image(build_config, iso_file, work_dir): + from vyos.system.disk import parttable_create + + if not os.path.exists(iso_file): + print(f"E: ISO file {iso_file} does not exist in the build directory") + sys.exit(1) + + with BuildContext(iso_file, work_dir, debug=True) as con: + con.build_config = build_config + version_data = read_version_data(con.iso_dir) + version = version_data['version'] + raw_file = f"vyos-{version}.raw" + print(f"I: Building raw file {raw_file}") + create_disk(raw_file, build_config["disk_size"]) + setup_loop_device(con, raw_file) + disk_details = parttable_create(con.loop_device, (int(build_config["disk_size"]) - 1) * 1024 * 1024) + con.disk_details = disk_details + mount_image(con) + install_image(con, version) + install_grub(con, version) + + return raw_file -- cgit v1.2.3 From d3696d878f9ed6eed46723550dff8c1b914b62da Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Mon, 29 Apr 2024 14:33:25 +0000 Subject: build: T3664: fail the build on external command errors (cherry picked from commit 7dfd9232da787a7befbc4338d4eb21fee4325174) --- scripts/image-build/build-vyos-image | 17 +++++++---------- scripts/image-build/raw_image.py | 6 +----- scripts/image-build/utils.py | 6 ++++++ 3 files changed, 14 insertions(+), 15 deletions(-) (limited to 'scripts/image-build/raw_image.py') diff --git a/scripts/image-build/build-vyos-image b/scripts/image-build/build-vyos-image index 2295df1f..c6e76208 100755 --- a/scripts/image-build/build-vyos-image +++ b/scripts/image-build/build-vyos-image @@ -78,6 +78,8 @@ import utils import defaults import raw_image +from utils import cmd + # argparse converts hyphens to underscores, # so for lookups in the original options hash we have to convert them back def field_to_option(s): @@ -450,7 +452,7 @@ if __name__ == "__main__": ## Clean up earlier build state and artifacts print("I: Cleaning the build workspace") - os.system("lb clean") + cmd("lb clean") #iter(lambda p: shutil.rmtree(p, ignore_errors=True), # ['config/binary', 'config/bootstrap', 'config/chroot', 'config/common', 'config/source']) artifacts = functools.reduce( @@ -575,10 +577,7 @@ if __name__ == "__main__": print("D: live-build configuration command") print(lb_config_command) - result = os.system(lb_config_command) - if result > 0: - print("E: live-build config failed") - sys.exit(1) + cmd(lb_config_command) ## In dry-run mode, exit at this point if build_config["dry_run"]: @@ -595,9 +594,7 @@ if __name__ == "__main__": print("I: Starting image build") if debug: print("D: It's not like I'm building this specially for you or anything!") - res = os.system("lb build 2>&1") - if res > 0: - sys.exit(res) + cmd("lb build 2>&1") # Copy the image shutil.copy("live-image-{0}.hybrid.iso".format(build_config["architecture"]), iso_file) @@ -613,11 +610,11 @@ if __name__ == "__main__": # For those cases, we support running a post-build hook on the raw image. # The image_format field should be 'raw' if a post-build hook is used. hook_path = build_config["post_build_hook"] - os.system(f"{hook_path} {raw_image}") + cmd(f"{hook_path} {raw_image}") else: # Most other formats, thankfully, can be produced with just `qemu-img convert` other_formats = filter(lambda x: x not in ["iso", "raw"], build_config["image_format"]) for f in other_formats: target = f"{os.path.splitext(raw_image)[0]}.{f}" print(f"I: Building {f} file {target}") - os.system(f"qemu-img convert -f raw -O {f} {raw_image} {target}") + cmd(f"qemu-img convert -f raw -O {f} {raw_image} {target}") diff --git a/scripts/image-build/raw_image.py b/scripts/image-build/raw_image.py index 67039a24..ae061990 100644 --- a/scripts/image-build/raw_image.py +++ b/scripts/image-build/raw_image.py @@ -25,11 +25,7 @@ import vyos.utils.process SQUASHFS_FILE = 'live/filesystem.squashfs' VERSION_FILE = 'version.json' - -def cmd(command): - res = vyos.utils.process.call(command, shell=True) - if res > 0: - raise OSError(f"Command '{command}' failed") +from utils import cmd def mkdir(path): os.makedirs(path, exist_ok=True) diff --git a/scripts/image-build/utils.py b/scripts/image-build/utils.py index 6906c52d..8c3ccbab 100644 --- a/scripts/image-build/utils.py +++ b/scripts/image-build/utils.py @@ -23,6 +23,7 @@ from distutils.spawn import find_executable # Local modules import defaults +import vyos def check_build_config(): if not os.path.exists(defaults.BUILD_CONFIG): @@ -76,3 +77,8 @@ def check_system_dependencies(deps): raise OSError(checker.format_missing_dependencies()) else: pass + +def cmd(command): + res = vyos.utils.process.call(command, shell=True) + if res > 0: + raise OSError(f"Command '{command}' failed") -- cgit v1.2.3