From 1cda2d42bb5064868c6435c14a5d3e4b047500d8 Mon Sep 17 00:00:00 2001 From: Gabin-CC Date: Thu, 5 Jun 2025 03:37:56 +0200 Subject: T7453: Enhance raw/qcow2 image creation Description This pull request introduces improvements to the raw_image.py script responsible for building raw disk images in the VyOS build process. Main Changes Added use of kpartx to reliably map EFI and root partitions from the loop device. Introduced disk_details as an attribute on the BuildContext object to pass partition metadata through the image build steps. Improved the __exit__ method for BuildContext to unmount all mount points and clean up kpartx mappings and loop devices, even in failure cases. Fixed a crash in mount_image() when con.disk_details was not set. Added useful debug logs for loop device usage and partition mapping. Motivation The previous implementation assumed partitions like /dev/loopXp3 would appear automatically, which is unreliable across some environments (especially containers or newer systems). This PR makes the process more reliable by explicitly mapping partitions with kpartx, a tool designed for this purpose. It also ensures proper resource cleanup by unmounting and detaching everything cleanly, preventing leaked loop devices or stale mount points. Test Instructions Flavor : cloud-init.toml packages = [ "cloud-init", "qemu-guest-agent" ] image_format = ["qcow2"] disk_size = 10 [boot_settings] console_type = "ttyS0" Run: sudo ./build-vyos-image --architecture amd64 \ --build-by "you@example.com" \ --reuse-iso vyos-1.5-rolling-*.iso \ cloud-init Expected behavior: The build completes without errors. The .qcow2 image file is generated and bootable (e.g., in KVM or Proxmox). Partitions are mounted correctly via /dev/mapper/loopXp*. Signed-off-by: Gabin-CC --- scripts/image-build/raw_image.py | 79 ++++++++++++++++++++++++++++++---------- 1 file changed, 60 insertions(+), 19 deletions(-) (limited to 'scripts/image-build/raw_image.py') diff --git a/scripts/image-build/raw_image.py b/scripts/image-build/raw_image.py index d850eead..2edfecd6 100644 --- a/scripts/image-build/raw_image.py +++ b/scripts/image-build/raw_image.py @@ -63,22 +63,38 @@ class BuildContext: return self - def __exit__(self, exc_type, exc_value, exc_tb): + def __exit__(self, exc_type, exc_value, traceback): print(f"I: Tearing down the raw image build environment in {self.work_dir}") - cmd(f"""umount {self.squash_dir}/dev/""") - cmd(f"""umount {self.squash_dir}/proc/""") - cmd(f"""umount {self.squash_dir}/sys/""") - - cmd(f"umount {self.squash_dir}/boot/efi") - cmd(f"umount {self.squash_dir}/boot") - - cmd(f"""umount {self.squash_dir}""") - cmd(f"""umount {self.iso_dir}""") - cmd(f"""umount {self.raw_dir}""") - cmd(f"""umount {self.efi_dir}""") + for mount in [ + f"{self.squash_dir}/dev/", + f"{self.squash_dir}/proc/", + f"{self.squash_dir}/sys/", + f"{self.squash_dir}/boot/efi", + f"{self.squash_dir}/boot", + f"{self.squash_dir}", + f"{self.iso_dir}", + f"{self.raw_dir}", + f"{self.efi_dir}" + ]: + if os.path.ismount(mount): + try: + cmd(f"umount {mount}") + except Exception as e: + print(f"W: Failed to umount {mount}: {e}") + + # Remove kpartx mappings if self.loop_device: - cmd(f"""losetup -d {self.loop_device}""") + mapper_base = os.path.basename(self.loop_device) + try: + cmd(f"kpartx -d {self.loop_device}") + except Exception as e: + print(f"W: Failed to remove kpartx mappings for {mapper_base}: {e}") + + try: + cmd(f"losetup -d {self.loop_device}") + except Exception as e: + print(f"W: Failed to detach loop device {self.loop_device}: {e}") def create_disk(path, size): cmd(f"""qemu-img create -f raw "{path}" {size}G""") @@ -106,14 +122,23 @@ def setup_loop_device(con, raw_file): def mount_image(con): import vyos.system.disk - from subprocess import Popen, PIPE, STDOUT - from re import match + try: + root = con.disk_details.partition['root'] + efi = con.disk_details.partition['efi'] + except (AttributeError, KeyError): + raise RuntimeError("E: No valid root or EFI partition found in disk details") + + vyos.system.disk.filesystem_create(efi, 'efi') + vyos.system.disk.filesystem_create(root, 'ext4') - vyos.system.disk.filesystem_create(con.disk_details.partition['efi'], 'efi') - vyos.system.disk.filesystem_create(con.disk_details.partition['root'], 'ext4') + print(f"I: Mounting root: {root} to {con.raw_dir}") + cmd(f"mount -t ext4 {root} {con.raw_dir}") + cmd(f"mount -t vfat {efi} {con.efi_dir}") - cmd(f"mount -t ext4 {con.disk_details.partition['root']} {con.raw_dir}") - cmd(f"mount -t vfat {con.disk_details.partition['efi']} {con.efi_dir}") + if not os.path.ismount(con.efi_dir): + cmd(f"mount -t vfat {con.disk_details.partition['efi']} {con.efi_dir}") + else: + print(f"I: {con.disk_details.partition['efi']} already mounted on {con.efi_dir}") def install_image(con, version): from glob import glob @@ -205,6 +230,22 @@ def create_raw_image(build_config, iso_file, work_dir): create_disk(raw_file, build_config["disk_size"]) setup_loop_device(con, raw_file) disk_details = parttable_create(con.loop_device, (int(build_config["disk_size"]) - 1) * 1024 * 1024) + + # Map partitions using kpartx + cmd(f"kpartx -av {con.loop_device}") + cmd("udevadm settle") + + # Resolve mapper names (example: /dev/mapper/loop0p2) + from glob import glob + mapper_base = os.path.basename(con.loop_device).replace("/dev/", "") + mapped_parts = sorted(glob(f"/dev/mapper/{mapper_base}p*")) + + if len(mapped_parts) < 3: + raise RuntimeError("E: Expected at least 3 partitions created by kpartx") + + disk_details.partition['efi'] = mapped_parts[1] + disk_details.partition['root'] = mapped_parts[2] + con.disk_details = disk_details mount_image(con) install_image(con, version) -- cgit v1.2.3 From 02c2e306228a3de24623946499923604341884d3 Mon Sep 17 00:00:00 2001 From: Gabin-CC Date: Fri, 6 Jun 2025 20:54:03 +0200 Subject: T7453: handle dynamic partition mapping in raw image build Enhanced the raw image creation logic to dynamically detect and assign EFI and root partitions based on the number of partitions created by kpartx. - Supports both 2-partition and 3-partition layouts - Adds debug output for mapped partitions - Avoids hardcoded assumptions about partition order - Improves resilience in cloud-init and containerized build contexts Fixes build failure when /dev/loopXp3 is missing or not mapped properly. Signed-off-by: Gabin-CC --- scripts/image-build/raw_image.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) (limited to 'scripts/image-build/raw_image.py') diff --git a/scripts/image-build/raw_image.py b/scripts/image-build/raw_image.py index 2edfecd6..399bb268 100644 --- a/scripts/image-build/raw_image.py +++ b/scripts/image-build/raw_image.py @@ -232,19 +232,34 @@ def create_raw_image(build_config, iso_file, work_dir): disk_details = parttable_create(con.loop_device, (int(build_config["disk_size"]) - 1) * 1024 * 1024) # Map partitions using kpartx + print("I: Mapping partitions using kpartx...") cmd(f"kpartx -av {con.loop_device}") cmd("udevadm settle") - # Resolve mapper names (example: /dev/mapper/loop0p2) + cmd("ls -l /dev/mapper") # debug output + + # Detect mapped partitions from glob import glob + import time + mapper_base = os.path.basename(con.loop_device).replace("/dev/", "") mapped_parts = sorted(glob(f"/dev/mapper/{mapper_base}p*")) - if len(mapped_parts) < 3: - raise RuntimeError("E: Expected at least 3 partitions created by kpartx") - - disk_details.partition['efi'] = mapped_parts[1] - disk_details.partition['root'] = mapped_parts[2] + if not mapped_parts: + raise RuntimeError(f"E: No partitions were found in /dev/mapper for {mapper_base}") + + print(f"I: Found mapped partitions: {mapped_parts}") + + if len(mapped_parts) == 2: + # Assume [0] = EFI, [1] = root + disk_details.partition['efi'] = mapped_parts[0] + disk_details.partition['root'] = mapped_parts[1] + elif len(mapped_parts) >= 3: + # Common layout: [1] = EFI, [2] = root (skip 0 if it's BIOS boot) + disk_details.partition['efi'] = mapped_parts[1] + disk_details.partition['root'] = mapped_parts[2] + else: + raise RuntimeError(f"E: Unexpected partition layout: {mapped_parts}") con.disk_details = disk_details mount_image(con) -- cgit v1.2.3 From 75f72ab9013c7ae5226497526e4db6e36dd43678 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Tue, 10 Jun 2025 15:43:29 +0100 Subject: Remove a stray debug output --- scripts/image-build/raw_image.py | 1 - 1 file changed, 1 deletion(-) (limited to 'scripts/image-build/raw_image.py') diff --git a/scripts/image-build/raw_image.py b/scripts/image-build/raw_image.py index 399bb268..a88ed020 100644 --- a/scripts/image-build/raw_image.py +++ b/scripts/image-build/raw_image.py @@ -236,7 +236,6 @@ def create_raw_image(build_config, iso_file, work_dir): cmd(f"kpartx -av {con.loop_device}") cmd("udevadm settle") - cmd("ls -l /dev/mapper") # debug output # Detect mapped partitions from glob import glob -- cgit v1.2.3