build: T3664: add support for building non-ISO flavors

(cherry picked from commit a896176ad8a1e1c7ef440a31c5afcfad358ed309)

3 files changed, 569 insertions, 286 deletions

diff --git a/scripts/image-build/build-vyos-image b/scripts/image-build/build-vyos-image
index 59ea0338..aa397843 100755
--- a/scripts/image-build/build-vyos-image
+++ b/scripts/image-build/build-vyos-image
@@ -17,32 +17,43 @@
 # File: build-vyos-image
 # Purpose: builds VyOS images using a fork of Debian's live-build tool
 
+# Import Python's standard library modules
 import re
 import os
 import sys
 import uuid
 import glob
+import json
 import shutil
 import getpass
 import platform
 import argparse
 import datetime
 import functools
-import string
 
-import json
+# Add the vyos-1x submodule directory to the Python path
+# so that we can import modules from it.
+VYOS1X_DIR = os.path.join(os.getcwd(), 'vyos-1x/python')
+if not os.path.exists(VYOS1X_DIR):
+    print("E: vyos-1x subdirectory does not exist, did you initialize submodules?")
+else:
+    sys.path.append(VYOS1X_DIR)
 
+# Import third-party modules
 try:
     import tomli
     import jinja2
     import git
+    import psutil
 except ModuleNotFoundError as e:
-    print(f"Cannot load a required library: {e}")
-    print("Please make sure the following Python3 modules are installed: tomli jinja2 git")
+    print(f"E: Cannot load required library {e}")
+    print("E: Please make sure the following Python3 modules are installed: tomli jinja2 git psutil")
 
-# Local modules
+# Import local modules from scripts/image-build
+# They rely on modules from vyos-1x
 import utils
 import defaults
+import raw_image
 
 # argparse converts hyphens to underscores,
 # so for lookups in the original options hash we have to convert them back
@@ -58,7 +69,7 @@ def get_validator(optdict, name):
     except KeyError:
         return None
 
-def merge_dicts(source, destination):
+def merge_dicts(source, destination, skip_none=False):
     """ Merge two dictionaries and return a new dict which has the merged key/value pairs.
     Merging logic is as follows:
       Sub-dicts are merged.
@@ -75,7 +86,7 @@ def merge_dicts(source, destination):
             tmp[key] = merge_dicts(source[key], tmp[key])
         elif isinstance(source[key], list):
             tmp[key] = source[key] + tmp[key]
-        else:
+        elif not skip_none or source[key] is not None:
             tmp[key] = source[key]
 
     return tmp
@@ -108,7 +119,10 @@ if __name__ == "__main__":
            'devscripts',
            'python3-pystache',
            'python3-git',
-           'qemu-utils'
+           'qemu-utils',
+           'gdisk',
+           'kpartx',
+           'dosfstools'
         ],
        'binaries': []
     }
@@ -117,7 +131,7 @@ if __name__ == "__main__":
     try:
         checker = utils.check_system_dependencies(deps)
     except OSError as e:
-        print(e)
+        print(f"E: {e}")
         sys.exit(1)
 
     ## Load the file with default build configuration options
@@ -125,11 +139,18 @@ if __name__ == "__main__":
         with open(defaults.DEFAULTS_FILE, 'rb') as f:
             build_defaults = tomli.load(f)
     except Exception as e:
-        print("Failed to open the defaults file {0}: {1}".format(defaults.DEFAULTS_FILE, e))
+        print("E: Failed to open the defaults file {0}: {1}".format(defaults.DEFAULTS_FILE, e))
         sys.exit(1)
 
     ## Get a list of available build flavors
-    build_flavors = list(map(lambda f: os.path.splitext(f)[0], os.listdir(defaults.BUILD_FLAVORS_DIR)))
+    flavor_dir_env = os.getenv("VYOS_BUILD_FLAVORS_DIR")
+    if flavor_dir_env:
+        flavor_dir = flavor_dir_env
+    else:
+        flavor_dir = defaults.BUILD_FLAVORS_DIR
+
+    print(f"I: using build flavors directory {flavor_dir}")
+    build_flavors = list(map(lambda f: os.path.splitext(f)[0], os.listdir(flavor_dir)))
 
     ## Set up the option parser
     ## XXX: It uses values from the default configuration file for its option defaults,
@@ -138,14 +159,13 @@ if __name__ == "__main__":
     # Options dict format:
     # '$option_name_without_leading_dashes': { ('$help_string', $default_value_generator_thunk, $value_checker_thunk) }
     options = {
-       'architecture': ('Image target architecture (amd64 or arm64)',
-          lambda: build_defaults['architecture'], lambda x: x in ['amd64', 'arm64']),
+       'architecture': ('Image target architecture (amd64 or arm64)', None, lambda x: x in ['amd64', 'arm64', None]),
        'build-by': ('Builder identifier (e.g. jrandomhacker@example.net)', get_default_build_by, None),
-       'debian-mirror': ('Debian repository mirror', lambda: build_defaults['debian_mirror'], None),
-       'debian-security-mirror': ('Debian security updates mirror', lambda: build_defaults['debian_security_mirror'], None),
-       'pbuilder-debian-mirror': ('Debian repository mirror for pbuilder env bootstrap', lambda: build_defaults['debian_mirror'], None),
-       'vyos-mirror': ('VyOS package mirror', lambda: build_defaults["vyos_mirror"], None),
-       'build-type': ('Build type, release or development', lambda: 'development', lambda x: x in ['release', 'development']),
+       'debian-mirror': ('Debian repository mirror', None, None),
+       'debian-security-mirror': ('Debian security updates mirror', None, None),
+       'pbuilder-debian-mirror': ('Debian repository mirror for pbuilder env bootstrap', None, None),
+       'vyos-mirror': ('VyOS package mirror', None, None),
+       'build-type': ('Build type, release or development', None, lambda x: x in ['release', 'development']),
        'version': ('Version number (release builds only)', None, None),
        'build-comment': ('Optional build comment', lambda: '', None)
     }
@@ -159,9 +179,8 @@ if __name__ == "__main__":
         else:
             parser.add_argument('--' + k, type=str, help=help_string, default=default_value_thunk())
 
-    # The debug option is a bit special since it different type is different
+    # Debug options
     parser.add_argument('--debug', help='Enable debug output', action='store_true')
-
     parser.add_argument('--dry-run', help='Check build configuration and exit', action='store_true')
 
     # Custom APT entry and APT key options can be used multiple times
@@ -169,6 +188,10 @@ if __name__ == "__main__":
     parser.add_argument('--custom-apt-key', help="Custom APT key file", action='append', default=[])
     parser.add_argument('--custom-package', help="Custom package to install from repositories", action='append', default=[])
 
+    # Options relevant for non-ISO format flavors
+    parser.add_argument('--reuse-iso', help='Use an existing ISO file to build additional image formats', type=str, action='store', default=None)
+    parser.add_argument('--disk-size', help='Disk size for non-ISO image formats', type=int, action='store', default=10)
+
     # Build flavor is a positional argument
     parser.add_argument('build_flavor', help='Build flavor', nargs='?', action='store')
 
@@ -182,7 +205,7 @@ if __name__ == "__main__":
         func = get_validator(options, k)
         if func is not None:
             if not func(v):
-                print("{v} is not a valid value for --{o} option".format(o=key, v=v))
+                print("E: {v} is not a valid value for --{o} option".format(o=key, v=v))
                 sys.exit(1)
 
     if not args["build_flavor"]:
@@ -198,7 +221,8 @@ if __name__ == "__main__":
     flavor_config = {}
     build_flavor = args["build_flavor"]
     try:
-        with open(make_toml_path(defaults.BUILD_FLAVORS_DIR, args["build_flavor"]), 'rb') as f:
+        toml_flavor_file = make_toml_path(flavor_dir, args["build_flavor"])
+        with open(toml_flavor_file, 'rb') as f:
             flavor_config = tomli.load(f)
             pre_build_config = merge_dicts(flavor_config, pre_build_config)
     except FileNotFoundError:
@@ -214,44 +238,40 @@ if __name__ == "__main__":
     # Some fixup for mirror settings.
     # The idea is: if --debian-mirror is specified but --pbuilder-debian-mirror is not,
     # use the --debian-mirror value for both lb and pbuilder bootstrap
-    if (args['debian_mirror'] != build_defaults["debian_mirror"]) and \
-       (args['pbuilder_debian_mirror'] == build_defaults["debian_mirror"]):
-        args['pbuilder_debian_mirror'] = args['debian_mirror']
+    if pre_build_config['debian_mirror'] is None or pre_build_config['debian_security_mirror'] is None:
+        print("E: debian_mirror and debian_security_mirror cannot be empty")
+        sys.exit(1)
+
+    if pre_build_config['pbuilder_debian_mirror'] is None:
+        args['pbuilder_debian_mirror'] = pre_build_config['pbuilder_debian_mirror'] = pre_build_config['debian_mirror']
 
     # Version can only be set for release builds,
     # for dev builds it hardly makes any sense
-    if args['build_type'] == 'development':
+    if pre_build_config['build_type'] == 'development':
         if args['version'] is not None:
-            print("Version can only be set for release builds")
+            print("E: Version can only be set for release builds")
             print("Use --build-type=release option if you want to set version number")
             sys.exit(1)
 
-    # Validate characters in version name
-    if 'version' in args and args['version'] != None:
-        allowed = string.ascii_letters + string.digits + '.' + '-' + '+'
-        if not set(args['version']) <= set(allowed):
-            print(f'Version contained illegal character(s), allowed: {allowed}')
-            sys.exit(1)
-
     ## Inject some useful hardcoded options
     args['build_dir'] = defaults.BUILD_DIR
     args['pbuilder_config'] = os.path.join(defaults.BUILD_DIR, defaults.PBUILDER_CONFIG)
 
     ## Combine the arguments with non-configurable defaults
-    build_config = merge_dicts(args, build_defaults)
+    build_config = merge_dicts({}, build_defaults)
 
-    ## Load the flavor file and mix-ins
-    with open(make_toml_path(defaults.BUILD_TYPES_DIR, build_config["build_type"]), 'rb') as f:
+    ## Load correct mix-ins
+    with open(make_toml_path(defaults.BUILD_TYPES_DIR, pre_build_config["build_type"]), 'rb') as f:
         build_type_config = tomli.load(f)
         build_config = merge_dicts(build_type_config, build_config)
 
-    with open(make_toml_path(defaults.BUILD_ARCHES_DIR, build_config["architecture"]), 'rb') as f:
+    with open(make_toml_path(defaults.BUILD_ARCHES_DIR, pre_build_config["architecture"]), 'rb') as f:
         build_arch_config = tomli.load(f)
         build_config = merge_dicts(build_arch_config, build_config)
 
-    with open(make_toml_path(defaults.BUILD_FLAVORS_DIR, build_config["build_flavor"]), 'rb') as f:
-        flavor_config = tomli.load(f)
-        build_config = merge_dicts(flavor_config, build_config)
+    ## Override with flavor and then CLI arguments
+    build_config = merge_dicts(flavor_config, build_config)
+    build_config = merge_dicts(args, build_config, skip_none=True)
 
     ## Rename and merge some fields for simplicity
     ## E.g. --custom-packages is for the user, but internally
@@ -264,7 +284,14 @@ if __name__ == "__main__":
     if has_nonempty_key(build_config, "architectures"):
         arch = build_config["architecture"]
         if arch in build_config["architectures"]:
-            build_config["packages"] += build_config["architectures"][arch]["packages"]
+            if has_nonempty_key(build_config["architectures"], "packages"):
+                build_config["packages"] += build_config["architectures"][arch]["packages"]
+
+    ## Add default boot settings if needed
+    if "boot_settings" not in build_config:
+        build_config["boot_settings"] = defaults.boot_settings
+    else:
+        build_config["boot_settings"] = merge_dicts(defaults.default_consolede, build_config["boot_settings"])
 
     ## Dump the complete config if the user enabled debug mode
     if debug:
@@ -274,269 +301,301 @@ if __name__ == "__main__":
 
     ## Clean up the old build config and set up a fresh copy
     lb_config_dir = os.path.join(defaults.BUILD_DIR, defaults.LB_CONFIG_DIR)
-    print(lb_config_dir)
     shutil.rmtree(lb_config_dir, ignore_errors=True)
     shutil.copytree("data/live-build-config/", lb_config_dir)
     os.makedirs(lb_config_dir, exist_ok=True)
 
-    ## Create the version file
-
-    # Create a build timestamp
-    now = datetime.datetime.today()
-    build_timestamp = now.strftime("%Y%m%d%H%M")
-
-    # FIXME: use aware rather than naive object
-    build_date = now.strftime("%a %d %b %Y %H:%M UTC")
+    # Switch to the build directory, this is crucial for the live-build work
+    # because the efective build config files etc. are there.
+    #
+    # All directory paths from this point must be relative to BUILD_DIR,
+    # not to the vyos-build repository root.
+    os.chdir(defaults.BUILD_DIR)
 
-    # Assign a (hopefully) unique identifier to the build (UUID)
-    build_uuid = str(uuid.uuid4())
+    iso_file = None
 
-    # Initialize Git object from our repository
-    try:
-        repo = git.Repo('.')
-
-        # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient
-        build_git = repo.head.object.hexsha[:14]
-        # If somone played around with the source tree and the build is "dirty", mark it
-        if repo.is_dirty():
-            build_git += "-dirty"
-
-        # Retrieve git branch name or current tag
-        # Building a tagged release might leave us checking out a git tag that is not the tip of a named branch (detached HEAD)
-        # Check if the current HEAD is associated with a tag and use its name instead of an unavailable branch name.
-        git_branch = next((tag.name for tag in repo.tags if tag.commit == repo.head.commit), None)
-        if git_branch is None:
-            git_branch = repo.active_branch.name
-    except Exception as e:
-      exit(f'Could not retrieve information from git: {e}')
-      build_git = ""
-      git_branch = ""
+    if build_config["reuse_iso"]:
+        iso_file = build_config["reuse_iso"]
+    else:
+        ## Create the version file
 
-    # Create the build version string
-    if build_config['build_type'] == 'development':
-        try:
-            if not git_branch:
-                raise ValueError("git branch could not be determined")
+        # Create a build timestamp
+        now = datetime.datetime.today()
+        build_timestamp = now.strftime("%Y%m%d%H%M")
 
-            # Load the branch to version mapping file
-            with open('data/versions') as f:
-                version_mapping = json.load(f)
+        # FIXME: use aware rather than naive object
+        build_date = now.strftime("%a %d %b %Y %H:%M UTC")
 
-            branch_version = version_mapping[git_branch]
+        # Assign a (hopefully) unique identifier to the build (UUID)
+        build_uuid = str(uuid.uuid4())
 
-            version = "{0}-rolling-{1}".format(branch_version, build_timestamp)
+        # Initialize Git object from our repository
+        try:
+            repo = git.Repo('.')
+
+            # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient
+            build_git = repo.head.object.hexsha[:14]
+            # If somone played around with the source tree and the build is "dirty", mark it
+            if repo.is_dirty():
+                build_git += "-dirty"
+
+            # Retrieve git branch name or current tag
+            # Building a tagged release might leave us checking out a git tag that is not the tip of a named branch (detached HEAD)
+            # Check if the current HEAD is associated with a tag and use its name instead of an unavailable branch name.
+            git_branch = next((tag.name for tag in repo.tags if tag.commit == repo.head.commit), None)
+            if git_branch is None:
+                git_branch = repo.active_branch.name
         except Exception as e:
-            print("Could not build a version string specific to git branch, falling back to default: {0}".format(str(e)))
-            version = "999.{0}".format(build_timestamp)
-    else:
-        # Release build, use the version from ./configure arguments
-        version = build_config['version']
+          exit(f'Could not retrieve information from git: {e}')
+          build_git = ""
+          git_branch = ""
+
+        # Create the build version string
+        if build_config['build_type'] == 'development':
+            try:
+                if not git_branch:
+                    raise ValueError("git branch could not be determined")
+
+                # Load the branch to version mapping file
+                with open('data/versions') as f:
+                    version_mapping = json.load(f)
+
+                branch_version = version_mapping[git_branch]
+
+                version = "{0}-rolling-{1}".format(branch_version, build_timestamp)
+            except Exception as e:
+                print("W: Could not build a version string specific to git branch, falling back to default: {0}".format(str(e)))
+                version = "999.{0}".format(build_timestamp)
+        else:
+            # Release build, use the version from ./configure arguments
+            version = build_config['version']
 
-    if build_config['build_type'] == 'development':
-        lts_build = False
-    else:
-        lts_build = True
-
-    version_data = {
-        'version': version,
-        'built_by': build_config['build_by'],
-        'built_on': build_date,
-        'build_uuid': build_uuid,
-        'build_git': build_git,
-        'build_branch': git_branch,
-        'release_train': build_config['release_train'],
-        'lts_build': lts_build,
-        'build_comment': build_config['build_comment'],
-        'bugtracker_url': build_config['bugtracker_url'],
-        'documentation_url': build_config['documentation_url'],
-        'project_news_url': build_config['project_news_url'],
-    }
+        if build_config['build_type'] == 'development':
+            lts_build = False
+        else:
+            lts_build = True
+
+        version_data = {
+            'version': version,
+            'built_by': build_config['build_by'],
+            'built_on': build_date,
+            'build_uuid': build_uuid,
+            'build_git': build_git,
+            'build_branch': git_branch,
+            'release_train': build_config['release_train'],
+            'lts_build': lts_build,
+            'build_comment': build_config['build_comment'],
+            'bugtracker_url': build_config['bugtracker_url'],
+            'documentation_url': build_config['documentation_url'],
+            'project_news_url': build_config['project_news_url'],
+        }
+
+        # Multi line strings needs to be un-indented to not have leading
+        # whitespaces in the resulting file
+        os_release = f"""
+    PRETTY_NAME="VyOS {version} ({build_config['release_train']})"
+    NAME="VyOS"
+    VERSION_ID="{version}"
+    VERSION="{version} ({build_config['release_train']})"
+    VERSION_CODENAME={build_defaults['debian_distribution']}
+    ID=vyos
+    BUILD_ID="{build_git}"
+    HOME_URL="{build_defaults['website_url']}"
+    SUPPORT_URL="{build_defaults['support_url']}"
+    BUG_REPORT_URL="{build_defaults['bugtracker_url']}"
+    DOCUMENTATION_URL="{build_config['documentation_url']}"
+        """
+
+        # Reminder: all paths relative to the build dir, not to the repository root
+        chroot_includes_dir = defaults.CHROOT_INCLUDES_DIR
+        binary_includes_dir = defaults.BINARY_INCLUDES_DIR
+        vyos_data_dir = os.path.join(chroot_includes_dir, "usr/share/vyos")
+        os.makedirs(vyos_data_dir, exist_ok=True)
+        with open(os.path.join(vyos_data_dir, 'version.json'), 'w') as f:
+            json.dump(version_data, f)
+        with open(os.path.join(binary_includes_dir, 'version.json'), 'w') as f:
+            json.dump(version_data, f)
+
+        # For backwards compatibility with 'add system image' script from older versions
+        # we need a file in the old format so that script can find out the version of the image
+        # for upgrade
+        os.makedirs(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/'), exist_ok=True)
+        with open(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/version'), 'w') as f:
+            print("Version: {0}".format(version), file=f)
+
+        # Define variables that influence to welcome message on boot
+        os.makedirs(os.path.join(chroot_includes_dir, 'usr/lib/'), exist_ok=True)
+        with open(os.path.join(chroot_includes_dir, 'usr/lib/os-release'), 'w') as f:
+            print(os_release, file=f)
+
+        ## Clean up earlier build state and artifacts
+        print("I: Cleaning the build workspace")
+        os.system("lb clean")
+        #iter(lambda p: shutil.rmtree(p, ignore_errors=True),
+        #  ['config/binary', 'config/bootstrap', 'config/chroot', 'config/common', 'config/source'])
+        artifacts = functools.reduce(
+          lambda x, y: x + y,
+          map(glob.glob, ['*.iso', '*.raw', '*.img', '*.xz', '*.ova', '*.ovf']))
+        iter(os.remove, artifacts)
+
+        ## Create the target ISO file path
+        iso_file = "vyos-{0}-{1}.iso".format(version_data["version"], build_config["architecture"])
+
+        ## Create live-build configuration files
+
+        # Add the additional repositories to package lists
+        print("I: Setting up additional APT entries")
+        vyos_repo_entry = "deb {vyos_mirror} {vyos_branch} main\n".format(**build_config)
+
+        apt_file = defaults.VYOS_REPO_FILE
 
-    # Multi line strings needs to be un-indented to not have leading
-    # whitespaces in the resulting file
-    os_release = f"""
-PRETTY_NAME="VyOS {version} ({build_config['release_train']})"
-NAME="VyOS"
-VERSION_ID="{version}"
-VERSION="{version} ({build_config['release_train']})"
-VERSION_CODENAME={build_defaults['debian_distribution']}
-ID=vyos
-BUILD_ID="{build_git}"
-HOME_URL="{build_defaults['website_url']}"
-SUPPORT_URL="{build_defaults['support_url']}"
-BUG_REPORT_URL="{build_defaults['bugtracker_url']}"
-DOCUMENTATION_URL="{build_config['documentation_url']}"
+        if debug:
+            print(f"D: Adding these entries to {apt_file}:")
+            print("\t", vyos_repo_entry)
+
+        with open(apt_file, 'w') as f:
+            f.write(vyos_repo_entry)
+
+        # Add custom APT entries
+        if build_config.get('additional_repositories', False):
+            build_config['custom_apt_entry'] += build_config['additional_repositories']
+
+        if build_config.get('custom_apt_entry', False):
+            custom_apt_file = defaults.CUSTOM_REPO_FILE
+            entries = "\n".join(build_config['custom_apt_entry'])
+            if debug:
+                print("D: Adding custom APT entries:")
+                print(entries)
+            with open(custom_apt_file, 'w') as f:
+                f.write(entries)
+                f.write("\n")
+
+        # Add custom APT keys
+        if has_nonempty_key(build_config, 'custom_apt_key'):
+            key_dir = defaults.ARCHIVES_DIR
+            for k in build_config['custom_apt_key']:
+                dst_name = '{0}.key.chroot'.format(os.path.basename(k))
+                shutil.copy(k, os.path.join(key_dir, dst_name))
+
+        # Add custom packages
+        if has_nonempty_key(build_config, 'packages'):
+            package_list_file = defaults.PACKAGE_LIST_FILE
+            packages = "\n".join(build_config['packages'])
+            with open (package_list_file, 'w') as f:
+                f.write(packages)
+
+        ## Create includes
+        if has_nonempty_key(build_config, "includes_chroot"):
+            for i in build_config["includes_chroot"]:
+                file_path = os.path.join(chroot_includes_dir, i["path"])
+                if debug:
+                    print(f"D: Creating chroot include file: {file_path}")
+                os.makedirs(os.path.dirname(file_path), exist_ok=True)
+                with open(file_path, 'w') as f:
+                    f.write(i["data"])
+
+        ## Create the default config
+        ## Technically it's just another includes.chroot entry,
+        ## but it's special enough to warrant making it easier for flavor writers
+        if has_nonempty_key(build_config, "default_config"):
+            file_path = os.path.join(chroot_includes_dir, "opt/vyatta/etc/config.boot.default")
+            os.makedirs(os.path.dirname(file_path), exist_ok=True)
+            with open(file_path, 'w') as f:
+                f.write(build_config["default_config"])
+
+        ## Configure live-build
+        lb_config_tmpl = jinja2.Template("""
+        lb config noauto \
+                --apt-indices false \
+                --apt-options "--yes -oAPT::Get::allow-downgrades=true" \
+                --apt-recommends false \
+                --architecture {{architecture}} \
+                --archive-areas {{debian_archive_areas}} \
+                --backports true \
+                --binary-image iso-hybrid \
+                --bootappend-live "boot=live components hostname=vyos username=live nopersistence noautologin nonetworking union=overlay console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \
+                --bootappend-live-failsafe "live components memtest noapic noapm nodma nomce nolapic nomodeset nosmp nosplash vga=normal console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \
+                --bootloaders {{bootloaders}} \
+                --checksums 'sha256 md5' \
+                --chroot-squashfs-compression-type "{{squashfs_compression_type}}" \
+                --debian-installer none \
+                --debootstrap-options "--variant=minbase --exclude=isc-dhcp-client,isc-dhcp-common,ifupdown --include=apt-utils,ca-certificates,gnupg2" \
+                --distribution {{debian_distribution}} \
+                --firmware-binary false \
+                --firmware-chroot false \
+                --iso-application "VyOS" \
+                --iso-publisher "{{build_by}}" \
+                --iso-volume "VyOS" \
+                --linux-flavours {{kernel_flavor}} \
+                --linux-packages linux-image-{{kernel_version}} \
+                --mirror-binary {{debian_mirror}} \
+                --mirror-binary-security {{debian_security_mirror}} \
+                --mirror-bootstrap {{debian_mirror}} \
+                --mirror-chroot {{debian_mirror}} \
+                --mirror-chroot-security {{debian_security_mirror}} \
+                --security true \
+                --updates true
+                "${@}"
+        """)
+
+        lb_config_command = lb_config_tmpl.render(build_config)
+
+        ## Pin release for VyOS packages
+        apt_pin = f"""Package: *
+    Pin: release n={build_config['release_train']}
+    Pin-Priority: 600
     """
 
-    chroot_includes_dir = os.path.join(defaults.BUILD_DIR, defaults.CHROOT_INCLUDES_DIR)
-    binary_includes_dir = os.path.join(defaults.BUILD_DIR, defaults.BINARY_INCLUDES_DIR)
-    vyos_data_dir = os.path.join(chroot_includes_dir, "usr/share/vyos")
-    os.makedirs(vyos_data_dir, exist_ok=True)
-    with open(os.path.join(vyos_data_dir, 'version.json'), 'w') as f:
-        json.dump(version_data, f)
-    with open(os.path.join(binary_includes_dir, 'version.json'), 'w') as f:
-        json.dump(version_data, f)
-
-    # For backwards compatibility with 'add system image' script from older versions
-    # we need a file in the old format so that script can find out the version of the image
-    # for upgrade
-    os.makedirs(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/'), exist_ok=True)
-    with open(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/version'), 'w') as f:
-        print("Version: {0}".format(version), file=f)
-
+        with open(defaults.VYOS_PIN_FILE, 'w') as f:
+            f.write(apt_pin)
 
-    # Define variables that influence to welcome message on boot
-    os.makedirs(os.path.join(chroot_includes_dir, 'usr/lib/'), exist_ok=True)
-    with open(os.path.join(chroot_includes_dir, 'usr/lib/os-release'), 'w') as f:
-        print(os_release, file=f)
+        print("I: Configuring live-build")
 
-
-    ## Clean up earlier build state and artifacts
-    print("I: Cleaning the build workspace")
-    os.system("lb clean")
-    #iter(lambda p: shutil.rmtree(p, ignore_errors=True),
-    #  ['config/binary', 'config/bootstrap', 'config/chroot', 'config/common', 'config/source'])
-    artifacts = functools.reduce(
-      lambda x, y: x + y,
-      map(glob.glob, ['*.iso', '*.raw', '*.img', '*.xz', '*.ova', '*.ovf']))
-    iter(os.remove, artifacts)
-
-    ## Create live-build configuration files
-
-    # Add the additional repositories to package lists
-    print("I: Setting up additional APT entries")
-    vyos_repo_entry = "deb {vyos_mirror} {vyos_branch} main\n".format(**build_config)
-
-    apt_file = defaults.VYOS_REPO_FILE
-
-    if debug:
-        print(f"D: Adding these entries to {apt_file}:")
-        print("\t", vyos_repo_entry)
-
-    with open(apt_file, 'w') as f:
-        f.write(vyos_repo_entry)
-
-    # Add custom APT entries
-    if build_config.get('additional_repositories', False):
-        build_config['custom_apt_entry'] += build_config['additional_repositories']
-
-    if build_config.get('custom_apt_entry', False):
-        custom_apt_file = defaults.CUSTOM_REPO_FILE
-        entries = "\n".join(build_config['custom_apt_entry'])
         if debug:
-            print("D: Adding custom APT entries:")
-            print(entries)
-        with open(custom_apt_file, 'w') as f:
-            f.write(entries)
-            f.write("\n")
-
-    # Add custom APT keys
-    if has_nonempty_key(build_config, 'custom_apt_key'):
-        key_dir = defaults.ARCHIVES_DIR
-        for k in build_config['custom_apt_key']:
-            dst_name = '{0}.key.chroot'.format(os.path.basename(k))
-            shutil.copy(k, os.path.join(key_dir, dst_name))
-
-    # Add custom packages
-    if has_nonempty_key(build_config, 'packages'):
-        package_list_file = defaults.PACKAGE_LIST_FILE
-        packages = "\n".join(build_config['packages'])
-        with open (package_list_file, 'w') as f:
-            f.write(packages)
-
-    ## Create includes
-    if has_nonempty_key(build_config, "includes_chroot"):
-        for i in build_config["includes_chroot"]:
-            file_path = os.path.join(chroot_includes_dir, i["path"])
-            os.makedirs(os.path.dirname(file_path), exist_ok=True)
-            with open(file_path, 'w') as f:
-                f.write(i["data"])
-
-    ## Create the default config
-    ## Technically it's just another includes.chroot entry,
-    ## but it's special enough to warrant making it easier for flavor writers
-    if has_nonempty_key(build_config, "default_config"):
-        file_path = os.path.join(chroot_includes_dir, "opt/vyatta/etc/config.boot.default")
-        os.makedirs(os.path.dirname(file_path), exist_ok=True)
-        with open(file_path, 'w') as f:
-            f.write(build_config["default_config"])
-
-    ## Configure live-build
-    lb_config_tmpl = jinja2.Template("""
-    lb config noauto \
-            --apt-indices false \
-            --apt-options "--yes -oAPT::Get::allow-downgrades=true" \
-            --apt-recommends false \
-            --architecture {{architecture}} \
-            --archive-areas {{debian_archive_areas}} \
-            --backports true \
-            --binary-image iso-hybrid \
-            --bootappend-live "boot=live components hostname=vyos username=live nopersistence noautologin nonetworking union=overlay console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \
-            --bootappend-live-failsafe "live components memtest noapic noapm nodma nomce nolapic nomodeset nosmp nosplash vga=normal console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \
-            --bootloaders {{bootloaders}} \
-            --checksums 'sha256 md5' \
-            --chroot-squashfs-compression-type "{{squashfs_compression_type}}" \
-            --debian-installer none \
-            --debootstrap-options "--variant=minbase --exclude=isc-dhcp-client,isc-dhcp-common,ifupdown --include=apt-utils,ca-certificates,gnupg2" \
-            --distribution {{debian_distribution}} \
-            --firmware-binary false \
-            --firmware-chroot false \
-            --iso-application "VyOS" \
-            --iso-publisher "{{build_by}}" \
-            --iso-volume "VyOS" \
-            --linux-flavours {{kernel_flavor}} \
-            --linux-packages linux-image-{{kernel_version}} \
-            --mirror-binary {{debian_mirror}} \
-            --mirror-binary-security {{debian_security_mirror}} \
-            --mirror-bootstrap {{debian_mirror}} \
-            --mirror-chroot {{debian_mirror}} \
-            --mirror-chroot-security {{debian_security_mirror}} \
-            --security true \
-            --updates true
-            "${@}"
-    """)
-
-    lb_config_command = lb_config_tmpl.render(build_config)
-
-    ## Pin release for VyOS packages
-    apt_pin = f"""Package: *
-Pin: release n={build_config['release_train']}
-Pin-Priority: 600
-"""
-
-    with open(defaults.VYOS_PIN_FILE, 'w') as f:
-        f.write(apt_pin)
-
-    print("I: Configuring live-build")
+            print("D: live-build configuration command")
+            print(lb_config_command)
 
-    if debug:
-        print("D: live-build configuration command")
-        print(lb_config_command)
-
-    result = os.system(lb_config_command)
-    if result > 0:
-        print("E: live-build config failed")
-        sys.exit(1)
+        result = os.system(lb_config_command)
+        if result > 0:
+            print("E: live-build config failed")
+            sys.exit(1)
 
-    ## In dry-run mode, exit at this point
-    if build_config["dry_run"]:
-        print("I: dry-run, not starting image build")
-        sys.exit(0)
+        ## In dry-run mode, exit at this point
+        if build_config["dry_run"]:
+            print("I: dry-run, not starting image build")
+            sys.exit(0)
 
-    ## Add local packages
-    local_packages = glob.glob('../packages/*.deb')
-    if local_packages:
-        for f in local_packages:
-            shutil.copy(f, os.path.join(defaults.LOCAL_PACKAGES_PATH, os.path.basename(f)))
+        ## Add local packages
+        local_packages = glob.glob('../packages/*.deb')
+        if local_packages:
+            for f in local_packages:
+                shutil.copy(f, os.path.join(defaults.LOCAL_PACKAGES_PATH, os.path.basename(f)))
 
-    ## Build the image
-    print("I: Starting image build")
-    if debug:
-        print("D: It's not like I'm building this specially for you or anything!")
-    res = os.system("lb build 2>&1")
-    if res > 0:
-        sys.exit(res)
-
-    # Copy the image
-    shutil.copy("live-image-{0}.hybrid.iso".format(build_config["architecture"]),
-      "vyos-{0}-{1}.iso".format(version_data["version"], build_config["architecture"]))
+        ## Build the image
+        print("I: Starting image build")
+        if debug:
+            print("D: It's not like I'm building this specially for you or anything!")
+        res = os.system("lb build 2>&1")
+        if res > 0:
+            sys.exit(res)
+
+        # Copy the image
+        shutil.copy("live-image-{0}.hybrid.iso".format(build_config["architecture"]), iso_file)
+
+    # Build additional flavors from the ISO,
+    # if the flavor calls for them
+    if build_config["image_format"] != ["iso"]:
+        raw_image = raw_image.create_raw_image(build_config, iso_file, "tmp/")
+
+        other_formats = filter(lambda x: x not in ["iso", "raw"], build_config["image_format"])
+        for f in other_formats:
+            target = f"{os.path.splitext(raw_image)[0]}.{f}"
+            print(f"I: building {f} file {target}")
+            os.system(f"qemu-img convert -f raw -O {f} {raw_image} {target}")
+
+        # Some flavors require special procedures that aren't covered by qemu-img
+        # (most notable, the VMware OVA that requires a custom tool to make and sign the image).
+        # Such procedures are executed as post-build hooks.
+        if has_nonempty_key(build_config, "post_build_hook"):
+            hook_path = build_config["post_build_hook"]
+            os.system(f"{hook_path} {raw_image}")
diff --git a/scripts/image-build/defaults.py b/scripts/image-build/defaults.py
index 1d7141ea..9fd5eeed 100644
--- a/scripts/image-build/defaults.py
+++ b/scripts/image-build/defaults.py
@@ -18,6 +18,15 @@
 
 import os
 
+# Default boot settings
+boot_settings: dict[str, str] = {
+    'timeout': '5',
+    'console_type': 'tty',
+    'console_num': '0',
+    'console_speed': '115200',
+    'bootmode': 'normal'
+}
+
 # Relative to the repository directory
 
 BUILD_DIR = 'build'
diff --git a/scripts/image-build/raw_image.py b/scripts/image-build/raw_image.py
new file mode 100644
index 00000000..67039a24
--- /dev/null
+++ b/scripts/image-build/raw_image.py
@@ -0,0 +1,215 @@
+# Copyright (C) 2024 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# File: raw_image.py
+# Purpose: Helper functions for building raw images.
+
+import os
+import sys
+import shutil
+import traceback
+
+import vyos.utils.process
+
+SQUASHFS_FILE = 'live/filesystem.squashfs'
+VERSION_FILE = 'version.json'
+
+
+def cmd(command):
+    res = vyos.utils.process.call(command, shell=True)
+    if res > 0:
+        raise OSError(f"Command '{command}' failed")
+
+def mkdir(path):
+    os.makedirs(path, exist_ok=True)
+
+
+class BuildContext:
+    def __init__(self, iso_path, work_dir, debug=False):
+        self.work_dir = work_dir
+        self.iso_path = iso_path
+        self.debug = debug
+        self.loop_device = None
+
+    def __enter__(self):
+        print(f"I: Setting up a raw image build directory in {self.work_dir}")
+
+        self.iso_dir = os.path.join(self.work_dir, "iso")
+        self.squash_dir = os.path.join(self.work_dir, "squash")
+        self.raw_dir = os.path.join(self.work_dir, "raw")
+        self.efi_dir = os.path.join(self.work_dir, "efi")
+
+        # Create mount point directories
+        mkdir(self.iso_dir)
+        mkdir(self.squash_dir)
+        mkdir(self.raw_dir)
+        mkdir(self.efi_dir)
+
+        # Mount the ISO image
+        cmd(f"""mount -t iso9660 -o ro,loop {self.iso_path} {self.iso_dir}""")
+
+        # Mount the SquashFS image
+        cmd(f"""mount -t squashfs -o ro,loop {self.iso_dir}/{SQUASHFS_FILE} {self.squash_dir}""")
+
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        print(f"I: Tearing down the raw image build environment in {self.work_dir}")
+        cmd(f"""umount {self.squash_dir}/dev/""")
+        cmd(f"""umount {self.squash_dir}/proc/""")
+        cmd(f"""umount {self.squash_dir}/sys/""")
+
+        cmd(f"umount {self.squash_dir}/boot/efi")
+        cmd(f"umount {self.squash_dir}/boot")
+
+        cmd(f"""umount {self.squash_dir}""")
+        cmd(f"""umount {self.iso_dir}""")
+        cmd(f"""umount {self.raw_dir}""")
+        cmd(f"""umount {self.efi_dir}""")
+
+        if self.loop_device:
+            cmd(f"""losetup -d {self.loop_device}""")
+
+def create_disk(path, size):
+    cmd(f"""qemu-img create -f raw "{path}" {size}G""")
+
+def read_version_data(iso_dir):
+    from json import load
+    with open(os.path.join(iso_dir, VERSION_FILE), 'r') as f:
+        data = load(f)
+    return data
+
+def setup_loop_device(con, raw_file):
+    from subprocess import Popen, PIPE, STDOUT
+    from re import match
+    command = f'losetup --show -f {raw_file}'
+    p = Popen(command, stderr=PIPE, stdout=PIPE, stdin=PIPE, shell=True)
+    (stdout, stderr) = p.communicate()
+
+    if p.returncode > 0:
+        raise OSError(f"Could not set up a loop device: {stderr.decode()}")
+
+    con.loop_device = stdout.decode().strip()
+    if con.debug:
+        print(f"I: Using loop device {con.loop_device}")
+
+def mount_image(con):
+    import vyos.system.disk
+
+    from subprocess import Popen, PIPE, STDOUT
+    from re import match
+
+    vyos.system.disk.filesystem_create(con.disk_details.partition['efi'], 'efi')
+    vyos.system.disk.filesystem_create(con.disk_details.partition['root'], 'ext4')
+
+    cmd(f"mount -t ext4 {con.disk_details.partition['root']} {con.raw_dir}")
+    cmd(f"mount -t vfat {con.disk_details.partition['efi']} {con.efi_dir}")
+
+def install_image(con, version):
+    from glob import glob
+
+    vyos_dir = os.path.join(con.raw_dir, f'boot/{version}/')
+    mkdir(vyos_dir)
+    mkdir(os.path.join(vyos_dir, 'work/work'))
+    mkdir(os.path.join(vyos_dir, 'rw'))
+
+    shutil.copy(f"{con.iso_dir}/{SQUASHFS_FILE}", f"{vyos_dir}/{version}.squashfs")
+
+    boot_files = glob(f'{con.squash_dir}/boot/*')
+    boot_files = [f for f in boot_files if os.path.isfile(f)]
+
+    for f in boot_files:
+        print(f"I: Copying file {f}")
+        shutil.copy(f, vyos_dir)
+
+    with open(f"{con.raw_dir}/persistence.conf", 'w') as f:
+        f.write("/ union\n")
+
+def setup_grub_configuration(build_config, root_dir) -> None:
+    """Install GRUB configurations
+
+    Args:
+        root_dir (str): a path to the root of target filesystem
+    """
+    from vyos.template import render
+    from vyos.system import grub
+
+    print('I: Installing GRUB configuration files')
+    grub_cfg_main = f'{root_dir}/{grub.GRUB_DIR_MAIN}/grub.cfg'
+    grub_cfg_vars = f'{root_dir}/{grub.CFG_VYOS_VARS}'
+    grub_cfg_modules = f'{root_dir}/{grub.CFG_VYOS_MODULES}'
+    grub_cfg_menu = f'{root_dir}/{grub.CFG_VYOS_MENU}'
+    grub_cfg_options = f'{root_dir}/{grub.CFG_VYOS_OPTIONS}'
+
+    # create new files
+    render(grub_cfg_main, grub.TMPL_GRUB_MAIN, {})
+    grub.common_write(root_dir)
+    grub.vars_write(grub_cfg_vars, build_config["boot_settings"])
+    grub.modules_write(grub_cfg_modules, [])
+    grub.write_cfg_ver(1, root_dir)
+    render(grub_cfg_menu, grub.TMPL_GRUB_MENU, {})
+    render(grub_cfg_options, grub.TMPL_GRUB_OPTS, {})
+
+def install_grub(con, version):
+    from re import match
+    from vyos.system import disk, grub
+
+    # Mount the required virtual filesystems
+    os.makedirs(f"{con.raw_dir}/boot/efi", exist_ok=True)
+    cmd(f"mount --bind /dev {con.squash_dir}/dev")
+    cmd(f"mount --bind /proc {con.squash_dir}/proc")
+    cmd(f"mount --bind /sys {con.squash_dir}/sys")
+
+    cmd(f"mount --bind {con.raw_dir}/boot {con.squash_dir}/boot")
+    cmd(f"mount --bind {con.efi_dir} {con.squash_dir}/boot/efi")
+
+    DIR_DST_ROOT = con.raw_dir
+
+    setup_grub_configuration(con.build_config, DIR_DST_ROOT)
+    # add information about version
+    grub.create_structure(DIR_DST_ROOT)
+    grub.version_add(version, DIR_DST_ROOT)
+    grub.set_default(version, DIR_DST_ROOT)
+    grub.set_console_type(con.build_config["boot_settings"]["console_type"], DIR_DST_ROOT)
+
+    print('I: Installing GRUB to the disk image')
+    grub.install(con.loop_device, f'/boot/', f'/boot/efi', chroot=con.squash_dir)
+
+    # sort inodes (to make GRUB read config files in alphabetical order)
+    grub.sort_inodes(f'{DIR_DST_ROOT}/{grub.GRUB_DIR_VYOS}')
+    grub.sort_inodes(f'{DIR_DST_ROOT}/{grub.GRUB_DIR_VYOS_VERS}')
+
+
+def create_raw_image(build_config, iso_file, work_dir):
+    from vyos.system.disk import parttable_create
+
+    if not os.path.exists(iso_file):
+        print(f"E: ISO file {iso_file} does not exist in the build directory")
+        sys.exit(1)
+
+    with BuildContext(iso_file, work_dir, debug=True) as con:
+        con.build_config = build_config
+        version_data = read_version_data(con.iso_dir)
+        version = version_data['version']
+        raw_file = f"vyos-{version}.raw"
+        print(f"I: Building raw file {raw_file}")
+        create_disk(raw_file, build_config["disk_size"])
+        setup_loop_device(con, raw_file)
+        disk_details = parttable_create(con.loop_device, (int(build_config["disk_size"]) - 1) * 1024 * 1024)
+        con.disk_details = disk_details
+        mount_image(con)
+        install_image(con, version)
+        install_grub(con, version)
+
+        return raw_file