diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/build-config | 168 | ||||
| -rwxr-xr-x | scripts/build-flavour | 37 | ||||
| -rwxr-xr-x | scripts/build-vyos-image | 485 | ||||
| -rwxr-xr-x | scripts/check-config | 38 | ||||
| -rwxr-xr-x | scripts/copy-image | 7 | ||||
| -rwxr-xr-x | scripts/live-build-config | 123 | ||||
| -rwxr-xr-x | scripts/query-json | 42 | ||||
| -rw-r--r-- | scripts/vyos_build_defaults.py (renamed from scripts/defaults.py) | 23 | ||||
| -rw-r--r-- | scripts/vyos_build_utils.py (renamed from scripts/util.py) | 20 |
9 files changed, 517 insertions, 426 deletions
diff --git a/scripts/build-config b/scripts/build-config deleted file mode 100755 index 4f4a8c08..00000000 --- a/scripts/build-config +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2019, VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -# -# File: build-config -# Purpose: -# This script serves the same purpose as ./configure in traditional -# autoconf setups. -# It takes build configuration options from command line, checks them, -# builds a config dictionary, augments it with some default and/or -# computed values and saves it to build/build-config.json -# for other build scripts to read. - -import argparse -import re -import sys -import os -import getpass -import platform -import json - -import defaults - -# argparse converts hyphens to underscores, -# so for lookups in the original options hash we have to -# convert them back -def field_to_option(s): - return re.sub(r'_', '-', s) - -def get_default_build_by(): - return "{user}@{host}".format(user= getpass.getuser(), host=platform.node()) - -def get_validator(optdict, name): - try: - return optdict[name][2] - except KeyError: - return None - -def load_config(filename): - with open(filename, 'r') as f: - print(f'Loading {filename}') - this_config = json.load(f) - - if not 'inherit_from' in this_config: - print(f'No inheritance detected') - return this_config - - inherited_config = load_config(this_config['inherit_from']) - del this_config['inherit_from'] - inherited_config.update(this_config) - return inherited_config - - -# Load the build flavor file -build_flavor = os.getenv('VYOS_BUILD_FLAVOR') -if build_flavor is None: - build_flavor = defaults.DEFAULT_BUILD_FLAVOR -try: - build_defaults = load_config(build_flavor) -except Exception as e: - print("Failed to open the build flavor file {0}: {1}".format(build_flavor, e)) - sys.exit(1) - - -# Options dict format: -# '$option_name_without_leading_dashes': { ('$help_string', $default_value_generator_thunk, $value_checker_thunk) } -options = { - 'architecture': ('Image target architecture (amd64 or i386 or armhf or arm64)', lambda: build_defaults['architecture'], lambda x: x in ['amd64', 'i386', 'armhf', 'arm64']), - 'build-by': ('Builder identifier (e.g. jrandomhacker@example.net)', get_default_build_by, None), - 'debian-mirror': ('Debian repository mirror for ISO build', lambda: build_defaults['debian_mirror'], None), - 'debian-security-mirror': ('Debian security updates mirror', lambda: build_defaults['debian_security_mirror'], None), - 'pbuilder-debian-mirror': ('Debian repository mirror for pbuilder env bootstrap', lambda: build_defaults['debian_mirror'], None), - 'vyos-mirror': ('VyOS package mirror', lambda: build_defaults["vyos_mirror"], None), - 'build-type': ('Build type, release or development', lambda: 'development', lambda x: x in ['release', 'development']), - 'version': ('Version number (release builds only)', None, None), - 'build-comment': ('Optional build comment', lambda: '', None) -} - -# Create the option parser -parser = argparse.ArgumentParser() -for k, v in options.items(): - help_string, default_value_thunk = v[0], v[1] - if default_value_thunk is None: - parser.add_argument('--' + k, type=str, help=help_string) - else: - parser.add_argument('--' + k, type=str, help=help_string, default=default_value_thunk()) - -# The debug option is a bit special since it's different type -parser.add_argument('--debug', help="Enable debug output", action='store_true') - -# Custom APT entry and APT key options can be used multiple times -parser.add_argument('--custom-apt-entry', help="Custom APT entry", action='append') -parser.add_argument('--custom-apt-key', help="Custom APT key file", action='append') -parser.add_argument('--custom-package', help="Custom package to install from repositories", action='append') - -args = vars(parser.parse_args()) - -# Validate options -for k, v in args.items(): - key = field_to_option(k) - func = get_validator(options, k) - if func is not None: - if not func(v): - print("{v} is not a valid value for --{o} option".format(o=key, v=v)) - sys.exit(1) - -# Some fixup for mirror settings. -# The idea is: if --debian-mirror is specified but --pbuilder-debian-mirror is not, -# use the --debian-mirror value for both lb and pbuilder bootstrap -if (args['debian_mirror'] != build_defaults["debian_mirror"]) and \ - (args['pbuilder_debian_mirror'] == build_defaults["debian_mirror"]): - args['pbuilder_debian_mirror'] = args['debian_mirror'] - -# Version can only be set for release builds, -# for dev builds it hardly makes any sense -if args['build_type'] == 'development': - if args['version'] is not None: - print("Version can only be set for release builds") - print("Use --build-type=release option if you want to set version number") - sys.exit(1) - -# Populate some defaults that are not configurable, -# but that are handy to have in the options hash -args['distribution'] = build_defaults["debian_distribution"] -args['build_dir'] = defaults.BUILD_DIR -args['pbuilder_config'] = defaults.PBUILDER_CONFIG -args['vyos_branch'] = build_defaults["vyos_branch"] -args['release_train'] = build_defaults["release_train"] - -# Add custom packages from build defaults -if not args['custom_package']: - args['custom_package'] = [] -args['custom_package'] = args['custom_package'] + build_defaults['custom_packages'] - -if not args['custom_apt_entry']: - args['custom_apt_entry'] = [] -args['custom_apt_entry'] = args['custom_apt_entry'] + build_defaults['additional_repositories'] - - -# Check the build environment and dependencies -env_check_retval = os.system("scripts/check-build-env") -if env_check_retval > 0: - print("Build environment check failed, fix the issues and retry") - -args['kernel_version'] = build_defaults['kernel_version'] -args['kernel_flavor'] = build_defaults['kernel_flavor'] -args['bootloaders'] = build_defaults['bootloaders'] - - -# Save to file -os.makedirs(defaults.BUILD_DIR, exist_ok=True) -print("Saving the build config to {0}".format(defaults.BUILD_CONFIG)) -with open(defaults.BUILD_CONFIG, 'w') as f: - json.dump(args, f, indent=4, sort_keys=True) - print("\n", file=f) - diff --git a/scripts/build-flavour b/scripts/build-flavour deleted file mode 100755 index 5e76672b..00000000 --- a/scripts/build-flavour +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Copyright (C) 2016 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -# -# File: build-flavour -# Purpose: Adds various data files to the build config -# depending on the build flavour. - - -BUILD_TYPE=$(scripts/query-json build/build-config.json build_type) -BUILD_ARCH=$(scripts/query-json build/build-config.json architecture) - -# Add debug tools if it's a development image -if [ $BUILD_TYPE = "development" ]; then - cp data/package-lists/vyos-dev.list.chroot build/config/package-lists/ -fi - -# Install grub-pc if it's an x86 build -if [ $BUILD_ARCH = 'amd64' -o $BUILD_ARCH = 'i386' ]; then - cp data/package-lists/vyos-x86.list.chroot build/config/package-lists/ -fi - -# Install grub-efi-arm if it's an arm build -if [ $BUILD_ARCH = 'armhf' -o $BUILD_ARCH = 'armel' -o $BUILD_ARCH = 'arm' ]; then - cp data/package-lists/vyos-arm.list.chroot build/config/package-lists/ -fi diff --git a/scripts/build-vyos-image b/scripts/build-vyos-image new file mode 100755 index 00000000..efe11af4 --- /dev/null +++ b/scripts/build-vyos-image @@ -0,0 +1,485 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# File: build-vyos-image +# Purpose: builds VyOS images using a fork of Debian's live-build tool + +import re +import os +import sys +import uuid +import glob +import shutil +import getpass +import platform +import argparse +import datetime +import functools + +import json + + +try: + import toml + import jinja2 + import git +except ModuleNotFoundError as e: + print("Cannot load a required library: {}".format(e)) + print("Please make sure the following Python3 modules are installed: toml jinja2 git") + +import vyos_build_utils as utils +import vyos_build_defaults as defaults + +# argparse converts hyphens to underscores, +# so for lookups in the original options hash we have to convert them back +def field_to_option(s): + return re.sub(r'_', '-', s) + +def get_default_build_by(): + return "{user}@{host}".format(user= getpass.getuser(), host=platform.node()) + +def get_validator(optdict, name): + try: + return optdict[name][2] + except KeyError: + return None + +def merge_dicts(source, destination): + """ Merge two dictionaries and return a new dict which has the merged key/value pairs. + Merging logic is as follows: + Sub-dicts are merged. + List values are combined. + Scalar values are set to those from the source dict. + """ + from copy import deepcopy + tmp = deepcopy(destination) + + for key, value in source.items(): + if key not in tmp: + tmp[key] = value + elif isinstance(source[key], dict): + tmp[key] = dict_merge(source[key], tmp[key]) + elif isinstance(source[key], list): + tmp[key] = source[key] + tmp[key] + else: + tmp[key] = source[key] + + return tmp + +def has_nonempty_key(config, key): + if key in config: + if config[key]: + return True + return False + +def make_toml_path(dir, file_basename): + return os.path.join(dir, file_basename + ".toml") + + +if __name__ == "__main__": + ## Check if the script is running wirh root permissions + ## Since live-build requires privileged calls such as chroot(), + ## there's no real way around it. + if os.getuid() != 0: + print("E: this script requires root privileges") + sys.exit(1) + + ## Check if there are missing build dependencies + deps = { + 'packages': [ + 'sudo', + 'make', + 'live-build', + 'pbuilder', + 'devscripts', + 'python3-pystache', + 'python3-git', + 'qemu-utils' + ], + 'binaries': [] + } + + print("I: Checking if packages required for VyOS image build are installed") + try: + checker = utils.check_system_dependencies(deps) + except OSError as e: + print(e) + sys.exit(1) + + ## Load the file with default build configuration options + try: + with open(defaults.DEFAULTS_FILE, 'r') as f: + build_defaults = toml.load(f) + except Exception as e: + print("Failed to open the defaults file {0}: {1}".format(defaults.DEFAULTS_FILE, e)) + sys.exit(1) + + ## Get a list of available build flavors + build_flavors = list(map(lambda f: os.path.splitext(f)[0], os.listdir(defaults.BUILD_FLAVORS_DIR))) + + ## Set up the option parser + ## XXX: It uses values from the default configuration file for its option defaults, + ## which is why it's defined after loading the defaults.toml file data. + + # Options dict format: + # '$option_name_without_leading_dashes': { ('$help_string', $default_value_generator_thunk, $value_checker_thunk) } + options = { + 'architecture': ('Image target architecture (amd64 or arm64)', + lambda: build_defaults['architecture'], lambda x: x in ['amd64', 'arm64']), + 'build-by': ('Builder identifier (e.g. jrandomhacker@example.net)', get_default_build_by, None), + 'debian-mirror': ('Debian repository mirror', lambda: build_defaults['debian_mirror'], None), + 'debian-security-mirror': ('Debian security updates mirror', lambda: build_defaults['debian_security_mirror'], None), + 'pbuilder-debian-mirror': ('Debian repository mirror for pbuilder env bootstrap', lambda: build_defaults['debian_mirror'], None), + 'vyos-mirror': ('VyOS package mirror', lambda: build_defaults["vyos_mirror"], None), + 'build-type': ('Build type, release or development', lambda: 'development', lambda x: x in ['release', 'development']), + 'version': ('Version number (release builds only)', None, None), + 'build-comment': ('Optional build comment', lambda: '', None) + } + + # Create the option parser + parser = argparse.ArgumentParser() + for k, v in options.items(): + help_string, default_value_thunk = v[0], v[1] + if default_value_thunk is None: + parser.add_argument('--' + k, type=str, help=help_string) + else: + parser.add_argument('--' + k, type=str, help=help_string, default=default_value_thunk()) + + # The debug option is a bit special since it different type is different + parser.add_argument('--debug', help='Enable debug output', action='store_true') + + parser.add_argument('--dry-run', help='Check build configuration and exit', action='store_true') + + # Custom APT entry and APT key options can be used multiple times + parser.add_argument('--custom-apt-entry', help="Custom APT entry", action='append') + parser.add_argument('--custom-apt-key', help="Custom APT key file", action='append') + parser.add_argument('--custom-package', help="Custom package to install from repositories", action='append') + + # Build flavor is a positional argument + parser.add_argument('build_flavor', help='Build flavor', nargs='?', action='store') + + args = vars(parser.parse_args()) + + debug = args["debug"] + + # Validate options + for k, v in args.items(): + key = field_to_option(k) + func = get_validator(options, k) + if func is not None: + if not func(v): + print("{v} is not a valid value for --{o} option".format(o=key, v=v)) + sys.exit(1) + + if not args["build_flavor"]: + print("E: Build flavor is not specified!") + print("E: For example, to build the generic ISO, run {} iso".format(sys.argv[0])) + print("Available build flavors:\n") + print("\n".join(build_flavors)) + sys.exit(1) + + # Some fixup for mirror settings. + # The idea is: if --debian-mirror is specified but --pbuilder-debian-mirror is not, + # use the --debian-mirror value for both lb and pbuilder bootstrap + if (args['debian_mirror'] != build_defaults["debian_mirror"]) and \ + (args['pbuilder_debian_mirror'] == build_defaults["debian_mirror"]): + args['pbuilder_debian_mirror'] = args['debian_mirror'] + + # Version can only be set for release builds, + # for dev builds it hardly makes any sense + if args['build_type'] == 'development': + if args['version'] is not None: + print("Version can only be set for release builds") + print("Use --build-type=release option if you want to set version number") + sys.exit(1) + + if not args['custom_apt_entry']: + args['custom_apt_entry'] = [] + args['custom_apt_entry'] = args['custom_apt_entry'] + build_defaults['additional_repositories'] + + ## Inject some useful hardcoded options + args['build_dir'] = defaults.BUILD_DIR + args['pbuilder_config'] = os.path.join(defaults.BUILD_DIR, defaults.PBUILDER_CONFIG) + + ## Combine the arguments with non-configurable defaults + build_config = merge_dicts(build_defaults, args) + + ## Load the flavor file and mix-ins + with open(make_toml_path(defaults.BUILD_TYPES_DIR, build_config["build_type"]), 'r') as f: + build_type_config = toml.load(f) + build_config = merge_dicts(build_config, build_type_config) + + with open(make_toml_path(defaults.BUILD_ARCHES_DIR, build_config["architecture"]), 'r') as f: + build_arch_config = toml.load(f) + build_config = merge_dicts(build_config, build_arch_config) + + with open(make_toml_path(defaults.BUILD_FLAVORS_DIR, build_config["build_flavor"]), 'r') as f: + flavor_config = toml.load(f) + build_config = merge_dicts(build_config, flavor_config) + + ## Rename and merge some fields for simplicity + ## E.g. --custom-packages is for the user, but internally + ## it's added to the same package list as everything else + if has_nonempty_key(build_config, "custom_package"): + build_config["packages"] += build_config["custom_package"] + del build_config["custom_package"] + + ## Add architecture-dependent packages from the flavor + if has_nonempty_key(build_config, "architectures"): + arch = build_config["architecture"] + if arch in build_config["architectures"]: + build_config["packages"] += build_config["architectures"][arch]["packages"] + + ## Dump the complete config if the user enabled debug mode + if debug: + import json + print("D: Effective build config:\n") + print(json.dumps(build_config, indent=4)) + + ## Clean up the old build config and set up a fresh copy + lb_config_dir = os.path.join(defaults.BUILD_DIR, defaults.LB_CONFIG_DIR) + print(lb_config_dir) + shutil.rmtree(lb_config_dir, ignore_errors=True) + shutil.copytree("data/live-build-config/", lb_config_dir) + os.makedirs(lb_config_dir, exist_ok=True) + + ## Create the version file + + # Create a build timestamp + now = datetime.datetime.today() + build_timestamp = now.strftime("%Y%m%d%H%M") + + # FIXME: use aware rather than naive object + build_date = now.strftime("%a %d %b %Y %H:%M UTC") + + # Assign a (hopefully) unique identifier to the build (UUID) + build_uuid = str(uuid.uuid4()) + + # Initialize Git object from our repository + try: + repo = git.Repo('.') + + # Retrieve the Git commit ID of the repository, 14 charaters will be sufficient + build_git = repo.head.object.hexsha[:14] + # If somone played around with the source tree and the build is "dirty", mark it + if repo.is_dirty(): + build_git += "-dirty" + + # Retrieve git branch name + git_branch = repo.active_branch.name + except Exception as e: + print("Could not retrieve information from git: {0}".format(str(e))) + build_git = "" + git_branch = "" + git_commit = "" + + # Create the build version string + if build_config['build_type'] == 'development': + try: + if not git_branch: + raise ValueError("git branch could not be determined") + + # Load the branch to version mapping file + with open('data/versions') as f: + version_mapping = json.load(f) + + branch_version = version_mapping[git_branch] + + version = "{0}-rolling-{1}".format(branch_version, build_timestamp) + except Exception as e: + print("Could not build a version string specific to git branch, falling back to default: {0}".format(str(e))) + version = "999.{0}".format(build_timestamp) + else: + # Release build, use the version from ./configure arguments + version = build_config['version'] + + if build_config['build_type'] == 'development': + lts_build = False + else: + lts_build = True + + version_data = { + 'version': version, + 'built_by': build_config['build_by'], + 'built_on': build_date, + 'build_uuid': build_uuid, + 'build_git': build_git, + 'build_branch': git_branch, + 'release_train': build_config['release_train'], + 'lts_build': lts_build, + 'build_comment': build_config['build_comment'] + } + + os_release = f""" + PRETTY_NAME="VyOS {version} ({build_config['release_train']})" + NAME="VyOS" + VERSION_ID="{version}" + VERSION="{version} ({build_config['release_train']})" + VERSION_CODENAME=buster + ID=vyos + HOME_URL="https://vyos.io" + SUPPORT_URL="https://support.vyos.io" + BUG_REPORT_URL="https://phabricator.vyos.net" + """ + + chroot_includes_dir = os.path.join(defaults.BUILD_DIR, defaults.CHROOT_INCLUDES_DIR) + vyos_data_dir = os.path.join(chroot_includes_dir, "usr/share/vyos") + os.makedirs(vyos_data_dir, exist_ok=True) + with open(os.path.join(vyos_data_dir, 'version.json'), 'w') as f: + json.dump(version_data, f) + + # For backwards compatibility with 'add system image' script from older versions + # we need a file in the old format so that script can find out the version of the image + # for upgrade + os.makedirs(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/'), exist_ok=True) + with open(os.path.join(chroot_includes_dir, 'opt/vyatta/etc/version'), 'w') as f: + print("Version: {0}".format(version), file=f) + + + # Define variables that influence to welcome message on boot + os.makedirs(os.path.join(chroot_includes_dir, 'usr/lib/'), exist_ok=True) + with open(os.path.join(chroot_includes_dir, 'usr/lib//os-release'), 'w') as f: + print(os_release, file=f) + + + ## Switch to the build directory, this is crucial for the live-build work work + ## because the efective build config files etc. are there + os.chdir(defaults.BUILD_DIR) + + ## Clean up earlier build state and artifacts + print("I: Cleaning the build workspace") + os.system("lb clean") + #iter(lambda p: shutil.rmtree(p, ignore_errors=True), + # ['config/binary', 'config/bootstrap', 'config/chroot', 'config/common', 'config/source']) + artifacts = functools.reduce( + lambda x, y: x + y, + map(glob.glob, ['*.iso', '*.raw', '*.img', '*.xz', '*.ova', '*.ovf'])) + iter(os.remove, artifacts) + + ## Create live-build configuration files + + # Add the additional repositories to package lists + print("I: Setting up additional APT entries") + vyos_repo_entry = "deb {0} {1} main\n".format(build_config['vyos_mirror'], build_config['vyos_branch']) + + apt_file = defaults.VYOS_REPO_FILE + + if debug: + print("D: Adding these entries to {0}:".format(apt_file)) + print("\t", vyos_repo_entry) + + with open(apt_file, 'w') as f: + f.write(vyos_repo_entry) + + # Add custom APT entries + if build_config['custom_apt_entry']: + custom_apt_file = defaults.CUSTOM_REPO_FILE + entries = "\n".join(build_config['custom_apt_entry']) + if debug: + print("D: Adding custom APT entries:") + print(entries) + with open(custom_apt_file, 'w') as f: + f.write(entries) + f.write("\n") + + # Add custom APT keys + if has_nonempty_key(build_config, 'custom_apt_key'): + key_dir = ARCHIVES_DIR + for k in build_config['custom_apt_key']: + dst_name = '{0}.key.chroot'.format(os.path.basename(k)) + shutil.copy(k, os.path.join(key_dir, dst_name)) + + # Add custom packages + if has_nonempty_key(build_config, 'packages'): + package_list_file = defaults.PACKAGE_LIST_FILE + packages = "\n".join(build_config['packages']) + with open (package_list_file, 'w') as f: + f.write(packages) + + ## Create includes + if has_nonempty_key(build_config, "includes_chroot"): + for i in build_config["includes_chroot"]: + file_path = os.path.join(includes_chroot_dir, i["path"]) + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, 'w') as f: + f.write(i["data"]) + + + ## Configure live-build + lb_config_tmpl = jinja2.Template(""" + lb config noauto \ + --architectures {{architecture}} \ + --bootappend-live "boot=live components hostname=vyos username=live nopersistence noautologin nonetworking union=overlay console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \ + --bootappend-live-failsafe "live components memtest noapic noapm nodma nomce nolapic nomodeset nosmp nosplash vga=normal console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \ + --linux-flavours {{kernel_flavor}} \ + --linux-packages linux-image-{{kernel_version}} \ + --bootloader syslinux,grub-efi \ + --binary-images iso-hybrid \ + --checksums 'sha256 md5' \ + --debian-installer none \ + --distribution {{debian_distribution}} \ + --iso-application "VyOS" \ + --iso-publisher "{{build_by}}" \ + --iso-volume "VyOS" \ + --debootstrap-options "--variant=minbase --exclude=isc-dhcp-client,isc-dhcp-common,ifupdown --include=apt-utils,ca-certificates,gnupg2" \ + --mirror-bootstrap {{debian_mirror}} \ + --mirror-chroot {{debian_mirror}} \ + --mirror-chroot-security {{debian_security_mirror}} \ + --mirror-binary {{debian_mirror}} \ + --mirror-binary-security {{debian_security_mirror}} \ + --archive-areas "main contrib non-free" \ + --firmware-chroot false \ + --firmware-binary false \ + --updates true \ + --security true \ + --backports true \ + --apt-recommends false \ + --apt-options "--yes -oAPT::Default-Release="{{release_train}}" -oAPT::Get::allow-downgrades=true" \ + --apt-indices false + "${@}" + """) + + + lb_config_command = lb_config_tmpl.render(build_config) + + print("I: Configuring live-build") + + if debug: + print("D: live-build configuration command") + print(lb_config_command) + + result = os.system(lb_config_command) + if result > 0: + print("E: live-build config failed") + sys.exit(1) + + ## In dry-run mode, exit at this point + if build_config["dry_run"]: + print("I: dry-run, not starting image build") + sys.exit(0) + + + ## Build the image + print("I: Starting image build") + if debug: + print("D: It's not like I'm building this specially for you or anything!") + res = os.system("lb build 2>&1") + if res > 0: + sys.exit(res) + + # Copy the image + shutil.copy("live-image-{0}.hybrid.iso".format(build_config["architecture"]), + "vyos-{0}-{1}.iso".format(version_data["version"], build_config["architecture"])) diff --git a/scripts/check-config b/scripts/check-config deleted file mode 100755 index d2236619..00000000 --- a/scripts/check-config +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2015 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -# -# File: check-config -# Purpose: -# Checks if the build config file (build/build-config.json) exists. -# This is to prevent accidental execution of plumbing make targets -# from going too far and failing with confusing errors. - - -import sys -import json - -import defaults - - -print("Checking build configuration") - -try: - with open(defaults.BUILD_CONFIG, 'r') as f: - build_config = json.load(f) -except: - print("Build config does not exist or is not a valid JSON file") - print("Please run the ./configure script and try again") - sys.exit(1) diff --git a/scripts/copy-image b/scripts/copy-image deleted file mode 100755 index 4196d06d..00000000 --- a/scripts/copy-image +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -BUILD_DIR="$(scripts/query-json build/build-config.json build_dir)" -BUILD_ARCH="$(scripts/query-json build/build-config.json architecture)" -VERSION="$(cat $BUILD_DIR/version)" - -cp "$BUILD_DIR/live-image-$BUILD_ARCH.hybrid.iso" "$BUILD_DIR/vyos-$VERSION-$BUILD_ARCH.iso" diff --git a/scripts/live-build-config b/scripts/live-build-config deleted file mode 100755 index a797a32c..00000000 --- a/scripts/live-build-config +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2018 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -# -# File: live-build-config -# Purpose: -# Creates a live-build config command from template using the build config -# and executes it, to prepare the system for building the installation ISO. - - -import sys -import os -import shutil -import json - -import pystache - -import defaults -import util - -util.check_build_config() - -lb_config_tmpl = """ -lb config noauto \ - --architectures {{architecture}} \ - --bootappend-live "boot=live components hostname=vyos username=live nopersistence noautologin nonetworking union=overlay console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \ - --bootappend-live-failsafe "live components memtest noapic noapm nodma nomce nolapic nomodeset nosmp nosplash vga=normal console=ttyS0,115200 console=tty0 net.ifnames=0 biosdevname=0" \ - --linux-flavours {{kernel_flavor}} \ - --linux-packages linux-image-{{kernel_version}} \ - --bootloader {{bootloaders}} \ - --binary-images iso-hybrid \ - --checksums 'sha256 md5' \ - --debian-installer none \ - --distribution {{distribution}} \ - --iso-application "VyOS" \ - --iso-publisher "{{build_by}}" \ - --iso-volume "VyOS" \ - --debootstrap-options "--variant=minbase --exclude=isc-dhcp-client,isc-dhcp-common,ifupdown --include=apt-utils,ca-certificates,gnupg2" \ - --mirror-bootstrap {{debian_mirror}} \ - --mirror-chroot {{debian_mirror}} \ - --mirror-chroot-security {{debian_security_mirror}} \ - --mirror-binary {{debian_mirror}} \ - --mirror-binary-security {{debian_security_mirror}} \ - --archive-areas "main contrib non-free" \ - --firmware-chroot false \ - --firmware-binary false \ - --updates true \ - --security false \ - --backports true \ - --utc-time true \ - --debug \ - --apt-recommends false \ - --apt-options "--yes -oAPT::Get::allow-downgrades=true" \ - --apt-indices false - "${@}" -""" - -with open(defaults.BUILD_CONFIG, 'r') as f: - build_config = json.load(f) - -debug = build_config['debug'] - -# Add the additional repositories to package lists -print("Setting up additional APT entries") -vyos_repo_entry = "deb {0} {1} main\n".format(build_config['vyos_mirror'], build_config['vyos_branch']) - -apt_file = os.path.join(build_config['build_dir'], defaults.VYOS_REPO_FILE) - -if debug: - print("Adding these entries to {0}:".format(apt_file)) - print("\t", vyos_repo_entry) - -with open(apt_file, 'w') as f: - f.write(vyos_repo_entry) - -# Add custom APT entries -if build_config['custom_apt_entry']: - custom_apt_file = os.path.join(build_config['build_dir'], defaults.CUSTOM_REPO_FILE) - entries = "\n".join(build_config['custom_apt_entry']) - if debug: - print("Adding custom APT entries:") - print(entries) - with open(custom_apt_file, 'w') as f: - f.write(entries) - f.write("\n") - -# Add custom APT keys -if build_config['custom_apt_key']: - key_dir = os.path.join(build_config['build_dir'], defaults.ARCHIVES_DIR) - for k in build_config['custom_apt_key']: - dst_name = '{0}.key.chroot'.format(os.path.basename(k)) - shutil.copy(k, os.path.join(key_dir, dst_name)) - -# Add custom packages -if build_config['custom_package']: - package_list_file = os.path.join(build_config['build_dir'], defaults.CUSTOM_PACKAGE_LIST_FILE) - packages = "\n".join(build_config['custom_package']) - with open (package_list_file, 'w') as f: - f.write(packages) - -# Configure live-build - -lb_config_command = pystache.render(lb_config_tmpl, build_config) - -print("Configuring live-build") - -os.chdir(defaults.BUILD_DIR) -result = os.system(lb_config_command) -if result > 0: - print("live-build config failed") - sys.exit(1) diff --git a/scripts/query-json b/scripts/query-json deleted file mode 100755 index 2f1ea32f..00000000 --- a/scripts/query-json +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/python3 -# -# Copyright (C) 2016 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -# -# File: query-config -# Purpose: Extracts field values from a flat JSON file, -# for use in languages that can't handle JSON easily, -# (I'm looking at you, Bourne shell!) - - -import sys -import json - -import defaults -import util - -if len(sys.argv) < 3: - print("Usage: {0} <flat JSON file> <config field name>".format(sys.argv[0])) - sys.exit(1) - -# Note: lack of error handling is deliberate, if some field is expected to be there -# but isn't, it's better if the failure will be obvious and spectacular - -file = sys.argv[1] -key = sys.argv[2] - -with open(file, 'r') as f: - json_data = json.load(f) - -print(json_data[key]) diff --git a/scripts/defaults.py b/scripts/vyos_build_defaults.py index 5d489e48..ac82dbc4 100644 --- a/scripts/defaults.py +++ b/scripts/vyos_build_defaults.py @@ -18,19 +18,28 @@ import os +# Relative to the repository directory + BUILD_DIR = 'build' -BUILD_CONFIG = os.path.join(BUILD_DIR, 'build-config.json') +BUILD_CONFIG = os.path.join(BUILD_DIR, 'build-config.toml') + +DEFAULTS_FILE = 'data/defaults.toml' + +BUILD_TYPES_DIR = 'data/build-types' +BUILD_ARCHES_DIR = 'data/architectures' +BUILD_FLAVORS_DIR = 'data/build-flavors' + +# Relative to the build directory -PBUILDER_CONFIG = os.path.join(BUILD_DIR, 'pbuilderrc') -PBUILDER_DIR = os.path.join(BUILD_DIR, 'pbuilder') +PBUILDER_CONFIG = 'pbuilderrc' +PBUILDER_DIR = 'pbuilder' -LB_CONFIG_DIR = os.path.join(BUILD_DIR, 'config') -CHROOT_INCLUDES_DIR = os.path.join(LB_CONFIG_DIR, 'includes.chroot') +LB_CONFIG_DIR = 'config' +CHROOT_INCLUDES_DIR = 'config/includes.chroot' ARCHIVES_DIR = 'config/archives/' VYOS_REPO_FILE = 'config/archives/vyos.list.chroot' CUSTOM_REPO_FILE = 'config/archives/custom.list.chroot' -CUSTOM_PACKAGE_LIST_FILE = 'config/package-lists/custom.list.chroot' +PACKAGE_LIST_FILE = 'config/package-lists/custom.list.chroot' -DEFAULT_BUILD_FLAVOR = 'data/defaults.json' diff --git a/scripts/util.py b/scripts/vyos_build_utils.py index 7cc33364..ed358848 100644 --- a/scripts/util.py +++ b/scripts/vyos_build_utils.py @@ -21,7 +21,7 @@ import sys import os from distutils.spawn import find_executable -import defaults +import vyos_build_defaults as defaults def check_build_config(): if not os.path.exists(defaults.BUILD_CONFIG): @@ -60,6 +60,18 @@ class DependencyChecker(object): return self.__missing return None - def print_missing_deps(self): - print("Missing packages: " + " ".join(self.__missing['packages'])) - print("Missing binaries: " + " ".join(self.__missing['binaries'])) + def format_missing_dependencies(self): + msg = "E: There are missing system dependencies!\n" + if self.__missing['packages']: + msg += "E: Missing packages: " + " ".join(self.__missing['packages']) + if self.__missing['binaries']: + msg += "E: Missing binaries: " + " ".join(self.__missing['binaries']) + return msg + +def check_system_dependencies(deps): + checker = DependencyChecker(deps) + missing = checker.get_missing_dependencies() + if missing: + raise OSError(checker.format_missing_dependencies()) + else: + pass |