From 8f94262e8fa2477700c50303ea6e2c6ddad72adb Mon Sep 17 00:00:00 2001 From: zsdc Date: Thu, 19 Jan 2023 20:18:42 +0200 Subject: image: T4516: Added system image tools This commit adds the whole set of system image tools written from the scratch in Python that allows performing all the operations on images: * check information * perform installation and deletion * versions management Also, it contains a new service that will update the GRUB menu and keep tracking its version in the future. WARNING: The commit contains non-reversible changes. Because of boot menu changes, it will not be possible to manage images from older VyOS versions after an update. --- src/op_mode/image_info.py | 108 ++++++++ src/op_mode/image_installer.py | 557 +++++++++++++++++++++++++++++++++++++++++ src/op_mode/image_manager.py | 190 ++++++++++++++ 3 files changed, 855 insertions(+) create mode 100644 src/op_mode/image_info.py create mode 100644 src/op_mode/image_installer.py create mode 100644 src/op_mode/image_manager.py (limited to 'src/op_mode') diff --git a/src/op_mode/image_info.py b/src/op_mode/image_info.py new file mode 100644 index 000000000..ae0677196 --- /dev/null +++ b/src/op_mode/image_info.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# +# Copyright 2022 VyOS maintainers and contributors +# +# This file is part of VyOS. +# +# VyOS is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# VyOS is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# VyOS. If not, see . + +import sys +from typing import List, Union + +from hurry.filesize import size +from tabulate import tabulate + +from vyos import opmode +from vyos.system import disk, grub, image + + +def _format_show_images_summary(images_summary: image.BootDetails) -> str: + headers: list[str] = ['Name', 'Default boot', 'Running'] + table_data: list[list[str]] = list() + for image_item in images_summary.get('images_available', []): + name: str = image_item + if images_summary.get('image_default') == name: + default: str = 'Yes' + else: + default: str = '' + + if images_summary.get('image_running') == name: + running: str = 'Yes' + else: + running: str = '' + + table_data.append([name, default, running]) + tabulated: str = tabulate(table_data, headers) + + return tabulated + + +def _format_show_images_details( + images_details: list[image.ImageDetails]) -> str: + headers: list[str] = [ + 'Name', 'Version', 'Storage Read-Only', 'Storage Read-Write', + 'Storage Total' + ] + table_data: list[list[Union[str, int]]] = list() + for image_item in images_details: + name: str = image_item.get('name') + version: str = image_item.get('version') + disk_ro: int = size(image_item.get('disk_ro')) + disk_rw: int = size(image_item.get('disk_rw')) + disk_total: int = size(image_item.get('disk_total')) + table_data.append([name, version, disk_ro, disk_rw, disk_total]) + tabulated: str = tabulate(table_data, headers) + + return tabulated + + +def show_images_summary(raw: bool) -> Union[image.BootDetails, str]: + images_available: list[str] = grub.version_list() + root_dir: str = disk.find_persistence() + boot_vars: dict = grub.vars_read(f'{root_dir}/{image.CFG_VYOS_VARS}') + + images_summary: image.BootDetails = dict() + + images_summary['image_default'] = image.get_default_image() + images_summary['image_running'] = image.get_running_image() + images_summary['images_available'] = images_available + images_summary['console_type'] = boot_vars.get('console_type') + images_summary['console_num'] = boot_vars.get('console_num') + + if raw: + return images_summary + else: + return _format_show_images_summary(images_summary) + + +def show_images_details(raw: bool) -> Union[list[image.ImageDetails], str]: + images: list[str] = grub.version_list() + images_details: list[image.ImageDetails] = list() + for image_name in images: + images_details.append(image.get_details(image_name)) + + if raw: + return images_details + else: + return _format_show_images_details(images_details) + + +if __name__ == '__main__': + try: + res = opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, opmode.Error) as e: + print(e) + sys.exit(1) diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py new file mode 100644 index 000000000..6ebb38e46 --- /dev/null +++ b/src/op_mode/image_installer.py @@ -0,0 +1,557 @@ +#!/usr/bin/env python3 +# +# Copyright 2022 VyOS maintainers and contributors +# +# This file is part of VyOS. +# +# VyOS is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# VyOS is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# VyOS. If not, see . + +from argparse import ArgumentParser, Namespace +from pathlib import Path +from shutil import copy, rmtree, copytree +from sys import exit +from urllib.parse import urlparse + +from psutil import disk_partitions + +from vyos.configtree import ConfigTree +from vyos.remote import download +from vyos.system import disk, grub, image +from vyos.template import render +from vyos.util import ask_input, ask_yes_no, run + +# define text messages +MSG_ERR_NOT_LIVE: str = 'The system is already installed. Please use "add system image" instead.' +MSG_ERR_LIVE: str = 'The system is in live-boot mode. Please use "install image" instead.' +MSG_ERR_NO_DISK: str = 'No suitable disk was found. There must be at least one disk of 2GB or greater size.' +MSG_INFO_INSTALL_WELCOME: str = 'Welcome to VyOS installation!\nThis command will install the VyOS to your permanent storage.' +MSG_INFO_INSTALL_EXIT: str = 'Exitting from VyOS installation' +MSG_INFO_INSTALL_SUCCESS: str = 'The image installed successfully' +MSG_INFO_INSTALL_DISKS_LIST: str = 'Were found the next disks:' +MSG_INFO_INSTALL_DISK_SELECT: str = 'Which one should be used for installation?' +MSG_INFO_INSTALL_DISK_CONFIRM: str = 'Installation will delete all data on the drive. Continue?' +MSG_INFO_INSTALL_PARTITONING: str = 'Creating partition table...' +MSG_INPUT_CONFIG_FOUND: str = 'An active configuration was found. Would you like to copy it to the new image?' +MSG_INPUT_IMAGE_NAME: str = 'What would you like to name this image?' +MSG_INPUT_IMAGE_DEFAULT: str = 'Would you like to set a new image as default one for boot?' +MSG_INPUT_PASSWORD: str = 'Please enter a password for the "vyos" user' +MSG_INPUT_ROOT_SIZE_ALL: str = 'Would you like to use all free space on the drive?' +MSG_INPUT_ROOT_SIZE_SET: str = 'What should be a size (in GB) of the root partition (min is 1.5 GB)?' +MSG_INPUT_CONSOLE_TYPE: str = 'What console should be used by default? (K: KVM, S: Serial, U: USB-Serial)?' +MSG_WARN_ISO_SIGN_INVALID: str = 'Signature is not valid. Do you want to continue with installation?' +MSG_WARN_ISO_SIGN_UNAVAL: str = 'Signature is not available. Do you want to continue with installation?' +MSG_WARN_ROOT_SIZE_TOOBIG: str = 'The size is too big. Try again.' +MSG_WARN_ROOT_SIZE_TOOSMALL: str = 'The size is too small. Try again' +MSG_WARN_IMAGE_NAME_WRONG: str = 'The suggested name is unsupported!\n' +'It must be between 1 and 32 characters long and contains only the next characters: .+-_ a-z A-Z 0-9' +CONST_MIN_DISK_SIZE: int = 2147483648 # 2 GB +CONST_MIN_ROOT_SIZE: int = 1610612736 # 1.5 GB +# a reserved space: 2MB for header, 1 MB for BIOS partition, 256 MB for EFI +CONST_RESERVED_SPACE: int = (2 + 1 + 256) * 1024**2 + +# define directories and paths +DIR_INSTALLATION: str = '/mnt/installation' +DIR_ROOTFS_SRC: str = f'{DIR_INSTALLATION}/root_src' +DIR_ROOTFS_DST: str = f'{DIR_INSTALLATION}/root_dst' +DIR_ISO_MOUNT: str = f'{DIR_INSTALLATION}/iso_src' +DIR_DST_ROOT: str = f'{DIR_INSTALLATION}/disk_dst' +DIR_KERNEL_SRC: str = '/boot/' +FILE_ROOTFS_SRC: str = '/usr/lib/live/mount/medium/live/filesystem.squashfs' +ISO_DOWNLOAD_PATH: str = '/tmp/vyos_installation.iso' + +# default boot variables +DEFAULT_BOOT_VARS: dict[str, str] = { + 'timeout': '5', + 'console_type': 'tty', + 'console_num': '0', + 'bootmode': 'normal' +} + + +def bytes_to_gb(size: int) -> float: + """Convert Bytes to GBytes, rounded to 1 decimal number + + Args: + size (int): input size in bytes + + Returns: + float: size in GB + """ + return round(size / 1024**3, 1) + + +def gb_to_bytes(size: float) -> int: + """Convert GBytes to Bytes + + Args: + size (float): input size in GBytes + + Returns: + int: size in bytes + """ + return int(size * 1024**3) + + +def find_disk() -> tuple[str, int]: + """Find a target disk for installation + + Returns: + tuple[str, int]: disk name and size in bytes + """ + # check for available disks + disks_available: dict[str, int] = disk.disks_size() + for disk_name, disk_size in disks_available.copy().items(): + if disk_size < CONST_MIN_DISK_SIZE: + del disks_available[disk_name] + if not disks_available: + print(MSG_ERR_NO_DISK) + exit(MSG_INFO_INSTALL_EXIT) + + # select one as a target + print(MSG_INFO_INSTALL_DISKS_LIST) + default_disk: str = list(disks_available)[0] + for disk_name, disk_size in disks_available.items(): + disk_size_human: str = bytes_to_gb(disk_size) + print(f'Drive: {disk_name} ({disk_size_human} GB)') + disk_selected: str = ask_input(MSG_INFO_INSTALL_DISK_SELECT, + default=default_disk, + valid_responses=list(disks_available)) + + return disk_selected, disks_available[disk_selected] + + +def ask_root_size(available_space: int) -> int: + """Define a size of root partition + + Args: + available_space (int): available space in bytes for a root partition + + Returns: + int: defined size + """ + if ask_yes_no(MSG_INPUT_ROOT_SIZE_ALL, default=True): + return available_space + + while True: + root_size_gb: str = ask_input(MSG_INPUT_ROOT_SIZE_SET) + root_size_kbytes: int = (gb_to_bytes(float(root_size_gb))) // 1024 + + if root_size_kbytes > available_space: + print(MSG_WARN_ROOT_SIZE_TOOBIG) + continue + if root_size_kbytes < CONST_MIN_ROOT_SIZE / 1024: + print(MSG_WARN_ROOT_SIZE_TOOSMALL) + continue + + return root_size_kbytes + + +def prepare_tmp_disr() -> None: + """Create temporary directories for installation + """ + print('Creating temporary directories') + for dir in [DIR_ROOTFS_SRC, DIR_ROOTFS_DST, DIR_DST_ROOT]: + dirpath = Path(dir) + dirpath.mkdir(mode=0o755, parents=True) + + +def setup_grub(root_dir: str) -> None: + """Install GRUB configurations + + Args: + root_dir (str): a path to the root of target filesystem + """ + print('Installing GRUB configuration files') + grub_cfg_main = f'{root_dir}/{grub.GRUB_DIR_MAIN}/grub.cfg' + grub_cfg_vars = f'{root_dir}/{grub.CFG_VYOS_VARS}' + grub_cfg_modules = f'{root_dir}/{grub.CFG_VYOS_MODULES}' + grub_cfg_menu = f'{root_dir}/{grub.CFG_VYOS_MENU}' + grub_cfg_options = f'{root_dir}/{grub.CFG_VYOS_OPTIONS}' + + # create new files + render(grub_cfg_main, grub.TMPL_GRUB_MAIN, {}) + grub.common_write(root_dir) + grub.vars_write(grub_cfg_vars, DEFAULT_BOOT_VARS) + grub.modules_write(grub_cfg_modules, []) + grub.write_cfg_ver(1, root_dir) + render(grub_cfg_menu, grub.TMPL_GRUB_MENU, {}) + render(grub_cfg_options, grub.TMPL_GRUB_OPTS, {}) + + +def configure_authentication(config_file: str, password: str) -> None: + config = ConfigTree(config_file) + config.set([ + 'system', 'login', 'user', 'vyos', 'authentication', + 'plaintext-password' + ], + value=password, + replace=True) + config.set_tag(['system', 'login', 'user']) + + +def validate_signature(file_path: str, sign_type: str) -> None: + """Validate a file by signature and delete a signature file + + Args: + file_path (str): a path to file + sign_type (str): a signature type + """ + print('Validating signature') + signature_valid: bool = False + # validate with minisig + if sign_type == 'minisig': + for pubkey in [ + '/usr/share/vyos/keys/vyos-release.minisign.pub', + '/usr/share/vyos/keys/vyos-backup.minisign.pub' + ]: + if run(f'minisign -V -q -p {pubkey} -m {file_path} -x {file_path}.minisig' + ) == 0: + signature_valid = True + break + Path(f'{file_path}.minisig').unlink() + # validate with GPG + if sign_type == 'asc': + if run(f'gpg --verify ${file_path}.asc ${file_path}') == 0: + signature_valid = True + Path(f'{file_path}.asc').unlink() + + # warn or pass + if not signature_valid: + if not ask_yes_no(MSG_WARN_ISO_SIGN_INVALID, default=False): + exit(MSG_INFO_INSTALL_EXIT) + else: + print('Signature is valid') + + +def image_fetch(image_path: str) -> Path: + """Fetch an ISO image + + Args: + image_path (str): a path, remote or local + + Returns: + Path: a path to a local file + """ + try: + # check a type of path + if urlparse(image_path).scheme: + # download an image + download(ISO_DOWNLOAD_PATH, image_path, True, True) + # download a signature + sign_file = (False, '') + for sign_type in ['minisig', 'asc']: + try: + download(f'{ISO_DOWNLOAD_PATH}.{sign_type}', + f'{image_path}.{sign_type}') + sign_file = (True, sign_type) + break + except Exception: + print(f'{sign_type} signature is not available') + # validate a signature if it is available + if sign_file[0]: + validate_signature(ISO_DOWNLOAD_PATH, sign_file[1]) + else: + if not ask_yes_no(MSG_WARN_ISO_SIGN_UNAVAL, default=False): + cleanup() + exit(MSG_INFO_INSTALL_EXIT) + + return Path(ISO_DOWNLOAD_PATH) + else: + local_path: Path = Path(image_path) + if local_path.is_file(): + return local_path + else: + raise + except Exception: + print(f'The image cannot be fetched from: {image_path}') + exit(1) + + +def migrate_config() -> bool: + """Check for active config and ask user for migration + + Returns: + bool: user's decision + """ + active_config_path: Path = Path('/opt/vyatta/etc/config/config.boot') + if active_config_path.exists(): + if ask_yes_no(MSG_INPUT_CONFIG_FOUND, default=True): + return True + return False + + +def cleanup(mounts: list[str] = [], remove_items: list[str] = []) -> None: + """Clean up after installation + + Args: + mounts (list[str], optional): List of mounts to unmount. + Defaults to []. + remove_items (list[str], optional): List of files or directories + to remove. Defaults to []. + """ + print('Cleaning up') + # clean up installation directory by default + mounts_all = disk_partitions(all=True) + for mounted_device in mounts_all: + if mounted_device.mountpoint.startswith(DIR_INSTALLATION) and not ( + mounted_device.device in mounts or + mounted_device.mountpoint in mounts): + mounts.append(mounted_device.mountpoint) + # add installation dir to cleanup list + if DIR_INSTALLATION not in remove_items: + remove_items.append(DIR_INSTALLATION) + # also delete an ISO file + if Path(ISO_DOWNLOAD_PATH).exists( + ) and ISO_DOWNLOAD_PATH not in remove_items: + remove_items.append(ISO_DOWNLOAD_PATH) + + if mounts: + print('Unmounting target filesystems') + for mountpoint in mounts: + disk.partition_umount(mountpoint) + if remove_items: + print('Removing temporary files') + for remove_item in remove_items: + if Path(remove_item).exists(): + if Path(remove_item).is_file(): + Path(remove_item).unlink() + if Path(remove_item).is_dir(): + rmtree(remove_item) + + +def install_image() -> None: + """Install an image to a disk + """ + if not image.is_live_boot(): + exit(MSG_ERR_NOT_LIVE) + + print(MSG_INFO_INSTALL_WELCOME) + if not ask_yes_no('Would you like to continue?'): + print(MSG_INFO_INSTALL_EXIT) + exit() + + try: + # configure image name + running_image_name: str = image.get_running_image() + while True: + image_name: str = ask_input(MSG_INPUT_IMAGE_NAME, + running_image_name) + if image.validate_name(image_name): + break + print(MSG_WARN_IMAGE_NAME_WRONG) + + # define target drive + install_target, target_size = find_disk() + + # define target rootfs size in KB (smallest unit acceptable by sgdisk) + availabe_size: int = (target_size - CONST_RESERVED_SPACE) // 1024 + rootfs_size: int = ask_root_size(availabe_size) + + # ask for password + user_password: str = ask_input(MSG_INPUT_PASSWORD, default='vyos') + + # ask for default console + console_type: str = ask_input(MSG_INPUT_CONSOLE_TYPE, + default='K', + valid_responses=['K', 'S', 'U']) + console_dict: dict[str, str] = {'K': 'tty', 'S': 'ttyS', 'U': 'ttyUSB'} + + # create partitions + if not ask_yes_no(MSG_INFO_INSTALL_DISK_CONFIRM): + print(MSG_INFO_INSTALL_EXIT) + exit() + print(MSG_INFO_INSTALL_PARTITONING) + disk.disk_cleanup(install_target) + disk.parttable_create(install_target, rootfs_size) + disk.filesystem_create(f'{install_target}2', 'efi') + disk.filesystem_create(f'{install_target}3', 'ext4') + + # create directiroes for installation media + prepare_tmp_disr() + + # mount target filesystem and create required dirs inside + print('Mounting new partitions') + disk.partition_mount(f'{install_target}3', DIR_DST_ROOT) + Path(f'{DIR_DST_ROOT}/boot/efi').mkdir(parents=True) + disk.partition_mount(f'{install_target}2', f'{DIR_DST_ROOT}/boot/efi') + + # a config dir. It is the deepest one, so the comand will + # create all the rest in a single step + print('Creating a configuration file') + target_config_dir: str = f'{DIR_DST_ROOT}/boot/{image_name}/rw/opt/vyatta/etc/config/' + Path(target_config_dir).mkdir(parents=True) + # copy config + if migrate_config(): + copy('/opt/vyatta/etc/config/config.boot', target_config_dir) + else: + copy('/opt/vyatta/etc/config.boot.default', + f'{target_config_dir}/config.boot') + configure_authentication(f'{target_config_dir}/config.boot', + user_password) + Path(f'{target_config_dir}/.vyatta_config').touch() + + # create a persistence.conf + Path(f'{DIR_DST_ROOT}/persistence.conf').write_text('/ union\n') + + # copy system image and kernel files + print('Copying system image files') + for file in Path(DIR_KERNEL_SRC).iterdir(): + if file.is_file(): + copy(file, f'{DIR_DST_ROOT}/boot/{image_name}/') + copy(FILE_ROOTFS_SRC, + f'{DIR_DST_ROOT}/boot/{image_name}/{image_name}.squashfs') + + # install GRUB + print('Installing GRUB to the drive') + grub.install(install_target, f'{DIR_DST_ROOT}/boot/', + f'{DIR_DST_ROOT}/boot/efi') + setup_grub(DIR_DST_ROOT) + # add information about version + grub.create_structure() + grub.version_add(image_name, DIR_DST_ROOT) + grub.set_default(image_name, DIR_DST_ROOT) + grub.set_console_type(console_dict[console_type], DIR_DST_ROOT) + + # umount filesystems and remove temporary files + cleanup([f'{install_target}2', f'{install_target}3'], + ['/mnt/installation']) + + # we are done + print(MSG_INFO_INSTALL_SUCCESS) + exit() + + except Exception as err: + print(f'Unable to install VyOS: {err}') + # unmount filesystems and clenup + try: + cleanup([f'{install_target}2', f'{install_target}3'], + ['/mnt/installation']) + except Exception as err: + print(f'Cleanup failed: {err}') + + exit(1) + + +def add_image(image_path: str) -> None: + """Add a new image + + Args: + image_path (str): a path to an ISO image + """ + if image.is_live_boot(): + exit(MSG_ERR_LIVE) + + # fetch an image + iso_path: Path = image_fetch(image_path) + try: + # mount an ISO + Path(DIR_ISO_MOUNT).mkdir(mode=0o755, parents=True) + disk.partition_mount(iso_path, DIR_ISO_MOUNT, 'iso9660') + + # check sums + print('Validating image checksums') + if run(f'cd {DIR_ISO_MOUNT} && sha256sum --status -c sha256sum.txt'): + cleanup() + exit('Image checksum verification failed.') + + # mount rootfs (to get a system version) + Path(DIR_ROOTFS_SRC).mkdir(mode=0o755, parents=True) + disk.partition_mount(f'{DIR_ISO_MOUNT}/live/filesystem.squashfs', + DIR_ROOTFS_SRC, 'squashfs') + version_file: str = Path( + f'{DIR_ROOTFS_SRC}/opt/vyatta/etc/version').read_text() + disk.partition_umount(f'{DIR_ISO_MOUNT}/live/filesystem.squashfs') + version_name: str = version_file.lstrip('Version: ').strip() + image_name: str = ask_input(MSG_INPUT_IMAGE_NAME, version_name) + set_as_default: bool = ask_yes_no(MSG_INPUT_IMAGE_DEFAULT, default=True) + + # find target directory + root_dir: str = disk.find_persistence() + + # a config dir. It is the deepest one, so the comand will + # create all the rest in a single step + target_config_dir: str = f'{root_dir}/boot/{image_name}/rw/opt/vyatta/etc/config/' + # copy config + if migrate_config(): + print('Copying configuration directory') + copytree('/opt/vyatta/etc/config/', target_config_dir) + else: + Path(target_config_dir).mkdir(parents=True) + Path(f'{target_config_dir}/.vyatta_config').touch() + + # copy system image and kernel files + print('Copying system image files') + for file in Path(f'{DIR_ISO_MOUNT}/live').iterdir(): + if file.is_file() and (file.match('initrd*') or + file.match('vmlinuz*')): + copy(file, f'{root_dir}/boot/{image_name}/') + copy(f'{DIR_ISO_MOUNT}/live/filesystem.squashfs', + f'{root_dir}/boot/{image_name}/{image_name}.squashfs') + + # unmount an ISO and cleanup + cleanup([str(iso_path)]) + + # add information about version + grub.version_add(image_name, root_dir) + if set_as_default: + grub.set_default(image_name, root_dir) + + except Exception as err: + # unmount an ISO and cleanup + cleanup([str(iso_path)]) + exit(f'Whooops: {err}') + + +def parse_arguments() -> Namespace: + """Parse arguments + + Returns: + Namespace: a namespace with parsed arguments + """ + parser: ArgumentParser = ArgumentParser( + description='Install new system images') + parser.add_argument('--action', + choices=['install', 'add'], + required=True, + help='action to perform with an image') + parser.add_argument( + '--image_path', + help='a path (HTTP or local file) to an image that needs to be installed' + ) + # parser.add_argument('--image_new_name', help='a new name for image') + args: Namespace = parser.parse_args() + # Validate arguments + if args.action == 'add' and not args.image_path: + exit('A path to image is required for add action') + + return args + + +if __name__ == '__main__': + try: + args: Namespace = parse_arguments() + if args.action == 'install': + install_image() + if args.action == 'add': + add_image(args.image_path) + + exit() + + except KeyboardInterrupt: + print('Stopped by Ctrl+C') + cleanup() + exit() + + except Exception as err: + exit(f'{err}') diff --git a/src/op_mode/image_manager.py b/src/op_mode/image_manager.py new file mode 100644 index 000000000..ac889da38 --- /dev/null +++ b/src/op_mode/image_manager.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +# +# Copyright 2022 VyOS maintainers and contributors +# +# This file is part of VyOS. +# +# VyOS is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# VyOS is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# VyOS. If not, see . + +from argparse import ArgumentParser, Namespace +from pathlib import Path +from shutil import rmtree +from sys import exit + +from vyos.system import disk, grub, image +from vyos.util import ask_yes_no + + +def delete_image(image_name: str) -> None: + """Remove installed image files and boot entry + + Args: + image_name (str): a name of image to delete + """ + if image_name == image.get_running_image(): + exit('Currently running image cannot be deleted') + if image_name == image.get_default_image(): + exit('Default image cannot be deleted') + available_images: list[str] = grub.version_list() + if image_name not in available_images: + exit(f'The image "{image_name}" cannot be found') + presistence_storage: str = disk.find_persistence() + if not presistence_storage: + exit('Persistence storage cannot be found') + + if not ask_yes_no(f'Do you really want to delete the image {image_name}?', + default=False): + exit() + + # remove files and menu entry + version_path: Path = Path(f'{presistence_storage}/boot/{image_name}') + try: + rmtree(version_path) + grub.version_del(image_name, presistence_storage) + print(f'The image "{image_name}" was successfully deleted') + except Exception as err: + exit(f'Unable to remove the image "{image_name}": {err}') + + +def set_image(image_name: str) -> None: + """Set default boot image + + Args: + image_name (str): an image name + """ + if image_name == image.get_default_image(): + exit(f'The image "{image_name}" already configured as default') + available_images: list[str] = grub.version_list() + if image_name not in available_images: + exit(f'The image "{image_name}" cannot be found') + presistence_storage: str = disk.find_persistence() + if not presistence_storage: + exit('Persistence storage cannot be found') + + if not ask_yes_no( + f'Do you really want to set the image {image_name} ' + 'as default boot image?', + default=False): + exit() + + # set default boot image + try: + grub.set_default(image_name, presistence_storage) + print(f'The image "{image_name}" is now default boot image') + except Exception as err: + exit(f'Unable to set default image "{image_name}": {err}') + + +def rename_image(name_old: str, name_new: str) -> None: + """Rename installed image + + Args: + name_old (str): old name + name_new (str): new name + """ + if name_old == image.get_running_image(): + exit('Currently running image cannot be renamed') + available_images: list[str] = grub.version_list() + if name_old not in available_images: + exit(f'The image "{name_old}" cannot be found') + if name_new in available_images: + exit(f'The image "{name_new}" already exists') + if not image.validate_name(name_new): + exit(f'The image name "{name_new}" is not allowed') + + presistence_storage: str = disk.find_persistence() + if not presistence_storage: + exit('Persistence storage cannot be found') + + if not ask_yes_no( + f'Do you really want to rename the image {name_old} ' + f'to the {name_new}?', + default=False): + exit() + + try: + # replace default boot item + if name_old == image.get_default_image(): + grub.set_default(name_new, presistence_storage) + + # rename files and dirs + old_path: Path = Path(f'{presistence_storage}/boot/{name_old}') + new_path: Path = Path(f'{presistence_storage}/boot/{name_new}') + old_path.rename(new_path) + + # replace boot item + grub.version_del(name_old, presistence_storage) + grub.version_add(name_new, presistence_storage) + + print(f'The image "{name_old}" was renamed to "{name_new}"') + except Exception as err: + exit(f'Unable to rename image "{name_old}" to "{name_new}": {err}') + + +def list_images() -> None: + """Print list of available images for CLI hints""" + images_list: list[str] = grub.version_list() + for image_name in images_list: + print(image_name) + + +def parse_arguments() -> Namespace: + """Parse arguments + + Returns: + Namespace: a namespace with parsed arguments + """ + parser: ArgumentParser = ArgumentParser(description='Manage system images') + parser.add_argument('--action', + choices=['delete', 'set', 'rename', 'list'], + required=True, + help='action to perform with an image') + parser.add_argument( + '--image_name', + help= + 'a name of an image to add, delete, install, rename, or set as default') + parser.add_argument('--image_new_name', help='a new name for image') + args: Namespace = parser.parse_args() + # Validate arguments + if args.action == 'delete' and not args.image_name: + exit('An image name is required for delete action') + if args.action == 'set' and not args.image_name: + exit('An image name is required for set action') + if args.action == 'rename' and (not args.image_name or + not args.image_new_name): + exit('Both old and new image names are required for rename action') + + return args + + +if __name__ == '__main__': + try: + args: Namespace = parse_arguments() + if args.action == 'delete': + delete_image(args.image_name) + if args.action == 'set': + set_image(args.image_name) + if args.action == 'rename': + rename_image(args.image_name, args.image_new_name) + if args.action == 'list': + list_images() + + exit() + + except KeyboardInterrupt: + print('Stopped by Ctrl+C') + exit() + + except Exception as err: + exit(f'{err}') -- cgit v1.2.3 From 74b00c1f6961d1bd3a59768021f154bdb64c154e Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Mon, 10 Apr 2023 14:04:00 -0500 Subject: image: T4516: correct permissions on creation of config directory --- python/vyos/utils/file.py | 6 ++++++ src/op_mode/image_installer.py | 17 ++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) (limited to 'src/op_mode') diff --git a/python/vyos/utils/file.py b/python/vyos/utils/file.py index 667a2464b..9f27a7fb9 100644 --- a/python/vyos/utils/file.py +++ b/python/vyos/utils/file.py @@ -134,6 +134,12 @@ def chmod_755(path): S_IROTH | S_IXOTH chmod(path, bitmask) +def chmod_2775(path): + """ user/group permissions with set-group-id bit set """ + from stat import S_ISGID, S_IRWXU, S_IRWXG, S_IROTH, S_IXOTH + + bitmask = S_ISGID | S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH + chmod(path, bitmask) def makedir(path, user=None, group=None): if os.path.exists(path): diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index 6ebb38e46..77bb6460f 100644 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -19,7 +19,7 @@ from argparse import ArgumentParser, Namespace from pathlib import Path -from shutil import copy, rmtree, copytree +from shutil import copy, chown, rmtree, copytree from sys import exit from urllib.parse import urlparse @@ -29,7 +29,9 @@ from vyos.configtree import ConfigTree from vyos.remote import download from vyos.system import disk, grub, image from vyos.template import render -from vyos.util import ask_input, ask_yes_no, run +from vyos.utils.io import ask_input, ask_yes_no +from vyos.utils.file import chmod_2775 +from vyos.util import run # define text messages MSG_ERR_NOT_LIVE: str = 'The system is already installed. Please use "add system image" instead.' @@ -391,6 +393,8 @@ def install_image() -> None: print('Creating a configuration file') target_config_dir: str = f'{DIR_DST_ROOT}/boot/{image_name}/rw/opt/vyatta/etc/config/' Path(target_config_dir).mkdir(parents=True) + chown(target_config_dir, group='vyattacfg') + chmod_2775(target_config_dir) # copy config if migrate_config(): copy('/opt/vyatta/etc/config/config.boot', target_config_dir) @@ -485,9 +489,16 @@ def add_image(image_path: str) -> None: # copy config if migrate_config(): print('Copying configuration directory') - copytree('/opt/vyatta/etc/config/', target_config_dir) + # copytree preserves perms but not ownership: + Path(target_config_dir).mkdir(parents=True) + chown(target_config_dir, group='vyattacfg') + chmod_2775(target_config_dir) + copytree('/opt/vyatta/etc/config/', target_config_dir, + dirs_exist_ok=True) else: Path(target_config_dir).mkdir(parents=True) + chown(target_config_dir, group='vyattacfg') + chmod_2775(target_config_dir) Path(f'{target_config_dir}/.vyatta_config').touch() # copy system image and kernel files -- cgit v1.2.3 From 169c9ff01287cb558850479afb733dd53fb6ae5d Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 11 Apr 2023 14:51:30 -0500 Subject: image: T4516: correct implementation of configure_authentication --- src/op_mode/image_installer.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) (limited to 'src/op_mode') diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index 77bb6460f..1f3245316 100644 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -21,6 +21,7 @@ from argparse import ArgumentParser, Namespace from pathlib import Path from shutil import copy, chown, rmtree, copytree from sys import exit +from passlib.hosts import linux_context from urllib.parse import urlparse from psutil import disk_partitions @@ -192,15 +193,33 @@ def setup_grub(root_dir: str) -> None: def configure_authentication(config_file: str, password: str) -> None: - config = ConfigTree(config_file) + """Write encrypted password to config file + + Args: + config_file (str): path of target config file + password (str): plaintext password + + N.B. this can not be deferred by simply setting the plaintext password + and relying on the config mode script to process at boot, as the config + will not automatically be saved in that case, thus leaving the + plaintext exposed + """ + encrypted_password = linux_context.hash(password) + + with open(config_file) as f: + config_string = f.read() + + config = ConfigTree(config_string) config.set([ 'system', 'login', 'user', 'vyos', 'authentication', - 'plaintext-password' + 'encrypted-password' ], - value=password, + value=encrypted_password, replace=True) config.set_tag(['system', 'login', 'user']) + with open(config_file, 'w') as f: + f.write(config.to_string()) def validate_signature(file_path: str, sign_type: str) -> None: """Validate a file by signature and delete a signature file -- cgit v1.2.3 From b31092cc33685628c74845f2aa1e94f0e7879e87 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 23 May 2023 11:19:03 -0500 Subject: image: T4516: do not prompt for config copy on live install --- src/op_mode/image_installer.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'src/op_mode') diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index 1f3245316..32ec075e4 100644 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -415,11 +415,7 @@ def install_image() -> None: chown(target_config_dir, group='vyattacfg') chmod_2775(target_config_dir) # copy config - if migrate_config(): - copy('/opt/vyatta/etc/config/config.boot', target_config_dir) - else: - copy('/opt/vyatta/etc/config.boot.default', - f'{target_config_dir}/config.boot') + copy('/opt/vyatta/etc/config/config.boot', target_config_dir) configure_authentication(f'{target_config_dir}/config.boot', user_password) Path(f'{target_config_dir}/.vyatta_config').touch() -- cgit v1.2.3 From d88168b8e26e46d512e3b175cd2eacecae0e596a Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 23 May 2023 11:29:46 -0500 Subject: image: T4516: set op-mode files executable --- src/op_mode/image_info.py | 0 src/op_mode/image_installer.py | 0 src/op_mode/image_manager.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 src/op_mode/image_info.py mode change 100644 => 100755 src/op_mode/image_installer.py mode change 100644 => 100755 src/op_mode/image_manager.py (limited to 'src/op_mode') diff --git a/src/op_mode/image_info.py b/src/op_mode/image_info.py old mode 100644 new mode 100755 diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py old mode 100644 new mode 100755 diff --git a/src/op_mode/image_manager.py b/src/op_mode/image_manager.py old mode 100644 new mode 100755 -- cgit v1.2.3 From a604d5d56d93a6958d879b838066bbe2df131bc5 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 23 May 2023 12:02:12 -0500 Subject: image: T4516: restore reboot reminder message --- src/op_mode/image_installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/op_mode') diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index 32ec075e4..972faef8c 100755 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -40,7 +40,7 @@ MSG_ERR_LIVE: str = 'The system is in live-boot mode. Please use "install image" MSG_ERR_NO_DISK: str = 'No suitable disk was found. There must be at least one disk of 2GB or greater size.' MSG_INFO_INSTALL_WELCOME: str = 'Welcome to VyOS installation!\nThis command will install the VyOS to your permanent storage.' MSG_INFO_INSTALL_EXIT: str = 'Exitting from VyOS installation' -MSG_INFO_INSTALL_SUCCESS: str = 'The image installed successfully' +MSG_INFO_INSTALL_SUCCESS: str = 'The image installed successfully; please reboot now.' MSG_INFO_INSTALL_DISKS_LIST: str = 'Were found the next disks:' MSG_INFO_INSTALL_DISK_SELECT: str = 'Which one should be used for installation?' MSG_INFO_INSTALL_DISK_CONFIRM: str = 'Installation will delete all data on the drive. Continue?' -- cgit v1.2.3 From fcded7930b5426193e8490c6df2a70e300a60e31 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 20 Sep 2023 14:53:24 -0500 Subject: image: T5195: vyos.util -> vyos.utils package refactoring --- python/vyos/system/disk.py | 2 +- python/vyos/system/grub.py | 2 +- src/op_mode/image_installer.py | 2 +- src/op_mode/image_manager.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) (limited to 'src/op_mode') diff --git a/python/vyos/system/disk.py b/python/vyos/system/disk.py index e20cf32be..b78190c06 100644 --- a/python/vyos/system/disk.py +++ b/python/vyos/system/disk.py @@ -18,7 +18,7 @@ from os import sync from psutil import disk_partitions -from vyos.util import run, cmd +from vyos.utils.process import run, cmd def disk_cleanup(drive_path: str) -> None: diff --git a/python/vyos/system/grub.py b/python/vyos/system/grub.py index 11c214675..47c674de8 100644 --- a/python/vyos/system/grub.py +++ b/python/vyos/system/grub.py @@ -19,7 +19,7 @@ from typing import Union from uuid import uuid5, NAMESPACE_URL, UUID from vyos.template import render -from vyos.util import run, cmd +from vyos.utils.process import run, cmd from vyos.system import disk # Define variables diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index 972faef8c..12d32968c 100755 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -32,7 +32,7 @@ from vyos.system import disk, grub, image from vyos.template import render from vyos.utils.io import ask_input, ask_yes_no from vyos.utils.file import chmod_2775 -from vyos.util import run +from vyos.utils.process import run # define text messages MSG_ERR_NOT_LIVE: str = 'The system is already installed. Please use "add system image" instead.' diff --git a/src/op_mode/image_manager.py b/src/op_mode/image_manager.py index ac889da38..76fc4367f 100755 --- a/src/op_mode/image_manager.py +++ b/src/op_mode/image_manager.py @@ -23,7 +23,7 @@ from shutil import rmtree from sys import exit from vyos.system import disk, grub, image -from vyos.util import ask_yes_no +from vyos.utils.io import ask_yes_no def delete_image(image_name: str) -> None: -- cgit v1.2.3 From 8efab9ee8cdb0e65dddb9d3ba97de8ddcf3666dc Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 20 Oct 2023 23:47:20 -0500 Subject: image: T4516: improve format of 'show system image details' --- python/vyos/utils/convert.py | 5 ++++- src/op_mode/image_info.py | 14 +++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) (limited to 'src/op_mode') diff --git a/python/vyos/utils/convert.py b/python/vyos/utils/convert.py index 9a8a1ff7d..c02f0071e 100644 --- a/python/vyos/utils/convert.py +++ b/python/vyos/utils/convert.py @@ -52,7 +52,8 @@ def seconds_to_human(s, separator=""): return result -def bytes_to_human(bytes, initial_exponent=0, precision=2): +def bytes_to_human(bytes, initial_exponent=0, precision=2, + int_below_exponent=0): """ Converts a value in bytes to a human-readable size string like 640 KB The initial_exponent parameter is the exponent of 2, @@ -68,6 +69,8 @@ def bytes_to_human(bytes, initial_exponent=0, precision=2): # log2 is a float, while range checking requires an int exponent = int(log2(bytes)) + if exponent < int_below_exponent: + precision = 0 if exponent < 10: value = bytes diff --git a/src/op_mode/image_info.py b/src/op_mode/image_info.py index ae0677196..14dca7476 100755 --- a/src/op_mode/image_info.py +++ b/src/op_mode/image_info.py @@ -20,11 +20,11 @@ import sys from typing import List, Union -from hurry.filesize import size from tabulate import tabulate from vyos import opmode from vyos.system import disk, grub, image +from vyos.utils.convert import bytes_to_human def _format_show_images_summary(images_summary: image.BootDetails) -> str: @@ -58,11 +58,15 @@ def _format_show_images_details( for image_item in images_details: name: str = image_item.get('name') version: str = image_item.get('version') - disk_ro: int = size(image_item.get('disk_ro')) - disk_rw: int = size(image_item.get('disk_rw')) - disk_total: int = size(image_item.get('disk_total')) + disk_ro: str = bytes_to_human(image_item.get('disk_ro'), + precision=1, int_below_exponent=30) + disk_rw: str = bytes_to_human(image_item.get('disk_rw'), + precision=1, int_below_exponent=30) + disk_total: str = bytes_to_human(image_item.get('disk_total'), + precision=1, int_below_exponent=30) table_data.append([name, version, disk_ro, disk_rw, disk_total]) - tabulated: str = tabulate(table_data, headers) + tabulated: str = tabulate(table_data, headers, + colalign=('left', 'left', 'right', 'right', 'right')) return tabulated -- cgit v1.2.3 From 96b65e90fbfa1fe63d97929ac86fc910abb0caa9 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sat, 21 Oct 2023 13:33:37 -0500 Subject: image: T4516: support for interoperability of legacy/new image tools This commit allows management of system images with either new or legacy tools: 'add/delete/rename system image' and 'set default' are translated appropriately on booting between images with the old and new tools. Consequently, the warning of the initial commit of T4516 is dropped. --- data/templates/grub/grub_compat.j2 | 58 +++++++ python/vyos/system/__init__.py | 4 +- python/vyos/system/compat.py | 307 +++++++++++++++++++++++++++++++++++++ python/vyos/system/disk.py | 2 +- python/vyos/system/grub.py | 7 +- python/vyos/system/image.py | 88 +++++++++-- src/op_mode/image_info.py | 7 +- src/op_mode/image_installer.py | 17 +- src/op_mode/image_manager.py | 7 +- src/system/grub_update.py | 141 ++--------------- 10 files changed, 486 insertions(+), 152 deletions(-) create mode 100644 data/templates/grub/grub_compat.j2 create mode 100644 python/vyos/system/compat.py (limited to 'src/op_mode') diff --git a/data/templates/grub/grub_compat.j2 b/data/templates/grub/grub_compat.j2 new file mode 100644 index 000000000..935172005 --- /dev/null +++ b/data/templates/grub/grub_compat.j2 @@ -0,0 +1,58 @@ +{# j2lint: disable=S6 #} +### Generated by VyOS image-tools v.{{ tools_version }} ### +{% macro menu_name(mode) -%} +{% if mode == 'normal' -%} + VyOS +{%- elif mode == 'pw_reset' -%} + Lost password change +{%- else -%} + Unknown +{%- endif %} +{%- endmacro %} +{% macro console_name(type) -%} +{% if type == 'tty' -%} + KVM +{%- elif type == 'ttyS' -%} + Serial +{%- elif type == 'ttyUSB' -%} + USB +{%- else -%} + Unknown +{%- endif %} +{%- endmacro %} +{% macro console_opts(type) -%} +{% if type == 'tty' -%} + console=ttyS0,115200 console=tty0 +{%- elif type == 'ttyS' -%} + console=tty0 console=ttyS0,115200 +{%- elif type == 'ttyUSB' -%} + console=tty0 console=ttyUSB0,115200 +{%- else -%} + console=tty0 console=ttyS0,115200 +{%- endif %} +{%- endmacro %} +{% macro passwd_opts(mode) -%} +{% if mode == 'pw_reset' -%} + init=/opt/vyatta/sbin/standalone_root_pw_reset +{%- endif %} +{%- endmacro %} +set default={{ default }} +set timeout={{ timeout }} +{% if console_type == 'ttyS' %} +serial --unit={{ console_num }} --speed=115200 +{% else %} +serial --unit=0 --speed=115200 +{% endif %} +terminal_output --append serial +terminal_input serial console +{% if efi %} +insmod efi_gop +insmod efi_uga +{% endif %} + +{% for v in versions %} +menuentry "{{ menu_name(v.bootmode) }} {{ v.version }} ({{ console_name(v.console_type) }} console)" { + linux /boot/{{ v.version }}/vmlinuz {{ v.boot_opts }} {{ console_opts(v.console_type) }} {{ passwd_opts(v.bootmode) }} + initrd /boot/{{ v.version }}/initrd.img +} +{% endfor %} diff --git a/python/vyos/system/__init__.py b/python/vyos/system/__init__.py index 403738e20..0c91330ba 100644 --- a/python/vyos/system/__init__.py +++ b/python/vyos/system/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors +# Copyright 2023 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -14,3 +14,5 @@ # License along with this library. If not, see . __all_: list[str] = ['disk', 'grub', 'image'] +# define image-tools version +SYSTEM_CFG_VER = 1 diff --git a/python/vyos/system/compat.py b/python/vyos/system/compat.py new file mode 100644 index 000000000..aa9b0b4b5 --- /dev/null +++ b/python/vyos/system/compat.py @@ -0,0 +1,307 @@ +# Copyright 2023 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see . + +from pathlib import Path +from re import compile, MULTILINE, DOTALL +from functools import wraps +from copy import deepcopy +from typing import Union + +from vyos.system import disk, grub, image, SYSTEM_CFG_VER +from vyos.template import render + +TMPL_GRUB_COMPAT: str = 'grub/grub_compat.j2' + +# define regexes and variables +REGEX_VERSION = r'^menuentry "[^\n]*{\n[^}]*\s+linux /boot/(?P\S+)/[^}]*}' +REGEX_MENUENTRY = r'^menuentry "[^\n]*{\n[^}]*\s+linux /boot/(?P\S+)/vmlinuz (?P[^\n]+)\n[^}]*}' +REGEX_CONSOLE = r'^.*console=(?P[^\s\d]+)(?P[\d]+).*$' +REGEX_SANIT_CONSOLE = r'\ ?console=[^\s\d]+[\d]+(,\d+)?\ ?' +REGEX_SANIT_INIT = r'\ ?init=\S*\ ?' +REGEX_SANIT_QUIET = r'\ ?quiet\ ?' +PW_RESET_OPTION = 'init=/opt/vyatta/sbin/standalone_root_pw_reset' + + +class DowngradingImageTools(Exception): + """Raised when attempting to add an image with an earlier version + of image-tools than the current system, as indicated by the value + of SYSTEM_CFG_VER or absence thereof.""" + pass + + +def mode(): + if grub.get_cfg_ver() >= SYSTEM_CFG_VER: + return False + + return True + + +def find_versions(menu_entries: list) -> list: + """Find unique VyOS versions from menu entries + + Args: + menu_entries (list): a list with menu entries + + Returns: + list: List of installed versions + """ + versions = [] + for vyos_ver in menu_entries: + versions.append(vyos_ver.get('version')) + # remove duplicates + versions = list(set(versions)) + return versions + + +def filter_unparsed(grub_path: str) -> str: + """Find currently installed VyOS version + + Args: + grub_path (str): a path to the grub.cfg file + + Returns: + str: unparsed grub.cfg items + """ + config_text = Path(grub_path).read_text() + regex_filter = compile(REGEX_VERSION, MULTILINE | DOTALL) + filtered = regex_filter.sub('', config_text) + regex_filter = compile(grub.REGEX_GRUB_VARS, MULTILINE) + filtered = regex_filter.sub('', filtered) + regex_filter = compile(grub.REGEX_GRUB_MODULES, MULTILINE) + filtered = regex_filter.sub('', filtered) + # strip extra new lines + filtered = filtered.strip() + return filtered + + +def sanitize_boot_opts(boot_opts: str) -> str: + """Sanitize boot options from console and init + + Args: + boot_opts (str): boot options + + Returns: + str: sanitized boot options + """ + regex_filter = compile(REGEX_SANIT_CONSOLE) + boot_opts = regex_filter.sub('', boot_opts) + regex_filter = compile(REGEX_SANIT_INIT) + boot_opts = regex_filter.sub('', boot_opts) + # legacy tools add 'quiet' on add system image; this is not desired + regex_filter = compile(REGEX_SANIT_QUIET) + boot_opts = regex_filter.sub(' ', boot_opts) + + return boot_opts + + +def parse_entry(entry: tuple) -> dict: + """Parse GRUB menuentry + + Args: + entry (tuple): tuple of (version, options) + + Returns: + dict: dictionary with parsed options + """ + # save version to dict + entry_dict = {'version': entry[0]} + # detect boot mode type + if PW_RESET_OPTION in entry[1]: + entry_dict['bootmode'] = 'pw_reset' + else: + entry_dict['bootmode'] = 'normal' + # find console type and number + regex_filter = compile(REGEX_CONSOLE) + entry_dict.update(regex_filter.match(entry[1]).groupdict()) + entry_dict['boot_opts'] = sanitize_boot_opts(entry[1]) + + return entry_dict + + +def parse_menuentries(grub_path: str) -> list: + """Parse all GRUB menuentries + + Args: + grub_path (str): a path to GRUB config file + + Returns: + list: list with menu items (each item is a dict) + """ + menuentries = [] + # read configuration file + config_text = Path(grub_path).read_text() + # parse menuentries to tuples (version, options) + regex_filter = compile(REGEX_MENUENTRY, MULTILINE) + filter_results = regex_filter.findall(config_text) + # parse each entry + for entry in filter_results: + menuentries.append(parse_entry(entry)) + + return menuentries + + +def prune_vyos_versions(root_dir: str = '') -> None: + """Delete vyos-versions files of registered images subsequently deleted + or renamed by legacy image-tools + + Args: + root_dir (str): an optional path to the root directory + """ + if not root_dir: + root_dir = disk.find_persistence() + + for version in grub.version_list(): + if not Path(f'{root_dir}/boot/{version}').is_dir(): + grub.version_del(version) + + +def update_cfg_ver(root_dir:str = '') -> int: + """Get minumum version of image-tools across all installed images + + Args: + root_dir (str): an optional path to the root directory + + Returns: + int: minimum version of image-tools + """ + if not root_dir: + root_dir = disk.find_persistence() + + prune_vyos_versions(root_dir) + + images_details = image.get_images_details() + cfg_version = min(d['tools_version'] for d in images_details) + + return cfg_version + + +def get_default(menu_entries: list, root_dir: str = '') -> Union[int, None]: + """Translate default version to menuentry index + + Args: + menu_entries (list): list of dicts of installed version boot data + root_dir (str): an optional path to the root directory + + Returns: + int: index of default version in menu_entries or None + """ + if not root_dir: + root_dir = disk.find_persistence() + + grub_cfg_main = f'{root_dir}/{grub.GRUB_CFG_MAIN}' + + image_name = image.get_default_image() + + sublist = list(filter(lambda x: x.get('version') == image_name, + menu_entries)) + if sublist: + return menu_entries.index(sublist[0]) + + return None + + +def update_version_list(root_dir: str = '') -> list[dict]: + """Update list of dicts of installed version boot data + + Args: + root_dir (str): an optional path to the root directory + + Returns: + list: list of dicts of installed version boot data + """ + if not root_dir: + root_dir = disk.find_persistence() + + grub_cfg_main = f'{root_dir}/{grub.GRUB_CFG_MAIN}' + + # get list of versions in menuentries + menu_entries = parse_menuentries(grub_cfg_main) + menu_versions = find_versions(menu_entries) + + # get list of versions added/removed by image-tools + current_versions = grub.version_list(root_dir) + + remove = list(set(menu_versions) - set(current_versions)) + for ver in remove: + menu_entries = list(filter(lambda x: x.get('version') != ver, + menu_entries)) + + add = list(set(current_versions) - set(menu_versions)) + for ver in add: + last = menu_entries[0].get('version') + new = deepcopy(list(filter(lambda x: x.get('version') == last, + menu_entries))) + for e in new: + boot_opts = e.get('boot_opts').replace(last, ver) + e.update({'version': ver, 'boot_opts': boot_opts}) + + menu_entries = new + menu_entries + + return menu_entries + + +def grub_cfg_fields(root_dir: str = '') -> dict: + """Gather fields for rendering grub.cfg + + Args: + root_dir (str): an optional path to the root directory + + Returns: + dict: dictionary for rendering TMPL_GRUB_COMPAT + """ + if not root_dir: + root_dir = disk.find_persistence() + + grub_cfg_main = f'{root_dir}/{grub.GRUB_CFG_MAIN}' + + fields = {'default': 0, 'timeout': 5} + # 'default' and 'timeout' from legacy grub.cfg + fields |= grub.vars_read(grub_cfg_main) + fields['tools_version'] = SYSTEM_CFG_VER + menu_entries = update_version_list(root_dir) + fields['versions'] = menu_entries + default = get_default(menu_entries, root_dir) + if default is not None: + fields['default'] = default + + p = Path('/sys/firmware/efi') + if p.is_dir(): + fields['efi'] = True + else: + fields['efi'] = False + + return fields + + +def render_grub_cfg(root_dir: str = '') -> None: + """Render grub.cfg for legacy compatibility""" + if not root_dir: + root_dir = disk.find_persistence() + + grub_cfg_main = f'{root_dir}/{grub.GRUB_CFG_MAIN}' + + fields = grub_cfg_fields(root_dir) + render(grub_cfg_main, TMPL_GRUB_COMPAT, fields) + + +def grub_cfg_update(func): + """Decorator to update grub.cfg after function call""" + @wraps(func) + def wrapper(*args, **kwargs): + ret = func(*args, **kwargs) + if mode(): + render_grub_cfg() + return ret + return wrapper diff --git a/python/vyos/system/disk.py b/python/vyos/system/disk.py index b78190c06..882b4eb39 100644 --- a/python/vyos/system/disk.py +++ b/python/vyos/system/disk.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors +# Copyright 2023 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/system/grub.py b/python/vyos/system/grub.py index 47c674de8..9ac205c03 100644 --- a/python/vyos/system/grub.py +++ b/python/vyos/system/grub.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors +# Copyright 2023 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -24,6 +24,7 @@ from vyos.system import disk # Define variables GRUB_DIR_MAIN: str = '/boot/grub' +GRUB_CFG_MAIN: str = f'{GRUB_DIR_MAIN}/grub.cfg' GRUB_DIR_VYOS: str = f'{GRUB_DIR_MAIN}/grub.cfg.d' CFG_VYOS_HEADER: str = f'{GRUB_DIR_VYOS}/00-vyos-header.cfg' CFG_VYOS_MODULES: str = f'{GRUB_DIR_VYOS}/10-vyos-modules-autoload.cfg' @@ -181,8 +182,8 @@ def get_cfg_ver(root_dir: str = '') -> int: if not root_dir: root_dir = disk.find_persistence() - cfg_ver: Union[str, None] = vars_read(f'{root_dir}/{CFG_VYOS_HEADER}').get( - 'VYOS_CFG_VER') + cfg_ver: str = vars_read(f'{root_dir}/{CFG_VYOS_HEADER}').get( + 'VYOS_CFG_VER') if cfg_ver: cfg_ver_int: int = int(cfg_ver) else: diff --git a/python/vyos/system/image.py b/python/vyos/system/image.py index b77c3563f..6c4e3bba5 100644 --- a/python/vyos/system/image.py +++ b/python/vyos/system/image.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors +# Copyright 2023 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -28,12 +28,14 @@ CFG_VYOS_VARS: str = f'{GRUB_DIR_VYOS}/20-vyos-defaults-autoload.cfg' GRUB_DIR_VYOS_VERS: str = f'{GRUB_DIR_VYOS}/vyos-versions' # prepare regexes REGEX_KERNEL_CMDLINE: str = r'^BOOT_IMAGE=/(?Pboot|live)/((?P.+)/)?vmlinuz.*$' +REGEX_SYSTEM_CFG_VER: str = r'(\r\n|\r|\n)SYSTEM_CFG_VER\s*=\s*(?P\d+)(\r\n|\r|\n)' # structures definitions class ImageDetails(TypedDict): name: str version: str + tools_version: int disk_ro: int disk_rw: int disk_total: int @@ -59,26 +61,73 @@ def bootmode_detect() -> str: return 'bios' -def get_version(image_name: str, root_dir: str) -> str: - """Extract version name from rootfs based on image name +def get_image_version(mount_path: str) -> str: + """Extract version name from rootfs mounted at mount_path Args: - image_name (str): a name of image (from boot menu) - root_dir (str): a root directory of persistence storage + mount_path (str): mount path of rootfs Returns: str: version name """ + version_file: str = Path( + f'{mount_path}/opt/vyatta/etc/version').read_text() + version_name: str = version_file.lstrip('Version: ').strip() + + return version_name + + +def get_image_tools_version(mount_path: str) -> int: + """Extract image-tools version from rootfs mounted at mount_path + + Args: + mount_path (str): mount path of rootfs + + Returns: + str: image-tools version + """ + try: + version_file: str = Path( + f'{mount_path}/usr/lib/python3/dist-packages/vyos/system/__init__.py').read_text() + except FileNotFoundError: + system_cfg_ver: int = 0 + else: + res = re_compile(REGEX_SYSTEM_CFG_VER).search(version_file) + system_cfg_ver: int = int(res.groupdict().get('cfg_ver', 0)) + + return system_cfg_ver + + +def get_versions(image_name: str, root_dir: str = '') -> dict[str, str]: + """Return versions of image and image-tools + + Args: + image_name (str): a name of an image + root_dir (str, optional): an optional path to the root directory. + Defaults to ''. + + Returns: + dict[str, int]: a dictionary with versions of image and image-tools + """ + if not root_dir: + root_dir = disk.find_persistence() + squashfs_file: str = next( Path(f'{root_dir}/boot/{image_name}').glob('*.squashfs')).as_posix() with TemporaryDirectory() as squashfs_mounted: disk.partition_mount(squashfs_file, squashfs_mounted, 'squashfs') - version_file: str = Path( - f'{squashfs_mounted}/opt/vyatta/etc/version').read_text() + + image_version: str = get_image_version(squashfs_mounted) + image_tools_version: int = get_image_tools_version(squashfs_mounted) + disk.partition_umount(squashfs_file) - version_name: str = version_file.lstrip('Version: ').strip() - return version_name + versions: dict[str, int] = { + 'image': image_version, + 'image-tools': image_tools_version + } + + return versions def get_details(image_name: str, root_dir: str = '') -> ImageDetails: @@ -95,7 +144,9 @@ def get_details(image_name: str, root_dir: str = '') -> ImageDetails: if not root_dir: root_dir = disk.find_persistence() - image_version: str = get_version(image_name, root_dir) + versions = get_versions(image_name, root_dir) + image_version: str = versions.get('image', '') + image_tools_version: int = versions.get('image-tools', 0) image_path: Path = Path(f'{root_dir}/boot/{image_name}') image_path_rw: Path = Path(f'{root_dir}/boot/{image_name}/rw') @@ -113,6 +164,7 @@ def get_details(image_name: str, root_dir: str = '') -> ImageDetails: image_details: ImageDetails = { 'name': image_name, 'version': image_version, + 'tools_version': image_tools_version, 'disk_ro': image_disk_ro, 'disk_rw': image_disk_rw, 'disk_total': image_disk_ro + image_disk_rw @@ -121,6 +173,20 @@ def get_details(image_name: str, root_dir: str = '') -> ImageDetails: return image_details +def get_images_details() -> list[ImageDetails]: + """Return information about all images + + Returns: + list[ImageDetails]: a list of dictionaries with details about images + """ + images: list[str] = grub.version_list() + images_details: list[ImageDetails] = list() + for image_name in images: + images_details.append(get_details(image_name)) + + return images_details + + def get_running_image() -> str: """Find currently running image name @@ -134,7 +200,7 @@ def get_running_image() -> str: if running_image_result: running_image: str = running_image_result.groupdict().get( 'image_version', '') - # we need to have a fallbak for live systems + # we need to have a fallback for live systems if not running_image: running_image: str = version.get_version() diff --git a/src/op_mode/image_info.py b/src/op_mode/image_info.py index 14dca7476..791001e00 100755 --- a/src/op_mode/image_info.py +++ b/src/op_mode/image_info.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2022 VyOS maintainers and contributors +# Copyright 2023 VyOS maintainers and contributors # # This file is part of VyOS. # @@ -91,10 +91,7 @@ def show_images_summary(raw: bool) -> Union[image.BootDetails, str]: def show_images_details(raw: bool) -> Union[list[image.ImageDetails], str]: - images: list[str] = grub.version_list() - images_details: list[image.ImageDetails] = list() - for image_name in images: - images_details.append(image.get_details(image_name)) + images_details = image.get_images_details() if raw: return images_details diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index 12d32968c..2d998f5e1 100755 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2022 VyOS maintainers and contributors +# Copyright 2023 VyOS maintainers and contributors # # This file is part of VyOS. # @@ -28,7 +28,7 @@ from psutil import disk_partitions from vyos.configtree import ConfigTree from vyos.remote import download -from vyos.system import disk, grub, image +from vyos.system import disk, grub, image, compat, SYSTEM_CFG_VER from vyos.template import render from vyos.utils.io import ask_input, ask_yes_no from vyos.utils.file import chmod_2775 @@ -462,6 +462,7 @@ def install_image() -> None: exit(1) +@compat.grub_cfg_update def add_image(image_path: str) -> None: """Add a new image @@ -488,10 +489,16 @@ def add_image(image_path: str) -> None: Path(DIR_ROOTFS_SRC).mkdir(mode=0o755, parents=True) disk.partition_mount(f'{DIR_ISO_MOUNT}/live/filesystem.squashfs', DIR_ROOTFS_SRC, 'squashfs') - version_file: str = Path( - f'{DIR_ROOTFS_SRC}/opt/vyatta/etc/version').read_text() + + cfg_ver: str = image.get_image_tools_version(DIR_ROOTFS_SRC) + version_name: str = image.get_image_version(DIR_ROOTFS_SRC) + disk.partition_umount(f'{DIR_ISO_MOUNT}/live/filesystem.squashfs') - version_name: str = version_file.lstrip('Version: ').strip() + + if cfg_ver < SYSTEM_CFG_VER: + raise compat.DowngradingImageTools( + f'Adding image would downgrade image tools to v.{cfg_ver}; disallowed') + image_name: str = ask_input(MSG_INPUT_IMAGE_NAME, version_name) set_as_default: bool = ask_yes_no(MSG_INPUT_IMAGE_DEFAULT, default=True) diff --git a/src/op_mode/image_manager.py b/src/op_mode/image_manager.py index 76fc4367f..725c40613 100755 --- a/src/op_mode/image_manager.py +++ b/src/op_mode/image_manager.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2022 VyOS maintainers and contributors +# Copyright 2023 VyOS maintainers and contributors # # This file is part of VyOS. # @@ -22,10 +22,11 @@ from pathlib import Path from shutil import rmtree from sys import exit -from vyos.system import disk, grub, image +from vyos.system import disk, grub, image, compat from vyos.utils.io import ask_yes_no +@compat.grub_cfg_update def delete_image(image_name: str) -> None: """Remove installed image files and boot entry @@ -57,6 +58,7 @@ def delete_image(image_name: str) -> None: exit(f'Unable to remove the image "{image_name}": {err}') +@compat.grub_cfg_update def set_image(image_name: str) -> None: """Set default boot image @@ -86,6 +88,7 @@ def set_image(image_name: str) -> None: exit(f'Unable to set default image "{image_name}": {err}') +@compat.grub_cfg_update def rename_image(name_old: str, name_new: str) -> None: """Rename installed image diff --git a/src/system/grub_update.py b/src/system/grub_update.py index 1ae66464b..da1986e9d 100644 --- a/src/system/grub_update.py +++ b/src/system/grub_update.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2022 VyOS maintainers and contributors +# Copyright 2023 VyOS maintainers and contributors # # This file is part of VyOS. # @@ -18,23 +18,11 @@ # VyOS. If not, see . from pathlib import Path -from re import compile, MULTILINE, DOTALL from sys import exit -from vyos.system import disk, grub, image +from vyos.system import disk, grub, image, compat, SYSTEM_CFG_VER from vyos.template import render -# define configuration version -CFG_VER = 1 - -# define regexes and variables -REGEX_VERSION = r'^menuentry "[^\n]*{\n[^}]*\s+linux /boot/(?P\S+)/[^}]*}' -REGEX_MENUENTRY = r'^menuentry "[^\n]*{\n[^}]*\s+linux /boot/(?P\S+)/vmlinuz (?P[^\n]+)\n[^}]*}' -REGEX_CONSOLE = r'^.*console=(?P[^\s\d]+)(?P[\d]+).*$' -REGEX_SANIT_CONSOLE = r'\ ?console=[^\s\d]+[\d]+(,\d+)?\ ?' -REGEX_SANIT_INIT = r'\ ?init=\S*\ ?' -PW_RESET_OPTION = 'init=/opt/vyatta/sbin/standalone_root_pw_reset' - def cfg_check_update() -> bool: """Check if GRUB structure update is required @@ -43,111 +31,10 @@ def cfg_check_update() -> bool: bool: False if not required, True if required """ current_ver = grub.get_cfg_ver() - if current_ver and current_ver >= CFG_VER: + if current_ver and current_ver >= SYSTEM_CFG_VER: return False - else: - return True - - -def find_versions(menu_entries: list) -> list: - """Find unique VyOS versions from menu entries - - Args: - menu_entries (list): a list with menu entries - - Returns: - list: List of installed versions - """ - versions = [] - for vyos_ver in menu_entries: - versions.append(vyos_ver.get('version')) - # remove duplicates - versions = list(set(versions)) - return versions - -def filter_unparsed(grub_path: str) -> str: - """Find currently installed VyOS version - - Args: - grub_path (str): a path to the grub.cfg file - - Returns: - str: unparsed grub.cfg items - """ - config_text = Path(grub_path).read_text() - regex_filter = compile(REGEX_VERSION, MULTILINE | DOTALL) - filtered = regex_filter.sub('', config_text) - regex_filter = compile(grub.REGEX_GRUB_VARS, MULTILINE) - filtered = regex_filter.sub('', filtered) - regex_filter = compile(grub.REGEX_GRUB_MODULES, MULTILINE) - filtered = regex_filter.sub('', filtered) - # strip extra new lines - filtered = filtered.strip() - return filtered - - -def sanitize_boot_opts(boot_opts: str) -> str: - """Sanitize boot options from console and init - - Args: - boot_opts (str): boot options - - Returns: - str: sanitized boot options - """ - regex_filter = compile(REGEX_SANIT_CONSOLE) - boot_opts = regex_filter.sub('', boot_opts) - regex_filter = compile(REGEX_SANIT_INIT) - boot_opts = regex_filter.sub('', boot_opts) - - return boot_opts - - -def parse_entry(entry: tuple) -> dict: - """Parse GRUB menuentry - - Args: - entry (tuple): tuple of (version, options) - - Returns: - dict: dictionary with parsed options - """ - # save version to dict - entry_dict = {'version': entry[0]} - # detect boot mode type - if PW_RESET_OPTION in entry[1]: - entry_dict['bootmode'] = 'pw_reset' - else: - entry_dict['bootmode'] = 'normal' - # find console type and number - regex_filter = compile(REGEX_CONSOLE) - entry_dict.update(regex_filter.match(entry[1]).groupdict()) - entry_dict['boot_opts'] = sanitize_boot_opts(entry[1]) - - return entry_dict - - -def parse_menuntries(grub_path: str) -> list: - """Parse all GRUB menuentries - - Args: - grub_path (str): a path to GRUB config file - - Returns: - list: list with menu items (each item is a dict) - """ - menuentries = [] - # read configuration file - config_text = Path(grub_path).read_text() - # parse menuentries to tuples (version, options) - regex_filter = compile(REGEX_MENUENTRY, MULTILINE) - filter_results = regex_filter.findall(config_text) - # parse each entry - for entry in filter_results: - menuentries.append(parse_entry(entry)) - - return menuentries + return True if __name__ == '__main__': @@ -162,12 +49,12 @@ if __name__ == '__main__': root_dir = disk.find_persistence() # read current GRUB config - grub_cfg_main = f'{root_dir}/{image.GRUB_DIR_MAIN}/grub.cfg' + grub_cfg_main = f'{root_dir}/{grub.GRUB_CFG_MAIN}' vars = grub.vars_read(grub_cfg_main) modules = grub.modules_read(grub_cfg_main) - vyos_menuentries = parse_menuntries(grub_cfg_main) - vyos_versions = find_versions(vyos_menuentries) - unparsed_items = filter_unparsed(grub_cfg_main) + vyos_menuentries = compat.parse_menuentries(grub_cfg_main) + vyos_versions = compat.find_versions(vyos_menuentries) + unparsed_items = compat.filter_unparsed(grub_cfg_main) # find default values default_entry = vyos_menuentries[int(vars['default'])] @@ -185,13 +72,12 @@ if __name__ == '__main__': # print(f'unparsed_items: {unparsed_items}') # create new files - grub_cfg_vars = f'{root_dir}/{image.CFG_VYOS_VARS}' + grub_cfg_vars = f'{root_dir}/{grub.CFG_VYOS_VARS}' grub_cfg_modules = f'{root_dir}/{grub.CFG_VYOS_MODULES}' grub_cfg_platform = f'{root_dir}/{grub.CFG_VYOS_PLATFORM}' grub_cfg_menu = f'{root_dir}/{grub.CFG_VYOS_MENU}' grub_cfg_options = f'{root_dir}/{grub.CFG_VYOS_OPTIONS}' - render(grub_cfg_main, grub.TMPL_GRUB_MAIN, {}) Path(image.GRUB_DIR_VYOS).mkdir(exist_ok=True) grub.vars_write(grub_cfg_vars, vars) grub.modules_write(grub_cfg_modules, modules) @@ -210,5 +96,12 @@ if __name__ == '__main__': grub.version_add(vyos_ver, root_dir, boot_opts) # update structure version - grub.write_cfg_ver(CFG_VER, root_dir) + cfg_ver = compat.update_cfg_ver(root_dir) + grub.write_cfg_ver(cfg_ver, root_dir) + + if compat.mode(): + compat.render_grub_cfg(root_dir) + else: + render(grub_cfg_main, grub.TMPL_GRUB_MAIN, {}) + exit(0) -- cgit v1.2.3 From aae1247da61206d7a1b0b4d6ee20d36d194dbaba Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 29 Oct 2023 19:13:31 -0500 Subject: image: T4516: add clearer error msg on attempt to upgrade to 1.2.x An attempt to upgrade to 1.2.x is caught, but error is of failed checksum verification; add check and message. --- src/op_mode/image_installer.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/op_mode') diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index 2d998f5e1..894eb6ee9 100755 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -38,6 +38,7 @@ from vyos.utils.process import run MSG_ERR_NOT_LIVE: str = 'The system is already installed. Please use "add system image" instead.' MSG_ERR_LIVE: str = 'The system is in live-boot mode. Please use "install image" instead.' MSG_ERR_NO_DISK: str = 'No suitable disk was found. There must be at least one disk of 2GB or greater size.' +MSG_ERR_IMPROPER_IMAGE: str = 'Missing sha256sum.txt.\nEither this image is corrupted, or of era 1.2.x (md5sum) and would downgrade image tools;\ndisallowed in either case.' MSG_INFO_INSTALL_WELCOME: str = 'Welcome to VyOS installation!\nThis command will install the VyOS to your permanent storage.' MSG_INFO_INSTALL_EXIT: str = 'Exitting from VyOS installation' MSG_INFO_INSTALL_SUCCESS: str = 'The image installed successfully; please reboot now.' @@ -481,6 +482,9 @@ def add_image(image_path: str) -> None: # check sums print('Validating image checksums') + if not Path(DIR_ISO_MOUNT).joinpath('sha256sum.txt').exists(): + cleanup() + exit(MSG_ERR_IMPROPER_IMAGE) if run(f'cd {DIR_ISO_MOUNT} && sha256sum --status -c sha256sum.txt'): cleanup() exit('Image checksum verification failed.') -- cgit v1.2.3 From cdc5fddfd796ccf7cfe35d2501cb1da380df53b2 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 29 Oct 2023 20:07:55 -0500 Subject: image: T4516: reword some messages and prompts --- src/op_mode/image_installer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src/op_mode') diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index 894eb6ee9..88d5eeb48 100755 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -39,10 +39,10 @@ MSG_ERR_NOT_LIVE: str = 'The system is already installed. Please use "add system MSG_ERR_LIVE: str = 'The system is in live-boot mode. Please use "install image" instead.' MSG_ERR_NO_DISK: str = 'No suitable disk was found. There must be at least one disk of 2GB or greater size.' MSG_ERR_IMPROPER_IMAGE: str = 'Missing sha256sum.txt.\nEither this image is corrupted, or of era 1.2.x (md5sum) and would downgrade image tools;\ndisallowed in either case.' -MSG_INFO_INSTALL_WELCOME: str = 'Welcome to VyOS installation!\nThis command will install the VyOS to your permanent storage.' -MSG_INFO_INSTALL_EXIT: str = 'Exitting from VyOS installation' +MSG_INFO_INSTALL_WELCOME: str = 'Welcome to VyOS installation!\nThis command will install VyOS to your permanent storage.' +MSG_INFO_INSTALL_EXIT: str = 'Exiting from VyOS installation' MSG_INFO_INSTALL_SUCCESS: str = 'The image installed successfully; please reboot now.' -MSG_INFO_INSTALL_DISKS_LIST: str = 'Were found the next disks:' +MSG_INFO_INSTALL_DISKS_LIST: str = 'The following disks were found:' MSG_INFO_INSTALL_DISK_SELECT: str = 'Which one should be used for installation?' MSG_INFO_INSTALL_DISK_CONFIRM: str = 'Installation will delete all data on the drive. Continue?' MSG_INFO_INSTALL_PARTITONING: str = 'Creating partition table...' @@ -50,8 +50,8 @@ MSG_INPUT_CONFIG_FOUND: str = 'An active configuration was found. Would you like MSG_INPUT_IMAGE_NAME: str = 'What would you like to name this image?' MSG_INPUT_IMAGE_DEFAULT: str = 'Would you like to set a new image as default one for boot?' MSG_INPUT_PASSWORD: str = 'Please enter a password for the "vyos" user' -MSG_INPUT_ROOT_SIZE_ALL: str = 'Would you like to use all free space on the drive?' -MSG_INPUT_ROOT_SIZE_SET: str = 'What should be a size (in GB) of the root partition (min is 1.5 GB)?' +MSG_INPUT_ROOT_SIZE_ALL: str = 'Would you like to use all the free space on the drive?' +MSG_INPUT_ROOT_SIZE_SET: str = 'Please specify the size (in GB) of the root partition (min is 1.5 GB)?' MSG_INPUT_CONSOLE_TYPE: str = 'What console should be used by default? (K: KVM, S: Serial, U: USB-Serial)?' MSG_WARN_ISO_SIGN_INVALID: str = 'Signature is not valid. Do you want to continue with installation?' MSG_WARN_ISO_SIGN_UNAVAL: str = 'Signature is not available. Do you want to continue with installation?' -- cgit v1.2.3 From 3d15cfd484e8c2732d9f10e4065f2282f1f5d334 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 31 Oct 2023 10:42:18 -0500 Subject: image: T4516: do not prompt for confirmation when setting default --- src/op_mode/image_manager.py | 6 ------ 1 file changed, 6 deletions(-) (limited to 'src/op_mode') diff --git a/src/op_mode/image_manager.py b/src/op_mode/image_manager.py index 725c40613..55fd5c07d 100755 --- a/src/op_mode/image_manager.py +++ b/src/op_mode/image_manager.py @@ -74,12 +74,6 @@ def set_image(image_name: str) -> None: if not presistence_storage: exit('Persistence storage cannot be found') - if not ask_yes_no( - f'Do you really want to set the image {image_name} ' - 'as default boot image?', - default=False): - exit() - # set default boot image try: grub.set_default(image_name, presistence_storage) -- cgit v1.2.3 From 9ffa3e82d951756696367578dd5e82ef0f690065 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 31 Oct 2023 12:53:52 -0500 Subject: image: T4516: restore select entry to set/delete image --- op-mode-definitions/system-image.xml.in | 12 ++++++++++ python/vyos/utils/io.py | 19 +++++++++++++++ src/op_mode/image_manager.py | 42 ++++++++++++++++++++++++--------- 3 files changed, 62 insertions(+), 11 deletions(-) (limited to 'src/op_mode') diff --git a/op-mode-definitions/system-image.xml.in b/op-mode-definitions/system-image.xml.in index 57aeb7bb4..463b985d6 100644 --- a/op-mode-definitions/system-image.xml.in +++ b/op-mode-definitions/system-image.xml.in @@ -77,6 +77,12 @@ Set system image parameters + + + Set default image to boot. + + sudo ${vyos_op_scripts_dir}/image_manager.py --action set + Set default image to boot. @@ -115,6 +121,12 @@ Delete system objects + + + Remove an installed image from the system + + sudo ${vyos_op_scripts_dir}/image_manager.py --action delete + Remove an installed image from the system diff --git a/python/vyos/utils/io.py b/python/vyos/utils/io.py index 8790cbaac..e34a1ba32 100644 --- a/python/vyos/utils/io.py +++ b/python/vyos/utils/io.py @@ -72,3 +72,22 @@ def is_dumb_terminal(): """Check if the current TTY is dumb, so that we can disable advanced terminal features.""" import os return os.getenv('TERM') in ['vt100', 'dumb'] + +def select_entry(l: list, list_msg: str = '', prompt_msg: str = '') -> str: + """Select an entry from a list + + Args: + l (list): a list of entries + list_msg (str): a message to print before listing the entries + prompt_msg (str): a message to print as prompt for selection + + Returns: + str: a selected entry + """ + en = list(enumerate(l, 1)) + print(list_msg) + for i, e in en: + print(f'\t{i}: {e}') + select = ask_input(prompt_msg, numeric_only=True, + valid_responses=range(1, len(l)+1)) + return next(filter(lambda x: x[0] == select, en))[1] diff --git a/src/op_mode/image_manager.py b/src/op_mode/image_manager.py index 55fd5c07d..de53c4cf0 100755 --- a/src/op_mode/image_manager.py +++ b/src/op_mode/image_manager.py @@ -21,23 +21,39 @@ from argparse import ArgumentParser, Namespace from pathlib import Path from shutil import rmtree from sys import exit +from typing import Optional from vyos.system import disk, grub, image, compat -from vyos.utils.io import ask_yes_no +from vyos.utils.io import ask_yes_no, select_entry + +SET_IMAGE_LIST_MSG: str = 'The following images are available:' +SET_IMAGE_PROMPT_MSG: str = 'Select an image to set as default:' +DELETE_IMAGE_LIST_MSG: str = 'The following images are installed:' +DELETE_IMAGE_PROMPT_MSG: str = 'Select an image to delete:' +MSG_DELETE_IMAGE_RUNNING: str = 'Currently running image cannot be deleted; reboot into another image first' +MSG_DELETE_IMAGE_DEFAULT: str = 'Default image cannot be deleted; set another image as default first' @compat.grub_cfg_update -def delete_image(image_name: str) -> None: +def delete_image(image_name: Optional[str] = None, + prompt: bool = True) -> None: """Remove installed image files and boot entry Args: image_name (str): a name of image to delete """ + available_images: list[str] = grub.version_list() + if image_name is None: + if not prompt: + exit('An image name is required for delete action') + else: + image_name = select_entry(available_images, + DELETE_IMAGE_LIST_MSG, + DELETE_IMAGE_PROMPT_MSG) if image_name == image.get_running_image(): - exit('Currently running image cannot be deleted') + exit(MSG_DELETE_IMAGE_RUNNING) if image_name == image.get_default_image(): - exit('Default image cannot be deleted') - available_images: list[str] = grub.version_list() + exit(MSG_DELETE_IMAGE_DEFAULT) if image_name not in available_images: exit(f'The image "{image_name}" cannot be found') presistence_storage: str = disk.find_persistence() @@ -59,15 +75,23 @@ def delete_image(image_name: str) -> None: @compat.grub_cfg_update -def set_image(image_name: str) -> None: +def set_image(image_name: Optional[str] = None, + prompt: bool = True) -> None: """Set default boot image Args: image_name (str): an image name """ + available_images: list[str] = grub.version_list() + if image_name is None: + if not prompt: + exit('An image name is required for set action') + else: + image_name = select_entry(available_images, + SET_IMAGE_LIST_MSG, + SET_IMAGE_PROMPT_MSG) if image_name == image.get_default_image(): exit(f'The image "{image_name}" already configured as default') - available_images: list[str] = grub.version_list() if image_name not in available_images: exit(f'The image "{image_name}" cannot be found') presistence_storage: str = disk.find_persistence() @@ -154,10 +178,6 @@ def parse_arguments() -> Namespace: parser.add_argument('--image_new_name', help='a new name for image') args: Namespace = parser.parse_args() # Validate arguments - if args.action == 'delete' and not args.image_name: - exit('An image name is required for delete action') - if args.action == 'set' and not args.image_name: - exit('An image name is required for set action') if args.action == 'rename' and (not args.image_name or not args.image_new_name): exit('Both old and new image names are required for rename action') -- cgit v1.2.3 From fc5dc00a3892fa26d03213854ea5091d6b0c2c18 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 31 Oct 2023 15:48:12 -0500 Subject: image: T4516: variable name spelling --- src/op_mode/image_manager.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'src/op_mode') diff --git a/src/op_mode/image_manager.py b/src/op_mode/image_manager.py index de53c4cf0..e4b2f4833 100755 --- a/src/op_mode/image_manager.py +++ b/src/op_mode/image_manager.py @@ -56,8 +56,8 @@ def delete_image(image_name: Optional[str] = None, exit(MSG_DELETE_IMAGE_DEFAULT) if image_name not in available_images: exit(f'The image "{image_name}" cannot be found') - presistence_storage: str = disk.find_persistence() - if not presistence_storage: + persistence_storage: str = disk.find_persistence() + if not persistence_storage: exit('Persistence storage cannot be found') if not ask_yes_no(f'Do you really want to delete the image {image_name}?', @@ -65,10 +65,10 @@ def delete_image(image_name: Optional[str] = None, exit() # remove files and menu entry - version_path: Path = Path(f'{presistence_storage}/boot/{image_name}') + version_path: Path = Path(f'{persistence_storage}/boot/{image_name}') try: rmtree(version_path) - grub.version_del(image_name, presistence_storage) + grub.version_del(image_name, persistence_storage) print(f'The image "{image_name}" was successfully deleted') except Exception as err: exit(f'Unable to remove the image "{image_name}": {err}') @@ -94,13 +94,13 @@ def set_image(image_name: Optional[str] = None, exit(f'The image "{image_name}" already configured as default') if image_name not in available_images: exit(f'The image "{image_name}" cannot be found') - presistence_storage: str = disk.find_persistence() - if not presistence_storage: + persistence_storage: str = disk.find_persistence() + if not persistence_storage: exit('Persistence storage cannot be found') # set default boot image try: - grub.set_default(image_name, presistence_storage) + grub.set_default(image_name, persistence_storage) print(f'The image "{image_name}" is now default boot image') except Exception as err: exit(f'Unable to set default image "{image_name}": {err}') @@ -124,8 +124,8 @@ def rename_image(name_old: str, name_new: str) -> None: if not image.validate_name(name_new): exit(f'The image name "{name_new}" is not allowed') - presistence_storage: str = disk.find_persistence() - if not presistence_storage: + persistence_storage: str = disk.find_persistence() + if not persistence_storage: exit('Persistence storage cannot be found') if not ask_yes_no( @@ -137,16 +137,16 @@ def rename_image(name_old: str, name_new: str) -> None: try: # replace default boot item if name_old == image.get_default_image(): - grub.set_default(name_new, presistence_storage) + grub.set_default(name_new, persistence_storage) # rename files and dirs - old_path: Path = Path(f'{presistence_storage}/boot/{name_old}') - new_path: Path = Path(f'{presistence_storage}/boot/{name_new}') + old_path: Path = Path(f'{persistence_storage}/boot/{name_old}') + new_path: Path = Path(f'{persistence_storage}/boot/{name_new}') old_path.rename(new_path) # replace boot item - grub.version_del(name_old, presistence_storage) - grub.version_add(name_new, presistence_storage) + grub.version_del(name_old, persistence_storage) + grub.version_add(name_new, persistence_storage) print(f'The image "{name_old}" was renamed to "{name_new}"') except Exception as err: -- cgit v1.2.3 From e036f783bc85e4d2bad5f5cbfd688a03a352223e Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 15 Nov 2023 11:56:52 -0600 Subject: image: T4516: add raid-1 install support --- python/vyos/system/disk.py | 83 +++++++++--- python/vyos/system/grub.py | 15 ++- python/vyos/system/raid.py | 115 ++++++++++++++++ python/vyos/utils/io.py | 10 +- src/op_mode/image_installer.py | 295 ++++++++++++++++++++++++++++++++--------- 5 files changed, 426 insertions(+), 92 deletions(-) create mode 100644 python/vyos/system/raid.py (limited to 'src/op_mode') diff --git a/python/vyos/system/disk.py b/python/vyos/system/disk.py index 882b4eb39..49e6b5c5e 100644 --- a/python/vyos/system/disk.py +++ b/python/vyos/system/disk.py @@ -15,12 +15,20 @@ from json import loads as json_loads from os import sync +from dataclasses import dataclass from psutil import disk_partitions from vyos.utils.process import run, cmd +@dataclass +class DiskDetails: + """Disk details""" + name: str + partition: dict[str, str] + + def disk_cleanup(drive_path: str) -> None: """Clean up disk partition table (MBR and GPT) Zeroize primary and secondary headers - first and last 17408 bytes @@ -67,6 +75,62 @@ def parttable_create(drive_path: str, root_size: int) -> None: sync() run(f'partprobe {drive_path}') + partitions: list[str] = partition_list(drive_path) + + disk: DiskDetails = DiskDetails( + name = drive_path, + partition = { + 'efi': next(x for x in partitions if x.endswith('2')), + 'root': next(x for x in partitions if x.endswith('3')) + } + ) + + return disk + + +def partition_list(drive_path: str) -> list[str]: + """Get a list of partitions on a drive + + Args: + drive_path (str): path to a drive + + Returns: + list[str]: a list of partition paths + """ + lsblk: str = cmd(f'lsblk -Jp {drive_path}') + drive_info: dict = json_loads(lsblk) + device: list = drive_info.get('blockdevices') + children: list[str] = device[0].get('children', []) if device else [] + partitions: list[str] = [child.get('name') for child in children] + return partitions + + +def partition_parent(partition_path: str) -> str: + """Get a parent device for a partition + + Args: + partition (str): path to a partition + + Returns: + str: path to a parent device + """ + parent: str = cmd(f'lsblk -ndpo pkname {partition_path}') + return parent + + +def from_partition(partition_path: str) -> DiskDetails: + drive_path: str = partition_parent(partition_path) + partitions: list[str] = partition_list(drive_path) + + disk: DiskDetails = DiskDetails( + name = drive_path, + partition = { + 'efi': next(x for x in partitions if x.endswith('2')), + 'root': next(x for x in partitions if x.endswith('3')) + } + ) + + return disk def filesystem_create(partition: str, fstype: str) -> None: """Create a filesystem on a partition @@ -138,25 +202,6 @@ def find_device(mountpoint: str) -> str: return '' -def raid_create(raid_name: str, - raid_members: list[str], - raid_level: str = 'raid1') -> None: - """Create a RAID array - - Args: - raid_name (str): a name of array (data, backup, test, etc.) - raid_members (list[str]): a list of array members - raid_level (str, optional): an array level. Defaults to 'raid1'. - """ - raid_devices_num: int = len(raid_members) - raid_members_str: str = ' '.join(raid_members) - command: str = f'mdadm --create /dev/md/{raid_name} --metadata=1.2 \ - --raid-devices={raid_devices_num} --level={raid_level} \ - {raid_members_str}' - - run(command) - - def disks_size() -> dict[str, int]: """Get a dictionary with physical disks and their sizes diff --git a/python/vyos/system/grub.py b/python/vyos/system/grub.py index 9ac205c03..0ac16af9a 100644 --- a/python/vyos/system/grub.py +++ b/python/vyos/system/grub.py @@ -19,7 +19,7 @@ from typing import Union from uuid import uuid5, NAMESPACE_URL, UUID from vyos.template import render -from vyos.utils.process import run, cmd +from vyos.utils.process import cmd from vyos.system import disk # Define variables @@ -49,7 +49,7 @@ REGEX_GRUB_MODULES: str = r'^insmod (?P.+)$' REGEX_KERNEL_CMDLINE: str = r'^BOOT_IMAGE=/(?Pboot|live)/((?P.+)/)?vmlinuz.*$' -def install(drive_path: str, boot_dir: str, efi_dir: str) -> None: +def install(drive_path: str, boot_dir: str, efi_dir: str, id: str = 'VyOS') -> None: """Install GRUB for both BIOS and EFI modes (hybrid boot) Args: @@ -62,11 +62,11 @@ def install(drive_path: str, boot_dir: str, efi_dir: str) -> None: {drive_path} --force', f'grub-install --no-floppy --recheck --target=x86_64-efi \ --force-extra-removable --boot-directory={boot_dir} \ - --efi-directory={efi_dir} --bootloader-id="VyOS" \ + --efi-directory={efi_dir} --bootloader-id="{id}" \ --no-uefi-secure-boot' ] for command in commands: - run(command) + cmd(command) def gen_version_uuid(version_name: str) -> str: @@ -294,7 +294,7 @@ def set_default(version_name: str, root_dir: str = '') -> None: vars_write(vars_file, vars_current) -def common_write(root_dir: str = '') -> None: +def common_write(root_dir: str = '', grub_common: dict[str, str] = {}) -> None: """Write common GRUB configuration file (overwrite everything) Args: @@ -304,7 +304,7 @@ def common_write(root_dir: str = '') -> None: if not root_dir: root_dir = disk.find_persistence() common_config = f'{root_dir}/{CFG_VYOS_COMMON}' - render(common_config, TMPL_GRUB_COMMON, {}) + render(common_config, TMPL_GRUB_COMMON, grub_common) def create_structure(root_dir: str = '') -> None: @@ -335,3 +335,6 @@ def set_console_type(console_type: str, root_dir: str = '') -> None: vars_current: dict[str, str] = vars_read(vars_file) vars_current['console_type'] = str(console_type) vars_write(vars_file, vars_current) + +def set_raid(root_dir: str = '') -> None: + pass diff --git a/python/vyos/system/raid.py b/python/vyos/system/raid.py new file mode 100644 index 000000000..13b99fa69 --- /dev/null +++ b/python/vyos/system/raid.py @@ -0,0 +1,115 @@ +# Copyright 2023 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see . + +"""RAID related functions""" + +from pathlib import Path +from shutil import copy +from dataclasses import dataclass + +from vyos.utils.process import cmd +from vyos.system import disk + + +@dataclass +class RaidDetails: + """RAID type""" + name: str + level: str + members: list[str] + disks: list[disk.DiskDetails] + + +def raid_create(raid_members: list[str], + raid_name: str = 'md0', + raid_level: str = 'raid1') -> None: + """Create a RAID array + + Args: + raid_name (str): a name of array (data, backup, test, etc.) + raid_members (list[str]): a list of array members + raid_level (str, optional): an array level. Defaults to 'raid1'. + """ + raid_devices_num: int = len(raid_members) + raid_members_str: str = ' '.join(raid_members) + if Path('/sys/firmware/efi').exists(): + for part in raid_members: + drive: str = disk.partition_parent(part) + command: str = f'sgdisk --typecode=3:A19D880F-05FC-4D3B-A006-743F0F84911E {drive}' + cmd(command) + else: + for part in raid_members: + drive: str = disk.partition_parent(part) + command: str = f'sgdisk --typecode=3:A19D880F-05FC-4D3B-A006-743F0F84911E {drive}' + cmd(command) + for part in raid_members: + command: str = f'mdadm --zero-superblock {part}' + cmd(command) + command: str = f'mdadm --create /dev/{raid_name} -R --metadata=1.0 \ + --raid-devices={raid_devices_num} --level={raid_level} \ + {raid_members_str}' + + cmd(command) + + raid = RaidDetails( + name = f'/dev/{raid_name}', + level = raid_level, + members = raid_members, + disks = [disk.from_partition(m) for m in raid_members] + ) + + return raid + +def update_initramfs() -> None: + """Update initramfs""" + mdadm_script = '/etc/initramfs-tools/scripts/local-top/mdadm' + copy('/usr/share/initramfs-tools/scripts/local-block/mdadm', mdadm_script) + p = Path(mdadm_script) + p.write_text(p.read_text().replace('$((COUNT + 1))', '20')) + command: str = 'update-initramfs -u' + cmd(command) + +def update_default(target_dir: str) -> None: + """Update /etc/default/mdadm to start MD monitoring daemon at boot + """ + source_mdadm_config = '/etc/default/mdadm' + target_mdadm_config = Path(target_dir).joinpath('/etc/default/mdadm') + target_mdadm_config_dir = Path(target_mdadm_config).parent + Path.mkdir(target_mdadm_config_dir, parents=True, exist_ok=True) + s = Path(source_mdadm_config).read_text().replace('START_DAEMON=false', + 'START_DAEMON=true') + Path(target_mdadm_config).write_text(s) + +def get_uuid(device: str) -> str: + """Get UUID of a device""" + command: str = f'tune2fs -l {device}' + l = cmd(command).splitlines() + uuid = next((x for x in l if x.startswith('Filesystem UUID')), '') + return uuid.split(':')[1].strip() if uuid else '' + +def get_uuids(raid_details: RaidDetails) -> tuple[str]: + """Get UUIDs of RAID members + + Args: + raid_name (str): a name of array (data, backup, test, etc.) + + Returns: + tuple[str]: root_disk uuid, root_md uuid + """ + raid_name: str = raid_details.name + root_partition: str = raid_details.members[0] + uuid_root_disk: str = get_uuid(root_partition) + uuid_root_md: str = get_uuid(raid_name) + return uuid_root_disk, uuid_root_md diff --git a/python/vyos/utils/io.py b/python/vyos/utils/io.py index e34a1ba32..74099b502 100644 --- a/python/vyos/utils/io.py +++ b/python/vyos/utils/io.py @@ -13,6 +13,8 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . +from typing import Callable + def print_error(str='', end='\n'): """ Print `str` to stderr, terminated with `end`. @@ -73,7 +75,8 @@ def is_dumb_terminal(): import os return os.getenv('TERM') in ['vt100', 'dumb'] -def select_entry(l: list, list_msg: str = '', prompt_msg: str = '') -> str: +def select_entry(l: list, list_msg: str = '', prompt_msg: str = '', + list_format: Callable = None,) -> str: """Select an entry from a list Args: @@ -87,7 +90,10 @@ def select_entry(l: list, list_msg: str = '', prompt_msg: str = '') -> str: en = list(enumerate(l, 1)) print(list_msg) for i, e in en: - print(f'\t{i}: {e}') + if list_format: + print(f'\t{i}: {list_format(e)}') + else: + print(f'\t{i}: {e}') select = ask_input(prompt_msg, numeric_only=True, valid_responses=range(1, len(l)+1)) return next(filter(lambda x: x[0] == select, en))[1] diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index 88d5eeb48..aa4cf301b 100755 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -21,18 +21,20 @@ from argparse import ArgumentParser, Namespace from pathlib import Path from shutil import copy, chown, rmtree, copytree from sys import exit -from passlib.hosts import linux_context +from time import sleep +from typing import Union from urllib.parse import urlparse +from passlib.hosts import linux_context from psutil import disk_partitions from vyos.configtree import ConfigTree from vyos.remote import download -from vyos.system import disk, grub, image, compat, SYSTEM_CFG_VER +from vyos.system import disk, grub, image, compat, raid, SYSTEM_CFG_VER from vyos.template import render -from vyos.utils.io import ask_input, ask_yes_no +from vyos.utils.io import ask_input, ask_yes_no, select_entry from vyos.utils.file import chmod_2775 -from vyos.utils.process import run +from vyos.utils.process import cmd, run # define text messages MSG_ERR_NOT_LIVE: str = 'The system is already installed. Please use "add system image" instead.' @@ -44,11 +46,15 @@ MSG_INFO_INSTALL_EXIT: str = 'Exiting from VyOS installation' MSG_INFO_INSTALL_SUCCESS: str = 'The image installed successfully; please reboot now.' MSG_INFO_INSTALL_DISKS_LIST: str = 'The following disks were found:' MSG_INFO_INSTALL_DISK_SELECT: str = 'Which one should be used for installation?' +MSG_INFO_INSTALL_RAID_CONFIGURE: str = 'Would you like to configure RAID-1 mirroring?' +MSG_INFO_INSTALL_RAID_FOUND_DISKS: str = 'Would you like to configure RAID-1 mirroring on them?' +MSG_INFO_INSTALL_RAID_CHOOSE_DISKS: str = 'Would you like to choose two disks for RAID-1 mirroring?' MSG_INFO_INSTALL_DISK_CONFIRM: str = 'Installation will delete all data on the drive. Continue?' +MSG_INFO_INSTALL_RAID_CONFIRM: str = 'Installation will delete all data on both drives. Continue?' MSG_INFO_INSTALL_PARTITONING: str = 'Creating partition table...' MSG_INPUT_CONFIG_FOUND: str = 'An active configuration was found. Would you like to copy it to the new image?' MSG_INPUT_IMAGE_NAME: str = 'What would you like to name this image?' -MSG_INPUT_IMAGE_DEFAULT: str = 'Would you like to set a new image as default one for boot?' +MSG_INPUT_IMAGE_DEFAULT: str = 'Would you like to set the new image as the default one for boot?' MSG_INPUT_PASSWORD: str = 'Please enter a password for the "vyos" user' MSG_INPUT_ROOT_SIZE_ALL: str = 'Would you like to use all the free space on the drive?' MSG_INPUT_ROOT_SIZE_SET: str = 'Please specify the size (in GB) of the root partition (min is 1.5 GB)?' @@ -107,13 +113,14 @@ def gb_to_bytes(size: float) -> int: return int(size * 1024**3) -def find_disk() -> tuple[str, int]: +def find_disks() -> dict[str, int]: """Find a target disk for installation Returns: - tuple[str, int]: disk name and size in bytes + dict[str, int]: a list of available disks by name and size """ # check for available disks + print('Probing disks') disks_available: dict[str, int] = disk.disks_size() for disk_name, disk_size in disks_available.copy().items(): if disk_size < CONST_MIN_DISK_SIZE: @@ -122,17 +129,10 @@ def find_disk() -> tuple[str, int]: print(MSG_ERR_NO_DISK) exit(MSG_INFO_INSTALL_EXIT) - # select one as a target - print(MSG_INFO_INSTALL_DISKS_LIST) - default_disk: str = list(disks_available)[0] - for disk_name, disk_size in disks_available.items(): - disk_size_human: str = bytes_to_gb(disk_size) - print(f'Drive: {disk_name} ({disk_size_human} GB)') - disk_selected: str = ask_input(MSG_INFO_INSTALL_DISK_SELECT, - default=default_disk, - valid_responses=list(disks_available)) + num_disks: int = len(disks_available) + print(f'{num_disks} disk(s) found') - return disk_selected, disks_available[disk_selected] + return disks_available def ask_root_size(available_space: int) -> int: @@ -160,6 +160,126 @@ def ask_root_size(available_space: int) -> int: return root_size_kbytes +def create_partitions(target_disk: str, target_size: int, + prompt: bool = True) -> None: + """Create partitions on a target disk + + Args: + target_disk (str): a target disk + target_size (int): size of disk in bytes + """ + # define target rootfs size in KB (smallest unit acceptable by sgdisk) + available_size: int = (target_size - CONST_RESERVED_SPACE) // 1024 + if prompt: + rootfs_size: int = ask_root_size(available_size) + else: + rootfs_size: int = available_size + + print(MSG_INFO_INSTALL_PARTITONING) + disk.disk_cleanup(target_disk) + disk_details: disk.DiskDetails = disk.parttable_create(target_disk, + rootfs_size) + + return disk_details + + +def ask_single_disk(disks_available: dict[str, int]) -> str: + """Ask user to select a disk for installation + + Args: + disks_available (dict[str, int]): a list of available disks + """ + print(MSG_INFO_INSTALL_DISKS_LIST) + default_disk: str = list(disks_available)[0] + for disk_name, disk_size in disks_available.items(): + disk_size_human: str = bytes_to_gb(disk_size) + print(f'Drive: {disk_name} ({disk_size_human} GB)') + disk_selected: str = ask_input(MSG_INFO_INSTALL_DISK_SELECT, + default=default_disk, + valid_responses=list(disks_available)) + + # create partitions + if not ask_yes_no(MSG_INFO_INSTALL_DISK_CONFIRM): + print(MSG_INFO_INSTALL_EXIT) + exit() + + disk_details: disk.DiskDetails = create_partitions(disk_selected, + disks_available[disk_selected]) + + disk.filesystem_create(disk_details.partition['efi'], 'efi') + disk.filesystem_create(disk_details.partition['root'], 'ext4') + + return disk_details + + +def check_raid_install(disks_available: dict[str, int]) -> Union[str, None]: + """Ask user to select disks for RAID installation + + Args: + disks_available (dict[str, int]): a list of available disks + """ + if len(disks_available) < 2: + return None + + if not ask_yes_no(MSG_INFO_INSTALL_RAID_CONFIGURE, default=True): + return None + + def format_selection(disk_name: str) -> str: + return f'{disk_name}\t({bytes_to_gb(disks_available[disk_name])} GB)' + + disk0, disk1 = list(disks_available)[0], list(disks_available)[1] + disks_selected: dict[str, int] = { disk0: disks_available[disk0], + disk1: disks_available[disk1] } + + target_size: int = min(disks_selected[disk0], disks_selected[disk1]) + + print(MSG_INFO_INSTALL_DISKS_LIST) + for disk_name, disk_size in disks_selected.items(): + disk_size_human: str = bytes_to_gb(disk_size) + print(f'\t{disk_name} ({disk_size_human} GB)') + if not ask_yes_no(MSG_INFO_INSTALL_RAID_FOUND_DISKS, default=True): + if not ask_yes_no(MSG_INFO_INSTALL_RAID_CHOOSE_DISKS, default=True): + return None + else: + disks_selected = {} + disk0 = select_entry(list(disks_available), 'Disks available:', + 'Select first disk:', format_selection) + + disks_selected[disk0] = disks_available[disk0] + del disks_available[disk0] + disk1 = select_entry(list(disks_available), 'Remaining disks:', + 'Select second disk:', format_selection) + disks_selected[disk1] = disks_available[disk1] + + target_size: int = min(disks_selected[disk0], + disks_selected[disk1]) + + # create partitions + if not ask_yes_no(MSG_INFO_INSTALL_RAID_CONFIRM): + print(MSG_INFO_INSTALL_EXIT) + exit() + + disks: list[disk.DiskDetails] = [] + for disk_selected in list(disks_selected): + print(f'Creating partitions on {disk_selected}') + disk_details = create_partitions(disk_selected, target_size, + prompt=False) + disk.filesystem_create(disk_details.partition['efi'], 'efi') + + disks.append(disk_details) + + print('Creating RAID array') + members = [disk.partition['root'] for disk in disks] + raid_details: raid.RaidDetails = raid.raid_create(members) + # raid init stuff + print('Updating initramfs') + raid.update_initramfs() + # end init + print('Creating filesystem on RAID array') + disk.filesystem_create(raid_details.name, 'ext4') + + return raid_details + def prepare_tmp_disr() -> None: """Create temporary directories for installation @@ -294,7 +414,7 @@ def image_fetch(image_path: str) -> Path: if local_path.is_file(): return local_path else: - raise + raise FileNotFoundError except Exception: print(f'The image cannot be fetched from: {image_path}') exit(1) @@ -351,6 +471,27 @@ def cleanup(mounts: list[str] = [], remove_items: list[str] = []) -> None: if Path(remove_item).is_dir(): rmtree(remove_item) +def cleanup_raid(details: raid.RaidDetails) -> None: + efiparts = [] + for raid_disk in details.disks: + efiparts.append(raid_disk.partition['efi']) + cleanup([details.name, *efiparts], + ['/mnt/installation']) + + +def is_raid_install(install_object: Union[disk.DiskDetails, raid.RaidDetails]) -> bool: + """Check if installation target is a RAID array + + Args: + install_object (Union[disk.DiskDetails, raid.RaidDetails]): a target disk + + Returns: + bool: True if it is a RAID array + """ + if isinstance(install_object, raid.RaidDetails): + return True + return False + def install_image() -> None: """Install an image to a disk @@ -363,50 +504,44 @@ def install_image() -> None: print(MSG_INFO_INSTALL_EXIT) exit() + # configure image name + running_image_name: str = image.get_running_image() + while True: + image_name: str = ask_input(MSG_INPUT_IMAGE_NAME, + running_image_name) + if image.validate_name(image_name): + break + print(MSG_WARN_IMAGE_NAME_WRONG) + + # ask for password + user_password: str = ask_input(MSG_INPUT_PASSWORD, default='vyos') + + # ask for default console + console_type: str = ask_input(MSG_INPUT_CONSOLE_TYPE, + default='K', + valid_responses=['K', 'S', 'U']) + console_dict: dict[str, str] = {'K': 'tty', 'S': 'ttyS', 'U': 'ttyUSB'} + + disks: dict[str, int] = find_disks() + + install_target: Union[disk.DiskDetails, raid.RaidDetails, None] = None try: - # configure image name - running_image_name: str = image.get_running_image() - while True: - image_name: str = ask_input(MSG_INPUT_IMAGE_NAME, - running_image_name) - if image.validate_name(image_name): - break - print(MSG_WARN_IMAGE_NAME_WRONG) - - # define target drive - install_target, target_size = find_disk() - - # define target rootfs size in KB (smallest unit acceptable by sgdisk) - availabe_size: int = (target_size - CONST_RESERVED_SPACE) // 1024 - rootfs_size: int = ask_root_size(availabe_size) - - # ask for password - user_password: str = ask_input(MSG_INPUT_PASSWORD, default='vyos') - - # ask for default console - console_type: str = ask_input(MSG_INPUT_CONSOLE_TYPE, - default='K', - valid_responses=['K', 'S', 'U']) - console_dict: dict[str, str] = {'K': 'tty', 'S': 'ttyS', 'U': 'ttyUSB'} - - # create partitions - if not ask_yes_no(MSG_INFO_INSTALL_DISK_CONFIRM): - print(MSG_INFO_INSTALL_EXIT) - exit() - print(MSG_INFO_INSTALL_PARTITONING) - disk.disk_cleanup(install_target) - disk.parttable_create(install_target, rootfs_size) - disk.filesystem_create(f'{install_target}2', 'efi') - disk.filesystem_create(f'{install_target}3', 'ext4') - - # create directiroes for installation media + install_target = check_raid_install(disks) + if install_target is None: + install_target = ask_single_disk(disks) + + # create directories for installation media prepare_tmp_disr() # mount target filesystem and create required dirs inside print('Mounting new partitions') - disk.partition_mount(f'{install_target}3', DIR_DST_ROOT) - Path(f'{DIR_DST_ROOT}/boot/efi').mkdir(parents=True) - disk.partition_mount(f'{install_target}2', f'{DIR_DST_ROOT}/boot/efi') + if is_raid_install(install_target): + disk.partition_mount(install_target.name, DIR_DST_ROOT) + Path(f'{DIR_DST_ROOT}/boot/efi').mkdir(parents=True) + else: + disk.partition_mount(install_target.partition['root'], DIR_DST_ROOT) + Path(f'{DIR_DST_ROOT}/boot/efi').mkdir(parents=True) + disk.partition_mount(install_target.partition['efi'], f'{DIR_DST_ROOT}/boot/efi') # a config dir. It is the deepest one, so the comand will # create all the rest in a single step @@ -432,10 +567,10 @@ def install_image() -> None: copy(FILE_ROOTFS_SRC, f'{DIR_DST_ROOT}/boot/{image_name}/{image_name}.squashfs') - # install GRUB - print('Installing GRUB to the drive') - grub.install(install_target, f'{DIR_DST_ROOT}/boot/', - f'{DIR_DST_ROOT}/boot/efi') + if is_raid_install(install_target): + write_dir: str = f'{DIR_DST_ROOT}/boot/{image_name}/rw' + raid.update_default(write_dir) + setup_grub(DIR_DST_ROOT) # add information about version grub.create_structure() @@ -443,9 +578,34 @@ def install_image() -> None: grub.set_default(image_name, DIR_DST_ROOT) grub.set_console_type(console_dict[console_type], DIR_DST_ROOT) + if is_raid_install(install_target): + # add RAID specific modules + grub.modules_write(f'{DIR_DST_ROOT}/{grub.CFG_VYOS_MODULES}', + ['part_msdos', 'part_gpt', 'diskfilter', + 'ext2','mdraid1x']) + # install GRUB + if is_raid_install(install_target): + print('Installing GRUB to the drives') + l = install_target.disks + for disk_target in l: + disk.partition_mount(disk_target.partition['efi'], f'{DIR_DST_ROOT}/boot/efi') + grub.install(disk_target.name, f'{DIR_DST_ROOT}/boot/', + f'{DIR_DST_ROOT}/boot/efi', + id=f'VyOS (RAID disk {l.index(disk_target) + 1})') + disk.partition_umount(disk_target.partition['efi']) + else: + print('Installing GRUB to the drive') + grub.install(install_target.name, f'{DIR_DST_ROOT}/boot/', + f'{DIR_DST_ROOT}/boot/efi') + # umount filesystems and remove temporary files - cleanup([f'{install_target}2', f'{install_target}3'], - ['/mnt/installation']) + if is_raid_install(install_target): + cleanup([install_target.name], + ['/mnt/installation']) + else: + cleanup([install_target.partition['efi'], + install_target.partition['root']], + ['/mnt/installation']) # we are done print(MSG_INFO_INSTALL_SUCCESS) @@ -455,8 +615,13 @@ def install_image() -> None: print(f'Unable to install VyOS: {err}') # unmount filesystems and clenup try: - cleanup([f'{install_target}2', f'{install_target}3'], - ['/mnt/installation']) + if install_target is not None: + if is_raid_install(install_target): + cleanup_raid(install_target) + else: + cleanup([install_target.partition['efi'], + install_target.partition['root']], + ['/mnt/installation']) except Exception as err: print(f'Cleanup failed: {err}') -- cgit v1.2.3