From 601e07c34205fb379d729c4faf654f95add90471 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 19 Jun 2024 20:45:53 -0500 Subject: migration: T6007: add version object for config file and system --- python/vyos/component_version.py | 152 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 150 insertions(+), 2 deletions(-) (limited to 'python/vyos/component_version.py') diff --git a/python/vyos/component_version.py b/python/vyos/component_version.py index 9662ebfcf..648d690b9 100644 --- a/python/vyos/component_version.py +++ b/python/vyos/component_version.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors +# Copyright 2022-2024 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -35,14 +35,162 @@ VyOS 1.2: import os import re import sys -import fileinput +from dataclasses import dataclass +from dataclasses import replace +from typing import Optional from vyos.xml_ref import component_version +from vyos.utils.file import write_file from vyos.version import get_version from vyos.defaults import directories DEFAULT_CONFIG_PATH = os.path.join(directories['config'], 'config.boot') +REGEX_WARN_VYOS = r'(// Warning: Do not remove the following line.)' +REGEX_WARN_VYATTA = r'(/\* Warning: Do not remove the following line. \*/)' +REGEX_COMPONENT_VERSION_VYOS = r'// vyos-config-version:\s+"([\w@:-]+)"\s*' +REGEX_COMPONENT_VERSION_VYATTA = r'/\* === vyatta-config-version:\s+"([\w@:-]+)"\s+=== \*/' +REGEX_RELEASE_VERSION_VYOS = r'// Release version:\s+(\S*)\s*' +REGEX_RELEASE_VERSION_VYATTA = r'/\* Release version:\s+(\S*)\s*\*/' + +CONFIG_FILE_VERSION = """\ +// Warning: Do not remove the following line. +// vyos-config-version: "{}" +// Release version: {}\n +""" + +warn_filter_vyos = re.compile(REGEX_WARN_VYOS) +warn_filter_vyatta = re.compile(REGEX_WARN_VYATTA) + +regex_filter = { 'vyos': dict(zip(['component', 'release'], + [re.compile(REGEX_COMPONENT_VERSION_VYOS), + re.compile(REGEX_RELEASE_VERSION_VYOS)])), + 'vyatta': dict(zip(['component', 'release'], + [re.compile(REGEX_COMPONENT_VERSION_VYATTA), + re.compile(REGEX_RELEASE_VERSION_VYATTA)])) } + +@dataclass +class VersionInfo: + component: Optional[dict[str,int]] = None + release: str = get_version() + vintage: str = 'vyos' + config_body: Optional[str] = None + footer_lines: Optional[list[str]] = None + + def component_is_none(self) -> bool: + return bool(self.component is None) + + def config_body_is_none(self) -> bool: + return bool(self.config_body is None) + + def update_footer(self): + f = CONFIG_FILE_VERSION.format(component_to_string(self.component), + self.release) + self.footer_lines = f.splitlines() + + def update_syntax(self): + self.vintage = 'vyos' + self.update_footer() + + def update_release(self, release: str): + self.release = release + self.update_footer() + + def update_component(self, key: str, version: int): + if not isinstance(version, int): + raise ValueError('version must be int') + if self.component is None: + self.component = {} + self.component[key] = version + self.component = dict(sorted(self.component.items(), key=lambda x: x[0])) + self.update_footer() + + def update_config_body(self, config_str: str): + self.config_body = config_str + + def write_string(self) -> str: + config_body = '' if self.config_body is None else self.config_body + footer_lines = [] if self.footer_lines is None else self.footer_lines + + return config_body + '\n' + '\n'.join(footer_lines) + '\n' + + def write(self, config_file): + string = self.write_string() + try: + write_file(config_file, string, mode=0o660) + except Exception as e: + raise ValueError(e) from e + +def component_to_string(component: dict) -> str: + l = [f'{k}@{v}' for k, v in sorted(component.items(), key=lambda x: x[0])] + return ':'.join(l) + +def component_from_string(string: str) -> dict: + return {k: int(v) for k, v in re.findall(r'([\w,-]+)@(\d+)', string)} + +def version_info_from_file(config_file) -> VersionInfo: + version_info = VersionInfo() + try: + with open(config_file) as f: + config_str = f.read() + except OSError: + return None + + if len(parts := warn_filter_vyos.split(config_str)) > 1: + vintage = 'vyos' + elif len(parts := warn_filter_vyatta.split(config_str)) > 1: + vintage = 'vyatta' + else: + version_info.config_body = parts[0] if parts else None + return version_info + + version_info.vintage = vintage + version_info.config_body = parts[0] + version_lines = ''.join(parts[1:]).splitlines() + version_lines = [k for k in version_lines if k] + if len(version_lines) != 3: + raise ValueError(f'Malformed version strings: {version_lines}') + + m = regex_filter[vintage]['component'].match(version_lines[1]) + if not m: + raise ValueError(f'Malformed component string: {version_lines[1]}') + version_info.component = component_from_string(m.group(1)) + + m = regex_filter[vintage]['release'].match(version_lines[2]) + if not m: + raise ValueError(f'Malformed component string: {version_lines[2]}') + version_info.release = m.group(1) + + version_info.footer_lines = version_lines + + return version_info + +def version_info_from_system() -> VersionInfo: + """ + Return system component versions. + """ + d = component_version() + sort_d = dict(sorted(d.items(), key=lambda x: x[0])) + version_info = VersionInfo( + component = sort_d, + release = get_version(), + vintage = 'vyos' + ) + + return version_info + +def version_info_copy(v: VersionInfo) -> VersionInfo: + """ + Make a copy of dataclass. + """ + return replace(v) + +def version_info_prune_component(x: VersionInfo, y: VersionInfo) -> VersionInfo: + """ + In place pruning of component keys of x not in y. + """ + x.component = { k: v for k,v in x.component.items() if k in y.component } + def from_string(string_line, vintage='vyos'): """ Get component version dictionary from string. -- cgit v1.2.3 From 51865448599ec40283fffe4dc15729f88f389886 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 23 Jun 2024 12:40:36 -0500 Subject: migration: T6007: add util add_system_version to replace *_system_footer --- python/vyos/component_version.py | 15 +++++++++++++++ src/helpers/add-system-version.py | 20 ++++++++++++++++++++ src/helpers/system-versions-foot.py | 28 ---------------------------- src/helpers/vyos-save-config.py | 15 +++++++-------- src/init/vyos-router | 4 ++-- 5 files changed, 44 insertions(+), 38 deletions(-) create mode 100755 src/helpers/add-system-version.py delete mode 100755 src/helpers/system-versions-foot.py (limited to 'python/vyos/component_version.py') diff --git a/python/vyos/component_version.py b/python/vyos/component_version.py index 648d690b9..1513ac5ed 100644 --- a/python/vyos/component_version.py +++ b/python/vyos/component_version.py @@ -191,6 +191,21 @@ def version_info_prune_component(x: VersionInfo, y: VersionInfo) -> VersionInfo: """ x.component = { k: v for k,v in x.component.items() if k in y.component } +def add_system_version(config_str: str = None, out_file: str = None): + """ + Wrap config string with system version and write to out_file. + For convenience, calling with no argument will write system version + string to stdout, for use in bash scripts. + """ + version_info = version_info_from_system() + if config_str is not None: + version_info.update_config_body(config_str) + version_info.update_footer() + if out_file is not None: + version_info.write(out_file) + else: + sys.stdout.write(version_info.write_string()) + def from_string(string_line, vintage='vyos'): """ Get component version dictionary from string. diff --git a/src/helpers/add-system-version.py b/src/helpers/add-system-version.py new file mode 100755 index 000000000..5270ee7d3 --- /dev/null +++ b/src/helpers/add-system-version.py @@ -0,0 +1,20 @@ +#!/usr/bin/python3 + +# Copyright 2019-2024 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see . + +from vyos.component_version import add_system_version + +add_system_version() diff --git a/src/helpers/system-versions-foot.py b/src/helpers/system-versions-foot.py deleted file mode 100755 index 9614f0d28..000000000 --- a/src/helpers/system-versions-foot.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/python3 - -# Copyright 2019, 2022 VyOS maintainers and contributors -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see . - -import sys -import vyos.defaults -from vyos.component_version import write_system_footer - -sys.stdout.write("\n\n") -if vyos.defaults.cfg_vintage == 'vyos': - write_system_footer(None, vintage='vyos') -elif vyos.defaults.cfg_vintage == 'vyatta': - write_system_footer(None, vintage='vyatta') -else: - write_system_footer(None, vintage='vyos') diff --git a/src/helpers/vyos-save-config.py b/src/helpers/vyos-save-config.py index 518bd9864..fa2ea0ce4 100755 --- a/src/helpers/vyos-save-config.py +++ b/src/helpers/vyos-save-config.py @@ -23,7 +23,7 @@ from argparse import ArgumentParser from vyos.config import Config from vyos.remote import urlc -from vyos.component_version import system_footer +from vyos.component_version import add_system_version from vyos.defaults import directories DEFAULT_CONFIG_PATH = os.path.join(directories['config'], 'config.boot') @@ -50,14 +50,13 @@ if re.match(r'\w+:/', save_file): config = Config() ct = config.get_config_tree(effective=True) +# pylint: disable=consider-using-with write_file = save_file if remote_save is None else NamedTemporaryFile(delete=False).name -with open(write_file, 'w') as f: - # config_tree is None before boot configuration is complete; - # automated saves should check boot_configuration_complete - if ct is not None: - f.write(ct.to_string()) - f.write("\n") - f.write(system_footer()) + +# config_tree is None before boot configuration is complete; +# automated saves should check boot_configuration_complete +config_str = None if ct is None else ct.to_string() +add_system_version(config_str, write_file) if json_file is not None and ct is not None: try: diff --git a/src/init/vyos-router b/src/init/vyos-router index 59004fdc1..8825cc16a 100755 --- a/src/init/vyos-router +++ b/src/init/vyos-router @@ -134,14 +134,14 @@ init_bootfile () { if [ ! -r $DEFAULT_BOOTFILE ]; then if [ -f $vyos_data_dir/config.boot.default ]; then cp $vyos_data_dir/config.boot.default $DEFAULT_BOOTFILE - $vyos_libexec_dir/system-versions-foot.py >> $DEFAULT_BOOTFILE + $vyos_libexec_dir/add-system-version.py >> $DEFAULT_BOOTFILE fi fi if [ ! -r $BOOTFILE ] ; then if [ -f $DEFAULT_BOOTFILE ]; then cp $DEFAULT_BOOTFILE $BOOTFILE else - $vyos_libexec_dir/system-versions-foot.py > $BOOTFILE + $vyos_libexec_dir/add-system-version.py > $BOOTFILE fi chgrp ${GROUP} $BOOTFILE chmod 660 $BOOTFILE -- cgit v1.2.3 From ed0cb7ffc2c627b9de96d64b45c7978c3bce7ed3 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 23 Jun 2024 16:33:01 -0500 Subject: migration: T6007: remove obsoleted --- python/vyos/component_version.py | 123 --------------------- python/vyos/migrator.py | 226 --------------------------------------- 2 files changed, 349 deletions(-) delete mode 100644 python/vyos/migrator.py (limited to 'python/vyos/component_version.py') diff --git a/python/vyos/component_version.py b/python/vyos/component_version.py index 1513ac5ed..6db1768d3 100644 --- a/python/vyos/component_version.py +++ b/python/vyos/component_version.py @@ -205,126 +205,3 @@ def add_system_version(config_str: str = None, out_file: str = None): version_info.write(out_file) else: sys.stdout.write(version_info.write_string()) - -def from_string(string_line, vintage='vyos'): - """ - Get component version dictionary from string. - Return empty dictionary if string contains no config information - or raise error if component version string malformed. - """ - version_dict = {} - - if vintage == 'vyos': - if re.match(r'// vyos-config-version:.+', string_line): - if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', string_line): - raise ValueError(f"malformed configuration string: {string_line}") - - for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): - version_dict[pair[0]] = int(pair[1]) - - elif vintage == 'vyatta': - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line): - if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line): - raise ValueError(f"malformed configuration string: {string_line}") - - for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): - version_dict[pair[0]] = int(pair[1]) - else: - raise ValueError("Unknown config string vintage") - - return version_dict - -def from_file(config_file_name=DEFAULT_CONFIG_PATH, vintage='vyos'): - """ - Get component version dictionary parsing config file line by line - """ - with open(config_file_name, 'r') as f: - for line_in_config in f: - version_dict = from_string(line_in_config, vintage=vintage) - if version_dict: - return version_dict - - # no version information - return {} - -def from_system(): - """ - Get system component version dict. - """ - return component_version() - -def format_string(ver: dict) -> str: - """ - Version dict to string. - """ - keys = list(ver) - keys.sort() - l = [] - for k in keys: - v = ver[k] - l.append(f'{k}@{v}') - sep = ':' - return sep.join(l) - -def version_footer(ver: dict, vintage='vyos') -> str: - """ - Version footer as string. - """ - ver_str = format_string(ver) - release = get_version() - if vintage == 'vyos': - ret_str = (f'// Warning: Do not remove the following line.\n' - + f'// vyos-config-version: "{ver_str}"\n' - + f'// Release version: {release}\n') - elif vintage == 'vyatta': - ret_str = (f'/* Warning: Do not remove the following line. */\n' - + f'/* === vyatta-config-version: "{ver_str}" === */\n' - + f'/* Release version: {release} */\n') - else: - raise ValueError("Unknown config string vintage") - - return ret_str - -def system_footer(vintage='vyos') -> str: - """ - System version footer as string. - """ - ver_d = from_system() - return version_footer(ver_d, vintage=vintage) - -def write_version_footer(ver: dict, file_name, vintage='vyos'): - """ - Write version footer to file. - """ - footer = version_footer(ver=ver, vintage=vintage) - if file_name: - with open(file_name, 'a') as f: - f.write(footer) - else: - sys.stdout.write(footer) - -def write_system_footer(file_name, vintage='vyos'): - """ - Write system version footer to file. - """ - ver_d = from_system() - return write_version_footer(ver_d, file_name=file_name, vintage=vintage) - -def remove_footer(file_name): - """ - Remove old version footer. - """ - for line in fileinput.input(file_name, inplace=True): - if re.match(r'/\* Warning:.+ \*/$', line): - continue - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line): - continue - if re.match(r'/\* Release version:.+ \*/$', line): - continue - if re.match('// vyos-config-version:.+', line): - continue - if re.match('// Warning:.+', line): - continue - if re.match('// Release version:.+', line): - continue - sys.stdout.write(line) diff --git a/python/vyos/migrator.py b/python/vyos/migrator.py deleted file mode 100644 index 872682bc0..000000000 --- a/python/vyos/migrator.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright 2019-2022 VyOS maintainers and contributors -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see . - -import sys -import os -import json -import logging - -import vyos.defaults -import vyos.component_version as component_version -from vyos.utils.process import cmd - -log_file = os.path.join(vyos.defaults.directories['config'], 'vyos-migrate.log') - -class MigratorError(Exception): - pass - -class Migrator(object): - def __init__(self, config_file, force=False, set_vintage='vyos'): - self._config_file = config_file - self._force = force - self._set_vintage = set_vintage - self._config_file_vintage = None - self._changed = False - - def init_logger(self): - self.logger = logging.getLogger(__name__) - self.logger.setLevel(logging.DEBUG) - - # on adding the file handler, allow write permission for cfg_group; - # restore original umask on exit - mask = os.umask(0o113) - fh = logging.FileHandler(log_file) - formatter = logging.Formatter('%(message)s') - fh.setFormatter(formatter) - self.logger.addHandler(fh) - os.umask(mask) - - def read_config_file_versions(self): - """ - Get component versions from config file footer and set vintage; - return empty dictionary if config string is missing. - """ - cfg_file = self._config_file - component_versions = {} - - cfg_versions = component_version.from_file(cfg_file, vintage='vyatta') - - if cfg_versions: - self._config_file_vintage = 'vyatta' - component_versions = cfg_versions - - cfg_versions = component_version.from_file(cfg_file, vintage='vyos') - - if cfg_versions: - self._config_file_vintage = 'vyos' - component_versions = cfg_versions - - return component_versions - - def update_vintage(self): - old_vintage = self._config_file_vintage - - if self._set_vintage: - self._config_file_vintage = self._set_vintage - - if self._config_file_vintage not in ['vyatta', 'vyos']: - raise MigratorError("Unknown vintage.") - - if self._config_file_vintage == old_vintage: - return False - else: - return True - - def run_migration_scripts(self, config_file_versions, system_versions): - """ - Run migration scripts iteratively, until config file version equals - system component version. - """ - os.environ['VYOS_MIGRATION'] = '1' - self.init_logger() - - self.logger.info("List of executed migration scripts:") - - cfg_versions = config_file_versions - sys_versions = system_versions - - sys_keys = list(sys_versions.keys()) - sys_keys.sort() - - # XXX 'bgp' needs to follow 'quagga': - if 'bgp' in sys_keys and 'quagga' in sys_keys: - sys_keys.insert(sys_keys.index('quagga'), - sys_keys.pop(sys_keys.index('bgp'))) - - rev_versions = {} - - for key in sys_keys: - sys_ver = sys_versions[key] - if key in cfg_versions: - cfg_ver = cfg_versions[key] - else: - cfg_ver = 0 - - migrate_script_dir = os.path.join( - vyos.defaults.directories['migrate'], key) - - while cfg_ver < sys_ver: - next_ver = cfg_ver + 1 - - migrate_script = os.path.join(migrate_script_dir, - '{}-to-{}'.format(cfg_ver, next_ver)) - - try: - out = cmd([migrate_script, self._config_file]) - self.logger.info(f'{migrate_script}') - if out: self.logger.info(out) - except FileNotFoundError: - pass - except Exception as err: - print("\nMigration script error: {0}: {1}." - "".format(migrate_script, err)) - sys.exit(1) - - cfg_ver = next_ver - rev_versions[key] = cfg_ver - - del os.environ['VYOS_MIGRATION'] - return rev_versions - - def write_config_file_versions(self, cfg_versions): - """ - Write new versions string. - """ - if self._config_file_vintage == 'vyatta': - component_version.write_version_footer(cfg_versions, - self._config_file, - vintage='vyatta') - - if self._config_file_vintage == 'vyos': - component_version.write_version_footer(cfg_versions, - self._config_file, - vintage='vyos') - - def save_json_record(self, component_versions: dict): - """ - Write component versions to a json file - """ - mask = os.umask(0o113) - version_file = vyos.defaults.component_version_json - try: - with open(version_file, 'w') as f: - f.write(json.dumps(component_versions, indent=2, sort_keys=True)) - except OSError: - pass - finally: - os.umask(mask) - - def run(self): - """ - Gather component versions from config file and system. - Run migration scripts. - Update vintage ('vyatta' or 'vyos'), if needed. - If changed, remove old versions string from config file, and - write new versions string. - """ - cfg_file = self._config_file - - cfg_versions = self.read_config_file_versions() - if self._force: - # This will force calling all migration scripts: - cfg_versions = {} - - sys_versions = component_version.from_system() - - # save system component versions in json file for easy reference - self.save_json_record(sys_versions) - - rev_versions = self.run_migration_scripts(cfg_versions, sys_versions) - - if rev_versions != cfg_versions: - self._changed = True - - if self.update_vintage(): - self._changed = True - - if not self._changed: - return - - component_version.remove_footer(cfg_file) - - self.write_config_file_versions(rev_versions) - - def config_changed(self): - return self._changed - -class VirtualMigrator(Migrator): - def run(self): - cfg_file = self._config_file - - cfg_versions = self.read_config_file_versions() - if not cfg_versions: - return - - if self.update_vintage(): - self._changed = True - - if not self._changed: - return - - component_version.remove_footer(cfg_file) - - self.write_config_file_versions(cfg_versions) - -- cgit v1.2.3 From 8b4c2fcba2fe49af8c8ee87d3bb1f7b5803a08ea Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Mon, 24 Jun 2024 11:09:02 -0500 Subject: migration: T6007: drop restrictive perms, already set on config.boot --- python/vyos/component_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'python/vyos/component_version.py') diff --git a/python/vyos/component_version.py b/python/vyos/component_version.py index 6db1768d3..aebf90a0f 100644 --- a/python/vyos/component_version.py +++ b/python/vyos/component_version.py @@ -117,7 +117,7 @@ class VersionInfo: def write(self, config_file): string = self.write_string() try: - write_file(config_file, string, mode=0o660) + write_file(config_file, string) except Exception as e: raise ValueError(e) from e -- cgit v1.2.3 From 5502a75b1747caf94e2b69982c89088281c8ca1f Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Mon, 24 Jun 2024 19:39:32 -0500 Subject: migration: T6007: fix cosmetic issue of extra newline --- python/vyos/component_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'python/vyos/component_version.py') diff --git a/python/vyos/component_version.py b/python/vyos/component_version.py index aebf90a0f..0c305e5e0 100644 --- a/python/vyos/component_version.py +++ b/python/vyos/component_version.py @@ -56,7 +56,7 @@ REGEX_RELEASE_VERSION_VYATTA = r'/\* Release version:\s+(\S*)\s*\*/' CONFIG_FILE_VERSION = """\ // Warning: Do not remove the following line. // vyos-config-version: "{}" -// Release version: {}\n +// Release version: {} """ warn_filter_vyos = re.compile(REGEX_WARN_VYOS) -- cgit v1.2.3