diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/vyos/base.py | 7 | ||||
| -rw-r--r-- | python/vyos/component_version.py | 277 | ||||
| -rw-r--r-- | python/vyos/compose_config.py | 12 | ||||
| -rw-r--r-- | python/vyos/configtree.py | 13 | ||||
| -rw-r--r-- | python/vyos/load_config.py | 31 | ||||
| -rw-r--r-- | python/vyos/migrate.py | 283 | ||||
| -rw-r--r-- | python/vyos/migrator.py | 226 | ||||
| -rw-r--r-- | python/vyos/utils/dict.py | 2 | ||||
| -rw-r--r-- | python/vyos/utils/system.py | 12 |
9 files changed, 482 insertions, 381 deletions
diff --git a/python/vyos/base.py b/python/vyos/base.py index 054b1d837..ca96d96ce 100644 --- a/python/vyos/base.py +++ b/python/vyos/base.py @@ -63,3 +63,10 @@ class ConfigError(Exception): message = fill(message, width=72) # Call the base class constructor with the parameters it needs super().__init__(message) + +class MigrationError(Exception): + def __init__(self, message): + # Reformat the message and trim it to 72 characters in length + message = fill(message, width=72) + # Call the base class constructor with the parameters it needs + super().__init__(message) diff --git a/python/vyos/component_version.py b/python/vyos/component_version.py index 9662ebfcf..94215531d 100644 --- a/python/vyos/component_version.py +++ b/python/vyos/component_version.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2022-2024 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -35,133 +35,170 @@ VyOS 1.2: import os import re import sys -import fileinput +from dataclasses import dataclass +from dataclasses import replace +from typing import Optional from vyos.xml_ref import component_version +from vyos.utils.file import write_file from vyos.version import get_version from vyos.defaults import directories DEFAULT_CONFIG_PATH = os.path.join(directories['config'], 'config.boot') -def from_string(string_line, vintage='vyos'): - """ - Get component version dictionary from string. - Return empty dictionary if string contains no config information - or raise error if component version string malformed. - """ - version_dict = {} - - if vintage == 'vyos': - if re.match(r'// vyos-config-version:.+', string_line): - if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', string_line): - raise ValueError(f"malformed configuration string: {string_line}") +REGEX_WARN_VYOS = r'(// Warning: Do not remove the following line.)' +REGEX_WARN_VYATTA = r'(/\* Warning: Do not remove the following line. \*/)' +REGEX_COMPONENT_VERSION_VYOS = r'// vyos-config-version:\s+"([\w@:-]+)"\s*' +REGEX_COMPONENT_VERSION_VYATTA = r'/\* === vyatta-config-version:\s+"([\w@:-]+)"\s+=== \*/' +REGEX_RELEASE_VERSION_VYOS = r'// Release version:\s+(\S*)\s*' +REGEX_RELEASE_VERSION_VYATTA = r'/\* Release version:\s+(\S*)\s*\*/' - for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): - version_dict[pair[0]] = int(pair[1]) - - elif vintage == 'vyatta': - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line): - if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line): - raise ValueError(f"malformed configuration string: {string_line}") +CONFIG_FILE_VERSION = """\ +// Warning: Do not remove the following line. +// vyos-config-version: "{}" +// Release version: {} +""" - for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): - version_dict[pair[0]] = int(pair[1]) +warn_filter_vyos = re.compile(REGEX_WARN_VYOS) +warn_filter_vyatta = re.compile(REGEX_WARN_VYATTA) + +regex_filter = { 'vyos': dict(zip(['component', 'release'], + [re.compile(REGEX_COMPONENT_VERSION_VYOS), + re.compile(REGEX_RELEASE_VERSION_VYOS)])), + 'vyatta': dict(zip(['component', 'release'], + [re.compile(REGEX_COMPONENT_VERSION_VYATTA), + re.compile(REGEX_RELEASE_VERSION_VYATTA)])) } + +@dataclass +class VersionInfo: + component: Optional[dict[str,int]] = None + release: str = get_version() + vintage: str = 'vyos' + config_body: Optional[str] = None + footer_lines: Optional[list[str]] = None + + def component_is_none(self) -> bool: + return bool(self.component is None) + + def config_body_is_none(self) -> bool: + return bool(self.config_body is None) + + def update_footer(self): + f = CONFIG_FILE_VERSION.format(component_to_string(self.component), + self.release) + self.footer_lines = f.splitlines() + + def update_syntax(self): + self.vintage = 'vyos' + self.update_footer() + + def update_release(self, release: str): + self.release = release + self.update_footer() + + def update_component(self, key: str, version: int): + if not isinstance(version, int): + raise ValueError('version must be int') + if self.component is None: + self.component = {} + self.component[key] = version + self.component = dict(sorted(self.component.items(), key=lambda x: x[0])) + self.update_footer() + + def update_config_body(self, config_str: str): + self.config_body = config_str + + def write_string(self) -> str: + config_body = '' if self.config_body is None else self.config_body + footer_lines = [] if self.footer_lines is None else self.footer_lines + + return config_body + '\n' + '\n'.join(footer_lines) + '\n' + + def write(self, config_file): + string = self.write_string() + try: + write_file(config_file, string) + except Exception as e: + raise ValueError(e) from e + +def component_to_string(component: dict) -> str: + l = [f'{k}@{v}' for k, v in sorted(component.items(), key=lambda x: x[0])] + return ':'.join(l) + +def component_from_string(string: str) -> dict: + return {k: int(v) for k, v in re.findall(r'([\w,-]+)@(\d+)', string)} + +def version_info_from_file(config_file) -> VersionInfo: + """Return config file component and release version info.""" + version_info = VersionInfo() + try: + with open(config_file) as f: + config_str = f.read() + except OSError: + return None + + if len(parts := warn_filter_vyos.split(config_str)) > 1: + vintage = 'vyos' + elif len(parts := warn_filter_vyatta.split(config_str)) > 1: + vintage = 'vyatta' else: - raise ValueError("Unknown config string vintage") - - return version_dict - -def from_file(config_file_name=DEFAULT_CONFIG_PATH, vintage='vyos'): - """ - Get component version dictionary parsing config file line by line - """ - with open(config_file_name, 'r') as f: - for line_in_config in f: - version_dict = from_string(line_in_config, vintage=vintage) - if version_dict: - return version_dict - - # no version information - return {} - -def from_system(): - """ - Get system component version dict. - """ - return component_version() - -def format_string(ver: dict) -> str: - """ - Version dict to string. - """ - keys = list(ver) - keys.sort() - l = [] - for k in keys: - v = ver[k] - l.append(f'{k}@{v}') - sep = ':' - return sep.join(l) - -def version_footer(ver: dict, vintage='vyos') -> str: - """ - Version footer as string. - """ - ver_str = format_string(ver) - release = get_version() - if vintage == 'vyos': - ret_str = (f'// Warning: Do not remove the following line.\n' - + f'// vyos-config-version: "{ver_str}"\n' - + f'// Release version: {release}\n') - elif vintage == 'vyatta': - ret_str = (f'/* Warning: Do not remove the following line. */\n' - + f'/* === vyatta-config-version: "{ver_str}" === */\n' - + f'/* Release version: {release} */\n') - else: - raise ValueError("Unknown config string vintage") - - return ret_str - -def system_footer(vintage='vyos') -> str: - """ - System version footer as string. - """ - ver_d = from_system() - return version_footer(ver_d, vintage=vintage) - -def write_version_footer(ver: dict, file_name, vintage='vyos'): - """ - Write version footer to file. - """ - footer = version_footer(ver=ver, vintage=vintage) - if file_name: - with open(file_name, 'a') as f: - f.write(footer) + version_info.config_body = parts[0] if parts else None + return version_info + + version_info.vintage = vintage + version_info.config_body = parts[0] + version_lines = ''.join(parts[1:]).splitlines() + version_lines = [k for k in version_lines if k] + if len(version_lines) != 3: + raise ValueError(f'Malformed version strings: {version_lines}') + + m = regex_filter[vintage]['component'].match(version_lines[1]) + if not m: + raise ValueError(f'Malformed component string: {version_lines[1]}') + version_info.component = component_from_string(m.group(1)) + + m = regex_filter[vintage]['release'].match(version_lines[2]) + if not m: + raise ValueError(f'Malformed component string: {version_lines[2]}') + version_info.release = m.group(1) + + version_info.footer_lines = version_lines + + return version_info + +def version_info_from_system() -> VersionInfo: + """Return system component and release version info.""" + d = component_version() + sort_d = dict(sorted(d.items(), key=lambda x: x[0])) + version_info = VersionInfo( + component = sort_d, + release = get_version(), + vintage = 'vyos' + ) + + return version_info + +def version_info_copy(v: VersionInfo) -> VersionInfo: + """Make a copy of dataclass.""" + return replace(v) + +def version_info_prune_component(x: VersionInfo, y: VersionInfo) -> VersionInfo: + """In place pruning of component keys of x not in y.""" + if x.component is None or y.component is None: + return + x.component = { k: v for k,v in x.component.items() if k in y.component } + +def add_system_version(config_str: str = None, out_file: str = None): + """Wrap config string with system version and write to out_file. + + For convenience, calling with no argument will write system version + string to stdout, for use in bash scripts. + """ + version_info = version_info_from_system() + if config_str is not None: + version_info.update_config_body(config_str) + version_info.update_footer() + if out_file is not None: + version_info.write(out_file) else: - sys.stdout.write(footer) - -def write_system_footer(file_name, vintage='vyos'): - """ - Write system version footer to file. - """ - ver_d = from_system() - return write_version_footer(ver_d, file_name=file_name, vintage=vintage) - -def remove_footer(file_name): - """ - Remove old version footer. - """ - for line in fileinput.input(file_name, inplace=True): - if re.match(r'/\* Warning:.+ \*/$', line): - continue - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line): - continue - if re.match(r'/\* Release version:.+ \*/$', line): - continue - if re.match('// vyos-config-version:.+', line): - continue - if re.match('// Warning:.+', line): - continue - if re.match('// Release version:.+', line): - continue - sys.stdout.write(line) + sys.stdout.write(version_info.write_string()) diff --git a/python/vyos/compose_config.py b/python/vyos/compose_config.py index efa28babe..79a8718c5 100644 --- a/python/vyos/compose_config.py +++ b/python/vyos/compose_config.py @@ -17,12 +17,13 @@ config. """ +import traceback from pathlib import Path from typing import TypeAlias, Union, Callable from vyos.configtree import ConfigTree from vyos.configtree import deep_copy as ct_deep_copy -from vyos.utils.system import load_as_module +from vyos.utils.system import load_as_module_source ConfigObj: TypeAlias = Union[str, ConfigTree] @@ -54,7 +55,8 @@ class ComposeConfig: try: func(self.config_tree) except Exception as e: - self.config_tree = self.checkpoint + if self.checkpoint_file is not None: + self.config_tree = self.checkpoint raise ComposeConfigError(e) from e def apply_file(self, func_file: str, func_name: str): @@ -62,7 +64,7 @@ class ComposeConfig: """ try: mod_name = Path(func_file).stem.replace('-', '_') - mod = load_as_module(mod_name, func_file) + mod = load_as_module_source(mod_name, func_file) func = getattr(mod, func_name) except Exception as e: raise ComposeConfigError(f'Error with {func_file}: {e}') from e @@ -70,7 +72,9 @@ class ComposeConfig: try: self.apply_func(func) except ComposeConfigError as e: - raise ComposeConfigError(f'Error in {func_file}: {e}') from e + msg = str(e) + tb = f'{traceback.format_exc()}' + raise ComposeConfigError(f'Error in {func_file}: {msg}\n{tb}') from e def to_string(self, with_version=False) -> str: """Return the rendered config tree. diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index afd6e030b..5775070e2 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -15,6 +15,7 @@ import os import re import json +import logging from ctypes import cdll, c_char_p, c_void_p, c_int, c_bool @@ -161,6 +162,8 @@ class ConfigTree(object): self.__version = '' self.__migration = os.environ.get('VYOS_MIGRATION') + if self.__migration: + self.migration_log = logging.getLogger('vyos.migrate') def __del__(self): if self.__config is not None: @@ -215,7 +218,7 @@ class ConfigTree(object): self.__set_add_value(self.__config, path_str, str(value).encode()) if self.__migration: - print(f"- op: set path: {path} value: {value} replace: {replace}") + self.migration_log.info(f"- op: set path: {path} value: {value} replace: {replace}") def delete(self, path): check_path(path) @@ -226,7 +229,7 @@ class ConfigTree(object): raise ConfigTreeError(f"Path doesn't exist: {path}") if self.__migration: - print(f"- op: delete path: {path}") + self.migration_log.info(f"- op: delete path: {path}") def delete_value(self, path, value): check_path(path) @@ -242,7 +245,7 @@ class ConfigTree(object): raise ConfigTreeError() if self.__migration: - print(f"- op: delete_value path: {path} value: {value}") + self.migration_log.info(f"- op: delete_value path: {path} value: {value}") def rename(self, path, new_name): check_path(path) @@ -258,7 +261,7 @@ class ConfigTree(object): raise ConfigTreeError("Path [{}] doesn't exist".format(path)) if self.__migration: - print(f"- op: rename old_path: {path} new_path: {new_path}") + self.migration_log.info(f"- op: rename old_path: {path} new_path: {new_path}") def copy(self, old_path, new_path): check_path(old_path) @@ -275,7 +278,7 @@ class ConfigTree(object): raise ConfigTreeError(msg) if self.__migration: - print(f"- op: copy old_path: {old_path} new_path: {new_path}") + self.migration_log.info(f"- op: copy old_path: {old_path} new_path: {new_path}") def exists(self, path): check_path(path) diff --git a/python/vyos/load_config.py b/python/vyos/load_config.py index af563614d..b910a2f92 100644 --- a/python/vyos/load_config.py +++ b/python/vyos/load_config.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -28,9 +28,7 @@ from typing import Union, Literal, TypeAlias, get_type_hints, get_args from vyos.config import Config from vyos.configtree import ConfigTree, DiffTree from vyos.configsource import ConfigSourceSession, VyOSError -from vyos.component_version import from_string as version_from_string -from vyos.component_version import from_system as version_from_system -from vyos.migrator import Migrator, VirtualMigrator, MigratorError +from vyos.migrate import ConfigMigrate, ConfigMigrateError from vyos.utils.process import popen, DEVNULL Variety: TypeAlias = Literal['explicit', 'batch', 'tree', 'legacy'] @@ -51,20 +49,6 @@ def get_proposed_config(config_file: str = None) -> ConfigTree: config_str = Path(config_file).read_text() return ConfigTree(config_str) -def migration_needed(config_obj: ConfigObj) -> bool: - """Check if a migration is needed for the config object. - """ - if not isinstance(config_obj, ConfigTree): - atree = get_proposed_config(config_obj) - else: - atree = config_obj - version_str = atree.get_version_string() - if not version_str: - return True - aversion = version_from_string(version_str.splitlines()[1]) - bversion = version_from_system() - return aversion != bversion - def check_session(strict: bool, switch: Variety) -> None: """Check if we are in a config session, with no uncommitted changes, if strict. This is not needed for legacy load, as these checks are @@ -128,12 +112,10 @@ def migrate(config_obj: ConfigObj) -> ConfigObj: else: config_file = config_obj - virtual_migration = VirtualMigrator(config_file) - migration = Migrator(config_file) + config_migrate = ConfigMigrate(config_file) try: - virtual_migration.run() - migration.run() - except MigratorError as e: + config_migrate.run() + except ConfigMigrateError as e: raise LoadConfigError(e) from e else: if isinstance(config_obj, ConfigTree): @@ -193,8 +175,7 @@ def load(config_obj: ConfigObj, strict: bool = True, check_session(strict, switch) - if migration_needed(config_obj): - config_obj = migrate(config_obj) + config_obj = migrate(config_obj) func = getattr(thismod, f'load_{switch}') func(config_obj) diff --git a/python/vyos/migrate.py b/python/vyos/migrate.py new file mode 100644 index 000000000..9d1613676 --- /dev/null +++ b/python/vyos/migrate.py @@ -0,0 +1,283 @@ +# Copyright 2019-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +import os +import re +import json +import logging +from pathlib import Path +from grp import getgrnam + +from vyos.component_version import VersionInfo +from vyos.component_version import version_info_from_system +from vyos.component_version import version_info_from_file +from vyos.component_version import version_info_copy +from vyos.component_version import version_info_prune_component +from vyos.compose_config import ComposeConfig +from vyos.compose_config import ComposeConfigError +from vyos.configtree import ConfigTree +from vyos.defaults import directories as default_dir +from vyos.defaults import component_version_json + + +log_file = Path(default_dir['config']).joinpath('vyos-migrate.log') + +class ConfigMigrateError(Exception): + """Raised on error in config migration.""" + +class ConfigMigrate: + # pylint: disable=too-many-instance-attributes + # the number is reasonable in this case + def __init__(self, config_file: str, force=False, + output_file: str = None, checkpoint_file: str = None): + self.config_file: str = config_file + self.force: bool = force + self.system_version: VersionInfo = version_info_from_system() + self.file_version: VersionInfo = version_info_from_file(self.config_file) + self.compose = None + self.output_file = output_file + self.checkpoint_file = checkpoint_file + self.logger = None + self.config_modified = True + + if self.file_version is None: + raise ConfigMigrateError(f'failed to read config file {self.config_file}') + + def migration_needed(self) -> bool: + return self.system_version.component != self.file_version.component + + def release_update_needed(self) -> bool: + return self.system_version.release != self.file_version.release + + def syntax_update_needed(self) -> bool: + return self.system_version.vintage != self.file_version.vintage + + def update_release(self): + """ + Update config file release version. + """ + self.file_version.update_release(self.system_version.release) + + def update_syntax(self): + """ + Update config file syntax. + """ + self.file_version.update_syntax() + + @staticmethod + def normalize_config_body(version_info: VersionInfo): + """ + This is an interim workaround for the cosmetic issue of node + ordering when composing operations on the internal config_tree: + ordering is performed on parsing, hence was maintained in the old + system which would parse/write on each application of a migration + script (~200). Here, we will take the cost of one extra parsing to + reorder before save, for easier review. + """ + if not version_info.config_body_is_none(): + ct = ConfigTree(version_info.config_body) + version_info.update_config_body(ct.to_string()) + + def write_config(self): + if self.output_file is not None: + config_file = self.output_file + else: + config_file = self.config_file + + try: + self.file_version.write(config_file) + except ValueError as e: + raise ConfigMigrateError(f'failed to write {config_file}: {e}') from e + + def init_logger(self): + self.logger = logging.getLogger(__name__) + self.logger.setLevel(logging.DEBUG) + + fh = ConfigMigrate.group_perm_file_handler(log_file, + group='vyattacfg', + mode='w') + fh.setLevel(logging.INFO) + fh_formatter = logging.Formatter('%(message)s') + fh.setFormatter(fh_formatter) + self.logger.addHandler(fh) + ch = logging.StreamHandler() + ch.setLevel(logging.WARNING) + ch_formatter = logging.Formatter('%(name)s - %(levelname)s - %(message)s') + ch.setFormatter(ch_formatter) + self.logger.addHandler(ch) + + @staticmethod + def group_perm_file_handler(filename, group=None, mode='a'): + # pylint: disable=consider-using-with + if group is None: + return logging.FileHandler(filename, mode) + gid = getgrnam(group).gr_gid + if not os.path.exists(filename): + open(filename, 'a').close() + os.chown(filename, -1, gid) + os.chmod(filename, 0o664) + return logging.FileHandler(filename, mode) + + @staticmethod + def sort_function(): + """ + Define sort function for migration files as tuples (n, m) for file + n-to-m. + """ + numbers = re.compile(r'(\d+)') + def func(p: Path): + parts = numbers.split(p.stem) + return list(map(int, parts[1::2])) + return func + + @staticmethod + def file_ext(file_path: Path) -> str: + """ + Return an identifier from file name for checkpoint file extension. + """ + return f'{file_path.parent.stem}_{file_path.stem}' + + def run_migration_scripts(self): + """ + Call migration files iteratively. + """ + os.environ['VYOS_MIGRATION'] = '1' + + self.init_logger() + self.logger.info("List of applied migration modules:") + + components = list(self.system_version.component) + components.sort() + + # T4382: 'bgp' needs to follow 'quagga': + if 'bgp' in components and 'quagga' in components: + components.insert(components.index('quagga'), + components.pop(components.index('bgp'))) + + revision: VersionInfo = version_info_copy(self.file_version) + # prune retired, for example, zone-policy + version_info_prune_component(revision, self.system_version) + + migrate_dir = Path(default_dir['migrate']) + sort_func = ConfigMigrate.sort_function() + + for key in components: + p = migrate_dir.joinpath(key) + script_list = list(p.glob('*-to-*')) + script_list = sorted(script_list, key=sort_func) + + if not self.file_version.component_is_none() and not self.force: + start = self.file_version.component.get(key, 0) + script_list = list(filter(lambda x, st=start: sort_func(x)[0] >= st, + script_list)) + + if not script_list: # no applicable migration scripts + revision.update_component(key, self.system_version.component[key]) + continue + + for file in script_list: + f = file.as_posix() + self.logger.info(f'applying {f}') + try: + self.compose.apply_file(f, func_name='migrate') + except ComposeConfigError as e: + self.logger.error(e) + if self.checkpoint_file: + check = f'{self.checkpoint_file}_{ConfigMigrate.file_ext(file)}' + revision.update_config_body(self.compose.to_string()) + ConfigMigrate.normalize_config_body(revision) + revision.write(check) + break + else: + revision.update_component(key, sort_func(file)[1]) + + revision.update_config_body(self.compose.to_string()) + ConfigMigrate.normalize_config_body(revision) + self.file_version = version_info_copy(revision) + + if revision.component != self.system_version.component: + raise ConfigMigrateError(f'incomplete migration: check {log_file} for error') + + del os.environ['VYOS_MIGRATION'] + + def save_json_record(self): + """ + Write component versions to a json file + """ + version_file = component_version_json + + try: + with open(version_file, 'w') as f: + f.write(json.dumps(self.system_version.component, + indent=2, sort_keys=True)) + except OSError: + pass + + def load_config(self): + """ + Instantiate a ComposeConfig object with the config string. + """ + + self.compose = ComposeConfig(self.file_version.config_body, self.checkpoint_file) + + def run(self): + """ + If migration needed, run migration scripts and update config file. + If only release version update needed, update release version. + """ + # save system component versions in json file for reference + self.save_json_record() + + if not self.migration_needed(): + if self.release_update_needed(): + self.update_release() + self.write_config() + else: + self.config_modified = False + return + + if self.syntax_update_needed(): + self.update_syntax() + self.write_config() + + self.load_config() + + self.run_migration_scripts() + + self.update_release() + self.write_config() + + def run_script(self, test_script: str): + """ + Run a single migration script. For testing this simply provides the + body for loading and writing the result; the component string is not + updated. + """ + + self.load_config() + self.init_logger() + + os.environ['VYOS_MIGRATION'] = '1' + + try: + self.compose.apply_file(test_script, func_name='migrate') + except ComposeConfigError as e: + self.logger.error(f'config-migration error in {test_script}: {e}') + else: + self.file_version.update_config_body(self.compose.to_string()) + + del os.environ['VYOS_MIGRATION'] + + self.write_config() diff --git a/python/vyos/migrator.py b/python/vyos/migrator.py deleted file mode 100644 index 872682bc0..000000000 --- a/python/vyos/migrator.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see <http://www.gnu.org/licenses/>. - -import sys -import os -import json -import logging - -import vyos.defaults -import vyos.component_version as component_version -from vyos.utils.process import cmd - -log_file = os.path.join(vyos.defaults.directories['config'], 'vyos-migrate.log') - -class MigratorError(Exception): - pass - -class Migrator(object): - def __init__(self, config_file, force=False, set_vintage='vyos'): - self._config_file = config_file - self._force = force - self._set_vintage = set_vintage - self._config_file_vintage = None - self._changed = False - - def init_logger(self): - self.logger = logging.getLogger(__name__) - self.logger.setLevel(logging.DEBUG) - - # on adding the file handler, allow write permission for cfg_group; - # restore original umask on exit - mask = os.umask(0o113) - fh = logging.FileHandler(log_file) - formatter = logging.Formatter('%(message)s') - fh.setFormatter(formatter) - self.logger.addHandler(fh) - os.umask(mask) - - def read_config_file_versions(self): - """ - Get component versions from config file footer and set vintage; - return empty dictionary if config string is missing. - """ - cfg_file = self._config_file - component_versions = {} - - cfg_versions = component_version.from_file(cfg_file, vintage='vyatta') - - if cfg_versions: - self._config_file_vintage = 'vyatta' - component_versions = cfg_versions - - cfg_versions = component_version.from_file(cfg_file, vintage='vyos') - - if cfg_versions: - self._config_file_vintage = 'vyos' - component_versions = cfg_versions - - return component_versions - - def update_vintage(self): - old_vintage = self._config_file_vintage - - if self._set_vintage: - self._config_file_vintage = self._set_vintage - - if self._config_file_vintage not in ['vyatta', 'vyos']: - raise MigratorError("Unknown vintage.") - - if self._config_file_vintage == old_vintage: - return False - else: - return True - - def run_migration_scripts(self, config_file_versions, system_versions): - """ - Run migration scripts iteratively, until config file version equals - system component version. - """ - os.environ['VYOS_MIGRATION'] = '1' - self.init_logger() - - self.logger.info("List of executed migration scripts:") - - cfg_versions = config_file_versions - sys_versions = system_versions - - sys_keys = list(sys_versions.keys()) - sys_keys.sort() - - # XXX 'bgp' needs to follow 'quagga': - if 'bgp' in sys_keys and 'quagga' in sys_keys: - sys_keys.insert(sys_keys.index('quagga'), - sys_keys.pop(sys_keys.index('bgp'))) - - rev_versions = {} - - for key in sys_keys: - sys_ver = sys_versions[key] - if key in cfg_versions: - cfg_ver = cfg_versions[key] - else: - cfg_ver = 0 - - migrate_script_dir = os.path.join( - vyos.defaults.directories['migrate'], key) - - while cfg_ver < sys_ver: - next_ver = cfg_ver + 1 - - migrate_script = os.path.join(migrate_script_dir, - '{}-to-{}'.format(cfg_ver, next_ver)) - - try: - out = cmd([migrate_script, self._config_file]) - self.logger.info(f'{migrate_script}') - if out: self.logger.info(out) - except FileNotFoundError: - pass - except Exception as err: - print("\nMigration script error: {0}: {1}." - "".format(migrate_script, err)) - sys.exit(1) - - cfg_ver = next_ver - rev_versions[key] = cfg_ver - - del os.environ['VYOS_MIGRATION'] - return rev_versions - - def write_config_file_versions(self, cfg_versions): - """ - Write new versions string. - """ - if self._config_file_vintage == 'vyatta': - component_version.write_version_footer(cfg_versions, - self._config_file, - vintage='vyatta') - - if self._config_file_vintage == 'vyos': - component_version.write_version_footer(cfg_versions, - self._config_file, - vintage='vyos') - - def save_json_record(self, component_versions: dict): - """ - Write component versions to a json file - """ - mask = os.umask(0o113) - version_file = vyos.defaults.component_version_json - try: - with open(version_file, 'w') as f: - f.write(json.dumps(component_versions, indent=2, sort_keys=True)) - except OSError: - pass - finally: - os.umask(mask) - - def run(self): - """ - Gather component versions from config file and system. - Run migration scripts. - Update vintage ('vyatta' or 'vyos'), if needed. - If changed, remove old versions string from config file, and - write new versions string. - """ - cfg_file = self._config_file - - cfg_versions = self.read_config_file_versions() - if self._force: - # This will force calling all migration scripts: - cfg_versions = {} - - sys_versions = component_version.from_system() - - # save system component versions in json file for easy reference - self.save_json_record(sys_versions) - - rev_versions = self.run_migration_scripts(cfg_versions, sys_versions) - - if rev_versions != cfg_versions: - self._changed = True - - if self.update_vintage(): - self._changed = True - - if not self._changed: - return - - component_version.remove_footer(cfg_file) - - self.write_config_file_versions(rev_versions) - - def config_changed(self): - return self._changed - -class VirtualMigrator(Migrator): - def run(self): - cfg_file = self._config_file - - cfg_versions = self.read_config_file_versions() - if not cfg_versions: - return - - if self.update_vintage(): - self._changed = True - - if not self._changed: - return - - component_version.remove_footer(cfg_file) - - self.write_config_file_versions(cfg_versions) - diff --git a/python/vyos/utils/dict.py b/python/vyos/utils/dict.py index 9a4671c5f..1a7a6b96f 100644 --- a/python/vyos/utils/dict.py +++ b/python/vyos/utils/dict.py @@ -34,7 +34,7 @@ def colon_separated_to_dict(data_string, uniquekeys=False): otherwise they are always lists of strings. """ import re - key_value_re = re.compile('([^:]+)\s*\:\s*(.*)') + key_value_re = re.compile(r'([^:]+)\s*\:\s*(.*)') data_raw = re.split('\n', data_string) diff --git a/python/vyos/utils/system.py b/python/vyos/utils/system.py index 55813a5f7..cfd5b142c 100644 --- a/python/vyos/utils/system.py +++ b/python/vyos/utils/system.py @@ -98,3 +98,15 @@ def load_as_module(name: str, path: str): mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod + +def load_as_module_source(name: str, path: str): + """ Necessary modification of load_as_module for files without *.py + extension """ + import importlib.util + from importlib.machinery import SourceFileLoader + + loader = SourceFileLoader(name, path) + spec = importlib.util.spec_from_loader(name, loader) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod |