diff options
Diffstat (limited to 'python')
179 files changed, 11860 insertions, 1258 deletions
diff --git a/python/setup.py b/python/setup.py index 96dc211f7..571b956ee 100644 --- a/python/setup.py +++ b/python/setup.py @@ -7,6 +7,9 @@ from setuptools.command.build_py import build_py sys.path.append('./vyos') from defaults import directories +def desc_out(f): + return os.path.splitext(f)[0] + '.desc' + def packages(directory): return [ _[0].replace('/','.') @@ -37,9 +40,17 @@ class GenerateProto(build_py): 'protoc', '--python_out=vyos/proto', f'--proto_path={self.proto_path}/', + f'--descriptor_set_out=vyos/proto/{desc_out(proto_file)}', proto_file, ] ) + subprocess.check_call( + [ + 'vyos/proto/generate_dataclass.py', + 'vyos/proto/vyconf.desc', + '--out-dir=vyos/proto', + ] + ) build_py.run(self) diff --git a/python/vyos/accel_ppp.py b/python/vyos/accel_ppp.py index bae695fc3..b1160dc76 100644 --- a/python/vyos/accel_ppp.py +++ b/python/vyos/accel_ppp.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2024 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as diff --git a/python/vyos/accel_ppp_util.py b/python/vyos/accel_ppp_util.py index ae75e6654..844bc56e5 100644 --- a/python/vyos/accel_ppp_util.py +++ b/python/vyos/accel_ppp_util.py @@ -1,4 +1,4 @@ -# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -130,7 +130,7 @@ def verify_accel_ppp_authentication(config, local_users=True): Common helper function which must be used by all Accel-PPP services based on get_config_dict() """ - # vertify auth settings + # verify auth settings if local_users and dict_search("authentication.mode", config) == "local": if ( dict_search("authentication.local_users", config) is None @@ -173,7 +173,7 @@ def verify_accel_ppp_authentication(config, local_users=True): user_config = config["authentication"]["interface"][interface] if "mac" not in user_config: raise ConfigError( - f'Users MAC addreses are not configured for interface "{interface}"') + f'Users MAC addresses are not configured for interface "{interface}"') if dict_search('authentication.radius.dynamic_author.server', config): if not dict_search('authentication.radius.dynamic_author.key', config): @@ -221,10 +221,12 @@ def verify_accel_ppp_ip_pool(vpn_config): for interface, interface_config in vpn_config['interface'].items(): if dict_search('client_subnet', interface_config): break + if dict_search('external_dhcp.dhcp_relay', interface_config): + break else: raise ConfigError( 'Local auth and noauth mode requires local client-ip-pool \ - or client-ipv6-pool or client-subnet to be configured!') + or client-ipv6-pool or client-subnet or dhcp-relay to be configured!') else: raise ConfigError( "Local auth mode requires local client-ip-pool \ diff --git a/python/vyos/airbag.py b/python/vyos/airbag.py index 3c7a144b7..69b44dc9d 100644 --- a/python/vyos/airbag.py +++ b/python/vyos/airbag.py @@ -1,4 +1,4 @@ -# Copyright 2019-2020 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -15,19 +15,23 @@ import sys from datetime import datetime +from collections import deque from vyos import debug from vyos.logger import syslog from vyos.version import get_full_version_data +from vyos.defaults import airbag_noteworthy_size -def enable(log=True): +def enable(log=False): + if 'nose2' in sys.modules: + return if log: _intercepting_logger() _intercepting_exceptions() -_noteworthy = [] +_noteworthy = deque(maxlen=airbag_noteworthy_size) def noteworthy(msg): @@ -72,7 +76,7 @@ def bug_report(dtype, value, trace): note = '' if _noteworthy: note = 'noteworthy:\n' - note += '\n'.join(_noteworthy) + note += '\n'.join(list(_noteworthy)) information.update({ 'date': datetime.now().strftime('%Y-%m-%d %H:%M:%S'), diff --git a/python/vyos/base.py b/python/vyos/base.py index ca96d96ce..67f92564e 100644 --- a/python/vyos/base.py +++ b/python/vyos/base.py @@ -1,4 +1,4 @@ -# Copyright 2018-2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -15,8 +15,7 @@ from textwrap import fill - -class BaseWarning: +class UserMessage: def __init__(self, header, message, **kwargs): self.message = message self.kwargs = kwargs @@ -33,7 +32,6 @@ class BaseWarning: messages = self.message.split('\n') isfirstmessage = True initial_indent = self.textinitindent - print('') for mes in messages: mes = fill(mes, initial_indent=initial_indent, subsequent_indent=self.standardindent, **self.kwargs) @@ -44,17 +42,24 @@ class BaseWarning: print('', flush=True) +class Message(): + def __init__(self, message, **kwargs): + self.Message = UserMessage('', message, **kwargs) + self.Message.print() + class Warning(): def __init__(self, message, **kwargs): - self.BaseWarn = BaseWarning('WARNING: ', message, **kwargs) - self.BaseWarn.print() + print('') + self.UserMessage = UserMessage('WARNING: ', message, **kwargs) + self.UserMessage.print() class DeprecationWarning(): def __init__(self, message, **kwargs): # Reformat the message and trim it to 72 characters in length - self.BaseWarn = BaseWarning('DEPRECATION WARNING: ', message, **kwargs) - self.BaseWarn.print() + print('') + self.UserMessage = UserMessage('DEPRECATION WARNING: ', message, **kwargs) + self.UserMessage.print() class ConfigError(Exception): diff --git a/python/vyos/component_version.py b/python/vyos/component_version.py index 94215531d..13fb8333f 100644 --- a/python/vyos/component_version.py +++ b/python/vyos/component_version.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -49,7 +49,9 @@ DEFAULT_CONFIG_PATH = os.path.join(directories['config'], 'config.boot') REGEX_WARN_VYOS = r'(// Warning: Do not remove the following line.)' REGEX_WARN_VYATTA = r'(/\* Warning: Do not remove the following line. \*/)' REGEX_COMPONENT_VERSION_VYOS = r'// vyos-config-version:\s+"([\w@:-]+)"\s*' -REGEX_COMPONENT_VERSION_VYATTA = r'/\* === vyatta-config-version:\s+"([\w@:-]+)"\s+=== \*/' +REGEX_COMPONENT_VERSION_VYATTA = ( + r'/\* === vyatta-config-version:\s+"([\w@:-]+)"\s+=== \*/' +) REGEX_RELEASE_VERSION_VYOS = r'// Release version:\s+(\S*)\s*' REGEX_RELEASE_VERSION_VYATTA = r'/\* Release version:\s+(\S*)\s*\*/' @@ -62,16 +64,31 @@ CONFIG_FILE_VERSION = """\ warn_filter_vyos = re.compile(REGEX_WARN_VYOS) warn_filter_vyatta = re.compile(REGEX_WARN_VYATTA) -regex_filter = { 'vyos': dict(zip(['component', 'release'], - [re.compile(REGEX_COMPONENT_VERSION_VYOS), - re.compile(REGEX_RELEASE_VERSION_VYOS)])), - 'vyatta': dict(zip(['component', 'release'], - [re.compile(REGEX_COMPONENT_VERSION_VYATTA), - re.compile(REGEX_RELEASE_VERSION_VYATTA)])) } +regex_filter = { + 'vyos': dict( + zip( + ['component', 'release'], + [ + re.compile(REGEX_COMPONENT_VERSION_VYOS), + re.compile(REGEX_RELEASE_VERSION_VYOS), + ], + ) + ), + 'vyatta': dict( + zip( + ['component', 'release'], + [ + re.compile(REGEX_COMPONENT_VERSION_VYATTA), + re.compile(REGEX_RELEASE_VERSION_VYATTA), + ], + ) + ), +} + @dataclass class VersionInfo: - component: Optional[dict[str,int]] = None + component: Optional[dict[str, int]] = None release: str = get_version() vintage: str = 'vyos' config_body: Optional[str] = None @@ -84,8 +101,9 @@ class VersionInfo: return bool(self.config_body is None) def update_footer(self): - f = CONFIG_FILE_VERSION.format(component_to_string(self.component), - self.release) + f = CONFIG_FILE_VERSION.format( + component_to_string(self.component), self.release + ) self.footer_lines = f.splitlines() def update_syntax(self): @@ -121,13 +139,16 @@ class VersionInfo: except Exception as e: raise ValueError(e) from e + def component_to_string(component: dict) -> str: - l = [f'{k}@{v}' for k, v in sorted(component.items(), key=lambda x: x[0])] + l = [f'{k}@{v}' for k, v in sorted(component.items(), key=lambda x: x[0])] # noqa: E741 return ':'.join(l) + def component_from_string(string: str) -> dict: return {k: int(v) for k, v in re.findall(r'([\w,-]+)@(\d+)', string)} + def version_info_from_file(config_file) -> VersionInfo: """Return config file component and release version info.""" version_info = VersionInfo() @@ -166,27 +187,37 @@ def version_info_from_file(config_file) -> VersionInfo: return version_info + def version_info_from_system() -> VersionInfo: """Return system component and release version info.""" d = component_version() sort_d = dict(sorted(d.items(), key=lambda x: x[0])) - version_info = VersionInfo( - component = sort_d, - release = get_version(), - vintage = 'vyos' - ) + version_info = VersionInfo(component=sort_d, release=get_version(), vintage='vyos') return version_info + def version_info_copy(v: VersionInfo) -> VersionInfo: """Make a copy of dataclass.""" return replace(v) + def version_info_prune_component(x: VersionInfo, y: VersionInfo) -> VersionInfo: """In place pruning of component keys of x not in y.""" if x.component is None or y.component is None: return - x.component = { k: v for k,v in x.component.items() if k in y.component } + x.component = {k: v for k, v in x.component.items() if k in y.component} + + +def add_system_version_string(config_str: str = None) -> str: + """Wrap config string with system version and return string.""" + version_info = version_info_from_system() + if config_str is not None: + version_info.update_config_body(config_str) + version_info.update_footer() + + return version_info.write_string() + def add_system_version(config_str: str = None, out_file: str = None): """Wrap config string with system version and write to out_file. @@ -202,3 +233,11 @@ def add_system_version(config_str: str = None, out_file: str = None): version_info.write(out_file) else: sys.stdout.write(version_info.write_string()) + + +def append_system_version(file: str): + """Append system version data to existing file""" + version_info = version_info_from_system() + version_info.update_footer() + with open(file, 'a') as f: + f.write(version_info.write_string()) diff --git a/python/vyos/compose_config.py b/python/vyos/compose_config.py index 79a8718c5..1e7837858 100644 --- a/python/vyos/compose_config.py +++ b/python/vyos/compose_config.py @@ -1,4 +1,4 @@ -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/config.py b/python/vyos/config.py index 1fab46761..827246d05 100644 --- a/python/vyos/config.py +++ b/python/vyos/config.py @@ -1,4 +1,4 @@ -# Copyright 2017-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -53,7 +53,7 @@ VyOS has two distinct modes: operational mode and configuration mode. When a use the CLI is in the operational mode. In this mode, only the running (effective) config is accessible for reading. When a user enters the "configure" command, a configuration session is setup. Every config session -has its *proposed* (or *session*) config built on top of the current running config. When changes are commited, if commit succeeds, +has its *proposed* (or *session*) config built on top of the current running config. When changes are committed, if commit succeeds, the proposed config is merged into the running config. In configuration mode, "base" functions like `exists`, `return_value` return values from the session config, @@ -67,14 +67,18 @@ import json from typing import Union import vyos.configtree +from vyos.base import Warning from vyos.xml_ref import multi_to_list from vyos.xml_ref import from_source from vyos.xml_ref import ext_dict_merge from vyos.xml_ref import relative_defaults from vyos.utils.dict import get_sub_dict from vyos.utils.dict import mangle_dict_keys +from vyos.utils.boot import boot_configuration_complete +from vyos.utils.backend import vyconf_backend from vyos.configsource import ConfigSource from vyos.configsource import ConfigSourceSession +from vyos.configsource import ConfigSourceVyconfSession class ConfigDict(dict): _from_defaults = {} @@ -118,7 +122,7 @@ def config_dict_mangle_acme(name, cli_dict): # install ACME based PEM keys into "regular" CLI config keys cli_dict.update({'certificate' : cert_base64, 'private' : {'key' : key_base64}}) except: - raise ConfigError(f'Unable to load ACME certificates for "{name}"!') + Warning(f'Unable to load ACME certificates for "{name}"!') return cli_dict @@ -131,8 +135,13 @@ class Config(object): subtrees. """ def __init__(self, session_env=None, config_source=None): + self.vyconf_session = None if config_source is None: - self._config_source = ConfigSourceSession(session_env) + if vyconf_backend() and boot_configuration_complete(): + self._config_source = ConfigSourceVyconfSession(session_env) + self.vyconf_session = self._config_source._vyconf_session + else: + self._config_source = ConfigSourceSession(session_env) else: if not isinstance(config_source, ConfigSource): raise TypeError("config_source not of type ConfigSource") @@ -149,6 +158,18 @@ class Config(object): return self._running_config return self._session_config + def get_bool_attr(self, attr) -> bool: + if not hasattr(self, attr): + return False + else: + tmp = getattr(self, attr) + if not isinstance(tmp, bool): + return False + return tmp + + def set_bool_attr(self, attr, val): + setattr(self, attr, val) + def _make_path(self, path): # Backwards-compatibility stuff: original implementation used string paths # libvyosconfig paths are lists, but since node names cannot contain whitespace, @@ -238,7 +259,7 @@ class Config(object): def session_changed(self): """ Returns: - True if the config session has uncommited changes, False otherwise. + True if the config session has uncommitted changes, False otherwise. """ return self._config_source.session_changed() @@ -342,7 +363,7 @@ class Config(object): pki_dict['certificate'][certificate] = config_dict_mangle_acme( certificate, pki_dict['certificate'][certificate]) - conf_dict['pki'] = pki_dict + conf_dict['pki'] = pki_dict interfaces_root = root_dict.get('interfaces', {}) setattr(conf_dict, 'interfaces_root', interfaces_root) @@ -523,7 +544,7 @@ class Config(object): Note: This function is safe to use in operational mode. In configuration mode, - it ignores uncommited changes. + it ignores uncommitted changes. """ if self._running_config is None: return False diff --git a/python/vyos/config_mgmt.py b/python/vyos/config_mgmt.py index dd8910afb..b3efdeac9 100644 --- a/python/vyos/config_mgmt.py +++ b/python/vyos/config_mgmt.py @@ -1,4 +1,4 @@ -# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -25,25 +25,32 @@ from filecmp import cmp from datetime import datetime from textwrap import dedent from pathlib import Path -from tabulate import tabulate from shutil import copy, chown +from subprocess import Popen +from subprocess import DEVNULL from urllib.parse import urlsplit from urllib.parse import urlunsplit +from tabulate import tabulate from vyos.config import Config from vyos.configtree import ConfigTree from vyos.configtree import ConfigTreeError from vyos.configsession import ConfigSession from vyos.configsession import ConfigSessionError -from vyos.configtree import show_diff +from vyos.configtree import diff_compare +from vyos.configtree import DiffTree from vyos.load_config import load from vyos.load_config import LoadConfigError from vyos.defaults import directories from vyos.version import get_full_version_data from vyos.utils.io import ask_yes_no +from vyos.utils.io import catch_broken_pipe from vyos.utils.boot import boot_configuration_complete from vyos.utils.process import is_systemd_service_active from vyos.utils.process import rc_cmd +from vyos.defaults import DEFAULT_COMMIT_CONFIRM_MINUTES +from vyos.component_version import append_system_version +from vyos.utils.file import file_compare SAVE_CONFIG = '/usr/libexec/vyos/vyos-save-config.py' config_json = '/run/vyatta/config/config.json' @@ -56,7 +63,6 @@ commit_hooks = { 'commit_archive': '02vyos-commit-archive', } -DEFAULT_TIME_MINUTES = 10 timer_name = 'commit-confirm' config_file = os.path.join(directories['config'], 'config.boot') @@ -94,7 +100,7 @@ def unsaved_commits(allow_missing_config=False) -> bool: return True tmp_save = '/tmp/config.running' save_config(tmp_save) - ret = not cmp(tmp_save, config_file, shallow=False) + ret = not file_compare(tmp_save, config_file) os.unlink(tmp_save) return ret @@ -144,14 +150,16 @@ class ConfigMgmt: ['system', 'config-management'], key_mangling=('-', '_'), get_first_key=True, - with_defaults=True, + with_recursive_defaults=True, ) self.max_revisions = int(d.get('commit_revisions', 0)) self.num_revisions = 0 self.locations = d.get('commit_archive', {}).get('location', []) self.source_address = d.get('commit_archive', {}).get('source_address', '') - self.reboot_unconfirmed = bool(d.get('commit_confirm') == 'reboot') + self.reboot_unconfirmed = bool( + d.get('commit_confirm', {}).get('action') == 'reboot' + ) self.config_dict = d if config.exists(['system', 'host-name']): @@ -165,11 +173,16 @@ class ConfigMgmt: # upload only on existence of effective values, notably, on boot. # one still needs session self.locations (above) for setting # post-commit hook in conf_mode script - path = ['system', 'config-management', 'commit-archive', 'location'] - if config.exists_effective(path): - self.effective_locations = config.return_effective_values(path) - else: - self.effective_locations = [] + base_path = ['system', 'config-management', 'commit-archive'] + location_path = base_path + ['location'] + self.effective_locations = None + if config.exists_effective(location_path): + self.effective_locations = config.return_effective_values(location_path) + + vrf_path = base_path + ['vrf'] + self.effective_vrf = None + if config.exists_effective(vrf_path): + self.effective_vrf = config.return_effective_value(vrf_path) # a call to compare without args is edit_level aware edit_level = os.getenv('VYATTA_EDIT_LEVEL', '') @@ -181,7 +194,7 @@ class ConfigMgmt: # Console script functions # def commit_confirm( - self, minutes: int = DEFAULT_TIME_MINUTES, no_prompt: bool = False + self, minutes: int = DEFAULT_COMMIT_CONFIRM_MINUTES, no_prompt: bool = False ) -> Tuple[str, int]: """Commit with reload/reboot to saved config in 'minutes' minutes if 'confirm' call is not issued. @@ -229,7 +242,14 @@ Proceed ?""" else: cmd = f'sudo -b /usr/libexec/vyos/commit-confirm-notify.py {minutes}' - os.system(cmd) + Popen( + cmd.split(), + stdout=DEVNULL, + stderr=DEVNULL, + stdin=DEVNULL, + close_fds=True, + preexec_fn=os.setsid, + ) if self.reboot_unconfirmed: msg = f'Initialized commit-confirm; {minutes} minutes to confirm before reboot' @@ -296,7 +316,11 @@ Proceed ?""" session = ConfigSession(os.getpid(), app='config-mgmt') try: - session.load_explicit(revert_ct) + if session.vyconf_backend(): + session.load_config_obj(revert_ct) + else: + session.load_explicit(revert_ct) + session.commit() except ConfigSessionError as e: raise ConfigMgmtError(e) from e @@ -397,9 +421,9 @@ Proceed ?""" path = [] if commands else self.edit_path try: if commands: - out = show_diff(ct1, ct2, path=path, commands=True) + out = diff_compare(ct1, ct2, path=path, commands=True) else: - out = show_diff(ct1, ct2, path=path) + out = diff_compare(ct1, ct2, path=path) except ConfigTreeError as e: return e, 1 @@ -428,6 +452,80 @@ Proceed ?""" return self.compare(commands=cmnds, rev1=r1, rev2=r2) + def _format_remote_diff(self, diff_tree: DiffTree, path: list, commands: bool): + add_tree = diff_tree.add + del_tree = diff_tree.delete + command_prefix = ' '.join(path) + + result_lines = [] + if commands: + # Process the deleted elements into command format and filter based on prefix (path) + for line in del_tree.to_commands(op='delete').splitlines(): + if line.startswith(f'delete {command_prefix}'): + result_lines.append(line) + + # Process the added elements into command format and filter based on prefix (path) + for line in add_tree.to_commands(op='set').splitlines(): + if line.startswith(f'set {command_prefix}'): + result_lines.append(line) + else: + with_node = len(path) > 1 + # Retrieve subtrees for the specified path from both added and deleted trees + del_tree = del_tree.get_subtree(path, with_node=with_node) + add_tree = add_tree.get_subtree(path, with_node=with_node) + + # Convert the subtrees to string lines for further processing + del_tree_lines = str(del_tree).splitlines() + add_tree_lines = str(add_tree).splitlines() + + # Format the lines with a prefix ('-', '+') and filter out empty lines + del_lines = [f'- {l}' for l in del_tree_lines if l.strip()] + add_lines = [f'+ {l}' for l in add_tree_lines if l.strip()] + + if del_lines or add_lines: + # Adjust command prefix if a node is present in the path + command_prefix = ' '.join(path[:-1]) if with_node else command_prefix + if command_prefix: + result_lines.append(f'[{command_prefix}]') + + # Combine both deleted and added lines and process them + result_lines.extend(del_lines + add_lines) + + # Join the result lines into a single string, excluding empty lines + return '\n'.join((line for line in result_lines if line.strip())) + + def remote_compare( + self, + source: str, + remote_tree: ConfigTree, + path: Optional[list] = None, + commands: bool = False, + ) -> str: + """ + Compares a local configuration tree with a remote + configuration tree based on the specified source ('running', 'candidate', 'saved'). + """ + path = path or [] + + # Determine the correct local configuration tree based on the 'source' parameter + if source == 'running': + local_tree = self.active_config + elif source == 'candidate': + local_tree = self.working_config + elif source == 'saved': + local_tree = self._get_saved_config_tree() + else: + raise ConfigMgmtError( + 'Invalid source, must be one of: running, candidate, saved' + ) + + try: + diff_tree = DiffTree(remote_tree, local_tree) + except ConfigTreeError as e: + raise ConfigMgmtError(e) from e + + return self._format_remote_diff(diff_tree, path, commands) + # Initialization and post-commit hooks for conf-mode # def initialize_revision(self): @@ -481,16 +579,13 @@ Proceed ?""" if self.effective_locations: print('Archiving config...') - for location in self.effective_locations: - url = urlsplit(location) - _, _, netloc = url.netloc.rpartition('@') - redacted_location = urlunsplit(url._replace(netloc=netloc)) - print(f' {redacted_location}', end=' ', flush=True) - upload( - archive_config_file, - f'{location}/{remote_file}', - source_host=source_address, - ) + for location in self.effective_locations: + url = urlsplit(location) + _, _, netloc = url.netloc.rpartition('@') + redacted_location = urlunsplit(url._replace(netloc=netloc)) + print(f' {redacted_location}', end=' ', flush=True) + upload(archive_config_file, f'{location}/{remote_file}', + source_host=source_address, vrf=self.effective_vrf) # op-mode functions # @@ -543,6 +638,7 @@ Proceed ?""" ret = tabulate(res_l, tablefmt='plain') return ret + @catch_broken_pipe def show_commit_diff( self, rev: int, rev2: Optional[int] = None, commands: bool = False ) -> str: @@ -594,14 +690,16 @@ Proceed ?""" conf_file.chmod(0o644) def _archive_active_config(self) -> bool: - save_to_tmp = boot_configuration_complete() or not os.path.isfile( - archive_config_file - ) + # on first boot/fresh install, add baseline archive_config_file + if not os.path.exists(archive_config_file): + append_system_version(archive_config_file) + mask = os.umask(0o113) ext = os.getpid() cmp_saved = f'/tmp/config.boot.{ext}' - if save_to_tmp: + + if boot_configuration_complete(): save_config(cmp_saved, json_out=config_json) else: copy(config_file, cmp_saved) @@ -781,6 +879,7 @@ Proceed ?""" # entry_point for console script # +@catch_broken_pipe def run(): from argparse import ArgumentParser, REMAINDER @@ -805,7 +904,7 @@ def run(): '-t', dest='minutes', type=int, - default=DEFAULT_TIME_MINUTES, + default=DEFAULT_COMMIT_CONFIRM_MINUTES, help="Minutes until reboot, unless 'confirm'", ) commit_confirm.add_argument( @@ -819,7 +918,7 @@ def run(): rollback = subparsers.add_parser('rollback', help='Rollback to earlier config') rollback.add_argument('--rev', type=int, help='Revision number for rollback') rollback.add_argument( - '-y', dest='no_prompt', action='store_true', help='Excute without prompt' + '-y', dest='no_prompt', action='store_true', help='Execute without prompt' ) rollback_soft = subparsers.add_parser( diff --git a/python/vyos/configdep.py b/python/vyos/configdep.py index cf7c9d543..f1dd7895a 100644 --- a/python/vyos/configdep.py +++ b/python/vyos/configdep.py @@ -1,4 +1,4 @@ -# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -102,11 +102,16 @@ def run_config_mode_script(target: str, config: 'Config'): mod = load_as_module(name, path) config.set_level([]) + dry_run = config.get_bool_attr('dry_run') try: c = mod.get_config(config) mod.verify(c) - mod.generate(c) - mod.apply(c) + if not dry_run: + mod.generate(c) + mod.apply(c) + else: + if hasattr(mod, 'call_dependents'): + mod.call_dependents() except (VyOSError, ConfigError) as e: raise ConfigError(str(e)) from e @@ -181,7 +186,7 @@ def graph_from_dependency_dict(d: dict) -> dict: for k in list(d): g[k] = set() # add the dependencies for every sub-case; should there be cases - # that are mutally exclusive in the future, the graphs will be + # that are mutually exclusive in the future, the graphs will be # distinguished for el in list(d[k]): g[k] |= set(d[k][el]) diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py index 78b98a3eb..0603b42b8 100644 --- a/python/vyos/configdict.py +++ b/python/vyos/configdict.py @@ -1,4 +1,4 @@ -# Copyright 2019-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -355,47 +355,48 @@ def is_source_interface(conf, interface, intftype=None): def get_dhcp_interfaces(conf, vrf=None): """ Common helper functions to retrieve all interfaces from current CLI - sessions that have DHCP configured. """ + sessions that have DHCP configured. Only DHCP interfaces in the defined VRF + will be returned. If vrf is None - the default VRF interfaces will be + returned """ + dhcp_interfaces = {} - dict = conf.get_config_dict(['interfaces'], get_first_key=True) - if not dict: + interface_dict = conf.get_config_dict(['interfaces'], get_first_key=True) + if not interface_dict: return dhcp_interfaces - def check_dhcp(config): - ifname = config['ifname'] + def check_dhcp(if_config: dict, vrf=None) -> dict: + ifname = if_config['ifname'] tmp = {} - if 'address' in config and 'dhcp' in config['address']: + if 'address' in if_config and 'dhcp' in if_config['address']: options = {} - if dict_search('dhcp_options.default_route_distance', config) != None: - options.update({'dhcp_options' : config['dhcp_options']}) - if 'vrf' in config: - if vrf == config['vrf']: tmp.update({ifname : options}) + if dict_search('dhcp_options.default_route_distance', if_config) != None: + options.update({'dhcp_options' : if_config['dhcp_options']}) + if 'vrf' in if_config: + if vrf == if_config['vrf']: tmp.update({ifname : options}) else: if vrf is None: tmp.update({ifname : options}) return tmp - for section, interface in dict.items(): + for section, interface in interface_dict.items(): for ifname in interface: - # always reset config level, as get_interface_dict() will alter it - conf.set_level([]) # we already have a dict representation of the config from get_config_dict(), # but with the extended information from get_interface_dict() we also # get the DHCP client default-route-distance default option if not specified. _, ifconfig = get_interface_dict(conf, ['interfaces', section], ifname) - tmp = check_dhcp(ifconfig) + tmp = check_dhcp(ifconfig, vrf=vrf) dhcp_interfaces.update(tmp) # check per VLAN interfaces for vif, vif_config in ifconfig.get('vif', {}).items(): - tmp = check_dhcp(vif_config) + tmp = check_dhcp(vif_config, vrf=vrf) dhcp_interfaces.update(tmp) # check QinQ VLAN interfaces for vif_s, vif_s_config in ifconfig.get('vif_s', {}).items(): - tmp = check_dhcp(vif_s_config) + tmp = check_dhcp(vif_s_config, vrf=vrf) dhcp_interfaces.update(tmp) for vif_c, vif_c_config in vif_s_config.get('vif_c', {}).items(): - tmp = check_dhcp(vif_c_config) + tmp = check_dhcp(vif_c_config, vrf=vrf) dhcp_interfaces.update(tmp) return dhcp_interfaces @@ -404,7 +405,7 @@ def get_pppoe_interfaces(conf, vrf=None): """ Common helper functions to retrieve all interfaces from current CLI sessions that have DHCP configured. """ pppoe_interfaces = {} - conf.set_level([]) + conf.set_level([]) # required for list_nodes() for ifname in conf.list_nodes(['interfaces', 'pppoe']): # always reset config level, as get_interface_dict() will alter it conf.set_level([]) @@ -429,7 +430,7 @@ def get_interface_dict(config, base, ifname='', recursive_defaults=True, with_pk """ Common utility function to retrieve and mangle the interfaces configuration from the CLI input nodes. All interfaces have a common base where value - retrival is identical. This function must be used whenever possible when + retrieval is identical. This function must be used whenever possible when working on the interfaces node! Return a dictionary with the necessary interface config keys. @@ -471,7 +472,7 @@ def get_interface_dict(config, base, ifname='', recursive_defaults=True, with_pk # Check if QoS policy applied on this interface - See ifconfig.interface.set_mirror_redirect() if config.exists(['qos', 'interface', ifname]): - dict.update({'traffic_policy': {}}) + dict.update({'qos': {}}) address = leaf_node_changed(config, base + [ifname, 'address']) if address: dict.update({'address_old' : address}) @@ -488,14 +489,14 @@ def get_interface_dict(config, base, ifname='', recursive_defaults=True, with_pk bond = is_member(config, ifname, 'bonding') if bond: dict.update({'is_bond_member' : bond}) - # Check if any DHCP options changed which require a client restat + # Check if any DHCP options changed which require a client restart dhcp = is_node_changed(config, base + [ifname, 'dhcp-options']) if dhcp: dict.update({'dhcp_options_changed' : {}}) dhcpv6 = is_node_changed(config, base + [ifname, 'dhcpv6-options']) if dhcpv6: dict.update({'dhcpv6_options_changed' : {}}) # Some interfaces come with a source_interface which must also not be part - # of any other bond or bridge interface as it is exclusivly assigned as the + # of any other bond or bridge interface as it is exclusively assigned as the # Kernels "lower" interface to this new "virtual/upper" interface. if 'source_interface' in dict: # Check if source interface is member of another bridge @@ -517,12 +518,20 @@ def get_interface_dict(config, base, ifname='', recursive_defaults=True, with_pk else: dict['ipv6']['address'].update({'eui64_old': eui64}) + interface_identifier = leaf_node_changed(config, base + [ifname, 'ipv6', 'address', 'interface-identifier']) + if interface_identifier: + tmp = dict_search('ipv6.address', dict) + if not tmp: + dict.update({'ipv6': {'address': {'interface_identifier_old': interface_identifier}}}) + else: + dict['ipv6']['address'].update({'interface_identifier_old': interface_identifier}) + for vif, vif_config in dict.get('vif', {}).items(): # Add subinterface name to dictionary dict['vif'][vif].update({'ifname' : f'{ifname}.{vif}'}) if config.exists(['qos', 'interface', f'{ifname}.{vif}']): - dict['vif'][vif].update({'traffic_policy': {}}) + dict['vif'][vif].update({'qos': {}}) if 'deleted' not in dict: address = leaf_node_changed(config, base + [ifname, 'vif', vif, 'address']) @@ -549,7 +558,7 @@ def get_interface_dict(config, base, ifname='', recursive_defaults=True, with_pk dict['vif_s'][vif_s].update({'ifname' : f'{ifname}.{vif_s}'}) if config.exists(['qos', 'interface', f'{ifname}.{vif_s}']): - dict['vif_s'][vif_s].update({'traffic_policy': {}}) + dict['vif_s'][vif_s].update({'qos': {}}) if 'deleted' not in dict: address = leaf_node_changed(config, base + [ifname, 'vif-s', vif_s, 'address']) @@ -577,7 +586,7 @@ def get_interface_dict(config, base, ifname='', recursive_defaults=True, with_pk dict['vif_s'][vif_s]['vif_c'][vif_c].update({'ifname' : f'{ifname}.{vif_s}.{vif_c}'}) if config.exists(['qos', 'interface', f'{ifname}.{vif_s}.{vif_c}']): - dict['vif_s'][vif_s]['vif_c'][vif_c].update({'traffic_policy': {}}) + dict['vif_s'][vif_s]['vif_c'][vif_c].update({'qos': {}}) if 'deleted' not in dict: address = leaf_node_changed(config, base + [ifname, 'vif-s', vif_s, 'vif-c', vif_c, 'address']) @@ -604,6 +613,16 @@ def get_interface_dict(config, base, ifname='', recursive_defaults=True, with_pk # Check vif, vif-s/vif-c VLAN interfaces for removal dict = get_removed_vlans(config, base + [ifname], dict) + + # Checks for the presence of static ARP entries on a given interface or VLAN + static_arp = config.get_config_dict( + ['protocols', 'static', 'arp', 'interface'], + key_mangling=('-', '_'), + get_first_key=True, + ) + if any(key == ifname or key.startswith(f'{ifname}.') for key in static_arp.keys()): + dict.update({'static_arp': {}}) + return ifname, dict def get_vlan_ids(interface): @@ -626,16 +645,34 @@ def get_vlan_ids(interface): return vlan_ids +def get_vlans_ids_and_range(interface): + vlan_ids = set() + + vlan_filter_status = json.loads(cmd(f'bridge -j -d vlan show dev {interface}')) + + if vlan_filter_status is not None: + for interface_status in vlan_filter_status: + for vlan_entry in interface_status.get("vlans", []): + start = vlan_entry["vlan"] + end = vlan_entry.get("vlanEnd") + if end: + vlan_ids.add(f"{start}-{end}") + else: + vlan_ids.add(str(start)) + + return vlan_ids + def get_accel_dict(config, base, chap_secrets, with_pki=False): """ Common utility function to retrieve and mangle the Accel-PPP configuration from different CLI input nodes. All Accel-PPP services have a common base - where value retrival is identical. This function must be used whenever + where value retrieval is identical. This function must be used whenever possible when working with Accel-PPP services! Return a dictionary with the necessary interface config keys. """ from vyos.utils.cpu import get_core_count + from vyos.utils.cpu import get_half_cpus from vyos.template import is_ipv4 dict = config.get_config_dict(base, key_mangling=('-', '_'), @@ -645,7 +682,16 @@ def get_accel_dict(config, base, chap_secrets, with_pki=False): with_pki=with_pki) # set CPUs cores to process requests - dict.update({'thread_count' : get_core_count()}) + match dict.get('thread_count'): + case 'all': + dict['thread_count'] = get_core_count() + case 'half': + dict['thread_count'] = get_half_cpus() + case str(x) if x.isdigit(): + dict['thread_count'] = int(x) + case _: + dict['thread_count'] = get_core_count() + # we need to store the path to the secrets file dict.update({'chap_secrets_file' : chap_secrets}) @@ -668,3 +714,18 @@ def get_accel_dict(config, base, chap_secrets, with_pki=False): dict['authentication']['radius']['server'][server]['acct_port'] = '0' return dict + +def get_flowtable_interfaces(config): + """ + Return all interfaces used in flowtables + """ + ft_base = ['firewall', 'flowtable'] + + if not config.exists(ft_base): + return [] + + ifaces = [] + for ft_name in config.list_nodes(ft_base): + ifaces += config.return_values(ft_base + [ft_name, 'interface']) + + return ifaces diff --git a/python/vyos/configdiff.py b/python/vyos/configdiff.py index b6d4a5558..b7c3ef7c6 100644 --- a/python/vyos/configdiff.py +++ b/python/vyos/configdiff.py @@ -1,4 +1,4 @@ -# Copyright 2020-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -263,7 +263,7 @@ class ConfigDiff(object): to provide full dict; for example, Diff.MERGE will expand dict['merge'] into dict under value - no_detaults=False: if expand_nodes & Diff.MERGE, do not merge default + no_defaults=False: if expand_nodes & Diff.MERGE, do not merge default values to ret['merge'] recursive: if true, use config_tree diff algorithm provided by diff_tree class @@ -343,7 +343,7 @@ class ConfigDiff(object): to provide full dict; for example, Diff.MERGE will expand dict['merge'] into dict under value - no_detaults=False: if expand_nodes & Diff.MERGE, do not merge default + no_defaults=False: if expand_nodes & Diff.MERGE, do not merge default values to ret['merge'] recursive: if true, use config_tree diff algorithm provided by diff_tree class diff --git a/python/vyos/configquery.py b/python/vyos/configquery.py index 4c4ead0a3..3f385d97b 100644 --- a/python/vyos/configquery.py +++ b/python/vyos/configquery.py @@ -1,4 +1,4 @@ -# Copyright 2021-2025 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,10 +13,10 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see <http://www.gnu.org/licenses/>. -''' +""" A small library that allows querying existence or value(s) of config settings from op mode, and execution of arbitrary op mode commands. -''' +""" import os import json @@ -37,12 +37,17 @@ from vyos.utils.error import cli_shell_api_err from vyos.xml_ref import multi_to_list from vyos.xml_ref import is_tag from vyos.base import Warning +from vyos.utils.backend import vyconf_backend +from vyos.configsource import ConfigSourceVyconfSession +from vyos.utils.list import list_strip config_file = os.path.join(directories['config'], 'config.boot') + class ConfigQueryError(Exception): pass + class GenericConfigQuery: def __init__(self): pass @@ -56,6 +61,7 @@ class GenericConfigQuery: def values(self, path: list): raise NotImplementedError + class GenericOpRun: def __init__(self): pass @@ -63,6 +69,7 @@ class GenericOpRun: def run(self, path: list, **kwargs): raise NotImplementedError + class CliShellApiConfigQuery(GenericConfigQuery): def __init__(self): super().__init__() @@ -88,22 +95,27 @@ class CliShellApiConfigQuery(GenericConfigQuery): raise ConfigQueryError('No values for given path') return out + class ConfigTreeQuery(GenericConfigQuery): def __init__(self): super().__init__() if boot_configuration_complete(): - config_source = ConfigSourceSession() + if vyconf_backend(): + config_source = ConfigSourceVyconfSession() + else: + config_source = ConfigSourceSession() self.config = Config(config_source=config_source) else: try: with open(config_file) as f: config_string = f.read() - except OSError as err: + except OSError: config_string = '' - config_source = ConfigSourceString(running_config_text=config_string, - session_config_text=config_string) + config_source = ConfigSourceString( + running_config_text=config_string, session_config_text=config_string + ) self.config = Config(config_source=config_source) def exists(self, path: list): @@ -118,16 +130,28 @@ class ConfigTreeQuery(GenericConfigQuery): def list_nodes(self, path: list): return self.config.list_nodes(path) - def get_config_dict(self, path=[], effective=False, key_mangling=None, - get_first_key=False, no_multi_convert=False, - no_tag_node_value_mangle=False, with_defaults=False, - with_recursive_defaults=False): - return self.config.get_config_dict(path, effective=effective, - key_mangling=key_mangling, get_first_key=get_first_key, - no_multi_convert=no_multi_convert, - no_tag_node_value_mangle=no_tag_node_value_mangle, - with_defaults=with_defaults, - with_recursive_defaults=with_recursive_defaults) + def get_config_dict( + self, + path=[], + effective=False, + key_mangling=None, + get_first_key=False, + no_multi_convert=False, + no_tag_node_value_mangle=False, + with_defaults=False, + with_recursive_defaults=False, + ): + return self.config.get_config_dict( + path, + effective=effective, + key_mangling=key_mangling, + get_first_key=get_first_key, + no_multi_convert=no_multi_convert, + no_tag_node_value_mangle=no_tag_node_value_mangle, + with_defaults=with_defaults, + with_recursive_defaults=with_recursive_defaults, + ) + class VbashOpRun(GenericOpRun): def __init__(self): @@ -135,40 +159,60 @@ class VbashOpRun(GenericOpRun): def run(self, path: list, **kwargs): cmd = ' '.join(path) - (out, err) = popen(f'/opt/vyatta/bin/vyatta-op-cmd-wrapper {cmd}', stderr=STDOUT, **kwargs) + (out, err) = popen( + f'/opt/vyatta/bin/vyatta-op-cmd-wrapper {cmd}', stderr=STDOUT, **kwargs + ) if err: raise ConfigQueryError(out) return out -def query_context(config_query_class=CliShellApiConfigQuery, - op_run_class=VbashOpRun): + +def query_context(config_query_class=CliShellApiConfigQuery, op_run_class=VbashOpRun): query = config_query_class() run = op_run_class() return query, run + def verify_mangling(key_mangling): - if not (isinstance(key_mangling, tuple) and - len(key_mangling) == 2 and - isinstance(key_mangling[0], str) and - isinstance(key_mangling[1], str)): - raise ValueError("key_mangling must be a tuple of two strings") + if not ( + isinstance(key_mangling, tuple) + and len(key_mangling) == 2 + and isinstance(key_mangling[0], str) + and isinstance(key_mangling[1], str) + ): + raise ValueError('key_mangling must be a tuple of two strings') + def op_mode_run(cmd): - """ low-level to avoid overhead """ + """low-level to avoid overhead""" p = subprocess.Popen(cmd, stdout=subprocess.PIPE) out = p.stdout.read() p.wait() return p.returncode, out.decode() -def op_mode_config_dict(path=None, key_mangling=None, - no_tag_node_value_mangle=False, - no_multi_convert=False, get_first_key=False): +def op_mode_config_dict( + path=None, + key_mangling=None, + no_tag_node_value_mangle=False, + no_multi_convert=False, + get_first_key=False, +): if path is None: path = [] + command = ['/bin/cli-shell-api', '--show-active-only', 'showConfig'] - rc, out = op_mode_run(command + path) + edit_level = os.environ.get('VYATTA_EDIT_LEVEL', '') + if edit_level: + tmp = edit_level.split('/') + edit_path = [el for el in tmp if el] + relative_path = list_strip(path, edit_path) + else: + relative_path = path + + rc, out = op_mode_run(command + relative_path) + if rc == cli_shell_api_err.VYOS_EMPTY_CONFIG: out = '' if rc == cli_shell_api_err.VYOS_INVALID_PATH: @@ -188,8 +232,11 @@ def op_mode_config_dict(path=None, key_mangling=None, if key_mangling is not None: verify_mangling(key_mangling) - config_dict = mangle_dict_keys(config_dict, - key_mangling[0], key_mangling[1], - no_tag_node_value_mangle=no_tag_node_value_mangle) + config_dict = mangle_dict_keys( + config_dict, + key_mangling[0], + key_mangling[1], + no_tag_node_value_mangle=no_tag_node_value_mangle, + ) return get_sub_dict(config_dict, path, get_first_key=get_first_key) diff --git a/python/vyos/configsession.py b/python/vyos/configsession.py index 90b96b88c..f2abd3a5b 100644 --- a/python/vyos/configsession.py +++ b/python/vyos/configsession.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2024 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or modify it under the terms of # the GNU Lesser General Public License as published by the Free Software Foundation; @@ -16,21 +16,42 @@ import os import re import sys +import json +import weakref import subprocess +from tempfile import NamedTemporaryFile +from typing import TypeAlias +from typing import Union from vyos.defaults import directories from vyos.utils.process import is_systemd_service_running from vyos.utils.dict import dict_to_paths +from vyos.utils.boot import boot_configuration_complete +from vyos.utils.backend import vyconf_backend +from vyos.vyconf_session import VyconfSession +from vyos.base import Warning as Warn +from vyos.defaults import DEFAULT_COMMIT_CONFIRM_MINUTES +from vyos.configtree import ConfigTree +from vyos.configtree import ConfigTreeError +from vyos.configtree import delete_dict_from_masks +from vyos.derivedtree import subtree_from_list_of_partial_paths + +# type of config file path or configtree +ConfigObj: TypeAlias = Union[str, ConfigTree] + CLI_SHELL_API = '/bin/cli-shell-api' SET = '/opt/vyatta/sbin/my_set' DELETE = '/opt/vyatta/sbin/my_delete' COMMENT = '/opt/vyatta/sbin/my_comment' COMMIT = '/opt/vyatta/sbin/my_commit' +COMMIT_CONFIRM = ['/usr/bin/config-mgmt', 'commit_confirm', '-y'] +CONFIRM = ['/usr/bin/config-mgmt', 'confirm'] DISCARD = '/opt/vyatta/sbin/my_discard' SHOW_CONFIG = ['/bin/cli-shell-api', 'showConfig'] LOAD_CONFIG = ['/bin/cli-shell-api', 'loadFile'] MIGRATE_LOAD_CONFIG = ['/usr/libexec/vyos/vyos-load-config.py'] +MERGE_CONFIG = ['/usr/libexec/vyos/vyos-merge-config.py'] SAVE_CONFIG = ['/usr/libexec/vyos/vyos-save-config.py'] INSTALL_IMAGE = [ '/usr/libexec/vyos/op_mode/image_installer.py', @@ -63,6 +84,7 @@ GENERATE = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'generate'] SHOW = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'show'] RESET = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'reset'] REBOOT = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'reboot'] +RENEW = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'renew'] POWEROFF = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'poweroff'] OP_CMD_ADD = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'add'] OP_CMD_DELETE = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'delete'] @@ -116,6 +138,10 @@ def inject_vyos_env(env): env['vyos_sbin_dir'] = '/usr/sbin' env['vyos_validators_dir'] = '/usr/libexec/vyos/validators' + # with the retirement of the Cstore backend, this will remain as the + # sole indication of legacy CLI config mode, as checked by VyconfSession + env['_OFR_CONFIGURE'] = 'ok' + # if running the vyos-configd daemon, inject the vyshim env var if is_systemd_service_running('vyos-configd.service'): env['vyshim'] = '/usr/sbin/vyshim' @@ -132,7 +158,7 @@ class ConfigSession(object): The write API of VyOS. """ - def __init__(self, session_id, app=APP): + def __init__(self, session_id, app=APP, shared=False): """ Creates a new config session. @@ -160,32 +186,61 @@ class ConfigSession(object): for k, v in env_list: session_env[k] = v + # replaces ambient instance of SESSION_PID, + # for use when running from a non-shared configsession + session_env['SESSION_PID'] = str(session_id) + self.__session_env = session_env self.__session_env['COMMIT_VIA'] = app self.__run_command([CLI_SHELL_API, 'setupSession']) + if vyconf_backend() and boot_configuration_complete(): + self._vyconf_session = VyconfSession( + pid=session_id, on_error=ConfigSessionError + ) + else: + self._vyconf_session = None + + self.shared = shared + + if not self.shared and self._vyconf_session: + self._finalizer = weakref.finalize( + self, self.finalize_vyconf, self._vyconf_session + ) + def __del__(self): - try: - output = ( - subprocess.check_output( - [CLI_SHELL_API, 'teardownSession'], env=self.__session_env + if self.shared: + return + if not vyconf_backend(): + try: + output = ( + subprocess.check_output( + [CLI_SHELL_API, 'teardownSession'], env=self.__session_env + ) + .decode() + .strip() ) - .decode() - .strip() - ) - if output: + if output: + print( + 'cli-shell-api teardownSession output for session {0}: {1}'.format( + self.__session_id, output + ), + file=sys.stderr, + ) + except Exception as e: print( - 'cli-shell-api teardownSession output for sesion {0}: {1}'.format( - self.__session_id, output - ), + 'Could not tear down session {0}: {1}'.format(self.__session_id, e), file=sys.stderr, ) - except Exception as e: - print( - 'Could not tear down session {0}: {1}'.format(self.__session_id, e), - file=sys.stderr, - ) + + @classmethod + def finalize_vyconf(cls, session: VyconfSession): + if session.session_changed(): + Warn('Exiting with uncommitted changes') + session.discard() + session.exit_config_mode() + session.teardown() def __run_command(self, cmd_list): p = subprocess.Popen( @@ -204,12 +259,18 @@ class ConfigSession(object): def get_session_env(self): return self.__session_env + def vyconf_backend(self) -> bool: + return bool(self._vyconf_session) + def set(self, path, value=None): if not value: value = [] else: value = [value] - self.__run_command([SET] + path + value) + if self._vyconf_session is None: + self.__run_command([SET] + path + value) + else: + self._vyconf_session.set(path + value) def set_section(self, path: list, d: dict): try: @@ -223,7 +284,10 @@ class ConfigSession(object): value = [] else: value = [value] - self.__run_command([DELETE] + path + value) + if self._vyconf_session is None: + self.__run_command([DELETE] + path + value) + else: + self._vyconf_session.delete(path + value) def load_section(self, path: list, d: dict): try: @@ -242,15 +306,33 @@ class ConfigSession(object): except (ValueError, ConfigSessionError) as e: raise ConfigSessionError(e) - def load_section_tree(self, mask: dict, d: dict): + def load_section_tree( + self, config_tree: ConfigTree, mask_dict: dict, config_dict: dict + ): + if ( + not mask_dict + or 'inclusive' not in mask_dict + or 'exclusive' not in mask_dict + ): + raise ConfigSessionError( + "Missing mask data can damage the config: expected keys 'inclusive' and 'exclusive'" + ) try: - if mask: - for p in dict_to_paths(mask): + mask_in = ConfigTree(internal_string=mask_dict['inclusive']) + + mask_ex_list = json.loads(mask_dict['exclusive']) + mask_ex = subtree_from_list_of_partial_paths(config_tree, mask_ex_list) + + delete_dict = delete_dict_from_masks(config_tree, mask_in, mask_ex) + + if delete_dict: + for p in dict_to_paths(delete_dict): self.delete(p) - if d: - for p in dict_to_paths(d): + + if config_dict: + for p in dict_to_paths(config_dict): self.set(p) - except (ValueError, ConfigSessionError) as e: + except (ValueError, ConfigSessionError, ConfigTreeError) as e: raise ConfigSessionError(e) def comment(self, path, value=None): @@ -261,20 +343,46 @@ class ConfigSession(object): self.__run_command([COMMENT] + path + value) def commit(self): - out = self.__run_command([COMMIT]) + if self._vyconf_session is None: + out = self.__run_command([COMMIT]) + else: + out, _ = self._vyconf_session.commit() + + return out + + def commit_confirm(self, minutes: int = DEFAULT_COMMIT_CONFIRM_MINUTES): + out = self.__run_command(COMMIT_CONFIRM + [f'-t {minutes}']) + + return out + + def confirm(self): + out = self.__run_command(CONFIRM) + return out def discard(self): - self.__run_command([DISCARD]) + if self._vyconf_session is None: + self.__run_command([DISCARD]) + else: + out, _ = self._vyconf_session.discard() def show_config(self, path, format='raw'): - config_data = self.__run_command(SHOW_CONFIG + path) + if self._vyconf_session is None: + config_data = self.__run_command(SHOW_CONFIG + path) + else: + config_data, _ = self._vyconf_session.show_config(path) if format == 'raw': return config_data - def load_config(self, file_path): - out = self.__run_command(LOAD_CONFIG + [file_path]) + def load_config(self, file_path, cached: bool = False): + if self._vyconf_session is None: + out = self.__run_command(LOAD_CONFIG + [file_path]) + else: + out, _ = self._vyconf_session.load_config( + file_name=file_path, cached=cached + ) + return out def load_explicit(self, file_path): @@ -286,8 +394,31 @@ class ConfigSession(object): except LoadConfigError as e: raise ConfigSessionError(e) from e + def load_config_obj(self, config_obj: ConfigObj): + if isinstance(config_obj, ConfigTree): + with NamedTemporaryFile() as f: + config_obj.write_cache(f.name) + self.load_config(f.name, cached=True) + else: + self.load_config(config_obj) + def migrate_and_load_config(self, file_path): - out = self.__run_command(MIGRATE_LOAD_CONFIG + [file_path]) + if self._vyconf_session is None: + out = self.__run_command(MIGRATE_LOAD_CONFIG + [file_path]) + else: + out, _ = self._vyconf_session.load_config(file_name=file_path, migrate=True) + + return out + + def merge_config(self, file_path, destructive=False): + if self._vyconf_session is None: + destr = ['--destructive'] if destructive else [] + out = self.__run_command(MERGE_CONFIG + [file_path] + destr) + else: + out, _ = self._vyconf_session.merge_config( + file_name=file_path, destructive=destructive + ) + return out def save_config(self, file_path): @@ -330,6 +461,10 @@ class ConfigSession(object): out = self.__run_command(RESET + path) return out + def renew(self, path): + out = self.__run_command(RENEW + path) + return out + def poweroff(self, path): out = self.__run_command(POWEROFF + path) return out diff --git a/python/vyos/configsource.py b/python/vyos/configsource.py index 65cef5333..b54f6283a 100644 --- a/python/vyos/configsource.py +++ b/python/vyos/configsource.py @@ -1,5 +1,5 @@ -# Copyright 2020-2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -17,9 +17,16 @@ import os import re import subprocess +from typing import Union from vyos.configtree import ConfigTree from vyos.utils.boot import boot_configuration_complete +from vyos.vyconf_session import VyconfSession +from vyos.vyconf_session import VyconfSessionError +from vyos.defaults import directories +from vyos.xml_ref import is_tag +from vyos.xml_ref import is_leaf +from vyos.xml_ref import is_multi class VyOSError(Exception): """ @@ -44,7 +51,7 @@ class ConfigSource: def session_changed(self): """ Returns: - True if the config session has uncommited changes, False otherwise. + True if the config session has uncommitted changes, False otherwise. """ raise NotImplementedError(f"function not available for {type(self)}") @@ -191,7 +198,7 @@ class ConfigSourceSession(ConfigSource): def session_changed(self): """ Returns: - True if the config session has uncommited changes, False otherwise. + True if the config session has uncommitted changes, False otherwise. """ try: self._run(self._make_command('sessionChanged', '')) @@ -238,7 +245,7 @@ class ConfigSourceSession(ConfigSource): # FIXUP: by default, showConfig will give you a diff # if there are uncommitted changes. # The config parser obviously cannot work with diffs, - # so we need to supress diff production using appropriate + # so we need to suppress diff production using appropriate # options for getting either running (active) # or proposed (working) config. if effective: @@ -310,6 +317,110 @@ class ConfigSourceSession(ConfigSource): except VyOSError: return False +class ConfigSourceVyconfSession(ConfigSource): + def __init__(self, session_env=None): + super().__init__() + + if session_env: + self.__session_env = session_env + else: + self.__session_env = None + + if session_env and 'SESSION_PID' in session_env: + self.pid = int(session_env['SESSION_PID']) + else: + pid = os.environ.get('SESSION_PID', '') + self.pid = int(pid) if pid else os.getppid() + + self._vyconf_session = VyconfSession(pid=self.pid) + try: + out = self._vyconf_session.get_config() + except VyconfSessionError as e: + raise ConfigSourceError(f'Init error in {type(self)}: {e}') + + session_dir = directories['vyconf_session_dir'] + + self.running_cache_path = os.path.join(session_dir, f'running_cache_{out}') + self.session_cache_path = os.path.join(session_dir, f'session_cache_{out}') + + self._running_config = ConfigTree(internal=self.running_cache_path) + self._session_config = ConfigTree(internal=self.session_cache_path) + + if os.path.isfile(self.running_cache_path): + os.remove(self.running_cache_path) + if os.path.isfile(self.session_cache_path): + os.remove(self.session_cache_path) + + # N.B. level not yet implemented pending integration with legacy CLI + # cf. T7374 + self._level = [] + + def get_level(self): + return self._level + + def set_level(self): + pass + + def session_changed(self): + """ + Returns: + True if the config session has uncommitted changes, False otherwise. + """ + try: + return self._vyconf_session.session_changed() + except VyconfSessionError: + # no actionable session info on error + return False + + def in_session(self): + """ + Returns: + True if called from a configuration session, False otherwise. + """ + return self._vyconf_session.in_session() + + def show_config(self, path: Union[str,list] = None, default: str = None, + effective: bool = False): + """ + Args: + path (str|list): Configuration tree path, or empty + default (str): Default value to return + + Returns: + str: working configuration + """ + + if path is None: + path = [] + if isinstance(path, str): + path = path.split() + + ct = self._running_config if effective else self._session_config + with_node = True if self.is_tag(path) else False + ct_at_path = ct.get_subtree(path, with_node=with_node) if path else ct + + res = ct_at_path.to_string().strip() + + return res if res else default + + def is_tag(self, path): + try: + return is_tag(path) + except ValueError: + return False + + def is_leaf(self, path): + try: + return is_leaf(path) + except ValueError: + return False + + def is_multi(self, path): + try: + return is_multi(path) + except ValueError: + return False + class ConfigSourceString(ConfigSource): def __init__(self, running_config_text=None, session_config_text=None): super().__init__() diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index dade852c7..47e56bc46 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -1,5 +1,5 @@ # configtree -- a standalone VyOS config file manipulation library (Python bindings) -# Copyright (C) 2018-2025 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or modify it under the terms of # the GNU Lesser General Public License as published by the Free Software Foundation; @@ -18,6 +18,12 @@ import json import logging from ctypes import cdll, c_char_p, c_void_p, c_int, c_bool +from typing import TYPE_CHECKING + +# https://peps.python.org/pep-0484/#forward-references +# for type 'ConfigDict' +if TYPE_CHECKING: + from vyos.referencetree import ReferenceTree BUILD_PATH = '/tmp/libvyosconfig/_build/libvyosconfig.so' INSTALL_PATH = '/usr/lib/libvyosconfig.so.0' @@ -67,11 +73,21 @@ class ConfigTreeError(Exception): class ConfigTree(object): def __init__( - self, config_string=None, address=None, internal=None, libpath=LIBPATH + self, + config_string=None, + address=None, + internal=None, + internal_string=None, + libpath=LIBPATH, ): - if config_string is None and address is None and internal is None: + if ( + config_string is None + and address is None + and internal is None + and internal_string is None + ): raise TypeError( - "ConfigTree() requires one of 'config_string', 'address', or 'internal'" + "ConfigTree() requires one of 'config_string', 'address', 'internal', or 'internal_string'" ) self.__config = None @@ -101,6 +117,14 @@ class ConfigTree(object): self.__write_internal = self.__lib.write_internal self.__write_internal.argtypes = [c_void_p, c_char_p] + self.__read_internal_string = self.__lib.read_internal_string + self.__read_internal_string.argtypes = [c_char_p] + self.__read_internal_string.restype = c_void_p + + self.__write_internal_string = self.__lib.write_internal_string + self.__write_internal_string.argtypes = [c_void_p] + self.__write_internal_string.restype = c_char_p + self.__to_json = self.__lib.to_json self.__to_json.argtypes = [c_void_p] self.__to_json.restype = c_char_p @@ -145,6 +169,10 @@ class ConfigTree(object): self.__exists.argtypes = [c_void_p, c_char_p] self.__exists.restype = c_int + self.__value_exists = self.__lib.value_exists + self.__value_exists.argtypes = [c_void_p, c_char_p, c_char_p] + self.__value_exists.restype = c_int + self.__list_nodes = self.__lib.list_nodes self.__list_nodes.argtypes = [c_void_p, c_char_p] self.__list_nodes.restype = c_char_p @@ -184,6 +212,17 @@ class ConfigTree(object): self.__equal.argtypes = [c_void_p, c_void_p] self.__equal.restype = c_bool + self.__config_dict = self.__lib.config_dict + self.__config_dict.argtypes = [ + c_void_p, + c_void_p, + c_void_p, + c_char_p, + c_bool, + c_bool, + ] + self.__config_dict.restype = c_char_p + if address is not None: self.__config = address self.__version = '' @@ -191,16 +230,29 @@ class ConfigTree(object): config = self.__read_internal(internal.encode()) if config is None: msg = self.__get_error().decode() - raise ValueError('Failed to read internal rep: {0}'.format(msg)) + raise ValueError( + f'Failed to read internal representation from file {internal}: {msg}' + ) else: self.__config = config + self.__version = '' + elif internal_string is not None: + config = self.__read_internal_string(internal_string.encode()) + if config is None: + msg = self.__get_error().decode() + raise ValueError( + f'Failed to read internal representation from string: {msg}' + ) + else: + self.__config = config + self.__version = '' elif config_string is not None: config_section, version_section = extract_version(config_string) config_section = escape_backslash(config_section) config = self.__from_string(config_section.encode()) if config is None: msg = self.__get_error().decode() - raise ValueError('Failed to parse config: {0}'.format(msg)) + raise ValueError(f'Failed to parse config: {msg}') else: self.__config = config self.__version = version_section @@ -219,20 +271,24 @@ class ConfigTree(object): def __eq__(self, other): if isinstance(other, ConfigTree): - return self.__equal(self._get_config(), other._get_config()) + return self.__equal(self.get_tree(), other.get_tree()) return False def __str__(self): return self.to_string() - def _get_config(self): + def get_tree(self): return self.__config def get_version_string(self): return self.__version def write_cache(self, file_name): - self.__write_internal(self._get_config(), file_name) + self.__write_internal(self.get_tree(), file_name.encode()) + + def write_internal_string(self) -> str: + res = self.__write_internal_string(self.get_tree()) + return res.decode() def to_string(self, ordered_values=False, no_version=False): config_string = self.__to_string(self.__config, ordered_values).decode() @@ -259,14 +315,15 @@ class ConfigTree(object): res = self.__create_node(self.__config, path_str) if res != 0: - raise ConfigTreeError(f'Path already exists: {path}') + msg = self.__get_error().decode() + raise ConfigTreeError(f'{msg}: {path}') def set(self, path, value=None, replace=True): """Set new entry in VyOS configuration. path: configuration path e.g. 'system dns forwarding listen-address' value: value to be added to node, e.g. '172.18.254.201' - replace: True: current occurance will be replaced - False: new value will be appended to current occurances - use + replace: True: current occurrence will be replaced + False: new value will be appended to current occurrences - use this for adding values to a multi node """ @@ -274,12 +331,20 @@ class ConfigTree(object): path_str = ' '.join(map(str, path)).encode() if value is None: - self.__set_valueless(self.__config, path_str) + res = self.__set_valueless(self.__config, path_str) else: if replace: - self.__set_replace_value(self.__config, path_str, str(value).encode()) + res = self.__set_replace_value( + self.__config, path_str, str(value).encode() + ) else: - self.__set_add_value(self.__config, path_str, str(value).encode()) + res = self.__set_add_value(self.__config, path_str, str(value).encode()) + + if res != 0: + msg = self.__get_error().decode() + raise ConfigTreeError( + f'{msg}: path "{path}" value "{value}" replace "{replace}"' + ) if self.__migration: self.migration_log.info( @@ -292,7 +357,8 @@ class ConfigTree(object): res = self.__delete(self.__config, path_str) if res != 0: - raise ConfigTreeError(f"Path doesn't exist: {path}") + msg = self.__get_error().decode() + raise ConfigTreeError(f'{msg}: path "{path}"') if self.__migration: self.migration_log.info(f'- op: delete path: {path}') @@ -303,12 +369,8 @@ class ConfigTree(object): res = self.__delete_value(self.__config, path_str, value.encode()) if res != 0: - if res == 1: - raise ConfigTreeError(f"Path doesn't exist: {path}") - elif res == 2: - raise ConfigTreeError(f"Value doesn't exist: '{value}'") - else: - raise ConfigTreeError() + msg = self.__get_error().decode() + raise ConfigTreeError(f'{msg}: path "{path}" value "{value}"') if self.__migration: self.migration_log.info(f'- op: delete_value path: {path} value: {value}') @@ -321,10 +383,12 @@ class ConfigTree(object): # Check if a node with intended new name already exists new_path = path[:-1] + [new_name] if self.exists(new_path): - raise ConfigTreeError() + raise ConfigTreeError(f'Name {new_name} already exists') + res = self.__rename(self.__config, path_str, newname_str) if res != 0: - raise ConfigTreeError("Path [{}] doesn't exist".format(path)) + msg = self.__get_error().decode() + raise ConfigTreeError(f'{msg}: {path}') if self.__migration: self.migration_log.info( @@ -360,6 +424,16 @@ class ConfigTree(object): else: return True + def value_exists(self, path, value): + check_path(path) + path_str = ' '.join(map(str, path)).encode() + + res = self.__value_exists(self.__config, path_str, value.encode()) + if res == 0: + return False + else: + return True + def list_nodes(self, path, path_must_exist=True): check_path(path) path_str = ' '.join(map(str, path)).encode() @@ -417,13 +491,18 @@ class ConfigTree(object): if res == 0: return True else: - raise ConfigTreeError("Path [{}] doesn't exist".format(path_str)) + msg = self.__get_error().decode() + raise ConfigTreeError(f'{msg}: {path}') def is_leaf(self, path): check_path(path) path_str = ' '.join(map(str, path)).encode() - return self.__is_leaf(self.__config, path_str) + res = self.__is_leaf(self.__config, path_str) + if res >= 1: + return True + else: + return False def set_leaf(self, path, value): check_path(path) @@ -433,7 +512,8 @@ class ConfigTree(object): if res == 0: return True else: - raise ConfigTreeError("Path [{}] doesn't exist".format(path_str)) + msg = self.__get_error().decode() + raise ConfigTreeError(f'{msg}: {path}') def get_subtree(self, path, with_node=False): check_path(path) @@ -443,8 +523,25 @@ class ConfigTree(object): subt = ConfigTree(address=res) return subt + def config_dict( + self, ref_tree, path, mask, get_first_key=False, with_defaults=False + ): + check_path(path) + path_str = ' '.join(map(str, path)).encode() -def show_diff(left, right, path=[], commands=False, libpath=LIBPATH): + res_json = self.__config_dict( + self.__config, + ref_tree.get_tree(), + mask.get_tree(), + path_str, + get_first_key, + with_defaults, + ).decode() + res = json.loads(res_json) + return res + + +def diff_compare(left, right, path=[], commands=False, libpath=LIBPATH): if left is None: left = ConfigTree(config_string='\n') if right is None: @@ -459,14 +556,14 @@ def show_diff(left, right, path=[], commands=False, libpath=LIBPATH): path_str = ' '.join(map(str, path)).encode() __lib = cdll.LoadLibrary(libpath) - __show_diff = __lib.show_diff - __show_diff.argtypes = [c_bool, c_char_p, c_void_p, c_void_p] - __show_diff.restype = c_char_p + __diff_compare = __lib.diff_compare + __diff_compare.argtypes = [c_bool, c_char_p, c_void_p, c_void_p] + __diff_compare.restype = c_char_p __get_error = __lib.get_error __get_error.argtypes = [] __get_error.restype = c_char_p - res = __show_diff(commands, path_str, left._get_config(), right._get_config()) + res = __diff_compare(commands, path_str, left.get_tree(), right.get_tree()) res = res.decode() if res == '#1@': msg = __get_error().decode() @@ -492,7 +589,29 @@ def union(left, right, libpath=LIBPATH): __get_error.argtypes = [] __get_error.restype = c_char_p - res = __tree_union(left._get_config(), right._get_config()) + res = __tree_union(left.get_tree(), right.get_tree()) + tree = ConfigTree(address=res) + + return tree + + +def merge(left, right, destructive=False, libpath=LIBPATH): + if left is None: + left = ConfigTree(config_string='\n') + if right is None: + right = ConfigTree(config_string='\n') + if not (isinstance(left, ConfigTree) and isinstance(right, ConfigTree)): + raise TypeError('Arguments must be instances of ConfigTree') + + __lib = cdll.LoadLibrary(libpath) + __tree_merge = __lib.tree_merge + __tree_merge.argtypes = [c_bool, c_void_p, c_void_p] + __tree_merge.restype = c_void_p + __get_error = __lib.get_error + __get_error.argtypes = [] + __get_error.restype = c_char_p + + res = __tree_merge(destructive, left.get_tree(), right.get_tree()) tree = ConfigTree(address=res) return tree @@ -505,13 +624,13 @@ def mask_inclusive(left, right, libpath=LIBPATH): try: __lib = cdll.LoadLibrary(libpath) __mask_tree = __lib.mask_tree - __mask_tree.argtypes = [c_void_p, c_void_p] + __mask_tree.argtypes = [c_void_p, c_void_p, c_bool] __mask_tree.restype = c_void_p __get_error = __lib.get_error __get_error.argtypes = [] __get_error.restype = c_char_p - res = __mask_tree(left._get_config(), right._get_config()) + res = __mask_tree(left.get_tree(), right.get_tree(), False) except Exception as e: raise ConfigTreeError(e) if not res: @@ -523,33 +642,93 @@ def mask_inclusive(left, right, libpath=LIBPATH): return tree -def show_commit_data(active_tree, proposed_tree, libpath=LIBPATH): - if not ( - isinstance(active_tree, ConfigTree) and isinstance(proposed_tree, ConfigTree) - ): +def mask_exclusive(left, right, libpath=LIBPATH): + if not (isinstance(left, ConfigTree) and isinstance(right, ConfigTree)): raise TypeError('Arguments must be instances of ConfigTree') - __lib = cdll.LoadLibrary(libpath) - __show_commit_data = __lib.show_commit_data - __show_commit_data.argtypes = [c_void_p, c_void_p] - __show_commit_data.restype = c_char_p + try: + __lib = cdll.LoadLibrary(libpath) + __mask_tree = __lib.mask_tree + __mask_tree.argtypes = [c_void_p, c_void_p, c_bool] + __mask_tree.restype = c_void_p + __get_error = __lib.get_error + __get_error.argtypes = [] + __get_error.restype = c_char_p - res = __show_commit_data(active_tree._get_config(), proposed_tree._get_config()) + res = __mask_tree(left.get_tree(), right.get_tree(), True) + except Exception as e: + raise ConfigTreeError(e) + if not res: + msg = __get_error().decode() + raise ConfigTreeError(msg) - return res.decode() + tree = ConfigTree(address=res) + return tree -def test_commit(active_tree, proposed_tree, libpath=LIBPATH): - if not ( - isinstance(active_tree, ConfigTree) and isinstance(proposed_tree, ConfigTree) - ): - raise TypeError('Arguments must be instances of ConfigTree') - __lib = cdll.LoadLibrary(libpath) - __test_commit = __lib.test_commit - __test_commit.argtypes = [c_void_p, c_void_p] +def delete_tree_from_masks( + config_tree: ConfigTree, include_mask: ConfigTree, exclude_mask: ConfigTree +): + masked_inc = mask_inclusive(config_tree, include_mask) + # Here we want the reversed stand-alone exclusion/inclusion. + # This simplifies definition of delete paths as (delete) + # difference between the two trees of config data. + masked_upper_bound = mask_exclusive(config_tree, include_mask) + masked_lower_bound = mask_inclusive(config_tree, exclude_mask) + masked_exc = union(masked_upper_bound, masked_lower_bound) + + ret = DiffTree(masked_inc, masked_exc) + return ret.delete + + +def delete_dict_from_masks( + config_tree: ConfigTree, include_mask: ConfigTree, exclude_mask: ConfigTree +): + ret = delete_tree_from_masks(config_tree, include_mask, exclude_mask) + return json.loads(ret.to_json()) + + +def subtree_from_partial( + config_tree: ConfigTree, + path: list[str], + reference_tree: 'ReferenceTree', + start: ConfigTree = None, + libpath=LIBPATH, +): + if start: + if not isinstance(start, ConfigTree): + raise TypeError("Argument 'start' must be an instance of ConfigTree") + else: + start = ConfigTree('') + + check_path(path) + path_str = ' '.join(map(str, path)).encode() + + try: + __lib = cdll.LoadLibrary(libpath) + __subtree_from_partial = __lib.subtree_from_partial + __subtree_from_partial.argtypes = [c_void_p, c_void_p, c_void_p, c_char_p] + __subtree_from_partial.restype = c_void_p + __get_error = __lib.get_error + __get_error.argtypes = [] + __get_error.restype = c_char_p - __test_commit(active_tree._get_config(), proposed_tree._get_config()) + res = __subtree_from_partial( + reference_tree.get_tree(), + config_tree.get_tree(), + start.get_tree(), + path_str, + ) + except Exception as e: + raise ConfigTreeError(e) + if not res: + msg = __get_error().decode() + raise ConfigTreeError(msg) + + tree = ConfigTree(address=res) + + return tree def reference_tree_to_json(from_dir, to_file, internal_cache='', libpath=LIBPATH): @@ -620,6 +799,44 @@ def reference_tree_cache_to_json(cache_path, render_file, libpath=LIBPATH): raise ConfigTreeError(msg) +# validate_tree_filter c_ptr rt_cache validator_dir +def validate_tree_filter( + config_tree, + cache_path='/usr/share/vyos/reftree.cache', + validator_dir='/usr/libexec/vyos/validators', + libpath=LIBPATH, +): + try: + __lib = cdll.LoadLibrary(libpath) + __validate_tree_filter = __lib.validate_tree_filter + __validate_tree_filter.argtypes = [c_void_p, c_char_p, c_char_p] + __get_error = __lib.get_error + __get_error.argtypes = [] + __get_error.restype = c_char_p + res = __validate_tree_filter( + config_tree.get_tree(), cache_path.encode(), validator_dir.encode() + ) + except Exception as e: + raise ConfigTreeError(e) + + msg = __get_error().decode() + tree = ConfigTree(address=res) + + return tree, msg + + +def validate_tree( + config_tree, + cache_path='/usr/share/vyos/reftree.cache', + validator_dir='/usr/libexec/vyos/validators', +): + _, out = validate_tree_filter( + config_tree, cache_path=cache_path, validator_dir=validator_dir + ) + + return out + + class DiffTree: def __init__(self, left, right, path=[], libpath=LIBPATH): if left is None: @@ -646,7 +863,7 @@ class DiffTree: check_path(path) path_str = ' '.join(map(str, path)).encode() - res = self.__diff_tree(path_str, left._get_config(), right._get_config()) + res = self.__diff_tree(path_str, left.get_tree(), right.get_tree()) # full diff config_tree and python dict representation self.full = ConfigTree(address=res) diff --git a/python/vyos/configverify.py b/python/vyos/configverify.py index 4084425b1..00cde7cdf 100644 --- a/python/vyos/configverify.py +++ b/python/vyos/configverify.py @@ -1,4 +1,4 @@ -# Copyright 2020-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -22,14 +22,18 @@ # makes use of it! from vyos import ConfigError +from vyos.base import Warning from vyos.utils.dict import dict_search +from vyos.utils.dict import dict_search_recursive +from vyos.utils.network import interface_exists + # pattern re-used in ipsec migration script dynamic_interface_pattern = r'(ppp|pppoe|sstpc|l2tp|ipoe)[0-9]+' def verify_mtu(config): """ Common helper function used by interface implementations to perform - recurring validation if the specified MTU can be used by the underlaying + recurring validation if the specified MTU can be used by the underlying hardware. """ from vyos.ifconfig import Interface @@ -92,12 +96,14 @@ def verify_mtu_ipv6(config): tmp = dict_search('ipv6.address.eui64', config) if tmp != None: raise ConfigError(error_msg) + tmp = dict_search('ipv6.address.interface_identifier', config) + if tmp != None: raise ConfigError(error_msg) + def verify_vrf(config): """ Common helper function used by interface implementations to perform recurring validation of VRF configuration. """ - from vyos.utils.network import interface_exists if 'vrf' in config: vrfs = config['vrf'] if isinstance(vrfs, str): @@ -174,10 +180,14 @@ def verify_mirror_redirect(config): It makes no sense to mirror traffic back at yourself! """ - from vyos.utils.network import interface_exists - if {'mirror', 'redirect'} <= set(config): + if 'mirror' in config and 'redirect' in config: raise ConfigError('Mirror and redirect can not be enabled at the same time!') + if 'mirror' in config and 'qos' in config: + # XXX: support combination of limiting and mirror - this is an artificial + # limitation from the past + raise ConfigError('Can not use QoS together with mirror!') + if 'mirror' in config: for direction, mirror_interface in config['mirror'].items(): if not interface_exists(mirror_interface): @@ -194,11 +204,6 @@ def verify_mirror_redirect(config): raise ConfigError(f'Requested redirect interface "{redirect_ifname}" '\ 'does not exist!') - if ('mirror' in config or 'redirect' in config) and dict_search('traffic_policy.in', config) is not None: - # XXX: support combination of limiting and redirect/mirror - this is an - # artificial limitation - raise ConfigError('Can not use ingress policy together with mirror or redirect!') - def verify_authentication(config): """ Common helper function used by interface implementations to perform @@ -244,10 +249,6 @@ def verify_interface_exists(config, ifname, state_required=False, warning_only=F if the interface is defined on the CLI, if it's not found we try if it exists at the OS level. """ - from vyos.base import Warning - from vyos.utils.dict import dict_search_recursive - from vyos.utils.network import interface_exists - if not state_required: # Check if interface is present in CLI config tmp = getattr(config, 'interfaces_root', {}) @@ -264,6 +265,47 @@ def verify_interface_exists(config, ifname, state_required=False, warning_only=F return False raise ConfigError(message) +def verify_virtual_interface_exists( + config, ifname, state_required=False, warning_only=False +): + """ + Verify the existence of a virtual network interface in the configuration or the Linux kernel. + + This function checks whether a specified virtual interface exists in the provided configuration + or in the Linux kernel. It can return a warning or raise an error based on the parameters provided. + """ + physical_ifname, vif_id = ifname.split('.', maxsplit=1) + + if vif_id and '.' in vif_id: + vif_s, vif_c = vif_id.split('.', maxsplit=1) + vif_id = None + else: + vif_s = vif_c = None + + if not state_required: + # Check if sub-interface is present in CLI config + interfaces_root = getattr(config, 'interfaces_root', {}) + + if vif_s and vif_c: + path = ['ethernet', physical_ifname, 'vif-s', vif_s, 'vif-c'] + key = vif_c + else: + path = ['ethernet', physical_ifname, 'vif'] + key = vif_id + + if bool(list(dict_search_recursive(interfaces_root, key, path=path))): + return True + + # Interface not found on CLI, try Linux Kernel + if interface_exists(ifname): + return True + + message = f'Virtual Interface "{ifname}" does not exist!' + if warning_only: + Warning(message) + return False + raise ConfigError(message) + def verify_source_interface(config): """ Common helper function used by interface implementations to @@ -271,7 +313,6 @@ def verify_source_interface(config): required by e.g. peth/MACvlan, MACsec ... """ import re - from vyos.utils.network import interface_exists ifname = config['ifname'] if 'source_interface' not in config: @@ -356,6 +397,7 @@ def verify_vlan_config(config): verify_vrf(vlan) verify_mirror_redirect(vlan) verify_mtu_parent(vlan, config) + verify_mtu_ipv6(vlan) # 802.1ad (Q-in-Q) VLANs for s_vlan_id in config.get('vif_s', {}): @@ -367,6 +409,7 @@ def verify_vlan_config(config): verify_vrf(s_vlan) verify_mirror_redirect(s_vlan) verify_mtu_parent(s_vlan, config) + verify_mtu_ipv6(s_vlan) for c_vlan_id in s_vlan.get('vif_c', {}): c_vlan = s_vlan['vif_c'][c_vlan_id] @@ -378,6 +421,7 @@ def verify_vlan_config(config): verify_mirror_redirect(c_vlan) verify_mtu_parent(c_vlan, config) verify_mtu_parent(c_vlan, s_vlan) + verify_mtu_ipv6(c_vlan) def verify_diffie_hellman_length(file, min_keysize): @@ -412,7 +456,7 @@ def verify_common_route_maps(config): # XXX: This function is called in combination with a previous call to: # tmp = conf.get_config_dict(['policy']) - see protocols_ospf.py as example. # We should NOT call this with the key_mangling option as this would rename - # route-map hypens '-' to underscores '_' and one could no longer distinguish + # route-map hyphens '-' to underscores '_' and one could no longer distinguish # what should have been the "proper" route-map name, as foo-bar and foo_bar # are two entire different route-map instances! for route_map in ['route-map', 'route_map']: @@ -495,7 +539,7 @@ def verify_pki_ca_certificate(config: dict, ca_name: str): pki_cert = config['pki']['ca'][ca_name] if 'certificate' not in pki_cert: - raise ConfigError(f'PEM CA certificate for "{cert_name}" missing in configuration!') + raise ConfigError(f'PEM CA certificate for "{ca_name}" missing in configuration!') def verify_pki_dh_parameters(config: dict, dh_name: str, min_key_size: int=0): """ @@ -521,6 +565,25 @@ def verify_pki_dh_parameters(config: dict, dh_name: str, min_key_size: int=0): if dh_bits < min_key_size: raise ConfigError(f'Minimum DH key-size is {min_key_size} bits!') +def verify_pki_openssh_key(config: dict, key_name: str): + """ + Common helper function user by PKI consumers to perform recurring + validation functions on OpenSSH keys + """ + if 'pki' not in config: + raise ConfigError('PKI is not configured!') + + if 'openssh' not in config['pki']: + raise ConfigError('PKI does not contain any OpenSSH keys!') + + if key_name not in config['pki']['openssh']: + raise ConfigError(f'OpenSSH key "{key_name}" not found in configuration!') + + if 'public' in config['pki']['openssh'][key_name]: + if not {'key', 'type'} <= set(config['pki']['openssh'][key_name]['public']): + raise ConfigError('Both public key and type must be defined for '\ + f'OpenSSH public key "{key_name}"!') + def verify_eapol(config: dict): """ Common helper function used by interface implementations to perform diff --git a/python/vyos/container.py b/python/vyos/container.py new file mode 100644 index 000000000..475d796f2 --- /dev/null +++ b/python/vyos/container.py @@ -0,0 +1,41 @@ +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from vyos.config import Config +from vyos.ifconfig import Interface +from vyos.utils.dict import dict_search +from vyos.utils.network import interface_exists + +def restart_network(config: Config) -> None: + """ + Start network and assign it to given VRF if requested. + + This can only be done after the containers got started as the podman network + interface will only be enabled by the first container and yet I do not know + how to enable the network interface in advance. + """ + if 'network' in config: + for network, network_config in config['network'].items(): + type_config = dict_search('type', network_config) + if not dict_search('macvlan', type_config): + network_name = f'pod-{network}' + # T5147: Networks are started only as soon as there is a consumer. + # If only a network is created in the first place, no need to assign + # it to a VRF as there's no consumer, yet. + if interface_exists(network_name): + tmp = Interface(network_name) + tmp.set_vrf(network_config.get('vrf', '')) + tmp.add_ipv6_eui64_address('fe80::/64') + + return None diff --git a/python/vyos/debug.py b/python/vyos/debug.py index 6ce42b173..3b71693ab 100644 --- a/python/vyos/debug.py +++ b/python/vyos/debug.py @@ -1,4 +1,4 @@ -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -38,13 +38,12 @@ def message(message, flag='', destination=sys.stdout): if not logfile: return enable + mask = os.umask(0o111) try: # at boot the file is created as root:vyattacfg # at runtime the file is created as user:vyattacfg # but the helper scripts are not run as this so it # need the default permission to be 666 (an not 660) - mask = os.umask(0o111) - with open(logfile, 'a') as f: f.write(_timed(_format('log', message))) finally: @@ -65,7 +64,7 @@ def enabled(flag): - command: print command run with result Having the flag setup on the filesystem is required to have - debuging at boot time, however, setting the flag via environment + debugging at boot time, however, setting the flag via environment does not require a seek to the filesystem and is more efficient it can be done on the shell on via .bashrc for the user diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index 2b08ff68e..22aa1f62a 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -1,4 +1,4 @@ -# Copyright 2018-2025 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -15,10 +15,10 @@ import os -base_dir = '/usr/libexec/vyos/' +base_dir = '/usr/libexec/vyos' directories = { - 'base' : base_dir, + 'base' : f'{base_dir}', 'data' : '/usr/share/vyos/', 'conf_mode' : f'{base_dir}/conf_mode', 'op_mode' : f'{base_dir}/op_mode', @@ -38,18 +38,33 @@ directories = { 'vyos_configdir' : '/opt/vyatta/config', 'completion_dir' : f'{base_dir}/completion', 'ca_certificates' : '/usr/local/share/ca-certificates/vyos', + 'podman_storage' : '/usr/lib/live/mount/persistence/container/storage', 'ppp_nexthop_dir' : '/run/ppp_nexthop', - 'proto_path' : '/usr/share/vyos/vyconf' + 'proto_path' : '/usr/share/vyos/vyconf', + 'vyconf_session_dir' : f'{base_dir}/vyconf/session' } systemd_services = { - 'rsyslog' : 'rsyslog.service', + 'haproxy' : 'haproxy.service', + 'openconnect': 'ocserv.service', + 'syslog' : 'syslog.service', 'snmpd' : 'snmpd.service', } +internal_ports = { + 'certbot_haproxy' : 65080, # Certbot running behind haproxy +} + +config_files = { + 'sshd_user_ca' : '/run/sshd/trusted_user_ca', + 'igmp_proxy' : '/run/igmpproxy/igmpproxy.conf', +} + config_status = '/tmp/vyos-config-status' api_config_state = '/run/http-api-state' frr_debug_enable = '/tmp/vyos.frr.debug' +static_route_dhcp_interfaces_path = '/tmp/static_dhcp_interfaces' +vyos_configd_socket_path = 'ipc:///run/vyos-configd.sock' cfg_group = 'vyattacfg' @@ -63,8 +78,8 @@ config_default = os.path.join(directories['data'], 'config.boot.default') rt_symbolic_names = { # Standard routing tables for Linux & reserved IDs for VyOS - 'default': 253, # Confusingly, a final fallthru, not the default. - 'main': 254, # The actual global table used by iproute2 unless told otherwise. + 'default': 253, # Confusingly, a final fallthru, not the default. + 'main': 254, # The actual global table used by iproute2 unless told otherwise. 'local': 255, # Special kernel loopback table. } @@ -72,3 +87,26 @@ rt_global_vrf = rt_symbolic_names['main'] rt_global_table = rt_symbolic_names['main'] vyconfd_conf = '/etc/vyos/vyconfd.conf' + +DEFAULT_COMMIT_CONFIRM_MINUTES = 10 + +commit_hooks = {'pre': '/etc/commit/pre-hooks.d', + 'post': '/etc/commit/post-hooks.d' + } + +airbag_noteworthy_size = 20 + +SSH_DSA_DEPRECATION_WARNING: str = \ +'Support for SSH-DSA keys is deprecated and will be removed in VyOS 1.6. ' \ +'Please update affected keys to a supported algorithm (e.g., RSA, ECDSA or ' \ +'ED25519) to avoid authentication failures after the upgrade.' + +reference_tree_cache = '/usr/share/vyos/reftree.cache' + +activation_list = os.path.join(directories['config'], 'activation-list') +activation_init = os.path.join(directories['data'], 'activation-init') +activation_hint = os.path.join(directories['data'], '.activation_hint') + +config_sync_exclusion_list = os.path.join( + directories['data'], 'config-sync-exclude.json' +) diff --git a/python/vyos/derivedtree.py b/python/vyos/derivedtree.py new file mode 100644 index 000000000..dc46b8eb3 --- /dev/null +++ b/python/vyos/derivedtree.py @@ -0,0 +1,63 @@ +# Copyright (C) VyOS Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + + +from vyos.referencetree import ReferenceTree +from vyos.configtree import ConfigTree +from vyos.configtree import ConfigTreeError +from vyos.configtree import subtree_from_partial + + +class DerivedTreeError(Exception): + """Error to be raised by functions of derivedtree""" + + +def subtree_from_list_of_partial_paths( + ctree: ConfigTree, + paths: list[list[str]], + accumulator: ConfigTree = None, + reference_tree: ReferenceTree = None, +): + """Return the union of subtrees of the ConfigTree argument matching each + of the 'partial' paths. A partial path is one that may or may not + contain intervening tag node values, in which case it will match for all + values that apply. + + An existing subtree may be passed as the initial value of accumulator. + + For testing or use outside of the canonical environment, an instance of + the ReferenceTree may be passed from an alternative cache location. + """ + if reference_tree is None: + reference_tree = ReferenceTree() + + if accumulator is not None: + if not isinstance(accumulator, ConfigTree): + raise TypeError("Argument 'accumulator' must be an instance of ConfigTree") + else: + accumulator = ConfigTree('') + + errors = [] + for path in paths: + try: + accumulator = subtree_from_partial(ctree, path, reference_tree, accumulator) + except ConfigTreeError as e: + errors.append(str(e)) + continue + + if errors: + raise DerivedTreeError(f'Nonsensical paths: {errors}') + + return accumulator diff --git a/python/vyos/ethtool.py b/python/vyos/ethtool.py index 4710a5d40..34c884e0b 100644 --- a/python/vyos/ethtool.py +++ b/python/vyos/ethtool.py @@ -1,4 +1,4 @@ -# Copyright 2021-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -14,10 +14,13 @@ # License along with this library. If not, see <http://www.gnu.org/licenses/>. import re +import contextlib from json import loads from vyos.utils.network import interface_exists from vyos.utils.process import popen +from vyos.netlink import coalesce +from vyos.netlink import timestamp # These drivers do not support using ethtool to change the speed, duplex, or # flow control settings @@ -25,11 +28,16 @@ _drivers_without_speed_duplex_flow = ['vmxnet3', 'virtio_net', 'xen_netfront', 'iavf', 'ice', 'i40e', 'hv_netvsc', 'veth', 'ixgbevf', 'tun', 'vif'] +_drivers_without_mac_change = ['ena'] +# enable interface bonding will change the interface MAC address, thus all drivers +# not supporting MAC address change, also do not support bonding +_drivers_without_bonding_support = _drivers_without_mac_change + [] + class Ethtool: """ - Class is used to retrive and cache information about an ethernet adapter + Class is used to retrieve and cache information about an ethernet adapter """ - # dictionary containing driver featurs, it will be populated on demand and + # dictionary containing driver features, it will be populated on demand and # the content will look like: # [{'esp-hw-offload': {'active': False, 'fixed': True, 'requested': False}, # 'esp-tx-csum-hw-offload': {'active': False, @@ -59,6 +67,9 @@ class Ethtool: _ring_buffer = None _driver_name = None _flow_control = None + _channels = '' + _coalesce = None + _hw_timestamp_filters = None def __init__(self, ifname): # Get driver used for interface @@ -70,7 +81,7 @@ class Ethtool: if driver: self._driver_name = driver.group(1) - # Build a dictinary of supported link-speed and dupley settings. + # Build a dictionary of supported link-speed and dupley settings. # [ { # "ifname": "eth0", # "supported-ports": [ "TP" ], @@ -111,6 +122,19 @@ class Ethtool: if not bool(err): self._flow_control = loads(out)[0] + # Get information about NIC channels + out, err = popen(f'ethtool --show-channels {ifname}') + if not bool(err): + self._channels = out.lower() + + # Get information about NIC coalesce settings + with contextlib.suppress(coalesce.CoalesceError, coalesce.GeneralNetlinkError): + self._coalesce = coalesce.get_coalesce(ifname) + + # Get supported hardware timestamp receive filters + with contextlib.suppress(timestamp.TsInfoError, timestamp.GeneralNetlinkError): + self._hw_timestamp_filters = timestamp.get_hw_timestamp_filters(ifname) + def check_auto_negotiation_supported(self): """ Check if the NIC supports changing auto-negotiation """ return self._base_settings['supports-auto-negotiation'] @@ -169,7 +193,7 @@ class Ethtool: def check_speed_duplex(self, speed, duplex): """ Check if the passed speed and duplex combination is supported by - the underlaying network adapter. """ + the underlying network adapter. """ if isinstance(speed, int): speed = str(speed) if speed != 'auto' and not speed.isdigit(): @@ -199,3 +223,45 @@ class Ethtool: 'flow-control settings!') return 'on' if bool(self._flow_control['autonegotiate']) else 'off' + + def get_channels(self, rx_tx_comb): + """ + Get both the pre-set maximum and current value for a given channel type. + + Args: + rx_tx_comb (str): Channel type, one of "rx", "tx", or "combined". + + Returns: + list[int]: [maximum, current] values for the channel, + or an empty list if not supported. + """ + if rx_tx_comb not in ['rx', 'tx', 'combined']: + raise ValueError('Channel type must be either "rx", "tx" or "combined"') + matches = re.findall(rf'{rx_tx_comb}:\s+(\d+)', self._channels) + + return [int(value) for value in matches] + + def check_mac_change(self) -> bool: + """ Check if ethernet drivers supports changing MAC address """ + return bool(self.get_driver_name() not in _drivers_without_mac_change) + + def check_bonding(self) -> bool: + """ Check if ethernet drivers supports bonding """ + return bool(self.get_driver_name() not in _drivers_without_bonding_support) + + def check_coalesce(self, setting_name=None): + """Check if the NIC supports 'coalesce' parameter(s)""" + + if not self._coalesce: + return False + + return self._coalesce.get(setting_name) is not None if setting_name else True + + def get_coalesce(self): + """Get all 'coalesce' parameters for the interface""" + + return self._coalesce.copy() if self._coalesce else {} + + def get_hw_timestamp_filters(self): + """Get supported hardware timestamp receive filter names""" + return self._hw_timestamp_filters or set() diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index 9f01f8be1..c655e2de8 100755 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021-2024 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -12,24 +12,16 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import csv -import gzip -import os import re -from pathlib import Path from socket import AF_INET from socket import AF_INET6 from socket import getaddrinfo -from time import strftime -from vyos.remote import download from vyos.template import is_ipv4 -from vyos.template import render from vyos.utils.dict import dict_search_args from vyos.utils.dict import dict_search_recursive from vyos.utils.process import cmd -from vyos.utils.process import run from vyos.utils.network import get_vrf_tableid from vyos.defaults import rt_global_table from vyos.defaults import rt_global_vrf @@ -233,6 +225,9 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): hook_name = 'prerouting' if hook == 'NAM': hook_name = f'name' + # for policy + if hook == 'route' or hook == 'route6': + hook_name = hook output.append(f'{ip_name} {prefix}addr {operator} @GEOIP_CC{def_suffix}_{hook_name}_{fw_name}_{rule_id}') if 'mac_address' in side_conf: @@ -316,7 +311,10 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): if group_name[0] == '!': operator = '!=' group_name = group_name[1:] - output.append(f'{ip_name} {prefix}addr {operator} @R_{group_name}') + if ip_name == 'ip': + output.append(f'{ip_name} {prefix}addr {operator} @R_{group_name}') + elif ip_name == 'ip6': + output.append(f'{ip_name} {prefix}addr {operator} @R6_{group_name}') if 'mac_group' in group: group_name = group['mac_group'] operator = '' @@ -355,7 +353,7 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): if iiface[0] == '!': operator = '!=' iiface = iiface[1:] - output.append(f'iifname {operator} {{{iiface}}}') + output.append(f'iifname {operator} {{"{iiface}"}}') elif 'group' in rule_conf['inbound_interface']: iiface = rule_conf['inbound_interface']['group'] if iiface[0] == '!': @@ -370,7 +368,7 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): if oiface[0] == '!': operator = '!=' oiface = oiface[1:] - output.append(f'oifname {operator} {{{oiface}}}') + output.append(f'oifname {operator} {{"{oiface}"}}') elif 'group' in rule_conf['outbound_interface']: oiface = rule_conf['outbound_interface']['group'] if oiface[0] == '!': @@ -468,14 +466,14 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): output.append('gre version 1') if gre_key: - # The offset of the key within the packet shifts depending on the C-flag. - # nftables cannot handle complex enough expressions to match multiple + # The offset of the key within the packet shifts depending on the C-flag. + # nftables cannot handle complex enough expressions to match multiple # offsets based on bitfields elsewhere. - # We enforce a specific match for the checksum flag in validation, so the - # gre_flags dict will always have a 'checksum' key when gre_key is populated. - if not gre_flags['checksum']: + # We enforce a specific match for the checksum flag in validation, so the + # gre_flags dict will always have a 'checksum' key when gre_key is populated. + if not gre_flags['checksum']: # No "unset" child node means C is set, we offset key lookup +32 bits - output.append(f'@th,64,32 == {gre_key}') + output.append(f'@th,64,32 == {gre_key}') else: output.append(f'@th,32,32 == {gre_key}') @@ -557,7 +555,7 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): timeout_value = side_conf['timeout'] output.append(f'set update ip{def_suffix} {prefix}addr timeout {timeout_value} @DA{def_suffix}_{dyn_group}') else: - output.append(f'set update ip{def_suffix} saddr @DA{def_suffix}_{dyn_group}') + output.append(f'set update ip{def_suffix} {prefix}addr @DA{def_suffix}_{dyn_group}') set_table = False if 'set' in rule_conf: @@ -634,7 +632,7 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): return " ".join(output) def parse_gre_flags(flags, force_keyed=False): - flag_map = { # nft does not have symbolic names for these. + flag_map = { # nft does not have symbolic names for these. 'checksum': 1<<0, 'routing': 1<<1, 'key': 1<<2, @@ -645,7 +643,7 @@ def parse_gre_flags(flags, force_keyed=False): include = 0 exclude = 0 for fl_name, fl_state in flags.items(): - if not fl_state: + if not fl_state: include |= flag_map[fl_name] else: # 'unset' child tag exclude |= flag_map[fl_name] @@ -664,6 +662,19 @@ def parse_tcp_flags(flags): exclude = list(flags['not']) if 'not' in flags else [] return f'tcp flags & ({"|".join(include + exclude)}) == {"|".join(include) if include else "0x0"}' +def expand_weekday(abbrev: str) -> str: + mapping = { + 'mon': 'monday', + 'tue': 'tuesday', + 'wed': 'wednesday', + 'thu': 'thursday', + 'fri': 'friday', + 'sat': 'saturday', + 'sun': 'sunday', + } + return mapping.get(abbrev.lower(), abbrev).lower() + + def parse_time(time): out = [] if 'startdate' in time: @@ -681,123 +692,7 @@ def parse_time(time): if 'stoptime' in time and 'stopdate' not in time: out.append(f'hour < "{time["stoptime"]}"') if 'weekdays' in time: - days = time['weekdays'].split(",") - out_days = [f'"{day}"' for day in days if day[0] != '!'] + days = [day.strip() for day in time['weekdays'].split(",") if day] + out_days = [f'"{expand_weekday(day).title()}"' for day in days if day[0] != '!'] out.append(f'day {{{",".join(out_days)}}}') return " ".join(out) - -# GeoIP - -nftables_geoip_conf = '/run/nftables-geoip.conf' -geoip_database = '/usr/share/vyos-geoip/dbip-country-lite.csv.gz' -geoip_lock_file = '/run/vyos-geoip.lock' - -def geoip_load_data(codes=[]): - data = None - - if not os.path.exists(geoip_database): - return [] - - try: - with gzip.open(geoip_database, mode='rt') as csv_fh: - reader = csv.reader(csv_fh) - out = [] - for start, end, code in reader: - if code.lower() in codes: - out.append([start, end, code.lower()]) - return out - except: - print('Error: Failed to open GeoIP database') - return [] - -def geoip_download_data(): - url = 'https://download.db-ip.com/free/dbip-country-lite-{}.csv.gz'.format(strftime("%Y-%m")) - try: - dirname = os.path.dirname(geoip_database) - if not os.path.exists(dirname): - os.mkdir(dirname) - - download(geoip_database, url) - print("Downloaded GeoIP database") - return True - except: - print("Error: Failed to download GeoIP database") - return False - -class GeoIPLock(object): - def __init__(self, file): - self.file = file - - def __enter__(self): - if os.path.exists(self.file): - return False - - Path(self.file).touch() - return True - - def __exit__(self, exc_type, exc_value, tb): - os.unlink(self.file) - -def geoip_update(firewall, force=False): - with GeoIPLock(geoip_lock_file) as lock: - if not lock: - print("Script is already running") - return False - - if not firewall: - print("Firewall is not configured") - return True - - if not os.path.exists(geoip_database): - if not geoip_download_data(): - return False - elif force: - geoip_download_data() - - ipv4_codes = {} - ipv6_codes = {} - - ipv4_sets = {} - ipv6_sets = {} - - # Map country codes to set names - for codes, path in dict_search_recursive(firewall, 'country_code'): - set_name = f'GEOIP_CC_{path[1]}_{path[2]}_{path[4]}' - if ( path[0] == 'ipv4'): - for code in codes: - ipv4_codes.setdefault(code, []).append(set_name) - elif ( path[0] == 'ipv6' ): - set_name = f'GEOIP_CC6_{path[1]}_{path[2]}_{path[4]}' - for code in codes: - ipv6_codes.setdefault(code, []).append(set_name) - - if not ipv4_codes and not ipv6_codes: - if force: - print("GeoIP not in use by firewall") - return True - - geoip_data = geoip_load_data([*ipv4_codes, *ipv6_codes]) - - # Iterate IP blocks to assign to sets - for start, end, code in geoip_data: - ipv4 = is_ipv4(start) - if code in ipv4_codes and ipv4: - ip_range = f'{start}-{end}' if start != end else start - for setname in ipv4_codes[code]: - ipv4_sets.setdefault(setname, []).append(ip_range) - if code in ipv6_codes and not ipv4: - ip_range = f'{start}-{end}' if start != end else start - for setname in ipv6_codes[code]: - ipv6_sets.setdefault(setname, []).append(ip_range) - - render(nftables_geoip_conf, 'firewall/nftables-geoip-update.j2', { - 'ipv4_sets': ipv4_sets, - 'ipv6_sets': ipv6_sets - }) - - result = run(f'nft --file {nftables_geoip_conf}') - if result != 0: - print('Error: GeoIP failed to update firewall') - return False - - return True diff --git a/python/vyos/flavor.py b/python/vyos/flavor.py new file mode 100644 index 000000000..a9dccf448 --- /dev/null +++ b/python/vyos/flavor.py @@ -0,0 +1,69 @@ +# Copyright (C) VyOS Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +""" +VyOS flavor data access library. + +VyOS stores its flavor specific data in a JSON file. This module provides a +convenient interface to reading it. + +Example of the version data dict:: + { + 'console_type': 'ttyS', + 'console_num': '0', + 'console_speed': '115200' + } +""" + +import os +import vyos.defaults + +from vyos.utils.file import read_json + +flavor_file = os.path.join(vyos.defaults.directories['data'], 'flavor.json') + +def get_flavor_data(fname=flavor_file): + """ + Get complete flavor data + + Args: + file (str): path to the flavor file + + Returns: + dict: flavor data, if it can not be found and empty dict + + The optional ``file`` argument comes in handy in upgrade scripts + that need to retrieve information from images other than the running image. + It should not be used on a running system since the location of that file + is an implementation detail and may change in the future, while the interface + of this module will stay the same. + """ + return read_json(flavor_file, {}) + +def get_image_serial_console(fname=flavor_file): + """ + Get serial console parameters baked into the image flavor. + + Args: + file (str): path to the flavor file + + Returns: + dict: serial interface data baked into the image flavor. Example: + {"console_type": "ttyS", "console_speed": "115200", "console_num":"0"} + """ + console_type = get_flavor_data(fname=fname).get('console_type', '') + console_num = get_flavor_data(fname=fname).get('console_num', '') + console_speed = get_flavor_data(fname=fname).get('console_speed', '') + return (console_type, console_num, console_speed) diff --git a/python/vyos/frrender.py b/python/vyos/frrender.py index 8d469e3e2..ff69f8266 100644 --- a/python/vyos/frrender.py +++ b/python/vyos/frrender.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -14,18 +14,31 @@ # along with this library. If not, see <http://www.gnu.org/licenses/>. """ -Library used to interface with FRRs mgmtd introduced in version 10.0 +Helper class attached to vyos-configd to interact between our CLI configuration +and FRR. Class will render one full FRR configuration and apply this via +frr-reload.py, if the configuration has no errors. + +Will fail early if the rendered configuration has any errors. """ import os +from copy import deepcopy from time import sleep +from vyos.config import Config +from vyos.config import config_dict_merge +from vyos.configdict import get_dhcp_interfaces +from vyos.configdict import get_pppoe_interfaces from vyos.defaults import frr_debug_enable +from vyos.defaults import static_route_dhcp_interfaces_path from vyos.utils.dict import dict_search +from vyos.utils.dict import dict_set_nested +from vyos.utils.file import read_file from vyos.utils.file import write_file from vyos.utils.process import cmd from vyos.utils.process import rc_cmd +from vyos.template import get_dhcp_router from vyos.template import render_to_string from vyos import ConfigError @@ -34,9 +47,30 @@ def debug(message): return print(message) -frr_protocols = ['babel', 'bfd', 'bgp', 'eigrp', 'isis', 'mpls', 'nhrp', - 'openfabric', 'ospf', 'ospfv3', 'pim', 'pim6', 'rip', - 'ripng', 'rpki', 'segment_routing', 'static'] +ERROR_RELOAD_TEST: str = 'The system encountered an error while rendering the ' \ + 'new routing daemon configuration. To ensure network stability and avoid ' \ + 'potential connectivity disruptions, the configuration was not applied!' + +frr_protocols = [ + 'babel', + 'bfd', + 'bgp', + 'eigrp', + 'isis', + 'mpls', + 'nhrp', + 'openfabric', + 'ospf', + 'ospfv3', + 'pim', + 'pim6', + 'rip', + 'ripng', + 'rpki', + 'segment_routing', + 'static', + 'traffic_engineering', +] babel_daemon = 'babeld' bfd_daemon = 'bfdd' @@ -54,12 +88,7 @@ ripng_daemon = 'ripngd' zebra_daemon = 'zebra' nhrp_daemon = 'nhrpd' -def get_frrender_dict(conf, argv=None) -> dict: - from copy import deepcopy - from vyos.config import config_dict_merge - from vyos.configdict import get_dhcp_interfaces - from vyos.configdict import get_pppoe_interfaces - +def get_frrender_dict(conf: Config, argv=None) -> dict: # We need to re-set the CLI path to the root level, as this function uses # conf.exists() with an absolute path form the CLI root conf.set_level([]) @@ -73,7 +102,7 @@ def get_frrender_dict(conf, argv=None) -> dict: def dict_helper_ospf_defaults(ospf, path): # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. + # options which we need to update into the dictionary retrieved. default_values = conf.get_config_defaults(path, key_mangling=('-', '_'), get_first_key=True, recursive=True) @@ -92,7 +121,7 @@ def get_frrender_dict(conf, argv=None) -> dict: if dict_search(f'area.{area_num}.area_type.nssa', ospf) is None: del default_values['area'][area_num]['area_type']['nssa'] - for protocol in ['babel', 'bgp', 'connected', 'isis', 'kernel', 'rip', 'static']: + for protocol in ['babel', 'bgp', 'connected', 'isis', 'kernel', 'nhrp', 'rip', 'static']: if dict_search(f'redistribute.{protocol}', ospf) is None: del default_values['redistribute'][protocol] if not bool(default_values['redistribute']): @@ -111,7 +140,7 @@ def get_frrender_dict(conf, argv=None) -> dict: def dict_helper_ospfv3_defaults(ospfv3, path): # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. + # options which we need to update into the dictionary retrieved. default_values = conf.get_config_defaults(path, key_mangling=('-', '_'), get_first_key=True, recursive=True) @@ -139,7 +168,7 @@ def get_frrender_dict(conf, argv=None) -> dict: def dict_helper_pim_defaults(pim, path): # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. + # options which we need to update into the dictionary retrieved. default_values = conf.get_config_defaults(path, key_mangling=('-', '_'), get_first_key=True, recursive=True) @@ -196,6 +225,8 @@ def get_frrender_dict(conf, argv=None) -> dict: 'security_profile'] = name return nhrp + deleted_protocol = {'deleted' : ''} + # Ethernet and bonding interfaces can participate in EVPN which is configured via FRR tmp = {} for if_type in ['ethernet', 'bonding']: @@ -221,6 +252,12 @@ def get_frrender_dict(conf, argv=None) -> dict: ip_dict['afi'] = ip_version dict.update({ip_version : ip_dict}) + # Get FRR profile + frr_system_cli_path = ['system', 'frr'] + dict['system_frr'] = conf.get_config_dict(frr_system_cli_path, key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True) + # Enable SNMP agentx support # SNMP AgentX support cannot be disabled once enabled if conf.exists(['service', 'snmp']): @@ -239,6 +276,8 @@ def get_frrender_dict(conf, argv=None) -> dict: get_first_key=True, with_recursive_defaults=True) dict.update({'babel' : babel}) + elif conf.exists_effective(babel_cli_path): + dict.update({'babel' : deleted_protocol}) # We need to check the CLI if the BFD node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -249,6 +288,8 @@ def get_frrender_dict(conf, argv=None) -> dict: no_tag_node_value_mangle=True, with_recursive_defaults=True) dict.update({'bfd' : bfd}) + elif conf.exists_effective(bfd_cli_path): + dict.update({'bfd' : deleted_protocol}) # We need to check the CLI if the BGP node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -273,7 +314,7 @@ def get_frrender_dict(conf, argv=None) -> dict: with_recursive_defaults=True) dict.update({'eigrp' : eigrp}) elif conf.exists_effective(eigrp_cli_path): - dict.update({'eigrp' : {'deleted' : ''}}) + dict.update({'eigrp' : deleted_protocol}) # We need to check the CLI if the ISIS node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -285,7 +326,7 @@ def get_frrender_dict(conf, argv=None) -> dict: with_recursive_defaults=True) dict.update({'isis' : isis}) elif conf.exists_effective(isis_cli_path): - dict.update({'isis' : {'deleted' : ''}}) + dict.update({'isis' : deleted_protocol}) # We need to check the CLI if the MPLS node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -295,7 +336,7 @@ def get_frrender_dict(conf, argv=None) -> dict: get_first_key=True) dict.update({'mpls' : mpls}) elif conf.exists_effective(mpls_cli_path): - dict.update({'mpls' : {'deleted' : ''}}) + dict.update({'mpls' : deleted_protocol}) # We need to check the CLI if the OPENFABRIC node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -306,7 +347,7 @@ def get_frrender_dict(conf, argv=None) -> dict: no_tag_node_value_mangle=True) dict.update({'openfabric' : openfabric}) elif conf.exists_effective(openfabric_cli_path): - dict.update({'openfabric' : {'deleted' : ''}}) + dict.update({'openfabric' : deleted_protocol}) # We need to check the CLI if the OSPF node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -317,7 +358,7 @@ def get_frrender_dict(conf, argv=None) -> dict: ospf = dict_helper_ospf_defaults(ospf, ospf_cli_path) dict.update({'ospf' : ospf}) elif conf.exists_effective(ospf_cli_path): - dict.update({'ospf' : {'deleted' : ''}}) + dict.update({'ospf' : deleted_protocol}) # We need to check the CLI if the OSPFv3 node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -328,7 +369,7 @@ def get_frrender_dict(conf, argv=None) -> dict: ospfv3 = dict_helper_ospfv3_defaults(ospfv3, ospfv3_cli_path) dict.update({'ospfv3' : ospfv3}) elif conf.exists_effective(ospfv3_cli_path): - dict.update({'ospfv3' : {'deleted' : ''}}) + dict.update({'ospfv3' : deleted_protocol}) # We need to check the CLI if the PIM node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -339,7 +380,7 @@ def get_frrender_dict(conf, argv=None) -> dict: pim = dict_helper_pim_defaults(pim, pim_cli_path) dict.update({'pim' : pim}) elif conf.exists_effective(pim_cli_path): - dict.update({'pim' : {'deleted' : ''}}) + dict.update({'pim' : deleted_protocol}) # We need to check the CLI if the PIM6 node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -350,7 +391,7 @@ def get_frrender_dict(conf, argv=None) -> dict: with_recursive_defaults=True) dict.update({'pim6' : pim6}) elif conf.exists_effective(pim6_cli_path): - dict.update({'pim6' : {'deleted' : ''}}) + dict.update({'pim6' : deleted_protocol}) # We need to check the CLI if the RIP node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -361,7 +402,7 @@ def get_frrender_dict(conf, argv=None) -> dict: with_recursive_defaults=True) dict.update({'rip' : rip}) elif conf.exists_effective(rip_cli_path): - dict.update({'rip' : {'deleted' : ''}}) + dict.update({'rip' : deleted_protocol}) # We need to check the CLI if the RIPng node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -372,7 +413,7 @@ def get_frrender_dict(conf, argv=None) -> dict: with_recursive_defaults=True) dict.update({'ripng' : ripng}) elif conf.exists_effective(ripng_cli_path): - dict.update({'ripng' : {'deleted' : ''}}) + dict.update({'ripng' : deleted_protocol}) # We need to check the CLI if the RPKI node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -388,7 +429,7 @@ def get_frrender_dict(conf, argv=None) -> dict: cache_config['ssh']['private_key_file'] = f'{rpki_ssh_key_base}_{cache}' dict.update({'rpki' : rpki}) elif conf.exists_effective(rpki_cli_path): - dict.update({'rpki' : {'deleted' : ''}}) + dict.update({'rpki' : deleted_protocol}) # We need to check the CLI if the Segment Routing node is present and thus load in # all the default values present on the CLI - that's why we have if conf.exists() @@ -400,7 +441,22 @@ def get_frrender_dict(conf, argv=None) -> dict: with_recursive_defaults=True) dict.update({'segment_routing' : sr}) elif conf.exists_effective(sr_cli_path): - dict.update({'segment_routing' : {'deleted' : ''}}) + dict.update({'segment_routing' : deleted_protocol}) + + # We need to check the CLI if the Traffic Engineering node is present and thus load in + # all the default values present on the CLI - that's why we have if conf.exists() + te_cli_path = ['protocols', 'traffic-engineering'] + if conf.exists(te_cli_path): + te = conf.get_config_dict( + te_cli_path, + key_mangling=('-', '_'), + get_first_key=True, + no_tag_node_value_mangle=True, + with_recursive_defaults=True, + ) + dict.update({'traffic_engineering': te}) + elif conf.exists_effective(te_cli_path): + dict.update({'traffic_engineering': deleted_protocol}) # We need to check the CLI if the static node is present and thus load in # all the default values present on the CLI - that's why we have if conf.exists() @@ -411,7 +467,7 @@ def get_frrender_dict(conf, argv=None) -> dict: no_tag_node_value_mangle=True) dict.update({'static' : static}) elif conf.exists_effective(static_cli_path): - dict.update({'static' : {'deleted' : ''}}) + dict.update({'static' : deleted_protocol}) # We need to check the CLI if the NHRP node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -423,21 +479,14 @@ def get_frrender_dict(conf, argv=None) -> dict: nhrp = dict_helper_nhrp_defaults(nhrp) dict.update({'nhrp' : nhrp}) elif conf.exists_effective(nhrp_cli_path): - dict.update({'nhrp' : {'deleted' : ''}}) + dict.update({'nhrp' : deleted_protocol}) # T3680 - get a list of all interfaces currently configured to use DHCP tmp = get_dhcp_interfaces(conf) - if tmp: - if 'static' in dict: - dict['static'].update({'dhcp' : tmp}) - else: - dict.update({'static' : {'dhcp' : tmp}}) + if tmp: dict_set_nested('static.dhcp', tmp, dict) + tmp = get_pppoe_interfaces(conf) - if tmp: - if 'static' in dict: - dict['static'].update({'pppoe' : tmp}) - else: - dict.update({'static' : {'pppoe' : tmp}}) + if tmp: dict_set_nested('static.pppoe', tmp, dict) # keep a re-usable list of dependent VRFs dependent_vrfs_default = {} @@ -456,10 +505,12 @@ def get_frrender_dict(conf, argv=None) -> dict: # come into place under the protocols tree, thus we can safely merge them with the # appropriate routing protocols for vrf_name, vrf_config in vrf['name'].items(): + protocol_dict_path = f'name.{vrf_name}.protocols' + bgp_vrf_path = ['vrf', 'name', vrf_name, 'protocols', 'bgp'] if 'bgp' in vrf_config.get('protocols', []): # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. + # options which we need to update into the dictionary retrieved. default_values = conf.get_config_defaults(bgp_vrf_path, key_mangling=('-', '_'), get_first_key=True, recursive=True) @@ -498,10 +549,7 @@ def get_frrender_dict(conf, argv=None) -> dict: if 'bgp' in dict: dict['bgp']['dependent_vrfs'].update({vrf_name : {'protocols': tmp} }) - if 'protocols' not in vrf['name'][vrf_name]: - vrf['name'][vrf_name].update({'protocols': {'bgp' : tmp}}) - else: - vrf['name'][vrf_name]['protocols'].update({'bgp' : tmp}) + dict_set_nested(f'{protocol_dict_path}.bgp', tmp, vrf) # We need to check the CLI if the EIGRP node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -509,9 +557,9 @@ def get_frrender_dict(conf, argv=None) -> dict: if 'eigrp' in vrf_config.get('protocols', []): eigrp = conf.get_config_dict(eigrp_vrf_path, key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) - vrf['name'][vrf_name]['protocols'].update({'eigrp' : isis}) + dict_set_nested(f'{protocol_dict_path}.eigrp', eigrp, vrf) elif conf.exists_effective(eigrp_vrf_path): - vrf['name'][vrf_name]['protocols'].update({'eigrp' : {'deleted' : ''}}) + dict_set_nested(f'{protocol_dict_path}.eigrp', deleted_protocol, vrf) # We need to check the CLI if the ISIS node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -519,9 +567,9 @@ def get_frrender_dict(conf, argv=None) -> dict: if 'isis' in vrf_config.get('protocols', []): isis = conf.get_config_dict(isis_vrf_path, key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True, with_recursive_defaults=True) - vrf['name'][vrf_name]['protocols'].update({'isis' : isis}) + dict_set_nested(f'{protocol_dict_path}.isis', isis, vrf) elif conf.exists_effective(isis_vrf_path): - vrf['name'][vrf_name]['protocols'].update({'isis' : {'deleted' : ''}}) + dict_set_nested(f'{protocol_dict_path}.isis', deleted_protocol, vrf) # We need to check the CLI if the OSPF node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -529,9 +577,9 @@ def get_frrender_dict(conf, argv=None) -> dict: if 'ospf' in vrf_config.get('protocols', []): ospf = conf.get_config_dict(ospf_vrf_path, key_mangling=('-', '_'), get_first_key=True) ospf = dict_helper_ospf_defaults(vrf_config['protocols']['ospf'], ospf_vrf_path) - vrf['name'][vrf_name]['protocols'].update({'ospf' : ospf}) + dict_set_nested(f'{protocol_dict_path}.ospf', ospf, vrf) elif conf.exists_effective(ospf_vrf_path): - vrf['name'][vrf_name]['protocols'].update({'ospf' : {'deleted' : ''}}) + dict_set_nested(f'{protocol_dict_path}.ospf', deleted_protocol, vrf) # We need to check the CLI if the OSPFv3 node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -539,9 +587,24 @@ def get_frrender_dict(conf, argv=None) -> dict: if 'ospfv3' in vrf_config.get('protocols', []): ospfv3 = conf.get_config_dict(ospfv3_vrf_path, key_mangling=('-', '_'), get_first_key=True) ospfv3 = dict_helper_ospfv3_defaults(vrf_config['protocols']['ospfv3'], ospfv3_vrf_path) - vrf['name'][vrf_name]['protocols'].update({'ospfv3' : ospfv3}) + dict_set_nested(f'{protocol_dict_path}.ospfv3', ospfv3, vrf) elif conf.exists_effective(ospfv3_vrf_path): - vrf['name'][vrf_name]['protocols'].update({'ospfv3' : {'deleted' : ''}}) + dict_set_nested(f'{protocol_dict_path}.ospfv3', deleted_protocol, vrf) + + # We need to check the CLI if the RPKI node is present and thus load in all the default + # values present on the CLI - that's why we have if conf.exists() + rpki_vrf_path = ['vrf', 'name', vrf_name, 'protocols', 'rpki'] + if 'rpki' in vrf_config.get('protocols', []): + rpki = conf.get_config_dict(rpki_vrf_path, key_mangling=('-', '_'), get_first_key=True, + with_pki=True, with_recursive_defaults=True) + rpki_ssh_key_base = '/run/frr/id_rpki' + for cache, cache_config in rpki.get('cache',{}).items(): + if 'ssh' in cache_config: + cache_config['ssh']['public_key_file'] = f'{rpki_ssh_key_base}_{cache}.pub' + cache_config['ssh']['private_key_file'] = f'{rpki_ssh_key_base}_{cache}' + dict_set_nested(f'{protocol_dict_path}.rpki', rpki, vrf) + elif conf.exists_effective(rpki_vrf_path): + dict_set_nested(f'{protocol_dict_path}.rpki', deleted_protocol, vrf) # We need to check the CLI if the static node is present and thus load in all the default # values present on the CLI - that's why we have if conf.exists() @@ -550,15 +613,16 @@ def get_frrender_dict(conf, argv=None) -> dict: static = conf.get_config_dict(static_vrf_path, key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) - # T3680 - get a list of all interfaces currently configured to use DHCP - tmp = get_dhcp_interfaces(conf, vrf_name) - if tmp: static.update({'dhcp' : tmp}) - tmp = get_pppoe_interfaces(conf, vrf_name) - if tmp: static.update({'pppoe' : tmp}) - - vrf['name'][vrf_name]['protocols'].update({'static': static}) + dict_set_nested(f'{protocol_dict_path}.static', static, vrf) elif conf.exists_effective(static_vrf_path): - vrf['name'][vrf_name]['protocols'].update({'static': {'deleted' : ''}}) + dict_set_nested(f'{protocol_dict_path}.static', deleted_protocol, vrf) + + # T3680 - get a list of all interfaces currently configured to use DHCP + tmp = get_dhcp_interfaces(conf, vrf_name) + if tmp: dict_set_nested(f'name.{vrf_name}.protocols.static.dhcp', tmp, vrf) + + tmp = get_pppoe_interfaces(conf, vrf_name) + if tmp: dict_set_nested(f'name.{vrf_name}.protocols.static.pppoe', tmp, vrf) vrf_vni_path = ['vrf', 'name', vrf_name, 'vni'] if conf.exists(vrf_vni_path): @@ -612,6 +676,7 @@ def get_frrender_dict(conf, argv=None) -> dict: class FRRender: cached_config_dict = {} + cached_dhcp_gateways = {} def __init__(self): self._frr_conf = '/run/frr/config/vyos.frr.conf' @@ -624,11 +689,20 @@ class FRRender: tmp = type(config_dict) raise ValueError(f'Config must be of type "dict" and not "{tmp}"!') + dhcp_gateways = { + interface: get_dhcp_router(interface) + for interface in read_file(static_route_dhcp_interfaces_path, '').split() + } - if self.cached_config_dict == config_dict: + if ( + self.cached_config_dict == config_dict + and self.cached_dhcp_gateways == dhcp_gateways + ): debug('FRR: NO CHANGES DETECTED') return False + self.cached_config_dict = config_dict + self.cached_dhcp_gateways = dhcp_gateways def inline_helper(config_dict) -> str: output = '!\n' @@ -675,11 +749,20 @@ class FRRender: output += render_to_string('frr/ripngd.frr.j2', config_dict['ripng']) output += '\n' if 'rpki' in config_dict and 'deleted' not in config_dict['rpki']: - output += render_to_string('frr/rpki.frr.j2', config_dict['rpki']) + output += render_to_string('frr/rpki.frr.j2', {'rpki': config_dict['rpki']}) output += '\n' if 'segment_routing' in config_dict and 'deleted' not in config_dict['segment_routing']: output += render_to_string('frr/zebra.segment_routing.frr.j2', config_dict['segment_routing']) output += '\n' + if ( + 'traffic_engineering' in config_dict + and 'deleted' not in config_dict['traffic_engineering'] + ): + output += render_to_string( + 'frr/zebra.traffic_engineering.frr.j2', + config_dict['traffic_engineering'], + ) + output += '\n' if 'static' in config_dict and 'deleted' not in config_dict['static']: output += render_to_string('frr/staticd.frr.j2', config_dict['static']) output += '\n' @@ -697,6 +780,23 @@ class FRRender: debug('FRR: START CONFIGURATION RENDERING') # we can not reload an empty file, thus we always embed the marker output = '!\n' + + # FRR profile configuration + tmp = dict_search('system_frr.profile', config_dict) + if tmp: + output += f'frr defaults {tmp}\n' + + # Enable FRR logging + output += 'log facility daemon\n' + output += 'log timestamp precision 3\n' + # Extend logging depending on operating mode + if os.path.exists(frr_debug_enable): + output += 'log syslog informational\n' + output += 'log unique-id\n' + else: + output += 'log syslog notifications\n' + output += 'no log unique-id\n' + # Enable SNMP agentx support # SNMP AgentX support cannot be disabled once enabled if 'snmp' in config_dict: @@ -718,7 +818,7 @@ class FRRender: output += inline_helper(vrf_config['protocols']) - # remove any accidently added empty newline to not confuse FRR + # remove any accidentally added empty newline to not confuse FRR output = os.linesep.join([s for s in output.splitlines() if s]) if '!!' in output: @@ -730,6 +830,13 @@ class FRRender: return True def apply(self, count_max=5): + # Do a config reload test + cmdline = f'/usr/lib/frr/frr-reload.py --test' + rc, emsg = rc_cmd(f'{cmdline} {self._frr_conf}') + if rc != 0: + debug(emsg) + raise ConfigError(ERROR_RELOAD_TEST) + count = 0 emsg = '' while count < count_max: diff --git a/python/vyos/geoip.py b/python/vyos/geoip.py new file mode 100644 index 000000000..db2313ab9 --- /dev/null +++ b/python/vyos/geoip.py @@ -0,0 +1,267 @@ + +import csv +import gzip +import os +import sqlite3 +import zipfile + +from io import TextIOWrapper +from pathlib import Path +from time import strftime + +from vyos.remote import download +from vyos.template import is_ipv4, render +from vyos.utils.dict import dict_search_recursive +from vyos.utils.process import run + +nftables_geoip_conf = '/run/nftables-geoip.conf' +dbip_database_raw = '/usr/share/vyos-geoip/dbip-country-lite.csv.gz' +mm_database_raw = '/usr/share/vyos-geoip/maxmind-country.zip' +geoip_database_path = '/var/cache/vyos/geoip-lookup.db' +geoip_lock_file = '/var/lock/vyos-geoip.lock' + +# Raw data + +def geoip_download_dbip(): + url = 'https://download.db-ip.com/free/dbip-country-lite-{}.csv.gz'.format(strftime("%Y-%m")) + try: + dirname = os.path.dirname(dbip_database_raw) + if not os.path.exists(dirname): + os.mkdir(dirname) + + download(dbip_database_raw, url) + return True + except: + return False + +def geoip_download_maxmind(account_id : str, license_key: str, lite : bool) -> bool: + db_str = 'GeoLite2' if lite else 'GeoIP2' + url = f'https://{account_id}:{license_key}@download.maxmind.com/geoip/databases/{db_str}-Country-CSV/download?suffix=zip' + try: + dirname = os.path.dirname(mm_database_raw) + if not os.path.exists(dirname): + os.mkdir(dirname) + + download(mm_database_raw, url) + return True + except: + return False + +# VyOS database + +def db_is_initialised(): + if not os.path.exists(geoip_database_path): + return False + + with sqlite3.connect(geoip_database_path) as conn: + cur = conn.cursor() + cur.execute("PRAGMA table_info(geoip_ranges);") + rows = cur.fetchall() + return len(rows) > 0 + +def db_initialise(): + dirname = os.path.dirname(geoip_database_path) + if not os.path.exists(dirname): + os.mkdir(dirname) + + with sqlite3.connect(geoip_database_path) as conn: + cur = conn.cursor() + cur.execute(""" + CREATE TABLE IF NOT EXISTS geoip_ranges ( + country_code TEXT NOT NULL, + range TEXT NOT NULL, + version INT NOT NULL + ) + """) + cur.execute('CREATE INDEX IF NOT EXISTS idx_cc_version ON geoip_ranges(country_code, version)') + conn.commit() + +def db_import_dbip_ranges(replace=True, delete_file=False): + if not os.path.exists(dbip_database_raw): + return False + + if not os.path.exists(geoip_database_path): + return False + + try: + with gzip.open(dbip_database_raw, mode='rt') as csv_fh: + reader = csv.reader(csv_fh) + + with sqlite3.connect(geoip_database_path) as conn: + cur = conn.cursor() + + if replace: + cur.execute('DELETE FROM geoip_ranges') + + for start, end, code in reader: + version = 4 if is_ipv4(start) else 6 + cur.execute('INSERT INTO geoip_ranges (country_code, range, version) VALUES (?, ?, ?)', (code.lower(), f'{start}-{end}', version)) + conn.commit() + + if delete_file: + os.unlink(dbip_database_raw) + + return True + except: + return False + +def db_import_maxmind_ranges(replace=True, delete_file=False): + if not os.path.exists(mm_database_raw): + return False + + if not zipfile.is_zipfile(mm_database_raw): + return False + + if not os.path.exists(geoip_database_path): + return False + + try: + with zipfile.ZipFile(mm_database_raw, mode='r') as zip_fh: + directory = os.path.dirname(zip_fh.namelist()[0]) + prefix = 'GeoLite2' if any(f.startswith('GeoLite2') for f in zip_fh.namelist()) else 'GeoIP2' + + ipv4_file = f'{directory}/{prefix}-Country-Blocks-IPv4.csv' + ipv6_file = f'{directory}/{prefix}-Country-Blocks-IPv6.csv' + locations_file = f'{directory}/{prefix}-Country-Locations-en.csv' + locations_map = {} + + with zip_fh.open(locations_file) as raw_csv_fh: + with TextIOWrapper(raw_csv_fh, encoding='utf-8') as csv_fh: + reader = csv.DictReader(csv_fh) + + for row in reader: + id = row['geoname_id'] + locations_map[id] = row['country_iso_code'] + + with sqlite3.connect(geoip_database_path) as conn: + cur = conn.cursor() + + if replace: + cur.execute('DELETE FROM geoip_ranges') + + with zip_fh.open(ipv4_file) as raw_csv_fh: + with TextIOWrapper(raw_csv_fh, encoding='utf-8') as csv_fh: + reader = csv.DictReader(csv_fh) + for row in reader: + id = row['geoname_id'] + + if not id or id not in locations_map: + continue + + code = locations_map[id] + cur.execute('INSERT INTO geoip_ranges (country_code, range, version) VALUES (?, ?, 4)', (code.lower(), row['network'])) + + with zip_fh.open(ipv6_file) as raw_csv_fh: + with TextIOWrapper(raw_csv_fh, encoding='utf-8') as csv_fh: + reader = csv.DictReader(csv_fh) + for row in reader: + id = row['geoname_id'] + + if not id or id not in locations_map: + continue + + code = locations_map[id] + cur.execute('INSERT INTO geoip_ranges (country_code, range, version) VALUES (?, ?, 6)', (code.lower(), row['network'])) + + conn.commit() + + if delete_file: + os.unlink(mm_database_raw) + + return True + except: + return False + +def db_return_ranges(codes, version): + out = [] + with sqlite3.connect(geoip_database_path) as conn: + cur = conn.cursor() + ph = ','.join(['?'] * len(codes)) + for row in cur.execute(f'SELECT range FROM geoip_ranges WHERE version = ? AND country_code IN ({ph})', [version, *codes]): + out.append(row[0]) + return out + +# Update + +def geoip_refresh(): + with GeoIPLock(geoip_lock_file) as lock: + if not lock: + return True + + if not os.path.exists(nftables_geoip_conf): + return False + + result = run(f'nft --file {nftables_geoip_conf}') + if result != 0: + return False + + return True + +def geoip_update(firewall=None, policy=None): + with GeoIPLock(geoip_lock_file) as lock: + if not lock: + print("Script is already running") + return False + + if not firewall and not policy: + print("Firewall and policy are not configured") + return True + + if not os.path.exists(geoip_database_path): + print("Running one-time database initialisation") + db_initialise() + db_import_dbip_ranges() + + firewall_sets = {'v4': {}, 'v6': {}} + policy_sets = {'v4': {}, 'v6': {}} + + if firewall: + for codes, path in dict_search_recursive(firewall, 'country_code'): + if path[0] == 'policy': + continue + + version = 6 if path[0] == 'ipv6' else 4 + vprefix = '6' if version == 6 else '' + set_name = f'GEOIP_CC{vprefix}_{path[1]}_{path[2]}_{path[4]}' + firewall_sets[f'v{version}'][set_name] = db_return_ranges(codes, version) + + if policy: + for codes, path in dict_search_recursive(policy, 'country_code'): + if path[0] == 'firewall': + continue + + version = 6 if path[0] == 'route6' else 4 + vprefix = '6' if version == 6 else '' + set_name = f'GEOIP_CC{vprefix}_{path[0]}_{path[1]}_{path[3]}' + policy_sets[f'v{version}'][set_name] = db_return_ranges(codes, version) + + render( + nftables_geoip_conf, + 'firewall/nftables-geoip-update.j2', + {'firewall_sets': firewall_sets, 'policy_sets': policy_sets}, + group='vyattacfg', + permission=0o664, + ) + + result = run(f'nft --file {nftables_geoip_conf}') + if result != 0: + print('Error: GeoIP failed to update firewall and/or policy') + return False + + return True + +# Utility + +class GeoIPLock(object): + def __init__(self, file): + self.file = file + + def __enter__(self): + if os.path.exists(self.file): + return False + + Path(self.file).touch() + return True + + def __exit__(self, exc_type, exc_value, tb): + os.unlink(self.file) diff --git a/python/vyos/http_api_client.py b/python/vyos/http_api_client.py new file mode 100644 index 000000000..ae16cfa54 --- /dev/null +++ b/python/vyos/http_api_client.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# +# Copyright (C) VyOS Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +import json +import urllib3 +import requests +from typing import Optional +from dataclasses import dataclass + +from vyos.version import get_version +from vyos.template import bracketize_ipv6 + + +class ApiError(Exception): + """Generic VyOS HTTP API client error""" + + +class ApiAuthError(ApiError): + """Authentication/authorization error""" + + +class ApiTransportError(ApiError): + """Network/transport error (timeouts, connection errors, etc.)""" + + +class ApiResponseError(ApiError): + """Server responded, but payload is invalid or indicates an error""" + + def __init__(self, message, response=None): + super().__init__(message) + self.response = response + + +@dataclass(frozen=True) +class ApiClientConfig: + host: str + key: str + port: int = 443 + timeout: Optional[int] = None + verify_tls: bool = False + + +class ApiClient: + """Small helper for talking to VyOS HTTP API using requests. + + Design goals: + - minimal surface area (thin wrapper around requests) + - consistent error handling + typed exceptions + - safe defaults for VyOS typical self-signed HTTPS usage (verify_tls=False) + """ + + _DEFAULT_HEADERS = { + 'Content-Type': 'application/json', + 'User-Agent': f'VyOS/{get_version()}', + } + + def __init__(self, config: ApiClientConfig): + assert isinstance(config, ApiClientConfig) + + self._cfg = config + self._host = bracketize_ipv6(config.host) + + self._session = requests.Session() + self._session.headers.update(self._DEFAULT_HEADERS) + + if not config.verify_tls: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + @property + def base_url(self) -> str: + return f'https://{self._host}:{self._cfg.port}' + + def post( + self, + endpoint: str, + payload: dict, + *, + params: Optional[dict] = None, + raise_on_error: bool = True, + ) -> dict: + """POST JSON to API and return decoded JSON response. + + Args: + endpoint: URL path, e.g. '/configure-section' or 'configure' + payload: dict that will be JSON-encoded and sent as request body + params: Optional query parameters + raise_on_error: If True, raise when response envelope contains bad status code + + Raises: + ApiTransportError: network problems/timeouts + ApiAuthError: 401/403 responses + ApiResponseError: non-2xx responses or invalid JSON + """ + if not endpoint.startswith('/'): + endpoint = f'/{endpoint}' + + # Most VyOS endpoints in this repo expect 'key' inside body. + body = dict(payload) + body.setdefault('key', self._cfg.key) + + url = f'{self.base_url}{endpoint}' + + try: + resp = self._session.post( + url, + data=json.dumps(body), + params=params, + timeout=self._cfg.timeout, + verify=self._cfg.verify_tls, + ) + except requests.exceptions.Timeout as e: + raise ApiTransportError(f'Request timed out: {e}') from e + except requests.exceptions.RequestException as e: + raise ApiTransportError(f'Request failed: {e}') from e + + if not resp.ok: + text = resp.text.strip() + err_msg = f'HTTP {resp.status_code} from {endpoint}: {text}' + + if resp.status_code in (401, 403): + raise ApiAuthError(err_msg) + elif resp.status_code >= 500: + raise ApiResponseError(err_msg, response=resp) + + if raise_on_error: + raise ApiResponseError(err_msg, response=resp) + + try: + return resp.json() + except json.JSONDecodeError as e: + raise ApiResponseError( + f'Invalid JSON response from {endpoint}: {e}', + response=resp, + ) from e diff --git a/python/vyos/ifconfig/__init__.py b/python/vyos/ifconfig/__init__.py index 206b2bba1..7838fa9a2 100644 --- a/python/vyos/ifconfig/__init__.py +++ b/python/vyos/ifconfig/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/ifconfig/afi.py b/python/vyos/ifconfig/afi.py index fd263d220..a391cb8a0 100644 --- a/python/vyos/ifconfig/afi.py +++ b/python/vyos/ifconfig/afi.py @@ -1,4 +1,4 @@ -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/ifconfig/bond.py b/python/vyos/ifconfig/bond.py index a659b9bd2..ace84c6f2 100644 --- a/python/vyos/ifconfig/bond.py +++ b/python/vyos/ifconfig/bond.py @@ -1,4 +1,4 @@ -# Copyright 2019-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,8 +13,6 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. -import os - from vyos.ifconfig.interface import Interface from vyos.utils.dict import dict_search from vyos.utils.assertion import assert_list @@ -90,6 +88,9 @@ class BondIf(Interface): 'bond_arp_ip_target': { 'location': '/sys/class/net/{ifname}/bonding/arp_ip_target', }, + 'bond_members': { + 'location': '/sys/class/net/{ifname}/bonding/slaves', + }, 'bond_mode': { 'location': '/sys/class/net/{ifname}/bonding/mode', } @@ -124,27 +125,27 @@ class BondIf(Interface): # when a bond member gets deleted, all members are placed in A/D state # even when they are enabled inside CLI. This will make the config # and system look async. - slave_list = [] - for s in self.get_slaves(): - slave = { + members = [] + for s in self.get_members(): + member = { 'ifname': s, 'state': Interface(s).get_admin_state() } - slave_list.append(slave) + members.append(member) # remove bond master which places members in disabled state super().remove() # replicate previous interface state before bond destruction back to # physical interface - for slave in slave_list: - i = Interface(slave['ifname']) - i.set_admin_state(slave['state']) + for member in members: + i = Interface(member['ifname']) + i.set_admin_state(member['state']) def set_hash_policy(self, mode): """ - Selects the transmit hash policy to use for slave selection in - balance-xor, 802.3ad, and tlb modes. Possible values are: layer2, + Selects the transmit hash policy to use for selecting forwarding link + in balance-xor, 802.3ad, and tlb modes. Possible values are: layer2, layer2+3, layer3+4, encap2+3, encap3+4. The default value is layer2 @@ -200,8 +201,8 @@ class BondIf(Interface): def set_miimon_interval(self, interval): """ - Specifies the MII link monitoring frequency in milliseconds. This - determines how often the link state of each slave is inspected for link + Specifies the MII link monitoring frequency in milliseconds. Determines + how often the link state of each physical member is inspected for link failures. A value of zero disables MII link monitoring. A value of 100 is a good starting point. @@ -217,10 +218,10 @@ class BondIf(Interface): """ Specifies the ARP link monitoring frequency in milliseconds. - The ARP monitor works by periodically checking the slave devices + The ARP monitor works by periodically checking the member devices to determine whether they have sent or received traffic recently (the precise criteria depends upon the bonding mode, and the - state of the slave). Regular traffic is generated via ARP probes + state of the member). Regular traffic is generated via ARP probes issued for the addresses specified by the arp_ip_target option. If ARP monitoring is used in an etherchannel compatible mode @@ -283,7 +284,7 @@ class BondIf(Interface): def add_port(self, interface): """ - Enslave physical interface to bond. + Add physical interface to bond as participating member. Example: >>> from vyos.ifconfig import BondIf @@ -301,19 +302,19 @@ class BondIf(Interface): # The kernel will ALWAYS place new bond members in "up" state regardless # what the CLI will tell us! - # Physical interface must be in admin down state before they can be - # enslaved. If this is not the case an error will be shown: + # Physical interface must be in admin down state before it can be + # added. If this is not the case an error will be shown: # bond0: eth0 is up - this may be due to an out of date ifenslave - slave = Interface(interface) - slave_state = slave.get_admin_state() - if slave_state == 'up': - slave.set_admin_state('down') + member = Interface(interface) + member_state = member.get_admin_state() + if member_state == 'up': + member.set_admin_state('down') ret = self.set_interface('bond_add_port', f'+{interface}') - # The kernel will ALWAYS place new bond members in "up" state regardless - # what the LI is configured for - thus we place the interface in its - # desired state - slave.set_admin_state(slave_state) + # The kernel will ALWAYS place new bond members in "up" state + # regardless what the interface is configured for - thus we place the + # interface in its desired state + member.set_admin_state(member_state) return ret def del_port(self, interface): @@ -326,26 +327,18 @@ class BondIf(Interface): """ return self.set_interface('bond_del_port', f'-{interface}') - def get_slaves(self): + def get_members(self) -> list: """ - Return a list with all configured slave interfaces on this bond. + Return a list with all configured physical member interfaces. Example: >>> from vyos.ifconfig import BondIf - >>> BondIf('bond0').get_slaves() + >>> BondIf('bond0').get_members() ['eth1', 'eth2'] """ - enslaved_ifs = [] - # retrieve real enslaved interfaces from OS kernel - sysfs_bond = '/sys/class/net/{}'.format(self.config['ifname']) - if os.path.isdir(sysfs_bond): - for directory in os.listdir(sysfs_bond): - if 'lower_' in directory: - enslaved_ifs.append(directory.replace('lower_', '')) + return self.get_interface('bond_members').split() - return enslaved_ifs - - def get_mode(self): + def get_mode(self) -> str: """ Return bond operation mode. @@ -354,17 +347,16 @@ class BondIf(Interface): >>> BondIf('bond0').get_mode() '802.3ad' """ - mode = self.get_interface('bond_mode') - # mode is now "802.3ad 4", we are only interested in "802.3ad" - return mode.split()[0] + mode_name, _ = self.get_interface('bond_mode').split() + return mode_name def set_primary(self, interface): """ - A string (eth0, eth2, etc) specifying which slave is the primary - device. The specified device will always be the active slave while it + A string (eth0, eth2, etc.) specifying which member is the primary + device. The specified device will always be the active member while it is available. Only when the primary is off-line will alternate devices - be used. This is useful when one slave is preferred over another, e.g., - when one slave has higher throughput than another. + be used. This is useful when one member is preferred over another, e.g., + when one member has higher throughput than another. The primary option is only valid for active-backup, balance-tlb and balance-alb mode. @@ -383,13 +375,15 @@ class BondIf(Interface): Possible values are: balance-rr, active-backup, balance-xor, broadcast, 802.3ad, balance-tlb, balance-alb - NOTE: the bonding mode can not be changed when the bond itself has - slaves + NOTE: the bond operation mode cannot be changed when the bond itself + has members attached! Example: >>> from vyos.ifconfig import BondIf >>> BondIf('bond0').set_mode('802.3ad') """ + if len(self.get_members()) != 0: + raise OSError(f'{self.ifname} still has member interfaces attached!') return self.set_interface('bond_mode', mode) def set_system_mac(self, mac): @@ -411,16 +405,17 @@ class BondIf(Interface): return self.set_interface('bond_system_mac', mac) def update(self, config): - """ General helper function which works on a dictionary retrived by + """ General helper function which works on a dictionary retrieved by get_config_dict(). It's main intention is to consolidate the scattered - interface setup code and provide a single point of entry when workin + interface setup code and provide a single point of entry when working on any interface. """ - # use ref-counting function to place an interface into admin down state. + # Use ref-counting function to place an interface into admin down state. # set_admin_state_up() must be called the same amount of times else the - # interface won't come up. This can/should be used to prevent link flapping - # when changing interface parameters require the interface to be down. - # We will disable it once before reconfiguration and enable it afterwards. + # interface won't come up. This can/should be used to prevent link + # flapping when changing interface parameters require the interface to + # be down. We will disable it once before reconfiguration and enable it + # afterwards. if 'shutdown_required' in config: self.set_admin_state('down') @@ -438,19 +433,17 @@ class BondIf(Interface): # Some interface options can only be changed if the interface is # administratively down + # + # We can not move the upper "shutdown_required" code path here - as this + # would break initial bond creation and initial mode assignment during + # interface creation! if self.get_admin_state() == 'down': - # Remove ALL bond member interfaces - for interface in self.get_slaves(): - self.del_port(interface) - - # Restore correct interface status based on config - if dict_search(f'member.interface.{interface}.disable', config) is not None or \ - dict_search(f'member.interface_remove.{interface}.disable', config) is not None: - Interface(interface).set_admin_state('down') - else: - Interface(interface).set_admin_state('up') - # Bonding policy/mode - default value, always present + # + # Changing bond operation mode can only happen when there is no + # physical member interface associated with the bond itself! + for member in self.get_members(): + self.del_port(member) self.set_mode(config['mode']) # LACPDU transmission rate - default value @@ -467,10 +460,10 @@ class BondIf(Interface): # result in the following exception: OSError: [Errno 22] Invalid argument. # # We remove ALL addresses prior to adding new ones, this will remove - # addresses manually added by the user too - but as we are limited to 16 adresses + # addresses manually added by the user too - but as we are limited to 16 addresses # from the kernel side this looks valid to me. We won't run into an error - # when a user added manual adresses which would result in having more - # then 16 adresses in total. + # when a user added manual addresses which would result in having more + # then 16 addresses in total. arp_tgt_addr = list(map(str, self.get_arp_ip_target().split())) for addr in arp_tgt_addr: self.set_arp_ip_target('-' + addr) @@ -483,17 +476,62 @@ class BondIf(Interface): for addr in value: self.set_arp_ip_target('+' + addr) - # Add (enslave) interfaces to bond - value = dict_search('member.interface', config) - for interface in (value or []): - # if we've come here we already verified the interface - # does not have an addresses configured so just flush - # any remaining ones - Interface(interface).flush_addrs() - self.add_port(interface) + # Remove bond interface members first - before adding new members to the link + bond_members = self.get_members() + # Add new interfaces to the bond first before removing no longer + # required members - this is to ensure that in theory there is always an + # active member link + is_first = True + for interface in dict_search('member.interface', config, default=[]): + # Only add interface to bond if it is not already a member + if interface in bond_members: + continue + + # Physical bond member interface instance + tmp_if = Interface(interface) + # At this point, we've confirmed that the interface has no + # configured addresses, so we can safely flush any remaining ones. + tmp_if.flush_addrs() + + # T7571: This behavior changed from Linux Kernel 5.4 (used in VyOS 1.3) + # to Kernel 6.6 (starting with VyOS 1.4). Previously, the MAC address of + # the first member interface in a bond was adopted as the bond's default MAC. + # In newer versions, a synthetic MAC address is assigned instead. + # + # Re-assign first underlay MAC address to the bond + if is_first: + self.set_mac(tmp_if.get_mac()) + is_first = False + + # Assign underlying interface to logical bond + self.add_port(interface) + bond_members.append(interface) + + # Restore correct interface status based on config + if dict_search(f'member.interface.{interface}.disable', config): + Interface(interface).set_admin_state('down') + else: + Interface(interface).set_admin_state('up') + + # Remove no longer needed interfaces from the bond - this should happen + # after adding "new" interfaces to the bond as members. + for interface in dict_search('member.interface_remove', config, default=[]): + if interface not in bond_members: + # print(f'Interface {interface} not found in bond member list') + continue + + self.del_port(interface) + bond_members.remove(interface) + + # Restore correct interface status based on config + if dict_search(f'member.interface_remove.{interface}.disable', config): + Interface(interface).set_admin_state('down') + else: + Interface(interface).set_admin_state('up') # Add system mac address for 802.3ad - default address is all zero - # mode is always present (defaultValue) + # mode is always present (defaultValue) - must happen after members got + # added to the bond if config['mode'] == '802.3ad': mac = '00:00:00:00:00:00' if 'system_mac' in config: diff --git a/python/vyos/ifconfig/bridge.py b/python/vyos/ifconfig/bridge.py index d534dade7..5f184f221 100644 --- a/python/vyos/ifconfig/bridge.py +++ b/python/vyos/ifconfig/bridge.py @@ -1,4 +1,4 @@ -# Copyright 2019-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -19,7 +19,7 @@ from vyos.utils.assertion import assert_list from vyos.utils.assertion import assert_positive from vyos.utils.dict import dict_search from vyos.utils.network import interface_exists -from vyos.configdict import get_vlan_ids +from vyos.configdict import get_vlans_ids_and_range from vyos.configdict import list_diff @Interface.register @@ -273,9 +273,9 @@ class BridgeIf(Interface): return self.set_interface('vlan_protocol', map[protocol]) def update(self, config): - """ General helper function which works on a dictionary retrived by + """ General helper function which works on a dictionary retrieved by get_config_dict(). It's main intention is to consolidate the scattered - interface setup code and provide a single point of entry when workin + interface setup code and provide a single point of entry when working on any interface. """ # Set ageing time @@ -376,11 +376,26 @@ class BridgeIf(Interface): if 'priority' in interface_config: lower.set_path_priority(interface_config['priority']) + # set BPDU guard + tmp = dict_search('bpdu_guard', interface_config) + value = '1' if (tmp != None) else '0' + lower.set_bpdu_guard(value) + + # set root guard + tmp = dict_search('root_guard', interface_config) + value = '1' if (tmp != None) else '0' + lower.set_root_guard(value) + + # set learning + disable_learning = dict_search('disable_learning', interface_config) + value = '1' if disable_learning is None else '0' + lower.set_learning(value) + if 'enable_vlan' in config: add_vlan = [] native_vlan_id = None allowed_vlan_ids= [] - cur_vlan_ids = get_vlan_ids(interface) + cur_vlan_ids = get_vlans_ids_and_range(interface) if 'native_vlan' in interface_config: vlan_id = interface_config['native_vlan'] @@ -389,14 +404,8 @@ class BridgeIf(Interface): if 'allowed_vlan' in interface_config: for vlan in interface_config['allowed_vlan']: - vlan_range = vlan.split('-') - if len(vlan_range) == 2: - for vlan_add in range(int(vlan_range[0]),int(vlan_range[1]) + 1): - add_vlan.append(str(vlan_add)) - allowed_vlan_ids.append(str(vlan_add)) - else: - add_vlan.append(vlan) - allowed_vlan_ids.append(vlan) + add_vlan.append(vlan) + allowed_vlan_ids.append(vlan) # Remove redundant VLANs from the system for vlan in list_diff(cur_vlan_ids, add_vlan): diff --git a/python/vyos/ifconfig/control.py b/python/vyos/ifconfig/control.py index a886c1b9e..1ef42f981 100644 --- a/python/vyos/ifconfig/control.py +++ b/python/vyos/ifconfig/control.py @@ -1,4 +1,4 @@ -# Copyright 2019-2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -29,15 +29,17 @@ class Control(Section): _command_get = {} _command_set = {} _signature = {} + config = {} + ifname = None def __init__(self, **kargs): - # some commands (such as operation comands - show interfaces, etc.) + # some commands (such as operation commands - show interfaces, etc.) # need to query the interface statistics. If the interface # code is used and the debugging is enabled, the screen output # will include both the command but also the debugging for that command - # to prevent this, debugging can be explicitely disabled + # to prevent this, debugging can be explicitly disabled - # if debug is not explicitely disabled the the config, enable it + # if debug is not explicitly disabled the the config, enable it self.debug = '' if kargs.get('debug', True) and debug.enabled('ifconfig'): self.debug = 'ifconfig' diff --git a/python/vyos/ifconfig/dummy.py b/python/vyos/ifconfig/dummy.py index 29a1965a3..93066c965 100644 --- a/python/vyos/ifconfig/dummy.py +++ b/python/vyos/ifconfig/dummy.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/ifconfig/ethernet.py b/python/vyos/ifconfig/ethernet.py index 93727bdf6..3c4c86f4b 100644 --- a/python/vyos/ifconfig/ethernet.py +++ b/python/vyos/ifconfig/ethernet.py @@ -1,4 +1,4 @@ -# Copyright 2019-2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -19,6 +19,7 @@ from glob import glob from vyos.base import Warning from vyos.ethtool import Ethtool +from vyos.netlink import coalesce from vyos.ifconfig import Section from vyos.ifconfig.interface import Interface from vyos.utils.dict import dict_search @@ -131,19 +132,17 @@ class EthernetIf(Interface): >>> i.remove() """ + # T7813: we do need to remove the VLAN subinterfaces first so we can + # properly stop the DHCP client and inform the DHCP server that we are + # returning the lease. + for vlan in Section.sub_interfaces(self.ifname): + Interface(vlan).remove() + if self.exists(self.ifname): # interface is placed in A/D state when removed from config! It # will remain visible for the operating system. self.set_admin_state('down') - # Remove all VLAN subinterfaces - filter with the VLAN dot - for vlan in [ - x - for x in Section.interfaces('ethernet') - if x.startswith(f'{self.ifname}.') - ]: - Interface(vlan).remove() - super().remove() def set_flow_control(self, enable): @@ -245,7 +244,7 @@ class EthernetIf(Interface): cmd += f' speed {speed} duplex {duplex} autoneg off' return self._cmd(cmd) except PermissionError: - # Some NICs do not tell that they don't suppport settings speed/duplex, + # Some NICs do not tell that they don't support settings speed/duplex, # but they do not actually support it either. # In that case it's probably better to ignore the error # than end up with a broken config. @@ -451,12 +450,80 @@ class EthernetIf(Interface): cmd = f'ethtool --set-ring {ifname} {rx_tx} {size}' output, code = self._popen(cmd) # ethtool error codes: - # 80 - value already setted + # 80 - value already set # 81 - does not possible to set value if code and code != 80: print(f'could not set "{rx_tx}" ring-buffer for {ifname}') return output + def set_interrupt_coalescing(self, params: dict): + """ + Apply ethtool coalesce settings to an interface. + + This method configures interrupt coalescing parameters for the interface + using the netlink API. + + Args: + params: dict containing any of the supported keys, e.g.: + - adaptive_rx, adaptive_tx, + - rx_usecs, rx_frames, rx_usecs_irq, rx_frames_irq, + - tx_usecs, tx_frames, tx_usecs_irq, tx_frames_irq, + - stats_block_usecs, + - pkt_rate_low, pkt_rate_high, + - rx_usecs_low, rx_frames_low, + - tx_usecs_low, tx_frames_low, + - rx_usecs_high, rx_frames_high, + - tx_usecs_high, tx_frames_high, + - sample_interval, + - cqe_mode_rx, cqe_mode_tx, + - tx_aggr_max_bytes, tx_aggr_max_frames, tx_aggr_time_usecs. + + Example: + >>> from vyos.ifconfig import EthernetIf + >>> i = EthernetIf('eth0') + >>> i.set_interrupt_coalescing({'rx_usecs': 8, 'tx_usecs': 16}) + """ + + ifname = self.config['ifname'] + output = '' + + # Nothing to apply + if not params: + return None + + # Override boolean parameters to true if they exist and supported by NIC driver + for boolean_param in coalesce.get_all_params(boolean=True): + supported = self.ethtool.check_coalesce(boolean_param) + if supported: + params[boolean_param] = boolean_param in params + + # Update interrupt coalescing parameters + try: + coalesce.set_coalesce(ifname, **params) + except coalesce.CoalesceError as e: + print(f'interrupt coalescing error: {e}') + except coalesce.GeneralNetlinkError as e: + print(f'netlink error: {e}') + + return output + + def set_channels(self, rx_tx_comb, queues): + """ + Example: + >>> from vyos.ifconfig import EthernetIf + >>> i = EthernetIf('eth0') + >>> i.set_channels('rx', 2) + """ + ifname = self.config['ifname'] + cmd = f'ethtool --set-channels {ifname} {rx_tx_comb} {queues}' + output, code = self._popen(cmd) + # ethtool error codes: + # 80 - value already set + # 81 - does not possible to set value + if code and code != 80: + print(f'could not set "{rx_tx_comb}" channel for {ifname}') + return output + def set_switchdev(self, enable): ifname = self.config['ifname'] addr, code = self._popen( @@ -483,9 +550,9 @@ class EthernetIf(Interface): self._cmd(f'/sbin/devlink dev eswitch set pci/{addr} mode legacy') def update(self, config): - """General helper function which works on a dictionary retrived by + """General helper function which works on a dictionary retrieved by get_config_dict(). It's main intention is to consolidate the scattered - interface setup code and provide a single point of entry when workin + interface setup code and provide a single point of entry when working on any interface.""" # disable ethernet flow control (pause frames) @@ -528,6 +595,10 @@ class EthernetIf(Interface): for rx_tx, size in config['ring_buffer'].items(): self.set_ring_buffer(rx_tx, size) + # Set coalesce settings for the interface + if 'interrupt_coalescing' in config: + self.set_interrupt_coalescing(config['interrupt_coalescing']) + self.set_switchdev('switchdev' in config) # call base class last diff --git a/python/vyos/ifconfig/geneve.py b/python/vyos/ifconfig/geneve.py index f53ef4166..7c5b7c0fb 100644 --- a/python/vyos/ifconfig/geneve.py +++ b/python/vyos/ifconfig/geneve.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/ifconfig/input.py b/python/vyos/ifconfig/input.py index 201d3cacb..6cb1eb64c 100644 --- a/python/vyos/ifconfig/input.py +++ b/python/vyos/ifconfig/input.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/ifconfig/interface.py b/python/vyos/ifconfig/interface.py index 979b62578..9cb76ee05 100644 --- a/python/vyos/ifconfig/interface.py +++ b/python/vyos/ifconfig/interface.py @@ -1,4 +1,4 @@ -# Copyright 2019-2025 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -22,10 +22,10 @@ from copy import deepcopy from glob import glob from ipaddress import IPv4Network -from netifaces import ifaddresses -# this is not the same as socket.AF_INET/INET6 -from netifaces import AF_INET -from netifaces import AF_INET6 +from ipaddress import IPv6Interface +from netifaces import ifaddresses # pylint: disable = no-name-in-module +from socket import AF_INET +from socket import AF_INET6 from netaddr import EUI from netaddr import mac_unix_expanded @@ -48,6 +48,7 @@ from vyos.utils.network import get_interface_namespace from vyos.utils.network import get_vrf_tableid from vyos.utils.network import is_netns_interface from vyos.utils.process import is_systemd_service_active +from vyos.utils.process import stop_systemd_unit from vyos.utils.process import run from vyos.utils.file import read_file from vyos.utils.file import write_file @@ -216,6 +217,21 @@ class Interface(Control): 'location': '/sys/class/net/{ifname}/brport/priority', 'errormsg': '{ifname} is not a bridge port member' }, + 'bpdu_guard': { + 'validate': assert_boolean, + 'location': '/sys/class/net/{ifname}/brport/bpdu_guard', + 'errormsg': '{ifname} is not a bridge port member' + }, + 'root_guard': { + 'validate': assert_boolean, + 'location': '/sys/class/net/{ifname}/brport/root_block', + 'errormsg': '{ifname} is not a bridge port member' + }, + 'learning': { + 'validate': assert_boolean, + 'location': '/sys/class/net/{ifname}/brport/learning', + 'errormsg': '{ifname} is not a bridge port member' + }, 'proxy_arp': { 'validate': assert_boolean, 'location': '/proc/sys/net/ipv4/conf/{ifname}/proxy_arp', @@ -361,7 +377,7 @@ class Interface(Control): if 'netns' in self.config: cmd = f'ip netns exec {netns} {cmd}' self._cmd(cmd) - def remove(self): + def remove(self, skip_delete=False): """ Remove interface from operating system. Removing the interface deconfigures all assigned IP addresses and clear possible DHCP(v6) @@ -373,17 +389,25 @@ class Interface(Control): >>> i.remove() """ # Stop WPA supplicant if EAPoL was in use - if is_systemd_service_active(f'wpa_supplicant-wired@{self.ifname}'): - self._cmd(f'systemctl stop wpa_supplicant-wired@{self.ifname}') + netns = self.config['netns'] if 'netns' in self.config else None + stop_systemd_unit(f'wpa_supplicant-wired@{self.ifname}', netns=netns) # remove all assigned IP addresses from interface - this is a bit redundant # as the kernel will remove all addresses on interface deletion, but we - # can not delete ALL interfaces, see below + # can not delete ALL interfaces, see below. + # + # This will internally stop DHCP(v6) if running self.flush_addrs() # remove interface from conntrack VRF interface map self._del_interface_from_ct_iface_map() + # Some interfaces - mainly veth pairs - should be properly de-configured + # but not deleted. Deleting one veth pair member will delete the other, + # we need need a way to skip the deletion. + if skip_delete: + return + # --------------------------------------------------------------------- # Any class can define an eternal regex in its definition # interface matching the regex will not be deleted @@ -406,17 +430,17 @@ class Interface(Control): def _nft_check_and_run(self, nft_command): # Check if deleting is possible first to avoid raising errors - _, err = self._popen(f'nft --check {nft_command}') + _, err = self._popen(f'nft --check {nft_command} 2>/dev/null') if not err: # Remove map element self._cmd(f'nft {nft_command}') def _del_interface_from_ct_iface_map(self): - nft_command = f'delete element inet vrf_zones ct_iface_map {{ "{self.ifname}" }}' + nft_command = f'delete element inet vrf_zones ct_iface_map {{ \'"{self.ifname}"\' }}' self._nft_check_and_run(nft_command) def _add_interface_to_ct_iface_map(self, vrf_table_id: int): - nft_command = f'add element inet vrf_zones ct_iface_map {{ "{self.ifname}" : {vrf_table_id} }}' + nft_command = f'add element inet vrf_zones ct_iface_map {{ \'"{self.ifname}"\' : {vrf_table_id} }}' self._nft_check_and_run(nft_command) def get_ifindex(self): @@ -480,7 +504,7 @@ class Interface(Control): def get_mac(self): """ - Get current interface MAC (Media Access Contrl) address used. + Get current interface MAC (Media Access Control) address used. Example: >>> from vyos.ifconfig import Interface @@ -536,7 +560,7 @@ class Interface(Control): def set_mac(self, mac): """ - Set interface MAC (Media Access Contrl) address to given value. + Set interface MAC (Media Access Control) address to given value. Example: >>> from vyos.ifconfig import Interface @@ -554,7 +578,7 @@ class Interface(Control): self.set_interface('mac', mac) - # Turn an interface to the 'up' state if it was changed to 'down' by this fucntion + # Turn an interface to the 'up' state if it was changed to 'down' by this function if prev_state == 'up': self.set_admin_state('up') @@ -609,7 +633,7 @@ class Interface(Control): if 'netns' in self.config: return False - tmp = self.get_interface('vrf') + tmp = self.get_vrf() if tmp == vrf: return False @@ -909,7 +933,11 @@ class Interface(Control): tmp = self.get_interface('ipv6_autoconf') if tmp == autoconf: return None - return self.set_interface('ipv6_autoconf', autoconf) + rc = self.set_interface('ipv6_autoconf', autoconf) + if autoconf == '0': + flushed = self.flush_ipv6_slaac_addrs() + self.flush_ipv6_slaac_routes(ra_addrs=flushed) + return rc def add_ipv6_eui64_address(self, prefix): """ @@ -937,6 +965,20 @@ class Interface(Control): prefixlen = prefix.split('/')[1] self.del_addr(f'{eui64}/{prefixlen}') + def set_ipv6_interface_identifier(self, identifier): + """ + Set the interface identifier for IPv6 autoconf. + """ + cmd = f'ip token set {identifier} dev {self.ifname}' + self._cmd(cmd) + + def del_ipv6_interface_identifier(self): + """ + Delete the interface identifier for IPv6 autoconf. + """ + cmd = f'ip token delete dev {self.ifname}' + self._cmd(cmd) + def set_ipv6_forwarding(self, forwarding): """ Configure IPv6 interface-specific Host/Router behaviour. @@ -1087,6 +1129,28 @@ class Interface(Control): """ self.set_interface('path_priority', priority) + def set_bpdu_guard(self, state): + """ + Set BPDU guard state for a bridge port. When enabled, the port will be + disabled if it receives a BPDU packet. + + Example: + >>> from vyos.ifconfig import Interface + >>> Interface('eth0').set_bpdu_guard(1) + """ + self.set_interface('bpdu_guard', state) + + def set_root_guard(self, state): + """ + Set root guard state for a bridge port. When enabled, the port will be + disabled if it receives a superior BPDU that would make it a root port. + + Example: + >>> from vyos.ifconfig import Interface + >>> Interface('eth0').set_root_guard(1) + """ + self.set_interface('root_guard', state) + def set_port_isolation(self, on_or_off): """ Controls whether a given port will be isolated, which means it will be @@ -1101,6 +1165,18 @@ class Interface(Control): """ self.set_interface('bridge_port_isolation', on_or_off) + def set_learning(self, state): + """ + Set MAC address learning state on a bridge port. When disabled, + the bridge will not learn source MAC addresses from incoming frames on + this port, causing all unknown unicast traffic to be flooded. + + Example: + >>> from vyos.ifconfig import Interface + >>> Interface('eth0').set_learning(0) + """ + self.set_interface('learning', state) + def set_proxy_arp(self, enable): """ Set per interface proxy ARP configuration @@ -1304,12 +1380,80 @@ class Interface(Control): self.set_dhcp(False) self.set_dhcpv6(False) + if not self.exists(self.ifname): + return + netns = get_interface_namespace(self.ifname) netns_cmd = f'ip netns exec {netns}' if netns else '' cmd = f'{netns_cmd} ip addr flush dev {self.ifname}' # flush all addresses self._cmd(cmd) + def flush_ipv6_slaac_addrs(self) -> list: + """ + Flush all IPv6 addresses installed in response to router advertisement + messages from this interface. + + Will raise an exception on error. + Will return a list of flushed IPv6 addresses. + """ + netns = get_interface_namespace(self.ifname) + netns_cmd = f'ip netns exec {netns}' if netns else '' + tmp = get_interface_address(self.ifname) + if not tmp or 'addr_info' not in tmp: + return + + # Parse interface IP addresses. Example data: + # {'family': 'inet6', 'local': '2001:db8:1111:0:250:56ff:feb3:38c5', + # 'prefixlen': 64, 'scope': 'global', 'dynamic': True, + # 'mngtmpaddr': True, 'protocol': 'kernel_ra', + # 'valid_life_time': 2591987, 'preferred_life_time': 14387} + flushed = [] + for addr_info in tmp['addr_info']: + if 'protocol' not in addr_info: + continue + if (addr_info['protocol'] == 'kernel_ra' and + addr_info['scope'] == 'global'): + # Flush IPv6 addresses installed by router advertisement + ra_addr = f"{addr_info['local']}/{addr_info['prefixlen']}" + flushed.append(ra_addr) + cmd = f'{netns_cmd} ip -6 addr del dev {self.ifname} {ra_addr}' + self._cmd(cmd) + return flushed + + def flush_ipv6_slaac_routes(self, ra_addrs: list=[]) -> None: + """ + Flush IPv6 default routes installed in response to router advertisement + messages from this interface. + + Will raise an exception on error. + """ + # Find IPv6 connected prefixes for flushed SLAAC addresses + connected = [] + for addr in ra_addrs if isinstance(ra_addrs, list) else []: + connected.append(str(IPv6Interface(addr).network)) + + netns = get_interface_namespace(self.ifname) + netns_cmd = f'ip netns exec {netns}' if netns else '' + + tmp = self._cmd(f'{netns_cmd} ip -j -6 route show dev {self.ifname}') + tmp = json.loads(tmp) + # Parse interface routes. Example data: + # {'dst': 'default', 'gateway': 'fe80::250:56ff:feb3:cdba', + # 'protocol': 'ra', 'metric': 1024, 'flags': [], 'expires': 1398, + # 'metrics': [{'hoplimit': 64}], 'pref': 'medium'} + for route in tmp: + # If it's a default route received from RA, delete it + if (dict_search('dst', route) == 'default' and + dict_search('protocol', route) == 'ra'): + self._cmd(f'{netns_cmd} ip -6 route del default via {route["gateway"]} dev {self.ifname}') + # Remove connected prefixes received from RA + if dict_search('dst', route) in connected: + # If it's a connected prefix, delete it + self._cmd(f'{netns_cmd} ip -6 route del {route["dst"]} dev {self.ifname}') + + return None + def add_to_bridge(self, bridge_dict): """ Adds the interface to the bridge with the passed port config. @@ -1320,8 +1464,6 @@ class Interface(Control): # drop all interface addresses first self.flush_addrs() - ifname = self.ifname - for bridge, bridge_config in bridge_dict.items(): # add interface to bridge - use Section.klass to get BridgeIf class Section.klass(bridge)(bridge, create=True).add_port(self.ifname) @@ -1332,12 +1474,12 @@ class Interface(Control): # set bridge port path priority if 'priority' in bridge_config: - self.set_path_cost(bridge_config['priority']) + self.set_path_priority(bridge_config['priority']) bridge_vlan_filter = Section.klass(bridge)(bridge, create=True).get_vlan_filter() if int(bridge_vlan_filter): - cur_vlan_ids = get_vlan_ids(ifname) + cur_vlan_ids = get_vlan_ids(self.ifname) add_vlan = [] native_vlan_id = None allowed_vlan_ids= [] @@ -1360,15 +1502,15 @@ class Interface(Control): # Remove redundant VLANs from the system for vlan in list_diff(cur_vlan_ids, add_vlan): - cmd = f'bridge vlan del dev {ifname} vid {vlan} master' + cmd = f'bridge vlan del dev {self.ifname} vid {vlan} master' self._cmd(cmd) for vlan in allowed_vlan_ids: - cmd = f'bridge vlan add dev {ifname} vid {vlan} master' + cmd = f'bridge vlan add dev {self.ifname} vid {vlan} master' self._cmd(cmd) # Setting native VLAN to system if native_vlan_id: - cmd = f'bridge vlan add dev {ifname} vid {native_vlan_id} pvid untagged master' + cmd = f'bridge vlan add dev {self.ifname} vid {native_vlan_id} pvid untagged master' self._cmd(cmd) def set_dhcp(self, enable: bool, vrf_changed: bool=False): @@ -1404,22 +1546,23 @@ class Interface(Control): render(systemd_override_file, 'dhcp-client/override.conf.j2', self.config) render(dhclient_config_file, 'dhcp-client/ipv4.j2', self.config) - # Reload systemd unit definitons as some options are dynamically generated + # Reload systemd unit definitions as some options are dynamically generated self._cmd('systemctl daemon-reload') + netns = self.config['netns'] if 'netns' in self.config else None # When the DHCP client is restarted a brief outage will occur, as # the old lease is released a new one is acquired (T4203). We will # only restart DHCP client if it's option changed, or if it's not # running, but it should be running (e.g. on system startup) if (vrf_changed or ('dhcp_options_changed' in self.config) or - (not is_systemd_service_active(systemd_service))): + (not is_systemd_service_active(systemd_service, netns=netns))): return self._cmd(f'systemctl restart {systemd_service}') else: - if is_systemd_service_active(systemd_service): - self._cmd(f'systemctl stop {systemd_service}') + netns = self.config['netns'] if 'netns' in self.config else None + stop_systemd_unit(systemd_service, netns=netns) - # Smoketests occationally fail if the lease is not removed from the Kernel fast enough: + # Smoketests occasionally fail if the lease is not removed from the Kernel fast enough: # AssertionError: 2 unexpectedly found in {17: [{'addr': '52:54:00:00:00:00', # 'broadcast': 'ff:ff:ff:ff:ff:ff'}], 2: [{'addr': '192.0.2.103', 'netmask': '255.255.255.0', # @@ -1447,12 +1590,11 @@ class Interface(Control): if enable not in [True, False]: raise ValueError() - ifname = self.ifname config_base = directories['dhcp6_client_dir'] - config_file = f'{config_base}/dhcp6c.{ifname}.conf' - script_file = f'/etc/wide-dhcpv6/dhcp6c.{ifname}.script' # can not live under /run b/c of noexec mount option - systemd_override_file = f'/run/systemd/system/dhcp6c@{ifname}.service.d/10-override.conf' - systemd_service = f'dhcp6c@{ifname}.service' + config_file = f'{config_base}/dhcp6c.{self.ifname}.conf' + script_file = f'/etc/wide-dhcpv6/dhcp6c.{self.ifname}.script' # can not live under /run b/c of noexec mount option + systemd_override_file = f'/run/systemd/system/dhcp6c@{self.ifname}.service.d/10-override.conf' + systemd_service = f'dhcp6c@{self.ifname}.service' # Rendered client configuration files require additional settings config = deepcopy(self.config) @@ -1464,18 +1606,19 @@ class Interface(Control): render(config_file, 'dhcp-client/ipv6.j2', config) render(script_file, 'dhcp-client/dhcp6c-script.j2', config, permission=0o755) - # Reload systemd unit definitons as some options are dynamically generated + # Reload systemd unit definitions as some options are dynamically generated self._cmd('systemctl daemon-reload') + netns = self.config['netns'] if 'netns' in self.config else None # We must ignore any return codes. This is required to enable # DHCPv6-PD for interfaces which are yet not up and running. if (vrf_changed or ('dhcpv6_options_changed' in self.config) or - (not is_systemd_service_active(systemd_service))): + (not is_systemd_service_active(systemd_service, netns=netns))): return self._popen(f'systemctl restart {systemd_service}') else: - if is_systemd_service_active(systemd_service): - self._cmd(f'systemctl stop {systemd_service}') + netns = self.config['netns'] if 'netns' in self.config else None + stop_systemd_unit(systemd_service, netns=netns) if os.path.isfile(config_file): os.remove(config_file) if os.path.isfile(script_file): @@ -1487,14 +1630,14 @@ class Interface(Control): # Please refer to the document for details # - https://man7.org/linux/man-pages/man8/tc.8.html # - https://man7.org/linux/man-pages/man8/tc-mirred.8.html - # Depening if we are the source or the target interface of the port + # Depending if we are the source or the target interface of the port # mirror we need to setup some variables. # Don't allow for netns yet if 'netns' in self.config: return None - source_if = self.config['ifname'] + source_if = self.ifname mirror_config = None if 'mirror' in self.config: @@ -1507,9 +1650,9 @@ class Interface(Control): # clear existing ingess - ignore errors (e.g. "Error: Cannot find specified # qdisc on specified device") - we simply cleanup all stuff here - if not 'traffic_policy' in self.config: - self._popen(f'tc qdisc del dev {source_if} parent ffff: 2>/dev/null'); - self._popen(f'tc qdisc del dev {source_if} parent 1: 2>/dev/null'); + if not 'qos' in self.config: + self._popen(f'tc qdisc del dev {source_if} root 2>/dev/null') + self._popen(f'tc qdisc del dev {source_if} ingress 2>/dev/null') # Apply interface mirror policy if mirror_config: @@ -1521,14 +1664,14 @@ class Interface(Control): handle = '1: root prio' parent = '1:' - # Mirror egress traffic + # Mirror traffic mirror_cmd = f'tc qdisc add dev {source_if} handle {handle}; ' # Export the mirrored traffic to the interface mirror_cmd += f'tc filter add dev {source_if} parent {parent} protocol '\ f'all prio 10 u32 match u32 0 0 flowid 1:1 action mirred '\ f'egress mirror dev {target_if}' _, err = self._popen(mirror_cmd) - if err: print('tc qdisc(filter for mirror port failed') + if err: print('tc filter for mirror port failed') # Apply interface traffic redirection policy elif 'redirect' in self.config: @@ -1539,7 +1682,7 @@ class Interface(Control): _, err = self._popen(f'tc filter add dev {source_if} parent ffff: protocol '\ f'all prio 10 u32 match u32 0 0 flowid 1:1 action mirred '\ f'egress redirect dev {target_if}') - if err: print('tc filter add for redirect failed') + if err: print('tc filter for redirect failed') def set_per_client_thread(self, enable): """ @@ -1621,9 +1764,9 @@ class Interface(Control): os.unlink(wpa_supplicant_conf) def update(self, config): - """ General helper function which works on a dictionary retrived by + """ General helper function which works on a dictionary retrieved by get_config_dict(). It's main intention is to consolidate the scattered - interface setup code and provide a single point of entry when workin + interface setup code and provide a single point of entry when working on any interface. """ if self.debug: @@ -1792,11 +1935,26 @@ class Interface(Control): value = '0' if (tmp != None) else '1' self.set_ipv6_forwarding(value) + # Delete old interface identifier + # This should be before setting the accept_ra value + old = dict_search('ipv6.address.interface_identifier_old', config) + now = dict_search('ipv6.address.interface_identifier', config) + if old and not now: + # accept_ra of ra is required to delete the interface identifier + self.set_ipv6_accept_ra('2') + self.del_ipv6_interface_identifier() + + # Set IPv6 Interface identifier + # This should be before setting the accept_ra value + tmp = dict_search('ipv6.address.interface_identifier', config) + if tmp: + # accept_ra is required to set the interface identifier + self.set_ipv6_accept_ra('2') + self.set_ipv6_interface_identifier(tmp) + # IPv6 router advertisements tmp = dict_search('ipv6.address.autoconf', config) - value = '2' if (tmp != None) else '1' - if 'dhcpv6' in new_addr: - value = '2' + value = '2' if (tmp != None) else '0' self.set_ipv6_accept_ra(value) # IPv6 address autoconfiguration diff --git a/python/vyos/ifconfig/l2tpv3.py b/python/vyos/ifconfig/l2tpv3.py index dfaa006aa..141a77e7c 100644 --- a/python/vyos/ifconfig/l2tpv3.py +++ b/python/vyos/ifconfig/l2tpv3.py @@ -1,4 +1,4 @@ -# Copyright 2019-2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -48,7 +48,7 @@ class L2TPv3If(Interface): definition = { **Interface.definition, **{ - 'section': 'l2tpeth', + 'section': 'l2tpv3', 'prefixes': ['l2tpeth', ], 'bridgeable': True, } diff --git a/python/vyos/ifconfig/loopback.py b/python/vyos/ifconfig/loopback.py index 13e8a2c50..e2cf69a15 100644 --- a/python/vyos/ifconfig/loopback.py +++ b/python/vyos/ifconfig/loopback.py @@ -1,4 +1,4 @@ -# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -56,9 +56,9 @@ class LoopbackIf(Interface): self.del_addr(addr) def update(self, config): - """ General helper function which works on a dictionary retrived by + """ General helper function which works on a dictionary retrieved by get_config_dict(). It's main intention is to consolidate the scattered - interface setup code and provide a single point of entry when workin + interface setup code and provide a single point of entry when working on any interface. """ address = config.get('address', []) diff --git a/python/vyos/ifconfig/macsec.py b/python/vyos/ifconfig/macsec.py index 3b4dc223f..4d76a1d46 100644 --- a/python/vyos/ifconfig/macsec.py +++ b/python/vyos/ifconfig/macsec.py @@ -1,4 +1,4 @@ -# Copyright 2020-2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/ifconfig/macvlan.py b/python/vyos/ifconfig/macvlan.py index fe948b920..7a26f9ef5 100644 --- a/python/vyos/ifconfig/macvlan.py +++ b/python/vyos/ifconfig/macvlan.py @@ -1,4 +1,4 @@ -# Copyright 2019-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/ifconfig/operational.py b/python/vyos/ifconfig/operational.py index dc2742123..e60518948 100644 --- a/python/vyos/ifconfig/operational.py +++ b/python/vyos/ifconfig/operational.py @@ -1,4 +1,4 @@ -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/ifconfig/pppoe.py b/python/vyos/ifconfig/pppoe.py index 85ca3877e..7c4a6dfbb 100644 --- a/python/vyos/ifconfig/pppoe.py +++ b/python/vyos/ifconfig/pppoe.py @@ -1,4 +1,4 @@ -# Copyright 2020-2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -15,6 +15,7 @@ from vyos.ifconfig.interface import Interface from vyos.utils.assertion import assert_range +from vyos.utils.dict import dict_search from vyos.utils.network import get_interface_config @Interface.register @@ -101,9 +102,9 @@ class PPPoEIf(Interface): self.set_interface('accept_ra_defrtr', enable) def update(self, config): - """ General helper function which works on a dictionary retrived by + """ General helper function which works on a dictionary retrieved by get_config_dict(). It's main intention is to consolidate the scattered - interface setup code and provide a single point of entry when workin + interface setup code and provide a single point of entry when working on any interface. """ # Cache the configuration - it will be reused inside e.g. DHCP handler @@ -139,3 +140,7 @@ class PPPoEIf(Interface): self._cmd(f'vtysh -c "conf t" {vrf} -c "ip route 0.0.0.0/0 {self.ifname} tag 210 {distance}"') if 'ipv6' in config: self._cmd(f'vtysh -c "conf t" {vrf} -c "ipv6 route ::/0 {self.ifname} tag 210 {distance}"') + + # kick RS when IPv6 is up. + if dict_search('ipv6.address.autoconf', config) is not None: + self._cmd(f'rdisc6 --single --retry 3 {self.ifname}') diff --git a/python/vyos/ifconfig/section.py b/python/vyos/ifconfig/section.py index 50273cf67..c3f803c76 100644 --- a/python/vyos/ifconfig/section.py +++ b/python/vyos/ifconfig/section.py @@ -1,4 +1,4 @@ -# Copyright 2020 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -97,7 +97,9 @@ class Section: for ifname in interfaces: ifsection = cls.section(ifname) - if not ifsection and not ifname.startswith('vrrp'): + if not ifsection and not ( + ifname.startswith('vrrp') or ifname.startswith('vpp') + ): continue if section and ifsection != section: @@ -144,10 +146,18 @@ class Section: if no section is provided, then it returns all configured interfaces. If vlan is True, also Vlan subinterfaces will be returned """ - return cls._sort_interfaces(cls._intf_under_section(section, vlan)) @classmethod + def sub_interfaces(cls, interface : str) -> list: + """ + return a list of subinterfaces (e.g. VLAN) derived from a given interface + """ + if_type = cls.section(interface) + res = [x for x in cls.interfaces(if_type) if x.startswith(f'{interface}.')] + return res + + @classmethod def _intf_with_feature(cls, feature=''): """ return a generator with the name of the configured interface which have @@ -176,7 +186,7 @@ class Section: return list(cls._prefixes.keys()) @classmethod - def get_config_path(cls, name): + def get_config_path(cls, name, delimiter=' '): """ get config path to interface with .vif or .vif-s.vif-c example: eth0.1.2 -> 'ethernet eth0 vif-s 1 vif-c 2' @@ -185,11 +195,11 @@ class Section: sect = cls.section(name) if sect: splinterface = name.split('.') - intfpath = f'{sect} {splinterface[0]}' + intfpath = f'{sect}{delimiter}{splinterface[0]}' if len(splinterface) == 2: - intfpath += f' vif {splinterface[1]}' + intfpath += f'{delimiter}vif{delimiter}{splinterface[1]}' elif len(splinterface) == 3: - intfpath += f' vif-s {splinterface[1]} vif-c {splinterface[2]}' + intfpath += f'{delimiter}vif-s{delimiter}{splinterface[1]}{delimiter}vif-c{delimiter}{splinterface[2]}' return intfpath else: return False diff --git a/python/vyos/ifconfig/sstpc.py b/python/vyos/ifconfig/sstpc.py index d92ef23dc..e43a2f177 100644 --- a/python/vyos/ifconfig/sstpc.py +++ b/python/vyos/ifconfig/sstpc.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/ifconfig/tunnel.py b/python/vyos/ifconfig/tunnel.py index df904f7d5..befaed8fd 100644 --- a/python/vyos/ifconfig/tunnel.py +++ b/python/vyos/ifconfig/tunnel.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -161,9 +161,9 @@ class TunnelIf(Interface): return self.set_interface('multicast', enable) def update(self, config): - """ General helper function which works on a dictionary retrived by + """ General helper function which works on a dictionary retrieved by get_config_dict(). It's main intention is to consolidate the scattered - interface setup code and provide a single point of entry when workin + interface setup code and provide a single point of entry when working on any interface. """ # Adjust iproute2 tunnel parameters if necessary self._change_options() diff --git a/python/vyos/ifconfig/veth.py b/python/vyos/ifconfig/veth.py index 2c8709d20..9868ea526 100644 --- a/python/vyos/ifconfig/veth.py +++ b/python/vyos/ifconfig/veth.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -14,7 +14,8 @@ # License along with this library. If not, see <http://www.gnu.org/licenses/>. from vyos.ifconfig.interface import Interface - +from vyos.utils.dict import dict_search +from vyos.utils.network import get_interface_config @Interface.register class VethIf(Interface): @@ -51,3 +52,18 @@ class VethIf(Interface): # interface is always A/D down. It needs to be enabled explicitly self.set_admin_state('down') + + def remove(self): + # The oddity with virtual-ethernet pairs is, if you delete one - the OS + # Kernel will immediately delete the other interface, leaving no + # possibility to properly shutdown the peer-interface. + # + # We deconfigure the second veth pair interface during deletion, but + # skip it's actual call to "ip link del dev ..." + tmp = get_interface_config(self.ifname) + peer = tmp.get('link', dict_search('linkinfo.info_kind', tmp)) + if peer != None: + Interface(peer).remove(skip_delete=True) + + # always forward to the base class + super().remove() diff --git a/python/vyos/ifconfig/vpp/__init__.py b/python/vyos/ifconfig/vpp/__init__.py new file mode 100644 index 000000000..a0a1f6930 --- /dev/null +++ b/python/vyos/ifconfig/vpp/__init__.py @@ -0,0 +1,36 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from .bond import VPPBondInterface +from .bridge import VPPBridgeInterface +from .gre import VPPGREInterface +from .interface import VPPInterface +from .ipip import VPPIPIPInterface +from .loopback import VPPLoopbackInterface +from .vxlan import VPPVXLANInterface +from .xconnect import VPPXconnectInterface + +__all__ = [ + 'VPPBondInterface', + 'VPPBridgeInterface', + 'VPPGREInterface', + 'VPPInterface', + 'VPPIPIPInterface', + 'VPPLoopbackInterface', + 'VPPVXLANInterface', + 'VPPXconnectInterface', +] diff --git a/python/vyos/ifconfig/vpp/bond.py b/python/vyos/ifconfig/vpp/bond.py new file mode 100644 index 000000000..b3bcce449 --- /dev/null +++ b/python/vyos/ifconfig/vpp/bond.py @@ -0,0 +1,150 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from vyos.ifconfig import Interface +from vyos.ifconfig.vpp.interface import VPPInterface + + +class VPPBondInterface(Interface, VPPInterface): + def __init__(self, ifname, config): + self.ifname = ifname + self.instance = int(ifname.removeprefix('vppbond')) + self.vpp_ifname = f'BondEthernet{self.instance}' + + # Initialize Interface (kernel) and VPP part + super().__init__(ifname) + VPPInterface.__init__(self, self.vpp_ifname) + + self.index = self.vpp.get_sw_if_index(self.vpp_ifname) + self.state = 'up' if 'disable' not in config else 'down' + self.mode = config.get('mode') + self.load_balance = config.get('hash_policy') + self.mac = config.get('mac') + + def _create(self): + pass + + def add_bond(self): + """Create Bond interface + https://github.com/FDio/vpp/blob/stable/2306/src/vnet/bonding/bond.api + Example: + from vyos.ifconfig.vpp import VPPBondInterface + a = BondInterface(ifname='vppbond0', config) + a.add_bond() + """ + # Create interface 'BondEthernetX' + create_args = { + 'id': self.instance, + 'mode': self.mode, + 'lb': self.load_balance, + } + if self.mac: + create_args.update({'use_custom_mac': True, 'mac_address': self.mac}) + self.vpp.api.bond_create2(**create_args) + # Add LCP pair (kernel) interface + self.kernel_add() + # Set interface state + self.set_state(self.state) + self.set_admin_state(self.state) + self.index = self.vpp.get_sw_if_index(self.vpp_ifname) + + def delete_bond(self): + """Delete Bond interface + Example: + from vyos.ifconfig.vpp import VPPBondInterface + a = VPPBondInterface(ifname='vppbond0', config) + a.delete_bond() + """ + self.vpp.api.bond_delete(sw_if_index=self.index) + + def add_member(self, interface): + """Add member to Bond interface + Example: + from vyos.ifconfig.vpp import VPPBondInterface + a = VPPBondInterface(ifname='vppbond0', config) + a.add_member(interface='eth0') + """ + member_if_index = self.vpp.get_sw_if_index(interface) + self.vpp.api.bond_add_member( + bond_sw_if_index=self.index, sw_if_index=member_if_index + ) + self.vpp.api.sw_interface_set_promisc( + sw_if_index=member_if_index, promisc_on=True + ) + + def detach_member(self, interface): + """Detach member from Bond interface + Example: + from vyos.ifconfig.vpp import VPPBondInterface + a = VPPBondInterface(ifname='vppbond0') + a.detach_member(interface='eth0') + """ + member_if_index = self.vpp.get_sw_if_index(interface) + self.vpp.api.bond_detach_member(sw_if_index=member_if_index) + + def get_members(self): + members = [] + tmp = self.vpp.api.sw_member_interface_dump(sw_if_index=self.index) + for member in tmp: + members.append(member.interface_name) + return members + + def kernel_add(self): + """Add LCP pair + Example: + from vyos.ifconfig.vpp import VPPBondInterface + a = VPPBondInterface(ifname='vppbond0') + a.kernel_add() + """ + self.vpp.lcp_pair_add(self.vpp_ifname, self.ifname) + + def kernel_delete(self): + """Delete LCP pair + Example: + from vyos.ifconfig.vpp import VPPBondInterface + a = VPPBondInterface(ifname='vppbond0') + a.kernel_delete() + """ + self.vpp.lcp_pair_del(self.vpp_ifname, self.ifname) + + def remove(self): + if self.index: + # Detach all existing members + members = self.get_members() + for member in members: + self.detach_member(interface=member) + + # Delete lcp pair interface + if self.vpp.lcp_pair_find(vpp_name_hw=self.vpp_ifname): + self.kernel_delete() + + # Delete bonding interface + self.delete_bond() + + def update(self, config): + # Add bond interface + self.add_bond() + + # Add members to bond + for member in config.get('member', {}).get('interface', []): + self.add_member(interface=member) + + # Apply VPP-specific interface settings + VPPInterface.update(self, config) + + # Apply all settings to the lcp pair (kernel) interface + super().update(config) diff --git a/python/vyos/ifconfig/vpp/bridge.py b/python/vyos/ifconfig/vpp/bridge.py new file mode 100644 index 000000000..379163501 --- /dev/null +++ b/python/vyos/ifconfig/vpp/bridge.py @@ -0,0 +1,140 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from vyos.ifconfig.vpp.interface import VPPInterface +from vyos.utils.dict import dict_search +from vyos.vpp.utils import iftunnel_transform + + +class VPPBridgeInterface(VPPInterface): + def __init__(self, ifname): + self.instance = int(ifname.removeprefix('vppbr')) + self.vpp_ifname = f'br{self.instance}' + + super().__init__(self.vpp_ifname) + + def add(self): + """Create Bridge interface + https://github.com/FDio/vpp/blob/stable/2306/src/vnet/l2/l2.api + + Bridge-domain 0 is reserved for the default bridge-domain. + + Example: + from vyos.ifconfig.vpp import VPPBridgeInterface + a = VPPBridgeInterface(ifname='br23') + a.add() + """ + self.vpp.api.bridge_domain_add_del_v2( + is_add=True, + bd_id=self.instance, + flood=True, + forward=True, + learn=True, + uu_flood=True, + arp_term=False, + ) + + def delete(self): + """Delete Bridge interface + + Bridge-members must be detached before deleting the bridge interface. + + Example: + from vyos.ifconfig.vpp import VPPBridgeInterface + a = VPPBridgeInterface(ifname='br23') + a.delete() + """ + self.vpp.api.bridge_domain_add_del_v2(is_add=False, bd_id=self.instance) + + def get_members(self): + bridge = self.vpp.api.bridge_domain_dump(bd_id=self.instance)[0] + members = [] + for member in bridge.sw_if_details: + members.append(member.sw_if_index) + return members + + def add_member(self, member: str | int, port_type: int = 0): + """Add member to Bridge interface + + Attaches a VPP interface to the Bridge interface specified by `interface_suffix`. + The `member` parameter can be either the name (str) or the index (int) of the network + VPP interface to be added as a member to the bridge. + + Args: + member (str or int): The name or index of the VPP network interface + to be added as a member to the bridge. + port_type: 0 - Normal port, 1 - BVI port + + Example: + from vyos.ifconfig.vpp import VPPBridgeInterface + a = VPPBridgeInterface(ifname='br23') + a.add_member(member='eth0') + """ + member_if_index = self.vpp.get_sw_if_index(member) + return self.vpp.api.sw_interface_set_l2_bridge( + rx_sw_if_index=member_if_index, bd_id=self.instance, port_type=port_type + ) + + def detach_member(self, member: int): + """Detach member from Bridge interface. + Bridge-domain 0 is reserved for the default bridge-domain. + The `member` parameter can be either the name (str) or the index (int) + of the network VPP interface + + Args: + member (str or int): The name or index of the VPP network interface + to be detached from the bridge. + + Example: + from vyos.ifconfig.vpp import VPPBridgeInterface + a = VPPBridgeInterface(ifname='br23') + a.detach_member(member='eth0') + """ + # enable=0, 0 = Enable L3 mode + return self.vpp.api.sw_interface_set_l2_bridge( + rx_sw_if_index=member, bd_id=0, port_type=0, enable=0 + ) + + def remove(self): + if self.vpp.api.bridge_domain_dump(bd_id=self.instance): + # Detach all existing members + members = self.get_members() + for member in members: + self.detach_member(member=member) + + # Delete bridge interface + self.delete() + + def update(self, config): + # Add bridge interface + self.add() + + # Add members to bridge + for member, member_config in dict_search( + 'member.interface', config, {} + ).items(): + member = member.removeprefix('vpp') + if member.startswith('vxlan'): + member = iftunnel_transform(member) + elif member.startswith('lo'): + # interface name in VPP is loopX + member = member.replace('lo', 'loop') + elif member.startswith('bond'): + # interface name in VPP is BondEthernetX + member = member.replace('bond', 'BondEthernet') + port = 1 if 'bvi' in member_config else 0 + self.add_member(member=member, port_type=port) diff --git a/python/vyos/ifconfig/vpp/gre.py b/python/vyos/ifconfig/vpp/gre.py new file mode 100644 index 000000000..4cdb27213 --- /dev/null +++ b/python/vyos/ifconfig/vpp/gre.py @@ -0,0 +1,142 @@ +# VyOS implementation of VPP GRE interface +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from vyos.ifconfig import Interface +from vyos.ifconfig.vpp.interface import VPPInterface + + +class VPPGREInterface(Interface, VPPInterface): + """ + Class representing a GRE (Generic Routing Encapsulation) interface. + """ + + # Mapping of tunnel types https://github.com/FDio/vpp/blob/stable/2406/src/plugins/gre/gre.api#L25-L35 + TUNNEL_TYPE_MAP = { + 'l3': 0, + 'teb': 1, + 'erspan': 2, + } + + MODE_MAP = { + 'point-to-point': 0, + # 'point-to-multipoint': 1, + } + + def __init__(self, ifname, config): + self.ifname = ifname + self.instance = int(ifname.removeprefix('vppgre')) + self.vpp_ifname = f'gre{self.instance}' + + # Initialize Interface (kernel) and VPP part + super().__init__(ifname) + VPPInterface.__init__(self, self.vpp_ifname) + + self.index = self.vpp.get_sw_if_index(self.vpp_ifname) + self.state = 'up' if 'disable' not in config else 'down' + self.src_address = config.get('source_address') + self.dst_address = config.get('remote') + self.tunnel_type = self.TUNNEL_TYPE_MAP.get(config.get('tunnel_type'), 0) + self.mode = self.MODE_MAP['point-to-point'] + self.key = int(config.get('key', 0)) + + def _create(self): + pass + + def get_gre(self): + tunnels = self.vpp.api.gre_tunnel_dump_v2(sw_if_index=self.index) + return tunnels if tunnels else None + + def add_gre(self): + """Create GRE interface + https://github.com/FDio/vpp/blob/stable/2406/src/plugins/gre/gre.api + Example: + from vyos.ifconfig.vpp import VPPGREInterface + a = VPPGREInterface(ifname='vppgre0', config) + a.add_gre() + """ + self.vpp.api.gre_tunnel_add_del_v2( + is_add=True, + tunnel={ + 'src': self.src_address, + 'dst': self.dst_address, + 'instance': self.instance, + 'mode': self.mode, + 'type': self.tunnel_type, + 'key': self.key, + }, + ) + # Add LCP pair (kernel) interface + self.kernel_add() + # Set interface state + self.set_state(self.state) + self.set_admin_state(self.state) + self.index = self.vpp.get_sw_if_index(self.vpp_ifname) + + def delete_gre(self): + """Delete GRE interface + Example: + from vyos.ifconfig.vpp import VPPGREInterface + a = VPPGREInterface(ifname='vppgre0', config) + a.delete_gre() + """ + gre = self.get_gre() + if gre: + return self.vpp.api.gre_tunnel_add_del_v2( + is_add=False, + tunnel={ + 'src': gre.tunnel.src, + 'dst': gre.tunnel.dst, + 'key': gre.tunnel.key, + }, + ) + + def kernel_add(self): + """Add LCP pair + Example: + from vyos.ifconfig.vpp import VPPGREInterface + a = VPPGREInterface(ifname='vppgre0', config) + a.kernel_add() + """ + self.vpp.lcp_pair_add(self.vpp_ifname, self.ifname, 'tun') + + def kernel_delete(self): + """Delete LCP pair + Example: + from vyos.ifconfig.vpp import VPPGREInterface + a = VPPGREInterface(ifname='vppgre0', config) + a.kernel_delete() + """ + self.vpp.lcp_pair_del(self.vpp_ifname, self.ifname) + + def remove(self): + if self.index: + # Delete lcp pair interface + if self.vpp.lcp_pair_find(vpp_name_hw=self.vpp_ifname): + self.kernel_delete() + + # Delete gre interface + self.delete_gre() + + def update(self, config): + # Add gre interface + self.add_gre() + + # Apply VPP-specific interface settings + VPPInterface.update(self, config) + + super().update(config) diff --git a/python/vyos/ifconfig/vpp/interface.py b/python/vyos/ifconfig/vpp/interface.py new file mode 100644 index 000000000..17be83814 --- /dev/null +++ b/python/vyos/ifconfig/vpp/interface.py @@ -0,0 +1,78 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from vyos.vpp import VPPControl + + +class VPPInterface: + def __init__(self, vpp_ifname): + self.vpp_ifname = vpp_ifname + self.vpp = VPPControl() + + def set_state(self, state: str): + """Set interface state to UP or DOWN + Args: + state (str): The state of the interface. Options are 'up' and 'down'. + Example: + from vyos.interface.vpp import VPPInterface + a = VPPInterface(vpp_ifname='eth0') + a.set_state(state='up') + """ + if state not in ['up', 'down']: + raise ValueError(f"Invalid state: {state}") + state_flag = 1 if state == 'up' else 0 + if_index = self.vpp.get_sw_if_index(self.vpp_ifname) + self.vpp.api.sw_interface_set_flags(sw_if_index=if_index, flags=state_flag) + + def get_state(self): + """Get interface state + Example: + from vyos.interface.vpp import VPPInterface + a = VPPInterface(vpp_ifname='eth0') + a.get_state() + """ + if_index = self.vpp.get_sw_if_index(self.vpp_ifname) + return self.vpp.api.sw_interface_dump(sw_if_index=if_index)[0]['flags'] + + def set_rx_mode(self, rx_mode): + lcp_pair = self.vpp.lcp_pair_find(vpp_name_hw=self.vpp_ifname) + if lcp_pair: + lcp_name = lcp_pair.get('vpp_name_kernel') + if lcp_name: + self.vpp.iface_rxmode(lcp_name, rx_mode) + + def set_mtu_vpp(self, mtu): + # Set MTU for the VPP interface + self.vpp.set_iface_mtu(self.vpp_ifname, mtu) + + # Set MTU for the LCP pair interface + lcp_pair = self.vpp.lcp_pair_find(vpp_name_hw=self.vpp_ifname) + if lcp_pair: + lcp_name = lcp_pair.get('vpp_name_kernel') + if lcp_name: + self.vpp.set_iface_mtu(lcp_name, mtu) + + def update(self, config): + # Set MTU + if 'mtu' in config: + mtu = int(config['mtu']) + self.set_mtu_vpp(mtu) + + # Set rx-mode + rx_mode = config.get('vpp_settings', {}).get('interface_rx_mode') + if rx_mode: + self.set_rx_mode(rx_mode) diff --git a/python/vyos/ifconfig/vpp/ipip.py b/python/vyos/ifconfig/vpp/ipip.py new file mode 100644 index 000000000..4605cfa8f --- /dev/null +++ b/python/vyos/ifconfig/vpp/ipip.py @@ -0,0 +1,107 @@ +# VyOS implementation of VPP IPIP interface +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from vyos.ifconfig import Interface +from vyos.ifconfig.vpp.interface import VPPInterface + + +class VPPIPIPInterface(Interface, VPPInterface): + def __init__(self, ifname, config): + self.ifname = ifname + self.instance = int(ifname.removeprefix('vppipip')) + self.vpp_ifname = self.ifname.removeprefix('vpp') + + # Initialize Interface (kernel) and VPP part + super().__init__(ifname) + VPPInterface.__init__(self, self.vpp_ifname) + + self.index = self.vpp.get_sw_if_index(self.vpp_ifname) + self.src_address = config.get('source_address') + self.dst_address = config.get('remote') + self.state = 'up' if 'disable' not in config else 'down' + + def _create(self): + pass + + def add_ipip(self): + """Create IPIP interface + https://github.com/FDio/vpp/blob/stable/2310/src/vnet/ipip/ipip.api + Example: + from vyos.ifconfig.vpp import VPPIPIPInterface + a = VPPIPIPInterface(ifname='vppipip0', config) + a.add_ipip() + """ + self.vpp.api.ipip_add_tunnel( + tunnel={ + 'src': self.src_address, + 'dst': self.dst_address, + 'instance': self.instance, + }, + ) + # Add LCP pair (kernel) interface + self.kernel_add() + # Set interface state + self.set_state(self.state) + self.set_admin_state(self.state) + self.index = self.vpp.get_sw_if_index(self.vpp_ifname) + + def delete_ipip(self): + """Delete IPIP interface + Example: + from vyos.ifconfig.vpp import VPPIPIPInterface + a = VPPIPIPInterface(ifname='vppipip0', config) + a.delete_ipip() + """ + return self.vpp.api.ipip_del_tunnel(sw_if_index=self.index) + + def kernel_add(self): + """Add LCP pair + Example: + from vyos.ifconfig.vpp import VPPIPIPInterface + a = VPPIPIPInterface(ifname='vppipip0', config) + a.kernel_add() + """ + self.vpp.lcp_pair_add(self.vpp_ifname, self.ifname, 'tun') + + def kernel_delete(self): + """Delete LCP pair + Example: + from vyos.ifconfig.vpp import VPPIPIPInterface + a = VPPIPIPInterface(ifname='vppipip0', config) + a.kernel_delete() + """ + self.vpp.lcp_pair_del(self.vpp_ifname, self.ifname) + + def remove(self): + if self.index: + # Delete lcp pair interface + if self.vpp.lcp_pair_find(vpp_name_hw=self.vpp_ifname): + self.kernel_delete() + + # Delete ipip interface + self.delete_ipip() + + def update(self, config): + # Add ipip interface + self.add_ipip() + + # Apply VPP-specific interface settings + VPPInterface.update(self, config) + + # Apply all settings to the lcp pair (kernel) interface + super().update(config) diff --git a/python/vyos/ifconfig/vpp/loopback.py b/python/vyos/ifconfig/vpp/loopback.py new file mode 100644 index 000000000..4d3325820 --- /dev/null +++ b/python/vyos/ifconfig/vpp/loopback.py @@ -0,0 +1,102 @@ +# VyOS implementation of VPP Loopback interface +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from vyos.ifconfig import Interface +from vyos.ifconfig.vpp.interface import VPPInterface + + +class VPPLoopbackInterface(Interface, VPPInterface): + """Interface Loopback""" + + def __init__(self, ifname, config): + self.ifname = ifname + self.instance = int(ifname.removeprefix('vpplo')) + self.vpp_ifname = f'loop{self.instance}' + + # Initialize Interface (kernel) and VPP part + super().__init__(ifname) + VPPInterface.__init__(self, self.vpp_ifname) + + self.index = self.vpp.get_sw_if_index(self.vpp_ifname) + self.state = 'up' if 'disable' not in config else 'down' + + def _create(self): + pass + + def add(self): + """Create Loopback interface + https://github.com/FDio/vpp/blob/stable/2306/src/vnet/interface.api + Example: + from vyos.ifconfig.vpp import VPPLoopbackInterface + a = VPPLoopbackInterface(ifname='vpplo1', config) + a.add() + """ + self.vpp.api.create_loopback_instance( + is_specified=True, user_instance=self.instance + ) + # Add LCP pair (kernel) interface + self.kernel_add() + # Set interface state + self.set_state(self.state) + self.set_admin_state(self.state) + self.index = self.vpp.get_sw_if_index(self.vpp_ifname) + + def delete(self): + """Delete Loopback interface + Example: + from vyos.ifconfig.vpp import VPPLoopbackInterface + a = VPPLoopbackInterface(ifname='vpplo1', config) + a.delete() + """ + return self.vpp.api.delete_loopback(sw_if_index=self.index) + + def kernel_add(self): + """Add LCP pair + Example: + from vyos.ifconfig.vpp import VPPLoopbackInterface + a = VPPLoopbackInterface(ifname='vpplo1') + a.kernel_add() + """ + self.vpp.lcp_pair_add(self.vpp_ifname, self.ifname) + + def kernel_delete(self): + """Delete LCP pair + Example: + from vyos.ifconfig.vpp import VPPLoopbackInterface + a = VPPLoopbackInterface(ifname='vpplo1') + a.kernel_delete() + """ + self.vpp.lcp_pair_del(self.vpp_ifname, self.ifname) + + def remove(self): + if self.index: + # Delete lcp pair interface + if self.vpp.lcp_pair_find(vpp_name_hw=self.vpp_ifname): + self.kernel_delete() + + # Delete loopback interface + self.delete() + + def update(self, config): + # Add loopback interface + self.add() + + VPPInterface.update(self, config) + + # Apply all settings to the lcp pair (kernel) interface + super().update(config) diff --git a/python/vyos/ifconfig/vpp/vxlan.py b/python/vyos/ifconfig/vpp/vxlan.py new file mode 100644 index 000000000..f09ed9be3 --- /dev/null +++ b/python/vyos/ifconfig/vpp/vxlan.py @@ -0,0 +1,122 @@ +# VyOS implementation of VPP VXLAN interface +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from vyos.ifconfig import Interface +from vyos.ifconfig.vpp.interface import VPPInterface + + +class VPPVXLANInterface(Interface, VPPInterface): + """Interface VXLAN""" + + def __init__(self, ifname, config): + self.ifname = ifname + self.instance = int(ifname.removeprefix('vppvxlan')) + self.vpp_ifname = f'vxlan_tunnel{self.instance}' + + super().__init__(ifname) + VPPInterface.__init__(self, self.vpp_ifname) + + self.index = self.vpp.get_sw_if_index(self.vpp_ifname) + self.src_address = config.get('source_address') + self.dst_address = config.get('remote') + self.vni = int(config.get('vni', 0)) + self.state = 'up' if 'disable' not in config else 'down' + + def _create(self): + pass + + def get_vxlan(self): + tunnels = self.vpp.api.vxlan_tunnel_dump(sw_if_index=self.index) + return tunnels[0] if tunnels else None + + def add_vxlan(self): + """Create VXLAN interface + https://github.com/FDio/vpp/blob/stable/2306/src/plugins/vxlan/vxlan.api + + Example: + from vyos.ifconfig.vpp import VPPVXLANInterface + a = VPPVXLANInterface(ifname='vppvxlan23', config) + a.add_vxlan() + """ + self.vpp.api.vxlan_add_del_tunnel_v3( + is_add=True, + src_address=self.src_address, + dst_address=self.dst_address, + vni=self.vni, + instance=self.instance, + decap_next_index=1, + is_l3=False, + ) + # Add LCP pair (kernel) interface + self.kernel_add() + # Set interface state + self.set_state(self.state) + self.set_admin_state(self.state) + self.index = self.vpp.get_sw_if_index(self.vpp_ifname) + + def delete_vxlan(self): + """Delete VXLAN interface + Example: + from vyos.ifconfig.vpp import VPPVXLANInterface + a = VPPVXLANInterface(ifname='vppvxlan23', config) + a.delete_vxlan() + """ + vxlan = self.get_vxlan() + if vxlan: + return self.vpp.api.vxlan_add_del_tunnel_v3( + is_add=False, + src_address=vxlan.src_address, + dst_address=vxlan.dst_address, + vni=vxlan.vni, + is_l3=False, + ) + + def kernel_add(self): + """Add LCP pair + Example: + from vyos.ifconfig.vpp import VPPVXLANInterface + a = VPPVXLANInterface(ifname='vppvxlan23', config) + a.kernel_add() + """ + self.vpp.lcp_pair_add(self.vpp_ifname, self.ifname) + + def kernel_delete(self): + """Delete LCP pair + Example: + from vyos.ifconfig.vpp import VPPVXLANInterface + a = VPPVXLANInterface(ifname='vppvxlan23', config) + a.kernel_delete() + """ + self.vpp.lcp_pair_del(self.vpp_ifname, self.ifname) + + def remove(self): + if self.index: + # Delete lcp pair interface + if self.vpp.lcp_pair_find(vpp_name_hw=self.vpp_ifname): + self.kernel_delete() + + # Delete vxlan interface + self.delete_vxlan() + + def update(self, config): + # Add vxlan interface + self.add_vxlan() + + VPPInterface.update(self, config) + + super().update(config) diff --git a/python/vyos/ifconfig/vpp/xconnect.py b/python/vyos/ifconfig/vpp/xconnect.py new file mode 100644 index 000000000..a5d7d15fd --- /dev/null +++ b/python/vyos/ifconfig/vpp/xconnect.py @@ -0,0 +1,98 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from vyos.ifconfig.vpp.interface import VPPInterface +from vyos.vpp.utils import iftunnel_transform + + +def _transform_members(members): + """ + Transform interface names to VPP names for xconnect operations. + + Args: + members (list of str): List of two interface names. + + Returns: + tuple: Transformed first and second interface names. + """ + interface_transform_filter = ('vxlan',) + first_member = members[0].removeprefix('vpp') + second_member = members[1].removeprefix('vpp') + + # Check if member in required filter to transform 'vxlanX' => 'vxlan_tunnelX' + if first_member.startswith(interface_transform_filter): + first_member = iftunnel_transform(first_member) + if second_member.startswith(interface_transform_filter): + second_member = iftunnel_transform(second_member) + + return first_member, second_member + + +class VPPXconnectInterface(VPPInterface): + def __init__(self, ifname): + self.vpp_ifname = ifname + + super().__init__(self.vpp_ifname) + + def add_l2_xconnect(self, first, second): + """Add l2 cross connect + Example: + from vyos.ifconfig.vpp import VPPXconnectInterface + a = VPPXconnectInterface(ifname='vppxcon0') + a.add_l2_xconnect(first, second) + """ + member_first_if_index = self.vpp.get_sw_if_index(first) + member_second_if_index = self.vpp.get_sw_if_index(second) + self.vpp.api.sw_interface_set_l2_xconnect( + rx_sw_if_index=member_first_if_index, + tx_sw_if_index=member_second_if_index, + enable=True, + ) + self.vpp.api.sw_interface_set_l2_xconnect( + rx_sw_if_index=member_second_if_index, + tx_sw_if_index=member_first_if_index, + enable=True, + ) + + def del_l2_xconnect(self, first, second): + """Move l2 cross connect member to mode l3 (delete xconnect) + Example: + from vyos.ifconfig.vpp import VPPXconnectInterface + a = VPPXconnectInterface(ifname='vppxcon0') + a.del_l2_xconnect(first, second) + """ + member_first_if_index = self.vpp.get_sw_if_index(first) + member_second_if_index = self.vpp.get_sw_if_index(second) + self.vpp.api.sw_interface_set_l2_xconnect( + rx_sw_if_index=member_first_if_index, + tx_sw_if_index=member_second_if_index, + enable=False, + ) + self.vpp.api.sw_interface_set_l2_xconnect( + rx_sw_if_index=member_second_if_index, + tx_sw_if_index=member_first_if_index, + enable=False, + ) + + def remove(self, members): + first, second = _transform_members(members) + self.del_l2_xconnect(first, second) + + def update(self, config): + members = config['member']['interface'] + first, second = _transform_members(members) + self.add_l2_xconnect(first, second) diff --git a/python/vyos/ifconfig/vrrp.py b/python/vyos/ifconfig/vrrp.py index 3ee22706c..4949fe571 100644 --- a/python/vyos/ifconfig/vrrp.py +++ b/python/vyos/ifconfig/vrrp.py @@ -1,4 +1,4 @@ -# Copyright 2019-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/ifconfig/vti.py b/python/vyos/ifconfig/vti.py index 78f5895f8..030aa1ed7 100644 --- a/python/vyos/ifconfig/vti.py +++ b/python/vyos/ifconfig/vti.py @@ -1,4 +1,4 @@ -# Copyright 2021-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/ifconfig/vtun.py b/python/vyos/ifconfig/vtun.py index ee790f275..e6963ce5d 100644 --- a/python/vyos/ifconfig/vtun.py +++ b/python/vyos/ifconfig/vtun.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/ifconfig/vxlan.py b/python/vyos/ifconfig/vxlan.py index 58844885b..974a53566 100644 --- a/python/vyos/ifconfig/vxlan.py +++ b/python/vyos/ifconfig/vxlan.py @@ -1,4 +1,4 @@ -# Copyright 2019-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -190,9 +190,9 @@ class VXLANIf(Interface): self._cmd(f'bridge vni add dev {self.ifname} vni {vni}') def update(self, config): - """ General helper function which works on a dictionary retrived by + """ General helper function which works on a dictionary retrieved by get_config_dict(). It's main intention is to consolidate the scattered - interface setup code and provide a single point of entry when workin + interface setup code and provide a single point of entry when working on any interface. """ # call base class last diff --git a/python/vyos/ifconfig/wireguard.py b/python/vyos/ifconfig/wireguard.py index f5217aecb..fb576bac0 100644 --- a/python/vyos/ifconfig/wireguard.py +++ b/python/vyos/ifconfig/wireguard.py @@ -1,4 +1,4 @@ -# Copyright 2019-2025 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -22,12 +22,13 @@ from tempfile import NamedTemporaryFile from hurry.filesize import size from hurry.filesize import alternative +from vyos.base import Warning from vyos.configquery import ConfigTreeQuery from vyos.ifconfig import Interface from vyos.ifconfig import Operational from vyos.template import is_ipv6 from vyos.template import is_ipv4 - +from vyos.utils.network import get_wireguard_peers class WireGuardOperational(Operational): def _dump(self): """Dump wireguard data in a python friendly way.""" @@ -51,7 +52,7 @@ class WireGuardOperational(Operational): 'private_key': None if private_key == '(none)' else private_key, 'public_key': None if public_key == '(none)' else public_key, 'listen_port': int(listen_port), - 'fw_mark': None if fw_mark == 'off' else int(fw_mark), + 'fw_mark': None if fw_mark == 'off' else int(fw_mark, 16), 'peers': {}, } else: @@ -251,92 +252,131 @@ class WireGuardIf(Interface): """Get a synthetic MAC address.""" return self.get_mac_synthetic() + def get_peer_public_keys(self, config, disabled=False): + """Get list of configured peer public keys""" + if 'peer' not in config: + return [] + + public_keys = [] + + for _, peer_config in config['peer'].items(): + if disabled == ('disable' in peer_config): + public_keys.append(peer_config['public_key']) + + return public_keys + def update(self, config): - """General helper function which works on a dictionary retrived by + """General helper function which works on a dictionary retrieved by get_config_dict(). It's main intention is to consolidate the scattered - interface setup code and provide a single point of entry when workin + interface setup code and provide a single point of entry when working on any interface.""" - tmp_file = NamedTemporaryFile('w') - tmp_file.write(config['private_key']) - tmp_file.flush() # Wireguard base command is identical for every peer base_cmd = f'wg set {self.ifname}' + interface_cmd = base_cmd if 'port' in config: interface_cmd += ' listen-port {port}' if 'fwmark' in config: interface_cmd += ' fwmark {fwmark}' - interface_cmd += f' private-key {tmp_file.name}' - interface_cmd = interface_cmd.format(**config) - # T6490: execute command to ensure interface configured - self._cmd(interface_cmd) + with NamedTemporaryFile('w') as tmp_file: + tmp_file.write(config['private_key']) + tmp_file.flush() - # If no PSK is given remove it by using /dev/null - passing keys via - # the shell (usually bash) is considered insecure, thus we use a file - no_psk_file = '/dev/null' + interface_cmd += f' private-key {tmp_file.name}' + interface_cmd = interface_cmd.format(**config) + # T6490: execute command to ensure interface configured + self._cmd(interface_cmd) + + current_peer_public_keys = get_wireguard_peers(self.ifname) + + if 'rebuild_required' in config: + # Remove all existing peers that no longer exist in config + current_public_keys = self.get_peer_public_keys(config) + cmd_remove_peers = [f' peer {public_key} remove' + for public_key in current_peer_public_keys + if public_key not in current_public_keys] + if cmd_remove_peers: + self._cmd(base_cmd + ''.join(cmd_remove_peers)) if 'peer' in config: + # Group removal of disabled peers in one command + current_disabled_peers = self.get_peer_public_keys(config, disabled=True) + cmd_disabled_peers = [f' peer {public_key} remove' + for public_key in current_disabled_peers] + if cmd_disabled_peers: + self._cmd(base_cmd + ''.join(cmd_disabled_peers)) + + peer_cmds = [] + peer_domain_cmds = [] + peer_psk_files = [] + for peer, peer_config in config['peer'].items(): # T4702: No need to configure this peer when it was explicitly # marked as disabled - also active sessions are terminated as # the public key was already removed when entering this method! if 'disable' in peer_config: - # remove peer if disabled, no error report even if peer not exists - cmd = base_cmd + ' peer {public_key} remove' - self._cmd(cmd.format(**peer_config)) continue - psk_file = no_psk_file - # start of with a fresh 'wg' command - peer_cmd = base_cmd + ' peer {public_key}' + peer_cmd = ' peer {public_key}' - try: - cmd = peer_cmd - - if 'preshared_key' in peer_config: - psk_file = '/tmp/tmp.wireguard.psk' - with open(psk_file, 'w') as f: - f.write(peer_config['preshared_key']) - cmd += f' preshared-key {psk_file}' - - # Persistent keepalive is optional - if 'persistent_keepalive' in peer_config: - cmd += ' persistent-keepalive {persistent_keepalive}' - - # Multiple allowed-ip ranges can be defined - ensure we are always - # dealing with a list - if isinstance(peer_config['allowed_ips'], str): - peer_config['allowed_ips'] = [peer_config['allowed_ips']] - cmd += ' allowed-ips ' + ','.join(peer_config['allowed_ips']) - - self._cmd(cmd.format(**peer_config)) - - cmd = peer_cmd - - # Ensure peer is created even if dns not working - if {'address', 'port'} <= set(peer_config): - if is_ipv6(peer_config['address']): - cmd += ' endpoint [{address}]:{port}' - elif is_ipv4(peer_config['address']): - cmd += ' endpoint {address}:{port}' - else: - # don't set endpoint if address uses domain name - continue - elif {'host_name', 'port'} <= set(peer_config): - cmd += ' endpoint {host_name}:{port}' - - self._cmd(cmd.format(**peer_config), env={ + cmd = peer_cmd + + if 'preshared_key' in peer_config: + with NamedTemporaryFile(mode='w', delete=False) as tmp_file: + tmp_file.write(peer_config['preshared_key']) + tmp_file.flush() + cmd += f' preshared-key {tmp_file.name}' + peer_psk_files.append(tmp_file.name) + else: + # If no PSK is given remove it by using /dev/null - passing keys via + # the shell (usually bash) is considered insecure, thus we use a file + cmd += f' preshared-key /dev/null' + + # Persistent keepalive is optional + if 'persistent_keepalive' in peer_config: + cmd += ' persistent-keepalive {persistent_keepalive}' + + # Multiple allowed-ip ranges can be defined - ensure we are always + # dealing with a list + if isinstance(peer_config['allowed_ips'], str): + peer_config['allowed_ips'] = [peer_config['allowed_ips']] + cmd += ' allowed-ips ' + ','.join(peer_config['allowed_ips']) + + peer_cmds.append(cmd.format(**peer_config)) + + cmd = peer_cmd + + # Ensure peer is created even if dns not working + if {'address', 'port'} <= set(peer_config): + if is_ipv6(peer_config['address']): + cmd += ' endpoint [{address}]:{port}' + elif is_ipv4(peer_config['address']): + cmd += ' endpoint {address}:{port}' + else: + # don't set endpoint if address uses domain name + continue + elif {'host_name', 'port'} <= set(peer_config): + cmd += ' endpoint {host_name}:{port}' + else: + continue + + peer_domain_cmds.append(cmd.format(**peer_config)) + + try: + if peer_cmds: + self._cmd(base_cmd + ''.join(peer_cmds)) + + if peer_domain_cmds: + self._cmd(base_cmd + ''.join(peer_domain_cmds), env={ 'WG_ENDPOINT_RESOLUTION_RETRIES': config['max_dns_retry']}) - except: - # todo: logging - pass - finally: - # PSK key file is not required to be stored persistently as its backed by CLI - if psk_file != no_psk_file and os.path.exists(psk_file): - os.remove(psk_file) + except Exception as e: + Warning(f'Failed to apply Wireguard peers on {self.ifname}: {e}') + finally: + for tmp in peer_psk_files: + os.unlink(tmp) # call base class super().update(config) diff --git a/python/vyos/ifconfig/wireless.py b/python/vyos/ifconfig/wireless.py index 121f56bd5..1026fcd57 100644 --- a/python/vyos/ifconfig/wireless.py +++ b/python/vyos/ifconfig/wireless.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -42,9 +42,9 @@ class WiFiIf(Interface): self._cmd(cmd) def update(self, config): - """ General helper function which works on a dictionary retrived by + """ General helper function which works on a dictionary retrieved by get_config_dict(). It's main intention is to consolidate the scattered - interface setup code and provide a single point of entry when workin + interface setup code and provide a single point of entry when working on any interface. """ # We can not call add_to_bridge() until wpa_supplicant is running, thus diff --git a/python/vyos/ifconfig/wwan.py b/python/vyos/ifconfig/wwan.py index 004a64b39..ffd3d0a67 100644 --- a/python/vyos/ifconfig/wwan.py +++ b/python/vyos/ifconfig/wwan.py @@ -1,4 +1,4 @@ -# Copyright 2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -26,6 +26,10 @@ class WWANIf(Interface): }, } + def _create(self): + # we can not create this interface as it is managed by the Kernel + pass + def remove(self): """ Remove interface from config. Removing the interface deconfigures all diff --git a/python/vyos/iflag.py b/python/vyos/iflag.py index 3ce73c1bf..179f33497 100644 --- a/python/vyos/iflag.py +++ b/python/vyos/iflag.py @@ -1,4 +1,4 @@ -# Copyright 2019-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/include/__init__.py b/python/vyos/include/__init__.py index 22e836531..ba196ffed 100644 --- a/python/vyos/include/__init__.py +++ b/python/vyos/include/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2025 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/include/uapi/__init__.py b/python/vyos/include/uapi/__init__.py index 22e836531..ba196ffed 100644 --- a/python/vyos/include/uapi/__init__.py +++ b/python/vyos/include/uapi/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2025 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/include/uapi/linux/__init__.py b/python/vyos/include/uapi/linux/__init__.py index 22e836531..ba196ffed 100644 --- a/python/vyos/include/uapi/linux/__init__.py +++ b/python/vyos/include/uapi/linux/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2025 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/include/uapi/linux/fib_rules.py b/python/vyos/include/uapi/linux/fib_rules.py index 72f0b18cb..83544f69b 100644 --- a/python/vyos/include/uapi/linux/fib_rules.py +++ b/python/vyos/include/uapi/linux/fib_rules.py @@ -1,4 +1,4 @@ -# Copyright (C) 2025 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as diff --git a/python/vyos/include/uapi/linux/icmpv6.py b/python/vyos/include/uapi/linux/icmpv6.py index 47e0c723c..cc30b76fd 100644 --- a/python/vyos/include/uapi/linux/icmpv6.py +++ b/python/vyos/include/uapi/linux/icmpv6.py @@ -1,4 +1,4 @@ -# Copyright (C) 2025 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as diff --git a/python/vyos/include/uapi/linux/if_arp.py b/python/vyos/include/uapi/linux/if_arp.py index 90cb66ebd..80c16a83d 100644 --- a/python/vyos/include/uapi/linux/if_arp.py +++ b/python/vyos/include/uapi/linux/if_arp.py @@ -1,4 +1,4 @@ -# Copyright (C) 2025 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as diff --git a/python/vyos/include/uapi/linux/lwtunnel.py b/python/vyos/include/uapi/linux/lwtunnel.py index 6797a762b..c598513a5 100644 --- a/python/vyos/include/uapi/linux/lwtunnel.py +++ b/python/vyos/include/uapi/linux/lwtunnel.py @@ -1,4 +1,4 @@ -# Copyright (C) 2025 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as diff --git a/python/vyos/include/uapi/linux/neighbour.py b/python/vyos/include/uapi/linux/neighbour.py index d5caf44b9..8878353e3 100644 --- a/python/vyos/include/uapi/linux/neighbour.py +++ b/python/vyos/include/uapi/linux/neighbour.py @@ -1,4 +1,4 @@ -# Copyright (C) 2025 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as diff --git a/python/vyos/include/uapi/linux/rtnetlink.py b/python/vyos/include/uapi/linux/rtnetlink.py index e31272460..f3778fa65 100644 --- a/python/vyos/include/uapi/linux/rtnetlink.py +++ b/python/vyos/include/uapi/linux/rtnetlink.py @@ -1,4 +1,4 @@ -# Copyright (C) 2025 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as diff --git a/python/vyos/initialsetup.py b/python/vyos/initialsetup.py index cb6b9e459..bff3adf20 100644 --- a/python/vyos/initialsetup.py +++ b/python/vyos/initialsetup.py @@ -1,7 +1,7 @@ # initialsetup -- functions for setting common values in config file, # for use in installation and first boot scripts # -# Copyright (C) 2018-2024 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or modify it under the terms of # the GNU Lesser General Public License as published by the Free Software Foundation; diff --git a/python/vyos/ioctl.py b/python/vyos/ioctl.py index 51574c1db..7f9ad226a 100644 --- a/python/vyos/ioctl.py +++ b/python/vyos/ioctl.py @@ -1,4 +1,4 @@ -# Copyright 2019-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/ipsec.py b/python/vyos/ipsec.py index 28f77565a..0d03f8cf9 100644 --- a/python/vyos/ipsec.py +++ b/python/vyos/ipsec.py @@ -1,4 +1,4 @@ -# Copyright 2020-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -65,7 +65,7 @@ def get_vici_sas_by_name(ike_name: str, tunnel: str) -> list: Find installed SAs by IKE_SA name and/or CHILD_SA name and return list with SASs info. If tunnel is not None return a list contained only - CHILD_SAs wich names equal tunnel value. + CHILD_SAs which names equal tunnel value. :param ike_name: IKE SA name :type ike_name: str :param tunnel: CHILD SA name diff --git a/python/vyos/ipt_netflow.py b/python/vyos/ipt_netflow.py new file mode 100644 index 000000000..14d2a458d --- /dev/null +++ b/python/vyos/ipt_netflow.py @@ -0,0 +1,178 @@ +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +# Package to stop/start ipt_NETFLOW kernel module + +# Provides functions stop(), start() and set_watched_iptables_interfaces() + +from vyos.utils.kernel import check_kmod +from vyos.utils.kernel import unload_kmod +from vyos.utils.process import cmd +from vyos import ConfigError + +module_name = 'ipt_NETFLOW' +iptables_ingress_netflow_table = 'raw' +iptables_ingress_netflow_chain = 'PREROUTING' +iptables_egress_netflow_table = 'mangle' +iptables_egress_netflow_chain = 'POSTROUTING' + + +# get iptables rule dict for chain in table +def _iptables_get_rules(command, chain, table): + # define list with rules + rules = [] + + # run iptables, save output and split it by lines + iptables_command = f'{command} -vn -t {table} -L {chain}' + tmp = cmd(iptables_command, message='Failed to get flows list') + lines = tmp.splitlines() + + # Sample output to parse: + # vyos@vyos:~$ sudo iptables -vn -t raw -L PREROUTING + # Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes) + # pkts bytes target prot opt in out source destination + # 0 0 NETFLOW 0 -- eth0 * 0.0.0.0/0 0.0.0.0/0 NETFLOW + + # Check that format is as expected + if len(lines) < 2: + raise ConfigError(f'Unexpected output from {command}, too few lines') + if not lines[0].startswith(f'Chain {chain}'): + raise ConfigError(f'Unexpected first line in output of {command}: "{lines[0]}"') + columns = lines[1].split() + + # parse each line and add information to list + rulenum = 0 + for current_rule in lines[2:]: + rulenum += 1 + current_rule_parsed = current_rule.split() + current_rule_parsed = { + columns[i]: current_rule_parsed[i] + for i in range(min(len(current_rule_parsed), len(columns))) + } + if current_rule_parsed.get('target', '') != 'NETFLOW': + continue + + rules.append( + { + 'interface-in': current_rule_parsed.get("in", ''), + 'interface-out': current_rule_parsed.get("out", ''), + 'table': table, + 'rulenum': rulenum, + } + ) + + # return list with rules + return rules + + +def _iptables_config(command, configured_ifaces, direction): + # define list of nftables commands to modify settings + iptables_commands = [] + + if direction == "ingress": + iptables_table = iptables_ingress_netflow_table + iptables_chain = iptables_ingress_netflow_chain + elif direction == "egress": + iptables_table = iptables_egress_netflow_table + iptables_chain = iptables_egress_netflow_chain + else: + raise ConfigError(f'_iptables_config: Unexpected direction="{direction}"') + + # prepare extended list with configured interfaces + configured_ifaces_extended = [] + for iface in configured_ifaces: + configured_ifaces_extended.append({'iface': iface}) + + # get currently configured interfaces with iptables rules + active_rules = _iptables_get_rules(command, iptables_chain, iptables_table) + + # compare current active list with configured one and delete excessive interfaces, add missed + active_ifaces = [] + interface_key = 'interface-out' if direction == "egress" else "interface-in" + rulenums_delete = [] + for rule in active_rules: + interface = rule[interface_key] + if interface not in configured_ifaces: + rulenums_delete.append(rule['rulenum']) + else: + active_ifaces.append({'iface': interface}) + + # It is important to delete rule with bigger rulenum first, so that other + # rulenums are not changed + rulenums_delete.sort(reverse=True) + for rulenum in rulenums_delete: + iptables_commands.append( + f'{command} -t {iptables_table} -D {iptables_chain} {rulenum}' + ) + + # do not create new rules for already configured interfaces + for iface in active_ifaces: + if iface in configured_ifaces_extended: + configured_ifaces_extended.remove(iface) + + # create missed rules + for iface_extended in configured_ifaces_extended: + iface = iface_extended['iface'] + iface_option = "o" if direction == "egress" else "i" + # iptables -t raw -A PREROUTING -j NETFLOW -i eth0 + rule_definition = f'{command} -t {iptables_table} -A {iptables_chain} -j NETFLOW -{iface_option} {iface}' + iptables_commands.append(rule_definition) + + # change iptables + for command in iptables_commands: + cmd(command, raising=ConfigError) + + +def _iptables_config_v4_and_v6(configured_ifaces, direction): + for command in 'iptables', 'ip6tables': + _iptables_config(command, configured_ifaces, direction) + + +def set_watched_iptables_interfaces(ingress_interfaces, egress_interfaces, ipv6=True): + """ + Update iptables and ip6tables rules so that ipt_NETFLOW watches + exact list of interfaces in ingress_interfaces for ingress table/chain + and egress_interfaces for egress table/chain + """ + commands = ['iptables'] + if ipv6: + commands.append('ip6tables') + for command in commands: + _iptables_config(command, ingress_interfaces, 'ingress') + _iptables_config(command, egress_interfaces, 'egress') + + +def stop(): + """ + Stop ipt_NETFLOW: remove all iptables rules that use it + and remove module + """ + set_watched_iptables_interfaces([], []) + + unload_kmod(module_name) + + +def start(ingress_interfaces, egress_interfaces, ipv6=True): + """ + Start ipt_NETFLOW: + + * Load ipt_NETFLOW kernel module + * Install iptables and ip6tables rules for + ingress_interfaces and egress_interfaces + """ + + check_kmod(module_name) + + set_watched_iptables_interfaces(ingress_interfaces, egress_interfaces, ipv6=ipv6) diff --git a/python/vyos/kea.py b/python/vyos/kea.py index c7947af3e..436c134c8 100644 --- a/python/vyos/kea.py +++ b/python/vyos/kea.py @@ -1,4 +1,4 @@ -# Copyright 2023-2025 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -15,17 +15,18 @@ import json import os +import re import socket from datetime import datetime from datetime import timezone +from vyos import ConfigError from vyos.template import is_ipv6 -from vyos.template import isc_static_route from vyos.template import netmask_from_cidr from vyos.utils.dict import dict_search_args from vyos.utils.file import file_permissions -from vyos.utils.process import run +from vyos.utils.process import run, rc_cmd kea4_options = { 'name_server': 'domain-name-servers', @@ -44,6 +45,9 @@ kea4_options = { 'wpad_url': 'wpad-url', 'ipv6_only_preferred': 'v6-only-preferred', 'captive_portal': 'v4-captive-portal', + 'capwap_controller': 'capwap-ac-v4', + 'interface_mtu': 'interface-mtu', + 'bootfile_name': 'boot-file-name', } kea6_options = { @@ -56,9 +60,10 @@ kea6_options = { 'nisplus_server': 'nisp-servers', 'sntp_server': 'sntp-servers', 'captive_portal': 'v6-captive-portal', + 'capwap_controller': 'capwap-ac-v6', } -kea_ctrl_socket = '/run/kea/dhcp{inet}-ctrl-socket' +kea_ctrl_socket = '/var/run/kea/dhcp{inet}{vrf_append}-ctrl-socket' def _format_hex_string(in_str): @@ -75,7 +80,7 @@ def _format_hex_string(in_str): def _find_list_of_dict_index(lst, key='ip', value=''): """ Find the index entry of list of dict matching the dict value - Exampe: + Example: % lst = [{'ip': '192.0.2.1'}, {'ip': '192.0.2.2'}] % _find_list_of_dict_index(lst, key='ip', value='192.0.2.2') % 1 @@ -84,6 +89,24 @@ def _find_list_of_dict_index(lst, key='ip', value=''): return idx +def _read_posix_timezone(tz_name): + try: + with open(f'/usr/share/zoneinfo/{tz_name}', 'rb') as f: + return f.read().split(b'\n')[-2].decode('utf-8').replace(',', '\\,') + except (FileNotFoundError, IOError, IndexError) as e: + raise ConfigError(f'Failed to read timezone data for: {tz_name}') from e + + +def kea_test_config(process: str, config_path: str) -> tuple[bool, str]: + result, output = rc_cmd(f'{process} -t {config_path}') + + if result == 0: + return (True, None) + + find = re.search(r'Error encountered:\s([^\n$]+)', output) + return (False, find[1] if find else None) + + def kea_parse_options(config): options = [] @@ -111,27 +134,24 @@ def kea_parse_options(config): default_route = '' if 'default_router' in config: - default_route = isc_static_route('0.0.0.0/0', config['default_router']) + default_route = f'0.0.0.0/0 - {config["default_router"]}' routes = [ - isc_static_route(route, route_options['next_hop']) + f'{route} - {route_options["next_hop"]}' for route, route_options in config['static_route'].items() ] options.append( { - 'name': 'rfc3442-static-route', + 'name': 'classless-static-route', 'data': ', '.join( routes if not default_route else routes + [default_route] ), } ) - options.append({'name': 'windows-static-route', 'data': ', '.join(routes)}) if 'time_zone' in config: - with open('/usr/share/zoneinfo/' + config['time_zone'], 'rb') as f: - tz_string = f.read().split(b'\n')[-2].decode('utf-8') - + tz_string = _read_posix_timezone(config['time_zone']) options.append({'name': 'pcode', 'data': tz_string}) options.append({'name': 'tcode', 'data': config['time_zone']}) @@ -139,15 +159,24 @@ def kea_parse_options(config): config, 'vendor_option', 'ubiquiti', 'unifi_controller' ) if unifi_controller: + options.append({'name': 'vendor-encapsulated-options'}) options.append( - {'name': 'unifi-controller', 'data': unifi_controller, 'space': 'ubnt'} + { + 'name': 'ubnt', + 'data': unifi_controller, + 'space': 'vendor-encapsulated-options-space', + } ) return options def kea_parse_subnet(subnet, config): - out = {'subnet': subnet, 'id': int(config['subnet_id'])} + out = { + 'subnet': subnet, + 'id': int(config['subnet_id']), + 'user-context': {'enable-ping-check': False}, + } if 'option' in config: out['option-data'] = kea_parse_options(config['option']) @@ -165,6 +194,12 @@ def kea_parse_subnet(subnet, config): out['valid-lifetime'] = int(config['lease']) out['max-valid-lifetime'] = int(config['lease']) + if 'ping_check' in config: + out['user-context']['enable-ping-check'] = True + + if 'client_class' in config: + out['client-class'] = config['client_class'] + if 'range' in config: pools = [] for num, range_config in config['range'].items(): @@ -180,6 +215,9 @@ def kea_parse_subnet(subnet, config): if 'bootfile_server' in range_config['option']: pool['next-server'] = range_config['option']['bootfile_server'] + if 'client_class' in range_config: + pool['client-class'] = range_config['client_class'] + pools.append(pool) out['pools'] = pools @@ -189,6 +227,9 @@ def kea_parse_subnet(subnet, config): if 'disable' in host_config: continue + if 'mac' not in host_config and 'duid' not in host_config: + continue + reservation = { 'hostname': host, } @@ -218,6 +259,9 @@ def kea_parse_subnet(subnet, config): reservations.append(reservation) out['reservations'] = reservations + if 'dynamic_dns_update' in config: + out.update(kea_parse_ddns_settings(config['dynamic_dns_update'])) + return out @@ -251,6 +295,11 @@ def kea6_parse_options(config): if hosts: options.append({'name': 'sip-server-dns', 'data': ', '.join(hosts)}) + if 'time_zone' in config: + tz_string = _read_posix_timezone(config['time_zone']) + options.append({'name': 'new-posix-timezone', 'data': tz_string}) + options.append({'name': 'new-tzdb-timezone', 'data': config['time_zone']}) + cisco_tftp = dict_search_args(config, 'vendor_option', 'cisco', 'tftp-server') if cisco_tftp: options.append( @@ -324,6 +373,9 @@ def kea6_parse_subnet(subnet, config): if 'disable' in host_config: continue + if 'mac' not in host_config and 'duid' not in host_config: + continue + reservation = {'hostname': host} if 'mac' in host_config: @@ -333,10 +385,10 @@ def kea6_parse_subnet(subnet, config): reservation['duid'] = host_config['duid'] if 'ipv6_address' in host_config: - reservation['ip-addresses'] = [host_config['ipv6_address']] + reservation['ip-addresses'] = host_config['ipv6_address'] if 'ipv6_prefix' in host_config: - reservation['prefixes'] = [host_config['ipv6_prefix']] + reservation['prefixes'] = host_config['ipv6_prefix'] if 'option' in host_config: reservation['option-data'] = kea6_parse_options(host_config['option']) @@ -348,8 +400,69 @@ def kea6_parse_subnet(subnet, config): return out -def _ctrl_socket_command(inet, command, args=None): - path = kea_ctrl_socket.format(inet=inet) +def kea_parse_tsig_algo(algo_spec): + translate = { + 'md5': 'HMAC-MD5', + 'sha1': 'HMAC-SHA1', + 'sha224': 'HMAC-SHA224', + 'sha256': 'HMAC-SHA256', + 'sha384': 'HMAC-SHA384', + 'sha512': 'HMAC-SHA512', + } + if algo_spec not in translate: + raise ConfigError(f'Unsupported TSIG algorithm: {algo_spec}') + return translate[algo_spec] + + +def kea_parse_enable_disable(value): + return True if value == 'enable' else False + + +def kea_parse_ddns_settings(config): + data = {} + + if send_updates := config.get('send_updates'): + data['ddns-send-updates'] = kea_parse_enable_disable(send_updates) + + if override_client_update := config.get('override_client_update'): + data['ddns-override-client-update'] = kea_parse_enable_disable( + override_client_update + ) + + if override_no_update := config.get('override_no_update'): + data['ddns-override-no-update'] = kea_parse_enable_disable(override_no_update) + + if update_on_renew := config.get('update_on_renew'): + data['ddns-update-on-renew'] = kea_parse_enable_disable(update_on_renew) + + if conflict_resolution := config.get('conflict_resolution'): + data['ddns-use-conflict-resolution'] = kea_parse_enable_disable( + conflict_resolution + ) + + if 'replace_client_name' in config: + data['ddns-replace-client-name'] = config['replace_client_name'] + if 'generated_prefix' in config: + data['ddns-generated-prefix'] = config['generated_prefix'] + if 'qualifying_suffix' in config: + data['ddns-qualifying-suffix'] = config['qualifying_suffix'] + if 'ttl_percent' in config: + data['ddns-ttl-percent'] = int(config['ttl_percent']) / 100 + if 'hostname_char_set' in config: + data['hostname-char-set'] = config['hostname_char_set'] + if 'hostname_char_replacement' in config: + data['hostname-char-replacement'] = config['hostname_char_replacement'] + + return data + + +def _ctrl_socket_command(inet, vrf_name, command, args=None): + if vrf_name: + vrf_append = f'-{vrf_name}' + else: + vrf_append = '' + + path = kea_ctrl_socket.format(inet=inet, vrf_append=vrf_append) if not os.path.exists(path): return None @@ -375,8 +488,8 @@ def _ctrl_socket_command(inet, command, args=None): return json.loads(result.decode('utf-8')) -def kea_get_leases(inet): - leases = _ctrl_socket_command(inet, f'lease{inet}-get-all') +def kea_get_leases(inet, vrf_name): + leases = _ctrl_socket_command(inet, vrf_name, f'lease{inet}-get-all') if not leases or 'result' not in leases or leases['result'] != 0: return [] @@ -386,6 +499,7 @@ def kea_get_leases(inet): def kea_add_lease( inet, + vrf_name, ip_address, host_name=None, mac_address=None, @@ -411,7 +525,7 @@ def kea_add_lease( if inet == '6' and iaid: args['iaid'] = iaid - result = _ctrl_socket_command(inet, f'lease{inet}-add', args) + result = _ctrl_socket_command(inet, vrf_name, f'lease{inet}-add', args) if result and 'result' in result: return result['result'] == 0 @@ -419,10 +533,10 @@ def kea_add_lease( return False -def kea_delete_lease(inet, ip_address): +def kea_delete_lease(inet, vrf_name, ip_address): args = {'ip-address': ip_address} - result = _ctrl_socket_command(inet, f'lease{inet}-del', args) + result = _ctrl_socket_command(inet, vrf_name, f'lease{inet}-del', args) if result and 'result' in result: return result['result'] == 0 @@ -430,8 +544,8 @@ def kea_delete_lease(inet, ip_address): return False -def kea_get_active_config(inet): - config = _ctrl_socket_command(inet, 'config-get') +def kea_get_active_config(inet, vrf_name): + config = _ctrl_socket_command(inet, vrf_name, 'config-get') if not config or 'result' not in config or config['result'] != 0: return None @@ -483,10 +597,10 @@ def kea_get_domain_from_subnet_id(config, inet, subnet_id): if option['name'] == 'domain-name': return option['data'] - # domain-name is not found in subnet, fallback to shared-network pool option - for option in network['option-data']: - if option['name'] == 'domain-name': - return option['data'] + # domain-name is not found in subnet, fallback to shared-network pool option + for option in network['option-data']: + if option['name'] == 'domain-name': + return option['data'] return None @@ -525,12 +639,14 @@ def kea_get_static_mappings(config, inet, pools=[]) -> list: return mappings -def kea_get_server_leases(config, inet, pools=[], state=[], origin=None) -> list: +def kea_get_server_leases( + config, inet, vrf_name, pools=[], state=[], origin=None +) -> list: """ Get DHCP server leases from active Kea DHCPv4 or DHCPv6 configuration :return list """ - leases = kea_get_leases(inet) + leases = kea_get_leases(inet, vrf_name) data = [] for lease in leases: @@ -563,9 +679,9 @@ def kea_get_server_leases(config, inet, pools=[], state=[], origin=None) -> list data_lease['origin'] = 'local' # TODO: Determine remote in HA # remove trailing dot in 'hostname' to ensure consistency for `vyos-hostsd-client` data_lease['hostname'] = lease.get('hostname', '').rstrip('.') or '-' + data_lease['mac'] = lease.get('hw-address', '-') if inet == '4': - data_lease['mac'] = lease['hw-address'] data_lease['start'] = lease['start_time'].timestamp() if inet == '6': @@ -581,9 +697,10 @@ def kea_get_server_leases(config, inet, pools=[], state=[], origin=None) -> list now = datetime.now(timezone.utc) if lease['valid-lft'] > 0 and lease['expire_time'] > now: - # substraction gives us a timedelta object which can't be formatted - # with strftime so we use str(), split gets rid of the microseconds - data_lease['remaining'] = str(lease['expire_time'] - now).split('.')[0] + # clear the microseconds before subtraction for visual clarity + data_lease['remaining'] = lease['expire_time'].replace( + microsecond=0 + ) - now.replace(microsecond=0) # Do not add old leases if ( @@ -606,3 +723,32 @@ def kea_get_server_leases(config, inet, pools=[], state=[], origin=None) -> list data.pop(idx) return data + + +def _build_relay_hex_condition(sub_option_index, value): + if value.startswith('0x'): + return f'relay4[{sub_option_index}].hex == {value}' + else: + return f'relay4[{sub_option_index}].hex == 0x{value.encode().hex().lower()}' + + +def kea_build_client_class_test(config): + conditions = [] + + if 'relay_agent_information' in config: + if 'circuit_id' in config['relay_agent_information']: + conditions.append( + _build_relay_hex_condition( + 1, config['relay_agent_information']['circuit_id'] + ) + ) + if 'remote_id' in config['relay_agent_information']: + conditions.append( + _build_relay_hex_condition( + 2, config['relay_agent_information']['remote_id'] + ) + ) + + test = ' and '.join(conditions) + + return test diff --git a/python/vyos/limericks.py b/python/vyos/limericks.py index 3c6744816..0c02d5292 100644 --- a/python/vyos/limericks.py +++ b/python/vyos/limericks.py @@ -1,4 +1,4 @@ -# Copyright 2015, 2018 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/load_config.py b/python/vyos/load_config.py index b910a2f92..f65e887f0 100644 --- a/python/vyos/load_config.py +++ b/python/vyos/load_config.py @@ -1,4 +1,4 @@ -# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/logger.py b/python/vyos/logger.py index f7cc964d5..207f95c1b 100644 --- a/python/vyos/logger.py +++ b/python/vyos/logger.py @@ -1,4 +1,4 @@ -# Copyright 2020 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/migrate.py b/python/vyos/migrate.py index 9d1613676..c06f6a76c 100644 --- a/python/vyos/migrate.py +++ b/python/vyos/migrate.py @@ -1,4 +1,4 @@ -# Copyright 2019-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/nat.py b/python/vyos/nat.py index 29f8e961b..7be957a0c 100644 --- a/python/vyos/nat.py +++ b/python/vyos/nat.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as diff --git a/python/vyos/netlink/__init__.py b/python/vyos/netlink/__init__.py new file mode 100644 index 000000000..db07bae58 --- /dev/null +++ b/python/vyos/netlink/__init__.py @@ -0,0 +1,14 @@ +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. diff --git a/python/vyos/netlink/coalesce.py b/python/vyos/netlink/coalesce.py new file mode 100644 index 000000000..d54a4dd07 --- /dev/null +++ b/python/vyos/netlink/coalesce.py @@ -0,0 +1,363 @@ +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + + +from pyroute2.netlink import genlmsg +from pyroute2.netlink import NLM_F_ACK +from pyroute2.netlink import NLM_F_REQUEST +from pyroute2.netlink.exceptions import NetlinkError +from pyroute2.netlink.generic import GenericNetlinkSocket +from pyroute2.netlink.generic.ethtool import ETHTOOL_GENL_NAME +from pyroute2.netlink.generic.ethtool import ETHTOOL_GENL_VERSION +from pyroute2.netlink.generic.ethtool import ethtoolheader + +# Netlink message types for coalesce (from Linux kernel) +ETHTOOL_MSG_COALESCE_GET = 0x13 +ETHTOOL_MSG_COALESCE_SET = 0x14 + +# Error codes from the kernel +EINVAL = 0x16 # Invalid argument +EOPNOTSUPP = 0x5F # Operation not supported + +# Mapping between Python attribute names and netlink attribute names, types +# https://docs.kernel.org/networking/ethtool-netlink.html#coalesce-get +# https://git.kernel.org/pub/scm/network/ethtool/ethtool.git/tree/netlink/coalesce.c +COALESCE_NL_ATTRS = { + 'adaptive_rx': 'ETHTOOL_A_COALESCE_USE_ADAPTIVE_RX', + 'adaptive_tx': 'ETHTOOL_A_COALESCE_USE_ADAPTIVE_TX', + 'cqe_mode_rx': 'ETHTOOL_A_COALESCE_USE_CQE_MODE_RX', + 'cqe_mode_tx': 'ETHTOOL_A_COALESCE_USE_CQE_MODE_TX', + 'pkt_rate_high': 'ETHTOOL_A_COALESCE_PKT_RATE_HIGH', + 'pkt_rate_low': 'ETHTOOL_A_COALESCE_PKT_RATE_LOW', + 'rx_frame_high': 'ETHTOOL_A_COALESCE_RX_MAX_FRAMES_HIGH', + 'rx_frame_low': 'ETHTOOL_A_COALESCE_RX_MAX_FRAMES_LOW', + 'rx_frames': 'ETHTOOL_A_COALESCE_RX_MAX_FRAMES', + 'rx_frames_irq': 'ETHTOOL_A_COALESCE_RX_MAX_FRAMES_IRQ', + 'rx_usecs': 'ETHTOOL_A_COALESCE_RX_USECS', + 'rx_usecs_high': 'ETHTOOL_A_COALESCE_RX_USECS_HIGH', + 'rx_usecs_irq': 'ETHTOOL_A_COALESCE_RX_USECS_IRQ', + 'rx_usecs_low': 'ETHTOOL_A_COALESCE_RX_USECS_LOW', + 'sample_interval': 'ETHTOOL_A_COALESCE_RATE_SAMPLE_INTERVAL', + 'stats_block_usecs': 'ETHTOOL_A_COALESCE_STATS_BLOCK_USECS', + 'tx_aggr_max_bytes': 'ETHTOOL_A_COALESCE_TX_AGGR_MAX_BYTES', + 'tx_aggr_max_frames': 'ETHTOOL_A_COALESCE_TX_AGGR_MAX_FRAMES', + 'tx_aggr_time_usecs': 'ETHTOOL_A_COALESCE_TX_AGGR_TIME_USECS', + 'tx_frame_high': 'ETHTOOL_A_COALESCE_TX_MAX_FRAMES_HIGH', + 'tx_frame_low': 'ETHTOOL_A_COALESCE_TX_MAX_FRAMES_LOW', + 'tx_frames': 'ETHTOOL_A_COALESCE_TX_MAX_FRAMES', + 'tx_frames_irq': 'ETHTOOL_A_COALESCE_TX_MAX_FRAMES_IRQ', + 'tx_usecs': 'ETHTOOL_A_COALESCE_TX_USECS', + 'tx_usecs_high': 'ETHTOOL_A_COALESCE_TX_USECS_HIGH', + 'tx_usecs_irq': 'ETHTOOL_A_COALESCE_TX_USECS_IRQ', + 'tx_usecs_low': 'ETHTOOL_A_COALESCE_TX_USECS_LOW', +} + + +class ethtool_coalesce_msg(genlmsg): + """Netlink message structure for coalesce parameters.""" + + ethtoolheader = ethtoolheader + + # https://docs.kernel.org/networking/ethtool-netlink.html#coalesce-get + nla_map = ( + ('ETHTOOL_A_COALESCE_UNSPEC', 'none'), + ('ETHTOOL_A_COALESCE_HEADER', 'ethtoolheader'), + ('ETHTOOL_A_COALESCE_RX_USECS', 'uint32'), + ('ETHTOOL_A_COALESCE_RX_MAX_FRAMES', 'uint32'), + ('ETHTOOL_A_COALESCE_RX_USECS_IRQ', 'uint32'), + ('ETHTOOL_A_COALESCE_RX_MAX_FRAMES_IRQ', 'uint32'), + ('ETHTOOL_A_COALESCE_TX_USECS', 'uint32'), + ('ETHTOOL_A_COALESCE_TX_MAX_FRAMES', 'uint32'), + ('ETHTOOL_A_COALESCE_TX_USECS_IRQ', 'uint32'), + ('ETHTOOL_A_COALESCE_TX_MAX_FRAMES_IRQ', 'uint32'), + ('ETHTOOL_A_COALESCE_STATS_BLOCK_USECS', 'uint32'), + ('ETHTOOL_A_COALESCE_USE_ADAPTIVE_RX', 'uint8'), + ('ETHTOOL_A_COALESCE_USE_ADAPTIVE_TX', 'uint8'), + ('ETHTOOL_A_COALESCE_PKT_RATE_LOW', 'uint32'), + ('ETHTOOL_A_COALESCE_RX_USECS_LOW', 'uint32'), + ('ETHTOOL_A_COALESCE_RX_MAX_FRAMES_LOW', 'uint32'), + ('ETHTOOL_A_COALESCE_TX_USECS_LOW', 'uint32'), + ('ETHTOOL_A_COALESCE_TX_MAX_FRAMES_LOW', 'uint32'), + ('ETHTOOL_A_COALESCE_PKT_RATE_HIGH', 'uint32'), + ('ETHTOOL_A_COALESCE_RX_USECS_HIGH', 'uint32'), + ('ETHTOOL_A_COALESCE_RX_MAX_FRAMES_HIGH', 'uint32'), + ('ETHTOOL_A_COALESCE_TX_USECS_HIGH', 'uint32'), + ('ETHTOOL_A_COALESCE_TX_MAX_FRAMES_HIGH', 'uint32'), + ('ETHTOOL_A_COALESCE_RATE_SAMPLE_INTERVAL', 'uint32'), + ('ETHTOOL_A_COALESCE_USE_CQE_MODE_TX', 'uint8'), + ('ETHTOOL_A_COALESCE_USE_CQE_MODE_RX', 'uint8'), + ('ETHTOOL_A_COALESCE_TX_AGGR_MAX_BYTES', 'uint32'), + ('ETHTOOL_A_COALESCE_TX_AGGR_MAX_FRAMES', 'uint32'), + ('ETHTOOL_A_COALESCE_TX_AGGR_TIME_USECS', 'uint32'), + ) + + @classmethod + def get_nl_attr_type(cls, nl_attr: str) -> str: + """Returns type of attribute using declared 'nla_map' field""" + + for nla_name, nla_type in cls.nla_map: + if nla_name == nl_attr: + return nla_type + + raise ValueError(f'Unknown netlink attribute name: {nl_attr}') + + +GeneralNetlinkError = NetlinkError + + +class CoalesceError(Exception): + """Base exception for coalesce operations""" + + pass + + +class CoalesceNotSupportedParam(CoalesceError): + """Raised when a coalesce parameter is not supported for modification""" + + pass + + +class CoalesceNotSupportedOperation(CoalesceError): + """Raised when a coalesce operation is not supported by NIC driver""" + + pass + + +class CoalesceInvalidValue(CoalesceError): + """Raised when a coalesce parameter value is invalid""" + + pass + + +class CoalesceNetlink(GenericNetlinkSocket): + """ + Interface coalesce management using `pyroute2` with netlink support. + + This class provides functions to read and set interface coalesce parameters + using the ethtool netlink interface, which properly handles "unsupported" values + (shown as `n/a` in `ethtool --show-coalesce` output). + + IMPORTANT: The kernel's ethtool netlink interface returns coalesce parameters + that have non-zero values, but this does NOT mean they are modifiable. The + driver may impose additional constraints: + + 1. Some parameters are read-only (e.g., `rx_usecs` may be reported but not settable) + 2. Some parameters only accept specific values (e.g., `rx_frames` may only accept 1) + 3. The `supported_coalesce_params` bitmask in the driver determines what's truly + supported, but this is not directly queryable via netlink. + + When setting parameters fails with EOPNOTSUPP (0x5F) or EINVAL (0x16), it typically + means the parameter is not modifiable or the value is not accepted by the driver. + """ + + def __init__(self, ifname: str): + super().__init__() + self._bound = False + self._ifname = ifname + + def _ensure_bound(self): + """Ensure the socket is bound to the ethtool generic netlink family""" + + if not self._bound: + self.bind(ETHTOOL_GENL_NAME, ethtool_coalesce_msg) + self._bound = True + + def _get_dev_header(self): + """Create device header for netlink message""" + + return {'attrs': [['ETHTOOL_A_HEADER_DEV_NAME', self._ifname]]} + + def get_coalesce(self): + """ + Get coalesce parameters for an interface using netlink. + + Example: + >>> cn = CoalesceNetlink('eth0') + >>> coalesce = cn.get_coalesce() + >>> print(coalesce) + {'rx_usecs': 3, 'rx_frames': None, 'tx_usecs': None, ... } + """ + self._ensure_bound() + + msg = ethtool_coalesce_msg() + msg['cmd'] = ETHTOOL_MSG_COALESCE_GET + msg['version'] = ETHTOOL_GENL_VERSION + msg['attrs'].append(('ETHTOOL_A_COALESCE_HEADER', self._get_dev_header())) + + try: + response = self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST + ) + except NetlinkError as e: + if e.code == EOPNOTSUPP: + raise CoalesceNotSupportedOperation( + f'Coalesce operation for {self._ifname} not supported by driver' + ) from e + raise + + if not response: + raise CoalesceError(f'No response for coalesce get on {self._ifname}') + + nl_msg = response[0] + + # Parse response - only include attributes that were returned + # (unsupported attributes won't be in the response) + result = {} + for py_attr, nl_attr in COALESCE_NL_ATTRS.items(): + value = nl_msg.get_attr(nl_attr) # Will be None if not present/supported + + if value is not None: + nl_type = ethtool_coalesce_msg.get_nl_attr_type(nl_attr) + + # Convert to boolean value if it is 'uint8' type + if nl_type == 'uint8': + value = bool(value) + + result[py_attr] = value + + return result + + def set_coalesce(self, **kwargs): + """ + Set coalesce parameters for an interface using netlink. + + Only the parameters that are explicitly passed will be set. + Unsupported parameters will raise an error from the kernel. + + Args: + **kwargs: Coalesce parameters to set. Valid keys are: + - rx_usecs, rx_frames, rx_usecs_irq, rx_frames_irq + - tx_usecs, tx_frames, tx_usecs_irq, tx_frames_irq + - stats_block_usecs + - adaptive_rx, adaptive_tx + - pkt_rate_low, pkt_rate_high + - rx_usecs_low, rx_frame_low, tx_usecs_low, tx_frame_low + - rx_usecs_high, rx_frame_high, tx_usecs_high, tx_frame_high + - sample_interval + - cqe_mode_tx, cqe_mode_rx + - tx_aggr_max_bytes, tx_aggr_max_frames, tx_aggr_time_usecs + + Example: + >>> cn = CoalesceNetlink('eth0') + >>> cn.set_coalesce(rx_usecs=10, tx_usecs=10) + """ + self._ensure_bound() + + msg = ethtool_coalesce_msg() + msg['cmd'] = ETHTOOL_MSG_COALESCE_SET + msg['version'] = ETHTOOL_GENL_VERSION + msg['attrs'].append(('ETHTOOL_A_COALESCE_HEADER', self._get_dev_header())) + + # Add only the parameters that were explicitly provided + for py_attr, value in kwargs.items(): + if py_attr not in COALESCE_NL_ATTRS: + raise CoalesceInvalidValue(f'Unknown coalesce parameter: {py_attr}') + + if value is not None: + nl_attr = COALESCE_NL_ATTRS[py_attr] + nl_type = ethtool_coalesce_msg.get_nl_attr_type(nl_attr) + + # Convert values to 'uint' type + if nl_type.startswith('uint'): + value = int(value) + + msg['attrs'].append((nl_attr, value)) + + try: + self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) + except NetlinkError as e: + params_str = ', '.join( + '='.join([k.replace('_', '-'), str(v)]) for k, v in kwargs.items() + ) + + if e.code == EOPNOTSUPP: + raise CoalesceNotSupportedParam( + f'Parameter(s) not supported for modification on ' + f'{self._ifname}: {params_str}' + ) from e + elif e.code == EINVAL: + raise CoalesceInvalidValue( + f'Invalid value for coalesce parameter(s) on ' + f'{self._ifname}: {params_str}' + ) from e + raise + + +def get_coalesce(ifname) -> dict: + """ + Get coalesce parameters for an interface. + + This is a convenience function that creates a CoalesceNetlink instance, + gets the coalesce parameters, and closes the socket. + + Args: + ifname: Interface name (e.g., 'eth0') + + Returns: + dict: Coalesce parameters with None for unsupported values. + """ + with CoalesceNetlink(ifname) as cn: + return cn.get_coalesce() + + +def set_coalesce(ifname, **kwargs): + """ + Set coalesce parameters for an interface. + + This is a convenience function that creates a CoalesceNetlink instance, + sets the coalesce parameters, and closes the socket. + + Args: + ifname: Interface name (e.g., 'eth0') + **kwargs: Coalesce parameters to set. + """ + with CoalesceNetlink(ifname) as cn: + cn.set_coalesce(**kwargs) + + +def get_all_params(boolean: bool = None) -> tuple: + """ + Get all available parameters by the Linux kernel. + + This is a function that gets the coalesce parameters + which are available by implementation and the Linux kernel. + + Args: + boolean: Filter parameters and return only/without boolean types. + + Returns: + tuple: All coalesce parameters + """ + + params = list(COALESCE_NL_ATTRS.keys()) + + def _param_is_bool(p): + nl_type = ethtool_coalesce_msg.get_nl_attr_type(COALESCE_NL_ATTRS[p]) + return nl_type == 'uint8' + + if boolean is not None: + boolean_params = list(filter(_param_is_bool, params)) + + if boolean: + # Use only boolean parameters + params = boolean_params + else: + # Use other parameters except booleans + for boolean_param in boolean_params: + params.remove(boolean_param) + + return tuple(params) diff --git a/python/vyos/netlink/timestamp.py b/python/vyos/netlink/timestamp.py new file mode 100644 index 000000000..d9f47678f --- /dev/null +++ b/python/vyos/netlink/timestamp.py @@ -0,0 +1,204 @@ +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from pyroute2.netlink import genlmsg +from pyroute2.netlink import nla +from pyroute2.netlink import NLM_F_REQUEST +from pyroute2.netlink.exceptions import NetlinkError +from pyroute2.netlink.generic import GenericNetlinkSocket +from pyroute2.netlink.generic.ethtool import ETHTOOL_GENL_NAME +from pyroute2.netlink.generic.ethtool import ETHTOOL_GENL_VERSION +from pyroute2.netlink.generic.ethtool import ethtoolheader + +# Netlink message type for tsinfo (from Linux kernel uapi/linux/ethtool_netlink.h) +ETHTOOL_MSG_TSINFO_GET = 0x19 + +# Operation not supported error code from the kernel (95 decimal) +EOPNOTSUPP = 0x5F + +# HWTSTAMP_FILTER_* types from <linux/net_tstamp.h> +# Each enum value N maps to bit (1 << N) in the rx_filters bitmask. +# 'ptp' is a combined mask of all PTP-related filter variants. +HWTSTAMP_FILTER = { + 'all': 1 << 1, # HWTSTAMP_FILTER_ALL + 'ntp': 1 << 15, # HWTSTAMP_FILTER_NTP_ALL + 'ptp': (1 << 3) # HWTSTAMP_FILTER_PTP_V1_L4_EVENT + | (1 << 4) # HWTSTAMP_FILTER_PTP_V1_L4_SYNC + | (1 << 5) # HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ + | (1 << 6) # HWTSTAMP_FILTER_PTP_V2_L4_EVENT + | (1 << 7) # HWTSTAMP_FILTER_PTP_V2_L4_SYNC + | (1 << 8) # HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ + | (1 << 9) # HWTSTAMP_FILTER_PTP_V2_L2_EVENT + | (1 << 10) # HWTSTAMP_FILTER_PTP_V2_L2_SYNC + | (1 << 11) # HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ + | (1 << 12) # HWTSTAMP_FILTER_PTP_V2_EVENT + | (1 << 13) # HWTSTAMP_FILTER_PTP_V2_SYNC + | (1 << 14), # HWTSTAMP_FILTER_PTP_V2_DELAY_REQ +} + + +class ethtool_bitset_bit(nla): + """Single bit entry inside a verbose ethtool bitset.""" + + nla_map = ( + ('ETHTOOL_A_BITSET_BIT_UNSPEC', 'none'), + ('ETHTOOL_A_BITSET_BIT_INDEX', 'uint32'), + ('ETHTOOL_A_BITSET_BIT_NAME', 'asciiz'), + ('ETHTOOL_A_BITSET_BIT_VALUE', 'flag'), + ) + + +class ethtool_bitset_bits(nla): + """Container layer for nesting bit entries in a verbose ethtool bitset.""" + + ethtool_bitset_bit = ethtool_bitset_bit + nla_map = ( + ('ETHTOOL_A_BITSET_BIT_UNSPEC', 'none'), + ('ETHTOOL_A_BITSET_BIT', 'ethtool_bitset_bit'), + ) + + +class ethtool_bitset(nla): + """Ethtool verbose bitset NLA structure. + + The kernel returns the verbose form by default (ETHTOOL_A_BITSET_BITS nested entries). + ETHTOOL_A_BITSET_VALUE and ETHTOOL_A_BITSET_MASK are listed for positional completeness + only — they are not read by this implementation. + """ + + ethtool_bitset_bits = ethtool_bitset_bits + nla_map = ( + ('ETHTOOL_A_BITSET_UNSPEC', 'none'), + ('ETHTOOL_A_BITSET_NOMASK', 'flag'), + ('ETHTOOL_A_BITSET_SIZE', 'uint32'), + ('ETHTOOL_A_BITSET_BITS', 'ethtool_bitset_bits'), + ('ETHTOOL_A_BITSET_VALUE', 'binary'), + ('ETHTOOL_A_BITSET_MASK', 'binary'), + ) + + +class ethtool_tsinfo_msg(genlmsg): + """Netlink message structure for ETHTOOL_MSG_TSINFO_GET. + + nla_map indices are strictly positional — all attributes up to the last needed index must be + listed. Intermediate unused attributes are typed 'hex' to skip full NLA decoding. + Reference: https://docs.kernel.org/networking/ethtool-netlink.html#tsinfo-get + """ + + ethtoolheader = ethtoolheader + ethtool_bitset = ethtool_bitset + nla_map = ( + ('ETHTOOL_A_TSINFO_UNSPEC', 'none'), + ('ETHTOOL_A_TSINFO_HEADER', 'ethtoolheader'), + ('ETHTOOL_A_TSINFO_TIMESTAMPING', 'hex'), + ('ETHTOOL_A_TSINFO_TX_TYPES', 'hex'), + ('ETHTOOL_A_TSINFO_RX_FILTERS', 'ethtool_bitset'), + ) + + +GeneralNetlinkError = NetlinkError + + +class TsInfoError(Exception): + """Custom exception for timestamp info retrieval errors.""" + + pass + + +def _bitset_to_int(attr) -> int: + """Reconstruct an integer bitmask from a verbose ethtool_bitset NLA.""" + if attr is None: + return 0 + bits_attr = attr.get_attr('ETHTOOL_A_BITSET_BITS') + if bits_attr is None: + return 0 + bitmask = 0 + for bit in bits_attr.get_attrs('ETHTOOL_A_BITSET_BIT'): + index = bit.get_attr('ETHTOOL_A_BITSET_BIT_INDEX') + if index is not None: + bitmask |= 1 << index + return bitmask + + +class TsInfoNetlink(GenericNetlinkSocket): + """Hardware timestamp info queries using pyroute2 with netlink support.""" + + def __init__(self, ifname: str): + super().__init__() + self._bound = False + self._ifname = ifname + + def _ensure_bound(self): + """Bind netlink socket to the generic ethtool family structure if not already active.""" + if not self._bound: + self.bind(ETHTOOL_GENL_NAME, ethtool_tsinfo_msg) + self._bound = True + + def get_rx_filters(self) -> set: + """ + Query supported hardware timestamp receive filters for the interface. + + Returns: + A set of supported filter names ('all', 'ntp', 'ptp'), or an empty set + if the driver or interface lacks hardware timestamping support. + + Example: + >>> with TsInfoNetlink('eth0') as ts: + ... print(ts.get_rx_filters()) + {'ptp'} + """ + self._ensure_bound() + + msg = ethtool_tsinfo_msg() + msg['cmd'] = ETHTOOL_MSG_TSINFO_GET + msg['version'] = ETHTOOL_GENL_VERSION + msg['attrs'].append( + ( + 'ETHTOOL_A_TSINFO_HEADER', + {'attrs': [['ETHTOOL_A_HEADER_DEV_NAME', self._ifname]]}, + ) + ) + + try: + response = self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST + ) + except NetlinkError as e: + if e.code == EOPNOTSUPP: + return set() + raise + + if not response: + return set() + + bitmask = _bitset_to_int(response[0].get_attr('ETHTOOL_A_TSINFO_RX_FILTERS')) + return {name for name, mask in HWTSTAMP_FILTER.items() if bitmask & mask} + + +def get_hw_timestamp_filters(ifname: str) -> set: + """ + Get supported hardware timestamp receive filter names for an interface. + + Args: + ifname: Target network interface string (e.g., 'eth0'). + + Returns: + A set of supported human-readable filters, or an empty set on error/lack of support. + """ + try: + with TsInfoNetlink(ifname) as ts: + return ts.get_rx_filters() + except (NetlinkError, OSError): + return set() diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index 7b11d36dd..421054f6e 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,8 +16,9 @@ import re import sys import typing -from humps import decamelize +from humps import decamelize +from vyos.configquery import ConfigTreeQuery class Error(Exception): """Any error that makes requested operation impossible to complete @@ -327,3 +328,28 @@ def run(module): # Other functions should not return anything, # although they may print their own warnings or status messages func(**args) + +def verify_cli_exists(cli_path: list, error_msg: str=''): + """Decorator checks if CLI config exists using a dynamic path and error message""" + def decorator(func): + from functools import wraps + + @wraps(func) + def _wrapper(*args, **kwargs): + config = ConfigTreeQuery() + interface = kwargs.get('intf_name') + + # Combine the base CLI path with the specific interface name + path = cli_path + [interface] if interface else cli_path + + if not config.exists(path): + if not error_msg: + unconf_message = f'CLI path [{" ".join(cli_path)}] unconfigured!' + else: + # Format the error message dynamically to allow {interface} injection + unconf_message = error_msg.format(interface=interface) + raise UnconfiguredSubsystem(unconf_message) + return func(*args, **kwargs) + + return _wrapper + return decorator diff --git a/python/vyos/pki.py b/python/vyos/pki.py index 55dc02631..4598c5daa 100644 --- a/python/vyos/pki.py +++ b/python/vyos/pki.py @@ -1,4 +1,4 @@ -# Copyright (C) 2023-2024 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as diff --git a/python/vyos/priority.py b/python/vyos/priority.py index ab4e6d411..e61281d3c 100644 --- a/python/vyos/priority.py +++ b/python/vyos/priority.py @@ -1,4 +1,4 @@ -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/progressbar.py b/python/vyos/progressbar.py index 8d1042672..eb8ed474a 100644 --- a/python/vyos/progressbar.py +++ b/python/vyos/progressbar.py @@ -1,4 +1,4 @@ -# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/proto/generate_dataclass.py b/python/vyos/proto/generate_dataclass.py new file mode 100755 index 000000000..64485cd10 --- /dev/null +++ b/python/vyos/proto/generate_dataclass.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +# +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# +import argparse +import os + +from google.protobuf.descriptor_pb2 import FileDescriptorSet # pylint: disable=no-name-in-module +from google.protobuf.descriptor_pb2 import FieldDescriptorProto # pylint: disable=no-name-in-module +from humps import decamelize + +HEADER = """\ +from enum import IntEnum +from dataclasses import dataclass +from dataclasses import field +""" + + +def normalize(s: str) -> str: + """Decamelize and avoid syntactic collision""" + t = decamelize(s) + return t + '_' if t in ['from'] else t + + +def generate_dataclass(descriptor_proto): + class_name = descriptor_proto.name + fields = [] + for field_p in descriptor_proto.field: + field_name = field_p.name + field_type, field_default = get_type(field_p.type, field_p.type_name) + match field_p.label: + case FieldDescriptorProto.LABEL_REPEATED: + field_type = f'list[{field_type}] = field(default_factory=list)' + case FieldDescriptorProto.LABEL_OPTIONAL: + field_type = f'{field_type} = None' + case _: + field_type = f'{field_type} = {field_default}' + + fields.append(f' {field_name}: {field_type}') + + code = f""" +@dataclass +class {class_name}: +{chr(10).join(fields) if fields else ' pass'} +""" + + return code + + +def generate_request(descriptor_proto): + class_name = descriptor_proto.name + fields = [] + f_vars = [] + for field_p in descriptor_proto.field: + field_name = field_p.name + field_type, field_default = get_type(field_p.type, field_p.type_name) + match field_p.label: + case FieldDescriptorProto.LABEL_REPEATED: + field_type = f'list[{field_type}] = []' + case FieldDescriptorProto.LABEL_OPTIONAL: + field_type = f'{field_type} = None' + case _: + field_type = f'{field_type} = {field_default}' + + fields.append(f'{normalize(field_name)}: {field_type}') + f_vars.append(f'{normalize(field_name)}') + + fields.insert(0, 'token: str = None') + + code = f""" +def set_request_{decamelize(class_name)}({', '.join(fields)}): + reqi = {class_name} ({', '.join(f_vars)}) + req = Request({decamelize(class_name)}=reqi) + req_env = RequestEnvelope(token, req) + return req_env +""" + + return code + + +def generate_nested_dataclass(descriptor_proto): + out = '' + for nested_p in descriptor_proto.nested_type: + out = out + generate_dataclass(nested_p) + + return out + + +def generate_nested_request(descriptor_proto): + out = '' + for nested_p in descriptor_proto.nested_type: + out = out + generate_request(nested_p) + + return out + + +def generate_enum_dataclass(descriptor_proto): + code = '' + for enum_p in descriptor_proto.enum_type: + enums = [] + enum_name = enum_p.name + for enum_val in enum_p.value: + enums.append(f' {enum_val.name} = {enum_val.number}') + + code += f""" +class {enum_name}(IntEnum): +{chr(10).join(enums)} +""" + + return code + + +def get_type(field_type, type_name): + res = 'Any', None + match field_type: + case FieldDescriptorProto.TYPE_STRING: + res = 'str', '""' + case FieldDescriptorProto.TYPE_INT32 | FieldDescriptorProto.TYPE_INT64: + res = 'int', 0 + case FieldDescriptorProto.TYPE_FLOAT | FieldDescriptorProto.TYPE_DOUBLE: + res = 'float', 0.0 + case FieldDescriptorProto.TYPE_BOOL: + res = 'bool', False + case FieldDescriptorProto.TYPE_MESSAGE | FieldDescriptorProto.TYPE_ENUM: + res = type_name.split('.')[-1], None + case _: + pass + + return res + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('descriptor_file', help='protobuf .desc file') + parser.add_argument('--out-dir', help='directory to write generated file') + args = parser.parse_args() + desc_file = args.descriptor_file + out_dir = args.out_dir + + with open(desc_file, 'rb') as f: + descriptor_set_data = f.read() + + descriptor_set = FileDescriptorSet() + descriptor_set.ParseFromString(descriptor_set_data) + + for file_proto in descriptor_set.file: + f = f'{file_proto.name.replace(".", "_")}.py' + f = os.path.join(out_dir, f) + dataclass_code = '' + nested_code = '' + enum_code = '' + request_code = '' + with open(f, 'w') as f: + enum_code += generate_enum_dataclass(file_proto) + for message_proto in file_proto.message_type: + dataclass_code += generate_dataclass(message_proto) + nested_code += generate_nested_dataclass(message_proto) + enum_code += generate_enum_dataclass(message_proto) + request_code += generate_nested_request(message_proto) + + f.write(HEADER) + f.write(enum_code) + f.write(nested_code) + f.write(dataclass_code) + f.write(request_code) diff --git a/python/vyos/proto/vycall_pb2.py b/python/vyos/proto/vycall_pb2.py new file mode 100644 index 000000000..e7c228499 --- /dev/null +++ b/python/vyos/proto/vycall_pb2.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: vycall.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cvycall.proto\"&\n\x06Status\x12\x0f\n\x07success\x18\x01 \x02(\x08\x12\x0b\n\x03out\x18\x02 \x02(\t\"Y\n\x04\x43\x61ll\x12\x13\n\x0bscript_name\x18\x01 \x02(\t\x12\x11\n\ttag_value\x18\x02 \x01(\t\x12\x11\n\targ_value\x18\x03 \x01(\t\x12\x16\n\x05reply\x18\x04 \x01(\x0b\x32\x07.Status\"\xb4\x01\n\x06\x43ommit\x12\x12\n\nsession_id\x18\x01 \x02(\t\x12\x13\n\x0bsession_pid\x18\x02 \x02(\x05\x12\x11\n\tsudo_user\x18\x03 \x02(\t\x12\x0c\n\x04user\x18\x04 \x02(\t\x12\x0f\n\x07\x64ry_run\x18\x05 \x02(\x08\x12\x0e\n\x06\x61tomic\x18\x06 \x02(\x08\x12\x12\n\nbackground\x18\x07 \x02(\x08\x12\x15\n\x04init\x18\x08 \x01(\x0b\x32\x07.Status\x12\x14\n\x05\x63\x61lls\x18\t \x03(\x0b\x32\x05.Call') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'vycall_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _STATUS._serialized_start=16 + _STATUS._serialized_end=54 + _CALL._serialized_start=56 + _CALL._serialized_end=145 + _COMMIT._serialized_start=148 + _COMMIT._serialized_end=328 +# @@protoc_insertion_point(module_scope) diff --git a/python/vyos/proto/vyconf_client.py b/python/vyos/proto/vyconf_client.py new file mode 100644 index 000000000..a3ba9864c --- /dev/null +++ b/python/vyos/proto/vyconf_client.py @@ -0,0 +1,89 @@ +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +import socket +from dataclasses import asdict + +from vyos.proto import vyconf_proto +from vyos.proto import vyconf_pb2 + +from google.protobuf.json_format import MessageToDict +from google.protobuf.json_format import ParseDict + +socket_path = '/var/run/vyconfd.sock' + + +def send_socket(msg: bytearray) -> bytes: + data = bytes() + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.connect(socket_path) + client.sendall(msg) + + data_length = client.recv(4) + if data_length: + length = int.from_bytes(data_length) + data = client.recv(length) + + client.close() + + return data + + +def request_to_msg(req: vyconf_proto.RequestEnvelope) -> vyconf_pb2.RequestEnvelope: + # pylint: disable=no-member + + msg = vyconf_pb2.RequestEnvelope() + msg = ParseDict(asdict(req), msg, ignore_unknown_fields=True) + return msg + + +def msg_to_response(msg: vyconf_pb2.Response) -> vyconf_proto.Response: + # pylint: disable=no-member + + d = MessageToDict( + msg, preserving_proto_field_name=True, use_integers_for_enums=True + ) + + response = vyconf_proto.Response(**d) + return response + + +def write_request(req: vyconf_proto.RequestEnvelope) -> bytearray: + req_msg = request_to_msg(req) + encoded_data = req_msg.SerializeToString() + byte_size = req_msg.ByteSize() + length_bytes = byte_size.to_bytes(4) + arr = bytearray(length_bytes) + arr.extend(encoded_data) + + return arr + + +def read_response(msg: bytes) -> vyconf_proto.Response: + response_msg = vyconf_pb2.Response() # pylint: disable=no-member + response_msg.ParseFromString(msg) + response = msg_to_response(response_msg) + + return response + + +def send_request(name, *args, **kwargs): + func = getattr(vyconf_proto, f'set_request_{name}') + request_env = func(*args, **kwargs) + msg = write_request(request_env) + response_msg = send_socket(msg) + response = read_response(response_msg) + + return response diff --git a/python/vyos/proto/vyconf_pb2.py b/python/vyos/proto/vyconf_pb2.py new file mode 100644 index 000000000..901843e9e --- /dev/null +++ b/python/vyos/proto/vyconf_pb2.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: vyconf.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cvyconf.proto\"\xb3\x1e\n\x07Request\x12!\n\x06prompt\x18\x01 \x01(\x0b\x32\x0f.Request.PromptH\x00\x12.\n\rsetup_session\x18\x02 \x01(\x0b\x32\x15.Request.SetupSessionH\x00\x12\x1b\n\x03set\x18\x03 \x01(\x0b\x32\x0c.Request.SetH\x00\x12!\n\x06\x64\x65lete\x18\x04 \x01(\x0b\x32\x0f.Request.DeleteH\x00\x12!\n\x06rename\x18\x05 \x01(\x0b\x32\x0f.Request.RenameH\x00\x12\x1d\n\x04\x63opy\x18\x06 \x01(\x0b\x32\r.Request.CopyH\x00\x12#\n\x07\x63omment\x18\x07 \x01(\x0b\x32\x10.Request.CommentH\x00\x12!\n\x06\x63ommit\x18\x08 \x01(\x0b\x32\x0f.Request.CommitH\x00\x12%\n\x08rollback\x18\t \x01(\x0b\x32\x11.Request.RollbackH\x00\x12\x1f\n\x05merge\x18\n \x01(\x0b\x32\x0e.Request.MergeH\x00\x12\x1d\n\x04save\x18\x0b \x01(\x0b\x32\r.Request.SaveH\x00\x12*\n\x0bshow_config\x18\x0c \x01(\x0b\x32\x13.Request.ShowConfigH\x00\x12!\n\x06\x65xists\x18\r \x01(\x0b\x32\x0f.Request.ExistsH\x00\x12&\n\tget_value\x18\x0e \x01(\x0b\x32\x11.Request.GetValueH\x00\x12(\n\nget_values\x18\x0f \x01(\x0b\x32\x12.Request.GetValuesH\x00\x12.\n\rlist_children\x18\x10 \x01(\x0b\x32\x15.Request.ListChildrenH\x00\x12)\n\x0brun_op_mode\x18\x11 \x01(\x0b\x32\x12.Request.RunOpModeH\x00\x12#\n\x07\x63onfirm\x18\x12 \x01(\x0b\x32\x10.Request.ConfirmH\x00\x12\x43\n\x18\x65nter_configuration_mode\x18\x13 \x01(\x0b\x32\x1f.Request.EnterConfigurationModeH\x00\x12\x41\n\x17\x65xit_configuration_mode\x18\x14 \x01(\x0b\x32\x1e.Request.ExitConfigurationModeH\x00\x12%\n\x08validate\x18\x15 \x01(\x0b\x32\x11.Request.ValidateH\x00\x12%\n\x08teardown\x18\x16 \x01(\x0b\x32\x11.Request.TeardownH\x00\x12\x30\n\x0ereload_reftree\x18\x17 \x01(\x0b\x32\x16.Request.ReloadReftreeH\x00\x12\x1d\n\x04load\x18\x18 \x01(\x0b\x32\r.Request.LoadH\x00\x12#\n\x07\x64iscard\x18\x19 \x01(\x0b\x32\x10.Request.DiscardH\x00\x12\x32\n\x0fsession_changed\x18\x1a \x01(\x0b\x32\x17.Request.SessionChangedH\x00\x12/\n\x0esession_of_pid\x18\x1b \x01(\x0b\x32\x15.Request.SessionOfPidH\x00\x12\x30\n\x0esession_exists\x18\x1c \x01(\x0b\x32\x16.Request.SessionExistsH\x00\x12(\n\nget_config\x18\x1d \x01(\x0b\x32\x12.Request.GetConfigH\x00\x12\"\n\x07\x61ux_set\x18\x1e \x01(\x0b\x32\x0f.Request.AuxSetH\x00\x12(\n\naux_delete\x18\x1f \x01(\x0b\x32\x12.Request.AuxDeleteH\x00\x12.\n\rshow_sessions\x18 \x01(\x0b\x32\x15.Request.ShowSessionsH\x00\x12/\n\x0eset_edit_level\x18! \x01(\x0b\x32\x15.Request.SetEditLevelH\x00\x12\x34\n\x11set_edit_level_up\x18\" \x01(\x0b\x32\x17.Request.SetEditLevelUpH\x00\x12\x33\n\x10reset_edit_level\x18# \x01(\x0b\x32\x17.Request.ResetEditLevelH\x00\x12/\n\x0eget_edit_level\x18$ \x01(\x0b\x32\x15.Request.GetEditLevelH\x00\x12\x31\n\x0f\x65\x64it_level_root\x18% \x01(\x0b\x32\x16.Request.EditLevelRootH\x00\x12\x30\n\x0e\x63onfig_unsaved\x18& \x01(\x0b\x32\x16.Request.ConfigUnsavedH\x00\x12=\n\x15reference_path_exists\x18\' \x01(\x0b\x32\x1c.Request.ReferencePathExistsH\x00\x12-\n\rget_path_type\x18( \x01(\x0b\x32\x14.Request.GetPathTypeH\x00\x12\x37\n\x12get_completion_env\x18) \x01(\x0b\x32\x19.Request.GetCompletionEnvH\x00\x1a\x08\n\x06Prompt\x1a\x83\x01\n\x0cSetupSession\x12\x12\n\nclient_pid\x18\x01 \x02(\x05\x12\x1a\n\x12\x63lient_application\x18\x02 \x01(\t\x12\x14\n\x0con_behalf_of\x18\x03 \x01(\x05\x12\x13\n\x0b\x63lient_user\x18\x04 \x01(\t\x12\x18\n\x10\x63lient_sudo_user\x18\x05 \x01(\t\x1a\"\n\x0cSessionOfPid\x12\x12\n\nclient_pid\x18\x01 \x02(\x05\x1a\x1e\n\rSessionExists\x12\r\n\x05\x64ummy\x18\x01 \x01(\x05\x1a\x1a\n\tGetConfig\x12\r\n\x05\x64ummy\x18\x01 \x01(\x05\x1a \n\x08Teardown\x12\x14\n\x0con_behalf_of\x18\x01 \x01(\x05\x1a\x46\n\x08Validate\x12\x0c\n\x04Path\x18\x01 \x03(\t\x12,\n\routput_format\x18\x02 \x01(\x0e\x32\x15.Request.OutputFormat\x1a\x13\n\x03Set\x12\x0c\n\x04path\x18\x01 \x03(\t\x1a\x16\n\x06\x44\x65lete\x12\x0c\n\x04path\x18\x01 \x03(\t\x1a>\n\x06\x41uxSet\x12\x0c\n\x04path\x18\x01 \x03(\t\x12\x13\n\x0bscript_name\x18\x02 \x02(\t\x12\x11\n\ttag_value\x18\x03 \x01(\t\x1a\x41\n\tAuxDelete\x12\x0c\n\x04path\x18\x01 \x03(\t\x12\x13\n\x0bscript_name\x18\x02 \x02(\t\x12\x11\n\ttag_value\x18\x03 \x01(\t\x1a\x18\n\x07\x44iscard\x12\r\n\x05\x64ummy\x18\x01 \x01(\x05\x1a\x1f\n\x0eSessionChanged\x12\r\n\x05\x64ummy\x18\x01 \x01(\x05\x1a+\n\x04\x43opy\x12\x0e\n\x06source\x18\x01 \x03(\t\x12\x13\n\x0b\x64\x65stination\x18\x02 \x03(\t\x1a-\n\x06Rename\x12\x0e\n\x06source\x18\x01 \x03(\t\x12\x13\n\x0b\x64\x65stination\x18\x02 \x03(\t\x1a(\n\x07\x43omment\x12\x0c\n\x04path\x18\x01 \x03(\t\x12\x0f\n\x07\x63omment\x18\x02 \x02(\t\x1aT\n\x06\x43ommit\x12\x0f\n\x07\x63onfirm\x18\x01 \x01(\x08\x12\x17\n\x0f\x63onfirm_timeout\x18\x02 \x01(\x05\x12\x0f\n\x07\x63omment\x18\x03 \x01(\t\x12\x0f\n\x07\x64ry_run\x18\x04 \x01(\x08\x1a\x1c\n\x08Rollback\x12\x10\n\x08revision\x18\x01 \x02(\x05\x1aO\n\x04Load\x12\x10\n\x08location\x18\x01 \x02(\t\x12\x0e\n\x06\x63\x61\x63hed\x18\x02 \x02(\x08\x12%\n\x06\x66ormat\x18\x03 \x01(\x0e\x32\x15.Request.ConfigFormat\x1aU\n\x05Merge\x12\x10\n\x08location\x18\x01 \x02(\t\x12\x13\n\x0b\x64\x65structive\x18\x02 \x02(\x08\x12%\n\x06\x66ormat\x18\x03 \x01(\x0e\x32\x15.Request.ConfigFormat\x1a?\n\x04Save\x12\x10\n\x08location\x18\x01 \x02(\t\x12%\n\x06\x66ormat\x18\x02 \x01(\x0e\x32\x15.Request.ConfigFormat\x1a\x41\n\nShowConfig\x12\x0c\n\x04path\x18\x01 \x03(\t\x12%\n\x06\x66ormat\x18\x02 \x01(\x0e\x32\x15.Request.ConfigFormat\x1a\x16\n\x06\x45xists\x12\x0c\n\x04path\x18\x01 \x03(\t\x1a\x46\n\x08GetValue\x12\x0c\n\x04path\x18\x01 \x03(\t\x12,\n\routput_format\x18\x02 \x01(\x0e\x32\x15.Request.OutputFormat\x1aG\n\tGetValues\x12\x0c\n\x04path\x18\x01 \x03(\t\x12,\n\routput_format\x18\x02 \x01(\x0e\x32\x15.Request.OutputFormat\x1aJ\n\x0cListChildren\x12\x0c\n\x04path\x18\x01 \x03(\t\x12,\n\routput_format\x18\x02 \x01(\x0e\x32\x15.Request.OutputFormat\x1aG\n\tRunOpMode\x12\x0c\n\x04path\x18\x01 \x03(\t\x12,\n\routput_format\x18\x02 \x01(\x0e\x32\x15.Request.OutputFormat\x1a\t\n\x07\x43onfirm\x1aG\n\x16\x45nterConfigurationMode\x12\x11\n\texclusive\x18\x01 \x02(\x08\x12\x1a\n\x12override_exclusive\x18\x02 \x02(\x08\x1a\x17\n\x15\x45xitConfigurationMode\x1a%\n\rReloadReftree\x12\x14\n\x0con_behalf_of\x18\x01 \x01(\x05\x1a;\n\x0cShowSessions\x12\x14\n\x0c\x65xclude_self\x18\x01 \x02(\x08\x12\x15\n\rexclude_other\x18\x02 \x02(\x08\x1a\x1c\n\x0cSetEditLevel\x12\x0c\n\x04path\x18\x01 \x03(\t\x1a\x1f\n\x0eSetEditLevelUp\x12\r\n\x05\x64ummy\x18\x01 \x01(\x05\x1a\x1f\n\x0eResetEditLevel\x12\r\n\x05\x64ummy\x18\x01 \x01(\x05\x1a\x1d\n\x0cGetEditLevel\x12\r\n\x05\x64ummy\x18\x01 \x01(\x05\x1a\x1e\n\rEditLevelRoot\x12\r\n\x05\x64ummy\x18\x01 \x01(\x05\x1a\x1d\n\rConfigUnsaved\x12\x0c\n\x04\x66ile\x18\x01 \x01(\t\x1a#\n\x13ReferencePathExists\x12\x0c\n\x04path\x18\x01 \x03(\t\x1a\x32\n\x0bGetPathType\x12\x0c\n\x04path\x18\x01 \x03(\t\x12\x15\n\rlegacy_format\x18\x02 \x02(\x08\x1a\x37\n\x10GetCompletionEnv\x12\x0c\n\x04path\x18\x01 \x03(\t\x12\x15\n\rlegacy_format\x18\x02 \x02(\x08\"#\n\x0c\x43onfigFormat\x12\t\n\x05\x43URLY\x10\x00\x12\x08\n\x04JSON\x10\x01\")\n\x0cOutputFormat\x12\x0c\n\x08OutPlain\x10\x00\x12\x0b\n\x07OutJSON\x10\x01\x42\x05\n\x03msg\";\n\x0fRequestEnvelope\x12\r\n\x05token\x18\x01 \x01(\t\x12\x19\n\x07request\x18\x02 \x02(\x0b\x32\x08.Request\"S\n\x08Response\x12\x17\n\x06status\x18\x01 \x02(\x0e\x32\x07.Errnum\x12\x0e\n\x06output\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12\x0f\n\x07warning\x18\x04 \x01(\t*\xd2\x01\n\x06\x45rrnum\x12\x0b\n\x07SUCCESS\x10\x00\x12\x08\n\x04\x46\x41IL\x10\x01\x12\x10\n\x0cINVALID_PATH\x10\x02\x12\x11\n\rINVALID_VALUE\x10\x03\x12\x16\n\x12\x43OMMIT_IN_PROGRESS\x10\x04\x12\x18\n\x14\x43ONFIGURATION_LOCKED\x10\x05\x12\x12\n\x0eINTERNAL_ERROR\x10\x06\x12\x15\n\x11PERMISSION_DENIED\x10\x07\x12\x17\n\x13PATH_ALREADY_EXISTS\x10\x08\x12\x16\n\x12UNCOMMITED_CHANGES\x10\t') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'vyconf_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _ERRNUM._serialized_start=4057 + _ERRNUM._serialized_end=4267 + _REQUEST._serialized_start=17 + _REQUEST._serialized_end=3908 + _REQUEST_PROMPT._serialized_start=1829 + _REQUEST_PROMPT._serialized_end=1837 + _REQUEST_SETUPSESSION._serialized_start=1840 + _REQUEST_SETUPSESSION._serialized_end=1971 + _REQUEST_SESSIONOFPID._serialized_start=1973 + _REQUEST_SESSIONOFPID._serialized_end=2007 + _REQUEST_SESSIONEXISTS._serialized_start=2009 + _REQUEST_SESSIONEXISTS._serialized_end=2039 + _REQUEST_GETCONFIG._serialized_start=2041 + _REQUEST_GETCONFIG._serialized_end=2067 + _REQUEST_TEARDOWN._serialized_start=2069 + _REQUEST_TEARDOWN._serialized_end=2101 + _REQUEST_VALIDATE._serialized_start=2103 + _REQUEST_VALIDATE._serialized_end=2173 + _REQUEST_SET._serialized_start=2175 + _REQUEST_SET._serialized_end=2194 + _REQUEST_DELETE._serialized_start=2196 + _REQUEST_DELETE._serialized_end=2218 + _REQUEST_AUXSET._serialized_start=2220 + _REQUEST_AUXSET._serialized_end=2282 + _REQUEST_AUXDELETE._serialized_start=2284 + _REQUEST_AUXDELETE._serialized_end=2349 + _REQUEST_DISCARD._serialized_start=2351 + _REQUEST_DISCARD._serialized_end=2375 + _REQUEST_SESSIONCHANGED._serialized_start=2377 + _REQUEST_SESSIONCHANGED._serialized_end=2408 + _REQUEST_COPY._serialized_start=2410 + _REQUEST_COPY._serialized_end=2453 + _REQUEST_RENAME._serialized_start=2455 + _REQUEST_RENAME._serialized_end=2500 + _REQUEST_COMMENT._serialized_start=2502 + _REQUEST_COMMENT._serialized_end=2542 + _REQUEST_COMMIT._serialized_start=2544 + _REQUEST_COMMIT._serialized_end=2628 + _REQUEST_ROLLBACK._serialized_start=2630 + _REQUEST_ROLLBACK._serialized_end=2658 + _REQUEST_LOAD._serialized_start=2660 + _REQUEST_LOAD._serialized_end=2739 + _REQUEST_MERGE._serialized_start=2741 + _REQUEST_MERGE._serialized_end=2826 + _REQUEST_SAVE._serialized_start=2828 + _REQUEST_SAVE._serialized_end=2891 + _REQUEST_SHOWCONFIG._serialized_start=2893 + _REQUEST_SHOWCONFIG._serialized_end=2958 + _REQUEST_EXISTS._serialized_start=2960 + _REQUEST_EXISTS._serialized_end=2982 + _REQUEST_GETVALUE._serialized_start=2984 + _REQUEST_GETVALUE._serialized_end=3054 + _REQUEST_GETVALUES._serialized_start=3056 + _REQUEST_GETVALUES._serialized_end=3127 + _REQUEST_LISTCHILDREN._serialized_start=3129 + _REQUEST_LISTCHILDREN._serialized_end=3203 + _REQUEST_RUNOPMODE._serialized_start=3205 + _REQUEST_RUNOPMODE._serialized_end=3276 + _REQUEST_CONFIRM._serialized_start=3278 + _REQUEST_CONFIRM._serialized_end=3287 + _REQUEST_ENTERCONFIGURATIONMODE._serialized_start=3289 + _REQUEST_ENTERCONFIGURATIONMODE._serialized_end=3360 + _REQUEST_EXITCONFIGURATIONMODE._serialized_start=3362 + _REQUEST_EXITCONFIGURATIONMODE._serialized_end=3385 + _REQUEST_RELOADREFTREE._serialized_start=3387 + _REQUEST_RELOADREFTREE._serialized_end=3424 + _REQUEST_SHOWSESSIONS._serialized_start=3426 + _REQUEST_SHOWSESSIONS._serialized_end=3485 + _REQUEST_SETEDITLEVEL._serialized_start=3487 + _REQUEST_SETEDITLEVEL._serialized_end=3515 + _REQUEST_SETEDITLEVELUP._serialized_start=3517 + _REQUEST_SETEDITLEVELUP._serialized_end=3548 + _REQUEST_RESETEDITLEVEL._serialized_start=3550 + _REQUEST_RESETEDITLEVEL._serialized_end=3581 + _REQUEST_GETEDITLEVEL._serialized_start=3583 + _REQUEST_GETEDITLEVEL._serialized_end=3612 + _REQUEST_EDITLEVELROOT._serialized_start=3614 + _REQUEST_EDITLEVELROOT._serialized_end=3644 + _REQUEST_CONFIGUNSAVED._serialized_start=3646 + _REQUEST_CONFIGUNSAVED._serialized_end=3675 + _REQUEST_REFERENCEPATHEXISTS._serialized_start=3677 + _REQUEST_REFERENCEPATHEXISTS._serialized_end=3712 + _REQUEST_GETPATHTYPE._serialized_start=3714 + _REQUEST_GETPATHTYPE._serialized_end=3764 + _REQUEST_GETCOMPLETIONENV._serialized_start=3766 + _REQUEST_GETCOMPLETIONENV._serialized_end=3821 + _REQUEST_CONFIGFORMAT._serialized_start=3823 + _REQUEST_CONFIGFORMAT._serialized_end=3858 + _REQUEST_OUTPUTFORMAT._serialized_start=3860 + _REQUEST_OUTPUTFORMAT._serialized_end=3901 + _REQUESTENVELOPE._serialized_start=3910 + _REQUESTENVELOPE._serialized_end=3969 + _RESPONSE._serialized_start=3971 + _RESPONSE._serialized_end=4054 +# @@protoc_insertion_point(module_scope) diff --git a/python/vyos/proto/vyconf_proto.py b/python/vyos/proto/vyconf_proto.py new file mode 100644 index 000000000..b05294e74 --- /dev/null +++ b/python/vyos/proto/vyconf_proto.py @@ -0,0 +1,518 @@ +from enum import IntEnum +from dataclasses import dataclass +from dataclasses import field + +class Errnum(IntEnum): + SUCCESS = 0 + FAIL = 1 + INVALID_PATH = 2 + INVALID_VALUE = 3 + COMMIT_IN_PROGRESS = 4 + CONFIGURATION_LOCKED = 5 + INTERNAL_ERROR = 6 + PERMISSION_DENIED = 7 + PATH_ALREADY_EXISTS = 8 + UNCOMMITED_CHANGES = 9 + +class ConfigFormat(IntEnum): + CURLY = 0 + JSON = 1 + +class OutputFormat(IntEnum): + OutPlain = 0 + OutJSON = 1 + +@dataclass +class Prompt: + pass + +@dataclass +class SetupSession: + client_pid: int = 0 + client_application: str = None + on_behalf_of: int = None + client_user: str = None + client_sudo_user: str = None + +@dataclass +class SessionOfPid: + client_pid: int = 0 + +@dataclass +class SessionExists: + dummy: int = None + +@dataclass +class GetConfig: + dummy: int = None + +@dataclass +class Teardown: + on_behalf_of: int = None + +@dataclass +class Validate: + Path: list[str] = field(default_factory=list) + output_format: OutputFormat = None + +@dataclass +class Set: + path: list[str] = field(default_factory=list) + +@dataclass +class Delete: + path: list[str] = field(default_factory=list) + +@dataclass +class AuxSet: + path: list[str] = field(default_factory=list) + script_name: str = "" + tag_value: str = None + +@dataclass +class AuxDelete: + path: list[str] = field(default_factory=list) + script_name: str = "" + tag_value: str = None + +@dataclass +class Discard: + dummy: int = None + +@dataclass +class SessionChanged: + dummy: int = None + +@dataclass +class Copy: + source: list[str] = field(default_factory=list) + destination: list[str] = field(default_factory=list) + +@dataclass +class Rename: + source: list[str] = field(default_factory=list) + destination: list[str] = field(default_factory=list) + +@dataclass +class Comment: + path: list[str] = field(default_factory=list) + comment: str = "" + +@dataclass +class Commit: + confirm: bool = None + confirm_timeout: int = None + comment: str = None + dry_run: bool = None + +@dataclass +class Rollback: + revision: int = 0 + +@dataclass +class Load: + location: str = "" + cached: bool = False + format: ConfigFormat = None + +@dataclass +class Merge: + location: str = "" + destructive: bool = False + format: ConfigFormat = None + +@dataclass +class Save: + location: str = "" + format: ConfigFormat = None + +@dataclass +class ShowConfig: + path: list[str] = field(default_factory=list) + format: ConfigFormat = None + +@dataclass +class Exists: + path: list[str] = field(default_factory=list) + +@dataclass +class GetValue: + path: list[str] = field(default_factory=list) + output_format: OutputFormat = None + +@dataclass +class GetValues: + path: list[str] = field(default_factory=list) + output_format: OutputFormat = None + +@dataclass +class ListChildren: + path: list[str] = field(default_factory=list) + output_format: OutputFormat = None + +@dataclass +class RunOpMode: + path: list[str] = field(default_factory=list) + output_format: OutputFormat = None + +@dataclass +class Confirm: + pass + +@dataclass +class EnterConfigurationMode: + exclusive: bool = False + override_exclusive: bool = False + +@dataclass +class ExitConfigurationMode: + pass + +@dataclass +class ReloadReftree: + on_behalf_of: int = None + +@dataclass +class ShowSessions: + exclude_self: bool = False + exclude_other: bool = False + +@dataclass +class SetEditLevel: + path: list[str] = field(default_factory=list) + +@dataclass +class SetEditLevelUp: + dummy: int = None + +@dataclass +class ResetEditLevel: + dummy: int = None + +@dataclass +class GetEditLevel: + dummy: int = None + +@dataclass +class EditLevelRoot: + dummy: int = None + +@dataclass +class ConfigUnsaved: + file: str = None + +@dataclass +class ReferencePathExists: + path: list[str] = field(default_factory=list) + +@dataclass +class GetPathType: + path: list[str] = field(default_factory=list) + legacy_format: bool = False + +@dataclass +class GetCompletionEnv: + path: list[str] = field(default_factory=list) + legacy_format: bool = False + +@dataclass +class Request: + prompt: Prompt = None + setup_session: SetupSession = None + set: Set = None + delete: Delete = None + rename: Rename = None + copy: Copy = None + comment: Comment = None + commit: Commit = None + rollback: Rollback = None + merge: Merge = None + save: Save = None + show_config: ShowConfig = None + exists: Exists = None + get_value: GetValue = None + get_values: GetValues = None + list_children: ListChildren = None + run_op_mode: RunOpMode = None + confirm: Confirm = None + enter_configuration_mode: EnterConfigurationMode = None + exit_configuration_mode: ExitConfigurationMode = None + validate: Validate = None + teardown: Teardown = None + reload_reftree: ReloadReftree = None + load: Load = None + discard: Discard = None + session_changed: SessionChanged = None + session_of_pid: SessionOfPid = None + session_exists: SessionExists = None + get_config: GetConfig = None + aux_set: AuxSet = None + aux_delete: AuxDelete = None + show_sessions: ShowSessions = None + set_edit_level: SetEditLevel = None + set_edit_level_up: SetEditLevelUp = None + reset_edit_level: ResetEditLevel = None + get_edit_level: GetEditLevel = None + edit_level_root: EditLevelRoot = None + config_unsaved: ConfigUnsaved = None + reference_path_exists: ReferencePathExists = None + get_path_type: GetPathType = None + get_completion_env: GetCompletionEnv = None + +@dataclass +class RequestEnvelope: + token: str = None + request: Request = None + +@dataclass +class Response: + status: Errnum = None + output: str = None + error: str = None + warning: str = None + +def set_request_prompt(token: str = None): + reqi = Prompt () + req = Request(prompt=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_setup_session(token: str = None, client_pid: int = 0, client_application: str = None, on_behalf_of: int = None, client_user: str = None, client_sudo_user: str = None): + reqi = SetupSession (client_pid, client_application, on_behalf_of, client_user, client_sudo_user) + req = Request(setup_session=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_session_of_pid(token: str = None, client_pid: int = 0): + reqi = SessionOfPid (client_pid) + req = Request(session_of_pid=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_session_exists(token: str = None, dummy: int = None): + reqi = SessionExists (dummy) + req = Request(session_exists=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_get_config(token: str = None, dummy: int = None): + reqi = GetConfig (dummy) + req = Request(get_config=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_teardown(token: str = None, on_behalf_of: int = None): + reqi = Teardown (on_behalf_of) + req = Request(teardown=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_validate(token: str = None, path: list[str] = [], output_format: OutputFormat = None): + reqi = Validate (path, output_format) + req = Request(validate=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_set(token: str = None, path: list[str] = []): + reqi = Set (path) + req = Request(set=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_delete(token: str = None, path: list[str] = []): + reqi = Delete (path) + req = Request(delete=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_aux_set(token: str = None, path: list[str] = [], script_name: str = "", tag_value: str = None): + reqi = AuxSet (path, script_name, tag_value) + req = Request(aux_set=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_aux_delete(token: str = None, path: list[str] = [], script_name: str = "", tag_value: str = None): + reqi = AuxDelete (path, script_name, tag_value) + req = Request(aux_delete=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_discard(token: str = None, dummy: int = None): + reqi = Discard (dummy) + req = Request(discard=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_session_changed(token: str = None, dummy: int = None): + reqi = SessionChanged (dummy) + req = Request(session_changed=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_copy(token: str = None, source: list[str] = [], destination: list[str] = []): + reqi = Copy (source, destination) + req = Request(copy=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_rename(token: str = None, source: list[str] = [], destination: list[str] = []): + reqi = Rename (source, destination) + req = Request(rename=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_comment(token: str = None, path: list[str] = [], comment: str = ""): + reqi = Comment (path, comment) + req = Request(comment=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_commit(token: str = None, confirm: bool = None, confirm_timeout: int = None, comment: str = None, dry_run: bool = None): + reqi = Commit (confirm, confirm_timeout, comment, dry_run) + req = Request(commit=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_rollback(token: str = None, revision: int = 0): + reqi = Rollback (revision) + req = Request(rollback=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_load(token: str = None, location: str = "", cached: bool = False, format: ConfigFormat = None): + reqi = Load (location, cached, format) + req = Request(load=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_merge(token: str = None, location: str = "", destructive: bool = False, format: ConfigFormat = None): + reqi = Merge (location, destructive, format) + req = Request(merge=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_save(token: str = None, location: str = "", format: ConfigFormat = None): + reqi = Save (location, format) + req = Request(save=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_show_config(token: str = None, path: list[str] = [], format: ConfigFormat = None): + reqi = ShowConfig (path, format) + req = Request(show_config=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_exists(token: str = None, path: list[str] = []): + reqi = Exists (path) + req = Request(exists=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_get_value(token: str = None, path: list[str] = [], output_format: OutputFormat = None): + reqi = GetValue (path, output_format) + req = Request(get_value=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_get_values(token: str = None, path: list[str] = [], output_format: OutputFormat = None): + reqi = GetValues (path, output_format) + req = Request(get_values=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_list_children(token: str = None, path: list[str] = [], output_format: OutputFormat = None): + reqi = ListChildren (path, output_format) + req = Request(list_children=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_run_op_mode(token: str = None, path: list[str] = [], output_format: OutputFormat = None): + reqi = RunOpMode (path, output_format) + req = Request(run_op_mode=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_confirm(token: str = None): + reqi = Confirm () + req = Request(confirm=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_enter_configuration_mode(token: str = None, exclusive: bool = False, override_exclusive: bool = False): + reqi = EnterConfigurationMode (exclusive, override_exclusive) + req = Request(enter_configuration_mode=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_exit_configuration_mode(token: str = None): + reqi = ExitConfigurationMode () + req = Request(exit_configuration_mode=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_reload_reftree(token: str = None, on_behalf_of: int = None): + reqi = ReloadReftree (on_behalf_of) + req = Request(reload_reftree=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_show_sessions(token: str = None, exclude_self: bool = False, exclude_other: bool = False): + reqi = ShowSessions (exclude_self, exclude_other) + req = Request(show_sessions=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_set_edit_level(token: str = None, path: list[str] = []): + reqi = SetEditLevel (path) + req = Request(set_edit_level=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_set_edit_level_up(token: str = None, dummy: int = None): + reqi = SetEditLevelUp (dummy) + req = Request(set_edit_level_up=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_reset_edit_level(token: str = None, dummy: int = None): + reqi = ResetEditLevel (dummy) + req = Request(reset_edit_level=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_get_edit_level(token: str = None, dummy: int = None): + reqi = GetEditLevel (dummy) + req = Request(get_edit_level=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_edit_level_root(token: str = None, dummy: int = None): + reqi = EditLevelRoot (dummy) + req = Request(edit_level_root=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_config_unsaved(token: str = None, file: str = None): + reqi = ConfigUnsaved (file) + req = Request(config_unsaved=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_reference_path_exists(token: str = None, path: list[str] = []): + reqi = ReferencePathExists (path) + req = Request(reference_path_exists=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_get_path_type(token: str = None, path: list[str] = [], legacy_format: bool = False): + reqi = GetPathType (path, legacy_format) + req = Request(get_path_type=reqi) + req_env = RequestEnvelope(token, req) + return req_env + +def set_request_get_completion_env(token: str = None, path: list[str] = [], legacy_format: bool = False): + reqi = GetCompletionEnv (path, legacy_format) + req = Request(get_completion_env=reqi) + req_env = RequestEnvelope(token, req) + return req_env diff --git a/python/vyos/qos/__init__.py b/python/vyos/qos/__init__.py index a2980ccde..4bffda2d2 100644 --- a/python/vyos/qos/__init__.py +++ b/python/vyos/qos/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/qos/base.py b/python/vyos/qos/base.py index b477b5b5e..487249714 100644 --- a/python/vyos/qos/base.py +++ b/python/vyos/qos/base.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/qos/cake.py b/python/vyos/qos/cake.py index ca5a26917..05a737649 100644 --- a/python/vyos/qos/cake.py +++ b/python/vyos/qos/cake.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -54,7 +54,16 @@ class CAKE(QoSBase): f'Invalid flow isolation parameter: {config["flow_isolation"]}' ) + if 'ack_filter' in config: + if 'aggressive' in config['ack_filter']: + tmp += ' ack-filter-aggressive' + else: + tmp += ' ack-filter' + else: + tmp += ' no-ack-filter' + tmp += ' nat' if 'flow_isolation_nat' in config else ' nonat' + tmp += ' no-split-gso' if 'no_split_gso' in config else ' split-gso' self._cmd(tmp) diff --git a/python/vyos/qos/droptail.py b/python/vyos/qos/droptail.py index 427d43d19..223ab1e64 100644 --- a/python/vyos/qos/droptail.py +++ b/python/vyos/qos/droptail.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/qos/fairqueue.py b/python/vyos/qos/fairqueue.py index f41d098fb..8f4fe2d47 100644 --- a/python/vyos/qos/fairqueue.py +++ b/python/vyos/qos/fairqueue.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/qos/fqcodel.py b/python/vyos/qos/fqcodel.py index cd2340aa2..d574226ef 100644 --- a/python/vyos/qos/fqcodel.py +++ b/python/vyos/qos/fqcodel.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/qos/limiter.py b/python/vyos/qos/limiter.py index 3f5c11112..dce376d3e 100644 --- a/python/vyos/qos/limiter.py +++ b/python/vyos/qos/limiter.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/qos/netem.py b/python/vyos/qos/netem.py index 8bdef300b..8fdd75387 100644 --- a/python/vyos/qos/netem.py +++ b/python/vyos/qos/netem.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/qos/priority.py b/python/vyos/qos/priority.py index 66d27a639..5f373f696 100644 --- a/python/vyos/qos/priority.py +++ b/python/vyos/qos/priority.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/qos/randomdetect.py b/python/vyos/qos/randomdetect.py index a3a39da36..63445bb62 100644 --- a/python/vyos/qos/randomdetect.py +++ b/python/vyos/qos/randomdetect.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/qos/ratelimiter.py b/python/vyos/qos/ratelimiter.py index a4f80a1be..b0d7b3072 100644 --- a/python/vyos/qos/ratelimiter.py +++ b/python/vyos/qos/ratelimiter.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/qos/roundrobin.py b/python/vyos/qos/roundrobin.py index 509c4069f..d07dc0f52 100644 --- a/python/vyos/qos/roundrobin.py +++ b/python/vyos/qos/roundrobin.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/qos/trafficshaper.py b/python/vyos/qos/trafficshaper.py index 9f92ccd8b..3840e7d0e 100644 --- a/python/vyos/qos/trafficshaper.py +++ b/python/vyos/qos/trafficshaper.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/raid.py b/python/vyos/raid.py index 7fb794817..4ae63a100 100644 --- a/python/vyos/raid.py +++ b/python/vyos/raid.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/referencetree.py b/python/vyos/referencetree.py new file mode 100644 index 000000000..87ed310b4 --- /dev/null +++ b/python/vyos/referencetree.py @@ -0,0 +1,81 @@ +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +from ctypes import cdll, c_char_p, c_void_p, c_bool + +from vyos.defaults import reference_tree_cache + +LIBPATH = '/usr/lib/libvyosconfig.so.0' + + +class ReferenceTreeError(Exception): + pass + + +class ReferenceTree: + # pylint: disable=too-many-instance-attributes + def __init__(self, cache_file=reference_tree_cache, libpath=LIBPATH): + self.__pointer = None + self.__lib = cdll.LoadLibrary(libpath) + + # Import functions + self.__get_error = self.__lib.get_error + self.__get_error.argtypes = [] + self.__get_error.restype = c_char_p + + self.__read_internal = self.__lib.read_internal_reference_tree + self.__read_internal.argtypes = [c_char_p] + self.__read_internal.restype = c_void_p + + self.__write_internal = self.__lib.write_internal_reference_tree + self.__write_internal.argtypes = [c_void_p, c_char_p] + + self.__to_json = self.__lib.to_json_reference_tree + self.__to_json.argtypes = [c_void_p] + self.__to_json.restype = c_char_p + + self.__destroy = self.__lib.destroy + self.__destroy.argtypes = [c_void_p] + + self.__equal = self.__lib.equal + self.__equal.argtypes = [c_void_p, c_void_p] + self.__equal.restype = c_bool + + pointer = self.__read_internal(cache_file.encode()) + if pointer is None: + msg = self.__get_error().decode() + raise ValueError(f'Failed to read internal rep: {msg}') + self.__pointer = pointer + + def __del__(self): + if self.__pointer is not None: + self.__destroy(self.__pointer) + + def __eq__(self, other): + if isinstance(other, ReferenceTree): + return self.__equal(self.get_tree(), other.get_tree()) + return False + + def __str__(self): + return self.to_json() + + def get_tree(self): + return self.__pointer + + def write_cache(self, file_name): + self.__write_internal(self.get_tree(), file_name.encode()) + + def to_json(self): + return self.__to_json(self.__pointer).decode() diff --git a/python/vyos/remote.py b/python/vyos/remote.py index c54fb6031..b69e8d32b 100644 --- a/python/vyos/remote.py +++ b/python/vyos/remote.py @@ -1,4 +1,4 @@ -# Copyright 2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -22,28 +22,42 @@ import stat import sys import tempfile import urllib.parse +import gzip from contextlib import contextmanager from pathlib import Path from ftplib import FTP from ftplib import FTP_TLS +from ftplib import _GLOBAL_DEFAULT_TIMEOUT +from ftplib import error_reply +from ftplib import parse150 -from paramiko import SSHClient, SSHException +from paramiko import SSHClient +from paramiko import SSHException from paramiko import MissingHostKeyPolicy - +from ftplib import FTP +from ftplib import FTP_TLS +from ftplib import _GLOBAL_DEFAULT_TIMEOUT +from ftplib import error_reply +from ftplib import parse150 from requests import Session from requests.adapters import HTTPAdapter -from requests.packages.urllib3 import PoolManager +from urllib.parse import urlsplit +from urllib.parse import urlunsplit +from urllib3 import PoolManager +from urllib3.connection import HTTPConnection from vyos.progressbar import Progressbar from vyos.utils.io import ask_yes_no from vyos.utils.io import is_interactive from vyos.utils.io import print_error from vyos.utils.misc import begin -from vyos.utils.process import cmd, rc_cmd +from vyos.utils.process import cmd +from vyos.utils.process import rc_cmd from vyos.version import get_version from vyos.base import Warning +from vyos.defaults import directories CHUNK_SIZE = 8192 @@ -70,16 +84,27 @@ class SourceAdapter(HTTPAdapter): """ urllib3 transport adapter for setting source addresses per session. """ - def __init__(self, source_pair, *args, **kwargs): - # A source pair is a tuple of a source host string and source port respectively. - # Supply '' and 0 respectively for default values. + def __init__(self, source_pair, vrf=None, *args, **kwargs): self._source_pair = source_pair + self._socket_options = list(HTTPConnection.default_socket_options) + + # Linux VRF binding (requires privileges) + if vrf: + self._socket_options.append( + (socket.SOL_SOCKET, socket.SO_BINDTODEVICE, (vrf + "\0").encode()) + ) + super(SourceAdapter, self).__init__(*args, **kwargs) - def init_poolmanager(self, connections, maxsize, block=False): + def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs): + pool_kwargs["source_address"] = self._source_pair + pool_kwargs["socket_options"] = self._socket_options self.poolmanager = PoolManager( - num_pools=connections, maxsize=maxsize, - block=block, source_address=self._source_pair) + num_pools=connections, + maxsize=maxsize, + block=block, + **pool_kwargs + ) @contextmanager def umask(mask: int): @@ -113,6 +138,99 @@ def check_storage(path, size): if size > shutil.disk_usage(directory).free: raise OSError(f'Not enough disk space available in "{directory}".') +class VRFFTPMixin: + """Shared VRF support for FTP/FTPS sockets (control + data).""" + def __init__(self, *args, vrf=None, **kwargs): + self._vrf = vrf.encode() + b"\0" if vrf else None + super().__init__(*args, **kwargs) + + def _bind_vrf(self, sock): + if self._vrf: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_BINDTODEVICE, self._vrf) + return sock + + def _create_vrf_connection(self, address, timeout, source_address=None): + host, port = address + err = None + for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): + af, socktype, proto, _, sa = res + sock = None + try: + sock = socket.socket(af, socktype, proto) + self._bind_vrf(sock) # bind VRF BEFORE connect + if timeout is not _GLOBAL_DEFAULT_TIMEOUT: + sock.settimeout(timeout) + if source_address: + sock.bind(source_address) + sock.connect(sa) + return sock + except OSError as e: + err = e + if sock is not None: + sock.close() + if err is not None: + raise err + raise OSError("getaddrinfo returns an empty list") + + def connect(self, host="", port=0, timeout=-999, source_address=None): + if host: + self.host = host + if port: + self.port = port + if timeout != -999: + self.timeout = timeout + + self.sock = self._create_vrf_connection( + (self.host, self.port), self.timeout, source_address + ) + self.af = self.sock.family + self.file = self.sock.makefile('r', encoding=self.encoding) + self.welcome = self.getresp() + return self.welcome + + def ntransfercmd(self, cmd, rest=None): + size = None + if self.passiveserver: + host, port = self.makepasv() + conn = self._create_vrf_connection( + (host, port), self.timeout, self.source_address + ) + try: + if rest is not None: + self.sendcmd("REST %s" % rest) + resp = self.sendcmd(cmd) + if resp[0] == '2': + resp = self.getresp() + if resp[0] != '1': + raise error_reply(resp) + except Exception: + conn.close() + raise + else: + with self.makeport() as sock: + if rest is not None: + self.sendcmd("REST %s" % rest) + resp = self.sendcmd(cmd) + if resp[0] == '2': + resp = self.getresp() + if resp[0] != '1': + raise error_reply(resp) + conn, _ = sock.accept() + if self.timeout is not _GLOBAL_DEFAULT_TIMEOUT: + conn.settimeout(self.timeout) + self._bind_vrf(conn) + + if resp[:3] == '150': + size = parse150(resp) + return conn, size + +class VRF_FTP(VRFFTPMixin, FTP): + """FTP client with VRF binding support.""" + pass + +class VRF_FTP_TLS(VRFFTPMixin, FTP_TLS): + """FTPS client with VRF binding support.""" + pass class FtpC: def __init__(self, @@ -121,7 +239,8 @@ class FtpC: check_space=False, source_host='', source_port=0, - timeout=10): + timeout=10, + vrf=None): self.secure = url.scheme == 'ftps' self.hostname = url.hostname self.path = url.path @@ -132,14 +251,18 @@ class FtpC: self.progressbar = progressbar self.check_space = check_space self.timeout = timeout + self.vrf = vrf def _establish(self): if self.secure: - return FTP_TLS(source_address=self.source, - context=ssl.create_default_context(), - timeout=self.timeout) + return VRF_FTP_TLS(source_address=self.source, + context=ssl.create_default_context(), + timeout=self.timeout, + vrf=self.vrf) else: - return FTP(source_address=self.source, timeout=self.timeout) + return VRF_FTP(source_address=self.source, + timeout=self.timeout, + vrf=self.vrf) def download(self, location: str): # Open the file upfront before establishing connection. @@ -182,7 +305,8 @@ class SshC: check_space=False, source_host='', source_port=0, - timeout=10.0): + timeout=10.0, + vrf=None): self.hostname = url.hostname self.path = url.path self.username = url.username or os.getenv('REMOTE_USERNAME') @@ -192,6 +316,7 @@ class SshC: self.progressbar = progressbar self.check_space = check_space self.timeout = timeout + self.vrf = vrf def _establish(self): ssh = SSHClient() @@ -200,9 +325,30 @@ class SshC: if os.path.exists(self.known_hosts): ssh.load_host_keys(self.known_hosts) ssh.set_missing_host_key_policy(InteractivePolicy()) - # `socket.create_connection()` automatically picks a NIC and an IPv4/IPv6 address family - # for us on dual-stack systems. - sock = socket.create_connection((self.hostname, self.port), self.timeout, self.source) + # Create the socket manually so VRF/device binding is applied before + # connect(), while still trying IPv4/IPv6 candidates on dual-stack systems. + sock = None + last_error = None + for family, socktype, proto, _, sockaddr in socket.getaddrinfo( + self.hostname, self.port, type=socket.SOCK_STREAM): + try: + sock = socket.socket(family, socktype, proto) + sock.settimeout(self.timeout) + if self.source != ('', 0): + sock.bind(self.source) + if self.vrf: + vrf = (self.vrf + "\0").encode() + sock.setsockopt(socket.SOL_SOCKET, socket.SO_BINDTODEVICE, vrf) + sock.connect(sockaddr) + break + except OSError as err: + last_error = err + if sock is not None: + sock.close() + sock = None + if sock is None: + raise last_error + ssh.connect(self.hostname, self.port, self.username, self.password, sock=sock) return ssh @@ -218,23 +364,21 @@ class SshC: def upload(self, location: str): with self._establish() as ssh, ssh.open_sftp() as sftp: + # A file exists at this destination. We're simply going to clobber it. + # This is our default fallback + path = self.path try: # If the remote path is a directory, use the original filename. if stat.S_ISDIR(sftp.stat(self.path).st_mode): path = os.path.join(self.path, os.path.basename(location)) - # A file exists at this destination. We're simply going to clobber it. - else: - path = self.path - # This path doesn't point at any existing file. We can freely use this filename. except IOError: - path = self.path - finally: - if self.progressbar: - with Progressbar() as p: - sftp.put(location, path, callback=p.progress) - else: - sftp.put(location, path) + pass + if self.progressbar: + with Progressbar() as p: + sftp.put(location, path, callback=p.progress) + else: + sftp.put(location, path) class HttpC: def __init__(self, @@ -243,7 +387,8 @@ class HttpC: check_space=False, source_host='', source_port=0, - timeout=10.0): + timeout=10.0, + vrf=None): self.urlstring = urllib.parse.urlunsplit(url) self.progressbar = progressbar self.check_space = check_space @@ -251,10 +396,12 @@ class HttpC: self.username = url.username or os.getenv('REMOTE_USERNAME') self.password = url.password or os.getenv('REMOTE_PASSWORD') self.timeout = timeout + self.vrf = vrf def _establish(self): session = Session() - session.mount(self.urlstring, SourceAdapter(self.source_pair)) + adapter = SourceAdapter(self.source_pair, vrf=self.vrf) + session.mount(self.urlstring, adapter) session.headers.update({'User-Agent': 'VyOS/' + get_version()}) if self.username: session.auth = self.username, self.password @@ -315,30 +462,36 @@ class TftpC: check_space=False, source_host=None, source_port=0, - timeout=10): + timeout=10, + vrf=None): source_option = f'--interface {source_host} --local-port {source_port}' if source_host else '' - progress_flag = '--progress-bar' if progressbar else '-s' + progress_flag = '--progress-bar' if progressbar else '--silent' self.command = f'curl {source_option} {progress_flag} --connect-timeout {timeout}' self.urlstring = urllib.parse.urlunsplit(url) + self.vrf = vrf def download(self, location: str): with open(location, 'wb') as f: - f.write(cmd(f'{self.command} "{self.urlstring}"').encode()) + f.write(cmd(f'{self.command} "{self.urlstring}"', + vrf=self.vrf).encode()) def upload(self, location: str): + print(f'{self.command} "{self.urlstring}"') with open(location, 'rb') as f: - cmd(f'{self.command} -T - "{self.urlstring}"', input=f.read()) + cmd(f'{self.command} --upload-file - "{self.urlstring}"', + input=f.read(), vrf=self.vrf) class GitC: def __init__(self, - url, - progressbar=False, - check_space=False, - source_host=None, - source_port=0, - timeout=10, - ): - self.command = 'git' + url, + progressbar=False, + check_space=False, + source_host=None, + source_port=0, + timeout=10, + vrf=None): + self.command = ['git'] + self.vrf = vrf self.url = url self.urlstring = urllib.parse.urlunsplit(url) if self.urlstring.startswith("git+"): @@ -384,9 +537,10 @@ class GitC: path_repository = Path(directory) / "repository" scheme = f"{scheme}://" if scheme else "" rc, out = rc_cmd( - [self.command, "clone", f"{scheme}{netloc}{url}", str(path_repository), "--depth=1"], + self.command + ["clone", f"{scheme}{netloc}{url}", str(path_repository), "--depth=1"], env=env, shell=False, + vrf=self.vrf, ) if rc: raise Exception(out) @@ -396,7 +550,7 @@ class GitC: dst = path_repository / filename shutil.copy2(location, dst) rc, out = rc_cmd( - [self.command, "-C", str(path_repository), "add", filename], + self.command + ["-C", str(path_repository), "add", filename], env=env, shell=False, ) @@ -404,16 +558,17 @@ class GitC: # git commit -m commit_message = os.environ.get("COMMIT_COMMENT", "commit") rc, out = rc_cmd( - [self.command, "-C", str(path_repository), "commit", "-m", commit_message], + self.command + ["-C", str(path_repository), "commit", "-m", commit_message], env=env, shell=False, ) # git push rc, out = rc_cmd( - [self.command, "-C", str(path_repository), "push"], + self.command + ["-C", str(path_repository), "push"], env=env, shell=False, + vrf=self.vrf, ) if rc: raise Exception(out) @@ -456,12 +611,15 @@ def download(local_path, urlstring, progressbar=False, check_space=False, sys.exit(1) def upload(local_path, urlstring, progressbar=False, - source_host='', source_port=0, timeout=10.0): + source_host='', source_port=0, timeout=10.0, vrf=None): try: progressbar = progressbar and is_interactive() - urlc(urlstring, progressbar, False, source_host, source_port, timeout).upload(local_path) + urlc(urlstring, progressbar, False, source_host, source_port, timeout, vrf).upload(local_path) except Exception as err: - print_error(f'Unable to upload "{urlstring}": {err}') + url = urlsplit(urlstring) + _, _, netloc = url.netloc.rpartition('@') + redacted_location = urlunsplit(url._replace(netloc=netloc)) + print_error(f'Unable to upload "{redacted_location}": {err}') sys.exit(1) except KeyboardInterrupt: print_error('\nUpload aborted by user.') @@ -478,3 +636,45 @@ def get_remote_config(urlstring, source_host='', source_port=0): return f.read() finally: os.remove(temp) + + +def get_config_file(file_in: str, file_out: str, source_host='', source_port=0): + protocols = ['scp', 'sftp', 'http', 'https', 'ftp', 'tftp'] + config_dir = directories['config'] + + with tempfile.NamedTemporaryFile() as tmp_file: + if any(file_in.startswith(f'{x}://') for x in protocols): + try: + download( + tmp_file.name, + file_in, + check_space=True, + source_host='', + source_port=0, + raise_error=True, + ) + except Exception as e: + return e + file_name = tmp_file.name + else: + full_path = os.path.realpath(file_in) + if os.path.isfile(full_path): + file_in = full_path + else: + file_in = os.path.join(config_dir, file_in) + if not os.path.isfile(file_in): + return ValueError(f'No such file {file_in}') + + file_name = file_in + + if file_in.endswith('.gz'): + try: + with gzip.open(file_name, 'rb') as f_in: + with open(file_out, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + except Exception as e: + return e + else: + shutil.copyfile(file_name, file_out) + + return None diff --git a/python/vyos/snmpv3_hashgen.py b/python/vyos/snmpv3_hashgen.py index 324c3274d..57dba07a0 100644 --- a/python/vyos/snmpv3_hashgen.py +++ b/python/vyos/snmpv3_hashgen.py @@ -1,4 +1,4 @@ -# Copyright 2020 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/system/__init__.py b/python/vyos/system/__init__.py index 0c91330ba..42af8e3e8 100644 --- a/python/vyos/system/__init__.py +++ b/python/vyos/system/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/system/compat.py b/python/vyos/system/compat.py index d35bddea2..40b38b366 100644 --- a/python/vyos/system/compat.py +++ b/python/vyos/system/compat.py @@ -1,4 +1,4 @@ -# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -14,12 +14,18 @@ # along with this library. If not, see <http://www.gnu.org/licenses/>. from pathlib import Path -from re import compile, MULTILINE, DOTALL +from re import compile +from re import MULTILINE +from re import DOTALL from functools import wraps from copy import deepcopy from typing import Union -from vyos.system import disk, grub, image, SYSTEM_CFG_VER +from vyos.flavor import get_image_serial_console +from vyos.system import disk +from vyos.system import grub +from vyos.system import image +from vyos.system import SYSTEM_CFG_VER from vyos.template import render TMPL_GRUB_COMPAT: str = 'grub/grub_compat.j2' @@ -128,11 +134,15 @@ def parse_entry(entry: tuple) -> dict: entry_dict['bootmode'] = 'pw_reset' else: entry_dict['bootmode'] = 'normal' + (_, _, default_speed) = get_image_serial_console() # find console type and number regex_filter = compile(REGEX_CONSOLE) entry_dict.update(regex_filter.match(entry[1]).groupdict()) + # Set new or default console speed - this line must always be present to + # keep backward compatibility. It is needed to boot into old images and + # use the serial console speed speed = entry_dict.get('console_speed', None) - entry_dict['console_speed'] = speed if speed is not None else '115200' + entry_dict['console_speed'] = speed if speed is not None else default_speed entry_dict['boot_opts'] = sanitize_boot_opts(entry[1]) return entry_dict @@ -179,7 +189,7 @@ def prune_vyos_versions(root_dir: str = '') -> None: def update_cfg_ver(root_dir:str = '') -> int: - """Get minumum version of image-tools across all installed images + """Get minimum version of image-tools across all installed images Args: root_dir (str): an optional path to the root directory diff --git a/python/vyos/system/disk.py b/python/vyos/system/disk.py index c8908cd5c..7ae9a15bb 100644 --- a/python/vyos/system/disk.py +++ b/python/vyos/system/disk.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -50,7 +50,7 @@ def find_persistence() -> str: """Find a mountpoint for persistence storage Returns: - str: Path where 'persistance' pertition is mounted, Empty if not found + str: Path where 'persistence' pertition is mounted, Empty if not found """ mounted_partitions = disk_partitions() for partition in mounted_partitions: @@ -77,7 +77,7 @@ def parttable_create(drive_path: str, root_size: int) -> None: -n3:0:+{root_size_text}K -t3:8300 {drive_path}' run(command) - # update partitons in kernel + # update partitions in kernel sync() run(f'partx -u {drive_path}') diff --git a/python/vyos/system/grub.py b/python/vyos/system/grub.py index de8303ee2..9435d18d2 100644 --- a/python/vyos/system/grub.py +++ b/python/vyos/system/grub.py @@ -1,4 +1,4 @@ -# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,13 +16,18 @@ import platform from pathlib import Path -from re import MULTILINE, compile as re_compile +from re import MULTILINE +from re import compile as re_compile from shutil import copy2 -from uuid import uuid5, NAMESPACE_URL, UUID +from uuid import uuid5 +from uuid import NAMESPACE_URL +from uuid import UUID -from vyos.template import render -from vyos.utils.process import cmd, rc_cmd +from vyos.flavor import get_image_serial_console from vyos.system import disk +from vyos.template import render +from vyos.utils.process import cmd +from vyos.utils.process import rc_cmd # Define variables GRUB_DIR_MAIN: str = '/boot/grub' @@ -385,7 +390,7 @@ def set_console_type(console_type: str, root_dir: str = '') -> None: """Write default console type to GRUB configuration Args: - console_type (str): a default console type + console_type (str): GRUB default console type, e.g. tty, ttyS or ttyAMA root_dir (str, optional): an optional path to the root directory. Defaults to empty. """ @@ -397,10 +402,13 @@ def set_console_type(console_type: str, root_dir: str = '') -> None: vars_current['console_type'] = str(console_type) vars_write(vars_file, vars_current) -def set_console_speed(console_speed: str, root_dir: str = '') -> None: +def set_serial_console(console_type: str, console_num: str, + console_speed: str, root_dir: str = '') -> None: """Write default console speed to GRUB configuration Args: + console_type (str): console device, e.g. 'ttyS' or 'ttyAMA' + console_num (str): console instance, e.g. '0' console_speed (str): default console speed root_dir (str, optional): an optional path to the root directory. Defaults to empty. @@ -408,9 +416,13 @@ def set_console_speed(console_speed: str, root_dir: str = '') -> None: if not root_dir: root_dir = disk.find_persistence() + (default_type, default_num, default_speed) = get_image_serial_console() + vars_file: str = f'{root_dir}/{CFG_VYOS_VARS}' vars_current: dict[str, str] = vars_read(vars_file) - vars_current['console_speed'] = str(console_speed) + vars_current['console_type'] = console_type if console_type else default_type + vars_current['console_num'] = console_num if console_num else default_num + vars_current['console_speed'] = console_speed if console_speed else default_speed vars_write(vars_file, vars_current) def set_kernel_cmdline_options(cmdline_options: str, version_name: str, diff --git a/python/vyos/system/grub_util.py b/python/vyos/system/grub_util.py index 4a3d8795e..edaea8ad3 100644 --- a/python/vyos/system/grub_util.py +++ b/python/vyos/system/grub_util.py @@ -1,4 +1,4 @@ -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,10 +13,14 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see <http://www.gnu.org/licenses/>. -from vyos.system import disk, grub, image, compat +from vyos.system import disk +from vyos.system import grub +from vyos.system import image +from vyos.system import compat @compat.grub_cfg_update -def set_console_speed(console_speed: str, root_dir: str = '') -> None: +def set_serial_console(console_type: str, console_num: str, + console_speed: str, root_dir: str = '') -> None: """Write default console speed to GRUB configuration Args: @@ -27,10 +31,11 @@ def set_console_speed(console_speed: str, root_dir: str = '') -> None: if not root_dir: root_dir = disk.find_persistence() - grub.set_console_speed(console_speed, root_dir) + grub.set_serial_console(console_type, console_num, console_speed, root_dir) @image.if_not_live_boot -def update_console_speed(console_speed: str, root_dir: str = '') -> None: +def update_serial_console(console_type: str, console_num: str, + console_speed: str, root_dir: str = '') -> None: """Update console_speed if different from current value""" if not root_dir: @@ -38,9 +43,15 @@ def update_console_speed(console_speed: str, root_dir: str = '') -> None: vars_file: str = f'{root_dir}/{grub.CFG_VYOS_VARS}' vars_current: dict[str, str] = grub.vars_read(vars_file) - console_speed_current = vars_current.get('console_speed', None) - if console_speed != console_speed_current: - set_console_speed(console_speed, root_dir) + + console_type_current = vars_current.get('console_type') + console_num_current = vars_current.get('console_num') + console_speed_current = vars_current.get('console_speed') + + if console_type != console_type_current or \ + console_num != console_num_current or \ + console_speed != console_speed_current: + set_serial_console(console_type, console_num, console_speed, root_dir) @compat.grub_cfg_update def set_kernel_cmdline_options(cmdline_options: str, version: str = '', @@ -56,13 +67,12 @@ def set_kernel_cmdline_options(cmdline_options: str, version: str = '', @image.if_not_live_boot def update_kernel_cmdline_options(cmdline_options: str, - root_dir: str = '') -> None: + root_dir: str = '', + version = image.get_running_image()) -> None: """Update Kernel custom cmdline options""" if not root_dir: root_dir = disk.find_persistence() - version = image.get_running_image() - boot_opts_current = grub.get_boot_opts(version, root_dir) boot_opts_proposed = grub.BOOT_OPTS_STEM + f'{version} {cmdline_options}' diff --git a/python/vyos/system/image.py b/python/vyos/system/image.py index aae52e770..ed8a96fbb 100644 --- a/python/vyos/system/image.py +++ b/python/vyos/system/image.py @@ -1,4 +1,4 @@ -# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/system/raid.py b/python/vyos/system/raid.py index 5b33d34da..c03764ad1 100644 --- a/python/vyos/system/raid.py +++ b/python/vyos/system/raid.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/template.py b/python/vyos/template.py index e75db1a8d..9d309370e 100755 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -1,4 +1,4 @@ -# Copyright 2019-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -36,6 +36,7 @@ DEFAULT_TEMPLATE_DIR = directories["templates"] # Holds template filters registered via register_filter() _FILTERS = {} _TESTS = {} +_CLEVER_FUNCTIONS = {} # reuse Environments with identical settings to improve performance @functools.lru_cache(maxsize=2) @@ -54,10 +55,11 @@ def _get_environment(location=None): loader=loc_loader, trim_blocks=True, undefined=ChainableUndefined, - extensions=['jinja2.ext.loopcontrols'] + extensions=['jinja2.ext.loopcontrols', 'jinja2.ext.do'] ) env.filters.update(_FILTERS) env.tests.update(_TESTS) + env.globals.update(_CLEVER_FUNCTIONS) return env @@ -77,7 +79,7 @@ def register_filter(name, func=None): "Filters can only be registered before rendering the first template" ) if name in _FILTERS: - raise ValueError(f"A filter with name {name!r} was registered already") + raise ValueError(f"A filter with name {name!r} was already registered") _FILTERS[name] = func return func @@ -97,10 +99,30 @@ def register_test(name, func=None): "Tests can only be registered before rendering the first template" ) if name in _TESTS: - raise ValueError(f"A test with name {name!r} was registered already") + raise ValueError(f"A test with name {name!r} was already registered") _TESTS[name] = func return func +def register_clever_function(name, func=None): + """Register a function to be available as test in templates under given name. + + It can also be used as a decorator, see below in this module for examples. + + :raise RuntimeError: + when trying to register a test after a template has been rendered already + :raise ValueError: when trying to register a name which was taken already + """ + if func is None: + return functools.partial(register_clever_function, name) + if _get_environment.cache_info().currsize: + raise RuntimeError( + "Clever functions can only be registered before rendering the" \ + "first template") + if name in _CLEVER_FUNCTIONS: + raise ValueError(f"A clever function with name {name!r} was already "\ + "registered") + _CLEVER_FUNCTIONS[name] = func + return func def render_to_string(template, content, formater=None, location=None): """Render a template from the template directory, raise on any errors. @@ -150,6 +172,8 @@ def render( # As we are opening the file with 'w', we are performing the rendering before # calling open() to not accidentally erase the file if rendering fails rendered = render_to_string(template, content, formater, location) + # Remove any trailing character and always add a new line at the end + rendered = rendered.rstrip() + "\n" # Write to file with open(destination, "w") as file: @@ -250,9 +274,9 @@ def netmask_from_ipv4(address): Example: - 172.18.201.10 -> 255.255.255.128 """ - from netifaces import interfaces - from netifaces import ifaddresses - from netifaces import AF_INET + from netifaces import interfaces # pylint: disable = no-name-in-module + from netifaces import ifaddresses # pylint: disable = no-name-in-module + from socket import AF_INET for interface in interfaces(): tmp = ifaddresses(interface) if AF_INET in tmp: @@ -390,28 +414,6 @@ def compare_netmask(netmask1, netmask2): except: return False -@register_filter('isc_static_route') -def isc_static_route(subnet, router): - # https://ercpe.de/blog/pushing-static-routes-with-isc-dhcp-server - # Option format is: - # <netmask>, <network-byte1>, <network-byte2>, <network-byte3>, <router-byte1>, <router-byte2>, <router-byte3> - # where bytes with the value 0 are omitted. - from ipaddress import ip_network - net = ip_network(subnet) - # add netmask - string = str(net.prefixlen) + ',' - # add network bytes - if net.prefixlen: - width = net.prefixlen // 8 - if net.prefixlen % 8: - width += 1 - string += ','.join(map(str,tuple(net.network_address.packed)[:width])) + ',' - - # add router bytes - string += ','.join(router.split('.')) - - return string - @register_filter('is_file') def is_file(filename): if os.path.exists(filename): @@ -420,21 +422,25 @@ def is_file(filename): @register_filter('get_dhcp_router') def get_dhcp_router(interface): - """ Static routes can point to a router received by a DHCP reply. This + """Static routes can point to a router received by a DHCP reply. This helper is used to get the current default router from the DHCP reply. - Returns False of no router is found, returns the IP address as string if + Returns None if no router is found, returns the IP address as string if a router is found. """ - lease_file = directories['isc_dhclient_dir'] + f'/dhclient_{interface}.leases' + lease_file = directories['isc_dhclient_dir'] + f'/dhclient_{interface}.lease' if not os.path.exists(lease_file): return None from vyos.utils.file import read_file for line in read_file(lease_file).splitlines(): - if 'option routers' in line: - (_, _, address) = line.split() - return address.rstrip(';') + if 'new_routers' in line: + (_, address, _) = line.split("'") + if not address: + return None + # Take first one if there are several + address = address.split()[0] + return address @register_filter('natural_sort') def natural_sort(iterable): @@ -566,6 +572,11 @@ def get_openvpn_data_ciphers(ciphers): out.append(cipher) return ':'.join(out).upper() + +@register_filter('openvpn_data_ciphers_fallback') +def get_openvpn_data_ciphers_fallback(cipher): + return get_openvpn_cipher(cipher) + @register_filter('snmp_auth_oid') def snmp_auth_oid(type): if type not in ['md5', 'sha', 'aes', 'des', 'none']: @@ -580,6 +591,10 @@ def snmp_auth_oid(type): } return OIDs[type] +@register_filter('quoted_join') +def quoted_join(input_list, join_str, quote='"'): + return str(join_str).join(f'{quote}{elem}{quote}' for elem in input_list) + @register_filter('nft_action') def nft_action(vyos_action): if vyos_action == 'accept': @@ -672,6 +687,29 @@ def nft_nested_group(out_list, includes, groups, key): add_includes(name) return out_list +@register_filter('nft_accept_invalid') +def nft_accept_invalid(ether_type): + ether_type_mapping = { + 'dhcp': 'udp sport 67 udp dport 68', + 'arp': 'arp', + 'pppoe-discovery': '0x8863', + 'pppoe': '0x8864', + '802.1q': '8021q', + '802.1ad': '8021ad', + 'wol': '0x0842', + } + if ether_type not in ether_type_mapping: + raise RuntimeError(f'Ethernet type "{ether_type}" not found in ' \ + 'available ethernet types!') + out = 'ct state invalid ' + + if ether_type != 'dhcp': + out += 'ether type ' + + out += f'{ether_type_mapping[ether_type]} counter accept' + + return out + @register_filter('nat_rule') def nat_rule(rule_conf, rule_id, nat_type, ipv6=False): from vyos.nat import parse_nat_rule @@ -726,7 +764,7 @@ def conntrack_rule(rule_conf, rule_id, action, ipv6=False): if port[0] == '!': operator = '!=' port = port[1:] - output.append(f'th {prefix}port {operator} {port}') + output.append(f'th {prefix}port {operator} {{ {port} }}') if 'group' in side_conf: group = side_conf['group'] @@ -881,10 +919,96 @@ def kea_high_availability_json(config): return dumps(data) +@register_filter('kea_client_class_json') +def kea_client_class_json(client_classes): + from vyos.kea import kea_build_client_class_test + from json import dumps + out = [] + + for name, config in client_classes.items(): + if 'disable' in config: + continue + + client_class = { + 'name': name, + 'test': kea_build_client_class_test(config) + } + + out.append(client_class) + + return dumps(out, indent=4) + +@register_filter('kea_dynamic_dns_update_main_json') +def kea_dynamic_dns_update_main_json(config): + from vyos.kea import kea_parse_ddns_settings + from json import dumps + + data = kea_parse_ddns_settings(config) + + if len(data) == 0: + return '' + + return dumps(data, indent=8)[1:-1] + ',' + +@register_filter('kea_dynamic_dns_update_tsig_key_json') +def kea_dynamic_dns_update_tsig_key_json(config): + from vyos.kea import kea_parse_tsig_algo + from json import dumps + out = [] + + if 'tsig_key' not in config: + return dumps(out) + + tsig_keys = config['tsig_key'] + + for tsig_key_name, tsig_key_config in tsig_keys.items(): + tsig_key = { + 'name': tsig_key_name, + 'algorithm': kea_parse_tsig_algo(tsig_key_config['algorithm']), + 'secret': tsig_key_config['secret'] + } + out.append(tsig_key) + + return dumps(out, indent=12) + +@register_filter('kea_dynamic_dns_update_domains') +def kea_dynamic_dns_update_domains(config, type_key): + from json import dumps + out = [] + + if type_key not in config: + return dumps(out) + + domains = config[type_key] + + for domain_name, domain_config in domains.items(): + domain = { + 'name': domain_name, + + } + if 'key_name' in domain_config: + domain['key-name'] = domain_config['key_name'] + + if 'dns_server' in domain_config: + dns_servers = [] + for dns_server_config in domain_config['dns_server'].values(): + dns_server = { + 'ip-address': dns_server_config['address'] + } + if 'port' in dns_server_config: + dns_server['port'] = int(dns_server_config['port']) + dns_servers.append(dns_server) + domain['dns-servers'] = dns_servers + + out.append(domain) + + return dumps(out, indent=12) + @register_filter('kea_shared_network_json') def kea_shared_network_json(shared_networks): from vyos.kea import kea_parse_options from vyos.kea import kea_parse_subnet + from vyos.kea import kea_parse_ddns_settings from json import dumps out = [] @@ -895,9 +1019,13 @@ def kea_shared_network_json(shared_networks): network = { 'name': name, 'authoritative': ('authoritative' in config), - 'subnet4': [] + 'subnet4': [], + 'user-context': {'enable-ping-check': False} } + if 'dynamic_dns_update' in config: + network.update(kea_parse_ddns_settings(config['dynamic_dns_update'])) + if 'option' in config: network['option-data'] = kea_parse_options(config['option']) @@ -907,12 +1035,21 @@ def kea_shared_network_json(shared_networks): if 'bootfile_server' in config['option']: network['next-server'] = config['option']['bootfile_server'] + subnet_ping_check = False + if 'subnet' in config: for subnet, subnet_config in config['subnet'].items(): if 'disable' in subnet_config: continue + + if 'ping_check' in subnet_config: + subnet_ping_check = True + network['subnet4'].append(kea_parse_subnet(subnet, subnet_config)) + if 'ping_check' in config or subnet_ping_check: + network['user-context']['enable-ping-check'] = True + out.append(network) return dumps(out, indent=4) @@ -998,3 +1135,48 @@ def vyos_defined(value, test_value=None, var_type=None): else: # Valid value and is matching optional argument if provided - return true return True + +@register_clever_function('get_default_port') +def get_default_port(service): + """ + Jinja2 plugin to retrieve common service port number from vyos.defaults + class from a Jinja2 template. This removes the need to hardcode, or pass in + the data using the general dictionary. + + Added to remove code complexity and make it easier to read. + + Example: + {{ get_default_port('certbot_haproxy') }} + """ + from vyos.defaults import internal_ports + if service not in internal_ports: + raise RuntimeError(f'Service "{service}" not found in internal ' \ + 'vyos.defaults.internal_ports dict!') + return internal_ports[service] + +@register_clever_function('get_default_config_file') +def get_default_config_file(filename): + """ + Jinja2 plugin to retrieve a common configuration file path from + vyos.defaults class from a Jinja2 template. This removes the need to + hardcode, or pass in the data using the general dictionary. + + Added to remove code complexity and make it easier to read. + + Example: + {{ get_default_config_file('certbot_haproxy') }} + """ + from vyos.defaults import config_files + if filename not in config_files: + raise RuntimeError(f'Configuration file "{filename}" not found in '\ + 'internal vyos.defaults.config_files dict!') + return config_files[filename] + + +@register_filter('parse_url') +def parse_url(url): + """Parse the given URL and return a urllib.parse.ParseResult object""" + from urllib.parse import urlparse + + parsed = urlparse(url) + return parsed diff --git a/python/vyos/tpm.py b/python/vyos/tpm.py index a24f149fd..663490dec 100644 --- a/python/vyos/tpm.py +++ b/python/vyos/tpm.py @@ -1,4 +1,4 @@ -# Copyright (C) 2024 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as diff --git a/python/vyos/utils/__init__.py b/python/vyos/utils/__init__.py index 3759b2125..280cde17f 100644 --- a/python/vyos/utils/__init__.py +++ b/python/vyos/utils/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/utils/activate.py b/python/vyos/utils/activate.py new file mode 100644 index 000000000..51b92a4aa --- /dev/null +++ b/python/vyos/utils/activate.py @@ -0,0 +1,126 @@ +# Copyright (C) VyOS Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + + +import json +import typing +from pathlib import Path + +from vyos.base import Warning as Warn +from vyos.defaults import activation_list +from vyos.defaults import activation_init +from vyos.defaults import activation_hint +from vyos.defaults import directories + + +ActiveOpt = typing.Literal['enabled', 'once', 'off', 'never'] + + +def get_activation_scripts() -> dict: + list_path = Path(activation_list) + return json.loads(list_path.read_text()) + + +def set_activation(file_name: str, value: ActiveOpt): + script_dict = get_activation_scripts() + file_key = Path(file_name).stem + script_dict[file_key] = value + list_path = Path(activation_list) + list_path.write_text(json.dumps(script_dict)) + + +def get_activation(file_name: str) -> ActiveOpt: + script_dict = get_activation_scripts() + file_key = Path(file_name).stem + return script_dict[file_key] + + +def is_active(file_name: str) -> bool: + script_dict = get_activation_scripts() + file_key = Path(file_name).stem + if script_dict[file_key] in ('enabled', 'once'): + return True + return False + + +def stable_update(new: dict, old: dict): + res = {} + for key in new.keys(): + res[key] = old[key] if key in old.keys() else new[key] + return res + + +def init_activation_list() -> bool: + """Init if activation_hint exists, left on install_image or if image was + built as raw_image""" + + init_hint = Path(activation_hint) + if not init_hint.exists(): + return False + + init_hint.unlink() + init_list = Path(activation_init) + data_list = Path(activation_list) + data_obj = json.loads(init_list.read_text()) + data_list.write_text(json.dumps(data_obj)) + + return True + + +def refresh_activation_list(): + """Refresh activation list, as will be needed after image update""" + + if init_activation_list(): + return + + new_list_path = Path(directories['data']).joinpath(Path(activation_list).name) + if not new_list_path.exists(): + return + + new_obj = json.loads(new_list_path.read_text()) + + orig_list_path = Path(activation_list) + if orig_list_path.exists(): + orig_obj = json.loads(orig_list_path.read_text()) + if orig_obj == new_obj: + return + + obj = stable_update(new_obj, orig_obj) + else: + obj = new_obj + + orig_list_path.write_text(json.dumps(obj)) + + +first_installed_boot_file = '/run/first_installed_boot' + + +def set_first_installed_boot(): + try: + Path(first_installed_boot_file).touch(exist_ok=False) + except FileExistsError: + Warn('redundant set of first_installed_boot') + + +def is_first_installed_boot(): + return Path(first_installed_boot_file).exists() + + +def set_config_path_hint(): + """The config hint allows subsequent installs to find previous disk + resident config data. It is traditionally added as part of the image + install procedure, however, for raw image builds an alternative is + needed.""" + Path(directories['config']).joinpath('.vyatta_config').touch(exist_ok=True) diff --git a/python/vyos/utils/assertion.py b/python/vyos/utils/assertion.py index c7fa220c3..35baa556b 100644 --- a/python/vyos/utils/assertion.py +++ b/python/vyos/utils/assertion.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -57,7 +57,7 @@ def assert_mac(m, test_all_zero=True): split = m.split(':') size = len(split) - # a mac address consits out of 6 octets + # a mac address consists out of 6 octets if size != 6: raise ValueError(f'wrong number of MAC octets ({size}): {m}') diff --git a/python/vyos/utils/auth.py b/python/vyos/utils/auth.py index 5d0e3464a..7123bd0a5 100644 --- a/python/vyos/utils/auth.py +++ b/python/vyos/utils/auth.py @@ -1,6 +1,6 @@ # authutils -- miscelanneous functions for handling passwords and publis keys # -# Copyright (C) 2023-2024 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or modify it under the terms of # the GNU Lesser General Public License as published by the Free Software Foundation; @@ -18,11 +18,39 @@ import math import re import string -from enum import StrEnum +from dataclasses import dataclass from decimal import Decimal -from vyos.utils.process import cmd +from enum import StrEnum +from typing import List +from typing import Optional +from vyos.utils.process import cmd +# Minimum UID used when adding system users +MIN_USER_UID: int = 1000 +# Maximum UID used when adding system users +MAX_USER_UID: int = 59999 +# List of local user accounts that must be preserved +SYSTEM_USER_SKIP_LIST: frozenset = { + 'radius_user', + 'radius_priv_user', + 'tacacs0', + 'tacacs1', + 'tacacs2', + 'tacacs3', + 'tacacs4', + 'tacacs5', + 'tacacs6', + 'tacacs7', + 'tacacs8', + 'tacacs9', + 'tacacs10', + 'tacacs11', + 'tacacs12', + 'tacacs13', + 'tacacs14', + 'tacacs15', +} DEFAULT_PASSWORD: str = 'vyos' LOW_ENTROPY_MSG: str = 'should be at least 8 characters long;' WEAK_PASSWORD_MSG: str = 'The password complexity is too low - @MSG@' @@ -90,7 +118,7 @@ def evaluate_strength(passwd: str) -> dict[str, str]: def make_password_hash(password): """ Makes a password hash for /etc/shadow using mkpasswd """ - mkpassword = 'mkpasswd --method=sha-512 --stdin' + mkpassword = 'mkpasswd --method=yescrypt --stdin' return cmd(mkpassword, input=password, timeout=5) def split_ssh_public_key(key_string, defaultname=""): @@ -119,3 +147,76 @@ def get_current_user() -> str: elif 'USER' in os.environ: current_user = os.environ['USER'] return current_user + +@dataclass +class PasswdEntry: + pw_name: str + pw_passwd: str + pw_uid: int + pw_gid: int + pw_gecos: str + pw_dir: str + pw_shell: str + +def get_local_passwd_entries(uid: Optional[int] = None) -> PasswdEntry | List[PasswdEntry] | None: + """ + If uid is None: return a list of all passwd entries. + If uid is given: return the matching entry or None. + """ + entries = [] + with open('/etc/passwd', 'r') as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split(":") + if len(parts) != 7: + continue + + try: + entry = PasswdEntry( + pw_name=parts[0], + pw_passwd=parts[1], + pw_uid=int(parts[2]), + pw_gid=int(parts[3]), + pw_gecos=parts[4], + pw_dir=parts[5], + pw_shell=parts[6], + ) + except ValueError: + # Skip entries with non-numeric UID or GID + continue + + # If searching for a specific UID, return immediately if found + if uid is not None and entry.pw_uid == uid: + return entry + + entries.append(entry) + + # uid given but not found + if uid is not None: + return None + + return entries + +def get_local_users(min_uid=MIN_USER_UID, max_uid=MAX_USER_UID) -> list: + """Return list of dynamically allocated users (see Debian Policy Manual)""" + local_users = [] + + for s_user in get_local_passwd_entries(): + if s_user.pw_uid < min_uid: + continue + if s_user.pw_uid > max_uid: + continue + if s_user.pw_name in SYSTEM_USER_SKIP_LIST: + continue + local_users.append(s_user.pw_name) + + return local_users + +def get_user_home_dir(user: str) -> str: + """Return user's home directory""" + for u in get_local_passwd_entries(): + if u.pw_name == user: + return u.pw_dir + raise KeyError(f"User '{user}' not found in /etc/passwd") diff --git a/python/vyos/utils/backend.py b/python/vyos/utils/backend.py new file mode 100644 index 000000000..d302a2efd --- /dev/null +++ b/python/vyos/utils/backend.py @@ -0,0 +1,94 @@ +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +# N.B. the following is a temporary addition for running smoketests under +# vyconf and is not to be called explicitly, at the risk of catastophe. + +# pylint: disable=wrong-import-position + +from pathlib import Path + +from vyos.utils.io import ask_yes_no +from vyos.utils.process import call +from vyos.utils.process import is_systemd_service_active + +VYCONF_SENTINEL = '/run/vyconf_backend' + +MSG_ENABLE_VYCONF = 'This will enable the vyconf backend for testing. Proceed?' +MSG_DISABLE_VYCONF = ( + 'This will restore the legacy backend; it requires a reboot. Proceed?' +) + +# read/set immutable file attribute without popen: +# https://www.geeklab.info/2021/04/chattr-and-lsattr-in-python/ +import fcntl # pylint: disable=C0411 # noqa: E402 +from array import array # pylint: disable=C0411 # noqa: E402 + +# FS constants - see /uapi/linux/fs.h in kernel source +# or <elixir.free-electrons.com/linux/latest/source/include/uapi/linux/fs.h> +FS_IOC_GETFLAGS = 0x80086601 +FS_IOC_SETFLAGS = 0x40086602 +FS_IMMUTABLE_FL = 0x010 + + +def chattri(filename: str, value: bool): + with open(filename, 'r') as f: + arg = array('L', [0]) + fcntl.ioctl(f.fileno(), FS_IOC_GETFLAGS, arg, True) + if value: + arg[0] = arg[0] | FS_IMMUTABLE_FL + else: + arg[0] = arg[0] & ~FS_IMMUTABLE_FL + fcntl.ioctl(f.fileno(), FS_IOC_SETFLAGS, arg, True) + + +def lsattri(filename: str) -> bool: + with open(filename, 'r') as f: + arg = array('L', [0]) + fcntl.ioctl(f.fileno(), FS_IOC_GETFLAGS, arg, True) + return bool(arg[0] & FS_IMMUTABLE_FL) + + +# End: read/set immutable file attribute without popen + + +def vyconf_backend() -> bool: + return Path(VYCONF_SENTINEL).exists() and lsattri(VYCONF_SENTINEL) + + +def set_vyconf_backend(value: bool, no_prompt: bool = False): + vyconfd_service = 'vyconfd.service' + commitd_service = 'vyos-commitd.service' + http_api_service = 'vyos-http-api.service' + match value: + case True: + if vyconf_backend(): + return + if not no_prompt and not ask_yes_no(MSG_ENABLE_VYCONF): + return + Path(VYCONF_SENTINEL).touch() + chattri(VYCONF_SENTINEL, True) + call(f'systemctl restart {vyconfd_service}') + call(f'systemctl restart {commitd_service}') + if is_systemd_service_active(http_api_service): + call(f'systemctl restart {http_api_service}') + case False: + if not vyconf_backend(): + return + if not no_prompt and not ask_yes_no(MSG_DISABLE_VYCONF): + return + chattri(VYCONF_SENTINEL, False) + Path(VYCONF_SENTINEL).unlink() + call('/sbin/shutdown -r now') diff --git a/python/vyos/utils/boot.py b/python/vyos/utils/boot.py index 708bef14d..f804cd94e 100644 --- a/python/vyos/utils/boot.py +++ b/python/vyos/utils/boot.py @@ -1,4 +1,4 @@ -# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/utils/commit.py b/python/vyos/utils/commit.py index 105aed8c2..1bbb236e4 100644 --- a/python/vyos/utils/commit.py +++ b/python/vyos/utils/commit.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,8 +13,70 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. +# pylint: disable=import-outside-toplevel + +from typing import IO + + +def _commit_lock_busy(lock_path: str) -> bool: + """Return True if another process holds a POSIX advisory lock on lock_path. + + Uses libc lockf(F_TEST): never acquires or releases a lock (no observer window + where this code holds LOCK_EX). Compatible with locks taken via fcntl.lockf / + fcntl F_SETLK on Linux. + """ + import ctypes + import errno + import os + + libc = ctypes.CDLL('libc.so.6', use_errno=True) + lockf_fn = libc.lockf + lockf_fn.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_long] + lockf_fn.restype = ctypes.c_int + + # Defined in glibc <unistd.h> / <fcntl.h> as F_TEST. + _F_TEST = 3 + + try: + fd = os.open(lock_path, os.O_RDONLY) + except FileNotFoundError: + # Lost a race with unlink or commit teardown. + return False + try: + os.lseek(fd, 0, os.SEEK_SET) + ctypes.set_errno(0) + ret = lockf_fn(fd, _F_TEST, 0) + err = ctypes.get_errno() + if ret == 0: + return False + if err in (errno.EACCES, errno.EAGAIN): + return True + raise OSError(err, os.strerror(err), lock_path) + finally: + os.close(fd) + +def commit_in_progress2(): + """ + Modern implementation of commit_in_progress() which is O(1) instead of O(n) + + The reason not everything is moved to this new implementation yet is to + give it heavy testing in vyos-netlinkd first. + """ + # Query advisory locks without acquiring them (see _commit_lock_busy). + # Requires read access to the lock file. + # If there is no read access otherwise os.open raises PermissionError. + + from pathlib import Path + from vyos.defaults import commit_lock + + lock_path = Path(commit_lock) + if not lock_path.exists(): + return False + + return _commit_lock_busy(str(lock_path)) + def commit_in_progress(): - """ Not to be used in normal op mode scripts! """ + """Not to be used in normal op mode scripts!""" # The CStore backend locks the config by opening a file # The file is not removed after commit, so just checking @@ -36,7 +98,9 @@ def commit_in_progress(): from vyos.defaults import commit_lock if getuser() != 'root': - raise OSError('This functions needs to be run as root to return correct results!') + raise OSError( + 'This functions needs to be run as root to return correct results!' + ) for proc in process_iter(): try: @@ -45,7 +109,7 @@ def commit_in_progress(): for f in files: if f.path == commit_lock: return True - except NoSuchProcess as err: + except NoSuchProcess: # Process died before we could examine it pass # Default case @@ -53,8 +117,71 @@ def commit_in_progress(): def wait_for_commit_lock(): - """ Not to be used in normal op mode scripts! """ + """Not to be used in normal op mode scripts!""" from time import sleep + # Very synchronous approach to multiprocessing while commit_in_progress(): sleep(1) + + +# For transitional compatibility with the legacy commit locking mechanism, +# we require a lockf/fcntl (POSIX-type) lock, hence the following in place +# of vyos.utils.locking + + +def acquire_commit_lock_file() -> tuple[IO, str]: + import fcntl + from pathlib import Path + from vyos.defaults import commit_lock + + try: + # pylint: disable=consider-using-with + lock_fd = Path(commit_lock).open('w') + except IOError as e: + out = f'Critical error opening commit lock file {e}' + return None, out + + try: + fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + return lock_fd, '' + except IOError: + out = 'Configuration system locked by another commit in progress' + lock_fd.close() + return None, out + + +def release_commit_lock_file(file_descr): + import fcntl + + if file_descr is None: + return + fcntl.lockf(file_descr, fcntl.LOCK_UN) + file_descr.close() + + +def call_commit_hooks(which: str): + import re + import os + from pathlib import Path + from vyos.defaults import commit_hooks + from vyos.utils.process import rc_cmd + + if which not in list(commit_hooks): + raise ValueError(f'no entry {which} in commit_hooks') + + hook_dir = commit_hooks[which] + file_list = list(Path(hook_dir).glob('*')) + regex = re.compile('^[a-zA-Z0-9._-]+$') + hook_list = sorted([str(f) for f in file_list if regex.match(f.name)]) + err = False + out = '' + for runf in hook_list: + try: + e, o = rc_cmd(runf) + except FileNotFoundError: + continue + err = err | bool(e) + out = out + o + + return out, int(err) diff --git a/python/vyos/utils/config.py b/python/vyos/utils/config.py index deda13c13..1032e22dc 100644 --- a/python/vyos/utils/config.py +++ b/python/vyos/utils/config.py @@ -1,4 +1,4 @@ -# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -44,6 +44,37 @@ def read_saved_value(path: list): return ' '.join(res) return res +def write_saved_value(path: list, value=None, config_path: str=config_file): + """Write or replace a node in a saved configuration. + + - If value is None, the node is treated as valueless. + - Tag nodes in the path are detected via XML reference tree and marked on + the ConfigTree prior to setting the node. + """ + if not isinstance(path, list) or not path: + raise ValueError('path must be a non-empty list') + + from vyos.configtree import ConfigTree + from vyos.utils.file import read_file + from vyos.utils.file import write_file + + config_string = read_file(config_path) + ct = ConfigTree(config_string) + + # ConfigTree.set_tag() requires the node to exist. + # Create missing nodes along the path so tag marking works even when + # writing a completely new subtree into config.boot. + for target in flag(path): + if not ct.exists(target): + ct.create_node(target) + + set_tags(ct, path) + + ct.set(path, value=value, replace=True) + set_leaf(ct, path) + + write_file(config_path, ct.to_string()) + def flag(l: list) -> list: res = [l[0:i] for i,_ in enumerate(l, start=1)] return res @@ -63,6 +94,11 @@ def set_tags(ct: 'ConfigTree', path: list) -> None: if condition: ct.set_tag(target) +def set_leaf(ct: 'ConfigTree', path: list) -> None: + from vyos.xml_ref import is_leaf + if is_leaf(path): + ct.set_leaf(path, True) + def parse_commands(cmds: str) -> dict: from re import split as re_split from shlex import split as shlex_split diff --git a/python/vyos/utils/configfs.py b/python/vyos/utils/configfs.py index 8617f0129..72332ae42 100644 --- a/python/vyos/utils/configfs.py +++ b/python/vyos/utils/configfs.py @@ -1,4 +1,4 @@ -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,25 +13,57 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. +# pylint: disable=import-outside-toplevel + import os +from vyos.utils.backend import vyconf_backend +from vyos.utils.boot import boot_configuration_complete + + def delete_cli_node(cli_path: list): - from shutil import rmtree - for config_dir in ['VYATTA_TEMP_CONFIG_DIR', 'VYATTA_CHANGES_ONLY_DIR']: - tmp = os.path.join(os.environ[config_dir], '/'.join(cli_path)) - # delete CLI node - if os.path.exists(tmp): - rmtree(tmp) - -def add_cli_node(cli_path: list, value: str=None): - from vyos.utils.auth import get_current_user - from vyos.utils.file import write_file - - current_user = get_current_user() - for config_dir in ['VYATTA_TEMP_CONFIG_DIR', 'VYATTA_CHANGES_ONLY_DIR']: - # store new value - tmp = os.path.join(os.environ[config_dir], '/'.join(cli_path)) - write_file(f'{tmp}/node.val', value, user=current_user, group='vyattacfg', mode=0o664) - # mark CLI node as modified - if config_dir == 'VYATTA_CHANGES_ONLY_DIR': - write_file(f'{tmp}/.modified', '', user=current_user, group='vyattacfg', mode=0o664) + if vyconf_backend() and boot_configuration_complete(): + # pylint: disable=redefined-outer-name + from vyos.utils.session import delete_cli_node + + delete_cli_node(cli_path) + else: + from shutil import rmtree + + for config_dir in ['VYATTA_TEMP_CONFIG_DIR', 'VYATTA_CHANGES_ONLY_DIR']: + tmp = os.path.join(os.environ[config_dir], '/'.join(cli_path)) + # delete CLI node + if os.path.exists(tmp): + rmtree(tmp) + + +def add_cli_node(cli_path: list, value: str = None): + if vyconf_backend() and boot_configuration_complete(): + # pylint: disable=redefined-outer-name + from vyos.utils.session import add_cli_node + + add_cli_node(cli_path, value) + else: + from vyos.utils.auth import get_current_user + from vyos.utils.file import write_file + + current_user = get_current_user() + for config_dir in ['VYATTA_TEMP_CONFIG_DIR', 'VYATTA_CHANGES_ONLY_DIR']: + # store new value + tmp = os.path.join(os.environ[config_dir], '/'.join(cli_path)) + write_file( + f'{tmp}/node.val', + value, + user=current_user, + group='vyattacfg', + mode=0o664, + ) + # mark CLI node as modified + if config_dir == 'VYATTA_CHANGES_ONLY_DIR': + write_file( + f'{tmp}/.modified', + '', + user=current_user, + group='vyattacfg', + mode=0o664, + ) diff --git a/python/vyos/utils/convert.py b/python/vyos/utils/convert.py index 2f587405d..bdc9fd549 100644 --- a/python/vyos/utils/convert.py +++ b/python/vyos/utils/convert.py @@ -1,4 +1,4 @@ -# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -261,3 +261,56 @@ def encode_to_base64(input_string): # Decode the base64 bytes back to a string return encoded_string.decode('utf-8') + + +def range_str_to_list(data: str) -> list[int]: + """ + Convert a string of ranges to a sorted list of unique integers + + Args: + data (str): Comma-separated ranges, e.g. '1-3,5,7-9' + + Returns: + Sorted list of unique integers, e.g. ``[1, 2, 3, 5, 7, 8, 9]``. + """ + if not data: + return [] + result = [] + for part in data.split(','): + if '-' in part: + start, end = part.split('-') + result.extend(range(int(start), int(end) + 1)) + else: + result.append(int(part)) + return sorted(set(result)) + + +def list_to_range_str(nums: list[int]) -> str: + """ + Convert a list of integers to a compact string of ranges + + Args: + nums: List of integers, e.g. [1, 2, 3, 5, 7, 8, 9]. + Duplicates are removed and the list is sorted internally. + + Returns: + Compact range string, e.g. '1-3,5,7-9'. + Returns '' for an empty input. + """ + nums = sorted(set(nums)) + if not nums: + return '' + + ranges = [] + start = end = nums[0] + + for n in nums[1:]: + if n == end + 1: + end = n + else: + ranges.append(str(start) if start == end else f'{start}-{end}') + start = end = n + + ranges.append(str(start) if start == end else f'{start}-{end}') + + return ','.join(ranges) diff --git a/python/vyos/utils/cpu.py b/python/vyos/utils/cpu.py index 8ace77d15..0f47123a4 100644 --- a/python/vyos/utils/cpu.py +++ b/python/vyos/utils/cpu.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2024 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -26,6 +26,7 @@ It has special cases for x86_64 and MAY work correctly on other architectures, but nothing is certain. """ +import os import re def _read_cpuinfo(): @@ -114,3 +115,8 @@ def get_available_cpus(): out = json.loads(cmd('lscpu --extended -b --json')) return out['cpus'] + + +def get_half_cpus(): + """ return 1/2 of the numbers of available CPUs """ + return max(1, os.cpu_count() // 2) diff --git a/python/vyos/utils/dict.py b/python/vyos/utils/dict.py index 1a7a6b96f..3fcda902f 100644 --- a/python/vyos/utils/dict.py +++ b/python/vyos/utils/dict.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -145,24 +145,33 @@ def get_sub_dict(source, lpath, get_first_key=False): return ret -def dict_search(path, dict_object): +def dict_search(path, dict_object, default=None): """ Traverse Python dictionary (dict_object) delimited by dot (.). - Return value of key if found, None otherwise. + + Args: + path (str): Dot-delimited key path, e.g. "foo.bar.baz". + dict_object (dict): The dictionary to search. + default (Any, optional): Value to return if the path is not found + or if dict_object is not a dict. Defaults to None. + + Returns: + Any: The value found at the given path, or None if not found. Optionally, + a default value can be provided to be returned. This is faster implementation then jmespath.search('foo.bar', dict_object)""" if not isinstance(dict_object, dict) or not path: - return None + return default parts = path.split('.') inside = parts[:-1] if not inside: if path not in dict_object: - return None + return default return dict_object[path] c = dict_object for p in parts[:-1]: c = c.get(p, {}) - return c.get(parts[-1], None) + return c.get(parts[-1], default) def dict_search_args(dict_object, *path): # Traverse dictionary using variable arguments @@ -178,7 +187,7 @@ def dict_search_args(dict_object, *path): return dict_object def dict_search_recursive(dict_object, key, path=[]): - """ Traverse a dictionary recurisvely and return the value of the key + """ Traverse a dictionary recursively and return the value of the key we are looking for. Thankfully copied from https://stackoverflow.com/a/19871956 @@ -211,6 +220,39 @@ def dict_set(key_path, value, dict_object): dynamic_dict = dynamic_dict[path_list[i]] dynamic_dict[path_list[len(path_list)-1]] = value +def dict_set_nested(key_path, value, dict_object): + """ + Set value to Python dictionary (dict_object) using a path to the key + delimited by dot ('.'). The key will be added if it does not exist. + Missing keys along the path will be created as nested dictionaries. + + Parameters + ---------- + key_path : str + Dot-delimited path to the key (e.g. "this.is.a.path"). + value : any + The value to set at the final key in the path. + dict_object : dict + Dictionary to modify. Will be updated in place. + + Examples + -------- + d = {} + dict_set_nested("this.is.a.path", 42, d) + # {'this': {'is': {'a': {'path': 42}}}} + + d = {"existing": {"branch": {}}} + dict_set_nested("existing.branch.leaf", "value", d) + # {'existing': {'branch': {'leaf': 'value'}}} + """ + path_list = key_path.split(".") + dynamic_dict = dict_object + for i in range(0, len(path_list) - 1): + if path_list[i] not in dynamic_dict or not isinstance(dynamic_dict[path_list[i]], dict): + dynamic_dict[path_list[i]] = {} + dynamic_dict = dynamic_dict[path_list[i]] + dynamic_dict[path_list[-1]] = value + def dict_delete(key_path, dict_object): """ Delete key in Python dictionary (dict_object) using path to key delimited by dot (.). """ @@ -338,7 +380,7 @@ def check_mutually_exclusive_options(d, keys, required=False): class FixedDict(dict): """ - FixedDict: A dictionnary not allowing new keys to be created after initialisation. + FixedDict: A dictionary not allowing new keys to be created after initialisation. >>> f = FixedDict(**{'count':1}) >>> f['count'] = 2 @@ -347,8 +389,6 @@ class FixedDict(dict): raise ConfigError(f'Option "{k}" has no defined default') """ - from vyos import ConfigError - def __init__(self, **options): self._allowed = options.keys() super().__init__(**options) @@ -368,7 +408,7 @@ class FixedDict(dict): >>> d {'key': 'value'} """ + from vyos import ConfigError if k not in self._allowed: raise ConfigError(f'Option "{k}" has no defined default') super().__setitem__(k, v) - diff --git a/python/vyos/utils/disk.py b/python/vyos/utils/disk.py index d4271ebe1..b822badde 100644 --- a/python/vyos/utils/disk.py +++ b/python/vyos/utils/disk.py @@ -1,4 +1,4 @@ -# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/utils/error.py b/python/vyos/utils/error.py index 8d4709bff..75ad813f3 100644 --- a/python/vyos/utils/error.py +++ b/python/vyos/utils/error.py @@ -1,4 +1,4 @@ -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/utils/file.py b/python/vyos/utils/file.py index eaebb57a3..f230977a7 100644 --- a/python/vyos/utils/file.py +++ b/python/vyos/utils/file.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -14,6 +14,9 @@ # License along with this library. If not, see <http://www.gnu.org/licenses/>. import os +import tempfile +import shutil + from vyos.utils.permission import chown def makedir(path, user=None, group=None): @@ -28,22 +31,28 @@ def file_is_persistent(path): absolute = os.path.abspath(os.path.dirname(path)) return re.match(location,absolute) -def read_file(fname, defaultonfailure=None): +def read_file(fname, defaultonfailure=None, sudo=False): """ read the content of a file, stripping any end characters (space, newlines) should defaultonfailure be not None, it is returned on failure to read """ try: - """ Read a file to string """ - with open(fname, 'r') as f: - data = f.read().strip() - return data + # Some files can only be read by root - emulate sudo cat call + if sudo: + from vyos.utils.process import cmd + data = cmd(['sudo', 'cat', fname]) + else: + # If not sudo, just read the file + with open(fname, 'r') as f: + data = f.read() + return data.strip() except Exception as e: if defaultonfailure is not None: return defaultonfailure raise e -def write_file(fname, data, defaultonfailure=None, user=None, group=None, mode=None, append=False): +def write_file(fname, data, defaultonfailure=None, user=None, group=None, + mode=None, append=False, trailing_newline=False): """ Write content of data to given fname, should defaultonfailure be not None, it is returned on failure to read. @@ -60,6 +69,9 @@ def write_file(fname, data, defaultonfailure=None, user=None, group=None, mode=N bytes = 0 with open(fname, 'w' if not append else 'a') as f: bytes = f.write(data) + if trailing_newline and not data.endswith('\n'): + f.write('\n') + bytes += 1 chown(fname, user, group) chmod(fname, mode) return bytes @@ -83,36 +95,6 @@ def read_json(fname, defaultonfailure=None): return defaultonfailure raise e -def chown(path, user=None, group=None, recursive=False): - """ change file/directory owner """ - from pwd import getpwnam - from grp import getgrnam - - if user is None and group is None: - return False - - # path may also be an open file descriptor - if not isinstance(path, int) and not os.path.exists(path): - return False - - # keep current value if not specified otherwise - uid = -1 - gid = -1 - - if user: - uid = getpwnam(user).pw_uid - if group: - gid = getgrnam(group).gr_gid - - if recursive: - for dirpath, dirnames, filenames in os.walk(path): - os.chown(dirpath, uid, gid) - for filename in filenames: - os.chown(os.path.join(dirpath, filename), uid, gid) - else: - os.chown(path, uid, gid) - return True - def chmod(path, bitmask): # path may also be an open file descriptor @@ -166,12 +148,6 @@ def file_permissions(path): """ Return file permissions in string format, e.g '0755' """ return oct(os.stat(path).st_mode)[4:] -def makedir(path, user=None, group=None): - if os.path.exists(path): - return - os.makedirs(path, mode=0o755) - chown(path, user, group) - def wait_for_inotify(file_path, pre_hook=None, event_type=None, timeout=None, sleep_interval=0.1): """ Waits for an inotify event to occur """ if not os.path.dirname(file_path): @@ -212,3 +188,169 @@ def wait_for_file_write_complete(file_path, pre_hook=None, timeout=None, sleep_i """ Waits for a process to close a file after opening it in write mode. """ wait_for_inotify(file_path, event_type='IN_CLOSE_WRITE', pre_hook=pre_hook, timeout=timeout, sleep_interval=sleep_interval) + + +def copy_chown(source, target): + # pylint: disable=import-outside-toplevel + import shutil + import stat + + shutil.copy2(source, target) + st = os.stat(source) + os.chown(target, st[stat.ST_UID], st[stat.ST_GID]) + + +def write_file_sync(file_path, data: str, mode='w'): + """Write file with explicit sync of file and directory""" + # pylint: disable=consider-using-with + file_dir = os.path.dirname(file_path) + + # write and sync file + try: + file = open(file_path, mode) + file.write(data) + file.flush() + os.fsync(file.fileno()) + file.close() + except OSError as e: + try: + file.close() + except OSError: + pass + raise e + + # sync directory entry + try: + fd = os.open(file_dir, os.O_DIRECTORY | os.O_RDONLY) + os.fsync(fd) + os.close(fd) + except OSError as e: + try: + os.close(fd) + except OSError: + pass + raise e + + +def write_file_atomic(file_path, data: str, mode='w'): + """Use os.rename for 'atomic' write. + + Note that this requires an euid/egid of that of the source file for the + chown operation. + + Note that this calls write_file_sync, above. + """ + # pylint: disable=consider-using-with,raise-missing-from + file_dir = os.path.dirname(file_path) + temp_file = tempfile.NamedTemporaryFile(delete=False, dir=file_dir).name + + def cleanup(): + if os.path.exists(temp_file): + try: + os.unlink(temp_file) + except OSError: + pass + + if os.path.exists(file_path): + try: + copy_chown(file_path, temp_file) + except OSError as e: + cleanup() + raise OSError(f'copy_chown {e}') + + try: + write_file_sync(temp_file, data, mode=mode) + except OSError as e: + cleanup() + raise OSError(f'write_file_sync {e}') + + try: + os.rename(temp_file, file_path) + except OSError as e: + cleanup() + raise OSError(f'rename {e}') + +def copy_recursive(src: str, dst: str, overwrite: bool = False): + """ + Recursively copy files from `src` to `dst`. + + :param src: Source directory + :param dst: Destination directory + :param overwrite: If True, overwrite existing files. If False, skip them. + """ + + if not os.path.exists(src): + raise FileNotFoundError(f"Source path does not exist: {src}") + + os.makedirs(dst, exist_ok=True) # Create destination directory if not exists + + for root, _, files in os.walk(src): + # Find relative path to maintain directory structure + rel_path = os.path.relpath(root, src) + target_dir = os.path.join(dst, rel_path) if rel_path != "." else dst + + os.makedirs(target_dir, exist_ok=True) + + for file in files: + src_file = os.path.join(root, file) + dst_file = os.path.join(target_dir, file) + + if not os.path.exists(dst_file) or overwrite: + shutil.copy2(src_file, dst_file) + + +def move_recursive(src: str, dst: str, overwrite=False): + """ + Recursively move files from `src` to `dst` and removing the source. + + :param src: Source directory + :param dst: Destination directory + :param overwrite: If True, overwrite existing files. If False, skip them. + """ + if not os.path.exists(src): + raise FileNotFoundError(f"Source path does not exist: {src}") + + copy_recursive(src, dst, overwrite=overwrite) + shutil.rmtree(src) + + +def file_compare(file1: str, file2: str) -> bool: + """ + Compare two files modulo blank lines, leading/trailing whitespace, final + newline. + + Returns: + bool: True if files are equivalent in the sense above. + + """ + + def non_empty(line): + return bool(line.strip()) + + with open(file1) as f1, open(file2) as f2: + it1 = filter(non_empty, f1) + it2 = filter(non_empty, f2) + + try: + for l1, l2 in zip(it1, it2, strict=True): + if l1.strip() != l2.strip(): + return False + except ValueError: + return False + + return True + + +def get_name_from_path(path) -> str: + """ + Extracts the base name (without extension) from a file path. + + Args: + path (str | pathlib.Path): The file path to extract the name from. + + Returns: + str: The base name without file extension. + """ + base = os.path.basename(path) + name, _ = base.split('.', maxsplit=1) + return name diff --git a/python/vyos/utils/func.py b/python/vyos/utils/func.py new file mode 100644 index 000000000..830b76c94 --- /dev/null +++ b/python/vyos/utils/func.py @@ -0,0 +1,28 @@ +# Copyright (C) VyOS Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + + +"""Module for functors and higher order functions.""" + + +class FalseCallable: + """Define falsy callable for use as default value for getattr of a + function from a module defined by vyos.utils.system.load_as_module""" + + def __call__(self, *args, **kwargs): + pass + + def __bool__(self): + return False diff --git a/python/vyos/utils/io.py b/python/vyos/utils/io.py index 205210b66..3efb55ddc 100644 --- a/python/vyos/utils/io.py +++ b/python/vyos/utils/io.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -111,3 +111,14 @@ def select_entry(l: list, list_msg: str = '', prompt_msg: str = '', select = ask_input(prompt_msg, default=default_entry, numeric_only=True, valid_responses=valid_entry) return next(filter(lambda x: x[0] == select, en))[1] + +def catch_broken_pipe(func): + import os + import sys + def wrapped(*args, **kwargs): + try: + func(*args, **kwargs) + except (BrokenPipeError, KeyboardInterrupt): + # Flush output to /dev/null and bail out. + os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno()) # pylint: disable = no-member + return wrapped diff --git a/python/vyos/utils/kernel.py b/python/vyos/utils/kernel.py index 05eac8a6a..726025605 100644 --- a/python/vyos/utils/kernel.py +++ b/python/vyos/utils/kernel.py @@ -1,4 +1,4 @@ -# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -14,32 +14,67 @@ # License along with this library. If not, see <http://www.gnu.org/licenses/>. import os +from typing import Tuple +from typing import Optional # A list of used Kernel constants # https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/net/wireguard/messages.h?h=linux-6.6.y#n45 WIREGUARD_REKEY_AFTER_TIME = 120 +def load_module(name: str, quiet: bool = True, dry_run: bool = False) -> int: + """Load a kernel module via modprobe. + + Returns the modprobe return code. + """ + + from vyos.utils.process import run + + if is_module_loaded(name): + return 0 + + cmd = ['modprobe'] + if dry_run: + cmd.append('-n') + if quiet: + cmd.append('-q') + cmd.append(name) + return run(cmd) + +def unload_module(name: str) -> int: + """Unload a kernel module via rmmod. + + Returns the rmmod return code. + """ + + from vyos.utils.process import run + + if not is_module_loaded(name): + return 0 + + return run(['rmmod', name]) + def check_kmod(k_mod): """ Common utility function to load required kernel modules on demand """ from vyos import ConfigError - from vyos.utils.process import call if isinstance(k_mod, str): k_mod = k_mod.split() for module in k_mod: - if not os.path.exists(f'/sys/module/{module}'): - if call(f'modprobe {module}') != 0: - raise ConfigError(f'Loading Kernel module {module} failed') + if load_module(module) != 0: + raise ConfigError(f'Loading Kernel module {module} failed') + + +def is_module_loaded(module): + """Common utility function to check whether module is loaded""" + return os.path.exists(f'/sys/module/{module}') def unload_kmod(k_mod): """ Common utility function to unload required kernel modules on demand """ from vyos import ConfigError - from vyos.utils.process import call if isinstance(k_mod, str): k_mod = k_mod.split() for module in k_mod: - if os.path.exists(f'/sys/module/{module}'): - if call(f'rmmod {module}') != 0: - raise ConfigError(f'Unloading Kernel module {module} failed') + if unload_module(module) != 0: + raise ConfigError(f'Unloading Kernel module {module} failed') def list_loaded_modules(): """ Returns the list of currently loaded kernel modules """ @@ -115,3 +150,25 @@ def lsmod(): for m in list_loaded_modules(): mods_data.append(get_module_data(m)) return mods_data + +def get_kernel_serial_console() -> Tuple[Optional[str], Optional[str], Optional[str]]: + """ + Extract the serial console type, number, and speed setting from the kernel + command line which was used during system boot. + """ + import re + from vyos.utils.file import read_file + + cmdline_console_re = re.compile( + r'(?:^|\s)console=(?P<console_type>tty(?:S|AMA))(?P<console_num>\d+),(?P<console_speed>\d+)(?=\s|$)' + ) + + kernel_cmdline = read_file('/proc/cmdline') + if m := cmdline_console_re.search(kernel_cmdline): + return ( + m.group('console_type'), + m.group('console_num'), + m.group('console_speed'), + ) + + return (None, None, None) diff --git a/python/vyos/utils/list.py b/python/vyos/utils/list.py index 63ef720ab..01a399026 100644 --- a/python/vyos/utils/list.py +++ b/python/vyos/utils/list.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -18,3 +18,46 @@ def is_list_equal(first: list, second: list) -> bool: if len(first) != len(second) or len(first) == 0: return False return sorted(first) == sorted(second) + + +def list_strip(lst: list, sub: list, right: bool = False) -> list: + """Remove list 'sub' from beginning (right=False), resp., end of list 'lst'""" + + if not right: + while sub: + if lst[:1] == sub[:1]: + lst = lst[1:] + sub = sub[1:] + else: + lst = [] + sub = [] + else: + while sub: + if lst[-1:] == sub[-1:]: + lst = lst[:-1] + sub = sub[:-1] + else: + lst = [] + sub = [] + + return lst + + +def list_contains_sublist(lst: list, sub: list) -> bool: + """ + Check if any sublist in lst contains any element from list sub. + + Parameters: + lst (list of list): A list of sublists to be searched. + sub (list): A list of elements to search for. + + Returns: + bool: True if any element from sub is found in any sublist of lst, otherwise False. + """ + + for sublist in lst: + if len(sub) == len(sublist): + # Ensure all elements the same and position in right position + if all(a == b for a, b in zip(sub, sublist, strict=True)): + return True + return False diff --git a/python/vyos/utils/locking.py b/python/vyos/utils/locking.py index 63cb1a816..f4cd6fd41 100644 --- a/python/vyos/utils/locking.py +++ b/python/vyos/utils/locking.py @@ -1,4 +1,4 @@ -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/utils/misc.py b/python/vyos/utils/misc.py index d82655914..2fe25da91 100644 --- a/python/vyos/utils/misc.py +++ b/python/vyos/utils/misc.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -12,6 +12,9 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. +import time + +from typing import Callable, Any def begin(*args): """ @@ -34,9 +37,13 @@ def install_into_config(conf, config_paths, override_prompt=True): if not config_paths: return None + import os + from vyos.config import Config + from vyos.defaults import base_dir from vyos.utils.io import ask_yes_no from vyos.utils.process import cmd + if not Config().in_session(): print('You are not in configure mode, commands to install manually from configure mode:') for path in config_paths: @@ -46,13 +53,17 @@ def install_into_config(conf, config_paths, override_prompt=True): count = 0 failed = [] + env = os.environ.copy() + env['vyos_libexec_dir'] = base_dir + env['vyos_validators_dir'] = f'{base_dir}/validators' + for path in config_paths: if override_prompt and conf.exists(path) and not conf.is_multi(path): - if not ask_yes_no(f'Config node "{node}" already exists. Do you want to overwrite it?'): + if not ask_yes_no(f'Config node "{path}" already exists. Do you want to overwrite it?'): continue try: - cmd(f'/opt/vyatta/sbin/my_set {path}') + cmd(f'/opt/vyatta/sbin/my_set {path}', env=env) count += 1 except: failed.append(path) @@ -64,3 +75,29 @@ def install_into_config(conf, config_paths, override_prompt=True): if count > 0: print(f'{count} value(s) installed. Use "compare" to see the pending changes, and "commit" to apply.') + +def wait_for( + func: Callable[..., Any], + *args, + interval: float = 1.0, + timeout: float = 5.0, + **kwargs +) -> bool: + """ + Repeatedly calls `func()` until it returns True or the timeout expires. + + Args: + func: A function with no arguments that returns a truthy value when ready. + interval: Seconds to wait between calls (default: 1.0). + timeout: Maximum time to wait in seconds (default: 5.0). + + Returns: + True if the function returned True within the timeout, otherwise False. + """ + start = time.monotonic() + while True: + if func(*args, **kwargs): + return True + if (time.monotonic() - start) >= timeout: + return False + time.sleep(interval) diff --git a/python/vyos/utils/network.py b/python/vyos/utils/network.py index 2f666f0ee..8544b2163 100644 --- a/python/vyos/utils/network.py +++ b/python/vyos/utils/network.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,9 +13,14 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. +import hashlib + +from json import loads +from socket import AF_INET +from socket import AF_INET6 +from vyos.utils.process import cmd + def _are_same_ip(one, two): - from socket import AF_INET - from socket import AF_INET6 from socket import inet_pton from vyos.template import is_ipv4 # compare the binary representation of the IP @@ -47,9 +52,63 @@ def is_netns_interface(interface, netns): return True return False +def get_host_identity() -> str: + """ + Build a stable host identity string for deterministic MAC generation. + + Combines: + • The system's hardware UUID (from /sys/class/dmi/id/product_uuid), + if available + • The system hostname + + Both are normalized (lowercase, dashes removed in UUID) and joined with a colon. + + Returns: + str: A string "<uuid>:<hostname>", used as part of the host-specific seed when + generating deterministic MAC addresses. + """ + import os.path + + uuid_file = '/sys/class/dmi/id/product_uuid' + + if os.path.exists(uuid_file): + uuid = cmd(f"sudo cat {uuid_file}").strip().replace("-", "").lower() + else: + uuid = None + + host = cmd("hostname").strip().lower() + + if uuid is not None: + return f"{uuid}:{host}" + else: + return host + +def gen_mac(name: str, addr: str, ident: str) -> str: + """ + Generate a deterministic locally-administered MAC address. + + The MAC is derived from: + • Host identity (UUID + hostname) + • Container name + • Concatenated address string (IPv4 and/or IPv6 addresses) + + A SHA-256 digest is computed from the combined string. The first 5 bytes + of the digest are used, prefixed with 0x02 to mark the address as + locally-administered and unicast. + + Args: + name (str): Container name to differentiate MACs. + addr (str): Concatenated list of container addresses (IPv4/IPv6). + + Returns: + str: Deterministic MAC address in standard "xx:xx:xx:xx:xx:xx" format. + """ + h = hashlib.sha256(f"{ident}:{name}:{addr}".encode()).hexdigest() + # 0x02 = locally-administered, unicast + b = [0x02] + [int(h[i:i+2], 16) for i in range(0, 10, 2)] # 5 bytes = 40 bits + return ":".join(f"{x:02x}" for x in b) + def get_netns_all() -> list: - from json import loads - from vyos.utils.process import cmd tmp = loads(cmd('ip --json netns ls')) return [ netns['name'] for netns in tmp ] @@ -59,14 +118,12 @@ def get_vrf_members(vrf: str) -> list: :param vrf: str :return: list """ - import json - from vyos.utils.process import cmd interfaces = [] try: if not interface_exists(vrf): raise ValueError(f'VRF "{vrf}" does not exist!') output = cmd(f'ip --json --brief link show vrf {vrf}') - answer = json.loads(output) + answer = loads(output) for data in answer: if 'ifname' in data: # Skip PIM interfaces which appears in VRF @@ -80,7 +137,10 @@ def get_interface_vrf(interface): """ Returns VRF of given interface """ from vyos.utils.dict import dict_search from vyos.utils.network import get_interface_config - tmp = get_interface_config(interface) + if isinstance(interface, str): + tmp = get_interface_config(interface) + elif isinstance(interface, dict): + tmp = interface if dict_search('linkinfo.info_slave_kind', tmp) == 'vrf': return tmp['master'] return 'default' @@ -104,8 +164,6 @@ def get_interface_config(interface): """ if not interface_exists(interface): return None - from json import loads - from vyos.utils.process import cmd tmp = loads(cmd(f'ip --detail --json link show dev {interface}'))[0] return tmp @@ -115,18 +173,13 @@ def get_interface_address(interface): """ if not interface_exists(interface): return None - from json import loads - from vyos.utils.process import cmd tmp = loads(cmd(f'ip --detail --json addr show dev {interface}'))[0] return tmp def get_interface_namespace(interface: str): """ - Returns wich netns the interface belongs to + Returns which netns the interface belongs to """ - from json import loads - from vyos.utils.process import cmd - # Bail out early if netns does not exist tmp = cmd(f'ip --json netns ls') if not tmp: return None @@ -154,14 +207,13 @@ def is_ipv6_tentative(iface: str, ipv6_address: str) -> bool: Returns: bool: True if the IPv6 address is tentative, False otherwise. """ - import json from vyos.utils.process import rc_cmd rc, out = rc_cmd(f'ip -6 --json address show dev {iface}') if rc: return False - data = json.loads(out) + data = loads(out) for addr_info in data[0]['addr_info']: if ( addr_info.get('local') == ipv6_address and @@ -173,9 +225,7 @@ def is_ipv6_tentative(iface: str, ipv6_address: str) -> bool: def is_wwan_connected(interface): """ Determine if a given WWAN interface, e.g. wwan0 is connected to the carrier network or not """ - import json from vyos.utils.dict import dict_search - from vyos.utils.process import cmd from vyos.utils.process import is_systemd_service_active if not interface.startswith('wwan'): @@ -188,8 +238,12 @@ def is_wwan_connected(interface): modem = interface.lstrip('wwan') - tmp = cmd(f'mmcli --modem {modem} --output-json') - tmp = json.loads(tmp) + try: + tmp = cmd(f'mmcli --modem {modem} --output-json') + except OSError: + return False + + tmp = loads(tmp) # return True/False if interface is in connected state return dict_search('modem.generic.state', tmp) == 'connected' @@ -198,15 +252,11 @@ def get_bridge_fdb(interface): """ Returns the forwarding database entries for a given interface """ if not interface_exists(interface): return None - from json import loads - from vyos.utils.process import cmd tmp = loads(cmd(f'bridge -j fdb show dev {interface}')) return tmp def get_all_vrfs(): """ Return a dictionary of all system wide known VRF instances """ - from json import loads - from vyos.utils.process import cmd tmp = loads(cmd('ip --json vrf list')) # Result is of type [{"name":"red","table":1000},{"name":"blue","table":2000}] # so we will re-arrange it to a more nicer representation: @@ -225,7 +275,6 @@ def interface_list() -> list: """ return Section.interfaces() - def vrf_list() -> list: """ Get list of VRFs in system @@ -256,62 +305,115 @@ def mac2eui64(mac, prefix=None): except: # pylint: disable=bare-except return -def check_port_availability(ipaddress, port, protocol): + +def check_port_availability(address: str = None, port: int = 0, + protocol: str = 'tcp', vrf: str = None) -> bool: """ - Check if port is available and not used by any service - Return False if a port is busy or IP address does not exists + Check if given port is available and not used by any service. + Should be used carefully for services that can start listening dynamically, because IP address may be dynamic too + + Args: + address: IPv4 or IPv6 address - if None, checks on all interfaces + port: TCP/UDP port number. + vrf: VRF name to test the bind in - when set, the socket is bound + to the VRF master device via SO_BINDTODEVICE so that the + port check runs in the correct L3 domain. + + Returns: + False if a port is busy or IP address does not exists + True if a port is free and IP address exists """ - from socketserver import TCPServer, UDPServer + import socket from ipaddress import ip_address + # treat None as "any address" + address = address or '::' + # verify arguments try: - ipaddress = ip_address(ipaddress).compressed - except: - raise ValueError(f'The {ipaddress} is not a valid IPv4 or IPv6 address') + address = ip_address(address).compressed + except ValueError: + raise ValueError(f'{address} is not a valid IPv4 or IPv6 address') if port not in range(1, 65536): - raise ValueError(f'The port number {port} is not in the 1-65535 range') + raise ValueError(f'Port {port} is not in range 1-65535') if protocol not in ['tcp', 'udp']: - raise ValueError(f'The protocol {protocol} is not supported. Only tcp and udp are allowed') + raise ValueError(f'{protocol} is not supported - only tcp and udp are allowed') - # check port availability + protocol = socket.SOCK_STREAM if protocol == 'tcp' else socket.SOCK_DGRAM try: - if protocol == 'tcp': - server = TCPServer((ipaddress, port), None, bind_and_activate=True) - if protocol == 'udp': - server = UDPServer((ipaddress, port), None, bind_and_activate=True) - server.server_close() - except Exception as e: - # errno.h: - #define EADDRINUSE 98 /* Address already in use */ - if e.errno == 98: - return False - - return True - -def is_listen_port_bind_service(port: int, service: str) -> bool: + addr_info = socket.getaddrinfo(address, port, socket.AF_UNSPEC, protocol) + except socket.gaierror as e: + print(f'Invalid address: {address}') + return False + + for family, socktype, proto, canonname, sockaddr in addr_info: + try: + with socket.socket(family, socktype, proto) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # When a VRF is specified, bind the socket to the VRF master + # device so the address is resolved in that VRF's L3 domain. + if vrf: + s.setsockopt(socket.SOL_SOCKET, socket.SO_BINDTODEVICE, + (vrf + '\0').encode()) + s.bind(sockaddr) + return True # port is free to use + except OSError: + return False # port is already in use + + # if we reach this point, no socket was tested and we assume the port is + # already in use - better safe then sorry + return False + + +def is_listen_port_bind_service(port: int, service: str, address: str = None) -> bool: """Check if listen port bound to expected program name :param port: Bind port :param service: Program name + :param address: IP address - if None, not consider this IP for filtering :return: bool Example: % is_listen_port_bind_service(443, 'nginx') True + % is_listen_port_bind_service(443, 'nginx', address='10.10.0.1') + False % is_listen_port_bind_service(443, 'ocserv-main') False """ from psutil import net_connections as connections from psutil import Process as process + from ipaddress import ip_address + + has_address = bool(address) + + if has_address: + try: + # Normalize address before comparison to handle IPv6 format variations: + # 0:0:0:0:0:0:0:1 vs ::1 + address = ip_address(address).compressed + except ValueError: + raise ValueError(f'{address} is not a valid IPv4 or IPv6 address') + for connection in connections(): addr = connection.laddr pid = connection.pid + + # The PID of the process that opened the socket may not be retrievable + if pid is None: + continue + pid_name = process(pid).name() pid_port = addr.port - if service == pid_name and port == pid_port: - return True + + if has_address: + if address == addr.ip and port == pid_port and service == pid_name: + return True + else: + if service == pid_name and port == pid_port: + return True + return False def is_ipv6_link_local(addr): @@ -327,7 +429,7 @@ def is_ipv6_link_local(addr): def is_addr_assigned(ip_address, vrf=None, return_ifname=False, include_vrf=False) -> bool | str: """ Verify if the given IPv4/IPv6 address is assigned to any interface """ - from netifaces import interfaces + from netifaces import interfaces # pylint: disable = no-name-in-module from vyos.utils.network import get_interface_config from vyos.utils.dict import dict_search @@ -350,7 +452,6 @@ def is_intf_addr_assigned(ifname: str, addr: str, netns: str=None) -> bool: It can check both a single IP address (e.g. 192.0.2.1 or a assigned CIDR address 192.0.2.1/24. """ - import json import jmespath from vyos.utils.process import rc_cmd @@ -359,7 +460,7 @@ def is_intf_addr_assigned(ifname: str, addr: str, netns: str=None) -> bool: netns_cmd = f'ip netns exec {netns}' if netns else '' rc, out = rc_cmd(f'{netns_cmd} ip --json address show dev {ifname}') if rc == 0: - json_out = json.loads(out) + json_out = loads(out) addresses = jmespath.search("[].addr_info[].{family: family, address: local, prefixlen: prefixlen}", json_out) for address_info in addresses: family = address_info['family'] @@ -389,13 +490,25 @@ def is_wireguard_key_pair(private_key: str, public_key:str) -> bool: :return: If public/private keys are keypair returns True else False :rtype: bool """ - from vyos.utils.process import cmd gen_public_key = cmd('wg pubkey', input=private_key) if gen_public_key == public_key: return True else: return False +def get_wireguard_peers(ifname: str) -> list: + """ + Return list of configured Wireguard peers for interface + :param ifname: Interface name + :type ifname: str + :return: list of public keys + :rtype: list + """ + if not interface_exists(ifname): + return [] + peers = cmd(f'wg show {ifname} peers') + return peers.splitlines() + def is_subnet_connected(subnet, primary=False): """ Verify is the given IPv4/IPv6 subnet is connected to any interface on this @@ -410,10 +523,8 @@ def is_subnet_connected(subnet, primary=False): from ipaddress import ip_address from ipaddress import ip_network - from netifaces import ifaddresses - from netifaces import interfaces - from netifaces import AF_INET - from netifaces import AF_INET6 + from netifaces import ifaddresses # pylint: disable = no-name-in-module + from netifaces import interfaces # pylint: disable = no-name-in-module from vyos.template import is_ipv6 @@ -447,9 +558,7 @@ def is_subnet_connected(subnet, primary=False): def is_afi_configured(interface: str, afi): """ Check if given address family is configured, or in other words - an IP address is assigned to the interface. """ - from netifaces import ifaddresses - from netifaces import AF_INET - from netifaces import AF_INET6 + from netifaces import ifaddresses # pylint: disable = no-name-in-module if afi not in [AF_INET, AF_INET6]: raise ValueError('Address family must be in [AF_INET, AF_INET6]') @@ -464,9 +573,6 @@ def is_afi_configured(interface: str, afi): def get_vxlan_vlan_tunnels(interface: str) -> list: """ Return a list of strings with VLAN IDs configured in the Kernel """ - from json import loads - from vyos.utils.process import cmd - if not interface.startswith('vxlan'): raise ValueError('Only applicable for VXLAN interfaces!') @@ -507,9 +613,6 @@ def get_vxlan_vlan_tunnels(interface: str) -> list: def get_vxlan_vni_filter(interface: str) -> list: """ Return a list of strings with VNIs configured in the Kernel""" - from json import loads - from vyos.utils.process import cmd - if not interface.startswith('vxlan'): raise ValueError('Only applicable for VXLAN interfaces!') @@ -580,9 +683,8 @@ def get_nft_vrf_zone_mapping() -> dict: {'interface': 'eth2', 'vrf_tableid': 1000}, {'interface': 'blue', 'vrf_tableid': 2000}] """ - from json import loads from jmespath import search - from vyos.utils.process import cmd + output = [] tmp = loads(cmd('sudo nft -j list table inet vrf_zones')) # {'nftables': [{'metainfo': {'json_schema_version': 1, @@ -615,3 +717,44 @@ def is_valid_ipv4_address_or_range(addr: str) -> bool: return ip_network(addr).version == 4 except: return False + +def is_valid_ipv6_address_or_range(addr: str) -> bool: + """ + Validates if the provided address is a valid IPv4, CIDR or IPv4 range + :param addr: address to test + :return: bool: True if provided address is valid + """ + from ipaddress import ip_network + try: + if '-' in addr: # If we are checking a range, validate both address's individually + split = addr.split('-') + return is_valid_ipv6_address_or_range(split[0]) and is_valid_ipv6_address_or_range(split[1]) + else: + return ip_network(addr).version == 6 + except: + return False + + +def get_interfaces_by_ip(ip_address: str) -> list: + """ + Return a list of all interface names assigned the given IP address. + Args: + ip_address (str): The IP address to search for. + Returns: + list: List of interface names (str) that have the given IP address assigned. + Returns an empty list if no interface has the IP assigned. + """ + import netifaces + from vyos.template import is_ipv6 + + addr_type = AF_INET + if is_ipv6(ip_address): + addr_type = AF_INET6 + + ifaces = [] + for interface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(interface) + for addr_info in addresses.get(addr_type, []): + if addr_info.get('addr') == ip_address: + ifaces.append(interface) + return ifaces diff --git a/python/vyos/utils/permission.py b/python/vyos/utils/permission.py index d938b494f..08bb7bdbc 100644 --- a/python/vyos/utils/permission.py +++ b/python/vyos/utils/permission.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -15,23 +15,37 @@ import os -def chown(path, user, group): +def chown(path, user=None, group=None, recursive=False): """ change file/directory owner """ from pwd import getpwnam from grp import getgrnam - if user is None or group is None: + if user is None and group is None: return False # path may also be an open file descriptor if not isinstance(path, int) and not os.path.exists(path): return False - uid = getpwnam(user).pw_uid - gid = getgrnam(group).gr_gid - os.chown(path, uid, gid) + # keep current value if not specified otherwise + uid = -1 + gid = -1 + + if user: + uid = getpwnam(user).pw_uid + if group: + gid = getgrnam(group).gr_gid + + if recursive: + for dirpath, dirnames, filenames in os.walk(path): + os.chown(dirpath, uid, gid) + for filename in filenames: + os.chown(os.path.join(dirpath, filename), uid, gid) + else: + os.chown(path, uid, gid) return True + def chmod(path, bitmask): # path may also be an open file descriptor if not isinstance(path, int) and not os.path.exists(path): diff --git a/python/vyos/utils/process.py b/python/vyos/utils/process.py index 121b6e240..e78c0b969 100644 --- a/python/vyos/utils/process.py +++ b/python/vyos/utils/process.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -14,27 +14,26 @@ # License along with this library. If not, see <http://www.gnu.org/licenses/>. import os +import shlex +import time from subprocess import Popen from subprocess import PIPE from subprocess import STDOUT from subprocess import DEVNULL - -def get_wrapper(vrf, netns, auth): - wrapper = '' +def get_wrapper(vrf, netns): + wrapper = None if vrf: - wrapper = f'ip vrf exec {vrf} ' + wrapper = ['ip', 'vrf', 'exec', vrf] elif netns: - wrapper = f'ip netns exec {netns} ' - if auth: - wrapper = f'{auth} {wrapper}' + wrapper = ['ip', 'netns', 'exec', netns] return wrapper def popen(command, flag='', shell=None, input=None, timeout=None, env=None, - stdout=PIPE, stderr=PIPE, decode='utf-8', auth='', vrf=None, - netns=None): + stdout=PIPE, stderr=PIPE, decode='utf-8', vrf=None, netns=None, + buffered=True): """ popen is a wrapper helper around subprocess.Popen with it default setting it will return a tuple (out, err) @@ -57,9 +56,12 @@ def popen(command, flag='', shell=None, input=None, timeout=None, env=None, - STDOUT, send the data to be merged with stdout - DEVNULL, discard the output decode: specify the expected text encoding (utf-8, ascii, ...) - the default is explicitely utf-8 which is python's own default + the default is explicitly utf-8 which is python's own default vrf: run command in a VRF context netns: run command in the named network namespace + buffered: define how process output shall be presented to stdout + - true: buffer output and return once after command finished + - false: immediately output strings on stdout - give live feedback usage: get both stdout and stderr: popen('command', stdout=PIPE, stderr=STDOUT) @@ -67,7 +69,7 @@ def popen(command, flag='', shell=None, input=None, timeout=None, env=None, """ # airbag must be left as an import in the function as otherwise we have a - # a circual import dependency + # a circular import dependency from vyos import debug from vyos import airbag @@ -75,19 +77,6 @@ def popen(command, flag='', shell=None, input=None, timeout=None, env=None, if not debug.enabled(flag): flag = 'command' - # Must be run as root to execute command in VRF or network namespace - if vrf or netns: - if os.getuid() != 0: - raise OSError( - 'Permission denied: cannot execute commands in VRF and netns contexts as an unprivileged user' - ) - - wrapper = get_wrapper(vrf, netns, auth) - command = f'{wrapper} {command}' if wrapper else command - - cmd_msg = f"cmd '{command}'" - debug.message(cmd_msg, flag) - use_shell = shell stdin = None if shell is None: @@ -97,42 +86,75 @@ def popen(command, flag='', shell=None, input=None, timeout=None, env=None, if env: use_shell = True + # Must be run as root to execute command in VRF or network namespace + wrapper = get_wrapper(vrf, netns) + if vrf or netns: + if os.getuid() != 0: + raise OSError('Permission denied: cannot execute commands in VRF ' \ + 'and netns contexts as an unprivileged user') + + if use_shell: + command = f'{shlex.join(wrapper)} {command}' + else: + if type(command) is not list: + command = [command] + command = wrapper + command + + cmd_msg = f"cmd '{command}'" if use_shell else f"cmd '{shlex.join(command)}'" + debug.message(cmd_msg, flag) + if input: stdin = PIPE input = input.encode() if type(input) is str else input + text = None + bufsize = -1 # default: system default of io.DEFAULT_BUFFER_SIZE is used + if not buffered: + text = True # Treat output as strings (not bytes) + bufsize = 1 # Enable line buffering + p = Popen(command, stdin=stdin, stdout=stdout, stderr=stderr, - env=env, shell=use_shell) + env=env, shell=use_shell, text=text, bufsize=bufsize) - pipe = p.communicate(input, timeout) + if buffered: + pipe = p.communicate(input, timeout) + rc = p.returncode - pipe_out = b'' - if stdout == PIPE: - pipe_out = pipe[0] + pipe_out = b'' + if stdout == PIPE: + pipe_out = pipe[0] - pipe_err = b'' - if stderr == PIPE: - pipe_err = pipe[1] + pipe_err = b'' + if stderr == PIPE: + pipe_err = pipe[1] - str_out = pipe_out.decode(decode).replace('\r\n', '\n').strip() - str_err = pipe_err.decode(decode).replace('\r\n', '\n').strip() + str_out = pipe_out.decode(decode).replace('\r\n', '\n').strip() + str_err = pipe_err.decode(decode).replace('\r\n', '\n').strip() - out_msg = f"returned (out):\n{str_out}" - if str_out: - debug.message(out_msg, flag) + out_msg = f"returned (out):\n{str_out}" + if str_out: + debug.message(out_msg, flag) - if str_err: - from sys import stderr - err_msg = f"returned (err):\n{str_err}" - # this message will also be send to syslog via airbag - debug.message(err_msg, flag, destination=stderr) + if str_err: + from sys import stderr + err_msg = f"returned (err):\n{str_err}" + # this message will also be send to syslog via airbag + debug.message(err_msg, flag, destination=stderr) - # should something go wrong, report this too via airbag - airbag.noteworthy(cmd_msg) - airbag.noteworthy(out_msg) - airbag.noteworthy(err_msg) + # should something go wrong, report this too via airbag + airbag.noteworthy(cmd_msg) + airbag.noteworthy(out_msg) + airbag.noteworthy(err_msg) + else: + output_lines = [] + for line in p.stdout: + print(line, end='', flush=True) # print each line as it arrives + output_lines.append(line) + p.stdout.close() + rc = p.wait() + str_out = ''.join(output_lines) - return str_out, p.returncode + return str_out, rc def run(command, flag='', shell=None, input=None, timeout=None, env=None, @@ -155,7 +177,7 @@ def run(command, flag='', shell=None, input=None, timeout=None, env=None, def cmd(command, flag='', shell=None, input=None, timeout=None, env=None, stdout=PIPE, stderr=PIPE, decode='utf-8', raising=None, message='', - expect=[0], auth='', vrf=None, netns=None): + expect=[0], vrf=None, netns=None): """ A wrapper around popen, which returns the stdout and will raise the error code of a command @@ -171,12 +193,11 @@ def cmd(command, flag='', shell=None, input=None, timeout=None, env=None, input=input, timeout=timeout, env=env, shell=shell, decode=decode, - auth=auth, vrf=vrf, netns=netns, ) if code not in expect: - wrapper = get_wrapper(vrf, netns, auth='') + wrapper = get_wrapper(vrf, netns) command = f'{wrapper} {command}' feedback = message + '\n' if message else '' feedback += f'failed to run command: {command}\n' @@ -189,9 +210,9 @@ def cmd(command, flag='', shell=None, input=None, timeout=None, env=None, raise raising(feedback) return decoded - def rc_cmd(command, flag='', shell=None, input=None, timeout=None, env=None, - stdout=PIPE, stderr=STDOUT, decode='utf-8', vrf=None, netns=None): + stdout=PIPE, stderr=STDOUT, decode='utf-8', vrf=None, netns=None, + buffered=True): """ A wrapper around popen, which returns the return code of a command and stdout @@ -209,6 +230,7 @@ def rc_cmd(command, flag='', shell=None, input=None, timeout=None, env=None, decode=decode, vrf=vrf, netns=netns, + buffered=buffered, ) return code, out @@ -256,7 +278,6 @@ def process_named_running(name: str, cmdline: str=None, timeout: int=0): return p.info['pid'] return None if timeout: - import time time_expire = time.time() + timeout while True: tmp = check_process(name, cmdline) @@ -270,13 +291,49 @@ def process_named_running(name: str, cmdline: str=None, timeout: int=0): return check_process(name, cmdline) return None -def is_systemd_service_active(service): +def is_systemd_service_active(service: str, vrf=None, netns=None) -> bool: """ Test is a specified systemd service is activated. Returns True if service is active, false otherwise. Copied from: https://unix.stackexchange.com/a/435317 """ - tmp = cmd(f'systemctl show --value -p ActiveState {service}') + tmp = cmd(f'systemctl show --value -p ActiveState {service}', + vrf=vrf, netns=netns) return bool((tmp == 'active')) +def stop_systemd_unit(service: str, retries: int=3, delay_s: float=0.250, + raise_on_failure: bool=True, vrf=None, netns=None) -> None: + """ + Stop systemd unit used during interface teardown (e.g. DHCP clients). + + Retries transient "systemctl stop| failures and verifies ActiveState is no + longer "active". Escalate to systemctl kill if stop attempts fail while + unit remains active. + """ + + if not is_systemd_service_active(service, vrf=vrf, netns=netns): + return None + + for _ in range(retries): + rc_cmd(f'systemctl stop {service}', vrf=vrf, netns=netns) + if not is_systemd_service_active(service, vrf=vrf, netns=netns): + # Service properly stopped - return early, this should be the default + return None + time.sleep(delay_s) + + rc_cmd(f'systemctl kill {service}', vrf=vrf, netns=netns) + time.sleep(delay_s) + if not is_systemd_service_active(service, vrf=vrf, netns=netns): + return None + + # This should not happen + if raise_on_failure: + code, out = rc_cmd(f'systemctl show --value -p ActiveState {service}', + vrf=vrf, netns=netns) + disp_state = out.strip() if code == 0 and out.strip() else 'unknown' + + raise RuntimeError(f'systemd unit {service} still has ActiveState={disp_state} ' \ + f'after {retries} stop attempts and systemctl kill') + return None + def is_systemd_service_running(service): """ Test is a specified systemd service is actually running. Returns True if service is running, false otherwise. @@ -298,3 +355,9 @@ def ip_cmd(args, json=True): else: res = cmd(f"ip {args}") return res + + +def wrap_op(cmd: str) -> str: + """Returns a command with the VyOS operational mode wrapper.""" + + return f'/opt/vyatta/bin/vyatta-op-cmd-wrapper {cmd}' diff --git a/python/vyos/utils/serial.py b/python/vyos/utils/serial.py index b646f881e..fcb7a21c9 100644 --- a/python/vyos/utils/serial.py +++ b/python/vyos/utils/serial.py @@ -1,4 +1,4 @@ -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,7 +13,9 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. -import os, re, json +import os +import re +import json from typing import List from vyos.base import Warning @@ -116,3 +118,19 @@ def restart_login_consoles(prompt_user=False, quiet=True, devices: List[str]=[]) cmd(f'systemctl stop {unit_name}') return True + +def is_tty(name: str, warning=False) -> bool: + """ Check if a given device file (e.g. /dev/ttyS0) is a TTY (teletypewriter) + device in Linux + """ + import os + path_tty = f'/dev/{name}' + if os.path.exists(path_tty): + with open(path_tty, 'rb') as f: + fd = f.fileno() + # True if filename is a TTY + return os.isatty(fd) + elif warning: + from vyos.base import Warning + Warning(f'Device "{name}" does not exist!') + return False diff --git a/python/vyos/utils/session.py b/python/vyos/utils/session.py new file mode 100644 index 000000000..80691a378 --- /dev/null +++ b/python/vyos/utils/session.py @@ -0,0 +1,70 @@ +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +# pylint: disable=import-outside-toplevel + +import os +from inspect import stack + + +def in_config_session(): + """Vyatta bash completion uses the following environment variable for + indication of the config mode environment, independent of legacy backend + initialization of Cstore""" + from os import environ + + return '_OFR_CONFIGURE' in environ + + +# utility for functions below +def get_caller_name() -> str: + filename = stack()[2].filename + return os.path.basename(filename) + + +# OOB operations used (rarely) to update the session config during commit +# execution of config mode scripts. +# The standard use is for replacing plaintext with encrypted passwords in +# the session config during commit. +def delete_cli_node(cli_path: list): + from vyos.vyconf_session import VyconfSession + from vyos.configsession import ConfigSessionError + + pid = os.environ.get('SESSION_PID', '') + if not pid: + raise ValueError('Missing env var SESSION_PID') + + script_name = get_caller_name() + tag_value = os.environ.get('VYOS_TAGNODE_VALUE', None) + + vs = VyconfSession(pid=pid, on_error=ConfigSessionError) + vs.aux_delete(cli_path, script_name, tag_value) + + +def add_cli_node(cli_path: list, value: str = None): + from vyos.vyconf_session import VyconfSession + from vyos.configsession import ConfigSessionError + + pid = os.environ.get('SESSION_PID', '') + if not pid: + raise ValueError('Missing env var SESSION_PID') + + script_name = get_caller_name() + tag_value = os.environ.get('VYOS_TAGNODE_VALUE', None) + + cli_path = cli_path + [value] if value else cli_path + + vs = VyconfSession(pid=pid, on_error=ConfigSessionError) + vs.aux_set(cli_path, script_name, tag_value) diff --git a/python/vyos/utils/strip_config.py b/python/vyos/utils/strip_config.py index 7a9c78c9f..17f6867cb 100644 --- a/python/vyos/utils/strip_config.py +++ b/python/vyos/utils/strip_config.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 # -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/python/vyos/utils/system.py b/python/vyos/utils/system.py index 6c112334b..fd5f49645 100644 --- a/python/vyos/utils/system.py +++ b/python/vyos/utils/system.py @@ -1,4 +1,4 @@ -# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,28 +16,34 @@ import os from subprocess import run -def sysctl_read(name: str) -> str: +def _sysctl_key(parts: list[str]) -> str: + normalized = (p.replace('.', '/') for p in parts) + return '.'.join(normalized) + +def sysctl_read(name: list[str]) -> str: """Read and return current value of sysctl() option Args: - name (str): sysctl key name + name (list[str]): sysctl key name components Returns: str: sysctl key value """ - tmp = run(['sysctl', '-nb', name], capture_output=True) - return tmp.stdout.decode() + key = _sysctl_key(name) + tmp = run(['sysctl', '-nb', key], capture_output=True) + return tmp.stdout.decode().strip() -def sysctl_write(name: str, value: str | int) -> bool: +def sysctl_write(name: list[str], value: str | int) -> bool: """Change value via sysctl() Args: - name (str): sysctl key name + name (list[str]): sysctl key name components value (str | int): sysctl key value Returns: bool: True if changed, False otherwise """ + key = _sysctl_key(name) # convert other types to string before comparison if not isinstance(value, str): value = str(value) @@ -45,7 +51,7 @@ def sysctl_write(name: str, value: str | int) -> bool: if sysctl_read(name) == value: return True # return False if sysctl call failed - if run(['sysctl', '-wq', f'{name}={value}']).returncode != 0: + if run(['sysctl', '-wq', f'{key}={value}']).returncode != 0: return False # compare old and new values # sysctl may apply value, but its actual value will be @@ -55,11 +61,11 @@ def sysctl_write(name: str, value: str | int) -> bool: # False in other cases return False -def sysctl_apply(sysctl_dict: dict[str, str], revert: bool = True) -> bool: +def sysctl_apply(sysctl_dict: dict[tuple[str, ...], str], revert: bool = True) -> bool: """Apply sysctl values. Args: - sysctl_dict (dict[str, str]): dictionary with sysctl keys with values + sysctl_dict (dict[tuple[str, ...], str]): dictionary with sysctl key components (tuple) with values revert (bool, optional): Revert to original values if new were not applied. Defaults to True. @@ -67,12 +73,12 @@ def sysctl_apply(sysctl_dict: dict[str, str], revert: bool = True) -> bool: bool: True if all params configured properly, False in other cases """ # get current values - sysctl_original: dict[str, str] = {} - for key_name in sysctl_dict.keys(): - sysctl_original[key_name] = sysctl_read(key_name) + sysctl_original: dict[tuple[str, ...], str] = {} + for key_parts in sysctl_dict.keys(): + sysctl_original[key_parts] = sysctl_read(list(key_parts)) # apply new values and revert in case one of them was not applied - for key_name, value in sysctl_dict.items(): - if not sysctl_write(key_name, value): + for key_parts, value in sysctl_dict.items(): + if not sysctl_write(list(key_parts), value): if revert: sysctl_apply(sysctl_original, revert=False) return False @@ -80,7 +86,7 @@ def sysctl_apply(sysctl_dict: dict[str, str], revert: bool = True) -> bool: return True def find_device_file(device): - """ Recurively search /dev for the given device file and return its full path. + """ Recursively search /dev for the given device file and return its full path. If no device file was found 'None' is returned """ from fnmatch import fnmatch diff --git a/python/vyos/utils/vti_updown_db.py b/python/vyos/utils/vti_updown_db.py index b491fc6f2..2931df86c 100644 --- a/python/vyos/utils/vti_updown_db.py +++ b/python/vyos/utils/vti_updown_db.py @@ -1,4 +1,4 @@ -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -111,7 +111,7 @@ class VTIUpDownDB: """ Removes a matching entry from the DB. - If no matching entry can be fonud, the operation returns successfully. + If no matching entry can be found, the operation returns successfully. """ ifspec = f"{interface}:{connection}:{protocol}" if (connection is not None and protocol is not None) else interface if ifspec in self._ifspecs: @@ -173,7 +173,7 @@ class VTIUpDownDB: self._fileHandle.truncate() for interface in self._ifsDown: - vti_link = get_interface_config(interface) + vti_link = get_interface_config(interface) or {} vti_link_up = (vti_link['operstate'] != 'DOWN' if 'operstate' in vti_link else False) if vti_link_up: call(f'sudo ip link set {interface} down') @@ -182,7 +182,7 @@ class VTIUpDownDB: self._ifsDown.clear() for interface in self._ifsUp: - vti_link = get_interface_config(interface) + vti_link = get_interface_config(interface) or {} vti_link_up = (vti_link['operstate'] != 'DOWN' if 'operstate' in vti_link else False) if not vti_link_up: vti = interface_dict_supplier(interface) diff --git a/python/vyos/version.py b/python/vyos/version.py index 86e96d0ec..d62778c4b 100644 --- a/python/vyos/version.py +++ b/python/vyos/version.py @@ -1,4 +1,4 @@ -# Copyright 2017-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -34,6 +34,7 @@ import os import requests import vyos.defaults from vyos.system.image import is_live_boot +from urllib3.util import retry from vyos.utils.file import read_file from vyos.utils.file import read_json @@ -124,9 +125,18 @@ def get_remote_version(url): } ] """ + headers = {} + session = requests.Session() + + # By default, `requests` lib. does not retry failed connections + # and this code implements automatic retries using the next wait time range: + # - 0.15, 0.30, 0.60, 1.20 and 2.40 seconds. + for adapter in session.adapters.values(): + adapter.max_retries = retry.Retry(total=5, backoff_factor=0.15) + try: - remote_data = requests.get(url=url, headers=headers) + remote_data = session.get(url=url, headers=headers) remote_data.raise_for_status() if remote_data.status_code != 200: return False diff --git a/python/vyos/vpp/__init__.py b/python/vyos/vpp/__init__.py new file mode 100644 index 000000000..2666de0ba --- /dev/null +++ b/python/vyos/vpp/__init__.py @@ -0,0 +1,24 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from .control_vpp import VPPControl + +__all__ = ['VPPControl'] + + +class VppNotRunningError(Exception): + pass diff --git a/python/vyos/vpp/acl/__init__.py b/python/vyos/vpp/acl/__init__.py new file mode 100644 index 000000000..6bd6cdb23 --- /dev/null +++ b/python/vyos/vpp/acl/__init__.py @@ -0,0 +1,3 @@ +from .acl import Acl + +__all__ = ['Acl'] diff --git a/python/vyos/vpp/acl/acl.py b/python/vyos/vpp/acl/acl.py new file mode 100644 index 000000000..c70b3484c --- /dev/null +++ b/python/vyos/vpp/acl/acl.py @@ -0,0 +1,106 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from vyos.vpp import VPPControl + + +NO_ACL_INDEX = 0xFFFFFFFF + + +class Acl: + def __init__(self): + self.vpp = VPPControl() + + def get_acl_index_by_tag(self, tag): + """Get ACL index by tag name + https://github.com/FDio/vpp/blob/21c641f9356da5137760cdc799127064c8c1fd31/src/plugins/acl/acl.api + """ + for acl in self.vpp.api.acl_dump(acl_index=NO_ACL_INDEX): + if acl.tag == tag: + return acl.acl_index + return NO_ACL_INDEX + + def add_replace_acl(self, tag, rules): + """Add new ACL or replace existing one""" + self.vpp.api.acl_add_replace( + tag=tag, + acl_index=self.get_acl_index_by_tag(tag), + count=len(rules), + r=rules, + ) + + def delete_acl(self, tag): + """Delete existing ACL""" + self.vpp.api.acl_del(acl_index=self.get_acl_index_by_tag(tag)) + + def add_acl_interface(self, interface, input_tags, output_tags): + """Add or replace ACLs on interface""" + acls = [] + for tag in input_tags: + acl_index = self.get_acl_index_by_tag(tag) + acls.append(acl_index) + for tag in output_tags: + acl_index = self.get_acl_index_by_tag(tag) + acls.append(acl_index) + self.vpp.api.acl_interface_set_acl_list( + sw_if_index=self.vpp.get_sw_if_index(interface), + count=len(acls), + n_input=len(input_tags), + acls=acls, + ) + + def delete_acl_interface(self, interface): + """Delete ACLs from interface""" + self.vpp.api.acl_interface_set_acl_list( + sw_if_index=self.vpp.get_sw_if_index(interface), + count=0, + ) + + def get_mac_acl_index_by_tag(self, tag): + """Get mac ACL by tag name""" + for acl in self.vpp.api.macip_acl_dump(): + if acl.tag == tag: + return acl.acl_index + return NO_ACL_INDEX + + def add_replace_acl_mac(self, tag, rules): + """Add or replace existing mac ACL""" + self.vpp.api.macip_acl_add_replace( + tag=tag, + acl_index=self.get_mac_acl_index_by_tag(tag), + count=len(rules), + r=rules, + ) + + def delete_acl_mac(self, tag): + """Delete existing mac ACL""" + self.vpp.api.macip_acl_del(acl_index=self.get_mac_acl_index_by_tag(tag)) + + def add_acl_mac_interface(self, interface, tag): + """Add or replace mac ACLs on interface""" + self.vpp.api.macip_acl_interface_add_del( + sw_if_index=self.vpp.get_sw_if_index(interface), + acl_index=self.get_mac_acl_index_by_tag(tag), + is_add=True, + ) + + def delete_acl_mac_interface(self, interface): + """Delete mac ACLs from interface""" + self.vpp.api.macip_acl_interface_add_del( + sw_if_index=self.vpp.get_sw_if_index(interface), + is_add=False, + ) diff --git a/python/vyos/vpp/config_deps.py b/python/vyos/vpp/config_deps.py new file mode 100644 index 000000000..a822825c3 --- /dev/null +++ b/python/vyos/vpp/config_deps.py @@ -0,0 +1,106 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +def deps_xconnect_dict(conf) -> dict[str, list[str]]: + """Get a dict of all xconnect interface members: + + keys: members + + values: xconnect interfaces + + Args: + conf (config): VyOS config object + + Returns: + dict[str, list[str]]: dict of members + """ + xconn_members_dict: dict[str, list[str]] = {} + config = conf.get_config_dict( + ['interfaces', 'vpp', 'xconnect'], + key_mangling=('-', '_'), + get_first_key=True, + no_tag_node_value_mangle=True, + ) + + for xconn_name, xconn_config in config.items(): + for member_name in xconn_config.get('member', {}).get('interface', []): + xconn_ifaces_list = xconn_members_dict.get(member_name, []) + xconn_ifaces_list.append(xconn_name) + xconn_members_dict.update({member_name: xconn_ifaces_list}) + + return xconn_members_dict + + +def deps_bridge_dict(conf) -> dict[str, list[str]]: + """Get a dict of all bridge interface members: + + keys: members + + values: bridge interfaces + + Args: + conf (config): VyOS config object + + Returns: + dict[str, list[str]]: dict of members + """ + bridge_members_dict: dict[str, list[str]] = {} + config = conf.get_config_dict( + ['interfaces', 'vpp', 'bridge'], + key_mangling=('-', '_'), + get_first_key=True, + no_tag_node_value_mangle=True, + ) + + for bridge_name, bridge_config in config.items(): + for member_name in bridge_config.get('member', {}).get('interface', []): + bridge_ifaces_list = bridge_members_dict.get(member_name, []) + bridge_ifaces_list.append(bridge_name) + bridge_members_dict.update({member_name: bridge_ifaces_list}) + + return bridge_members_dict + + +def deps_bond_dict(conf) -> dict[str, list[str]]: + """Get a dict of all bonding interface members: + + keys: members + + values: bridge interfaces + + Args: + conf (config): VyOS config object + + Returns: + dict[str, list[str]]: dict of members + """ + bond_members_dict: dict[str, list[str]] = {} + config = conf.get_config_dict( + ['interfaces', 'vpp', 'bonding'], + key_mangling=('-', '_'), + get_first_key=True, + no_tag_node_value_mangle=True, + ) + + for bond_name, bond_config in config.items(): + for member_name in bond_config.get('member', {}).get('interface', []): + bond_ifaces_list = bond_members_dict.get(member_name, []) + bond_ifaces_list.append(bond_name) + bond_members_dict.update({member_name: bond_ifaces_list}) + + return bond_members_dict diff --git a/python/vyos/vpp/config_filter.py b/python/vyos/vpp/config_filter.py new file mode 100644 index 000000000..d0c982f7c --- /dev/null +++ b/python/vyos/vpp/config_filter.py @@ -0,0 +1,62 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from vyos.base import Warning +from vyos.config import Config + + +def iface_filter_eth(config: Config, iface: str) -> None: + """Filter out unsupported config nodes from Ethernet interface config + + Args: + config (Config): config object + iface (str): Ethernet interface name to filter + """ + allowed_nodes: list[str] = [ + 'address', + 'description', + 'dhcp-options', + 'dhcpv6-options', + 'disable', + 'eapol', + 'hw-id', + 'ip', + 'ipv6', + 'mtu', + 'redirect', + 'vif', + 'vif-s', + 'vrf', + ] + + if not config._session_config.exists(['interfaces', 'ethernet', iface]): + return + + # get list of config nodes in a session configuration + iface_nodes = config._session_config.list_nodes(['interfaces', 'ethernet', iface]) + + # clean cached session config + if False in config._dict_cache: + del config._dict_cache[False] + + # remove unsupported config nodes + for cfg_node in iface_nodes: + if cfg_node not in allowed_nodes: + config._session_config.delete(['interfaces', 'ethernet', iface, cfg_node]) + Warning( + f'{cfg_node} option in {iface} settings is not supported by VPP interfaces. It will be ignored.' + ) diff --git a/python/vyos/vpp/config_resource_checks/__init__.py b/python/vyos/vpp/config_resource_checks/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/python/vyos/vpp/config_resource_checks/__init__.py diff --git a/python/vyos/vpp/config_resource_checks/memory.py b/python/vyos/vpp/config_resource_checks/memory.py new file mode 100644 index 000000000..c8073f64b --- /dev/null +++ b/python/vyos/vpp/config_resource_checks/memory.py @@ -0,0 +1,204 @@ +# Used for memory consumption calculations +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import re +import psutil + +from vyos.utils.process import cmd +from vyos.vpp.utils import human_memory_to_bytes +from vyos.vpp.config_resource_checks.resource_defaults import default_resource_map + + +# VPP buffers default per NUMA node +MIN_BUFFERS = 16_384 + + +def classify_page_size(page_size_bytes: int) -> str: + """ + Returns one of: '4K', '2M', '1G' based on page size. + """ + if page_size_bytes == 1 << 30: + return '1G' + if page_size_bytes == 2 << 20: + return '2M' + return '4K' + + +def get_hugepages_info() -> dict: + """ + Returns the information about HugePages + retrieved from /sys/kernel/mm/hugepages + """ + base_path = '/sys/kernel/mm/hugepages' + info = {} + + for entry in os.listdir(base_path): + page_size_kb = entry[10:] + page_size = human_memory_to_bytes(page_size_kb) + key = classify_page_size(page_size) + info[key] = {} + + with open(os.path.join(base_path, entry, 'nr_hugepages')) as f: + count = int(f.read().strip()) + info[key]['pages'] = count + info[key]['memory'] = page_size * count + + return info + + +def get_available_memory() -> dict: + memory = {size: info.get('memory') for size, info in get_hugepages_info().items()} + memory['4K'] = psutil.virtual_memory().available + + return memory + + +def get_vpp_used_memory() -> int: + """ + Returns memory currently used by VPP in bytes (RSS value) + """ + try: + out = cmd('ps -o rss= -p $(pidof vpp)') + except OSError: + out = 0 + return int(out) << 10 + + +def get_numa_count(): + """ + Run `numactl --hardware` and parse the 'available:' line. + """ + out = cmd('numactl --hardware') + # e.g. "available: 2 nodes (0-1)" + m = re.search(r'available:\s*(\d+)\s+nodes', out) + return int(m.group(1)) if m else 0 + + +def buffer_page_size(settings: dict) -> int: + page_size = settings['resource_allocation']['buffers']['page_size'] + return human_memory_to_bytes(page_size) + + +def buffer_size(settings: dict) -> int: + numa_count = get_numa_count() + buffers_per_numa = int( + settings['resource_allocation']['buffers']['buffers_per_numa'] + ) + data_size = int(settings['resource_allocation']['buffers']['data_size']) + buffers_memory = buffers_per_numa * data_size * numa_count + return buffers_memory + + +def main_heap_page_size(settings: dict) -> int: + heap_page_size = settings['resource_allocation']['memory']['main_heap_page_size'] + return human_memory_to_bytes(heap_page_size) + + +def memory_main_heap(settings: dict) -> int: + heap_size = settings['resource_allocation']['memory']['main_heap_size'] + return human_memory_to_bytes(heap_size) + + +def ipv6_heap_size(settings: dict) -> int: + heap_size = settings['resource_allocation']['ipv6']['heap_size'] + return human_memory_to_bytes(heap_size) + + +def total_heap_size(heap_size: int, heap_page_size: int) -> int: + return (heap_size + heap_page_size - 1) & ~(heap_page_size - 1) + + +def statseg_size(settings: dict) -> int: + statseg_memory = settings['resource_allocation']['memory']['stats']['size'] + return human_memory_to_bytes(statseg_memory) + + +def statseg_page_size(settings: dict) -> int: + page_size = settings['resource_allocation']['memory']['stats']['page_size'] + return human_memory_to_bytes(page_size) + + +def total_statseg_size(_statseg_size: int, _statseg_page: int) -> int: + return (_statseg_size + _statseg_page - 1) & ~(_statseg_page - 1) + + +def total_memory_required(settings: dict) -> dict: + memory = {'2M': 0, '1G': 0, '4K': 0} + + mem_stats = { + 'memory_buffers': (buffer_size(settings), buffer_page_size(settings)), + 'netlink_buffer_size': (default_resource_map.get('netlink_rx_buffer_size'), 0), + 'heap_size': ( + total_heap_size( + heap_size=memory_main_heap(settings), + heap_page_size=main_heap_page_size(settings), + ), + main_heap_page_size(settings), + ), + 'statseg_size': ( + total_statseg_size( + _statseg_size=statseg_size(settings), + _statseg_page=statseg_page_size(settings), + ), + statseg_page_size(settings), + ), + 'ipv6_heap_size': (ipv6_heap_size(settings), 0), + } + + for memory_size, page_size in mem_stats.values(): + memory[classify_page_size(page_size)] += memory_size + + return memory + + +def buffers_required(settings: dict) -> int: + """ + Calculate total VPP buffer requirements based on interface settings and workers. + """ + workers = int(settings['resource_allocation']['cpu_cores']) + buffers_total = 0 + for ifname, iface_config in settings.get('interface', {}).items(): + # Do not include XDP interfaces in buffer calculations. + # Unlike DPDK, XDP does not use VPP-managed mbufs for RX/TX rings, + # so buffer requirements cannot be derived from descriptors here. + # Buffers for XDP are handled internally by the kernel/XDP layer, + # not by VPP’s buffer allocator. + # if iface_config.get('driver') == 'xdp': + # continue + + rx_queues = int(iface_config.get('num_rx_queues', 1)) + rx_desc = int(iface_config.get('num_rx_desc')) + # default TX queues is equal to number of worker threads + tx_queues = int(iface_config.get('num_tx_queues', workers)) + tx_desc = int(iface_config.get('num_tx_desc')) + + # buffers for RX/TX queues for interface + buffers_total += rx_queues * rx_desc + tx_queues * tx_desc + + # per-thread buffer caches (approx. 256 buffers per worker) + buffers_total += workers * 256 + + # Safety margin for buffer calculations: + # traffic bursts, alignment/metadata overhead etc. + # The factor 2.5 is derived from VPP’s own calculations: + # https://github.com/FDio/vpp/blob/stable/2506/extras/vpp_config/vpplib/AutoConfig.py#L609 + buffers_total = int(buffers_total * 2.5) + + # Enforce minimum required by VPP (16K buffers) + return max(buffers_total, MIN_BUFFERS) diff --git a/python/vyos/vpp/config_resource_checks/resource_defaults.py b/python/vyos/vpp/config_resource_checks/resource_defaults.py new file mode 100644 index 000000000..aa1c9a8a0 --- /dev/null +++ b/python/vyos/vpp/config_resource_checks/resource_defaults.py @@ -0,0 +1,29 @@ +# Default values for resource consumption checks +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +default_resource_map = { + # Default size of buffers transferred via netlink + 'netlink_rx_buffer_size': 268435456, + # Minimal amount of memory required to start VPP + 'min_memory': '8G', + # Minimal number of physical CPU cores required to start VPP + 'min_cpus': 4, + # Reserve at least 2 physical cores + 'reserved_cpu_cores': 2, +} diff --git a/python/vyos/vpp/config_verify.py b/python/vyos/vpp/config_verify.py new file mode 100644 index 000000000..228fcbc09 --- /dev/null +++ b/python/vyos/vpp/config_verify.py @@ -0,0 +1,483 @@ +# Used for verifying configuration vpp interfaces +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import psutil + +from vyos import ConfigError +from vyos.base import Warning +from vyos.utils.convert import range_str_to_list +from vyos.utils.cpu import get_core_count as total_core_count, get_cpus +from vyos.utils.dict import dict_search +from vyos.utils.file import read_file + +from vyos.vpp.config_resource_checks import memory as mem_checks +from vyos.vpp.config_resource_checks.resource_defaults import default_resource_map +from vyos.vpp.utils import human_memory_to_bytes, bytes_to_human_memory + +# VPP feature paths that reference interfaces +_VPP_FEATURE_INTERFACE_REFS = [ + ('nat.cgnat.interface.inside', None, 'VPP CGNAT inside'), + ('nat.cgnat.interface.outside', None, 'VPP CGNAT outside'), + ('nat.nat44.interface.inside', None, 'VPP NAT44 inside'), + ('nat.nat44.interface.outside', None, 'VPP NAT44 outside'), + ( + 'nat.nat44.address_pool.translation.interface', + None, + 'VPP NAT44 translation pool', + ), + ('nat.nat44.address_pool.twice_nat.interface', None, 'VPP NAT44 twice-NAT pool'), + ('nat.nat44.exclude.rule', 'external_interface', 'VPP NAT44 exclude rule external'), + ('acl.ip.interface', None, 'VPP IP ACL'), + ('acl.mac.interface', None, 'VPP MAC ACL'), + ('ipfix.interface', None, 'IPFIX monitoring'), + ('sflow.interface', None, 'VPP sFlow'), +] +# VPP member configuration paths that reference interfaces +_VPP_MEMBER_INTERFACE_REFS = [ + ('interfaces_vpp.bonding', 'member.interface', 'VPP bonding member'), + ('interfaces_vpp.bridge', 'member.interface', 'VPP bridge member'), + ('interfaces_vpp.xconnect', 'member.interface', 'VPP xconnect member'), +] + +_VPP_INTERFACE_REFS = _VPP_FEATURE_INTERFACE_REFS + _VPP_MEMBER_INTERFACE_REFS + +# Line width used by ConfigError for message formatting +LINE_WIDTH = 72 + + +def vpp_interface_in_use( + iface: str, config: dict, match_vlans: bool = False, refs: list = None +): + """Check if an interface is referenced in VPP config. + + Args: + iface: interface name to check (e.g. 'eth0' or 'eth0.100') + config: config dict + match_vlans: if True, also match VLAN subinterfaces (e.g. 'eth0.100') + refs: list of (path, inner_path, feature_name) tuples to scan. + Defaults to _VPP_INTERFACE_REFS (all refs). + Use _VPP_FEATURE_INTERFACE_REFS or _VPP_MEMBER_INTERFACE_REFS to narrow scope. + + Returns: + feature_name (str) if found, None otherwise + """ + if refs is None: + refs = _VPP_INTERFACE_REFS + + def _matches(candidate): + """ + Return True if 'candidate' matches 'iface' (optionally match subinterfaces). + 'candidate' can be a string or a list/iterable of strings. + """ + values = [candidate] if isinstance(candidate, str) else list(candidate) + for name in values: + if name == iface: + return True + if match_vlans and name.startswith(f'{iface}.'): + return True + return False + + for path, inner_path, usage in refs: + data = dict_search(path, config) + if not data: + continue + + if inner_path is not None: + for item_key, item_conf in data.items(): + value = dict_search(inner_path, item_conf) + if value is not None and _matches(value): + return usage + else: + if _matches(data): + return usage + + return None + + +def verify_vpp_remove_interface(iface: str, config: dict, match_vlans: bool = False): + """ + Check that an interface is not referenced by any VPP feature. + Raises ConfigError if the interface is still referenced. + """ + feature = vpp_interface_in_use(iface, config, match_vlans) + if feature: + raise ConfigError( + f'Cannot remove interface "{iface}", ' + f'{"it or its VLAN " if match_vlans else "it "}is still configured as {feature} interface' + ) + + +def verify_vpp_interface_not_in_feature(iface: str, config: dict): + """Raise ConfigError if interface is used by a VPP feature (NAT, ACL, etc.). + + Called from VPP interfaces scripts (bonding/bridge/xconnect) before adding a member. + """ + feature = vpp_interface_in_use(iface, config, refs=_VPP_FEATURE_INTERFACE_REFS) + if feature: + raise ConfigError( + f'Interface {iface} is already used as {feature} interface ' + f'and cannot be added as a member' + ) + + +def verify_vpp_interface_not_a_member(iface: str, config: dict): + """Raise ConfigError if interface is a member of bonding/bridge/xconnect. + + Called from feature scripts (NAT, ACL, sFlow, etc.) before adding an interface. + """ + member = vpp_interface_in_use(iface, config, refs=_VPP_MEMBER_INTERFACE_REFS) + if member: + raise ConfigError( + f'Interface {iface} is already used as {member} interface ' + f'and cannot be added to a VPP feature' + ) + + +def verify_vpp_remove_xconnect_interface(config: dict): + if not 'deleted' in config: + return + for xconn_member, xconn_iface in config.get('xconn_members').items(): + if xconn_member == config.get('ifname'): + raise ConfigError( + f'interface "{xconn_member}" is still in use within "interfaces vpp xconnect". ' + f'Please remove it from "interfaces vpp xconnect {xconn_iface}" before proceeding.' + ) + + +def verify_vpp_remove_bridge_interface(config: dict): + if not 'deleted' in config: + return + for bridge_member, bridge_iface in config.get('bridge_members').items(): + if bridge_member == config.get('ifname'): + raise ConfigError( + f'interface "{bridge_member}" is still in use within "interfaces vpp bridge". ' + f'Please remove it from "interfaces vpp bridge {bridge_iface}" before proceeding.' + ) + + +def verify_vpp_tunnel_source_address(config: dict): + from vyos.utils.network import is_intf_addr_assigned + + address = config.get('source_address') + for iface in config.get('vpp_ether_vif_ifaces', []): + if is_intf_addr_assigned(iface, address): + return True + + raise ConfigError( + f'Source address "{address}" is not assigned on any Ethernet or VIF interface!' + ) + + +def verify_member_conflicts(iface, config, current_type): + """ + Check that a member interface is not already used by another interface type. + + Args: + iface (str): interface name to check + config (dict): configuration dictionary + current_type (str): the membership type being assigned + to the interface ('bridge', 'bond', or 'xconn') + + Raises: + ConfigError: If the interface is already a member of a conflicting interface type. + """ + iface_type_map = { + 'bridge': 'bridge', + 'bond': 'bonding', + 'xconn': 'xconnect', + } + for iface_type, label in iface_type_map.items(): + if iface_type == current_type: + continue + members = config.get(f'{iface_type}_members', {}).get(iface) + if members: + raise ConfigError( + f'Interface {iface} cannot be a member of {iface_type_map[current_type]} ' + f'because it already belongs to {label} interface(s): {", ".join(members)}.' + ) + + +def create_cpu_error_message(cpus_required: int, cpus_available: int = None) -> str: + cpu_info = get_cpus() + logical_cores = sum( + [int(s.get('siblings')) if 'siblings' in s else 1 for s in cpu_info] + ) + + reserved_cpus = default_resource_map.get('reserved_cpu_cores') + + available_str = ( + ( + '---'.ljust(LINE_WIDTH) + + 'Reserved:'.ljust(LINE_WIDTH) + + f'For system: {reserved_cpus}'.ljust(LINE_WIDTH) + + f'VPP main thread: 1'.ljust(LINE_WIDTH) + + '---'.ljust(LINE_WIDTH) + + 'Available:'.ljust(LINE_WIDTH) + + f'Physical cores: {max(cpus_available, 0)}' + ) + if cpus_available is not None + else '' + ) + + message = ( + '---'.ljust(LINE_WIDTH) + + 'Total in the system:'.ljust(LINE_WIDTH) + + f'Physical cores: {total_core_count()}'.ljust(LINE_WIDTH) + + f'Logical cores: {logical_cores}'.ljust(LINE_WIDTH) + + '---'.ljust(LINE_WIDTH) + + 'Required:'.ljust(LINE_WIDTH) + + f'Physical cores: {cpus_required}'.ljust(LINE_WIDTH) + + available_str + ) + + return message + + +def verify_vpp_minimum_cpus(): + """ + Verify that the host system has enough physical CPU cores + Current minimal requirement is 4 + """ + min_cpus = default_resource_map.get('min_cpus') + if total_core_count() < min_cpus: + raise ConfigError( + 'This system does not meet minimal requirements for VPP. '.ljust(LINE_WIDTH) + + create_cpu_error_message(min_cpus) + ) + + +def verify_vpp_minimum_memory(): + """ + Verify that the host system has enough RAM + Calculate by retrieving the amount of physical memory + And the minimal requirement (currently 8 GB). Round before comparing - + To avoid situations like when a machine nominally has 8192 MB (8 giga/gibibytes) + But the OS sees only 7.75 GB, creating a fail condition for this check + """ + min_mem = default_resource_map.get('min_memory') + total_memory = round(psutil.virtual_memory().total / (1024**3)) + min_memory = round(human_memory_to_bytes(min_mem) / (1024**3)) + + if total_memory < min_memory: + raise ConfigError( + 'This system does not meet minimal requirements for VPP. ' + f'Minimum {min_memory} GB of RAM are required.' + ) + + +def verify_vpp_main_heap_size(settings: dict): + main_heap_size = mem_checks.memory_main_heap(settings) + main_heap_page_size = mem_checks.main_heap_page_size(settings) + + if main_heap_size < 1 << 30: + raise ConfigError('The main heap size must be greater than or equal to 1G') + + readable_heap_page = bytes_to_human_memory(main_heap_page_size, 'K') + + if main_heap_page_size > main_heap_size: + raise ConfigError( + f'The main heap size must be greater than or equal to page-size ({readable_heap_page})' + ) + + +def verify_vpp_memory(config: dict): + memory_required = mem_checks.total_memory_required(config['settings']) + memory_available = mem_checks.get_available_memory() + + # Check if there is a config currently active + # If yes, calculate how much memory it consumes (only for 4k pages) + # and exclude it from required memory + if config.get('effective'): + memory_effective = mem_checks.total_memory_required( + config['effective']['settings'] + ) + + # If we want to reduce memory configs then we don't need + # to check 4K memory type + if memory_effective['4K'] >= memory_required['4K']: + del memory_required['4K'] + else: + # Get memory currently used by VPP and add it to available memory + memory_used = mem_checks.get_vpp_used_memory() + memory_available['4K'] += memory_used + + memory_required_gb = {k: round(v / 1024**3, 1) for k, v in memory_required.items()} + memory_available_gb = { + k: round(v / 1024**3, 1) for k, v in memory_available.items() + } + + errors = {} + for page_size, req_gb in memory_required_gb.items(): + avail_gb = memory_available_gb.get(page_size, 0) + + if req_gb > avail_gb: + label = 'System' if page_size == '4K' else f'{page_size} HugePages' + errors[page_size] = ( + f'{label} memory: available {avail_gb} GB, ' + f'required {memory_required_gb[page_size]} GB' + ) + + if errors: + raise ConfigError( + 'Not enough free memory to start VPP! '.ljust(LINE_WIDTH) + + '. '.join([line.ljust(LINE_WIDTH) for line in errors.values()]) + + ( + 'To add HugePages memory please use command '.ljust(LINE_WIDTH) + + '"set system option kernel memory hugepage-size ..." and reboot!' + if any(k in errors for k in ('2M', '1G')) + else '' + ) + ) + + +def verify_vpp_cpu_cores(cpu_cores: int): + """ + Verify that the system has enough available and isolated CPU cores. + + Checks performed: + 1. The host has enough physical cores (minus reserved) for the requested + cpu_cores count. + 2. The kernel actually has at least cpu_cores isolated CPUs + (read from /sys/devices/system/cpu/isolated). + + Args: + cpu_cores: Total number of VPP CPU cores (1 main + N-1 workers). + + Raises: + ConfigError: When any of the checks fail. + """ + total_cores = total_core_count() + reserved_cpus = default_resource_map.get('reserved_cpu_cores') + available_cores = total_cores - reserved_cpus + + if cpu_cores > available_cores: + raise ConfigError( + f'Not enough free physical CPU cores for {cpu_cores} "cpu-cores" '.ljust( + LINE_WIDTH + ) + + create_cpu_error_message(cpu_cores, available_cores) + ) + + # Read the set of CPUs the running kernel has actually isolated + isolated = read_file('/sys/devices/system/cpu/isolated') + isolated_cpus = range_str_to_list(isolated) + + if cpu_cores > len(isolated_cpus): + raise ConfigError( + 'Not enough isolated CPU cores available: '.ljust(LINE_WIDTH) + + f'{cpu_cores} requested, but only {len(isolated_cpus)} isolated' + f'{f" (CPU#{isolated})" if len(isolated_cpus) > 0 else ""}. '.ljust( + LINE_WIDTH + ) + + 'To isolate CPUs please use command '.ljust(LINE_WIDTH) + + '"set system option kernel cpu isolate-cpus ..." save and reboot!' + ) + + +def verify_vpp_statseg_size(settings: dict): + statseg_size = mem_checks.statseg_size(settings) + + if 'size' in settings['resource_allocation']['memory']['stats']: + if statseg_size < 128 << 20: + raise ConfigError('The "stats size" must be greater than or equal to 128M') + + if 'page_size' in settings['resource_allocation']['memory']['stats']: + statseg_page_size = mem_checks.statseg_page_size(settings) + if statseg_page_size > statseg_size: + readable_statseg_page = bytes_to_human_memory(statseg_page_size, 'K') + raise ConfigError( + f'The "stats size" must be greater than or equal to page-size ({readable_statseg_page})' + ) + + +def verify_vpp_interfaces_dpdk_num_queues(qtype: str, num_queues: int, workers: int): + """ + Verify that VPP has enough workers to run the given amount of RX/TX queues + 1 queue per 1 worker is assumed as default + """ + + if num_queues > workers: + raise ConfigError( + f'The number of {qtype} queues cannot be greater than the number of configured VPP "cpu-cores": ' + f'cpu-cores: {workers}, queues: {num_queues}' + ) + + +def verify_routes_count(settings: dict): + """ + Maximum routes count depending on main heap size, + statistics segment size and workers + """ + counters = 2 # 2 counters for each route + bytes = 16 # each counter consumes 16 bytes + statseg_scale = 2 + cpu_cores = int(settings['resource_allocation']['cpu_cores']) + statseg_size = settings['resource_allocation']['memory']['stats']['size'] + statseg_size_in_bytes = human_memory_to_bytes(statseg_size) + main_heap = settings['resource_allocation']['memory']['main_heap_size'] + main_heap_in_gb = human_memory_to_bytes(main_heap) >> 30 + + formula = cpu_cores * counters * bytes * statseg_scale + routes_count_statseg = statseg_size_in_bytes / formula + routes_count_statseg = round(routes_count_statseg / 1_000_000, 2) + routes_count_mh = main_heap_in_gb * 2 + routes_count_min = min(routes_count_statseg, routes_count_mh) + Warning( + f'NOTE: Current dataplane capacity (estimated): {routes_count_min} M IPv4 routes. ' + 'Exceeding these values will lead to a dataplane out-of-memory condition and a crash. ' + 'Extensive use of features like ACLs, NAT and others may reduce the numbers above. ' + 'Please read the documentation for details: https://docs.vyos.io/' + ) + + +def verify_vpp_buffers(settings: dict): + buffers_configured = int( + settings['resource_allocation']['buffers']['buffers_per_numa'] + ) + + buffers_required = mem_checks.buffers_required(settings) + + if buffers_required > buffers_configured: + raise ConfigError( + 'Not enough buffers to initialize RX/TX queues for interfaces. ' + f'Set "vpp settings resource-allocation buffers buffers-per-numa" to {buffers_required} or higher' + ) + + +def verify_nat_interfaces(config: dict, feature_name: str): + """ + Verify that interfaces are not already used in the selected NAT feature. + Example: + verify_nat_interfaces(config, 'nat44') + verify_nat_interfaces(config, 'cgnat') + """ + directions = ['inside', 'outside'] + interfaces = config.get('interface', {}) + nat_interfaces = config.get(f'{feature_name}_config', {}).get('interface', {}) + + for direction in directions: + for iface in interfaces.get(direction, []): + # Check if iface is used in any nat44/cgnat direction + nat_dir = next( + (d for d in directions if iface in nat_interfaces.get(d, [])), None + ) + if nat_dir: + raise ConfigError( + f'Cannot use {iface} as {direction} interface: ' + f'it is already configured as {nat_dir} interface in {feature_name.upper()}' + ) diff --git a/python/vyos/vpp/configdb.py b/python/vyos/vpp/configdb.py new file mode 100644 index 000000000..e86b32fb7 --- /dev/null +++ b/python/vyos/vpp/configdb.py @@ -0,0 +1,227 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from json import loads as json_loads, dumps as json_dumps +from pathlib import Path +from typing import Any + +STORAGE_LOCATION = '/run/vpp' + + +class JSONStorage: + def __init__(self, name: str = '') -> None: + """Initiate a file storage + + Args: + name (str, optional): Unique storage name. Defaults to '' (generate a name). + + Raises: + err: In case a file for storage cannot be created + """ + # If a name is not provided, this is a temporary one-time storage + # this use case is strange, but let's allow this + + # Indicates that the object has already been closed and cannot be used again + self.__closed = False + + if not name: + self.__temporary = True + self.__cache: dict[Any, Any] = {} + self.__locked = False + return + self.__temporary = False + + self.__storage = Path(f'{STORAGE_LOCATION}/{name}.json') + self.__lock_file = Path(f'{STORAGE_LOCATION}/{name}.lock') + + # prepare a folder + storage_dir = Path(STORAGE_LOCATION) + if not storage_dir.exists(): + storage_dir.mkdir(parents=True) + + # initialize lock status + self.__locked = False + if self.__storage_locked(): + raise FileExistsError(f'Cannot open locked storage: {self.__storage}') + self.__lock_file.touch() + + if not self.__storage.exists(): + try: + self.__storage.touch() + except Exception as err: + print(f'Unable to initiate storage: {err}') + raise err + # prepare an empty cache + self.__cache: dict[Any, Any] = {} + else: + # load a cache from file + self.__cache = self.__load_file() + + def __del__(self) -> None: + self.close() + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def close(self): + """Dump data to persistent storage and unlock it""" + + # Do not do anything if it is already closed storage + if self.__closed: + return + + try: + if self.__temporary: + return + # dump a cache to storage + if self.__cache: + self.__dump_file() + # or remove a file + else: + self.__storage.unlink(missing_ok=True) + # unlock a storage + self.__lock_file.unlink(missing_ok=True) + finally: + self.__closed = True + + def __check_types(self, data: Any) -> None: + """Check if all the data have supported types + + Args: + data (Any): object to validate + + Raises: + TypeError: If a data type is not supported + """ + if isinstance(data, str | int | float | bool | None): + return + if isinstance(data, list): + for item in data: + self.__check_types(item) + return + if isinstance(data, dict): + for item in data.values(): + self.__check_types(item) + return + raise TypeError(f'Object type "{type(data)}" is not allowed') + + def __load_file(self) -> dict[Any, Any]: + """Read a file to a dictionary + + Returns: + dict[Any, Any]: loaded dict object + """ + data: bytes = self.__storage.read_bytes() + return json_loads(data) + + def __dump_file(self) -> None: + """Dump cache to a file""" + data: str = json_dumps(self.__cache) + self.__storage.write_text(data) + + def __lock(self) -> None: + """Lock storage + + Raises: + FileExistsError: Raised if a storage is already locked + """ + if self.__locked: + raise FileExistsError(f'Access is already locked: {self.__storage}') + self.__locked = True + + def __unlock(self) -> None: + """Unlock storage + + Raises: + FileNotFoundError: Raised if a storage is already unlocked + """ + if not self.__locked: + raise FileNotFoundError(f'Access is already unlocked: {self.__storage}') + self.__locked = False + + def __storage_locked(self) -> bool: + """Check if a storage is locked + + Returns: + bool: Lock status + """ + if self.__lock_file.exists(): + return True + return False + + def delete(self, key: Any = None) -> None: + """Delete data from a storage or a full storage + + Raises: + FileExistsError: Raised if a storage is locked + """ + if self.__locked: + raise FileExistsError( + f'Storage locked and delete operation cannot be performed: {self.__storage}' + ) + if key: + if key not in self.__cache: + raise ValueError( + f'Object {key} does not exist in storage {self.__storage}' + ) + del self.__cache[key] + else: + self.__cache = {} + + def write(self, key: Any, value: Any) -> None: + # Check types first + self.__check_types(key) + self.__check_types(value) + # check lock status + if self.__locked: + raise FileExistsError( + f'Storage is locked and cannot be written: {self.__storage}' + ) + # write a data to a cache + self.__lock() + self.__cache[key] = value + self.__unlock() + + def read(self, key: Any, default: Any = None) -> Any: + """Read data from a storage + + Args: + key (Any): key name + default (Any, optional): Value to return if a key does not exist. Defaults to None. + + Raises: + FileExistsError: Raised if a storage is locked + + Returns: + Any: Value to return + """ + # Check types first + self.__check_types(key) + # check lock status + if self.__locked: + raise FileExistsError( + f'Storage is locked and it is not safe to read: {self.__storage}' + ) + # read a data from cache + self.__lock() + data: Any | None = self.__cache.get(key, default) + self.__unlock() + + return data diff --git a/python/vyos/vpp/control_host.py b/python/vyos/vpp/control_host.py new file mode 100644 index 000000000..707f56740 --- /dev/null +++ b/python/vyos/vpp/control_host.py @@ -0,0 +1,485 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +from pathlib import Path +from re import fullmatch as re_fullmatch +from subprocess import run +from time import sleep + +from pyroute2 import IPRoute + +from vyos.ethtool import Ethtool +from vyos.ifconfig import EthernetIf +from vyos.vpp.utils import EthtoolGDrvinfo + + +def pci_rescan(pci_addr: str = '') -> None: + """Rescan PCI device by removing it and rescan PCI bus + + If PCI address is not defined - just rescan PCI bus + + Args: + address (str, optional): PCI address of device. Defaults to ''. + """ + device_file = Path(f'/sys/bus/pci/devices/{pci_addr}/remove') + if pci_addr: + if device_file.exists(): + device_file.write_text('1') + # wait 10 seconds max until device will be removed + attempts = 100 + while device_file.exists() and attempts: + attempts -= 1 + sleep(0.1) + if device_file.exists(): + raise TimeoutError( + f'Timeout was reached for removing PCI device {pci_addr}' + ) + else: + raise FileNotFoundError(f'PCI device {pci_addr} does not exist') + rescan_file = Path('/sys/bus/pci/rescan') + rescan_file.write_text('1') + if pci_addr: + # wait 10 seconds max until device will be installed + attempts = 100 + while not device_file.exists() and attempts: + attempts -= 1 + sleep(0.1) + if not device_file.exists(): + raise TimeoutError( + f'Timeout was reached for installing PCI device {pci_addr}' + ) + + +def unbind_driver(bus_id: str, device_id: str) -> bool: + """Unbind a driver from a device + + Args: + bus_id (str): bus ID (pci, vmbus, etc.) + device_id (str): device id on the bus (PCI address, VMBus UUID) + + Returns: + bool: True if a driver has been unbound, False otherwise + """ + device_resolved: str = ( + Path(f'/sys/bus/{bus_id}/devices/{device_id}').resolve().as_posix() + ) + if not Path(f'{device_resolved}/driver').exists(): + return False + + Path(f'{device_resolved}/driver/unbind').write_text(device_id) + return True + + +def rebind_gve_driver(iface: str, bus_id: str, device_id: str) -> None: + """ + Rebind a device to the gve kernel driver. + + Args: + iface (str): Interface name + bus_id (str): Bus type (pci, vmbus, etc.) + device_id (str): device id on the bus (PCI address, VMBus UUID) + """ + set_status(iface, 'down') + + # Unbind the device from its current driver + unbind_driver(bus_id, device_id) + + # Clear driver override (if set) + device_path = Path(f'/sys/bus/{bus_id}/devices/{device_id}').resolve() + (device_path / 'driver_override').write_text('') + + # Bind the device to the gve kernel driver + Path(f'/sys/bus/{bus_id}/drivers/gve/bind').write_text(device_id) + + set_status(iface, 'up') + + +def probe_driver(bus_id: str, device_id: str) -> None: + """Probe driver for a device on a bus + + Args: + bus_id (str): bus ID (pci, vmbus, etc.) + device_id (str): device id on the bus (PCI address, VMBus UUID) + """ + Path(f'/sys/bus/{bus_id}/drivers_probe').write_text(device_id) + + +def load_kernel_module(module_name: str) -> None: + """Load a kernel module + + Args: + module_name (str): module name + """ + # check if a module already loaded + if Path(f'/sys/module/{module_name}').exists(): + return + + # execute modprobe with the specified module name + run(['/usr/sbin/modprobe', '-q', module_name], check=True) + + +def override_driver(bus_id: str, device_id: str, driver_name: str = '') -> None: + """Override a driver for a device + + Args: + bus_id (str): bus ID (pci, vmbus, etc.) + device_id (str): device id on the bus (PCI address, VMBus UUID) + driver_name (str, optional): Kernel module (driver) name. Defaults to '' - clear an override. + + Raises: + FileNotFoundError: A device does not support driver override + ChildProcessError: Failed to override a driver + """ + device_resolved: str = ( + Path(f'/sys/bus/{bus_id}/devices/{device_id}').resolve().as_posix() + ) + # check if a device supports driver override + if not Path(f'{device_resolved}/driver_override').exists(): + raise FileNotFoundError(f'{device_resolved} does not support driver override') + + if driver_name: + load_kernel_module(driver_name) + + unbind_driver(bus_id, device_id) + + # vfio-pci requires a different workflow: drivers must be explicitly bound by + # writing the vendor/device IDs to the vfio-pci `new_id` interface. The kernel + # does not provide a reliable way to check whether an ID has already been + # registered, so we simply attempt the write and ignore the FileExistsError. + # Any other failure is treated as a warning. + if driver_name == 'vfio-pci': + vendor: str = Path(f'{device_resolved}/vendor').read_text() + device: str = Path(f'{device_resolved}/device').read_text() + try: + Path('/sys/module/vfio_pci/drivers/pci:vfio-pci/new_id').write_text( + f'{vendor} {device}' + ) + except FileExistsError: + pass + except Exception as e: + print(f"Warning: failed to write new_id for vfio-pci: {e}") + + # override a driver + Path(f'{device_resolved}/driver_override').write_text(f'{driver_name}\n') + + # probe a driver + probe_driver(bus_id, device_id) + + # check the result + if not Path(f'{device_resolved}/driver').exists(): + raise ChildProcessError( + f'Failed to override a driver to {driver_name} for {bus_id}, {device_id}' + ) + + +def get_bus_name(iface: str) -> str: + """Get bus name + Works for PCI, VMbus, maybe something else. + Does not work for Virtio and other virtual devices + (however, it does not seem we need this for such kind of devices). + + Args: + iface (str): interface name + + Returns: + str: bus name + """ + device_resolved: Path = Path(f'/sys/class/net/{iface}/device').resolve() + + # Iterate upwards until a `bus` directory is found + current_path: Path = device_resolved + while True: + # Check if a bus info is available + subsystem_path = Path(f'{current_path}/subsystem') + if subsystem_path.is_symlink(): + # Read the link to determine the bus type + bus_path = subsystem_path.resolve() + # Check if the parent directory is a 'bus' directory in '/sys/bus/' + if bus_path.parent.name == 'bus': + # Return only the last name of the path, e.g., 'pci' + return bus_path.name + + # Move up one directory level + current_path = current_path.parent + if current_path == Path('/sys'): + break # Stop if we reach the root of /sys without finding a bus type + + return '' # Return None if no bus type was found + + +def get_eth_name(dev_id: str) -> str: + """Find Ethernet interface name by PCI address or UUID + + Args: + dev_id (str): PCI address or UUID + + Raises: + FileNotFoundError: no Ethernet interface was found + + Returns: + str: Ethernet interface name + """ + # find all PCI devices with eth* names + net_devs: dict[str, str] = {} + net_devs_dir = Path('/sys/class/net') + regex_filter = r'^/sys/devices/pci[\w/:\.]+/(?P<pci_addr>\w+:\w+:\w+\.\w+)/[\w/:\.]+/(?P<iface_name>eth\d+)$' + for dir in net_devs_dir.iterdir(): + # PCI devices + real_dir: str = dir.resolve().as_posix() + re_obj = re_fullmatch(regex_filter, real_dir) + if re_obj: + iface_name: str = re_obj.group('iface_name') + iface_addr: str = re_obj.group('pci_addr') + net_devs.update({iface_addr: iface_name}) + # UUID devices + else: + try: + bus_type: str = get_bus_name(dir.name) + iface_addr = EthtoolGDrvinfo(dir.name).bus_info_expand(bus_type) + net_devs.update({iface_addr: dir.name}) + except FileNotFoundError: + pass + + # match to provided PCI address or UUID and return a name if found + if dev_id in net_devs: + return net_devs[dev_id] + # raise error if device was not found + raise FileNotFoundError( + f'A device with ID {dev_id} not found in ethernet interfaces' + ) + + +def get_dev_id(iface: str) -> str: + """Get device ID by its interface name + + Args: + iface (str): interface name + + Raises: + FileNotFoundError: no Ethernet interface was found + + Returns: + str: device ID (PCI address or UUID) + """ + try: + # Try to get details via ethtool first + ethtool_info = EthtoolGDrvinfo(iface) + # For devices represented by UUID we need to expand them + # to their full representation + if ethtool_info.driver == 'hv_netvsc': + return ethtool_info.bus_info_expand('vmbus') + return ethtool_info.bus_info + except Exception: + # raise error if a device ID was not found + raise FileNotFoundError(f'Cannot find device ID for interface {iface}') + + +def get_eth_driver(iface: str) -> str: + """Find kernel module used for Ethernet interface + + Args: + iface (str): Ethernet interface name + + Raises: + FileNotFoundError: no Ethernet interface was found + + Returns: + str: kernel module name + """ + driver_dir = Path(f'/sys/class/net/{iface}/device/driver/module') + # Try to detect via sysfs (works for PCI devices) + if driver_dir.exists(): + return driver_dir.resolve().name + + # Fallback: use ethtool (works for veth, tun, etc.) + try: + return Ethtool(iface).get_driver_name() + except Exception as error: + raise Exception(f'Could not determine driver for "{iface}": {error}') from error + + +def get_pci_id(iface: str) -> str | None: + """ + Retrieves the PCI ID for a specified network interface. + + Args: + iface (str): The name of the network interface (e.g., 'eth0'). + + Raises: + OSError: The interface cannot be resolved or PCI ID files cannot be read + + Returns: + str: The PCI ID in the format 'vendor:device'. + None: In case a NIC is not on a PCI bus. + """ + dev_id = get_dev_id(iface) + + # Check if this device is on a PCI bus, return `None` if this is not a PCI device + pci_dev_path = Path(f'/sys/bus/pci/devices/{dev_id}') + if not pci_dev_path.exists(): + return None + + # Read vendor and device IDs + def _read_bus(file: str) -> str: + path = Path(f'/sys/bus/pci/devices/{dev_id}/{file}') + return path.read_text().removeprefix('0x').strip() + + try: + device_id, vendor_id = _read_bus('device'), _read_bus('vendor') + except OSError as e: + # If we reached this exception, then something really weird happened, + # but it is still right to have it + raise OSError(f'PCI IDs for {iface} cannot be retrieved') from e + + return f'{vendor_id}:{device_id}'.lower() + + +def unsafe_noiommu_mode(status: bool) -> None: + """Control unsafe_noiommu_mode parameter of vfio module + + Args: + status (bool): Target status + + Raises: + ChildProcessError: Raised if failed to set unsafe_noiommu_mode + """ + param_path = Path('/sys/module/vfio/parameters/enable_unsafe_noiommu_mode') + current_status: str = param_path.read_text().strip() + target_status: str = 'Y' if status else 'N' + if current_status != target_status: + param_path.write_text(target_status) + if param_path.read_text().strip() != target_status: + raise ChildProcessError('Failed to set unsafe_noiommu_mode') + + +def rename_iface(name_old: str, name_new: str) -> None: + """Rename interface + + Args: + name_old (str): old name + name_new (str): new name + """ + run(['ip', 'link', 'set', name_old, 'down']) + rename_cmd: list[str] = ['ip', 'link', 'set', name_old, 'name', name_new] + run(rename_cmd) + + +def set_promisc(iface_name: str, operation: str) -> None: + """Set promisc mode for interface + + Args: + iface_name (str): name of an interface + operation (str): operation (on, off) + """ + run(['ip', 'link', 'set', iface_name, 'promisc', operation]) + + +def set_mtu(iface_name: str, mtu: int) -> None: + """Set MTU for interface + + Args: + iface_name (str): name of an interface + mtu (int): MTU + """ + run(['ip', 'link', 'set', iface_name, 'mtu', str(mtu)]) + + +def get_eth_mac(iface_name: str) -> str: + """Get MAC address of an interface + + Args: + iface_name (str): name of an interface + + Raises: + FileNotFoundError: interface was not found + + Returns: + str: MAC address + """ + dev_addr_path = Path(f'/sys/class/net/{iface_name}/address') + if dev_addr_path.exists(): + return dev_addr_path.read_text().strip() + else: + # raise error if device was not found + raise FileNotFoundError(f'Interface {iface_name} not found') + + +def xdp_remove(iface_name: str) -> None: + """Remove XDP BPF program from an interface + + Args: + iface_name (str): name of an interface + """ + run(['ip', 'link', 'set', iface_name, 'xdp', 'off']) + + +def set_status(iface_name: str, status: str) -> None: + """Set interface status + + Args: + iface_name (str): name of an interface + status (str): status - "up" or "down" + """ + run(['ip', 'link', 'set', iface_name, status]) + + +def flush_ip(iface_name: str) -> None: + """Flush IP addresses from an interface + + Args: + iface_name (str): name of an interface + """ + iproute = IPRoute() + iproute.flush_addr(label=iface_name) + + +def get_eth_channels(iface_name: str) -> dict: + """ + Get the current hardware queue counts for channels of an interface using ethtool. + + Args: + iface_name (str): name of an interface + + Returns: + dict: Mapping of channel types to their current values: + - 'rx' (int | None): RX channel count. + - 'tx' (int | None): TX channel count. + - 'combined' (int | None): Combined channel count. + Returns None if the channel type is not supported. + """ + ethtool = Ethtool(iface_name) + + channels = {} + for channel in ['rx', 'tx', 'combined']: + queues_list = ethtool.get_channels(channel) + channels[channel] = queues_list[-1] if (len(queues_list) == 2) else None + + return channels + + +def set_eth_channels(iface_name: str, channels: dict) -> None: + """Configure the number of RX, TX, or combined channels for an interface. + + Args: + iface_name (str): name of an interface + channels (dict): channels to set + """ + interface = EthernetIf(iface_name) + for channel, value in channels.items(): + if value: + interface.set_channels(channel, value) diff --git a/python/vyos/vpp/control_vpp.py b/python/vyos/vpp/control_vpp.py new file mode 100644 index 000000000..1ad098b71 --- /dev/null +++ b/python/vyos/vpp/control_vpp.py @@ -0,0 +1,584 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import re + +from collections.abc import Callable +from functools import wraps +from systemd import journal +from time import sleep +from typing import TypeVar, ParamSpec, Literal + +from vpp_papi import VPPApiClient +from vpp_papi import VPPIOError, VPPValueError + +# define types for static type checkers +AnyType = TypeVar('AnyType') +AnyParam = ParamSpec('AnyParam') + + +class VPPControl: + """Control VPP network stack""" + + class _Decorators: + """Decorators for VPPControl""" + + @classmethod + def api_call( + cls, decorated_func: Callable[AnyParam, AnyType] + ) -> Callable[AnyParam, AnyType]: + """Check if API is connected before API call + + Args: + decorated_func: function to decorate + + Raises: + VPPIOError: Connection to API is not established + """ + + @wraps(decorated_func) + def api_safe_wrapper( + cls, *args: AnyParam.args, **kwargs: AnyParam.kwargs + ) -> AnyType: + if not cls.connected: + raise VPPIOError(2, 'VPP API is not connected') + return decorated_func(cls, *args, **kwargs) + + return api_safe_wrapper + + @classmethod + def check_retval( + cls, decorated_func: Callable[AnyParam, AnyType] + ) -> Callable[AnyParam, AnyType]: + """Check retval from API response + + Args: + decorated_func: function to decorate + + Raises: + VPPValueError: raised when retval is not 0 + """ + + @wraps(decorated_func) + def check_retval_wrapper( + cls, *args: AnyParam.args, **kwargs: AnyParam.kwargs + ) -> AnyType: + return_value = decorated_func(cls, *args, **kwargs) + if not return_value.retval == 0: + raise VPPValueError(f'VPP API call failed: {return_value.retval}') + return return_value + + return check_retval_wrapper + + def __init__(self, attempts: int = 5, interval: int = 1000) -> None: + """Create VPP API connection + + Args: + attempts (int, optional): attempts to connect. Defaults to 5. + interval (int, optional): interval between attempts in ms. Defaults to 1000. + + Raises: + VPPIOError: Connection to API cannot be established + """ + self.__vpp_api_client = VPPApiClient() + # connect with interval + while attempts: + try: + attempts -= 1 + self.__vpp_api_client.connect('vpp-vyos') + break + except (ConnectionRefusedError, FileNotFoundError) as err: + error_message = f'VPP API connection timeout: {err}' + journal.send(error_message, priority=journal.LOG_ERR) + sleep(interval / 1000) + # raise exception if connection was not successful in the end + if not self.__vpp_api_client.transport.connected: + raise VPPIOError(2, 'Cannot connect to VPP API') + + def __del__(self) -> None: + """Disconnect from VPP API (destructor)""" + self.disconnect() + + def disconnect(self) -> None: + """Disconnect from VPP API""" + if self.__vpp_api_client.transport.connected: + self.__vpp_api_client.disconnect() + + @_Decorators.check_retval + @_Decorators.api_call + def cli_cmd(self, command: str): + """Send raw CLI command + + Args: + command (str): command to send + + Returns: + vpp_papi.vpp_serializer.cli_inband_reply: CLI reply class + """ + return self.__vpp_api_client.api.cli_inband(cmd=command) + + @_Decorators.api_call + def get_mac(self, ifname: str) -> str: + """Find MAC address by interface name in VPP + + Args: + ifname (str): interface name inside VPP + + Returns: + str: MAC address + """ + for iface in self.__vpp_api_client.api.sw_interface_dump(): + if iface.interface_name == ifname: + return iface.l2_address.mac_string + return '' + + @_Decorators.api_call + def get_sw_if_index(self, ifname: str) -> int | None: + """Find interface index by interface name in VPP + + Args: + ifname (str): interface name inside VPP + + Returns: + int | None: Interface index or None (if was not fount) + """ + for iface in self.__vpp_api_client.api.sw_interface_dump(): + if iface.interface_name == ifname: + return iface.sw_if_index + return None + + @_Decorators.api_call + def get_interface_name(self, index: int) -> str | None: + """Find interface name by interface index in VPP + + Args: + index (int): interface index inside VPP + + Returns: + str | None: Interface name or None (if was not found) + """ + for iface in self.__vpp_api_client.api.sw_interface_dump(): + if iface.sw_if_index == index: + return iface.interface_name + return None + + @_Decorators.check_retval + @_Decorators.api_call + def lcp_pair_add( + self, + iface_name_vpp: str, + iface_name_kernel: str, + iface_type: Literal['tun', 'tap', ''] = '', + ) -> None: + """Create LCP interface pair between VPP and kernel + + Args: + iface_name_vpp (str): interface name in VPP + iface_name_kernel (str): interface name in kernel + iface_type (Literal['tun', 'tap', ''], optional): Use explicit interface type in kernel. Defaults to ''. + """ + iface_index = self.get_sw_if_index(iface_name_vpp) + if iface_index: + api_call_args: dict[str, bool | int | str] = { + 'is_add': True, + 'sw_if_index': iface_index, + 'host_if_name': iface_name_kernel, + } + if iface_type: + iface_type_resolve = {'tun': 1, 'tap': 0} + api_call_args['host_if_type'] = iface_type_resolve[iface_type] + return self.__vpp_api_client.api.lcp_itf_pair_add_del_v2(**api_call_args) + + @_Decorators.check_retval + @_Decorators.api_call + def lcp_pair_del(self, iface_name_vpp: str, iface_name_kernel: str) -> None: + """Delete LCP interface pair between VPP and kernel + + Args: + iface_name_vpp (str): interface name in VPP + iface_name_kernel (str): interface name in kernel + """ + iface_index = self.get_sw_if_index(iface_name_vpp) + if iface_index: + return self.__vpp_api_client.api.lcp_itf_pair_add_del_v2( + is_add=False, sw_if_index=iface_index, host_if_name=iface_name_kernel + ) + + @_Decorators.api_call + def lcp_pair_find( + self, + kernel_name: str = '', + vpp_index_hw: int | None = None, + vpp_index_kernel: int | None = None, + vpp_name_hw: str = '', + vpp_name_kernel: str = '', + ) -> dict[str, str | int] | None: + """Find LCP pair details + + Args: + kernel_name (str, optional): Interface name in the kernel. Defaults to ''. + vpp_index_hw (int | None, optional): Interface index in VPP (hardware). Defaults to None. + vpp_index_kernel (int | None, optional): Interface index in VPP (kernel). Defaults to None. + vpp_name_hw (str, optional): Interface name in VPP (hardware). Defaults to ''. + vpp_name_kernel (str, optional): Interface name in VPP (to kernel). Defaults to ''. + + Returns: + dict[str, str | int] | None: LCP pair details + """ + filter_dict = {} + for filter_name, filter_value in locals().items(): + if filter_value: + filter_dict[filter_name] = filter_value + + # Get list of pairs + lcp_pairs = self.lcp_pairs_list() + + # Check each pair + for pair in lcp_pairs: + pair_found = False + # For each item provided in function arguments + for filter_name, filter_value in filter_dict.items(): + # Stop if filter value is not as in a current pair + if filter_name in pair and pair[filter_name] != filter_value: + pair_found = False + break + # Set flag to True and check the next filter value + pair_found = True + + if pair_found: + return pair + + return None + + @_Decorators.api_call + def lcp_pairs_list(self) -> list[dict[str, str | int]]: + """List all LCP pairs + + Returns: + list[dict[str, str | int]]: LCP pairs details + """ + lcp_pairs_details = [] + + lcp_pairs = self.__vpp_api_client.api.lcp_itf_pair_get()[1] + vpp_ifaces = self.__vpp_api_client.api.sw_interface_dump() + for pair in lcp_pairs: + pair_details = { + 'kernel_name': pair.host_if_name, + 'vpp_index_hw': pair.phy_sw_if_index, + 'vpp_index_kernel': pair.host_sw_if_index, + } + for vpp_iface in vpp_ifaces: + if vpp_iface.sw_if_index == pair_details['vpp_index_hw']: + pair_details['vpp_name_hw'] = vpp_iface.interface_name + if vpp_iface.sw_if_index == pair_details['vpp_index_kernel']: + pair_details['vpp_name_kernel'] = vpp_iface.interface_name + + lcp_pairs_details.append(pair_details) + + return lcp_pairs_details + + @_Decorators.check_retval + @_Decorators.api_call + def lcp_resync(self) -> None: + """Resynchronize objects between kernel and VPP via Netlink + + This clears all routes in VPP configured by LCP and re-creates them + based on the current state of the kernel. + """ + return self.__vpp_api_client.api.lcp_nl_resync() + + @_Decorators.check_retval + @_Decorators.api_call + def iface_rxmode(self, iface_name: str, rx_mode: str) -> None: + """Set interface rx-mode in VPP + + Args: + iface_name (str): interface name in VPP + rx_mode (str): mode (polling, interrupt, adaptive) + """ + modes_dict: dict[str, int] = {'polling': 1, 'interrupt': 2, 'adaptive': 3} + if rx_mode not in modes_dict: + raise VPPValueError(f'Mode {rx_mode} is not known') + iface_index = self.get_sw_if_index(iface_name) + return self.__vpp_api_client.api.sw_interface_set_rx_mode( + sw_if_index=iface_index, mode=modes_dict[rx_mode] + ) + + @_Decorators.api_call + def get_pci_addr(self, ifname: str) -> str: + """Find PCI address of interface by interface name in VPP + + Args: + ifname (str): interface name inside VPP + + Returns: + str: PCI address + """ + hw_info = self.cli_cmd(f'show hardware-interfaces {ifname}').reply + + regex_filter = r'^\s+pci: device (?P<device>\w+:\w+) subsystem (?P<subsystem>\w+:\w+) address (?P<address>\w+:\w+:\w+\.\w+) numa (?P<numa>\w+)$' + re_obj = re.search(regex_filter, hw_info, re.MULTILINE) + + # return empty string if no interface or no PCI info was found + if not hw_info or not re_obj: + return '' + + address = re_obj.groupdict().get('address', '') + + # we need to modify address to match kernel style + # for example: 0000:06:14.00 -> 0000:06:14.0 + address_chunks: list[str] = address.split('.') + address_normalized: str = f'{address_chunks[0]}.{int(address_chunks[1])}' + + return address_normalized + + @_Decorators.check_retval + @_Decorators.api_call + def xdp_iface_create( + self, + host_if: str, + name: str, + rxq_num: int = 65535, + rxq_size: int = 0, + txq_size: int = 0, + mode: Literal['auto', 'copy', 'zero-copy'] = 'auto', + flags: Literal['no_syscall_lock', ''] = '', + ) -> None: + """Create XDP interface + + Args: + host_if (str): name of an interface in kernel + name (str): name of an interface in VPP + rxq_num (int, optional): Number of receive queues to connect to. Defaults to 0 (all). + rxq_size (int, optional): Size of receive queue. Defaults to 0. + txq_size (int, optional): Size of tranceive queue. Defaults to 0. + mode (Literal['auto', 'copy', 'zero-copy', optional): Zero-copy mode. Defaults to 'auto'. + flags (Literal['no_syscall_lock', ''], optional): Syscall lock mode. Defaults to ''. + """ + api_call_args: dict[str, int | str] = { + 'host_if': host_if, + 'name': name, + 'rxq_num': rxq_num, + 'rxq_size': rxq_size, + 'txq_size': txq_size, + } + if mode != 'auto': + mode_resolve: dict[str, int] = {'auto': 0, 'copy': 1, 'zero-copy': 2} + api_call_args['mode'] = mode_resolve[mode] + if flags == 'no_syscall_lock': + api_call_args['flags'] = 1 + return self.__vpp_api_client.api.af_xdp_create_v3(**api_call_args) + + @_Decorators.check_retval + @_Decorators.api_call + def xdp_iface_delete(self, iface_name_vpp: str) -> None: + """Delete XDP interface + + Args: + iface_name_vpp (str): Name of an interface in VPP + """ + iface_index = self.get_sw_if_index(iface_name_vpp) + if iface_index: + api_call_args: dict[str, int] = {'sw_if_index': iface_index} + return self.__vpp_api_client.api.af_xdp_delete(**api_call_args) + + @_Decorators.check_retval + @_Decorators.api_call + def set_iface_mac(self, iface_name_vpp: str, mac_address: str) -> None: + """Set MAC address of an interface + + Args: + iface_name_vpp (str): Name of an interface in VPP + mac_address (str): MAC address + """ + iface_index = self.get_sw_if_index(iface_name_vpp) + api_call_args: dict[str, str | int] = { + 'sw_if_index': iface_index, + 'mac_address': mac_address, + } + return self.__vpp_api_client.api.sw_interface_set_mac_address(**api_call_args) + + @_Decorators.check_retval + @_Decorators.api_call + def set_iface_mtu(self, iface_name_vpp: str, mtu: int) -> None: + """Set MTU for interface + + Args: + iface_name_vpp (str): Name of an interface in VPP + mtu (int): MTU + """ + iface_index = self.get_sw_if_index(iface_name_vpp) + api_call_args: dict[str, int | list[int]] = { + 'sw_if_index': iface_index, + 'mtu': [mtu, 0, 0, 0], + } + return self.__vpp_api_client.api.sw_interface_set_mtu(**api_call_args) + + @_Decorators.api_call + def get_sw_if_dev_type(self, ifname: str) -> int | None: + """Find interface device type by interface name in VPP + + Args: + ifname (str): interface name inside VPP + + Returns: + int | None: Interface device type or None (if was not fount) + """ + for iface in self.__vpp_api_client.api.sw_interface_dump(): + if iface.interface_name == ifname: + return iface.interface_dev_type + return None + + @property + def connected(self) -> bool: + """Check if VPP API is connected + + Returns: + bool: True if connected, False if not + """ + return self.__vpp_api_client.transport.connected + + @property + @_Decorators.api_call + def api(self): + """Call API + + Returns: + Callable[AnyParam, AnyType]: API functions + """ + return self.__vpp_api_client.api + + @_Decorators.api_call + def map_pppoe_interface(self, ifname: str) -> None: + """ + Create PPPoE mapping between data-plane and control-plane interfaces. + + Args: + ifname (str): Name of an interface in kernel. + """ + vpp_pair = self.lcp_pair_find(kernel_name=ifname) + if vpp_pair: + vpp_iface_index = vpp_pair['vpp_index_hw'] + vpp_pair_index = vpp_pair['vpp_index_kernel'] + self.__vpp_api_client.api.pppoe_add_del_cp( + dp_sw_if_index=vpp_iface_index, + cp_sw_if_index=vpp_pair_index, + is_add=True, + ) + + @_Decorators.api_call + def get_pppoe_interface_mapping(self) -> dict: + """ + Build a mapping between data-plane and control-plane interfaces. + + Returns: + dict: Mapping of data-plane interface indices to control-plane interface indices. + """ + result = {} + for binding in self.__vpp_api_client.api.pppoe_cp_binding_dump(): + dp_index = binding.dp_sw_if_index + cp_index = binding.cp_sw_if_index + result[dp_index] = cp_index + + return result + + @_Decorators.api_call + def delete_pppoe_mapping(self, dp_index: int, cp_index: int) -> None: + """ + Delete PPPoE mapping between data-plane and control-plane interfaces. + + Args: + dp_index (int): Index of an interface in data-plane. + cp_index (int): Index of an interface in control plane. + """ + self.__vpp_api_client.api.pppoe_add_del_cp( + dp_sw_if_index=dp_index, + cp_sw_if_index=cp_index, + is_add=False, + ) + + @_Decorators.api_call + def enable_dhcp_client(self, ifname: str) -> None: + """Enable DHCP client detection on a given interface + + Args: + ifname (str): name of an interface in kernel + """ + iface_index = self.get_sw_if_index(ifname) + feature_is_enabled = self.__vpp_api_client.api.feature_is_enabled( + sw_if_index=iface_index, + feature_name='ip4-dhcp-client-detect', + arc_name='ip4-unicast', + ) + if not feature_is_enabled.is_enabled: + self.__vpp_api_client.api.dhcp_client_detect_enable_disable( + sw_if_index=iface_index, + enable=True, + ) + + @_Decorators.api_call + def disable_dhcp_client(self, ifname: str) -> None: + """Disable DHCP client detection on a given interface + + Args: + ifname (str): name of an interface in kernel + """ + self.__vpp_api_client.api.dhcp_client_detect_enable_disable( + sw_if_index=self.get_sw_if_index(ifname), + enable=False, + ) + + @_Decorators.api_call + def enable_icmpv6_ra_punt(self, ifname: str) -> None: + """Enable ip6-icmp-ra-punt feature on a given interface for both ip6-unicast and ip6-multicast arcs. + + This ensures that IPv6 ICMP Router Advertisements (RA) are punted to the host, which is typically required + for features such as DHCPv6 client operation. + + Args: + ifname (str): name of an interface in kernel + """ + iface_index = self.get_sw_if_index(ifname) + for arc_name in ['ip6-unicast', 'ip6-multicast']: + feature_is_enabled = self.__vpp_api_client.api.feature_is_enabled( + sw_if_index=iface_index, + feature_name='ip6-icmp-ra-punt', + arc_name=arc_name, + ) + + if not feature_is_enabled.is_enabled: + self.__vpp_api_client.api.feature_enable_disable( + sw_if_index=iface_index, + enable=True, + arc_name=arc_name, + feature_name='ip6-icmp-ra-punt', + ) + + @_Decorators.api_call + def disable_icmpv6_ra_punt(self, ifname: str) -> None: + """Disable ip6-icmp-ra-punt feature on a given interface + + Args: + ifname (str): name of an interface in kernel + """ + for arc_name in ['ip6-unicast', 'ip6-multicast']: + self.__vpp_api_client.api.feature_enable_disable( + sw_if_index=self.get_sw_if_index(ifname), + enable=False, + arc_name=arc_name, + feature_name='ip6-icmp-ra-punt', + ) diff --git a/python/vyos/vpp/ipfix/__init__.py b/python/vyos/vpp/ipfix/__init__.py new file mode 100644 index 000000000..9696cf5c1 --- /dev/null +++ b/python/vyos/vpp/ipfix/__init__.py @@ -0,0 +1,3 @@ +from .ipfix import IPFIX + +__all__ = ['IPFIX'] diff --git a/python/vyos/vpp/ipfix/ipfix.py b/python/vyos/vpp/ipfix/ipfix.py new file mode 100644 index 000000000..1569b22f6 --- /dev/null +++ b/python/vyos/vpp/ipfix/ipfix.py @@ -0,0 +1,181 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from vpp_papi import VppEnum +from vyos.vpp import VPPControl + + +class IPFIX: + def __init__( + self, + collector_address: str = '0.0.0.0', + collector_port: int = 4739, + src_address: str = '0.0.0.0', + path_mtu: int = 0, + template_interval: int = 20, + udp_checksum: bool = False, + vrf_id: int = 0, + ): + self.vpp = VPPControl() + self.collector_address = collector_address + self.collector_port = collector_port + self.src_address = src_address + self.path_mtu = path_mtu + self.template_interval = template_interval + self.udp_checksum = udp_checksum + self.vrf_id = vrf_id + + # enums mapping + self.RECORD_FLAGS_MAP = { + 'l2': VppEnum.vl_api_flowprobe_record_flags_t.FLOWPROBE_RECORD_FLAG_L2, + 'l3': VppEnum.vl_api_flowprobe_record_flags_t.FLOWPROBE_RECORD_FLAG_L3, + 'l4': VppEnum.vl_api_flowprobe_record_flags_t.FLOWPROBE_RECORD_FLAG_L4, + } + + self.WHICH_FLAGS_MAP = { + 'ipv4': VppEnum.vl_api_flowprobe_which_t.FLOWPROBE_WHICH_IP4, + 'ipv6': VppEnum.vl_api_flowprobe_which_t.FLOWPROBE_WHICH_IP6, + 'l2': VppEnum.vl_api_flowprobe_which_t.FLOWPROBE_WHICH_L2, + } + + self.DIRECTION_MAP = { + 'rx': VppEnum.vl_api_flowprobe_direction_t.FLOWPROBE_DIRECTION_RX, + 'tx': VppEnum.vl_api_flowprobe_direction_t.FLOWPROBE_DIRECTION_TX, + 'both': VppEnum.vl_api_flowprobe_direction_t.FLOWPROBE_DIRECTION_BOTH, + } + + def ipfix_exporter_delete(self): + """Delete IPFIX exporter + https://github.com/FDio/vpp/blob/stable/2506/src/vnet/ipfix-export/ipfix_export.api + Example: + from vyos.vpp import ipfix + i = ipfix.IPFIX() + i.ipfix_exporter_delete() + """ + self.vpp.api.set_ipfix_exporter( + collector_port=0, + collector_address='0.0.0.0', + src_address='0.0.0.0', + path_mtu=0xFFFFFFFF, + template_interval=0, + udp_checksum=False, + vrf_id=4294967295, + ) + + def set_ipfix_exporter(self): + """Set IPFIX exporter parameters + Example: + from vyos.vpp import ipfix + i = ipfix.IPFIX(collector_address='192.0.2.2', src_address='192.0.2.1', collector_port=2055, template_interval=20, path_mtu=1450) + i.set_ipfix_exporter() + """ + self.vpp.api.set_ipfix_exporter( + collector_port=self.collector_port, + collector_address=self.collector_address, + src_address=self.src_address, + path_mtu=self.path_mtu, + template_interval=self.template_interval, + udp_checksum=self.udp_checksum, + vrf_id=self.vrf_id, + ) + + def flowprobe_interface_add( + self, + interface: str, + direction: str = 'both', + which: str = 'ipv4', + ): + """Add IPFIX flowprobe to interface + https://github.com/FDio/vpp/blob/stable/2506/src/plugins/flowprobe/flowprobe.api + Args: + interface (str): Interface name + direction (str): Direction of flowprobe ('rx', 'tx', 'both') + which (str): Which packets to probe ('ipv4', 'ipv6', 'l2') + Example: + from vyos.vpp import ipfix + i = ipfix.IPFIX(collector_address='192.0.2.2', src_address='192.0.2.1', collector_port=2055, template_interval=20, path_mtu=1450) + i.flowprobe_set_params(record_flags=['l2', 'l3'], active_timer=2, passive_timer=20) + i.flowprobe_interface_add('eth0') + """ + sw_if_index = self.vpp.get_sw_if_index(interface) + direction_flag = self.DIRECTION_MAP.get(direction, self.DIRECTION_MAP['both']) + which_flag = self.WHICH_FLAGS_MAP.get(which, self.WHICH_FLAGS_MAP['ipv4']) + + self.vpp.api.flowprobe_interface_add_del( + is_add=True, + sw_if_index=sw_if_index, + direction=direction_flag, + which=which_flag, + ) + + def flowprobe_interface_delete( + self, + interface: str, + direction: str = 'both', + which: str = 'ipv4', + ): + """Delete IPFIX flowprobe from interface + https://github.com/FDio/vpp/blob/stable/2506/src/plugins/flowprobe/flowprobe.api + Args: + interface (str): Interface name + Example: + from vyos.vpp import ipfix + i = ipfix.IPFIX(collector_address='192.0.2.2', src_address='192.0.2.1', collector_port=2055, template_interval=20, path_mtu=1450) + i.flowprobe_interface_delete('eth0') + """ + sw_if_index = self.vpp.get_sw_if_index(interface) + direction_flag = self.DIRECTION_MAP.get(direction, self.DIRECTION_MAP['both']) + which_flag = self.WHICH_FLAGS_MAP.get(which, self.WHICH_FLAGS_MAP['ipv4']) + + self.vpp.api.flowprobe_interface_add_del( + is_add=False, + sw_if_index=sw_if_index, + direction=direction_flag, + which=which_flag, + ) + + def flowprobe_set_params( + self, + active_timer: int = 15, + passive_timer: int = 120, + record_flags: list = None, + ): + """Set IPFIX flowprobe parameters + + Args: + active_timer (int): Active timer in seconds + passive_timer (int): Passive timer in seconds + record_flags: Record flags as list of 'l2', 'l3', 'l4' + Examples: ['l2'], ['l2', 'l3'], ['l2', 'l3', 'l4'] + Example: + from vyos.vpp import ipfix + i = ipfix.IPFIX(collector_address='192.0.2.2', src_address='192.0.2.1', collector_port=2055, template_interval=20, path_mtu=1450) + i.flowprobe_set_params(record_flags=['l2', 'l3'], active_timer=10, passive_timer=30) + i.flowprobe_interface_add('eth0') + """ + if record_flags is None: + record_flags = ['l2', 'l3', 'l4'] + # Calculate combined flags + record_flag = 0 + for flag in record_flags: + record_flag |= self.RECORD_FLAGS_MAP[flag] + + self.vpp.api.flowprobe_set_params( + active_timer=active_timer, + passive_timer=passive_timer, + record_flags=record_flag, + ) diff --git a/python/vyos/vpp/nat/__init__.py b/python/vyos/vpp/nat/__init__.py new file mode 100644 index 000000000..16685c9cc --- /dev/null +++ b/python/vyos/vpp/nat/__init__.py @@ -0,0 +1,4 @@ +from .nat44 import Nat44 +from .det44 import Det44 + +__all__ = ['Nat44', 'Det44'] diff --git a/python/vyos/vpp/nat/det44.py b/python/vyos/vpp/nat/det44.py new file mode 100644 index 000000000..600f02e8c --- /dev/null +++ b/python/vyos/vpp/nat/det44.py @@ -0,0 +1,147 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from vyos.vpp import VPPControl +from vyos.vpp.nat.nat44 import NAT_IS_NONE, NAT_IS_ADDR_ONLY + + +class Det44: + def __init__(self): + self.vpp = VPPControl() + + def enable_det44_plugin(self): + """Enable DET44 plugin + Example: + from vyos.vpp.nat import Det44 + det44 = Det44() + det44.enable_det44_plugin() + https://github.com/FDio/vpp/blob/stable/2410/src/plugins/nat/det44/det44.api + """ + self.vpp.api.det44_plugin_enable_disable(enable=True) + + def disable_det44_plugin(self): + """Disable DET44 plugin""" + self.vpp.api.det44_plugin_enable_disable(enable=False) + + def add_det44_interface_outside(self, interface_out): + """Add DET44 outside interface""" + self.vpp.api.det44_interface_add_del_feature( + sw_if_index=self.vpp.get_sw_if_index(interface_out), + is_inside=False, + is_add=True, + ) + + def delete_det44_interface_outside(self, interface_out_index): + """Delete DET44 outside interface""" + self.vpp.api.det44_interface_add_del_feature( + sw_if_index=interface_out_index, + is_inside=False, + is_add=False, + ) + + def add_det44_interface_inside(self, interface_in): + """Add DET44 inside interface""" + self.vpp.api.det44_interface_add_del_feature( + sw_if_index=self.vpp.get_sw_if_index(interface_in), + is_inside=True, + is_add=True, + ) + + def delete_det44_interface_inside(self, interface_in_index): + """Delete DET44 inside interface""" + self.vpp.api.det44_interface_add_del_feature( + sw_if_index=interface_in_index, + is_inside=True, + is_add=False, + ) + + def add_det44_mapping(self, in_addr, in_plen, out_addr, out_plen): + """Add DET44 mapping""" + self.vpp.api.det44_add_del_map( + in_addr=in_addr, + in_plen=in_plen, + out_addr=out_addr, + out_plen=out_plen, + is_add=True, + ) + + def delete_det44_mapping(self, in_addr, in_plen, out_addr, out_plen): + """Delete DET44 mapping""" + self.vpp.api.det44_add_del_map( + in_addr=in_addr, + in_plen=in_plen, + out_addr=out_addr, + out_plen=out_plen, + is_add=False, + ) + + def set_det44_timeouts( + self, icmp: int, udp: int, tcp_established: int, tcp_transitory: int + ): + """Set DET44 timeouts + Args: + tcp_established (int): TCP established timeout + tcp_transitory (int): TCP transitory timeout + udp (int): UDP timeout + icmp (int): ICMP timeout + """ + self.vpp.api.det44_set_timeouts( + icmp=icmp, + udp=udp, + tcp_established=tcp_established, + tcp_transitory=tcp_transitory, + ) + + def get_det44_interfaces_outside(self): + ifaces_outside = [] + for iface in self.vpp.api.det44_interface_dump(): + if iface.is_outside: + ifaces_outside.append(iface.sw_if_index) + return ifaces_outside + + def get_det44_interfaces_inside(self): + ifaces_inside = [] + for iface in self.vpp.api.det44_interface_dump(): + if iface.is_inside: + ifaces_inside.append(iface.sw_if_index) + return ifaces_inside + + def add_det44_identity_mapping(self, ip_address, protocol, port, tag=''): + """Add DET44 identity mapping (exclude rule)""" + flags = NAT_IS_ADDR_ONLY if not (protocol or port) else NAT_IS_NONE + + self.vpp.api.det44_add_del_identity_mapping( + is_add=True, + addr=ip_address, + protocol=protocol, + port=port, + flags=flags, + tag=tag if tag else '', + ) + + def delete_det44_identity_mapping(self, ip_address, protocol, port, tag=''): + """Delete DET44 identity mapping (exclude rule)""" + flags = NAT_IS_ADDR_ONLY if not (protocol or port) else NAT_IS_NONE + + self.vpp.api.det44_add_del_identity_mapping( + is_add=False, + addr=ip_address, + protocol=protocol, + port=port, + flags=flags, + tag=tag, + ) diff --git a/python/vyos/vpp/nat/nat44.py b/python/vyos/vpp/nat/nat44.py new file mode 100644 index 000000000..179e86c0c --- /dev/null +++ b/python/vyos/vpp/nat/nat44.py @@ -0,0 +1,243 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from vyos.vpp import VPPControl + + +# NAT44 flags +NAT_IS_NONE = 0x00 +NAT_IS_TWICE_NAT = 0x01 +NAT_IS_SELF_TWICE_NAT = 0x02 +NAT_IS_OUT2IN_ONLY = 0x04 +NAT_IS_ADDR_ONLY = 0x08 +NAT_IS_OUTSIDE = 0x10 +NAT_IS_INSIDE = 0x20 + +NO_INTERFACE = 0xFFFFFFFF + + +class Nat44: + def __init__(self): + self.vpp = VPPControl() + + def enable_nat44_ed(self): + """Enable NAT44 endpoint dependent plugin + Example: + from vyos.vpp.nat import Nat44 + nat44 = Nat44() + nat44.enable_nat44_ed() + https://github.com/FDio/vpp/blob/stable/2410/src/plugins/nat/nat44-ed/nat44_ed.api + """ + self.vpp.api.nat44_ed_plugin_enable_disable(enable=True) + + def disable_nat44_ed(self): + """Disable NAT44 endpoint dependent plugin""" + self.vpp.api.nat44_ed_plugin_enable_disable(enable=False) + + def enable_nat44_ei(self): + """Enable NAT44 endpoint independent plugin + Example: + from vyos.vpp.nat import Nat44 + nat44 = Nat44() + nat44.enable_nat44_ei() + """ + self.vpp.api.nat44_ei_plugin_enable_disable(enable=True) + + def add_nat44_interface_inside(self, interface_in): + """Add NAT44 interface""" + self.vpp.api.nat44_interface_add_del_feature( + flags=NAT_IS_INSIDE, + sw_if_index=self.vpp.get_sw_if_index(interface_in), + is_add=True, + ) + + def delete_nat44_interface_inside(self, interface_in): + """Delete NAT44 interface""" + self.vpp.api.nat44_interface_add_del_feature( + flags=NAT_IS_INSIDE, + sw_if_index=self.vpp.get_sw_if_index(interface_in), + is_add=False, + ) + + def add_nat44_interface_outside(self, interface_out): + """Add NAT44 interface""" + self.vpp.api.nat44_interface_add_del_feature( + flags=NAT_IS_OUTSIDE, + sw_if_index=self.vpp.get_sw_if_index(interface_out), + is_add=True, + ) + + def delete_nat44_interface_outside(self, interface_out): + """Delete NAT44 interface""" + self.vpp.api.nat44_interface_add_del_feature( + flags=NAT_IS_OUTSIDE, + sw_if_index=self.vpp.get_sw_if_index(interface_out), + is_add=False, + ) + + def add_nat44_address_range(self, addresses, twice_nat): + """Add NAT44 address range""" + if '-' not in addresses: + first_ip_address = last_ip_address = addresses + else: + first_ip_address, last_ip_address = addresses.split('-') + self.vpp.api.nat44_add_del_address_range( + flags=NAT_IS_TWICE_NAT if twice_nat else NAT_IS_NONE, + first_ip_address=first_ip_address, + last_ip_address=last_ip_address, + is_add=True, + ) + + def delete_nat44_address_range(self, addresses, twice_nat): + """Delete NAT44 address range""" + if '-' not in addresses: + first_ip_address = last_ip_address = addresses + else: + first_ip_address, last_ip_address = addresses.split('-') + self.vpp.api.nat44_add_del_address_range( + flags=NAT_IS_TWICE_NAT if twice_nat else NAT_IS_NONE, + first_ip_address=first_ip_address, + last_ip_address=last_ip_address, + is_add=False, + ) + + def add_nat44_interface_address(self, interface, twice_nat): + """Add NAT44 interface address""" + self.vpp.api.nat44_add_del_interface_addr( + flags=NAT_IS_TWICE_NAT if twice_nat else NAT_IS_NONE, + sw_if_index=self.vpp.get_sw_if_index(interface), + is_add=True, + ) + + def delete_nat44_interface_address(self, interface, twice_nat): + """Delete NAT44 interface address""" + self.vpp.api.nat44_add_del_interface_addr( + flags=NAT_IS_TWICE_NAT if twice_nat else NAT_IS_NONE, + sw_if_index=self.vpp.get_sw_if_index(interface), + is_add=False, + ) + + def add_nat44_static_mapping( + self, + local_ip, + external_ip, + local_port, + external_port, + protocol, + twice_nat, + self_twice_nat, + out2in, + pool_ip, + ): + """Add NAT44 static mapping""" + flags = NAT_IS_ADDR_ONLY if not (protocol or local_port) else NAT_IS_NONE + flags |= NAT_IS_TWICE_NAT if twice_nat else 0 + flags |= NAT_IS_SELF_TWICE_NAT if self_twice_nat else 0 + flags |= NAT_IS_OUT2IN_ONLY if out2in else 0 + self.vpp.api.nat44_add_del_static_mapping_v2( + local_ip_address=local_ip, + external_ip_address=external_ip, + protocol=protocol, + local_port=local_port, + external_port=external_port, + match_pool=True if pool_ip else False, + pool_ip_address=pool_ip if pool_ip else '', + flags=flags, + is_add=True, + ) + + def delete_nat44_static_mapping( + self, + local_ip, + external_ip, + local_port, + external_port, + protocol, + twice_nat, + self_twice_nat, + out2in, + pool_ip, + ): + """Delete NAT44 static mapping""" + flags = NAT_IS_ADDR_ONLY if not (protocol or local_port) else NAT_IS_NONE + flags |= NAT_IS_TWICE_NAT if twice_nat else 0 + flags |= NAT_IS_SELF_TWICE_NAT if self_twice_nat else 0 + flags |= NAT_IS_OUT2IN_ONLY if out2in else 0 + self.vpp.api.nat44_add_del_static_mapping_v2( + local_ip_address=local_ip, + external_ip_address=external_ip, + protocol=protocol, + local_port=local_port, + external_port=external_port, + match_pool=True if pool_ip else False, + pool_ip_address=pool_ip if pool_ip else '', + flags=flags, + is_add=False, + ) + + def add_nat44_identity_mapping(self, ip_address, protocol, port, interface): + """Add NAT44 identity mapping""" + self.vpp.api.nat44_add_del_identity_mapping( + ip_address=ip_address, + protocol=protocol, + port=port, + sw_if_index=( + self.vpp.get_sw_if_index(interface) if interface else NO_INTERFACE + ), + flags=NAT_IS_ADDR_ONLY if not (protocol or port) else NAT_IS_NONE, + is_add=True, + ) + + def delete_nat44_identity_mapping(self, ip_address, protocol, port, interface): + """Delete NAT44 identity mapping""" + self.vpp.api.nat44_add_del_identity_mapping( + ip_address=ip_address, + protocol=protocol, + port=port, + sw_if_index=( + self.vpp.get_sw_if_index(interface) if interface else NO_INTERFACE + ), + flags=NAT_IS_ADDR_ONLY if not (protocol or port) else NAT_IS_NONE, + is_add=False, + ) + + def set_nat_timeouts(self, icmp, udp, tcp_established, tcp_transitory): + """Set NAT timeouts""" + self.vpp.api.nat_set_timeouts( + icmp=icmp, + udp=udp, + tcp_established=tcp_established, + tcp_transitory=tcp_transitory, + ) + + def enable_disable_nat44_forwarding(self, enable): + """Enable/disable NAT44 forwarding""" + self.vpp.api.nat44_forwarding_enable_disable(enable=enable) + + def set_nat44_session_limit(self, session_limit: int) -> None: + """Set NAT44 session limit + + Args: + session_limit (int): Maximum number of sessions per thread + """ + self.vpp.api.nat44_set_session_limit( + session_limit=session_limit, + ) + + def enable_ipfix(self): + """Enable NAT44 IPFIX logging""" + self.vpp.api.nat44_ei_ipfix_enable_disable(enable=True) diff --git a/python/vyos/vpp/sflow/__init__.py b/python/vyos/vpp/sflow/__init__.py new file mode 100644 index 000000000..8d27be75a --- /dev/null +++ b/python/vyos/vpp/sflow/__init__.py @@ -0,0 +1,3 @@ +from .sflow import SFlow + +__all__ = ['SFlow'] diff --git a/python/vyos/vpp/sflow/sflow.py b/python/vyos/vpp/sflow/sflow.py new file mode 100644 index 000000000..644270d3a --- /dev/null +++ b/python/vyos/vpp/sflow/sflow.py @@ -0,0 +1,49 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from vyos.vpp import VPPControl + + +class SFlow: + def __init__(self): + self.vpp = VPPControl() + + def enable_sflow(self, interface): + """Enable sFlow on interface""" + self.vpp.api.sflow_enable_disable( + enable_disable=True, + hw_if_index=self.vpp.get_sw_if_index(interface), + ) + + def disable_sflow(self, interface): + """Disable sFlow on interface""" + self.vpp.api.sflow_enable_disable( + enable_disable=False, + hw_if_index=self.vpp.get_sw_if_index(interface), + ) + + def set_sampling_rate(self, sample_rate): + """Set sFlow sampling-rate""" + self.vpp.api.sflow_sampling_rate_set(sampling_N=sample_rate) + + def set_polling_interval(self, interval): + """Set sFlow polling interval""" + self.vpp.api.sflow_polling_interval_set(polling_S=interval) + + def set_header_bytes(self, header_bytes): + """Set sFlow maximum header length in bytes""" + self.vpp.api.sflow_header_bytes_set(header_B=header_bytes) diff --git a/python/vyos/vpp/utils.py b/python/vyos/vpp/utils.py new file mode 100644 index 000000000..10940b5a4 --- /dev/null +++ b/python/vyos/vpp/utils.py @@ -0,0 +1,402 @@ +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import ctypes +import os +import socket +from fcntl import ioctl +from pathlib import Path +from struct import pack + + +mem_shift = {'K': 10, 'KB': 10, 'M': 20, 'MB': 20, 'G': 30, 'GB': 30} + + +def iftunnel_transform(iface: str) -> str: + """Transform interface name from `vppxxxNN` to `xxx_tunnelNN` + + Args: + iface (str): original interface name + + Raises: + ValueError: Raised if an interface name does not start with a alpha and ends with decimal digit + + Returns: + str: Transformed interface name + """ + # Remove vpp prefix + iface = iface.removeprefix('vpp') + # Check format + if not iface[0].isascii() or not iface[-1].isdecimal(): + raise ValueError(f'Wrong interface name format: {iface}') + # Transform + iface_type: str = iface.rstrip('0123456789') + iface_num: str = iface.removeprefix(iface_type) + # Return transformed + return f'{iface_type}_tunnel{iface_num}' + + +def vpp_iface_name_transform(iface: str) -> str: + """Convert a CLI interface name to its corresponding VPP interface name format + + Args: + iface (str): Interface name as used in VyOS configuration (e.g., "bond0"). + + Returns: + str: Interface name formatted as recognized by VPP (e.g., "BondEthernet0"). + """ + vpp_iface_name = iface + if vpp_iface_name.startswith('vppbond'): + # interface name in VPP is BondEthernetX + vpp_iface_name = vpp_iface_name.replace('vppbond', 'BondEthernet') + return vpp_iface_name + + +def cli_ifaces_list(config_instance, mode: str = 'candidate') -> list[str]: + """List of all VPP interfaces (CLI names) + + Args: + config_instance (VyOS Config): VyOS Config instance + mode (str, optional): `candidate` or `running`. Defaults to 'candidate'. + + Returns: + list[str]: list of interfaces + """ + + effective_mode: bool = True if mode == 'running' else False + + # Read a config + config = config_instance.get_config_dict( + ['vpp'], + key_mangling=('-', '_'), + effective=effective_mode, + get_first_key=True, + no_tag_node_value_mangle=True, + with_recursive_defaults=True, + ) + + interfaces_config = config_instance.get_config_dict( + ['interfaces', 'vpp'], + key_mangling=('-', '_'), + effective=effective_mode, + get_first_key=True, + no_tag_node_value_mangle=True, + with_recursive_defaults=True, + ) + + vpp_ifaces: list[str] = [] + + # Get a list of Ethernet interfaces + for iface in config.get('settings', {}).get('interface', {}).keys(): + vpp_ifaces.append(iface) + + # Get a list of interfaces VPP + for iface_type in interfaces_config.keys(): + for iface in interfaces_config.get(iface_type, {}).keys(): + vpp_ifaces.append(iface) + + return vpp_ifaces + + +def cli_ethernet_with_vifs_ifaces( + config_instance, include_nested_vifs=False +) -> list[str]: + """List of all VPP Ethernet interfaces with VIFs + + Args: + config_instance (VyOS Config): VyOS Config instance + include_nested_vifs (bool): Include Q-in-Q/customer VIFs if True + + Returns: + list[str]: list of interfaces + """ + from vyos.configdict import get_interface_dict + + # Read a config + config = config_instance.get_config_dict( + ['vpp'], + key_mangling=('-', '_'), + get_first_key=True, + no_tag_node_value_mangle=True, + with_recursive_defaults=True, + ) + + ifaces: list[str] = [] + + # Get a list of Ethernet interfaces + parent_ifaces = list(config.get('settings', {}).get('interface', {}).keys()) + + # Add Ethernet interfaces with VIFs + for iface in parent_ifaces: + ifaces.append(iface) + _, iface_config = get_interface_dict( + config_instance, ['interfaces', 'ethernet'], ifname=iface + ) + ifaces.extend([f'{iface}.{vif}' for vif in iface_config.get('vif', {})]) + ifaces.extend([f'{iface}.{vif_s}' for vif_s in iface_config.get('vif_s', {})]) + + if include_nested_vifs: + for vif_s, vif_s_config in iface_config.get('vif_s', {}).items(): + ifaces.extend( + [ + f'{iface}.{vif_s}.{vif_c}' + for vif_c in vif_s_config.get('vif_c', {}) + ] + ) + + return ifaces + + +def vpp_ifaces_list(vpp_api) -> list[dict]: + """List interfaces in VPP + + Args: + vpp_api (_type_): VPP API object + + Returns: + list[dict]: list of dictionaries with interfaces + """ + ifaces_list: list[dict] = [] + sw_ifaces_dump = vpp_api.sw_interface_dump() + while sw_ifaces_dump: + iface_details = sw_ifaces_dump.pop() + ifaces_list.append(iface_details._asdict()) + + return ifaces_list + + +def vpp_ip_addresses_by_index(vpp_api, index: int, is_ipv6: bool = False) -> list[str]: + """List of IP addresses for interface by its index in VPP + + Args: + vpp_api (_type_): VPP API object + index (int): interface index in vpp + is_ipv6 (bool, optional): If True, return IPv6 addresses. Defaults to False. + + Returns: + list[str]: list of IP addresses (e.g. '192.0.2.1/24', '2001:db8::1/64') + """ + ip_addresses_list: list[str] = [] + ip_address_dump = vpp_api.ip_address_dump(sw_if_index=index, is_ipv6=is_ipv6) + while ip_address_dump: + ip_address_details = ip_address_dump.pop() + ip_addresses_list.append(str(ip_address_details._asdict().get('prefix'))) + return ip_addresses_list + + +def vpp_ifaces_stats( + iface_name: str = '', +) -> dict[str, dict[str, int | dict[str, int]]]: + from re import compile as re_compile + from vpp_papi import vpp_stats + + def total_value(val_list: vpp_stats.SimpleList) -> int | dict[str, int]: + """Helper for aggregation stats from multiple workers + + Args: + val_list (vpp_stats.SimpleList): list of stats for all workers + + Returns: + int | dict[str, int]: Summary stats + """ + # if all items are int return their sum + if all(isinstance(value, int) for value in val_list): + return sum(val_list) + # if all items are tuple + if all(isinstance(value, tuple) for value in val_list): + combined_stats = {} + # process individual workers + for worker_stats in val_list: + packets, octets = worker_stats + sum_packets = combined_stats.get('packets', 0) + packets + sum_octets = combined_stats.get('octets', 0) + octets + combined_stats = {'packets': sum_packets, 'bytes': sum_octets} + return combined_stats + + # items are something unknown, just return what we received + return val_list + + stats = vpp_stats.VPPStats() + + ifaces_stats: dict[str, dict[str, int | dict[str, int]]] = {} + + # prepare parser for stats output + regex_parser = re_compile(r'^/interfaces/(?P<iface>[^/]+)/(?P<param>[^/]+)') + # get list of available stats and dump them + stats_list: list[str] = stats.ls([f'^/interfaces/{iface_name}']) + stats_dump: list[dict[str, int]] = stats.dump(stats_list) + + # parse outputs and convert it to a dictionary + for stats_key, stats_value in stats_dump.items(): + parsed_key = regex_parser.search(stats_key).groupdict() + iface_name = parsed_key['iface'] + param = parsed_key['param'] + stats_item = {param: total_value(stats_value)} + if iface_name in ifaces_stats: + ifaces_stats[iface_name].update(stats_item) + else: + ifaces_stats[iface_name] = stats_item + + return ifaces_stats + + +def get_default_hugepage_size() -> int: + """ + Retrieve the system's default huge page size. + :return: The default huge page size in bytes. + """ + page_size = None + try: + # default huge page size + memfd = os.memfd_create('tmp', os.MFD_HUGETLB) + st = os.fstat(memfd) + page_size = st.st_blksize + os.close(memfd) + except OSError: + pass + + return page_size + + +def get_default_page_size() -> int: + """ + Retrieve the system's default page size. + :return: The default page size in bytes. + """ + return os.sysconf('SC_PAGESIZE') + + +def get_hugepage_sizes() -> list[int]: + """ + Retrieve all available huge page sizes from the system. + :return: A list of huge page sizes in bytes. + """ + huge_sizes = [] + path = '/sys/kernel/mm/hugepages/' + try: + entries = os.listdir(path) + for entry in entries: + if entry.startswith('hugepages-'): + try: + size_kb = int(entry.replace('hugepages-', '').replace('kB', '')) + huge_sizes.append(size_kb << 10) # Convert KB to bytes + except ValueError: + pass + except FileNotFoundError: + pass + + return huge_sizes + + +def human_memory_to_bytes(value: str) -> int: + """ + Convert a human-readable vpp memory format (K, M, G, xB) to a byte value. + + :param value: The string memory size in vpp human-readable format. + :return: A int representing the value. + """ + value = value.strip().upper() + try: + return int(value) + except ValueError: + for unit in sorted(mem_shift.keys(), key=len, reverse=True): + if value.endswith(unit): + num = value[: -len(unit)] + return int(num) << mem_shift[unit] + + +def bytes_to_human_memory(value: int, unit: str) -> str | None: + """ + Convert a byte value to a human-readable format (K, M, G). + + :param value: The size in bytes. + :param unit: The unit to convert to ('K', 'M', 'G'). + :return: A string representing the value in the specified unit, or None if zero. + """ + unit = unit.upper() + val = value >> mem_shift[unit] + return f'{val}{unit}' if val else None + + +class EthtoolGDrvinfo: + """Return interface details like `ethtol -i` does""" + + # TODO + # this probably need to be replaced with a code generator + # like ctypeslib or C extension + class EthtoolDrvinfo(ctypes.Structure): + _fields_ = [ + ('cmd', ctypes.c_uint32), + ('driver', ctypes.c_char * 32), # Driver short name + ('version', ctypes.c_char * 32), # Driver version + ('fw_version', ctypes.c_char * 32), # Firmware version + # Be careful: bus info can be longer than 32 chars and thus truncated + ('bus_info', ctypes.c_char * 32), # Bus info. + ('erom_version', ctypes.c_char * 32), # Expansion ROM version + ('reserved2', ctypes.c_char * 12), # Reserved for future use + ('n_priv_flags', ctypes.c_uint32), # Number of private flags + ('n_stats', ctypes.c_uint32), # Number of U64 stats + ('testinfo_len', ctypes.c_uint32), # Test info length + ('eedump_len', ctypes.c_uint32), # EEPROM dump length + ('regdump_len', ctypes.c_uint32), # Register dump length + ] + + def __init__(self, iface: str): + # Constants for ethtool + SIOCETHTOOL = 0x8946 # pretend to be ethtool + ETHTOOL_GDRVINFO = 0x00000003 # Command to get driver info + + # Create a dummy socket + sockfd = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + + # Prepare the request for getting driver info + drvinfo = self.EthtoolDrvinfo(cmd=ETHTOOL_GDRVINFO) + ifreq: bytes = pack('16sP', iface.encode('utf-8'), ctypes.addressof(drvinfo)) + + # Make an ioctl call to get the driver info + try: + ioctl(sockfd, SIOCETHTOOL, ifreq) + except OSError: + raise FileNotFoundError(f'There is no Ethernet device: {iface}') + + # Close the socket + sockfd.close() + + # save the information + self.driver: str = drvinfo.driver.decode('utf-8').strip('\x00') + self.version: str = drvinfo.version.decode('utf-8').strip('\x00') + self.fw_version: str = drvinfo.fw_version.decode('utf-8').strip('\x00') + self.bus_info: str = drvinfo.bus_info.decode('utf-8').strip('\x00') + self.erom_version: str = drvinfo.erom_version.decode('utf-8').strip('\x00') + self.reserved2: str = drvinfo.reserved2.decode('utf-8').strip('\x00') + self.n_priv_flags: int = drvinfo.n_priv_flags + self.testinfo_len: int = drvinfo.testinfo_len + self.eedump_len: int = drvinfo.eedump_len + self.regdump_len: int = drvinfo.regdump_len + + def bus_info_expand(self, bus_name: str) -> str: + bus_path = Path(f'/sys/bus/{bus_name}/devices').glob(f'{self.bus_info}*') + dev_ids = list(bus_path) + if not dev_ids: + raise FileNotFoundError( + f'No matching IDs on the bus: {self.bus_info} on {bus_name}' + ) + if len(dev_ids) > 1: + raise FileNotFoundError( + f'There are more than one matching IDs on the bus: {dev_ids} on {bus_name}' + ) + return dev_ids[0].name diff --git a/python/vyos/vyconf_session.py b/python/vyos/vyconf_session.py new file mode 100644 index 000000000..91e51338c --- /dev/null +++ b/python/vyos/vyconf_session.py @@ -0,0 +1,312 @@ +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# + +import os +import weakref +import tempfile +import json +from functools import wraps +from typing import Type + +from vyos.proto import vyconf_client +from vyos.migrate import ConfigMigrate +from vyos.migrate import ConfigMigrateError +from vyos.component_version import append_system_version +from vyos.utils.session import in_config_session +from vyos.proto.vyconf_proto import Errnum +from vyos.utils.commit import acquire_commit_lock_file +from vyos.utils.commit import release_commit_lock_file +from vyos.utils.commit import call_commit_hooks +from vyos.remote import get_config_file + + +class VyconfSessionError(Exception): + pass + + +def new_session(pid: int, sudo_user: str, user: str): + out = vyconf_client.send_request( + 'setup_session', + client_pid=pid, + client_sudo_user=sudo_user, + client_user=user, + ) + return out.output + + +class VyconfSession: + def __init__( + self, + pid: int = None, + token: str = None, + extant=False, + on_error: Type[Exception] = None, + ): + self.pid = pid if pid else os.getpid() + self.sudo_user = os.environ.get('SUDO_USER', None) + self.user = os.environ.get('USER', None) + + self.in_config_session = in_config_session() + + match token: + case None: + # config-mode sessions are persistent, and managed by caller (CLI or ConfigSession) + # + # op-mode sessions are ephemeral, unless forced with extant=True: + # --- open a new session on init; teardown in finalizer + if self.in_config_session: + out = vyconf_client.send_request( + 'session_of_pid', client_pid=self.pid + ) + if out.output is None: + self.__token = new_session(self.pid, self.sudo_user, self.user) + out = vyconf_client.send_request( + 'enter_configuration_mode', token=self.__token + ) + if out.status: + raise VyconfSessionError(self.output(out)) + else: + self.__token = out.output + else: + if not extant: + self.__token = new_session(self.pid, self.sudo_user, self.user) + else: + out = vyconf_client.send_request( + 'session_of_pid', client_pid=self.pid + ) + if out.output is None: + raise ValueError(f'No existing session for pid {self.pid}') + self.__token = out.output + case _: + out = vyconf_client.send_request('session_exists', token=token) + if out.status: + raise ValueError(f'No existing session for token: {token}') + self.__token = token + + if not self.in_config_session and not extant: + self._finalizer = weakref.finalize(self, self._teardown, self.__token) + + self.on_error = on_error + + @classmethod + def _teardown(cls, token): + vyconf_client.send_request('teardown', token) + + def teardown(self): + self._teardown(self.__token) + + def exit_config_mode(self): + if self.session_changed(): + return 'Uncommitted changes', Errnum.UNCOMMITED_CHANGES + out = vyconf_client.send_request('exit_configuration_mode', token=self.__token) + return self.output(out), out.status + + def in_session(self) -> bool: + return self.in_config_session + + def session_changed(self) -> bool: + out = vyconf_client.send_request('session_changed', token=self.__token) + return not bool(out.status) + + def get_config(self): + out = vyconf_client.send_request('get_config', token=self.__token) + if out.status: + raise VyconfSessionError(self.output(out)) + return out.output + + def show_sessions( + self, exclude_self: bool = False, exclude_other: bool = False + ) -> list | dict: + out = vyconf_client.send_request( + 'show_sessions', + token=self.__token, + exclude_self=exclude_self, + exclude_other=exclude_other, + ) + + lst = json.loads(out.output) + if len(lst) == 1: + return lst[0] + return lst + + @staticmethod + def config_mode(f): + @wraps(f) + def wrapped(self, *args, **kwargs): + msg = 'operation not available outside of config mode' + if not self.in_config_session: + if self.on_error is None: + raise VyconfSessionError(msg) + raise self.on_error(msg) + return f(self, *args, **kwargs) + + return wrapped + + @staticmethod + def raise_exception(f): + @wraps(f) + def wrapped(self, *args, **kwargs): + if self.on_error is None: + return f(self, *args, **kwargs) + o, e = f(self, *args, **kwargs) + if e: + raise self.on_error(o) + return o, e + + return wrapped + + @staticmethod + def output(o): + out = '' + for res in (o.output, o.error, o.warning): + if res is not None: + out = out + res + return out + + @config_mode + def discard(self) -> tuple[str, int]: + out = vyconf_client.send_request('discard', token=self.__token) + return self.output(out), out.status + + @raise_exception + @config_mode + def set(self, path: list[str]) -> tuple[str, int]: + out = vyconf_client.send_request('set', token=self.__token, path=path) + return self.output(out), out.status + + @raise_exception + @config_mode + def delete(self, path: list[str]) -> tuple[str, int]: + out = vyconf_client.send_request('delete', token=self.__token, path=path) + return self.output(out), out.status + + @raise_exception + def aux_set( + self, path: list[str], script_name: str, tag_value: str = None + ) -> tuple[str, int]: + out = vyconf_client.send_request( + 'aux_set', + token=self.__token, + path=path, + script_name=script_name, + tag_value=tag_value, + ) + return self.output(out), out.status + + @raise_exception + def aux_delete( + self, path: list[str], script_name: str, tag_value: str = None + ) -> tuple[str, int]: + out = vyconf_client.send_request( + 'aux_delete', + token=self.__token, + path=path, + script_name=script_name, + tag_value=tag_value, + ) + return self.output(out), out.status + + @raise_exception + @config_mode + def commit(self) -> tuple[str, int]: + if not self.session_changed(): + out = 'No changes to commit' + return out, 0 + + lock_fd, out = acquire_commit_lock_file() + if lock_fd is None: + return out, Errnum.COMMIT_IN_PROGRESS + + pre_out, _ = call_commit_hooks('pre') + out = vyconf_client.send_request('commit', token=self.__token) + os.environ['COMMIT_STATUS'] = 'FAILURE' if out.status else 'SUCCESS' + post_out, _ = call_commit_hooks('post') + + release_commit_lock_file(lock_fd) + + return pre_out + self.output(out) + post_out, out.status + + @raise_exception + @config_mode + def load_config( + self, file_name: str, migrate: bool = False, cached: bool = False + ) -> tuple[str, int]: + # pylint: disable=consider-using-with + file_path = tempfile.NamedTemporaryFile(delete=False).name + err = get_config_file(file_name, file_path) + if err: + os.remove(file_path) + return str(err), Errnum.INVALID_VALUE + if not cached: + if migrate: + config_migrate = ConfigMigrate(file_path) + try: + config_migrate.run() + except ConfigMigrateError as e: + os.remove(file_path) + return repr(e), 1 + + out = vyconf_client.send_request( + 'load', token=self.__token, location=file_path, cached=cached + ) + + if not cached: + os.remove(file_path) + + return self.output(out), out.status + + @raise_exception + @config_mode + def merge_config( + self, file_name: str, migrate: bool = False, destructive: bool = False + ) -> tuple[str, int]: + # pylint: disable=consider-using-with + file_path = tempfile.NamedTemporaryFile(delete=False).name + err = get_config_file(file_name, file_path) + if err: + os.remove(file_path) + return str(err), Errnum.INVALID_VALUE + if migrate: + config_migrate = ConfigMigrate(file_path) + try: + config_migrate.run() + except ConfigMigrateError as e: + os.remove(file_path) + return repr(e), 1 + + out = vyconf_client.send_request( + 'merge', token=self.__token, location=file_path, destructive=destructive + ) + + os.remove(file_path) + + return self.output(out), out.status + + @raise_exception + def save_config(self, file: str, append_version: bool = False) -> tuple[str, int]: + file = os.path.realpath(file) + out = vyconf_client.send_request('save', token=self.__token, location=file) + if append_version: + append_system_version(file) + return self.output(out), out.status + + @raise_exception + def show_config(self, path: list[str] = None) -> tuple[str, int]: + if path is None: + path = [] + out = vyconf_client.send_request('show_config', token=self.__token, path=path) + return self.output(out), out.status diff --git a/python/vyos/wanloadbalance.py b/python/vyos/wanloadbalance.py index 62e109f21..05ae6b536 100644 --- a/python/vyos/wanloadbalance.py +++ b/python/vyos/wanloadbalance.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2024 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -65,7 +65,46 @@ def nft_rule(rule_conf, rule_id, local=False, exclude=False, limit=False, weight if port[:1] == '!': operator = '!=' port = port[1:] - output.append(f'th {prefix}port {operator} {port}') + output.append(f'th {prefix}port {operator} {{ {port} }}') + + if 'group' in direction_conf: + group = direction_conf['group'] + if 'address_group' in group: + group_name = group['address_group'] + operator = '' + exclude = group_name[0] == "!" + if exclude: + operator = '!=' + group_name = group_name[1:] + output.append(f'ip {prefix}addr {operator} @A_{group_name}') + if 'network_group' in group: + group_name = group['network_group'] + operator = '' + if group_name[0] == "!": + operator = '!=' + group_name = group_name[1:] + output.append(f'ip {prefix}addr {operator} @N_{group_name}') + # Generate firewall group domain-group + if 'domain_group' in group: + group_name = group['domain_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'ip {prefix}addr {operator} @D_{group_name}') + if 'port_group' in group: + proto = rule_conf['protocol'] + group_name = group['port_group'] + + if proto == 'tcp_udp': + proto = 'th' + + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + + output.append(f'{proto} {prefix}port {operator} @P_{group_name}') if 'source_based_routing' not in rule_conf and not restore_mark: output.append('ct state new') @@ -115,7 +154,7 @@ def wlb_weight_interfaces(rule_conf, health_state): for ifname, weight in sorted(interfaces, key=lambda i: i[1]): # build weight ranges end = start + weight - 1 out.append((ifname, f'{start}-{end}' if end > start else start)) - start = weight + start += weight return out, total_weight diff --git a/python/vyos/xml_ref/__init__.py b/python/vyos/xml_ref/__init__.py index 99d8432d2..41a25049e 100644 --- a/python/vyos/xml_ref/__init__.py +++ b/python/vyos/xml_ref/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -14,6 +14,8 @@ # along with this library. If not, see <http://www.gnu.org/licenses/>. from typing import Optional, Union, TYPE_CHECKING +from typing import Callable +from typing import Any from vyos.xml_ref import definition from vyos.xml_ref import op_definition @@ -89,6 +91,7 @@ def from_source(d: dict, path: list) -> bool: def ext_dict_merge(source: dict, destination: Union[dict, 'ConfigDict']): return definition.ext_dict_merge(source, destination) + def load_op_reference(op_cache=[]): if op_cache: return op_cache[0] @@ -108,5 +111,26 @@ def load_op_reference(op_cache=[]): return op_xml -def get_op_ref_path(path: list) -> list[op_definition.PathData]: - return load_op_reference()._get_op_ref_path(path) + +def walk_op_data(func: Callable[[tuple, dict], Any]): + return load_op_reference().walk(func) + + +def walk_op_node_data(): + return load_op_reference().walk_node_data() + + +def lookup_op_data( + path: list, tag_values: bool = False, last_node_type: str = '' +) -> (dict, list[str]): + return load_op_reference().lookup( + path, tag_values=tag_values, last_node_type=last_node_type + ) + + +def lookup_op_node_data( + path: list, tag_values: bool = False, last_node_type: str = '' +) -> list[op_definition.NodeData]: + return load_op_reference().lookup_node_data( + path, tag_values=tag_values, last_node_type=last_node_type + ) diff --git a/python/vyos/xml_ref/definition.py b/python/vyos/xml_ref/definition.py index 4e755ab72..cee762b02 100644 --- a/python/vyos/xml_ref/definition.py +++ b/python/vyos/xml_ref/definition.py @@ -1,4 +1,4 @@ -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -104,7 +104,7 @@ class Xml: if self.exists(path): if self.is_valueless(path) or not self.is_leaf(path): # It's a complete path for a valueless node - # or a path to an empy non-leaf node + # or a path to an empty non-leaf node return (path, None) else: raise ValueError(f'Path "{path}" needs a value or children') diff --git a/python/vyos/xml_ref/generate_cache.py b/python/vyos/xml_ref/generate_cache.py index 093697993..daea50dc9 100755 --- a/python/vyos/xml_ref/generate_cache.py +++ b/python/vyos/xml_ref/generate_cache.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2023-2024 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -52,7 +52,7 @@ def non_trivial(s): return s def main(): - parser = ArgumentParser(description='generate and save dict from xml defintions') + parser = ArgumentParser(description='generate and save dict from xml definitions') parser.add_argument('--xml-dir', type=str, required=True, help='transcluded xml interface-definition directory') parser.add_argument('--internal-cache', type=str, required=True, diff --git a/python/vyos/xml_ref/generate_op_cache.py b/python/vyos/xml_ref/generate_op_cache.py index 95779d066..ca939f550 100755 --- a/python/vyos/xml_ref/generate_op_cache.py +++ b/python/vyos/xml_ref/generate_op_cache.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2024-2025 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -14,10 +14,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import os +import io import re import sys -import json import glob +import json +import atexit from argparse import ArgumentParser from os.path import join @@ -25,23 +28,43 @@ from os.path import abspath from os.path import dirname from xml.etree import ElementTree as ET from xml.etree.ElementTree import Element +from functools import cmp_to_key from typing import TypeAlias from typing import Optional +from op_definition import NodeData +from op_definition import OpKey # pylint: disable=unused-import # noqa: F401 +from op_definition import OpData # pylint: disable=unused-import # noqa: F401 +from op_definition import key_name +from op_definition import key_type +from op_definition import node_data_difference +from op_definition import get_node_data +from op_definition import collapse + _here = dirname(__file__) sys.path.append(join(_here, '..')) -from defaults import directories +# pylint: disable=wrong-import-position,wrong-import-order +from defaults import directories # noqa: E402 -from op_definition import PathData - -xml_op_cache_json = 'xml_op_cache.json' -xml_op_tmp = join('/tmp', xml_op_cache_json) op_ref_cache = abspath(join(_here, 'op_cache.py')) OptElement: TypeAlias = Optional[Element] -DEBUG = False + + +# It is expected that the node_data help txt contained in top-level nodes, +# shared across files, e.g.'show', will reveal inconsistencies; to list +# differences, use --check-xml-consistency +CHECK_XML_CONSISTENCY = False +err_buf = io.StringIO() + + +def write_err_buf(): + err_buf.seek(0) + out = err_buf.read() + print(out) + err_buf.close() def translate_exec(s: str) -> str: @@ -57,7 +80,7 @@ def translate_position(s: str, pos: list[str]) -> str: # preferred to .format(*list) to avoid collisions with braces for i, p in enumerate(pos): - t = t.replace(f'_place_holder_{i+1}_', p) + t = t.replace(f'_place_holder_{i+1}_', f'{{{{{p}}}}}') return t @@ -65,6 +88,14 @@ def translate_position(s: str, pos: list[str]) -> str: def translate_command(s: str, pos: list[str]) -> str: s = translate_exec(s) s = translate_position(s, pos) + + # If there are any untranslated occurrences of "_place_holder_", + # it means the command is incorrect, + # e.g., it references "$6" when it only has five words. + if re.search(r'_place_holder_', s): + print(f'Command translation failed: {s}') + sys.exit(1) + return s @@ -74,27 +105,123 @@ def translate_op_script(s: str) -> str: return s -def insert_node(n: Element, l: list[PathData], path=None) -> None: - # pylint: disable=too-many-locals,too-many-branches +def compare_keys(a, b): + # pylint: disable=too-many-return-statements + match key_type(a), key_type(b): + case None, None: + if key_name(a) == key_name(b): + return 0 + return -1 if key_name(a) < key_name(b) else 1 + case None, _: + return -1 + case _, None: + return 1 + case _, _: + if key_name(a) == key_name(b): + if key_type(a) == key_type(b): + return 0 + return -1 if key_type(a) < key_type(b) else 1 + return -1 if key_name(a) < key_name(b) else 1 + + +def sort_func(obj: dict, key_func): + if not obj or not isinstance(obj, dict): + return obj + k_list = list(obj.keys()) + if not isinstance(k_list[0], tuple): + return obj + k_list = sorted(k_list, key=key_func) + v_list = map(lambda t: sort_func(obj[t], key_func), k_list) + return dict(zip(k_list, v_list)) + + +def sort_op_data(obj): + key_func = cmp_to_key(compare_keys) + return sort_func(obj, key_func) + +def get_constraints(props): + if props is None: + return None + + # Put regexes and validators into separate dict fields. + # Since multiple constraints work like logical OR, + # it's better for the runner can evaluate regexes internally first, + # which is much faster than calling external validators. + constraints = { + 'regexes': [], + 'validators': [] + } + + constraint_elem = props.find('constraint') + + if constraint_elem is not None: + constraint_elems = list(constraint_elem) + if constraint_elems: + for ce in constraint_elems: + if ce.tag == 'regex': + constraints['regexes'].append(ce.text) + elif ce.tag == 'validator': + name = ce.get('name') + arg = ce.get('argument') + validator = {'name': name, 'argument': arg} + constraints['validators'].append(validator) + else: + print(f"Ignoring unknown validator type {ce.tag}") + + if constraints['regexes'] or constraints['validators']: + return constraints + else: + return None + +def insert_node( + n: Element, d: dict, path: list[str] = None, parent: NodeData = None, file: str = '' +) -> None: + # pylint: disable=too-many-locals,too-many-branches,too-many-statements prop: OptElement = n.find('properties') children: OptElement = n.find('children') command: OptElement = n.find('command') - # name is not None as required by schema - name: str = n.get('name', 'schema_error') + standalone: OptElement = n.find('standalone') + constraints: OptElement = get_constraints(prop) + constraint_error_message: OptElement = n.find('constraintErrorMessage') node_type: str = n.tag + + if node_type == 'virtualTagNode': + name = '__virtual_tag' + else: + name = n.get('name') + if not name: + raise ValueError( + 'Node name is required for all node types except <virtualTagNode>' + ) + if path is None: path = [] - path.append(name) + if node_type != 'virtualTagNode': + path.append(name) + if node_type == 'tagNode': path.append(f'{name}-tag_value') + if node_type == 'virtualTagNode': + path.append(f'{parent.name}-tag_value') + help_prop: OptElement = None if prop is None else prop.find('help') help_text = None if help_prop is None else help_prop.text command_text = None if command is None else command.text if command_text is not None: command_text = translate_command(command_text, path) + try: + standalone_command = translate_command(standalone.find('command').text, path) + except AttributeError: + standalone_command = None + + try: + standalone_help_text = translate_command(standalone.find('help').text, path) + except AttributeError: + standalone_help_text = None + comp_help = {} if prop is not None: che = prop.findall('completionHelp') @@ -124,53 +251,135 @@ def insert_node(n: Element, l: list[PathData], path=None) -> None: if comp_scripts: comp_help['script'] = comp_scripts - cur_node_dict = {} - cur_node_dict['name'] = name - cur_node_dict['type'] = node_type - cur_node_dict['comp_help'] = comp_help - cur_node_dict['help'] = help_text - cur_node_dict['command'] = command_text - cur_node_dict['path'] = path - cur_node_dict['children'] = [] - l.append(cur_node_dict) + new_node_data = NodeData() + new_node_data.name = name + new_node_data.node_type = node_type + new_node_data.comp_help = comp_help + new_node_data.help_text = help_text + new_node_data.command = command_text + new_node_data.standalone_help_text = standalone_help_text + new_node_data.standalone_command = standalone_command + new_node_data.constraints = constraints + new_node_data.constraint_error_message = constraint_error_message + new_node_data.path = path + new_node_data.files = [file] + + value = {('__node_data', None): new_node_data} + key = (name, node_type) + + cur_value = d.setdefault(key, value) + + if CHECK_XML_CONSISTENCY: + out = node_data_difference(get_node_data(cur_value), get_node_data(value)) + if out: + err_buf.write(out) + + # track the correct pointer reference: + cur_node_data = cur_value[('__node_data', None)] + + if file not in cur_node_data.files: + cur_node_data.files.append(file) + + if not cur_node_data.comp_help and comp_help: + cur_node_data.comp_help = comp_help + + if not cur_node_data.help_text and help_text: + cur_node_data.help_text = help_text + + if not cur_node_data.command and command_text: + cur_node_data.command = command_text + + if not cur_node_data.standalone_help_text and standalone_help_text: + cur_node_data.standalone_help_text = standalone_help_text + + if not cur_node_data.standalone_command and standalone_command: + cur_node_data.standalone_command = standalone_command + + if parent and key not in parent.children: + parent.children.append(key) if children is not None: inner_nodes = children.iterfind('*') for inner_n in inner_nodes: inner_path = path[:] - insert_node(inner_n, cur_node_dict['children'], inner_path) + insert_node(inner_n, d[key], inner_path, cur_node_data, file) -def parse_file(file_path, l): +def parse_file(file_path, d): tree = ET.parse(file_path) root = tree.getroot() + file = os.path.basename(file_path) for n in root.iterfind('*'): - insert_node(n, l) + insert_node(n, d, file=file) def main(): - parser = ArgumentParser(description='generate dict from xml defintions') + # pylint: disable=global-statement + global CHECK_XML_CONSISTENCY + + parser = ArgumentParser(description='generate dict from xml definitions') parser.add_argument( '--xml-dir', type=str, required=True, help='transcluded xml op-mode-definition file', ) + parser.add_argument( + '--check-xml-consistency', + action='store_true', + help='check consistency of node data across files', + ) + parser.add_argument( + '--select', + type=str, + help='limit cache to a subset of XML files: "power_ctl | multicast-group | ..."', + ) + + parser.add_argument( + '--export-json', + type=str, + help='Export a JSON version of the cache to a file', + ) args = vars(parser.parse_args()) + if args['check_xml_consistency']: + CHECK_XML_CONSISTENCY = True + atexit.register(write_err_buf) + xml_dir = abspath(args['xml_dir']) - l = [] + op_mode_data = {} + + select = args['select'] + if select: + select = [item.strip() for item in select.split('|')] - for fname in glob.glob(f'{xml_dir}/*.xml'): - parse_file(fname, l) + for fname in sorted(glob.glob(f'{xml_dir}/*.xml')): + file = os.path.basename(fname) + if not select or os.path.splitext(file)[0] in select: + parse_file(fname, op_mode_data) - with open(xml_op_tmp, 'w') as f: - json.dump(l, f, indent=2) + op_mode_data = sort_op_data(op_mode_data) + + res, out, err = collapse(op_mode_data) + if err: + print( + 'Failed to generate operational command definition cache due to duplicate paths.' + ) + print('Found the following duplicate paths:\n') + print(out) + sys.exit(1) + else: + op_mode_data = res with open(op_ref_cache, 'w') as f: - f.write(f'op_reference = {str(l)}') + f.write('from vyos.xml_ref.op_definition import NodeData\n') + f.write(f'op_reference = {str(op_mode_data)}') + + if args['export_json']: + with open(args['export_json'], 'w') as f: + json.dump(op_mode_data, f) if __name__ == '__main__': diff --git a/python/vyos/xml_ref/op_definition.py b/python/vyos/xml_ref/op_definition.py index 914f3a105..a172f890a 100644 --- a/python/vyos/xml_ref/op_definition.py +++ b/python/vyos/xml_ref/op_definition.py @@ -1,4 +1,4 @@ -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,37 +13,246 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see <http://www.gnu.org/licenses/>. -from typing import TypedDict from typing import TypeAlias -from typing import Optional from typing import Union +from typing import Optional +from typing import Iterator +from dataclasses import dataclass +from dataclasses import field +from dataclasses import fields +from dataclasses import asdict +from itertools import filterfalse + + +@dataclass +class NodeData: + # pylint: disable=too-many-instance-attributes + name: str = '' + node_type: str = 'node' + help_text: str = '' + comp_help: dict[str, list] = field(default_factory=dict) + command: str = '' + standalone_help_text: Optional[str] = None + standalone_command: Optional[str] = None + constraints: Optional[dict] = None + constraint_error_message: Optional[str] = None + path: list[str] = field(default_factory=list) + files: list[str] = field(default_factory=list) + children: list[tuple] = field(default_factory=list) + + +OpKey: TypeAlias = tuple[str, str] +OpData: TypeAlias = dict[OpKey, Union[NodeData, 'OpData']] + + +def key_name(k: OpKey): + return k[0] + + +def key_type(k: OpKey): + return k[1] + + +def key_names(l: list): # noqa: E741 + return list(map(lambda t: t[0], l)) + + +def keys_of_name(s: str, l: list): # noqa: E741 + filter(lambda t: t[0] == s, l) + + +def is_tag_node(t: tuple): + return t[1] == 'tagNode' + + +def subdict_of_name(s: str, d: dict) -> dict: + res = {} + for t, v in d.items(): + if not isinstance(t, tuple): + break + if key_name(t) == s: + res[t] = v + + return res + + +def next_keys(d: dict) -> list: + key_set = set() + for k in list(d.keys()): + if isinstance(d[k], dict): + key_set |= set(d[k].keys()) + return list(key_set) + + +def tuple_paths(d: dict) -> Iterator[list[tuple]]: + def func(d, path): + if isinstance(d, dict): + if not d: + yield path + for k, v in d.items(): + if isinstance(k, tuple) and key_name(k) != '__node_data': + for r in func(v, path + [k]): + yield r + else: + yield path + else: + yield path + for r in func(d, []): + yield r -class NodeData(TypedDict): - node_type: Optional[str] - help_text: Optional[str] - comp_help: Optional[dict[str, list]] - command: Optional[str] - path: Optional[list[str]] +def match_tuple_paths( + path: list[str], paths: list[list[tuple[str, str]]] +) -> list[list[tuple[str, str]]]: + return list(filter(lambda p: key_names(p) == path, paths)) -PathData: TypeAlias = dict[str, Union[NodeData|list['PathData']]] + +def get_node_data(d: dict) -> NodeData: + return d.get(('__node_data', None), {}) + + +def get_node_data_at_path(d: dict, tpath): + if not tpath: + return {} + # operates on actual paths, not names: + if not isinstance(tpath[0], tuple): + raise ValueError('must be path of tuples') + while tpath and d: + d = d.get(tpath[0], {}) + tpath = tpath[1:] + + return get_node_data(d) + + +def node_data_difference(a: NodeData, b: NodeData): + out = '' + for fld in fields(NodeData): + if fld.name in ('children', 'files'): + continue + a_fld = getattr(a, fld.name) + b_fld = getattr(b, fld.name) + if a_fld != b_fld: + out += f'prev: {a.files[-1:]} {a.path} {fld.name}: {a_fld}\n' + out += f'new: {b.files[-1:]} {b.path} {fld.name}: {b_fld}\n' + out += '\n' + + return out + + +def collapse(d: OpData, acc: dict = None) -> tuple[dict, str, bool]: + err = False + inner_err = False + out = '' + inner_out = '' + if acc is None: + acc = {} + if not isinstance(d, dict): + return d + for k, v in d.items(): + if isinstance(k, tuple): + name = key_name(k) + if name != '__node_data': + new_data = get_node_data(v) + if name in list(acc.keys()): + err = True + prev_data = acc[name].get('__node_data', {}) + if prev_data: + out += f'prev: {prev_data["file"]} {prev_data["path"]}\n' + else: + out += '\n' + out += f'new: {new_data.file} {new_data.path}\n\n' + else: + new_data.children = list(map(lambda t: t[0], new_data.children)) + acc[name] = {} + acc[name]['__node_data'] = asdict(new_data) + inner, o, e = collapse(v) + inner_err |= e + inner_out += o + acc[name].update(inner) + else: + name = k + acc[name] = v + + err |= inner_err + out += inner_out + + return acc, out, err class OpXml: def __init__(self): self.op_ref = {} - def define(self, op_ref: list[PathData]) -> None: + def define(self, op_ref: dict) -> None: self.op_ref = op_ref - def _get_op_ref_path(self, path: list[str]) -> list[PathData]: - def _get_path_list(path: list[str], l: list[PathData]) -> list[PathData]: - if not path: - return l - for d in l: - if path[0] in list(d): - return _get_path_list(path[1:], d[path[0]]) - return [] - l = self.op_ref - return _get_path_list(path, l) + def walk(self, func): + def walk_op_data(obj, func): + if isinstance(obj, dict): + for k, v in obj.items(): + if isinstance(k, tuple): + res = func(k, v) + yield res + yield from walk_op_data(v, func) + + return walk_op_data(self.op_ref, func) + + @staticmethod + def get_node_data_func(k, v): + if key_name(k) == '__node_data': + return v + return None + + def walk_node_data(self): + return filterfalse(lambda x: x is None, self.walk(self.get_node_data_func)) + + def lookup( + self, path: list[str], tag_values: bool = False, last_node_type: str = '' + ) -> (OpData, list[str]): + path = path[:] + + ref_path = [] + + def prune_tree(d: dict, p: list[str]): + p = p[:] + if not d or not isinstance(d, dict) or not p: + return d + op_data: dict = subdict_of_name(p[0], d) + op_keys = list(op_data.keys()) + ref_path.append(p[0]) + if len(p) < 2: + # check last node_type + if last_node_type: + keys = list(filter(lambda t: t[1] == last_node_type, op_keys)) + values = list(map(lambda t: op_data[t], keys)) + return dict(zip(keys, values)) + return op_data + + if p[1] not in key_names(next_keys(op_data)): + # check if tag_values + if tag_values: + p = p[2:] + keys = list(filter(is_tag_node, op_keys)) + values = list(map(lambda t: prune_tree(op_data[t], p), keys)) + return dict(zip(keys, values)) + return {} + + p = p[1:] + op_data = list(map(lambda t: prune_tree(op_data[t], p), op_keys)) + + return dict(zip(op_keys, op_data)) + + return prune_tree(self.op_ref, path), ref_path + + def lookup_node_data( + self, path: list[str], tag_values: bool = False, last_node_type: str = '' + ) -> list[NodeData]: + res = [] + d, ref_path = self.lookup(path, tag_values, last_node_type) + paths = list(tuple_paths(d)) + paths = match_tuple_paths(ref_path, paths) + for p in paths: + res.append(get_node_data_at_path(d, p)) + + return res diff --git a/python/vyos/xml_ref/update_cache.py b/python/vyos/xml_ref/update_cache.py index 0842bcbe9..6643f9dc4 100755 --- a/python/vyos/xml_ref/update_cache.py +++ b/python/vyos/xml_ref/update_cache.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2023 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as |