diff options
Diffstat (limited to 'python/vyos')
38 files changed, 2067 insertions, 98 deletions
diff --git a/python/vyos/accel_ppp.py b/python/vyos/accel_ppp.py index bfc8ee5a9..0af311e57 100644 --- a/python/vyos/accel_ppp.py +++ b/python/vyos/accel_ppp.py @@ -38,6 +38,9 @@ def get_server_statistics(accel_statistics, pattern, sep=':') -> dict: if key in ['starting', 'active', 'finishing']: stat_dict['sessions'][key] = value.strip() continue + if key == 'cpu': + stat_dict['cpu_load_percentage'] = int(re.sub(r'%', '', value.strip())) + continue stat_dict[key] = value.strip() return stat_dict diff --git a/python/vyos/config_mgmt.py b/python/vyos/config_mgmt.py new file mode 100644 index 000000000..fade3081c --- /dev/null +++ b/python/vyos/config_mgmt.py @@ -0,0 +1,669 @@ +# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +import os +import re +import sys +import gzip +import logging +from typing import Optional, Tuple, Union +from filecmp import cmp +from datetime import datetime +from tabulate import tabulate + +from vyos.config import Config +from vyos.configtree import ConfigTree, ConfigTreeError, show_diff +from vyos.defaults import directories +from vyos.util import is_systemd_service_active, ask_yes_no, rc_cmd + +SAVE_CONFIG = '/opt/vyatta/sbin/vyatta-save-config.pl' + +# created by vyatta-cfg-postinst +commit_post_hook_dir = '/etc/commit/post-hooks.d' + +commit_hooks = {'commit_revision': '01vyos-commit-revision', + 'commit_archive': '02vyos-commit-archive'} + +DEFAULT_TIME_MINUTES = 10 +timer_name = 'commit-confirm' + +config_file = os.path.join(directories['config'], 'config.boot') +archive_dir = os.path.join(directories['config'], 'archive') +archive_config_file = os.path.join(archive_dir, 'config.boot') +commit_log_file = os.path.join(archive_dir, 'commits') +logrotate_conf = os.path.join(archive_dir, 'lr.conf') +logrotate_state = os.path.join(archive_dir, 'lr.state') +rollback_config = os.path.join(archive_dir, 'config.boot-rollback') +prerollback_config = os.path.join(archive_dir, 'config.boot-prerollback') +tmp_log_entry = '/tmp/commit-rev-entry' + +logger = logging.getLogger('config_mgmt') +logger.setLevel(logging.INFO) +ch = logging.StreamHandler() +formatter = logging.Formatter('%(funcName)s: %(levelname)s:%(message)s') +ch.setFormatter(formatter) +logger.addHandler(ch) + +class ConfigMgmtError(Exception): + pass + +class ConfigMgmt: + def __init__(self, session_env=None, config=None): + if session_env: + self._session_env = session_env + else: + self._session_env = None + + if config is None: + config = Config() + + d = config.get_config_dict(['system', 'config-management'], + key_mangling=('-', '_'), + get_first_key=True) + + self.max_revisions = int(d.get('commit_revisions', 0)) + self.locations = d.get('commit_archive', {}).get('location', []) + self.source_address = d.get('commit_archive', + {}).get('source_address', '') + if config.exists(['system', 'host-name']): + self.hostname = config.return_value(['system', 'host-name']) + else: + self.hostname = 'vyos' + + # upload only on existence of effective values, notably, on boot. + # one still needs session self.locations (above) for setting + # post-commit hook in conf_mode script + path = ['system', 'config-management', 'commit-archive', 'location'] + if config.exists_effective(path): + self.effective_locations = config.return_effective_values(path) + else: + self.effective_locations = [] + + # a call to compare without args is edit_level aware + edit_level = os.getenv('VYATTA_EDIT_LEVEL', '') + self.edit_path = [l for l in edit_level.split('/') if l] + + self.active_config = config._running_config + self.working_config = config._session_config + + @staticmethod + def save_config(target): + cmd = f'{SAVE_CONFIG} {target}' + rc, out = rc_cmd(cmd) + if rc != 0: + logger.critical(f'save config failed: {out}') + + def _unsaved_commits(self) -> bool: + tmp_save = '/tmp/config.boot.check-save' + self.save_config(tmp_save) + ret = not cmp(tmp_save, config_file, shallow=False) + os.unlink(tmp_save) + return ret + + # Console script functions + # + def commit_confirm(self, minutes: int=DEFAULT_TIME_MINUTES, + no_prompt: bool=False) -> Tuple[str,int]: + """Commit with reboot to saved config in 'minutes' minutes if + 'confirm' call is not issued. + """ + if is_systemd_service_active(f'{timer_name}.timer'): + msg = 'Another confirm is pending' + return msg, 1 + + if self._unsaved_commits(): + W = '\nYou should save previous commits before commit-confirm !\n' + else: + W = '' + + prompt_str = f''' +commit-confirm will automatically reboot in {minutes} minutes unless changes +are confirmed.\n +Proceed ?''' + prompt_str = W + prompt_str + if not no_prompt and not ask_yes_no(prompt_str, default=True): + msg = 'commit-confirm canceled' + return msg, 1 + + action = 'sg vyattacfg "/usr/bin/config-mgmt revert"' + cmd = f'sudo systemd-run --quiet --on-active={minutes}m --unit={timer_name} {action}' + rc, out = rc_cmd(cmd) + if rc != 0: + raise ConfigMgmtError(out) + + # start notify + cmd = f'sudo -b /usr/libexec/vyos/commit-confirm-notify.py {minutes}' + os.system(cmd) + + msg = f'Initialized commit-confirm; {minutes} minutes to confirm before reboot' + return msg, 0 + + def confirm(self) -> Tuple[str,int]: + """Do not reboot to saved config following 'commit-confirm'. + Update commit log and archive. + """ + if not is_systemd_service_active(f'{timer_name}.timer'): + msg = 'No confirm pending' + return msg, 0 + + cmd = f'sudo systemctl stop --quiet {timer_name}.timer' + rc, out = rc_cmd(cmd) + if rc != 0: + raise ConfigMgmtError(out) + + # kill notify + cmd = 'sudo pkill -f commit-confirm-notify.py' + rc, out = rc_cmd(cmd) + if rc != 0: + raise ConfigMgmtError(out) + + entry = self._read_tmp_log_entry() + self._add_log_entry(**entry) + + if self._archive_active_config(): + self._update_archive() + + msg = 'Reboot timer stopped' + return msg, 0 + + def revert(self) -> Tuple[str,int]: + """Reboot to saved config, dropping commits from 'commit-confirm'. + """ + _ = self._read_tmp_log_entry() + + # archived config will be reverted on boot + rc, out = rc_cmd('sudo systemctl reboot') + if rc != 0: + raise ConfigMgmtError(out) + + return '', 0 + + def rollback(self, rev: int, no_prompt: bool=False) -> Tuple[str,int]: + """Reboot to config revision 'rev'. + """ + from shutil import copy + + msg = '' + + if not self._check_revision_number(rev): + msg = f'Invalid revision number {rev}: must be 0 < rev < {maxrev}' + return msg, 1 + + prompt_str = 'Proceed with reboot ?' + if not no_prompt and not ask_yes_no(prompt_str, default=True): + msg = 'Canceling rollback' + return msg, 0 + + rc, out = rc_cmd(f'sudo cp {archive_config_file} {prerollback_config}') + if rc != 0: + raise ConfigMgmtError(out) + + path = os.path.join(archive_dir, f'config.boot.{rev}.gz') + with gzip.open(path) as f: + config = f.read() + try: + with open(rollback_config, 'wb') as f: + f.write(config) + copy(rollback_config, config_file) + except OSError as e: + raise ConfigMgmtError from e + + rc, out = rc_cmd('sudo systemctl reboot') + if rc != 0: + raise ConfigMgmtError(out) + + return msg, 0 + + def compare(self, saved: bool=False, commands: bool=False, + rev1: Optional[int]=None, + rev2: Optional[int]=None) -> Tuple[str,int]: + """General compare function for config file revisions: + revision n vs. revision m; working version vs. active version; + or working version vs. saved version. + """ + ct1 = self.active_config + ct2 = self.working_config + msg = 'No changes between working and active configurations.\n' + if saved: + ct1 = self._get_saved_config_tree() + ct2 = self.working_config + msg = 'No changes between working and saved configurations.\n' + if rev1 is not None: + if not self._check_revision_number(rev1): + return f'Invalid revision number {rev1}', 1 + ct1 = self._get_config_tree_revision(rev1) + ct2 = self.working_config + msg = f'No changes between working and revision {rev1} configurations.\n' + if rev2 is not None: + if not self._check_revision_number(rev2): + return f'Invalid revision number {rev2}', 1 + # compare older to newer + ct2 = ct1 + ct1 = self._get_config_tree_revision(rev2) + msg = f'No changes between revisions {rev2} and {rev1} configurations.\n' + + out = '' + path = [] if commands else self.edit_path + try: + if commands: + out = show_diff(ct1, ct2, path=path, commands=True) + else: + out = show_diff(ct1, ct2, path=path) + except ConfigTreeError as e: + return e, 1 + + if out: + msg = out + + return msg, 0 + + def wrap_compare(self, options) -> Tuple[str,int]: + """Interface to vyatta-cfg-run: args collected as 'options' to parse + for compare. + """ + cmnds = False + r1 = None + r2 = None + if 'commands' in options: + cmnds=True + options.remove('commands') + for i in options: + if not i.isnumeric(): + options.remove(i) + if len(options) > 0: + r1 = int(options[0]) + if len(options) > 1: + r2 = int(options[1]) + + return self.compare(commands=cmnds, rev1=r1, rev2=r2) + + # Initialization and post-commit hooks for conf-mode + # + def initialize_revision(self): + """Initialize config archive, logrotate conf, and commit log. + """ + mask = os.umask(0o002) + os.makedirs(archive_dir, exist_ok=True) + + self._add_logrotate_conf() + + if (not os.path.exists(commit_log_file) or + self._get_number_of_revisions() == 0): + user = self._get_user() + via = 'init' + comment = '' + self._add_log_entry(user, via, comment) + # add empty init config before boot-config load for revision + # and diff consistency + if self._archive_active_config(): + self._update_archive() + + os.umask(mask) + + def commit_revision(self): + """Update commit log and rotate archived config.boot. + + commit_revision is called in post-commit-hooks, if + ['commit-archive', 'commit-revisions'] is configured. + """ + if os.getenv('IN_COMMIT_CONFIRM', ''): + self._new_log_entry(tmp_file=tmp_log_entry) + return + + self._add_log_entry() + + if self._archive_active_config(): + self._update_archive() + + def commit_archive(self): + """Upload config to remote archive. + """ + from vyos.remote import upload + + hostname = self.hostname + t = datetime.now() + timestamp = t.strftime('%Y%m%d_%H%M%S') + remote_file = f'config.boot-{hostname}.{timestamp}' + source_address = self.source_address + + for location in self.effective_locations: + upload(archive_config_file, f'{location}/{remote_file}', + source_host=source_address) + + # op-mode functions + # + def get_raw_log_data(self) -> list: + """Return list of dicts of log data: + keys: [timestamp, user, commit_via, commit_comment] + """ + log = self._get_log_entries() + res_l = [] + for line in log: + d = self._get_log_entry(line) + res_l.append(d) + + return res_l + + @staticmethod + def format_log_data(data: list) -> str: + """Return formatted log data as str. + """ + res_l = [] + for l_no, l in enumerate(data): + time_d = datetime.fromtimestamp(int(l['timestamp'])) + time_str = time_d.strftime("%Y-%m-%d %H:%M:%S") + + res_l.append([l_no, time_str, + f"by {l['user']}", f"via {l['commit_via']}"]) + + if l['commit_comment'] != 'commit': # default comment + res_l.append([None, l['commit_comment']]) + + ret = tabulate(res_l, tablefmt="plain") + return ret + + @staticmethod + def format_log_data_brief(data: list) -> str: + """Return 'brief' form of log data as str. + + Slightly compacted format used in completion help for + 'rollback'. + """ + res_l = [] + for l_no, l in enumerate(data): + time_d = datetime.fromtimestamp(int(l['timestamp'])) + time_str = time_d.strftime("%Y-%m-%d %H:%M:%S") + + res_l.append(['\t', l_no, time_str, + f"{l['user']}", f"by {l['commit_via']}"]) + + ret = tabulate(res_l, tablefmt="plain") + return ret + + def show_commit_diff(self, rev: int, rev2: Optional[int]=None, + commands: bool=False) -> str: + """Show commit diff at revision number, compared to previous + revision, or to another revision. + """ + if rev2 is None: + out, _ = self.compare(commands=commands, rev1=rev, rev2=(rev+1)) + return out + + out, _ = self.compare(commands=commands, rev1=rev, rev2=rev2) + return out + + def show_commit_file(self, rev: int) -> str: + return self._get_file_revision(rev) + + # utility functions + # + @staticmethod + def _strip_version(s): + return re.split(r'(^//)', s, maxsplit=1, flags=re.MULTILINE)[0] + + def _get_saved_config_tree(self): + with open(config_file) as f: + c = self._strip_version(f.read()) + return ConfigTree(c) + + def _get_file_revision(self, rev: int): + if rev not in range(0, self._get_number_of_revisions()): + raise ConfigMgmtError('revision not available') + revision = os.path.join(archive_dir, f'config.boot.{rev}.gz') + with gzip.open(revision) as f: + r = f.read().decode() + return r + + def _get_config_tree_revision(self, rev: int): + c = self._strip_version(self._get_file_revision(rev)) + return ConfigTree(c) + + def _add_logrotate_conf(self): + conf = f"""{archive_config_file} {{ + su root vyattacfg + rotate {self.max_revisions} + start 0 + compress + copy +}}""" + mask = os.umask(0o133) + + with open(logrotate_conf, 'w') as f: + f.write(conf) + + os.umask(mask) + + def _archive_active_config(self) -> bool: + mask = os.umask(0o113) + + ext = os.getpid() + tmp_save = f'/tmp/config.boot.{ext}' + self.save_config(tmp_save) + + try: + if cmp(tmp_save, archive_config_file, shallow=False): + # this will be the case on boot, as well as certain + # re-initialiation instances after delete/set + os.unlink(tmp_save) + return False + except FileNotFoundError: + pass + + rc, out = rc_cmd(f'sudo mv {tmp_save} {archive_config_file}') + os.umask(mask) + + if rc != 0: + logger.critical(f'mv file to archive failed: {out}') + return False + + return True + + @staticmethod + def _update_archive(): + cmd = f"sudo logrotate -f -s {logrotate_state} {logrotate_conf}" + rc, out = rc_cmd(cmd) + if rc != 0: + logger.critical(f'logrotate failure: {out}') + + @staticmethod + def _get_log_entries() -> list: + """Return lines of commit log as list of strings + """ + entries = [] + if os.path.exists(commit_log_file): + with open(commit_log_file) as f: + entries = f.readlines() + + return entries + + def _get_number_of_revisions(self) -> int: + l = self._get_log_entries() + return len(l) + + def _check_revision_number(self, rev: int) -> bool: + # exclude init revision: + maxrev = self._get_number_of_revisions() + if not 0 <= rev < maxrev - 1: + return False + return True + + @staticmethod + def _get_user() -> str: + import pwd + + try: + user = os.getlogin() + except OSError: + try: + user = pwd.getpwuid(os.geteuid())[0] + except KeyError: + user = 'unknown' + return user + + def _new_log_entry(self, user: str='', commit_via: str='', + commit_comment: str='', timestamp: Optional[int]=None, + tmp_file: str=None) -> Optional[str]: + # Format log entry and return str or write to file. + # + # Usage is within a post-commit hook, using env values. In case of + # commit-confirm, it can be written to a temporary file for + # inclusion on 'confirm'. + from time import time + + if timestamp is None: + timestamp = int(time()) + + if not user: + user = self._get_user() + if not commit_via: + commit_via = os.getenv('COMMIT_VIA', 'other') + if not commit_comment: + commit_comment = os.getenv('COMMIT_COMMENT', 'commit') + + # the commit log reserves '|' as field demarcation, so replace in + # comment if present; undo this in _get_log_entry, below + if re.search(r'\|', commit_comment): + commit_comment = commit_comment.replace('|', '%%') + + entry = f'|{timestamp}|{user}|{commit_via}|{commit_comment}|\n' + + mask = os.umask(0o113) + if tmp_file is not None: + try: + with open(tmp_file, 'w') as f: + f.write(entry) + except OSError as e: + logger.critical(f'write to {tmp_file} failed: {e}') + os.umask(mask) + return None + + os.umask(mask) + return entry + + @staticmethod + def _get_log_entry(line: str) -> dict: + log_fmt = re.compile(r'\|.*\|\n?$') + keys = ['user', 'commit_via', 'commit_comment', 'timestamp'] + if not log_fmt.match(line): + logger.critical(f'Invalid log format {line}') + return {} + + timestamp, user, commit_via, commit_comment = ( + tuple(line.strip().strip('|').split('|'))) + + commit_comment = commit_comment.replace('%%', '|') + d = dict(zip(keys, [user, commit_via, + commit_comment, timestamp])) + + return d + + def _read_tmp_log_entry(self) -> dict: + try: + with open(tmp_log_entry) as f: + entry = f.read() + os.unlink(tmp_log_entry) + except OSError as e: + logger.critical(f'error on file {tmp_log_entry}: {e}') + + return self._get_log_entry(entry) + + def _add_log_entry(self, user: str='', commit_via: str='', + commit_comment: str='', timestamp: Optional[int]=None): + mask = os.umask(0o113) + + entry = self._new_log_entry(user=user, commit_via=commit_via, + commit_comment=commit_comment, + timestamp=timestamp) + + log_entries = self._get_log_entries() + log_entries.insert(0, entry) + if len(log_entries) > self.max_revisions: + log_entries = log_entries[:-1] + + try: + with open(commit_log_file, 'w') as f: + f.writelines(log_entries) + except OSError as e: + logger.critical(e) + + os.umask(mask) + +# entry_point for console script +# +def run(): + from argparse import ArgumentParser, REMAINDER + + config_mgmt = ConfigMgmt() + + for s in list(commit_hooks): + if sys.argv[0].replace('-', '_').endswith(s): + func = getattr(config_mgmt, s) + try: + func() + except Exception as e: + print(f'{s}: {e}') + sys.exit(0) + + parser = ArgumentParser() + subparsers = parser.add_subparsers(dest='subcommand') + + commit_confirm = subparsers.add_parser('commit_confirm', + help="Commit with opt-out reboot to saved config") + commit_confirm.add_argument('-t', dest='minutes', type=int, + default=DEFAULT_TIME_MINUTES, + help="Minutes until reboot, unless 'confirm'") + commit_confirm.add_argument('-y', dest='no_prompt', action='store_true', + help="Execute without prompt") + + subparsers.add_parser('confirm', help="Confirm commit") + subparsers.add_parser('revert', help="Revert commit-confirm") + + rollback = subparsers.add_parser('rollback', + help="Rollback to earlier config") + rollback.add_argument('--rev', type=int, + help="Revision number for rollback") + rollback.add_argument('-y', dest='no_prompt', action='store_true', + help="Excute without prompt") + + compare = subparsers.add_parser('compare', + help="Compare config files") + + compare.add_argument('--saved', action='store_true', + help="Compare session config with saved config") + compare.add_argument('--commands', action='store_true', + help="Show difference between commands") + compare.add_argument('--rev1', type=int, default=None, + help="Compare revision with session config or other revision") + compare.add_argument('--rev2', type=int, default=None, + help="Compare revisions") + + wrap_compare = subparsers.add_parser('wrap_compare', + help="Wrapper interface for vyatta-cfg-run") + wrap_compare.add_argument('--options', nargs=REMAINDER) + + args = vars(parser.parse_args()) + + func = getattr(config_mgmt, args['subcommand']) + del args['subcommand'] + + res = '' + try: + res, rc = func(**args) + except ConfigMgmtError as e: + print(e) + sys.exit(1) + if res: + print(res) + sys.exit(rc) diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py index 53decfbf5..6ab5c252c 100644 --- a/python/vyos/configdict.py +++ b/python/vyos/configdict.py @@ -333,8 +333,9 @@ def get_dhcp_interfaces(conf, vrf=None): if dict_search('dhcp_options.default_route_distance', config) != None: options.update({'dhcp_options' : config['dhcp_options']}) if 'vrf' in config: - if vrf is config['vrf']: tmp.update({ifname : options}) - else: tmp.update({ifname : options}) + if vrf == config['vrf']: tmp.update({ifname : options}) + else: + if vrf is None: tmp.update({ifname : options}) return tmp @@ -382,8 +383,9 @@ def get_pppoe_interfaces(conf, vrf=None): if 'no_default_route' in ifconfig: options.update({'no_default_route' : {}}) if 'vrf' in ifconfig: - if vrf is ifconfig['vrf']: pppoe_interfaces.update({ifname : options}) - else: pppoe_interfaces.update({ifname : options}) + if vrf == ifconfig['vrf']: pppoe_interfaces.update({ifname : options}) + else: + if vrf is None: pppoe_interfaces.update({ifname : options}) return pppoe_interfaces @@ -427,6 +429,10 @@ def get_interface_dict(config, base, ifname=''): # Add interface instance name into dictionary dict.update({'ifname': ifname}) + # Check if QoS policy applied on this interface - See ifconfig.interface.set_mirror_redirect() + if config.exists(['qos', 'interface', ifname]): + dict.update({'traffic_policy': {}}) + # XXX: T2665: When there is no DHCPv6-PD configuration given, we can safely # remove the default values from the dict. if 'dhcpv6_options' not in dict: @@ -498,6 +504,9 @@ def get_interface_dict(config, base, ifname=''): # Add subinterface name to dictionary dict['vif'][vif].update({'ifname' : f'{ifname}.{vif}'}) + if config.exists(['qos', 'interface', f'{ifname}.{vif}']): + dict['vif'][vif].update({'traffic_policy': {}}) + default_vif_values = defaults(base + ['vif']) # XXX: T2665: When there is no DHCPv6-PD configuration given, we can safely # remove the default values from the dict. @@ -532,6 +541,9 @@ def get_interface_dict(config, base, ifname=''): # Add subinterface name to dictionary dict['vif_s'][vif_s].update({'ifname' : f'{ifname}.{vif_s}'}) + if config.exists(['qos', 'interface', f'{ifname}.{vif_s}']): + dict['vif_s'][vif_s].update({'traffic_policy': {}}) + default_vif_s_values = defaults(base + ['vif-s']) # XXX: T2665: we only wan't the vif-s defaults - do not care about vif-c if 'vif_c' in default_vif_s_values: del default_vif_s_values['vif_c'] @@ -571,6 +583,9 @@ def get_interface_dict(config, base, ifname=''): # Add subinterface name to dictionary dict['vif_s'][vif_s]['vif_c'][vif_c].update({'ifname' : f'{ifname}.{vif_s}.{vif_c}'}) + if config.exists(['qos', 'interface', f'{ifname}.{vif_s}.{vif_c}']): + dict['vif_s'][vif_s]['vif_c'][vif_c].update({'traffic_policy': {}}) + default_vif_c_values = defaults(base + ['vif-s', 'vif-c']) # XXX: T2665: When there is no DHCPv6-PD configuration given, we can safely diff --git a/python/vyos/configdiff.py b/python/vyos/configdiff.py index 9185575df..ac86af09c 100644 --- a/python/vyos/configdiff.py +++ b/python/vyos/configdiff.py @@ -78,23 +78,34 @@ def get_config_diff(config, key_mangling=None): isinstance(key_mangling[1], str)): raise ValueError("key_mangling must be a tuple of two strings") - diff_t = DiffTree(config._running_config, config._session_config) + if hasattr(config, 'cached_diff_tree'): + diff_t = getattr(config, 'cached_diff_tree') + else: + diff_t = DiffTree(config._running_config, config._session_config) + setattr(config, 'cached_diff_tree', diff_t) - return ConfigDiff(config, key_mangling, diff_tree=diff_t) + if hasattr(config, 'cached_diff_dict'): + diff_d = getattr(config, 'cached_diff_dict') + else: + diff_d = diff_t.dict + setattr(config, 'cached_diff_dict', diff_d) + + return ConfigDiff(config, key_mangling, diff_tree=diff_t, + diff_dict=diff_d) class ConfigDiff(object): """ The class of config changes as represented by comparison between the session config dict and the effective config dict. """ - def __init__(self, config, key_mangling=None, diff_tree=None): + def __init__(self, config, key_mangling=None, diff_tree=None, diff_dict=None): self._level = config.get_level() self._session_config_dict = config.get_cached_root_dict(effective=False) self._effective_config_dict = config.get_cached_root_dict(effective=True) self._key_mangling = key_mangling self._diff_tree = diff_tree - self._diff_dict = diff_tree.dict if diff_tree else {} + self._diff_dict = diff_dict # mirrored from Config; allow path arguments relative to level def _make_path(self, path): @@ -209,9 +220,9 @@ class ConfigDiff(object): if self._diff_tree is None: raise NotImplementedError("diff_tree class not available") else: - add = get_sub_dict(self._diff_tree.dict, ['add'], get_first_key=True) - sub = get_sub_dict(self._diff_tree.dict, ['sub'], get_first_key=True) - inter = get_sub_dict(self._diff_tree.dict, ['inter'], get_first_key=True) + add = get_sub_dict(self._diff_dict, ['add'], get_first_key=True) + sub = get_sub_dict(self._diff_dict, ['sub'], get_first_key=True) + inter = get_sub_dict(self._diff_dict, ['inter'], get_first_key=True) ret = {} ret[enum_to_key(Diff.MERGE)] = session_dict ret[enum_to_key(Diff.DELETE)] = get_sub_dict(sub, self._make_path(path), @@ -284,9 +295,9 @@ class ConfigDiff(object): if self._diff_tree is None: raise NotImplementedError("diff_tree class not available") else: - add = get_sub_dict(self._diff_tree.dict, ['add'], get_first_key=True) - sub = get_sub_dict(self._diff_tree.dict, ['sub'], get_first_key=True) - inter = get_sub_dict(self._diff_tree.dict, ['inter'], get_first_key=True) + add = get_sub_dict(self._diff_dict, ['add'], get_first_key=True) + sub = get_sub_dict(self._diff_dict, ['sub'], get_first_key=True) + inter = get_sub_dict(self._diff_dict, ['inter'], get_first_key=True) ret = {} ret[enum_to_key(Diff.MERGE)] = session_dict ret[enum_to_key(Diff.DELETE)] = get_sub_dict(sub, self._make_path(path)) diff --git a/python/vyos/configquery.py b/python/vyos/configquery.py index 5b097b312..85fef8777 100644 --- a/python/vyos/configquery.py +++ b/python/vyos/configquery.py @@ -88,7 +88,7 @@ class ConfigTreeQuery(GenericConfigQuery): with open(config_file) as f: config_string = f.read() except OSError as err: - raise ConfigQueryError('No config file available') from err + config_string = '' config_source = ConfigSourceString(running_config_text=config_string, session_config_text=config_string) diff --git a/python/vyos/configsession.py b/python/vyos/configsession.py index 3a60f6d92..df44fd8d6 100644 --- a/python/vyos/configsession.py +++ b/python/vyos/configsession.py @@ -34,6 +34,8 @@ REMOVE_IMAGE = ['/opt/vyatta/bin/vyatta-boot-image.pl', '--del'] GENERATE = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'generate'] SHOW = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'show'] RESET = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'reset'] +OP_CMD_ADD = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'add'] +OP_CMD_DELETE = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'delete'] # Default "commit via" string APP = "vyos-http-api" @@ -204,3 +206,15 @@ class ConfigSession(object): def reset(self, path): out = self.__run_command(RESET + path) return out + + def add_container_image(self, name): + out = self.__run_command(OP_CMD_ADD + ['container', 'image'] + [name]) + return out + + def delete_container_image(self, name): + out = self.__run_command(OP_CMD_DELETE + ['container', 'image'] + [name]) + return out + + def show_container_image(self): + out = self.__run_command(SHOW + ['container', 'image']) + return out diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index b88615513..9308bdde4 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -16,7 +16,7 @@ import os import re import json -from ctypes import cdll, c_char_p, c_void_p, c_int +from ctypes import cdll, c_char_p, c_void_p, c_int, c_bool LIBPATH = '/usr/lib/libvyosconfig.so.0' @@ -60,7 +60,7 @@ class ConfigTree(object): self.__get_error.restype = c_char_p self.__to_string = self.__lib.to_string - self.__to_string.argtypes = [c_void_p] + self.__to_string.argtypes = [c_void_p, c_bool] self.__to_string.restype = c_char_p self.__to_commands = self.__lib.to_commands @@ -160,8 +160,8 @@ class ConfigTree(object): def _get_config(self): return self.__config - def to_string(self): - config_string = self.__to_string(self.__config).decode() + def to_string(self, ordered_values=False): + config_string = self.__to_string(self.__config, ordered_values).decode() config_string = "{0}\n{1}".format(config_string, self.__version) return config_string @@ -242,7 +242,8 @@ class ConfigTree(object): raise ConfigTreeError() res = self.__copy(self.__config, oldpath_str, newpath_str) if (res != 0): - raise ConfigTreeError("Path [{}] doesn't exist".format(old_path)) + msg = self.__get_error().decode() + raise ConfigTreeError(msg) if self.__migration: print(f"- op: copy old_path: {old_path} new_path: {new_path}") @@ -321,6 +322,57 @@ class ConfigTree(object): subt = ConfigTree(address=res) return subt +def show_diff(left, right, path=[], commands=False, libpath=LIBPATH): + if left is None: + left = ConfigTree(config_string='\n') + if right is None: + right = ConfigTree(config_string='\n') + if not (isinstance(left, ConfigTree) and isinstance(right, ConfigTree)): + raise TypeError("Arguments must be instances of ConfigTree") + if path: + if (not left.exists(path)) and (not right.exists(path)): + raise ConfigTreeError(f"Path {path} doesn't exist") + + check_path(path) + path_str = " ".join(map(str, path)).encode() + + __lib = cdll.LoadLibrary(libpath) + __show_diff = __lib.show_diff + __show_diff.argtypes = [c_bool, c_char_p, c_void_p, c_void_p] + __show_diff.restype = c_char_p + __get_error = __lib.get_error + __get_error.argtypes = [] + __get_error.restype = c_char_p + + res = __show_diff(commands, path_str, left._get_config(), right._get_config()) + res = res.decode() + if res == "#1@": + msg = __get_error().decode() + raise ConfigTreeError(msg) + + return res + +def union(left, right, libpath=LIBPATH): + if left is None: + left = ConfigTree(config_string='\n') + if right is None: + right = ConfigTree(config_string='\n') + if not (isinstance(left, ConfigTree) and isinstance(right, ConfigTree)): + raise TypeError("Arguments must be instances of ConfigTree") + + __lib = cdll.LoadLibrary(libpath) + __tree_union = __lib.tree_union + __tree_union.argtypes = [c_void_p, c_void_p] + __tree_union.restype = c_void_p + __get_error = __lib.get_error + __get_error.argtypes = [] + __get_error.restype = c_char_p + + res = __tree_union( left._get_config(), right._get_config()) + tree = ConfigTree(address=res) + + return tree + class DiffTree: def __init__(self, left, right, path=[], libpath=LIBPATH): if left is None: diff --git a/python/vyos/configverify.py b/python/vyos/configverify.py index 8e0ce701e..8fddd91d0 100644 --- a/python/vyos/configverify.py +++ b/python/vyos/configverify.py @@ -1,4 +1,4 @@ -# Copyright 2020-2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2020-2023 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -23,6 +23,7 @@ from vyos import ConfigError from vyos.util import dict_search +from vyos.util import dict_search_recursive def verify_mtu(config): """ @@ -35,8 +36,14 @@ def verify_mtu(config): mtu = int(config['mtu']) tmp = Interface(config['ifname']) - min_mtu = tmp.get_min_mtu() - max_mtu = tmp.get_max_mtu() + # Not all interfaces support min/max MTU + # https://vyos.dev/T5011 + try: + min_mtu = tmp.get_min_mtu() + max_mtu = tmp.get_max_mtu() + except: # Fallback to defaults + min_mtu = 68 + max_mtu = 9000 if mtu < min_mtu: raise ConfigError(f'Interface MTU too low, ' \ @@ -232,7 +239,7 @@ def verify_authentication(config): """ if 'authentication' not in config: return - if not {'user', 'password'} <= set(config['authentication']): + if not {'username', 'password'} <= set(config['authentication']): raise ConfigError('Authentication requires both username and ' \ 'password to be set!') @@ -414,7 +421,18 @@ def verify_accel_ppp_base_service(config, local_users=True): if 'key' not in radius_config: raise ConfigError(f'Missing RADIUS secret key for server "{server}"') - if 'gateway_address' not in config: + # Check global gateway or gateway in named pool + gateway = False + if 'gateway_address' in config: + gateway = True + else: + if 'client_ip_pool' in config: + if dict_search_recursive(config, 'gateway_address', ['client_ip_pool', 'name']): + for _, v in config['client_ip_pool']['name'].items(): + if 'gateway_address' in v: + gateway = True + break + if not gateway: raise ConfigError('Server requires gateway-address to be configured!') if 'name_server_ipv4' in config: diff --git a/python/vyos/cpu.py b/python/vyos/cpu.py index 488ae79fb..d2e5f6504 100644 --- a/python/vyos/cpu.py +++ b/python/vyos/cpu.py @@ -73,7 +73,7 @@ def _find_physical_cpus(): # On other architectures, e.g. on ARM, there's no such field. # We just assume they are different CPUs, # whether single core ones or cores of physical CPUs. - phys_cpus[num] = cpu[num] + phys_cpus[num] = cpus[num] return phys_cpus diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index 7de458960..d4ffc249e 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -1,4 +1,4 @@ -# Copyright 2018 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2018-2023 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -15,19 +15,24 @@ import os +base_dir = '/usr/libexec/vyos/' + directories = { - "data": "/usr/share/vyos/", - "conf_mode": "/usr/libexec/vyos/conf_mode", - "op_mode": "/usr/libexec/vyos/op_mode", - "config": "/opt/vyatta/etc/config", - "current": "/opt/vyatta/etc/config-migrate/current", - "migrate": "/opt/vyatta/etc/config-migrate/migrate", - "log": "/var/log/vyatta", - "templates": "/usr/share/vyos/templates/", - "certbot": "/config/auth/letsencrypt", - "api_schema": "/usr/libexec/vyos/services/api/graphql/graphql/schema/", - "api_templates": "/usr/libexec/vyos/services/api/graphql/session/templates/", - "vyos_udev_dir": "/run/udev/vyos" + 'base' : base_dir, + 'data' : '/usr/share/vyos/', + 'conf_mode' : f'{base_dir}/conf_mode', + 'op_mode' : f'{base_dir}/op_mode', + 'services' : f'{base_dir}/services', + 'config' : '/opt/vyatta/etc/config', + 'current' : '/opt/vyatta/etc/config-migrate/current', + 'migrate' : '/opt/vyatta/etc/config-migrate/migrate', + 'log' : '/var/log/vyatta', + 'templates' : '/usr/share/vyos/templates/', + 'certbot' : '/config/auth/letsencrypt', + 'api_schema': f'{base_dir}/services/api/graphql/graphql/schema/', + 'api_client_op': f'{base_dir}/services/api/graphql/graphql/client_op/', + 'api_templates': f'{base_dir}/services/api/graphql/session/templates/', + 'vyos_udev_dir' : '/run/udev/vyos' } config_status = '/tmp/vyos-config-status' @@ -50,12 +55,12 @@ api_data = { 'socket' : False, 'strict' : False, 'debug' : False, - 'api_keys' : [ {"id": "testapp", "key": "qwerty"} ] + 'api_keys' : [ {'id' : 'testapp', 'key' : 'qwerty'} ] } vyos_cert_data = { - "conf": "/etc/nginx/snippets/vyos-cert.conf", - "crt": "/etc/ssl/certs/vyos-selfsigned.crt", - "key": "/etc/ssl/private/vyos-selfsign", - "lifetime": "365", + 'conf' : '/etc/nginx/snippets/vyos-cert.conf', + 'crt' : '/etc/ssl/certs/vyos-selfsigned.crt', + 'key' : '/etc/ssl/private/vyos-selfsign', + 'lifetime' : '365', } diff --git a/python/vyos/ethtool.py b/python/vyos/ethtool.py index 2b6012a73..bc3402059 100644 --- a/python/vyos/ethtool.py +++ b/python/vyos/ethtool.py @@ -1,4 +1,4 @@ -# Copyright 2021-2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2021-2023 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -21,7 +21,7 @@ from vyos.util import popen # These drivers do not support using ethtool to change the speed, duplex, or # flow control settings _drivers_without_speed_duplex_flow = ['vmxnet3', 'virtio_net', 'xen_netfront', - 'iavf', 'ice', 'i40e', 'hv_netvsc'] + 'iavf', 'ice', 'i40e', 'hv_netvsc', 'veth'] class Ethtool: """ @@ -56,10 +56,10 @@ class Ethtool: def __init__(self, ifname): # Get driver used for interface - sysfs_file = f'/sys/class/net/{ifname}/device/driver/module' - if os.path.exists(sysfs_file): - link = os.readlink(sysfs_file) - self._driver_name = os.path.basename(link) + out, err = popen(f'ethtool --driver {ifname}') + driver = re.search(r'driver:\s(\w+)', out) + if driver: + self._driver_name = driver.group(1) # Build a dictinary of supported link-speed and dupley settings. out, err = popen(f'ethtool {ifname}') diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index b4b9e67bb..919032a41 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -223,10 +223,23 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name): action = rule_conf['action'] if 'action' in rule_conf else 'accept' output.append(f'log prefix "[{fw_name[:19]}-{rule_id}-{action[:1].upper()}]"') - if 'log_level' in rule_conf: - log_level = rule_conf['log_level'] - output.append(f'level {log_level}') + if 'log_options' in rule_conf: + if 'level' in rule_conf['log_options']: + log_level = rule_conf['log_options']['level'] + output.append(f'log level {log_level}') + + if 'group' in rule_conf['log_options']: + log_group = rule_conf['log_options']['group'] + output.append(f'log group {log_group}') + + if 'queue_threshold' in rule_conf['log_options']: + queue_threshold = rule_conf['log_options']['queue_threshold'] + output.append(f'queue-threshold {queue_threshold}') + + if 'snapshot_length' in rule_conf['log_options']: + log_snaplen = rule_conf['log_options']['snapshot_length'] + output.append(f'snaplen {log_snaplen}') if 'hop_limit' in rule_conf: operators = {'eq': '==', 'gt': '>', 'lt': '<'} @@ -277,6 +290,9 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name): negated_lengths_str = ','.join(rule_conf['packet_length_exclude']) output.append(f'ip{def_suffix} length != {{{negated_lengths_str}}}') + if 'packet_type' in rule_conf: + output.append(f'pkttype ' + rule_conf['packet_type']) + if 'dscp' in rule_conf: dscp_str = ','.join(rule_conf['dscp']) output.append(f'ip{def_suffix} dscp {{{dscp_str}}}') @@ -337,6 +353,15 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name): target = rule_conf['jump_target'] output.append(f'NAME{def_suffix}_{target}') + if 'queue' in rule_conf['action']: + if 'queue' in rule_conf: + target = rule_conf['queue'] + output.append(f'num {target}') + + if 'queue_options' in rule_conf: + queue_opts = ','.join(rule_conf['queue_options']) + output.append(f'{queue_opts}') + else: output.append('return') diff --git a/python/vyos/frr.py b/python/vyos/frr.py index ccb132dd5..a84f183ef 100644 --- a/python/vyos/frr.py +++ b/python/vyos/frr.py @@ -85,7 +85,7 @@ LOG.addHandler(ch2) _frr_daemons = ['zebra', 'bgpd', 'fabricd', 'isisd', 'ospf6d', 'ospfd', 'pbrd', 'pimd', 'ripd', 'ripngd', 'sharpd', 'staticd', 'vrrpd', 'ldpd', - 'bfdd', 'eigrpd'] + 'bfdd', 'eigrpd', 'babeld'] path_vtysh = '/usr/bin/vtysh' path_frr_reload = '/usr/lib/frr/frr-reload.py' diff --git a/python/vyos/ifconfig/ethernet.py b/python/vyos/ifconfig/ethernet.py index 519cfc58c..5080144ff 100644 --- a/python/vyos/ifconfig/ethernet.py +++ b/python/vyos/ifconfig/ethernet.py @@ -239,7 +239,7 @@ class EthernetIf(Interface): if not isinstance(state, bool): raise ValueError('Value out of range') - rps_cpus = '0' + rps_cpus = 0 queues = len(glob(f'/sys/class/net/{self.ifname}/queues/rx-*')) if state: # Enable RPS on all available CPUs except CPU0 which we will not @@ -248,10 +248,16 @@ class EthernetIf(Interface): # representation of the CPUs which should participate on RPS, we # can enable more CPUs that are physically present on the system, # Linux will clip that internally! - rps_cpus = 'ffffffff,ffffffff,ffffffff,fffffffe' + rps_cpus = (1 << os.cpu_count()) -1 + + # XXX: we should probably reserve one core when the system is under + # high preasure so we can still have a core left for housekeeping. + # This is done by masking out the lowst bit so CPU0 is spared from + # receive packet steering. + rps_cpus &= ~1 for i in range(0, queues): - self._write_sysfs(f'/sys/class/net/{self.ifname}/queues/rx-{i}/rps_cpus', rps_cpus) + self._write_sysfs(f'/sys/class/net/{self.ifname}/queues/rx-{i}/rps_cpus', f'{rps_cpus:x}') # send bitmask representation as hex string without leading '0x' return True diff --git a/python/vyos/ifconfig/input.py b/python/vyos/ifconfig/input.py index db7d2b6b4..3e5f5790d 100644 --- a/python/vyos/ifconfig/input.py +++ b/python/vyos/ifconfig/input.py @@ -1,4 +1,4 @@ -# Copyright 2020 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -17,6 +17,16 @@ from vyos.ifconfig.interface import Interface @Interface.register class InputIf(Interface): + """ + The Intermediate Functional Block (ifb) pseudo network interface acts as a + QoS concentrator for multiple different sources of traffic. Packets from + or to other interfaces have to be redirected to it using the mirred action + in order to be handled, regularly routed traffic will be dropped. This way, + a single stack of qdiscs, classes and filters can be shared between + multiple interfaces. + """ + + iftype = 'ifb' definition = { **Interface.definition, **{ diff --git a/python/vyos/ifconfig/interface.py b/python/vyos/ifconfig/interface.py index c50ead89f..f62b9f7d2 100644 --- a/python/vyos/ifconfig/interface.py +++ b/python/vyos/ifconfig/interface.py @@ -751,8 +751,8 @@ class Interface(Control): elif all_rp_filter == 2: global_setting = 'loose' from vyos.base import Warning - Warning(f'Global source-validation is set to "{global_setting} '\ - f'this overrides per interface setting!') + Warning(f'Global source-validation is set to "{global_setting}", this '\ + f'overrides per interface setting on "{self.ifname}"!') tmp = self.get_interface('rp_filter') if int(tmp) == value: @@ -1365,7 +1365,7 @@ class Interface(Control): if not isinstance(state, bool): raise ValueError("Value out of range") - # https://phabricator.vyos.net/T3448 - there is (yet) no RPI support for XDP + # https://vyos.dev/T3448 - there is (yet) no RPI support for XDP if not os.path.exists('/usr/sbin/xdp_loader'): return @@ -1709,6 +1709,14 @@ class VLANIf(Interface): if self.exists(f'{self.ifname}'): return + # If source_interface or vlan_id was not explicitly defined (e.g. when + # calling VLANIf('eth0.1').remove() we can define source_interface and + # vlan_id here, as it's quiet obvious that it would be eth0 in that case. + if 'source_interface' not in self.config: + self.config['source_interface'] = '.'.join(self.ifname.split('.')[:-1]) + if 'vlan_id' not in self.config: + self.config['vlan_id'] = self.ifname.split('.')[-1] + cmd = 'ip link add link {source_interface} name {ifname} type vlan id {vlan_id}' if 'protocol' in self.config: cmd += ' protocol {protocol}' diff --git a/python/vyos/ifconfig/loopback.py b/python/vyos/ifconfig/loopback.py index b3babfadc..e1d041839 100644 --- a/python/vyos/ifconfig/loopback.py +++ b/python/vyos/ifconfig/loopback.py @@ -46,7 +46,7 @@ class LoopbackIf(Interface): if addr in self._persistent_addresses: # Do not allow deletion of the default loopback addresses as # this will cause weird system behavior like snmp/ssh no longer - # operating as expected, see https://phabricator.vyos.net/T2034. + # operating as expected, see https://vyos.dev/T2034. continue self.del_addr(addr) diff --git a/python/vyos/ifconfig/operational.py b/python/vyos/ifconfig/operational.py index 33e8614f0..dc2742123 100644 --- a/python/vyos/ifconfig/operational.py +++ b/python/vyos/ifconfig/operational.py @@ -143,15 +143,17 @@ class Operational(Control): except IOError: return no_stats - def clear_counters(self, counters=None): - clear = self._stats_all if counters is None else [] - stats = self.load_counters() + def clear_counters(self): + stats = self.get_stats() for counter, value in stats.items(): - stats[counter] = 0 if counter in clear else value + stats[counter] = value self.save_counters(stats) def reset_counters(self): - os.remove(self.cachefile(self.ifname)) + try: + os.remove(self.cachefile(self.ifname)) + except FileNotFoundError: + pass def get_stats(self): """ return a dict() with the value for each interface counter """ diff --git a/python/vyos/ifconfig/tunnel.py b/python/vyos/ifconfig/tunnel.py index 5258a2cb1..b7bf7d982 100644 --- a/python/vyos/ifconfig/tunnel.py +++ b/python/vyos/ifconfig/tunnel.py @@ -83,11 +83,6 @@ class TunnelIf(Interface): 'convert': enable_to_on, 'shellcmd': 'ip link set dev {ifname} multicast {value}', }, - 'allmulticast': { - 'validate': lambda v: assert_list(v, ['enable', 'disable']), - 'convert': enable_to_on, - 'shellcmd': 'ip link set dev {ifname} allmulticast {value}', - }, } } @@ -162,6 +157,10 @@ class TunnelIf(Interface): """ Get a synthetic MAC address. """ return self.get_mac_synthetic() + def set_multicast(self, enable): + """ Change the MULTICAST flag on the device """ + return self.set_interface('multicast', enable) + def update(self, config): """ General helper function which works on a dictionary retrived by get_config_dict(). It's main intention is to consolidate the scattered @@ -170,5 +169,10 @@ class TunnelIf(Interface): # Adjust iproute2 tunnel parameters if necessary self._change_options() + # IP Multicast + tmp = dict_search('enable_multicast', config) + value = 'enable' if (tmp != None) else 'disable' + self.set_multicast(value) + # call base class first super().update(config) diff --git a/python/vyos/ipsec.py b/python/vyos/ipsec.py new file mode 100644 index 000000000..bb5611025 --- /dev/null +++ b/python/vyos/ipsec.py @@ -0,0 +1,179 @@ +# Copyright 2020-2023 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +#Package to communicate with Strongswan VICI + +class ViciInitiateError(Exception): + """ + VICI can't initiate a session. + """ + pass +class ViciCommandError(Exception): + """ + VICI can't execute a command by any reason. + """ + pass + +def get_vici_sas(): + from vici import Session as vici_session + + try: + session = vici_session() + except Exception: + raise ViciInitiateError("IPsec not initialized") + sas = list(session.list_sas()) + return sas + + +def get_vici_connections(): + from vici import Session as vici_session + + try: + session = vici_session() + except Exception: + raise ViciInitiateError("IPsec not initialized") + connections = list(session.list_conns()) + return connections + + +def get_vici_sas_by_name(ike_name: str, tunnel: str) -> list: + """ + Find sas by IKE_SA name and/or CHILD_SA name + and return list of OrdinaryDicts with SASs info + If tunnel is not None return value is list of OrdenaryDicts contained only + CHILD_SAs wich names equal tunnel value. + :param ike_name: IKE SA name + :type ike_name: str + :param tunnel: CHILD SA name + :type tunnel: str + :return: list of Ordinary Dicts with SASs + :rtype: list + """ + from vici import Session as vici_session + + try: + session = vici_session() + except Exception: + raise ViciInitiateError("IPsec not initialized") + vici_dict = {} + if ike_name: + vici_dict['ike'] = ike_name + if tunnel: + vici_dict['child'] = tunnel + try: + sas = list(session.list_sas(vici_dict)) + return sas + except Exception: + raise ViciCommandError(f'Failed to get SAs') + + +def terminate_vici_ikeid_list(ike_id_list: list) -> None: + """ + Terminate IKE SAs by their id that contained in the list + :param ike_id_list: list of IKE SA id + :type ike_id_list: list + """ + from vici import Session as vici_session + + try: + session = vici_session() + except Exception: + raise ViciInitiateError("IPsec not initialized") + try: + for ikeid in ike_id_list: + session_generator = session.terminate( + {'ike-id': ikeid, 'timeout': '-1'}) + # a dummy `for` loop is required because of requirements + # from vici. Without a full iteration on the output, the + # command to vici may not be executed completely + for _ in session_generator: + pass + except Exception: + raise ViciCommandError( + f'Failed to terminate SA for IKE ids {ike_id_list}') + + +def terminate_vici_by_name(ike_name: str, child_name: str) -> None: + """ + Terminate IKE SAs by name if CHILD SA name is None. + Terminate CHILD SAs by name if CHILD SA name is specified + :param ike_name: IKE SA name + :type ike_name: str + :param child_name: CHILD SA name + :type child_name: str + """ + from vici import Session as vici_session + + try: + session = vici_session() + except Exception: + raise ViciInitiateError("IPsec not initialized") + try: + vici_dict: dict= {} + if ike_name: + vici_dict['ike'] = ike_name + if child_name: + vici_dict['child'] = child_name + session_generator = session.terminate(vici_dict) + # a dummy `for` loop is required because of requirements + # from vici. Without a full iteration on the output, the + # command to vici may not be executed completely + for _ in session_generator: + pass + except Exception: + if child_name: + raise ViciCommandError( + f'Failed to terminate SA for IPSEC {child_name}') + else: + raise ViciCommandError( + f'Failed to terminate SA for IKE {ike_name}') + + +def vici_initiate(ike_sa_name: str, child_sa_name: str, src_addr: str, + dst_addr: str) -> bool: + """Initiate IKE SA connection with specific peer + + Args: + ike_sa_name (str): an IKE SA connection name + child_sa_name (str): a child SA profile name + src_addr (str): source address + dst_addr (str): remote address + + Returns: + bool: a result of initiation command + """ + from vici import Session as vici_session + + try: + session = vici_session() + except Exception: + raise ViciInitiateError("IPsec not initialized") + + try: + session_generator = session.initiate({ + 'ike': ike_sa_name, + 'child': child_sa_name, + 'timeout': '-1', + 'my-host': src_addr, + 'other-host': dst_addr + }) + # a dummy `for` loop is required because of requirements + # from vici. Without a full iteration on the output, the + # command to vici may not be executed completely + for _ in session_generator: + pass + return True + except Exception: + raise ViciCommandError(f'Failed to initiate SA for IKE {ike_sa_name}')
\ No newline at end of file diff --git a/python/vyos/nat.py b/python/vyos/nat.py index 8a311045a..53fd7fb33 100644 --- a/python/vyos/nat.py +++ b/python/vyos/nat.py @@ -47,6 +47,9 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): protocol = '{ tcp, udp }' output.append(f'meta l4proto {protocol}') + if 'packet_type' in rule_conf: + output.append(f'pkttype ' + rule_conf['packet_type']) + if 'exclude' in rule_conf: translation_str = 'return' log_suffix = '-EXCL' diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index 5ff768859..d7172a0b5 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -1,4 +1,4 @@ -# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2022-2023 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -22,6 +22,10 @@ from humps import decamelize class Error(Exception): """ Any error that makes requested operation impossible to complete for reasons unrelated to the user input or script logic. + + This is the base class, scripts should not use it directly + and should raise more specific errors instead, + whenever possible. """ pass @@ -45,6 +49,17 @@ class PermissionDenied(Error): """ pass +class InsufficientResources(Error): + """ Requested operation and its arguments are valid but the system + does not have enough resources (such as drive space or memory) + to complete it. + """ + pass + +class UnsupportedOperation(Error): + """ Requested operation is technically valid but is not implemented yet. """ + pass + class IncorrectValue(Error): """ Requested operation is valid, but an argument provided has an incorrect value, preventing successful completion. @@ -66,13 +81,13 @@ class InternalError(Error): def _is_op_mode_function_name(name): - if re.match(r"^(show|clear|reset|restart)", name): + if re.match(r"^(show|clear|reset|restart|add|delete|generate|set)", name): return True else: return False -def _is_show(name): - if re.match(r"^show", name): +def _capture_output(name): + if re.match(r"^(show|generate)", name): return True else: return False @@ -113,6 +128,25 @@ def _get_arg_type(t): else: return t +def _is_literal_type(t): + if _is_optional_type(t): + t = _get_arg_type(t) + + if typing.get_origin(t) == typing.Literal: + return True + + return False + +def _get_literal_values(t): + """ Returns the tuple of allowed values for a Literal type + """ + if not _is_literal_type(t): + return tuple() + if _is_optional_type(t): + t = _get_arg_type(t) + + return typing.get_args(t) + def _normalize_field_name(name): # Convert the name to string if it is not # (in some cases they may be numbers) @@ -179,9 +213,21 @@ def run(module): subparser.add_argument(f"--{opt}", action='store_true') else: if _is_optional_type(th): - subparser.add_argument(f"--{opt}", type=_get_arg_type(th), default=None) + if _is_literal_type(th): + subparser.add_argument(f"--{opt}", + choices=list(_get_literal_values(th)), + default=None) + else: + subparser.add_argument(f"--{opt}", + type=_get_arg_type(th), default=None) else: - subparser.add_argument(f"--{opt}", type=_get_arg_type(th), required=True) + if _is_literal_type(th): + subparser.add_argument(f"--{opt}", + choices=list(_get_literal_values(th)), + required=True) + else: + subparser.add_argument(f"--{opt}", + type=_get_arg_type(th), required=True) # Get options as a dict rather than a namespace, # so that we can modify it and pack for passing to functions @@ -199,20 +245,23 @@ def run(module): # it would cause an extra argument error when we pass the dict to a function del args["subcommand"] - # Show commands must always get the "raw" argument, - # but other commands (clear/reset/restart) should not, + # Show and generate commands must always get the "raw" argument, + # but other commands (clear/reset/restart/add/delete) should not, # because they produce no output and it makes no sense for them. - if ("raw" not in args) and _is_show(function_name): + if ("raw" not in args) and _capture_output(function_name): args["raw"] = False - if re.match(r"^show", function_name): - # Show commands are slightly special: + if _capture_output(function_name): + # Show and generate commands are slightly special: # they may return human-formatted output # or a raw dict that we need to serialize in JSON for printing res = func(**args) if not args["raw"]: return res else: + if not isinstance(res, dict) and not isinstance(res, list): + raise InternalError(f"Bare literal is not an acceptable raw output, must be a list or an object.\ + The output was:{res}") res = decamelize(res) res = _normalize_field_names(res) from json import dumps diff --git a/python/vyos/qos/__init__.py b/python/vyos/qos/__init__.py new file mode 100644 index 000000000..a2980ccde --- /dev/null +++ b/python/vyos/qos/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from vyos.qos.base import QoSBase +from vyos.qos.cake import CAKE +from vyos.qos.droptail import DropTail +from vyos.qos.fairqueue import FairQueue +from vyos.qos.fqcodel import FQCodel +from vyos.qos.limiter import Limiter +from vyos.qos.netem import NetEm +from vyos.qos.priority import Priority +from vyos.qos.randomdetect import RandomDetect +from vyos.qos.ratelimiter import RateLimiter +from vyos.qos.roundrobin import RoundRobin +from vyos.qos.trafficshaper import TrafficShaper +from vyos.qos.trafficshaper import TrafficShaperHFSC diff --git a/python/vyos/qos/base.py b/python/vyos/qos/base.py new file mode 100644 index 000000000..33bb8ae28 --- /dev/null +++ b/python/vyos/qos/base.py @@ -0,0 +1,302 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +import os + +from vyos.base import Warning +from vyos.util import cmd +from vyos.util import dict_search +from vyos.util import read_file + +class QoSBase: + _debug = False + _direction = ['egress'] + _parent = 0xffff + + def __init__(self, interface): + if os.path.exists('/tmp/vyos.qos.debug'): + self._debug = True + self._interface = interface + + def _cmd(self, command): + if self._debug: + print(f'DEBUG/QoS: {command}') + return cmd(command) + + def get_direction(self) -> list: + return self._direction + + def _get_class_max_id(self, config) -> int: + if 'class' in config: + tmp = list(config['class'].keys()) + tmp.sort(key=lambda ii: int(ii)) + return tmp[-1] + return None + + def _build_base_qdisc(self, config : dict, cls_id : int): + """ + Add/replace qdisc for every class (also default is a class). This is + a genetic method which need an implementation "per" queue-type. + + This matches the old mapping as defined in Perl here: + https://github.com/vyos/vyatta-cfg-qos/blob/equuleus/lib/Vyatta/Qos/ShaperClass.pm#L223-L229 + """ + queue_type = dict_search('queue_type', config) + default_tc = f'tc qdisc replace dev {self._interface} parent {self._parent}:{cls_id:x}' + + if queue_type == 'priority': + handle = 0x4000 + cls_id + default_tc += f' handle {handle:x}: prio' + self._cmd(default_tc) + + queue_limit = dict_search('queue_limit', config) + for ii in range(1, 4): + tmp = f'tc qdisc replace dev {self._interface} parent {handle:x}:{ii:x} pfifo limit {queue_limit}' + self._cmd(tmp) + + elif queue_type == 'fair-queue': + default_tc += f' sfq' + + tmp = dict_search('queue_limit', config) + if tmp: default_tc += f' limit {tmp}' + + self._cmd(default_tc) + + elif queue_type == 'fq-codel': + default_tc += f' fq_codel' + tmp = dict_search('codel_quantum', config) + if tmp: default_tc += f' quantum {tmp}' + + tmp = dict_search('flows', config) + if tmp: default_tc += f' flows {tmp}' + + tmp = dict_search('interval', config) + if tmp: default_tc += f' interval {tmp}' + + tmp = dict_search('interval', config) + if tmp: default_tc += f' interval {tmp}' + + tmp = dict_search('queue_limit', config) + if tmp: default_tc += f' limit {tmp}' + + tmp = dict_search('target', config) + if tmp: default_tc += f' target {tmp}' + + default_tc += f' noecn' + + self._cmd(default_tc) + + elif queue_type == 'random-detect': + default_tc += f' red' + + self._cmd(default_tc) + + elif queue_type == 'drop-tail': + default_tc += f' pfifo' + + tmp = dict_search('queue_limit', config) + if tmp: default_tc += f' limit {tmp}' + + self._cmd(default_tc) + + def _rate_convert(self, rate) -> int: + rates = { + 'bit' : 1, + 'kbit' : 1000, + 'mbit' : 1000000, + 'gbit' : 1000000000, + 'tbit' : 1000000000000, + } + + if rate == 'auto' or rate.endswith('%'): + speed = 10 + # Not all interfaces have valid entries in the speed file. PPPoE + # interfaces have the appropriate speed file, but you can not read it: + # cat: /sys/class/net/pppoe7/speed: Invalid argument + try: + speed = read_file(f'/sys/class/net/{self._interface}/speed') + if not speed.isnumeric(): + Warning('Interface speed cannot be determined (assuming 10 Mbit/s)') + if rate.endswith('%'): + percent = rate.rstrip('%') + speed = int(speed) * int(percent) // 100 + except: + pass + + return int(speed) *1000000 # convert to MBit/s + + rate_numeric = int(''.join([n for n in rate if n.isdigit()])) + rate_scale = ''.join([n for n in rate if not n.isdigit()]) + + if int(rate_numeric) <= 0: + raise ValueError(f'{rate_numeric} is not a valid bandwidth <= 0') + + if rate_scale: + return int(rate_numeric * rates[rate_scale]) + else: + # No suffix implies Kbps just as Cisco IOS + return int(rate_numeric * 1000) + + def update(self, config, direction, priority=None): + """ method must be called from derived class after it has completed qdisc setup """ + if self._debug: + import pprint + pprint.pprint(config) + + if 'class' in config: + for cls, cls_config in config['class'].items(): + self._build_base_qdisc(cls_config, int(cls)) + + # every match criteria has it's tc instance + filter_cmd = f'tc filter replace dev {self._interface} parent {self._parent:x}:' + + if priority: + filter_cmd += f' prio {cls}' + elif 'priority' in cls_config: + prio = cls_config['priority'] + filter_cmd += f' prio {prio}' + + filter_cmd += ' protocol all' + + if 'match' in cls_config: + for match, match_config in cls_config['match'].items(): + if 'mark' in match_config: + mark = match_config['mark'] + filter_cmd += f' handle {mark} fw' + + for af in ['ip', 'ipv6']: + tc_af = af + if af == 'ipv6': + tc_af = 'ip6' + + if af in match_config: + filter_cmd += ' u32' + + tmp = dict_search(f'{af}.source.address', match_config) + if tmp: filter_cmd += f' match {tc_af} src {tmp}' + + tmp = dict_search(f'{af}.source.port', match_config) + if tmp: filter_cmd += f' match {tc_af} sport {tmp} 0xffff' + + tmp = dict_search(f'{af}.destination.address', match_config) + if tmp: filter_cmd += f' match {tc_af} dst {tmp}' + + tmp = dict_search(f'{af}.destination.port', match_config) + if tmp: filter_cmd += f' match {tc_af} dport {tmp} 0xffff' + + tmp = dict_search(f'{af}.protocol', match_config) + if tmp: filter_cmd += f' match {tc_af} protocol {tmp} 0xff' + + # Will match against total length of an IPv4 packet and + # payload length of an IPv6 packet. + # + # IPv4 : match u16 0x0000 ~MAXLEN at 2 + # IPv6 : match u16 0x0000 ~MAXLEN at 4 + tmp = dict_search(f'{af}.max_length', match_config) + if tmp: + # We need the 16 bit two's complement of the maximum + # packet length + tmp = hex(0xffff & ~int(tmp)) + + if af == 'ip': + filter_cmd += f' match u16 0x0000 {tmp} at 2' + elif af == 'ipv6': + filter_cmd += f' match u16 0x0000 {tmp} at 4' + + # We match against specific TCP flags - we assume the IPv4 + # header length is 20 bytes and assume the IPv6 packet is + # not using extension headers (hence a ip header length of 40 bytes) + # TCP Flags are set on byte 13 of the TCP header. + # IPv4 : match u8 X X at 33 + # IPv6 : match u8 X X at 53 + # with X = 0x02 for SYN and X = 0x10 for ACK + tmp = dict_search(f'{af}.tcp', match_config) + if tmp: + mask = 0 + if 'ack' in tmp: + mask |= 0x10 + if 'syn' in tmp: + mask |= 0x02 + mask = hex(mask) + + if af == 'ip': + filter_cmd += f' match u8 {mask} {mask} at 33' + elif af == 'ipv6': + filter_cmd += f' match u8 {mask} {mask} at 53' + + else: + + filter_cmd += ' basic' + + # The police block allows limiting of the byte or packet rate of + # traffic matched by the filter it is attached to. + # https://man7.org/linux/man-pages/man8/tc-police.8.html + if any(tmp in ['exceed', 'bandwidth', 'burst'] for tmp in cls_config): + filter_cmd += f' action police' + + if 'exceed' in cls_config: + action = cls_config['exceed'] + filter_cmd += f' conform-exceed {action}' + if 'not_exceed' in cls_config: + action = cls_config['not_exceed'] + filter_cmd += f'/{action}' + + if 'bandwidth' in cls_config: + rate = self._rate_convert(cls_config['bandwidth']) + filter_cmd += f' rate {rate}' + + if 'burst' in cls_config: + burst = cls_config['burst'] + filter_cmd += f' burst {burst}' + + cls = int(cls) + filter_cmd += f' flowid {self._parent:x}:{cls:x}' + self._cmd(filter_cmd) + + if 'default' in config: + if 'class' in config: + class_id_max = self._get_class_max_id(config) + default_cls_id = int(class_id_max) +1 + self._build_base_qdisc(config['default'], default_cls_id) + + filter_cmd = f'tc filter replace dev {self._interface} parent {self._parent:x}: ' + filter_cmd += 'prio 255 protocol all basic' + + # The police block allows limiting of the byte or packet rate of + # traffic matched by the filter it is attached to. + # https://man7.org/linux/man-pages/man8/tc-police.8.html + if any(tmp in ['exceed', 'bandwidth', 'burst'] for tmp in config['default']): + filter_cmd += f' action police' + + if 'exceed' in config['default']: + action = config['default']['exceed'] + filter_cmd += f' conform-exceed {action}' + if 'not_exceed' in config['default']: + action = config['default']['not_exceed'] + filter_cmd += f'/{action}' + + if 'bandwidth' in config['default']: + rate = self._rate_convert(config['default']['bandwidth']) + filter_cmd += f' rate {rate}' + + if 'burst' in config['default']: + burst = config['default']['burst'] + filter_cmd += f' burst {burst}' + + if 'class' in config: + filter_cmd += f' flowid {self._parent:x}:{default_cls_id:x}' + + self._cmd(filter_cmd) + diff --git a/python/vyos/qos/cake.py b/python/vyos/qos/cake.py new file mode 100644 index 000000000..a89b1de1e --- /dev/null +++ b/python/vyos/qos/cake.py @@ -0,0 +1,55 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from vyos.qos.base import QoSBase + +class CAKE(QoSBase): + _direction = ['egress'] + + # https://man7.org/linux/man-pages/man8/tc-cake.8.html + def update(self, config, direction): + tmp = f'tc qdisc add dev {self._interface} root handle 1: cake {direction}' + if 'bandwidth' in config: + bandwidth = self._rate_convert(config['bandwidth']) + tmp += f' bandwidth {bandwidth}' + + if 'rtt' in config: + rtt = config['rtt'] + tmp += f' rtt {rtt}ms' + + if 'flow_isolation' in config: + if 'blind' in config['flow_isolation']: + tmp += f' flowblind' + if 'dst_host' in config['flow_isolation']: + tmp += f' dsthost' + if 'dual_dst_host' in config['flow_isolation']: + tmp += f' dual-dsthost' + if 'dual_src_host' in config['flow_isolation']: + tmp += f' dual-srchost' + if 'flow' in config['flow_isolation']: + tmp += f' flows' + if 'host' in config['flow_isolation']: + tmp += f' hosts' + if 'nat' in config['flow_isolation']: + tmp += f' nat' + if 'src_host' in config['flow_isolation']: + tmp += f' srchost ' + else: + tmp += f' nonat' + + self._cmd(tmp) + + # call base class + super().update(config, direction) diff --git a/python/vyos/qos/droptail.py b/python/vyos/qos/droptail.py new file mode 100644 index 000000000..427d43d19 --- /dev/null +++ b/python/vyos/qos/droptail.py @@ -0,0 +1,28 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from vyos.qos.base import QoSBase + +class DropTail(QoSBase): + # https://man7.org/linux/man-pages/man8/tc-pfifo.8.html + def update(self, config, direction): + tmp = f'tc qdisc add dev {self._interface} root pfifo' + if 'queue_limit' in config: + limit = config["queue_limit"] + tmp += f' limit {limit}' + self._cmd(tmp) + + # call base class + super().update(config, direction) diff --git a/python/vyos/qos/fairqueue.py b/python/vyos/qos/fairqueue.py new file mode 100644 index 000000000..f41d098fb --- /dev/null +++ b/python/vyos/qos/fairqueue.py @@ -0,0 +1,31 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from vyos.qos.base import QoSBase + +class FairQueue(QoSBase): + # https://man7.org/linux/man-pages/man8/tc-sfq.8.html + def update(self, config, direction): + tmp = f'tc qdisc add dev {self._interface} root sfq' + + if 'hash_interval' in config: + tmp += f' perturb {config["hash_interval"]}' + if 'queue_limit' in config: + tmp += f' limit {config["queue_limit"]}' + + self._cmd(tmp) + + # call base class + super().update(config, direction) diff --git a/python/vyos/qos/fqcodel.py b/python/vyos/qos/fqcodel.py new file mode 100644 index 000000000..cd2340aa2 --- /dev/null +++ b/python/vyos/qos/fqcodel.py @@ -0,0 +1,40 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from vyos.qos.base import QoSBase + +class FQCodel(QoSBase): + # https://man7.org/linux/man-pages/man8/tc-fq_codel.8.html + def update(self, config, direction): + tmp = f'tc qdisc add dev {self._interface} root fq_codel' + + if 'codel_quantum' in config: + tmp += f' quantum {config["codel_quantum"]}' + if 'flows' in config: + tmp += f' flows {config["flows"]}' + if 'interval' in config: + interval = int(config['interval']) * 1000 + tmp += f' interval {interval}' + if 'queue_limit' in config: + tmp += f' limit {config["queue_limit"]}' + if 'target' in config: + target = int(config['target']) * 1000 + tmp += f' target {target}' + + tmp += f' noecn' + self._cmd(tmp) + + # call base class + super().update(config, direction) diff --git a/python/vyos/qos/limiter.py b/python/vyos/qos/limiter.py new file mode 100644 index 000000000..ace0c0b6c --- /dev/null +++ b/python/vyos/qos/limiter.py @@ -0,0 +1,27 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from vyos.qos.base import QoSBase + +class Limiter(QoSBase): + _direction = ['ingress'] + + def update(self, config, direction): + tmp = f'tc qdisc add dev {self._interface} handle {self._parent:x}: {direction}' + self._cmd(tmp) + + # base class must be called last + super().update(config, direction) + diff --git a/python/vyos/qos/netem.py b/python/vyos/qos/netem.py new file mode 100644 index 000000000..8bdef300b --- /dev/null +++ b/python/vyos/qos/netem.py @@ -0,0 +1,53 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from vyos.qos.base import QoSBase + +class NetEm(QoSBase): + # https://man7.org/linux/man-pages/man8/tc-netem.8.html + def update(self, config, direction): + tmp = f'tc qdisc add dev {self._interface} root netem' + if 'bandwidth' in config: + rate = self._rate_convert(config["bandwidth"]) + tmp += f' rate {rate}' + + if 'queue_limit' in config: + limit = config["queue_limit"] + tmp += f' limit {limit}' + + if 'delay' in config: + delay = config["delay"] + tmp += f' delay {delay}ms' + + if 'loss' in config: + drop = config["loss"] + tmp += f' drop {drop}%' + + if 'corruption' in config: + corrupt = config["corruption"] + tmp += f' corrupt {corrupt}%' + + if 'reordering' in config: + reorder = config["reordering"] + tmp += f' reorder {reorder}%' + + if 'duplicate' in config: + duplicate = config["duplicate"] + tmp += f' duplicate {duplicate}%' + + self._cmd(tmp) + + # call base class + super().update(config, direction) diff --git a/python/vyos/qos/priority.py b/python/vyos/qos/priority.py new file mode 100644 index 000000000..6d4a60a43 --- /dev/null +++ b/python/vyos/qos/priority.py @@ -0,0 +1,41 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from vyos.qos.base import QoSBase +from vyos.util import dict_search + +class Priority(QoSBase): + _parent = 1 + + # https://man7.org/linux/man-pages/man8/tc-prio.8.html + def update(self, config, direction): + if 'class' in config: + class_id_max = self._get_class_max_id(config) + bands = int(class_id_max) +1 + + tmp = f'tc qdisc add dev {self._interface} root handle {self._parent:x}: prio bands {bands} priomap ' \ + f'{class_id_max} {class_id_max} {class_id_max} {class_id_max} ' \ + f'{class_id_max} {class_id_max} {class_id_max} {class_id_max} ' \ + f'{class_id_max} {class_id_max} {class_id_max} {class_id_max} ' \ + f'{class_id_max} {class_id_max} {class_id_max} {class_id_max} ' + self._cmd(tmp) + + for cls in config['class']: + cls = int(cls) + tmp = f'tc qdisc add dev {self._interface} parent {self._parent:x}:{cls:x} pfifo' + self._cmd(tmp) + + # base class must be called last + super().update(config, direction, priority=True) diff --git a/python/vyos/qos/randomdetect.py b/python/vyos/qos/randomdetect.py new file mode 100644 index 000000000..d7d84260f --- /dev/null +++ b/python/vyos/qos/randomdetect.py @@ -0,0 +1,54 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from vyos.qos.base import QoSBase + +class RandomDetect(QoSBase): + _parent = 1 + + # https://man7.org/linux/man-pages/man8/tc.8.html + def update(self, config, direction): + + tmp = f'tc qdisc add dev {self._interface} root handle {self._parent}:0 dsmark indices 8 set_tc_index' + self._cmd(tmp) + + tmp = f'tc filter add dev {self._interface} parent {self._parent}:0 protocol ip prio 1 tcindex mask 0xe0 shift 5' + self._cmd(tmp) + + # Generalized Random Early Detection + handle = self._parent +1 + tmp = f'tc qdisc add dev {self._interface} parent {self._parent}:0 handle {handle}:0 gred setup DPs 8 default 0 grio' + self._cmd(tmp) + + bandwidth = self._rate_convert(config['bandwidth']) + + # set VQ (virtual queue) parameters + for precedence, precedence_config in config['precedence'].items(): + precedence = int(precedence) + avg_pkt = int(precedence_config['average_packet']) + limit = int(precedence_config['queue_limit']) * avg_pkt + min_val = int(precedence_config['minimum_threshold']) * avg_pkt + max_val = int(precedence_config['maximum_threshold']) * avg_pkt + + tmp = f'tc qdisc change dev {self._interface} handle {handle}:0 gred limit {limit} min {min_val} max {max_val} avpkt {avg_pkt} ' + + burst = (2 * int(precedence_config['minimum_threshold']) + int(precedence_config['maximum_threshold'])) // 3 + probability = 1 / int(precedence_config['mark_probability']) + tmp += f'burst {burst} bandwidth {bandwidth} probability {probability} DP {precedence} prio {8 - precedence:x}' + + self._cmd(tmp) + + # call base class + super().update(config, direction) diff --git a/python/vyos/qos/ratelimiter.py b/python/vyos/qos/ratelimiter.py new file mode 100644 index 000000000..a4f80a1be --- /dev/null +++ b/python/vyos/qos/ratelimiter.py @@ -0,0 +1,37 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from vyos.qos.base import QoSBase + +class RateLimiter(QoSBase): + # https://man7.org/linux/man-pages/man8/tc-tbf.8.html + def update(self, config, direction): + # call base class + super().update(config, direction) + + tmp = f'tc qdisc add dev {self._interface} root tbf' + if 'bandwidth' in config: + rate = self._rate_convert(config['bandwidth']) + tmp += f' rate {rate}' + + if 'burst' in config: + burst = config['burst'] + tmp += f' burst {burst}' + + if 'latency' in config: + latency = config['latency'] + tmp += f' latency {latency}ms' + + self._cmd(tmp) diff --git a/python/vyos/qos/roundrobin.py b/python/vyos/qos/roundrobin.py new file mode 100644 index 000000000..80814ddfb --- /dev/null +++ b/python/vyos/qos/roundrobin.py @@ -0,0 +1,44 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from vyos.qos.base import QoSBase + +class RoundRobin(QoSBase): + _parent = 1 + + # https://man7.org/linux/man-pages/man8/tc-drr.8.html + def update(self, config, direction): + tmp = f'tc qdisc add dev {self._interface} root handle 1: drr' + self._cmd(tmp) + + if 'class' in config: + for cls in config['class']: + cls = int(cls) + tmp = f'tc class replace dev {self._interface} parent 1:1 classid 1:{cls:x} drr' + self._cmd(tmp) + + tmp = f'tc qdisc replace dev {self._interface} parent 1:{cls:x} pfifo' + self._cmd(tmp) + + if 'default' in config: + class_id_max = self._get_class_max_id(config) + default_cls_id = int(class_id_max) +1 + + # class ID via CLI is in range 1-4095, thus 1000 hex = 4096 + tmp = f'tc class replace dev {self._interface} parent 1:1 classid 1:{default_cls_id:x} drr' + self._cmd(tmp) + + # call base class + super().update(config, direction, priority=True) diff --git a/python/vyos/qos/trafficshaper.py b/python/vyos/qos/trafficshaper.py new file mode 100644 index 000000000..f42f4d022 --- /dev/null +++ b/python/vyos/qos/trafficshaper.py @@ -0,0 +1,106 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from math import ceil +from vyos.qos.base import QoSBase + +# Kernel limits on quantum (bytes) +MAXQUANTUM = 200000 +MINQUANTUM = 1000 + +class TrafficShaper(QoSBase): + _parent = 1 + + # https://man7.org/linux/man-pages/man8/tc-htb.8.html + def update(self, config, direction): + class_id_max = 0 + if 'class' in config: + tmp = list(config['class']) + tmp.sort() + class_id_max = tmp[-1] + + r2q = 10 + # bandwidth is a mandatory CLI node + speed = self._rate_convert(config['bandwidth']) + speed_bps = int(speed) // 8 + + # need a bigger r2q if going fast than 16 mbits/sec + if (speed_bps // r2q) >= MAXQUANTUM: # integer division + r2q = ceil(speed_bps // MAXQUANTUM) + else: + # if there is a slow class then may need smaller value + if 'class' in config: + min_speed = speed_bps + for cls, cls_options in config['class'].items(): + # find class with the lowest bandwidth used + if 'bandwidth' in cls_options: + bw_bps = int(self._rate_convert(cls_options['bandwidth'])) // 8 # bandwidth in bytes per second + if bw_bps < min_speed: + min_speed = bw_bps + + while (r2q > 1) and (min_speed // r2q) < MINQUANTUM: + tmp = r2q -1 + if (speed_bps // tmp) >= MAXQUANTUM: + break + r2q = tmp + + + default_minor_id = int(class_id_max) +1 + tmp = f'tc qdisc replace dev {self._interface} root handle {self._parent:x}: htb r2q {r2q} default {default_minor_id:x}' # default is in hex + self._cmd(tmp) + + tmp = f'tc class replace dev {self._interface} parent {self._parent:x}: classid {self._parent:x}:1 htb rate {speed}' + self._cmd(tmp) + + if 'class' in config: + for cls, cls_config in config['class'].items(): + # class id is used later on and passed as hex, thus this needs to be an int + cls = int(cls) + + # bandwidth is a mandatory CLI node + rate = self._rate_convert(cls_config['bandwidth']) + burst = cls_config['burst'] + quantum = cls_config['codel_quantum'] + + tmp = f'tc class replace dev {self._interface} parent {self._parent:x}:1 classid {self._parent:x}:{cls:x} htb rate {rate} burst {burst} quantum {quantum}' + if 'priority' in cls_config: + priority = cls_config['priority'] + tmp += f' prio {priority}' + self._cmd(tmp) + + tmp = f'tc qdisc replace dev {self._interface} parent {self._parent:x}:{cls:x} sfq' + self._cmd(tmp) + + if 'default' in config: + rate = self._rate_convert(config['default']['bandwidth']) + burst = config['default']['burst'] + quantum = config['default']['codel_quantum'] + tmp = f'tc class replace dev {self._interface} parent {self._parent:x}:1 classid {self._parent:x}:{default_minor_id:x} htb rate {rate} burst {burst} quantum {quantum}' + if 'priority' in config['default']: + priority = config['default']['priority'] + tmp += f' prio {priority}' + self._cmd(tmp) + + tmp = f'tc qdisc replace dev {self._interface} parent {self._parent:x}:{default_minor_id:x} sfq' + self._cmd(tmp) + + # call base class + super().update(config, direction) + +class TrafficShaperHFSC(TrafficShaper): + def update(self, config, direction): + # call base class + super().update(config, direction) + diff --git a/python/vyos/template.py b/python/vyos/template.py index 2a4135f9e..254a15e3a 100644 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -1,4 +1,4 @@ -# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2023 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -44,6 +44,7 @@ def _get_environment(location=None): loader=loc_loader, trim_blocks=True, undefined=ChainableUndefined, + extensions=['jinja2.ext.loopcontrols'] ) env.filters.update(_FILTERS) env.tests.update(_TESTS) @@ -158,6 +159,24 @@ def force_to_list(value): else: return [value] +@register_filter('seconds_to_human') +def seconds_to_human(seconds, separator=""): + """ Convert seconds to human-readable values like 1d6h15m23s """ + from vyos.util import seconds_to_human + return seconds_to_human(seconds, separator=separator) + +@register_filter('bytes_to_human') +def bytes_to_human(bytes, initial_exponent=0, precision=2): + """ Convert bytes to human-readable values like 1.44M """ + from vyos.util import bytes_to_human + return bytes_to_human(bytes, initial_exponent=initial_exponent, precision=precision) + +@register_filter('human_to_bytes') +def human_to_bytes(value): + """ Convert a data amount with a unit suffix to bytes, like 2K to 2048 """ + from vyos.util import human_to_bytes + return human_to_bytes(value) + @register_filter('ip_from_cidr') def ip_from_cidr(prefix): """ Take an IPv4/IPv6 CIDR host and strip cidr mask. @@ -193,6 +212,16 @@ def dot_colon_to_dash(text): text = text.replace(".", "-") return text +@register_filter('generate_uuid4') +def generate_uuid4(text): + """ Generate random unique ID + Example: + % uuid4() + UUID('958ddf6a-ef14-4e81-8cfb-afb12456d1c5') + """ + from uuid import uuid4 + return uuid4() + @register_filter('netmask_from_cidr') def netmask_from_cidr(prefix): """ Take CIDR prefix and convert the prefix length to a "subnet mask". @@ -476,6 +505,8 @@ def get_esp_ike_cipher(group_config, ike_group=None): continue tmp = '{encryption}-{hash}'.format(**proposal) + if 'prf' in proposal: + tmp += '-' + proposal['prf'] if 'dh_group' in proposal: tmp += '-' + pfs_lut[ 'dh-group' + proposal['dh_group'] ] elif 'pfs' in group_config and group_config['pfs'] != 'disable': @@ -633,7 +664,24 @@ def nat_static_rule(rule_conf, rule_id, nat_type): @register_filter('range_to_regex') def range_to_regex(num_range): + """Convert range of numbers or list of ranges + to regex + + % range_to_regex('11-12') + '(1[1-2])' + % range_to_regex(['11-12', '14-15']) + '(1[1-2]|1[4-5])' + """ from vyos.range_regex import range_to_regex + if isinstance(num_range, list): + data = [] + for entry in num_range: + if '-' not in entry: + data.append(entry) + else: + data.append(range_to_regex(entry)) + return f'({"|".join(data)})' + if '-' not in num_range: return num_range diff --git a/python/vyos/util.py b/python/vyos/util.py index 6a828c0ac..0593184cc 100644 --- a/python/vyos/util.py +++ b/python/vyos/util.py @@ -348,9 +348,11 @@ def colon_separated_to_dict(data_string, uniquekeys=False): l = l.strip() if l: match = re.match(key_value_re, l) - if match: + if match and (len(match.groups()) == 2): key = match.groups()[0].strip() value = match.groups()[1].strip() + else: + raise ValueError(f"""Line "{l}" could not be parsed a colon-separated pair """, l) if key in data.keys(): if uniquekeys: raise ValueError("Data string has duplicate keys: {0}".format(key)) @@ -486,7 +488,7 @@ def is_listen_port_bind_service(port: int, service: str) -> bool: Example: % is_listen_port_bind_service(443, 'nginx') True - % is_listen_port_bind_service(443, 'ocservr-main') + % is_listen_port_bind_service(443, 'ocserv-main') False """ from psutil import net_connections as connections @@ -872,12 +874,16 @@ def convert_data(data): Returns: str | list | dict: converted data """ + from base64 import b64encode from collections import OrderedDict if isinstance(data, str): return data if isinstance(data, bytes): - return data.decode() + try: + return data.decode() + except UnicodeDecodeError: + return b64encode(data).decode() if isinstance(data, list): list_tmp = [] for item in data: diff --git a/python/vyos/xml/load.py b/python/vyos/xml/load.py index c3022f3d6..f842ff9ce 100644 --- a/python/vyos/xml/load.py +++ b/python/vyos/xml/load.py @@ -71,16 +71,12 @@ def _merge(dict1, dict2): continue if isinstance(dict1[k], dict) and isinstance(dict2[k], dict): dict1[k] = _merge(dict1[k], dict2[k]) - elif isinstance(dict1[k], dict) and isinstance(dict2[k], dict): + elif isinstance(dict1[k], list) and isinstance(dict2[k], list): dict1[k].extend(dict2[k]) elif dict1[k] == dict2[k]: - # A definition shared between multiple files - if k in (kw.valueless, kw.multi, kw.hidden, kw.node, kw.summary, kw.owner, kw.priority): - continue - _fatal() - raise RuntimeError('parsing issue - undefined leaf?') + continue else: - raise RuntimeError('parsing issue - we messed up?') + dict1[k] = dict2[k] return dict1 @@ -131,7 +127,7 @@ def _format_nodes(inside, conf, xml): name = node.pop('@name') into = inside + [name] if name in r: - r[name].update(_format_node(into, node, xml)) + _merge(r[name], _format_node(into, node, xml)) else: r[name] = _format_node(into, node, xml) r[name][kw.node] = nodename @@ -141,7 +137,7 @@ def _format_nodes(inside, conf, xml): name = node.pop('@name') into = inside + [name] if name in r: - r[name].update(_format_node(inside + [name], node, xml)) + _merge(r[name], _format_node(inside + [name], node, xml)) else: r[name] = _format_node(inside + [name], node, xml) r[name][kw.node] = nodename @@ -180,10 +176,10 @@ def _format_node(inside, conf, xml): if isinstance(conf, list): for child in children: - r = _safe_update(r, _format_nodes(inside, child, xml)) + _merge(r, _format_nodes(inside, child, xml)) else: child = children - r = _safe_update(r, _format_nodes(inside, child, xml)) + _merge(r, _format_nodes(inside, child, xml)) elif 'properties' in keys: properties = conf.pop('properties') |