diff options
Diffstat (limited to 'python')
34 files changed, 1939 insertions, 489 deletions
diff --git a/python/vyos/accel_ppp.py b/python/vyos/accel_ppp.py new file mode 100644 index 000000000..bfc8ee5a9 --- /dev/null +++ b/python/vyos/accel_ppp.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +import sys + +import vyos.opmode +from vyos.util import rc_cmd + + +def get_server_statistics(accel_statistics, pattern, sep=':') -> dict: + import re + + stat_dict = {'sessions': {}} + + cpu = re.search(r'cpu(.*)', accel_statistics).group(0) + # Find all lines with pattern, for example 'sstp:' + data = re.search(rf'{pattern}(.*)', accel_statistics, re.DOTALL).group(0) + session_starting = re.search(r'starting(.*)', data).group(0) + session_active = re.search(r'active(.*)', data).group(0) + + for entry in {cpu, session_starting, session_active}: + if sep in entry: + key, value = entry.split(sep) + if key in ['starting', 'active', 'finishing']: + stat_dict['sessions'][key] = value.strip() + continue + stat_dict[key] = value.strip() + return stat_dict + + +def accel_cmd(port: int, command: str) -> str: + _, output = rc_cmd(f'/usr/bin/accel-cmd -p{port} {command}') + return output + + +def accel_out_parse(accel_output: list[str]) -> list[dict[str, str]]: + """ Parse accel-cmd show sessions output """ + data_list: list[dict[str, str]] = list() + field_names: list[str] = list() + + field_names_unstripped: list[str] = accel_output.pop(0).split('|') + for field_name in field_names_unstripped: + field_names.append(field_name.strip()) + + while accel_output: + if '|' not in accel_output[0]: + accel_output.pop(0) + continue + + current_item: list[str] = accel_output.pop(0).split('|') + item_dict: dict[str, str] = {} + + for field_index in range(len(current_item)): + field_name: str = field_names[field_index] + field_value: str = current_item[field_index].strip() + item_dict[field_name] = field_value + + data_list.append(item_dict) + + return data_list diff --git a/python/vyos/base.py b/python/vyos/base.py index 78067d5b2..9b93cb2f2 100644 --- a/python/vyos/base.py +++ b/python/vyos/base.py @@ -15,17 +15,47 @@ from textwrap import fill + +class BaseWarning: + def __init__(self, header, message, **kwargs): + self.message = message + self.kwargs = kwargs + if 'width' not in kwargs: + self.width = 72 + if 'initial_indent' in kwargs: + del self.kwargs['initial_indent'] + if 'subsequent_indent' in kwargs: + del self.kwargs['subsequent_indent'] + self.textinitindent = header + self.standardindent = '' + + def print(self): + messages = self.message.split('\n') + isfirstmessage = True + initial_indent = self.textinitindent + print('') + for mes in messages: + mes = fill(mes, initial_indent=initial_indent, + subsequent_indent=self.standardindent, **self.kwargs) + if isfirstmessage: + isfirstmessage = False + initial_indent = self.standardindent + print(f'{mes}') + print('') + + class Warning(): - def __init__(self, message): - # Reformat the message and trim it to 72 characters in length - message = fill(message, width=72) - print(f'\nWARNING: {message}') + def __init__(self, message, **kwargs): + self.BaseWarn = BaseWarning('WARNING: ', message, **kwargs) + self.BaseWarn.print() + class DeprecationWarning(): - def __init__(self, message): + def __init__(self, message, **kwargs): # Reformat the message and trim it to 72 characters in length - message = fill(message, width=72) - print(f'\nDEPRECATION WARNING: {message}\n') + self.BaseWarn = BaseWarning('DEPRECATION WARNING: ', message, **kwargs) + self.BaseWarn.print() + class ConfigError(Exception): def __init__(self, message): diff --git a/python/vyos/component_version.py b/python/vyos/component_version.py new file mode 100644 index 000000000..a4e318d08 --- /dev/null +++ b/python/vyos/component_version.py @@ -0,0 +1,192 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +""" +Functions for reading/writing component versions. + +The config file version string has the following form: + +VyOS 1.3/1.4: + +// Warning: Do not remove the following line. +// vyos-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack@3:conntrack-sync@2:dhcp-relay@2:dhcp-server@6:dhcpv6-server@1:dns-forwarding@3:firewall@5:https@2:interfaces@22:ipoe-server@1:ipsec@5:isis@1:l2tp@3:lldp@1:mdns@1:nat@5:ntp@1:pppoe-server@5:pptp@2:qos@1:quagga@8:rpki@1:salt@1:snmp@2:ssh@2:sstp@3:system@21:vrrp@2:vyos-accel-ppp@2:wanloadbalance@3:webproxy@2:zone-policy@1" +// Release version: 1.3.0 + +VyOS 1.2: + +/* Warning: Do not remove the following line. */ +/* === vyatta-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack-sync@1:conntrack@1:dhcp-relay@2:dhcp-server@5:dns-forwarding@1:firewall@5:ipsec@5:l2tp@1:mdns@1:nat@4:ntp@1:pppoe-server@2:pptp@1:qos@1:quagga@7:snmp@1:ssh@1:system@10:vrrp@2:wanloadbalance@3:webgui@1:webproxy@2:zone-policy@1" === */ +/* Release version: 1.2.8 */ + +""" + +import os +import re +import sys +import fileinput + +from vyos.xml import component_version +from vyos.version import get_version +from vyos.defaults import directories + +DEFAULT_CONFIG_PATH = os.path.join(directories['config'], 'config.boot') + +def from_string(string_line, vintage='vyos'): + """ + Get component version dictionary from string. + Return empty dictionary if string contains no config information + or raise error if component version string malformed. + """ + version_dict = {} + + if vintage == 'vyos': + if re.match(r'// vyos-config-version:.+', string_line): + if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', string_line): + raise ValueError(f"malformed configuration string: {string_line}") + + for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): + version_dict[pair[0]] = int(pair[1]) + + elif vintage == 'vyatta': + if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line): + if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line): + raise ValueError(f"malformed configuration string: {string_line}") + + for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): + version_dict[pair[0]] = int(pair[1]) + else: + raise ValueError("Unknown config string vintage") + + return version_dict + +def from_file(config_file_name=DEFAULT_CONFIG_PATH, vintage='vyos'): + """ + Get component version dictionary parsing config file line by line + """ + with open(config_file_name, 'r') as f: + for line_in_config in f: + version_dict = from_string(line_in_config, vintage=vintage) + if version_dict: + return version_dict + + # no version information + return {} + +def from_system(): + """ + Get system component version dict. + """ + return component_version() + +def legacy_from_system(): + """ + Get system component version dict from legacy location. + This is for a transitional sanity check; the directory will eventually + be removed. + """ + system_versions = {} + legacy_dir = directories['current'] + + # To be removed: + if not os.path.isdir(legacy_dir): + return system_versions + + try: + version_info = os.listdir(legacy_dir) + except OSError as err: + sys.exit(repr(err)) + + for info in version_info: + if re.match(r'[\w,-]+@\d+', info): + pair = info.split('@') + system_versions[pair[0]] = int(pair[1]) + + return system_versions + +def format_string(ver: dict) -> str: + """ + Version dict to string. + """ + keys = list(ver) + keys.sort() + l = [] + for k in keys: + v = ver[k] + l.append(f'{k}@{v}') + sep = ':' + return sep.join(l) + +def version_footer(ver: dict, vintage='vyos') -> str: + """ + Version footer as string. + """ + ver_str = format_string(ver) + release = get_version() + if vintage == 'vyos': + ret_str = (f'// Warning: Do not remove the following line.\n' + + f'// vyos-config-version: "{ver_str}"\n' + + f'// Release version: {release}\n') + elif vintage == 'vyatta': + ret_str = (f'/* Warning: Do not remove the following line. */\n' + + f'/* === vyatta-config-version: "{ver_str}" === */\n' + + f'/* Release version: {release} */\n') + else: + raise ValueError("Unknown config string vintage") + + return ret_str + +def system_footer(vintage='vyos') -> str: + """ + System version footer as string. + """ + ver_d = from_system() + return version_footer(ver_d, vintage=vintage) + +def write_version_footer(ver: dict, file_name, vintage='vyos'): + """ + Write version footer to file. + """ + footer = version_footer(ver=ver, vintage=vintage) + if file_name: + with open(file_name, 'a') as f: + f.write(footer) + else: + sys.stdout.write(footer) + +def write_system_footer(file_name, vintage='vyos'): + """ + Write system version footer to file. + """ + ver_d = from_system() + return write_version_footer(ver_d, file_name=file_name, vintage=vintage) + +def remove_footer(file_name): + """ + Remove old version footer. + """ + for line in fileinput.input(file_name, inplace=True): + if re.match(r'/\* Warning:.+ \*/$', line): + continue + if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line): + continue + if re.match(r'/\* Release version:.+ \*/$', line): + continue + if re.match('// vyos-config-version:.+', line): + continue + if re.match('// Warning:.+', line): + continue + if re.match('// Release version:.+', line): + continue + sys.stdout.write(line) diff --git a/python/vyos/component_versions.py b/python/vyos/component_versions.py deleted file mode 100644 index 90b458aae..000000000 --- a/python/vyos/component_versions.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2017 VyOS maintainers and contributors <maintainers@vyos.io> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. - -""" -The version data looks like: - -/* Warning: Do not remove the following line. */ -/* === vyatta-config-version: -"cluster@1:config-management@1:conntrack-sync@1:conntrack@1:dhcp-relay@1:dhcp-server@4:firewall@5:ipsec@4:nat@4:qos@1:quagga@2:system@8:vrrp@1:wanloadbalance@3:webgui@1:webproxy@1:zone-policy@1" -=== */ -/* Release version: 1.2.0-rolling+201806131737 */ -""" - -import re - -def get_component_version(string_line): - """ - Get component version dictionary from string - return empty dictionary if string contains no config information - or raise error if component version string malformed - """ - return_value = {} - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line): - - if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line): - raise ValueError("malformed configuration string: " + str(string_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): - if pair[0] in return_value.keys(): - raise ValueError("duplicate unit name: \"" + str(pair[0]) + "\" in string: \"" + string_line + "\"") - return_value[pair[0]] = int(pair[1]) - - return return_value - - -def get_component_versions_from_file(config_file_name='/opt/vyatta/etc/config/config.boot'): - """ - Get component version dictionary parsing config file line by line - """ - f = open(config_file_name, 'r') - for line_in_config in f: - component_version = get_component_version(line_in_config) - if component_version: - return component_version - raise ValueError("no config string in file:", config_file_name) diff --git a/python/vyos/configdep.py b/python/vyos/configdep.py new file mode 100644 index 000000000..d4b2cc78f --- /dev/null +++ b/python/vyos/configdep.py @@ -0,0 +1,95 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +import os +import json +import typing +from inspect import stack + +from vyos.util import load_as_module +from vyos.defaults import directories +from vyos.configsource import VyOSError +from vyos import ConfigError + +# https://peps.python.org/pep-0484/#forward-references +# for type 'Config' +if typing.TYPE_CHECKING: + from vyos.config import Config + +dependent_func: dict[str, list[typing.Callable]] = {} + +def canon_name(name: str) -> str: + return os.path.splitext(name)[0].replace('-', '_') + +def canon_name_of_path(path: str) -> str: + script = os.path.basename(path) + return canon_name(script) + +def caller_name() -> str: + return stack()[-1].filename + +def read_dependency_dict() -> dict: + path = os.path.join(directories['data'], + 'config-mode-dependencies.json') + with open(path) as f: + d = json.load(f) + return d + +def get_dependency_dict(config: 'Config') -> dict: + if hasattr(config, 'cached_dependency_dict'): + d = getattr(config, 'cached_dependency_dict') + else: + d = read_dependency_dict() + setattr(config, 'cached_dependency_dict', d) + return d + +def run_config_mode_script(script: str, config: 'Config'): + path = os.path.join(directories['conf_mode'], script) + name = canon_name(script) + mod = load_as_module(name, path) + + config.set_level([]) + try: + c = mod.get_config(config) + mod.verify(c) + mod.generate(c) + mod.apply(c) + except (VyOSError, ConfigError) as e: + raise ConfigError(repr(e)) + +def def_closure(target: str, config: 'Config', + tagnode: typing.Optional[str] = None) -> typing.Callable: + script = target + '.py' + def func_impl(): + if tagnode: + os.environ['VYOS_TAGNODE_VALUE'] = tagnode + run_config_mode_script(script, config) + return func_impl + +def set_dependents(case: str, config: 'Config', + tagnode: typing.Optional[str] = None): + d = get_dependency_dict(config) + k = canon_name_of_path(caller_name()) + l = dependent_func.setdefault(k, []) + for target in d[k][case]: + func = def_closure(target, config, tagnode) + l.append(func) + +def call_dependents(): + k = canon_name_of_path(caller_name()) + l = dependent_func.get(k, []) + while l: + f = l.pop(0) + f() diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py index 04ddc10e9..53decfbf5 100644 --- a/python/vyos/configdict.py +++ b/python/vyos/configdict.py @@ -201,11 +201,12 @@ def is_member(conf, interface, intftype=None): intftype is optional, if not passed it will search all known types (currently bridge and bonding) - Returns: - None -> Interface is not a member - interface name -> Interface is a member of this interface - False -> interface type cannot have members + Returns: dict + empty -> Interface is not a member + key -> Interface is a member of this interface """ + from vyos.ifconfig import Section + ret_val = {} intftypes = ['bonding', 'bridge'] @@ -222,7 +223,8 @@ def is_member(conf, interface, intftype=None): member = base + [intf, 'member', 'interface', interface] if conf.exists(member): tmp = conf.get_config_dict(member, key_mangling=('-', '_'), - get_first_key=True, no_tag_node_value_mangle=True) + get_first_key=True, + no_tag_node_value_mangle=True) ret_val.update({intf : tmp}) return ret_val @@ -293,16 +295,23 @@ def is_source_interface(conf, interface, intftype=None): """ ret_val = None intftypes = ['macsec', 'pppoe', 'pseudo-ethernet', 'tunnel', 'vxlan'] - if intftype not in intftypes + [None]: + if not intftype: + intftype = intftypes + + if isinstance(intftype, str): + intftype = [intftype] + elif not isinstance(intftype, list): + raise ValueError(f'Interface type "{type(intftype)}" must be either str or list!') + + if not all(x in intftypes for x in intftype): raise ValueError(f'unknown interface type "{intftype}" or it can not ' 'have a source-interface') - intftype = intftypes if intftype == None else [intftype] for it in intftype: base = ['interfaces', it] for intf in conf.list_nodes(base): - lower_intf = base + [intf, 'source-interface'] - if conf.exists(lower_intf) and interface in conf.return_values(lower_intf): + src_intf = base + [intf, 'source-interface'] + if conf.exists(src_intf) and interface in conf.return_values(src_intf): ret_val = intf break @@ -358,13 +367,14 @@ def get_pppoe_interfaces(conf, vrf=None): """ Common helper functions to retrieve all interfaces from current CLI sessions that have DHCP configured. """ pppoe_interfaces = {} + conf.set_level([]) for ifname in conf.list_nodes(['interfaces', 'pppoe']): # always reset config level, as get_interface_dict() will alter it conf.set_level([]) # we already have a dict representation of the config from get_config_dict(), # but with the extended information from get_interface_dict() we also # get the DHCP client default-route-distance default option if not specified. - ifconfig = get_interface_dict(conf, ['interfaces', 'pppoe'], ifname) + _, ifconfig = get_interface_dict(conf, ['interfaces', 'pppoe'], ifname) options = {} if 'default_route_distance' in ifconfig: @@ -455,8 +465,8 @@ def get_interface_dict(config, base, ifname=''): if bond: dict.update({'is_bond_member' : bond}) # Check if any DHCP options changed which require a client restat - dhcp = node_changed(config, ['dhcp-options'], recursive=True) - if dhcp: dict.update({'dhcp_options_changed' : ''}) + dhcp = is_node_changed(config, base + [ifname, 'dhcp-options']) + if dhcp: dict.update({'dhcp_options_changed' : {}}) # Some interfaces come with a source_interface which must also not be part # of any other bond or bridge interface as it is exclusivly assigned as the @@ -515,8 +525,8 @@ def get_interface_dict(config, base, ifname=''): if bridge: dict['vif'][vif].update({'is_bridge_member' : bridge}) # Check if any DHCP options changed which require a client restat - dhcp = node_changed(config, ['vif', vif, 'dhcp-options'], recursive=True) - if dhcp: dict['vif'][vif].update({'dhcp_options_changed' : ''}) + dhcp = is_node_changed(config, base + [ifname, 'vif', vif, 'dhcp-options']) + if dhcp: dict['vif'][vif].update({'dhcp_options_changed' : {}}) for vif_s, vif_s_config in dict.get('vif_s', {}).items(): # Add subinterface name to dictionary @@ -554,8 +564,8 @@ def get_interface_dict(config, base, ifname=''): if bridge: dict['vif_s'][vif_s].update({'is_bridge_member' : bridge}) # Check if any DHCP options changed which require a client restat - dhcp = node_changed(config, ['vif-s', vif_s, 'dhcp-options'], recursive=True) - if dhcp: dict['vif_s'][vif_s].update({'dhcp_options_changed' : ''}) + dhcp = is_node_changed(config, base + [ifname, 'vif-s', vif_s, 'dhcp-options']) + if dhcp: dict['vif_s'][vif_s].update({'dhcp_options_changed' : {}}) for vif_c, vif_c_config in vif_s_config.get('vif_c', {}).items(): # Add subinterface name to dictionary @@ -594,8 +604,8 @@ def get_interface_dict(config, base, ifname=''): {'is_bridge_member' : bridge}) # Check if any DHCP options changed which require a client restat - dhcp = node_changed(config, ['vif-s', vif_s, 'vif-c', vif_c, 'dhcp-options'], recursive=True) - if dhcp: dict['vif_s'][vif_s]['vif_c'][vif_c].update({'dhcp_options_changed' : ''}) + dhcp = is_node_changed(config, base + [ifname, 'vif-s', vif_s, 'vif-c', vif_c, 'dhcp-options']) + if dhcp: dict['vif_s'][vif_s]['vif_c'][vif_c].update({'dhcp_options_changed' : {}}) # Check vif, vif-s/vif-c VLAN interfaces for removal dict = get_removed_vlans(config, base + [ifname], dict) @@ -633,7 +643,9 @@ def get_accel_dict(config, base, chap_secrets): from vyos.util import get_half_cpus from vyos.template import is_ipv4 - dict = config.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) + dict = config.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, + no_tag_node_value_mangle=True) # We have gathered the dict representation of the CLI, but there are default # options which we need to update into the dictionary retrived. @@ -653,6 +665,18 @@ def get_accel_dict(config, base, chap_secrets): # added to individual local users instead - so we can simply delete them if dict_search('client_ipv6_pool.prefix.mask', default_values): del default_values['client_ipv6_pool']['prefix']['mask'] + # delete empty dicts + if len (default_values['client_ipv6_pool']['prefix']) == 0: + del default_values['client_ipv6_pool']['prefix'] + if len (default_values['client_ipv6_pool']) == 0: + del default_values['client_ipv6_pool'] + + # T2665: IPoE only - it has an interface tag node + # added to individual local users instead - so we can simply delete them + if dict_search('authentication.interface', default_values): + del default_values['authentication']['interface'] + if dict_search('interface', default_values): + del default_values['interface'] dict = dict_merge(default_values, dict) @@ -674,11 +698,9 @@ def get_accel_dict(config, base, chap_secrets): dict.update({'name_server_ipv4' : ns_v4, 'name_server_ipv6' : ns_v6}) del dict['name_server'] - # Add individual RADIUS server default values + # T2665: Add individual RADIUS server default values if dict_search('authentication.radius.server', dict): - # T2665 default_values = defaults(base + ['authentication', 'radius', 'server']) - for server in dict_search('authentication.radius.server', dict): dict['authentication']['radius']['server'][server] = dict_merge( default_values, dict['authentication']['radius']['server'][server]) @@ -688,22 +710,31 @@ def get_accel_dict(config, base, chap_secrets): if 'disable_accounting' in dict['authentication']['radius']['server'][server]: dict['authentication']['radius']['server'][server]['acct_port'] = '0' - # Add individual local-user default values + # T2665: Add individual local-user default values if dict_search('authentication.local_users.username', dict): - # T2665 default_values = defaults(base + ['authentication', 'local-users', 'username']) - for username in dict_search('authentication.local_users.username', dict): dict['authentication']['local_users']['username'][username] = dict_merge( default_values, dict['authentication']['local_users']['username'][username]) - # Add individual IPv6 client-pool default mask if required + # T2665: Add individual IPv6 client-pool default mask if required if dict_search('client_ipv6_pool.prefix', dict): - # T2665 default_values = defaults(base + ['client-ipv6-pool', 'prefix']) - for prefix in dict_search('client_ipv6_pool.prefix', dict): dict['client_ipv6_pool']['prefix'][prefix] = dict_merge( default_values, dict['client_ipv6_pool']['prefix'][prefix]) + # T2665: IPoE only - add individual local-user default values + if dict_search('authentication.interface', dict): + default_values = defaults(base + ['authentication', 'interface']) + for interface in dict_search('authentication.interface', dict): + dict['authentication']['interface'][interface] = dict_merge( + default_values, dict['authentication']['interface'][interface]) + + if dict_search('interface', dict): + default_values = defaults(base + ['interface']) + for interface in dict_search('interface', dict): + dict['interface'][interface] = dict_merge(default_values, + dict['interface'][interface]) + return dict diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index e9cdb69e4..b88615513 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -1,5 +1,5 @@ # configtree -- a standalone VyOS config file manipulation library (Python bindings) -# Copyright (C) 2018 VyOS maintainers and contributors +# Copyright (C) 2018-2022 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or modify it under the terms of # the GNU Lesser General Public License as published by the Free Software Foundation; @@ -12,6 +12,7 @@ # You should have received a copy of the GNU Lesser General Public License along with this library; # if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +import os import re import json @@ -147,6 +148,8 @@ class ConfigTree(object): self.__config = address self.__version = '' + self.__migration = os.environ.get('VYOS_MIGRATION') + def __del__(self): if self.__config is not None: self.__destroy(self.__config) @@ -191,18 +194,27 @@ class ConfigTree(object): else: self.__set_add_value(self.__config, path_str, str(value).encode()) + if self.__migration: + print(f"- op: set path: {path} value: {value} replace: {replace}") + def delete(self, path): check_path(path) path_str = " ".join(map(str, path)).encode() self.__delete(self.__config, path_str) + if self.__migration: + print(f"- op: delete path: {path}") + def delete_value(self, path, value): check_path(path) path_str = " ".join(map(str, path)).encode() self.__delete_value(self.__config, path_str, value.encode()) + if self.__migration: + print(f"- op: delete_value path: {path} value: {value}") + def rename(self, path, new_name): check_path(path) path_str = " ".join(map(str, path)).encode() @@ -216,6 +228,9 @@ class ConfigTree(object): if (res != 0): raise ConfigTreeError("Path [{}] doesn't exist".format(path)) + if self.__migration: + print(f"- op: rename old_path: {path} new_path: {new_path}") + def copy(self, old_path, new_path): check_path(old_path) check_path(new_path) @@ -229,6 +244,9 @@ class ConfigTree(object): if (res != 0): raise ConfigTreeError("Path [{}] doesn't exist".format(old_path)) + if self.__migration: + print(f"- op: copy old_path: {old_path} new_path: {new_path}") + def exists(self, path): check_path(path) path_str = " ".join(map(str, path)).encode() diff --git a/python/vyos/configverify.py b/python/vyos/configverify.py index 438485d98..8e0ce701e 100644 --- a/python/vyos/configverify.py +++ b/python/vyos/configverify.py @@ -99,6 +99,18 @@ def verify_vrf(config): 'Interface "{ifname}" cannot be both a member of VRF "{vrf}" ' 'and bridge "{is_bridge_member}"!'.format(**config)) +def verify_bond_bridge_member(config): + """ + Checks if interface has a VRF configured and is also part of a bond or + bridge, which is not allowed! + """ + if 'vrf' in config: + ifname = config['ifname'] + if 'is_bond_member' in config: + raise ConfigError(f'Can not add interface "{ifname}" to bond, it has a VRF assigned!') + if 'is_bridge_member' in config: + raise ConfigError(f'Can not add interface "{ifname}" to bridge, it has a VRF assigned!') + def verify_tunnel(config): """ This helper is used to verify the common part of the tunnel @@ -231,10 +243,10 @@ def verify_address(config): of a bridge or bond. """ if {'is_bridge_member', 'address'} <= set(config): - raise ConfigError( - 'Cannot assign address to interface "{ifname}" as it is a ' - 'member of bridge "{is_bridge_member}"!'.format(**config)) - + interface = config['ifname'] + bridge_name = next(iter(config['is_bridge_member'])) + raise ConfigError(f'Cannot assign address to interface "{interface}" ' + f'as it is a member of bridge "{bridge_name}"!') def verify_bridge_delete(config): """ @@ -244,9 +256,9 @@ def verify_bridge_delete(config): """ if 'is_bridge_member' in config: interface = config['ifname'] - for bridge in config['is_bridge_member']: - raise ConfigError(f'Interface "{interface}" cannot be deleted as it ' - f'is a member of bridge "{bridge}"!') + bridge_name = next(iter(config['is_bridge_member'])) + raise ConfigError(f'Interface "{interface}" cannot be deleted as it ' + f'is a member of bridge "{bridge_name}"!') def verify_interface_exists(ifname): """ @@ -272,15 +284,22 @@ def verify_source_interface(config): raise ConfigError('Specified source-interface {source_interface} does ' 'not exist'.format(**config)) + src_ifname = config['source_interface'] if 'source_interface_is_bridge_member' in config: - raise ConfigError('Invalid source-interface {source_interface}. Interface ' - 'is already a member of bridge ' - '{source_interface_is_bridge_member}'.format(**config)) + bridge_name = next(iter(config['source_interface_is_bridge_member'])) + raise ConfigError(f'Invalid source-interface "{src_ifname}". Interface ' + f'is already a member of bridge "{bridge_name}"!') if 'source_interface_is_bond_member' in config: - raise ConfigError('Invalid source-interface {source_interface}. Interface ' - 'is already a member of bond ' - '{source_interface_is_bond_member}'.format(**config)) + bond_name = next(iter(config['source_interface_is_bond_member'])) + raise ConfigError(f'Invalid source-interface "{src_ifname}". Interface ' + f'is already a member of bond "{bond_name}"!') + + if 'is_source_interface' in config: + tmp = config['is_source_interface'] + src_ifname = config['source_interface'] + raise ConfigError(f'Can not use source-interface "{src_ifname}", it already ' \ + f'belongs to interface "{tmp}"!') def verify_dhcpv6(config): """ @@ -362,15 +381,17 @@ def verify_vlan_config(config): verify_mtu_parent(c_vlan, config) verify_mtu_parent(c_vlan, s_vlan) -def verify_accel_ppp_base_service(config): +def verify_accel_ppp_base_service(config, local_users=True): """ Common helper function which must be used by all Accel-PPP services based on get_config_dict() """ # vertify auth settings - if dict_search('authentication.mode', config) == 'local': - if not dict_search('authentication.local_users', config): - raise ConfigError('Authentication mode local requires local users to be configured!') + if local_users and dict_search('authentication.mode', config) == 'local': + if (dict_search(f'authentication.local_users', config) is None or + dict_search(f'authentication.local_users', config) == {}): + raise ConfigError( + 'Authentication mode local requires local users to be configured!') for user in dict_search('authentication.local_users.username', config): user_config = config['authentication']['local_users']['username'][user] diff --git a/python/vyos/cpu.py b/python/vyos/cpu.py index a0ef864be..488ae79fb 100644 --- a/python/vyos/cpu.py +++ b/python/vyos/cpu.py @@ -32,7 +32,8 @@ import re def _read_cpuinfo(): with open('/proc/cpuinfo', 'r') as f: - return f.readlines() + lines = f.read().strip() + return re.split(r'\n+', lines) def _split_line(l): l = l.strip() diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index fcb6a7fbc..7de458960 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -18,6 +18,7 @@ import os directories = { "data": "/usr/share/vyos/", "conf_mode": "/usr/libexec/vyos/conf_mode", + "op_mode": "/usr/libexec/vyos/op_mode", "config": "/opt/vyatta/etc/config", "current": "/opt/vyatta/etc/config-migrate/current", "migrate": "/opt/vyatta/etc/config-migrate/migrate", @@ -25,7 +26,7 @@ directories = { "templates": "/usr/share/vyos/templates/", "certbot": "/config/auth/letsencrypt", "api_schema": "/usr/libexec/vyos/services/api/graphql/graphql/schema/", - "api_templates": "/usr/libexec/vyos/services/api/graphql/recipes/templates/", + "api_templates": "/usr/libexec/vyos/services/api/graphql/session/templates/", "vyos_udev_dir": "/run/udev/vyos" } @@ -48,7 +49,6 @@ api_data = { 'port' : '8080', 'socket' : False, 'strict' : False, - 'gql' : False, 'debug' : False, 'api_keys' : [ {"id": "testapp", "key": "qwerty"} ] } diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index 04fd44173..48263eef5 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2022 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -14,10 +14,51 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import csv +import gzip +import os import re +from pathlib import Path +from socket import AF_INET +from socket import AF_INET6 +from socket import getaddrinfo +from time import strftime + +from vyos.remote import download +from vyos.template import is_ipv4 +from vyos.template import render +from vyos.util import call from vyos.util import cmd from vyos.util import dict_search_args +from vyos.util import dict_search_recursive +from vyos.util import run + +# Domain Resolver + +def fqdn_config_parse(firewall): + firewall['ip_fqdn'] = {} + firewall['ip6_fqdn'] = {} + + for domain, path in dict_search_recursive(firewall, 'fqdn'): + fw_name = path[1] # name/ipv6-name + rule = path[3] # rule id + suffix = path[4][0] # source/destination (1 char) + set_name = f'{fw_name}_{rule}_{suffix}' + + if path[0] == 'name': + firewall['ip_fqdn'][set_name] = domain + elif path[0] == 'ipv6_name': + firewall['ip6_fqdn'][set_name] = domain + +def fqdn_resolve(fqdn, ipv6=False): + try: + res = getaddrinfo(fqdn, None, AF_INET6 if ipv6 else AF_INET) + return set(item[4][0] for item in res) + except: + return None + +# End Domain Resolver def find_nftables_rule(table, chain, rule_matches=[]): # Find rule in table/chain that matches all criteria and return the handle @@ -72,12 +113,32 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name): if side in rule_conf: prefix = side[0] side_conf = rule_conf[side] + address_mask = side_conf.get('address_mask', None) if 'address' in side_conf: suffix = side_conf['address'] - if suffix[0] == '!': - suffix = f'!= {suffix[1:]}' - output.append(f'{ip_name} {prefix}addr {suffix}') + operator = '' + exclude = suffix[0] == '!' + if exclude: + operator = '!= ' + suffix = suffix[1:] + if address_mask: + operator = '!=' if exclude else '==' + operator = f'& {address_mask} {operator} ' + output.append(f'{ip_name} {prefix}addr {operator}{suffix}') + + if 'fqdn' in side_conf: + fqdn = side_conf['fqdn'] + operator = '' + if fqdn[0] == '!': + operator = '!=' + output.append(f'{ip_name} {prefix}addr {operator} @FQDN_{fw_name}_{rule_id}_{prefix}') + + if dict_search_args(side_conf, 'geoip', 'country_code'): + operator = '' + if dict_search_args(side_conf, 'geoip', 'inverse_match') != None: + operator = '!=' + output.append(f'{ip_name} {prefix}addr {operator} @GEOIP_CC_{fw_name}_{rule_id}') if 'mac_address' in side_conf: suffix = side_conf["mac_address"] @@ -114,24 +175,36 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name): if 'address_group' in group: group_name = group['address_group'] operator = '' + exclude = group_name[0] == "!" + if exclude: + operator = '!=' + group_name = group_name[1:] + if address_mask: + operator = '!=' if exclude else '==' + operator = f'& {address_mask} {operator}' + output.append(f'{ip_name} {prefix}addr {operator} @A{def_suffix}_{group_name}') + # Generate firewall group domain-group + elif 'domain_group' in group: + group_name = group['domain_group'] + operator = '' if group_name[0] == '!': operator = '!=' group_name = group_name[1:] - output.append(f'{ip_name} {prefix}addr {operator} $A{def_suffix}_{group_name}') + output.append(f'{ip_name} {prefix}addr {operator} @D_{group_name}') elif 'network_group' in group: group_name = group['network_group'] operator = '' if group_name[0] == '!': operator = '!=' group_name = group_name[1:] - output.append(f'{ip_name} {prefix}addr {operator} $N{def_suffix}_{group_name}') + output.append(f'{ip_name} {prefix}addr {operator} @N{def_suffix}_{group_name}') if 'mac_group' in group: group_name = group['mac_group'] operator = '' if group_name[0] == '!': operator = '!=' group_name = group_name[1:] - output.append(f'ether {prefix}addr {operator} $M_{group_name}') + output.append(f'ether {prefix}addr {operator} @M_{group_name}') if 'port_group' in group: proto = rule_conf['protocol'] group_name = group['port_group'] @@ -144,11 +217,16 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name): operator = '!=' group_name = group_name[1:] - output.append(f'{proto} {prefix}port {operator} $P_{group_name}') + output.append(f'{proto} {prefix}port {operator} @P_{group_name}') if 'log' in rule_conf and rule_conf['log'] == 'enable': action = rule_conf['action'] if 'action' in rule_conf else 'accept' - output.append(f'log prefix "[{fw_name[:19]}-{rule_id}-{action[:1].upper()}] "') + output.append(f'log prefix "[{fw_name[:19]}-{rule_id}-{action[:1].upper()}]"') + + if 'log_level' in rule_conf: + log_level = rule_conf['log_level'] + output.append(f'level {log_level}') + if 'hop_limit' in rule_conf: operators = {'eq': '==', 'gt': '>', 'lt': '<'} @@ -157,6 +235,21 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name): value = rule_conf['hop_limit'][op] output.append(f'ip6 hoplimit {operator} {value}') + if 'inbound_interface' in rule_conf: + iiface = rule_conf['inbound_interface'] + output.append(f'iifname {iiface}') + + if 'outbound_interface' in rule_conf: + oiface = rule_conf['outbound_interface'] + output.append(f'oifname {oiface}') + + if 'ttl' in rule_conf: + operators = {'eq': '==', 'gt': '>', 'lt': '<'} + for op, operator in operators.items(): + if op in rule_conf['ttl']: + value = rule_conf['ttl'][op] + output.append(f'ip ttl {operator} {value}') + for icmp in ['icmp', 'icmpv6']: if icmp in rule_conf: if 'type_name' in rule_conf[icmp]: @@ -167,6 +260,23 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name): if 'type' in rule_conf[icmp]: output.append(icmp + ' type ' + rule_conf[icmp]['type']) + + if 'packet_length' in rule_conf: + lengths_str = ','.join(rule_conf['packet_length']) + output.append(f'ip{def_suffix} length {{{lengths_str}}}') + + if 'packet_length_exclude' in rule_conf: + negated_lengths_str = ','.join(rule_conf['packet_length_exclude']) + output.append(f'ip{def_suffix} length != {{{negated_lengths_str}}}') + + if 'dscp' in rule_conf: + dscp_str = ','.join(rule_conf['dscp']) + output.append(f'ip{def_suffix} dscp {{{dscp_str}}}') + + if 'dscp_exclude' in rule_conf: + negated_dscp_str = ','.join(rule_conf['dscp_exclude']) + output.append(f'ip{def_suffix} dscp != {{{negated_dscp_str}}}') + if 'ipsec' in rule_conf: if 'match_ipsec' in rule_conf['ipsec']: output.append('meta ipsec == 1') @@ -199,6 +309,11 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name): if tcp_flags: output.append(parse_tcp_flags(tcp_flags)) + # TCP MSS + tcp_mss = dict_search_args(rule_conf, 'tcp', 'mss') + if tcp_mss: + output.append(f'tcp option maxseg size {tcp_mss}') + output.append('counter') if 'set' in rule_conf: @@ -206,6 +321,10 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name): if 'action' in rule_conf: output.append(nft_action(rule_conf['action'])) + if 'jump' in rule_conf['action']: + target = rule_conf['jump_target'] + output.append(f'NAME{def_suffix}_{target}') + else: output.append('return') @@ -257,3 +376,118 @@ def parse_policy_set(set_conf, def_suffix): mss = set_conf['tcp_mss'] out.append(f'tcp option maxseg size set {mss}') return " ".join(out) + +# GeoIP + +nftables_geoip_conf = '/run/nftables-geoip.conf' +geoip_database = '/usr/share/vyos-geoip/dbip-country-lite.csv.gz' +geoip_lock_file = '/run/vyos-geoip.lock' + +def geoip_load_data(codes=[]): + data = None + + if not os.path.exists(geoip_database): + return [] + + try: + with gzip.open(geoip_database, mode='rt') as csv_fh: + reader = csv.reader(csv_fh) + out = [] + for start, end, code in reader: + if code.lower() in codes: + out.append([start, end, code.lower()]) + return out + except: + print('Error: Failed to open GeoIP database') + return [] + +def geoip_download_data(): + url = 'https://download.db-ip.com/free/dbip-country-lite-{}.csv.gz'.format(strftime("%Y-%m")) + try: + dirname = os.path.dirname(geoip_database) + if not os.path.exists(dirname): + os.mkdir(dirname) + + download(geoip_database, url) + print("Downloaded GeoIP database") + return True + except: + print("Error: Failed to download GeoIP database") + return False + +class GeoIPLock(object): + def __init__(self, file): + self.file = file + + def __enter__(self): + if os.path.exists(self.file): + return False + + Path(self.file).touch() + return True + + def __exit__(self, exc_type, exc_value, tb): + os.unlink(self.file) + +def geoip_update(firewall, force=False): + with GeoIPLock(geoip_lock_file) as lock: + if not lock: + print("Script is already running") + return False + + if not firewall: + print("Firewall is not configured") + return True + + if not os.path.exists(geoip_database): + if not geoip_download_data(): + return False + elif force: + geoip_download_data() + + ipv4_codes = {} + ipv6_codes = {} + + ipv4_sets = {} + ipv6_sets = {} + + # Map country codes to set names + for codes, path in dict_search_recursive(firewall, 'country_code'): + set_name = f'GEOIP_CC_{path[1]}_{path[3]}' + if path[0] == 'name': + for code in codes: + ipv4_codes.setdefault(code, []).append(set_name) + elif path[0] == 'ipv6_name': + for code in codes: + ipv6_codes.setdefault(code, []).append(set_name) + + if not ipv4_codes and not ipv6_codes: + if force: + print("GeoIP not in use by firewall") + return True + + geoip_data = geoip_load_data([*ipv4_codes, *ipv6_codes]) + + # Iterate IP blocks to assign to sets + for start, end, code in geoip_data: + ipv4 = is_ipv4(start) + if code in ipv4_codes and ipv4: + ip_range = f'{start}-{end}' if start != end else start + for setname in ipv4_codes[code]: + ipv4_sets.setdefault(setname, []).append(ip_range) + if code in ipv6_codes and not ipv4: + ip_range = f'{start}-{end}' if start != end else start + for setname in ipv6_codes[code]: + ipv6_sets.setdefault(setname, []).append(ip_range) + + render(nftables_geoip_conf, 'firewall/nftables-geoip-update.j2', { + 'ipv4_sets': ipv4_sets, + 'ipv6_sets': ipv6_sets + }) + + result = run(f'nft -f {nftables_geoip_conf}') + if result != 0: + print('Error: GeoIP failed to update firewall') + return False + + return True diff --git a/python/vyos/formatversions.py b/python/vyos/formatversions.py deleted file mode 100644 index 29117a5d3..000000000 --- a/python/vyos/formatversions.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see <http://www.gnu.org/licenses/>. - -import sys -import os -import re -import fileinput - -def read_vyatta_versions(config_file): - config_file_versions = {} - - with open(config_file, 'r') as config_file_handle: - for config_line in config_file_handle: - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', config_line): - if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', config_line): - raise ValueError("malformed configuration string: " - "{}".format(config_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', config_line): - config_file_versions[pair[0]] = int(pair[1]) - - - return config_file_versions - -def read_vyos_versions(config_file): - config_file_versions = {} - - with open(config_file, 'r') as config_file_handle: - for config_line in config_file_handle: - if re.match(r'// vyos-config-version:.+', config_line): - if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', config_line): - raise ValueError("malformed configuration string: " - "{}".format(config_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', config_line): - config_file_versions[pair[0]] = int(pair[1]) - - return config_file_versions - -def remove_versions(config_file): - """ - Remove old version string. - """ - for line in fileinput.input(config_file, inplace=True): - if re.match(r'/\* Warning:.+ \*/$', line): - continue - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line): - continue - if re.match(r'/\* Release version:.+ \*/$', line): - continue - if re.match('// vyos-config-version:.+', line): - continue - if re.match('// Warning:.+', line): - continue - if re.match('// Release version:.+', line): - continue - sys.stdout.write(line) - -def format_versions_string(config_versions): - cfg_keys = list(config_versions.keys()) - cfg_keys.sort() - - component_version_strings = [] - - for key in cfg_keys: - cfg_vers = config_versions[key] - component_version_strings.append('{}@{}'.format(key, cfg_vers)) - - separator = ":" - component_version_string = separator.join(component_version_strings) - - return component_version_string - -def write_vyatta_versions_foot(config_file, component_version_string, - os_version_string): - if config_file: - with open(config_file, 'a') as config_file_handle: - config_file_handle.write('/* Warning: Do not remove the following line. */\n') - config_file_handle.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string)) - config_file_handle.write('/* Release version: {} */\n'.format(os_version_string)) - else: - sys.stdout.write('/* Warning: Do not remove the following line. */\n') - sys.stdout.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string)) - sys.stdout.write('/* Release version: {} */\n'.format(os_version_string)) - -def write_vyos_versions_foot(config_file, component_version_string, - os_version_string): - if config_file: - with open(config_file, 'a') as config_file_handle: - config_file_handle.write('// Warning: Do not remove the following line.\n') - config_file_handle.write('// vyos-config-version: "{}"\n'.format(component_version_string)) - config_file_handle.write('// Release version: {}\n'.format(os_version_string)) - else: - sys.stdout.write('// Warning: Do not remove the following line.\n') - sys.stdout.write('// vyos-config-version: "{}"\n'.format(component_version_string)) - sys.stdout.write('// Release version: {}\n'.format(os_version_string)) - diff --git a/python/vyos/frr.py b/python/vyos/frr.py index 0ffd5cba9..ccb132dd5 100644 --- a/python/vyos/frr.py +++ b/python/vyos/frr.py @@ -477,7 +477,7 @@ class FRRConfig: # for the listed FRR issues above pass if count >= count_max: - raise ConfigurationNotValid(f'Config commit retry counter ({count_max}) exceeded') + raise ConfigurationNotValid(f'Config commit retry counter ({count_max}) exceeded for {daemon} dameon!') # Save configuration to /run/frr/config/frr.conf save_configuration() diff --git a/python/vyos/ifconfig/__init__.py b/python/vyos/ifconfig/__init__.py index a37615c8f..206b2bba1 100644 --- a/python/vyos/ifconfig/__init__.py +++ b/python/vyos/ifconfig/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -36,4 +36,6 @@ from vyos.ifconfig.tunnel import TunnelIf from vyos.ifconfig.wireless import WiFiIf from vyos.ifconfig.l2tpv3 import L2TPv3If from vyos.ifconfig.macsec import MACsecIf +from vyos.ifconfig.veth import VethIf from vyos.ifconfig.wwan import WWANIf +from vyos.ifconfig.sstpc import SSTPCIf diff --git a/python/vyos/ifconfig/bond.py b/python/vyos/ifconfig/bond.py index 2b9afe109..0edd17055 100644 --- a/python/vyos/ifconfig/bond.py +++ b/python/vyos/ifconfig/bond.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -179,6 +179,21 @@ class BondIf(Interface): """ self.set_interface('bond_lacp_rate', slow_fast) + def set_miimon_interval(self, interval): + """ + Specifies the MII link monitoring frequency in milliseconds. This + determines how often the link state of each slave is inspected for link + failures. A value of zero disables MII link monitoring. A value of 100 + is a good starting point. + + The default value is 0. + + Example: + >>> from vyos.ifconfig import BondIf + >>> BondIf('bond0').set_miimon_interval('100') + """ + return self.set_interface('bond_miimon', interval) + def set_arp_interval(self, interval): """ Specifies the ARP link monitoring frequency in milliseconds. @@ -202,16 +217,7 @@ class BondIf(Interface): >>> from vyos.ifconfig import BondIf >>> BondIf('bond0').set_arp_interval('100') """ - if int(interval) == 0: - """ - Specifies the MII link monitoring frequency in milliseconds. - This determines how often the link state of each slave is - inspected for link failures. A value of zero disables MII - link monitoring. A value of 100 is a good starting point. - """ - return self.set_interface('bond_miimon', interval) - else: - return self.set_interface('bond_arp_interval', interval) + return self.set_interface('bond_arp_interval', interval) def get_arp_ip_target(self): """ @@ -381,26 +387,9 @@ class BondIf(Interface): if 'shutdown_required' in config: self.set_admin_state('down') - # ARP monitor targets need to be synchronized between sysfs and CLI. - # Unfortunately an address can't be send twice to sysfs as this will - # result in the following exception: OSError: [Errno 22] Invalid argument. - # - # We remove ALL addresses prior to adding new ones, this will remove - # addresses manually added by the user too - but as we are limited to 16 adresses - # from the kernel side this looks valid to me. We won't run into an error - # when a user added manual adresses which would result in having more - # then 16 adresses in total. - arp_tgt_addr = list(map(str, self.get_arp_ip_target().split())) - for addr in arp_tgt_addr: - self.set_arp_ip_target('-' + addr) - - # Add configured ARP target addresses - value = dict_search('arp_monitor.target', config) - if isinstance(value, str): - value = [value] - if value: - for addr in value: - self.set_arp_ip_target('+' + addr) + # Specifies the MII link monitoring frequency in milliseconds + value = config.get('mii_mon_interval') + self.set_miimon_interval(value) # Bonding transmit hash policy value = config.get('hash_policy') @@ -416,10 +405,12 @@ class BondIf(Interface): # Remove ALL bond member interfaces for interface in self.get_slaves(): self.del_port(interface) - # Removing an interface from a bond will always place the underlaying - # physical interface in admin-down state! If physical interface is - # not disabled, re-enable it. - if not dict_search(f'member.interface_remove.{interface}.disable', config): + + # Restore correct interface status based on config + if dict_search(f'member.interface.{interface}.disable', config) is not None or \ + dict_search(f'member.interface_remove.{interface}.disable', config) is not None: + Interface(interface).set_admin_state('down') + else: Interface(interface).set_admin_state('up') # Bonding policy/mode - default value, always present @@ -430,6 +421,32 @@ class BondIf(Interface): if mode == '802.3ad': self.set_lacp_rate(config.get('lacp_rate')) + if mode not in ['802.3ad', 'balance-tlb', 'balance-alb']: + tmp = dict_search('arp_monitor.interval', config) + value = tmp if (tmp != None) else '0' + self.set_arp_interval(value) + + # ARP monitor targets need to be synchronized between sysfs and CLI. + # Unfortunately an address can't be send twice to sysfs as this will + # result in the following exception: OSError: [Errno 22] Invalid argument. + # + # We remove ALL addresses prior to adding new ones, this will remove + # addresses manually added by the user too - but as we are limited to 16 adresses + # from the kernel side this looks valid to me. We won't run into an error + # when a user added manual adresses which would result in having more + # then 16 adresses in total. + arp_tgt_addr = list(map(str, self.get_arp_ip_target().split())) + for addr in arp_tgt_addr: + self.set_arp_ip_target('-' + addr) + + # Add configured ARP target addresses + value = dict_search('arp_monitor.target', config) + if isinstance(value, str): + value = [value] + if value: + for addr in value: + self.set_arp_ip_target('+' + addr) + # Add (enslave) interfaces to bond value = dict_search('member.interface', config) for interface in (value or []): diff --git a/python/vyos/ifconfig/bridge.py b/python/vyos/ifconfig/bridge.py index ffd9c590f..aa818bc5f 100644 --- a/python/vyos/ifconfig/bridge.py +++ b/python/vyos/ifconfig/bridge.py @@ -90,6 +90,10 @@ class BridgeIf(Interface): 'validate': assert_boolean, 'location': '/sys/class/net/{ifname}/bridge/multicast_querier', }, + 'multicast_snooping': { + 'validate': assert_boolean, + 'location': '/sys/class/net/{ifname}/bridge/multicast_snooping', + }, }} _command_set = {**Interface._command_set, **{ @@ -183,6 +187,11 @@ class BridgeIf(Interface): """ self.set_interface('vlan_filter', state) + # VLAN of bridge parent interface is always 1 + # VLAN 1 is the default VLAN for all unlabeled packets + cmd = f'bridge vlan add dev {self.ifname} vid 1 pvid untagged self' + self._cmd(cmd) + def set_multicast_querier(self, enable): """ Sets whether the bridge actively runs a multicast querier or not. When a @@ -198,6 +207,18 @@ class BridgeIf(Interface): """ self.set_interface('multicast_querier', enable) + def set_multicast_snooping(self, enable): + """ + Enable or disable multicast snooping on the bridge. + + Use enable=1 to enable or enable=0 to disable + + Example: + >>> from vyos.ifconfig import Interface + >>> BridgeIf('br0').set_multicast_snooping(1) + """ + self.set_interface('multicast_snooping', enable) + def add_port(self, interface): """ Add physical interface to bridge (member port) @@ -257,6 +278,11 @@ class BridgeIf(Interface): value = '1' if 'stp' in config else '0' self.set_stp(value) + # enable or disable multicast snooping + tmp = dict_search('igmp.snooping', config) + value = '1' if (tmp != None) else '0' + self.set_multicast_snooping(value) + # enable or disable IGMP querier tmp = dict_search('igmp.querier', config) value = '1' if (tmp != None) else '0' @@ -269,31 +295,23 @@ class BridgeIf(Interface): self.del_port(member) # enable/disable Vlan Filter - vlan_filter = '1' if 'enable_vlan' in config else '0' - self.set_vlan_filter(vlan_filter) - - ifname = config['ifname'] - if int(vlan_filter): - add_vlan = [] - cur_vlan_ids = get_vlan_ids(ifname) - - tmp = dict_search('vif', config) - if tmp: - for vif, vif_config in tmp.items(): - add_vlan.append(vif) - - # Remove redundant VLANs from the system - for vlan in list_diff(cur_vlan_ids, add_vlan): - cmd = f'bridge vlan del dev {ifname} vid {vlan} self' + tmp = '1' if 'enable_vlan' in config else '0' + self.set_vlan_filter(tmp) + + # add VLAN interfaces to local 'parent' bridge to allow forwarding + if 'enable_vlan' in config: + for vlan in config.get('vif_remove', {}): + # Remove old VLANs from the bridge + cmd = f'bridge vlan del dev {self.ifname} vid {vlan} self' self._cmd(cmd) - for vlan in add_vlan: - cmd = f'bridge vlan add dev {ifname} vid {vlan} self' + for vlan in config.get('vif', {}): + cmd = f'bridge vlan add dev {self.ifname} vid {vlan} self' self._cmd(cmd) - # VLAN of bridge parent interface is always 1 - # VLAN 1 is the default VLAN for all unlabeled packets - cmd = f'bridge vlan add dev {ifname} vid 1 pvid untagged self' + # VLAN of bridge parent interface is always 1. VLAN 1 is the default + # VLAN for all unlabeled packets + cmd = f'bridge vlan add dev {self.ifname} vid 1 pvid untagged self' self._cmd(cmd) tmp = dict_search('member.interface', config) @@ -325,15 +343,13 @@ class BridgeIf(Interface): # set bridge port path cost if 'cost' in interface_config: - value = interface_config.get('cost') - lower.set_path_cost(value) + lower.set_path_cost(interface_config['cost']) # set bridge port path priority if 'priority' in interface_config: - value = interface_config.get('priority') - lower.set_path_priority(value) + lower.set_path_priority(interface_config['priority']) - if int(vlan_filter): + if 'enable_vlan' in config: add_vlan = [] native_vlan_id = None allowed_vlan_ids= [] @@ -363,6 +379,7 @@ class BridgeIf(Interface): for vlan in allowed_vlan_ids: cmd = f'bridge vlan add dev {interface} vid {vlan} master' self._cmd(cmd) + # Setting native VLAN to system if native_vlan_id: cmd = f'bridge vlan add dev {interface} vid {native_vlan_id} pvid untagged master' diff --git a/python/vyos/ifconfig/ethernet.py b/python/vyos/ifconfig/ethernet.py index 1280fc238..519cfc58c 100644 --- a/python/vyos/ifconfig/ethernet.py +++ b/python/vyos/ifconfig/ethernet.py @@ -16,6 +16,7 @@ import os import re +from glob import glob from vyos.ethtool import Ethtool from vyos.ifconfig.interface import Interface from vyos.util import run @@ -69,13 +70,6 @@ class EthernetIf(Interface): }, }} - _sysfs_set = {**Interface._sysfs_set, **{ - 'rps': { - 'convert': lambda cpus: cpus if cpus else '0', - 'location': '/sys/class/net/{ifname}/queues/rx-0/rps_cpus', - }, - }} - def __init__(self, ifname, **kargs): super().__init__(ifname, **kargs) self.ethtool = Ethtool(ifname) @@ -236,7 +230,7 @@ class EthernetIf(Interface): enabled, fixed = self.ethtool.get_large_receive_offload() if enabled != state: if not fixed: - return self.set_interface('gro', 'on' if state else 'off') + return self.set_interface('lro', 'on' if state else 'off') else: print('Adapter does not support changing large-receive-offload settings!') return False @@ -246,6 +240,7 @@ class EthernetIf(Interface): raise ValueError('Value out of range') rps_cpus = '0' + queues = len(glob(f'/sys/class/net/{self.ifname}/queues/rx-*')) if state: # Enable RPS on all available CPUs except CPU0 which we will not # utilize so the system has one spare core when it's under high @@ -255,8 +250,23 @@ class EthernetIf(Interface): # Linux will clip that internally! rps_cpus = 'ffffffff,ffffffff,ffffffff,fffffffe' + for i in range(0, queues): + self._write_sysfs(f'/sys/class/net/{self.ifname}/queues/rx-{i}/rps_cpus', rps_cpus) + # send bitmask representation as hex string without leading '0x' - return self.set_interface('rps', rps_cpus) + return True + + def set_rfs(self, state): + rfs_flow = 0 + queues = len(glob(f'/sys/class/net/{self.ifname}/queues/rx-*')) + if state: + global_rfs_flow = 32768 + rfs_flow = int(global_rfs_flow/queues) + + for i in range(0, queues): + self._write_sysfs(f'/sys/class/net/{self.ifname}/queues/rx-{i}/rps_flow_cnt', rfs_flow) + + return True def set_sg(self, state): """ @@ -273,7 +283,7 @@ class EthernetIf(Interface): enabled, fixed = self.ethtool.get_scatter_gather() if enabled != state: if not fixed: - return self.set_interface('gro', 'on' if state else 'off') + return self.set_interface('sg', 'on' if state else 'off') else: print('Adapter does not support changing scatter-gather settings!') return False @@ -293,7 +303,7 @@ class EthernetIf(Interface): enabled, fixed = self.ethtool.get_tcp_segmentation_offload() if enabled != state: if not fixed: - return self.set_interface('gro', 'on' if state else 'off') + return self.set_interface('tso', 'on' if state else 'off') else: print('Adapter does not support changing tcp-segmentation-offload settings!') return False @@ -342,6 +352,9 @@ class EthernetIf(Interface): # RPS - Receive Packet Steering self.set_rps(dict_search('offload.rps', config) != None) + # RFS - Receive Flow Steering + self.set_rfs(dict_search('offload.rfs', config) != None) + # scatter-gather option self.set_sg(dict_search('offload.sg', config) != None) @@ -359,5 +372,5 @@ class EthernetIf(Interface): for rx_tx, size in config['ring_buffer'].items(): self.set_ring_buffer(rx_tx, size) - # call base class first + # call base class last super().update(config) diff --git a/python/vyos/ifconfig/interface.py b/python/vyos/ifconfig/interface.py index 22441d1d2..c50ead89f 100755..100644 --- a/python/vyos/ifconfig/interface.py +++ b/python/vyos/ifconfig/interface.py @@ -168,6 +168,10 @@ class Interface(Control): 'validate': assert_boolean, 'location': '/proc/sys/net/ipv4/conf/{ifname}/forwarding', }, + 'ipv4_directed_broadcast': { + 'validate': assert_boolean, + 'location': '/proc/sys/net/ipv4/conf/{ifname}/bc_forwarding', + }, 'rp_filter': { 'validate': lambda flt: assert_range(flt,0,3), 'location': '/proc/sys/net/ipv4/conf/{ifname}/rp_filter', @@ -234,6 +238,9 @@ class Interface(Control): 'ipv4_forwarding': { 'location': '/proc/sys/net/ipv4/conf/{ifname}/forwarding', }, + 'ipv4_directed_broadcast': { + 'location': '/proc/sys/net/ipv4/conf/{ifname}/bc_forwarding', + }, 'rp_filter': { 'location': '/proc/sys/net/ipv4/conf/{ifname}/rp_filter', }, @@ -713,6 +720,13 @@ class Interface(Control): return None return self.set_interface('ipv4_forwarding', forwarding) + def set_ipv4_directed_broadcast(self, forwarding): + """ Configure IPv4 directed broadcast forwarding. """ + tmp = self.get_interface('ipv4_directed_broadcast') + if tmp == forwarding: + return None + return self.set_interface('ipv4_directed_broadcast', forwarding) + def set_ipv4_source_validation(self, value): """ Help prevent attacks used by Spoofing IP Addresses. Reverse path @@ -1305,8 +1319,9 @@ class Interface(Control): # clear existing ingess - ignore errors (e.g. "Error: Cannot find specified # qdisc on specified device") - we simply cleanup all stuff here - self._popen(f'tc qdisc del dev {source_if} parent ffff: 2>/dev/null'); - self._popen(f'tc qdisc del dev {source_if} parent 1: 2>/dev/null'); + if not 'traffic_policy' in self._config: + self._popen(f'tc qdisc del dev {source_if} parent ffff: 2>/dev/null'); + self._popen(f'tc qdisc del dev {source_if} parent 1: 2>/dev/null'); # Apply interface mirror policy if mirror_config: @@ -1439,14 +1454,22 @@ class Interface(Control): if dhcpv6pd: self.set_dhcpv6(True) - # There are some items in the configuration which can only be applied - # if this instance is not bound to a bridge. This should be checked - # by the caller but better save then sorry! - if not any(k in ['is_bond_member', 'is_bridge_member'] for k in config): - # Bind interface to given VRF or unbind it if vrf node is not set. - # unbinding will call 'ip link set dev eth0 nomaster' which will - # also drop the interface out of a bridge or bond - thus this is - # checked before + # XXX: Bind interface to given VRF or unbind it if vrf is not set. Unbinding + # will call 'ip link set dev eth0 nomaster' which will also drop the + # interface out of any bridge or bond - thus this is checked before. + if 'is_bond_member' in config: + bond_if = next(iter(config['is_bond_member'])) + tmp = get_interface_config(config['ifname']) + if 'master' in tmp and tmp['master'] != bond_if: + self.set_vrf('') + + elif 'is_bridge_member' in config: + bridge_if = next(iter(config['is_bridge_member'])) + tmp = get_interface_config(config['ifname']) + if 'master' in tmp and tmp['master'] != bridge_if: + self.set_vrf('') + + else: self.set_vrf(config.get('vrf', '')) # Add this section after vrf T4331 @@ -1498,6 +1521,11 @@ class Interface(Control): value = '0' if (tmp != None) else '1' self.set_ipv4_forwarding(value) + # IPv4 directed broadcast forwarding + tmp = dict_search('ip.enable_directed_broadcast', config) + value = '1' if (tmp != None) else '0' + self.set_ipv4_directed_broadcast(value) + # IPv4 source-validation tmp = dict_search('ip.source_validation', config) value = tmp if (tmp != None) else '0' @@ -1507,7 +1535,7 @@ class Interface(Control): # before mangling any IPv6 option. If MTU is less then 1280 IPv6 will be # automatically disabled by the kernel. Also MTU must be increased before # configuring any IPv6 address on the interface. - if 'mtu' in config: + if 'mtu' in config and dict_search('dhcp_options.mtu', config) == None: self.set_mtu(config.get('mtu')) # Configure MSS value for IPv6 TCP connections @@ -1555,8 +1583,8 @@ class Interface(Control): # re-add ourselves to any bridge we might have fallen out of if 'is_bridge_member' in config: - bridge_dict = config.get('is_bridge_member') - self.add_to_bridge(bridge_dict) + tmp = config.get('is_bridge_member') + self.add_to_bridge(tmp) # eXpress Data Path - highly experimental self.set_xdp('xdp' in config) diff --git a/python/vyos/ifconfig/macvlan.py b/python/vyos/ifconfig/macvlan.py index 776014bc3..2266879ec 100644 --- a/python/vyos/ifconfig/macvlan.py +++ b/python/vyos/ifconfig/macvlan.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -30,10 +30,17 @@ class MACVLANIf(Interface): } def _create(self): + """ + Create MACvlan interface in OS kernel. Interface is administrative + down by default. + """ # please do not change the order when assembling the command cmd = 'ip link add {ifname} link {source_interface} type {type} mode {mode}' self._cmd(cmd.format(**self.config)) + # interface is always A/D down. It needs to be enabled explicitly + self.set_admin_state('down') + def set_mode(self, mode): ifname = self.config['ifname'] cmd = f'ip link set dev {ifname} type macvlan mode {mode}' diff --git a/python/vyos/ifconfig/pppoe.py b/python/vyos/ifconfig/pppoe.py index 63ffc8069..437fe0cae 100644 --- a/python/vyos/ifconfig/pppoe.py +++ b/python/vyos/ifconfig/pppoe.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2020-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -14,6 +14,7 @@ # License along with this library. If not, see <http://www.gnu.org/licenses/>. from vyos.ifconfig.interface import Interface +from vyos.validate import assert_range from vyos.util import get_interface_config @Interface.register @@ -27,6 +28,21 @@ class PPPoEIf(Interface): }, } + _sysfs_get = { + **Interface._sysfs_get,**{ + 'accept_ra_defrtr': { + 'location': '/proc/sys/net/ipv6/conf/{ifname}/accept_ra_defrtr', + } + } + } + + _sysfs_set = {**Interface._sysfs_set, **{ + 'accept_ra_defrtr': { + 'validate': lambda value: assert_range(value, 0, 2), + 'location': '/proc/sys/net/ipv6/conf/{ifname}/accept_ra_defrtr', + }, + }} + def _remove_routes(self, vrf=None): # Always delete default routes when interface is removed vrf_cmd = '' @@ -70,6 +86,21 @@ class PPPoEIf(Interface): """ Get a synthetic MAC address. """ return self.get_mac_synthetic() + def set_accept_ra_defrtr(self, enable): + """ + Learn default router in Router Advertisement. + 1: enabled + 0: disable + + Example: + >>> from vyos.ifconfig import PPPoEIf + >>> PPPoEIf('pppoe1').set_accept_ra_defrtr(0) + """ + tmp = self.get_interface('accept_ra_defrtr') + if tmp == enable: + return None + self.set_interface('accept_ra_defrtr', enable) + def update(self, config): """ General helper function which works on a dictionary retrived by get_config_dict(). It's main intention is to consolidate the scattered @@ -107,6 +138,10 @@ class PPPoEIf(Interface): tmp = config['vrf'] vrf = f'-c "vrf {tmp}"' + # learn default router in Router Advertisement. + tmp = '0' if 'no_default_route' in config else '1' + self.set_accept_ra_defrtr(tmp) + if 'no_default_route' not in config: # Set default route(s) pointing to PPPoE interface distance = config['default_route_distance'] diff --git a/python/vyos/ifconfig/section.py b/python/vyos/ifconfig/section.py index 91f667b65..5e98cd510 100644 --- a/python/vyos/ifconfig/section.py +++ b/python/vyos/ifconfig/section.py @@ -88,7 +88,7 @@ class Section: raise ValueError(f'No type found for interface name: {name}') @classmethod - def _intf_under_section (cls,section=''): + def _intf_under_section (cls,section='',vlan=True): """ return a generator with the name of the configured interface which are under a section @@ -103,6 +103,9 @@ class Section: if section and ifsection != section: continue + if vlan == False and '.' in ifname: + continue + yield ifname @classmethod @@ -135,13 +138,14 @@ class Section: return l @classmethod - def interfaces(cls, section=''): + def interfaces(cls, section='', vlan=True): """ return a list of the name of the configured interface which are under a section - if no section is provided, then it returns all configured interfaces + if no section is provided, then it returns all configured interfaces. + If vlan is True, also Vlan subinterfaces will be returned """ - return cls._sort_interfaces(cls._intf_under_section(section)) + return cls._sort_interfaces(cls._intf_under_section(section, vlan)) @classmethod def _intf_with_feature(cls, feature=''): diff --git a/python/vyos/ifconfig/sstpc.py b/python/vyos/ifconfig/sstpc.py new file mode 100644 index 000000000..50fc6ee6b --- /dev/null +++ b/python/vyos/ifconfig/sstpc.py @@ -0,0 +1,40 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from vyos.ifconfig.interface import Interface + +@Interface.register +class SSTPCIf(Interface): + iftype = 'sstpc' + definition = { + **Interface.definition, + **{ + 'section': 'sstpc', + 'prefixes': ['sstpc', ], + 'eternal': 'sstpc[0-9]+$', + }, + } + + def _create(self): + # we can not create this interface as it is managed outside + pass + + def _delete(self): + # we can not create this interface as it is managed outside + pass + + def get_mac(self): + """ Get a synthetic MAC address. """ + return self.get_mac_synthetic() diff --git a/python/vyos/ifconfig/veth.py b/python/vyos/ifconfig/veth.py new file mode 100644 index 000000000..aafbf226a --- /dev/null +++ b/python/vyos/ifconfig/veth.py @@ -0,0 +1,54 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from vyos.ifconfig.interface import Interface + + +@Interface.register +class VethIf(Interface): + """ + Abstraction of a Linux veth interface + """ + iftype = 'veth' + definition = { + **Interface.definition, + **{ + 'section': 'virtual-ethernet', + 'prefixes': ['veth', ], + 'bridgeable': True, + }, + } + + def _create(self): + """ + Create veth interface in OS kernel. Interface is administrative + down by default. + """ + # check before create, as we have 2 veth interfaces in our CLI + # interface virtual-ethernet veth0 peer-name 'veth1' + # interface virtual-ethernet veth1 peer-name 'veth0' + # + # but iproute2 creates the pair with one command: + # ip link add vet0 type veth peer name veth1 + if self.exists(self.config['peer_name']): + return + + # create virtual-ethernet interface + cmd = 'ip link add {ifname} type {type}'.format(**self.config) + cmd += f' peer name {self.config["peer_name"]}' + self._cmd(cmd) + + # interface is always A/D down. It needs to be enabled explicitly + self.set_admin_state('down') diff --git a/python/vyos/ifconfig/vti.py b/python/vyos/ifconfig/vti.py index c50cd5ce9..dc99d365a 100644 --- a/python/vyos/ifconfig/vti.py +++ b/python/vyos/ifconfig/vti.py @@ -1,4 +1,4 @@ -# Copyright 2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2021-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -53,3 +53,7 @@ class VTIIf(Interface): self._cmd(cmd.format(**self.config)) self.set_interface('admin_state', 'down') + + def get_mac(self): + """ Get a synthetic MAC address. """ + return self.get_mac_synthetic() diff --git a/python/vyos/ifconfig/wireguard.py b/python/vyos/ifconfig/wireguard.py index 28b5e2991..fe5e9c519 100644 --- a/python/vyos/ifconfig/wireguard.py +++ b/python/vyos/ifconfig/wireguard.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -17,11 +17,11 @@ import os import time from datetime import timedelta +from tempfile import NamedTemporaryFile from hurry.filesize import size from hurry.filesize import alternative -from vyos.config import Config from vyos.ifconfig import Interface from vyos.ifconfig import Operational from vyos.template import is_ipv6 @@ -71,10 +71,11 @@ class WireGuardOperational(Operational): return output def show_interface(self): - wgdump = self._dump().get(self.config['ifname'], None) - + from vyos.config import Config c = Config() + wgdump = self._dump().get(self.config['ifname'], None) + c.set_level(["interfaces", "wireguard", self.config['ifname']]) description = c.return_effective_value(["description"]) ips = c.return_effective_values(["address"]) @@ -167,64 +168,66 @@ class WireGuardIf(Interface): # remove no longer associated peers first if 'peer_remove' in config: - for tmp in config['peer_remove']: - peer = config['peer_remove'][tmp] - peer['ifname'] = config['ifname'] + for peer, public_key in config['peer_remove'].items(): + self._cmd(f'wg set {self.ifname} peer {public_key} remove') - cmd = 'wg set {ifname} peer {public_key} remove' - self._cmd(cmd.format(**peer)) - - config['private_key_file'] = '/tmp/tmp.wireguard.key' - with open(config['private_key_file'], 'w') as f: - f.write(config['private_key']) + tmp_file = NamedTemporaryFile('w') + tmp_file.write(config['private_key']) + tmp_file.flush() # Wireguard base command is identical for every peer - base_cmd = 'wg set {ifname} private-key {private_key_file}' + base_cmd = 'wg set {ifname}' if 'port' in config: base_cmd += ' listen-port {port}' if 'fwmark' in config: base_cmd += ' fwmark {fwmark}' + base_cmd += f' private-key {tmp_file.name}' base_cmd = base_cmd.format(**config) - for tmp in config['peer']: - peer = config['peer'][tmp] - - # start of with a fresh 'wg' command - cmd = base_cmd + ' peer {public_key}' - - # If no PSK is given remove it by using /dev/null - passing keys via - # the shell (usually bash) is considered insecure, thus we use a file - no_psk_file = '/dev/null' - psk_file = no_psk_file - if 'preshared_key' in peer: - psk_file = '/tmp/tmp.wireguard.psk' - with open(psk_file, 'w') as f: - f.write(peer['preshared_key']) - cmd += f' preshared-key {psk_file}' - - # Persistent keepalive is optional - if 'persistent_keepalive'in peer: - cmd += ' persistent-keepalive {persistent_keepalive}' - - # Multiple allowed-ip ranges can be defined - ensure we are always - # dealing with a list - if isinstance(peer['allowed_ips'], str): - peer['allowed_ips'] = [peer['allowed_ips']] - cmd += ' allowed-ips ' + ','.join(peer['allowed_ips']) - - # Endpoint configuration is optional - if {'address', 'port'} <= set(peer): - if is_ipv6(peer['address']): - cmd += ' endpoint [{address}]:{port}' - else: - cmd += ' endpoint {address}:{port}' + if 'peer' in config: + for peer, peer_config in config['peer'].items(): + # T4702: No need to configure this peer when it was explicitly + # marked as disabled - also active sessions are terminated as + # the public key was already removed when entering this method! + if 'disable' in peer_config: + continue + + # start of with a fresh 'wg' command + cmd = base_cmd + ' peer {public_key}' + + # If no PSK is given remove it by using /dev/null - passing keys via + # the shell (usually bash) is considered insecure, thus we use a file + no_psk_file = '/dev/null' + psk_file = no_psk_file + if 'preshared_key' in peer_config: + psk_file = '/tmp/tmp.wireguard.psk' + with open(psk_file, 'w') as f: + f.write(peer_config['preshared_key']) + cmd += f' preshared-key {psk_file}' + + # Persistent keepalive is optional + if 'persistent_keepalive'in peer_config: + cmd += ' persistent-keepalive {persistent_keepalive}' + + # Multiple allowed-ip ranges can be defined - ensure we are always + # dealing with a list + if isinstance(peer_config['allowed_ips'], str): + peer_config['allowed_ips'] = [peer_config['allowed_ips']] + cmd += ' allowed-ips ' + ','.join(peer_config['allowed_ips']) + + # Endpoint configuration is optional + if {'address', 'port'} <= set(peer_config): + if is_ipv6(peer_config['address']): + cmd += ' endpoint [{address}]:{port}' + else: + cmd += ' endpoint {address}:{port}' - self._cmd(cmd.format(**peer)) + self._cmd(cmd.format(**peer_config)) - # PSK key file is not required to be stored persistently as its backed by CLI - if psk_file != no_psk_file and os.path.exists(psk_file): - os.remove(psk_file) + # PSK key file is not required to be stored persistently as its backed by CLI + if psk_file != no_psk_file and os.path.exists(psk_file): + os.remove(psk_file) # call base class super().update(config) diff --git a/python/vyos/migrator.py b/python/vyos/migrator.py index c6e3435ca..87c74e1ea 100644 --- a/python/vyos/migrator.py +++ b/python/vyos/migrator.py @@ -1,4 +1,4 @@ -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,11 +16,13 @@ import sys import os import json -import subprocess -import vyos.version +import logging + import vyos.defaults -import vyos.systemversions as systemversions -import vyos.formatversions as formatversions +import vyos.component_version as component_version +from vyos.util import cmd + +log_file = os.path.join(vyos.defaults.directories['config'], 'vyos-migrate.log') class MigratorError(Exception): pass @@ -31,9 +33,21 @@ class Migrator(object): self._force = force self._set_vintage = set_vintage self._config_file_vintage = None - self._log_file = None self._changed = False + def init_logger(self): + self.logger = logging.getLogger(__name__) + self.logger.setLevel(logging.DEBUG) + + # on adding the file handler, allow write permission for cfg_group; + # restore original umask on exit + mask = os.umask(0o113) + fh = logging.FileHandler(log_file) + formatter = logging.Formatter('%(message)s') + fh.setFormatter(formatter) + self.logger.addHandler(fh) + os.umask(mask) + def read_config_file_versions(self): """ Get component versions from config file footer and set vintage; @@ -42,13 +56,13 @@ class Migrator(object): cfg_file = self._config_file component_versions = {} - cfg_versions = formatversions.read_vyatta_versions(cfg_file) + cfg_versions = component_version.from_file(cfg_file, vintage='vyatta') if cfg_versions: self._config_file_vintage = 'vyatta' component_versions = cfg_versions - cfg_versions = formatversions.read_vyos_versions(cfg_file) + cfg_versions = component_version.from_file(cfg_file, vintage='vyos') if cfg_versions: self._config_file_vintage = 'vyos' @@ -70,34 +84,15 @@ class Migrator(object): else: return True - def open_log_file(self): - """ - Open log file for migration, catching any error. - Note that, on boot, migration takes place before the canonical log - directory is created, hence write to the config file directory. - """ - self._log_file = os.path.join(vyos.defaults.directories['config'], - 'vyos-migrate.log') - # on creation, allow write permission for cfg_group; - # restore original umask on exit - mask = os.umask(0o113) - try: - log = open('{0}'.format(self._log_file), 'w') - log.write("List of executed migration scripts:\n") - except Exception as e: - os.umask(mask) - print("Logging error: {0}".format(e)) - return None - - os.umask(mask) - return log - def run_migration_scripts(self, config_file_versions, system_versions): """ Run migration scripts iteratively, until config file version equals system component version. """ - log = self.open_log_file() + os.environ['VYOS_MIGRATION'] = '1' + self.init_logger() + + self.logger.info("List of executed migration scripts:") cfg_versions = config_file_versions sys_versions = system_versions @@ -129,8 +124,9 @@ class Migrator(object): '{}-to-{}'.format(cfg_ver, next_ver)) try: - subprocess.check_call([migrate_script, - self._config_file]) + out = cmd([migrate_script, self._config_file]) + self.logger.info(f'{migrate_script}') + if out: self.logger.info(out) except FileNotFoundError: pass except Exception as err: @@ -138,38 +134,25 @@ class Migrator(object): "".format(migrate_script, err)) sys.exit(1) - if log: - try: - log.write('{0}\n'.format(migrate_script)) - except Exception as e: - print("Error writing log: {0}".format(e)) - cfg_ver = next_ver - rev_versions[key] = cfg_ver - if log: - log.close() - + del os.environ['VYOS_MIGRATION'] return rev_versions def write_config_file_versions(self, cfg_versions): """ Write new versions string. """ - versions_string = formatversions.format_versions_string(cfg_versions) - - os_version_string = vyos.version.get_version() - if self._config_file_vintage == 'vyatta': - formatversions.write_vyatta_versions_foot(self._config_file, - versions_string, - os_version_string) + component_version.write_version_footer(cfg_versions, + self._config_file, + vintage='vyatta') if self._config_file_vintage == 'vyos': - formatversions.write_vyos_versions_foot(self._config_file, - versions_string, - os_version_string) + component_version.write_version_footer(cfg_versions, + self._config_file, + vintage='vyos') def save_json_record(self, component_versions: dict): """ @@ -200,7 +183,7 @@ class Migrator(object): # This will force calling all migration scripts: cfg_versions = {} - sys_versions = systemversions.get_system_component_version() + sys_versions = component_version.from_system() # save system component versions in json file for easy reference self.save_json_record(sys_versions) @@ -216,7 +199,7 @@ class Migrator(object): if not self._changed: return - formatversions.remove_versions(cfg_file) + component_version.remove_footer(cfg_file) self.write_config_file_versions(rev_versions) @@ -237,7 +220,7 @@ class VirtualMigrator(Migrator): if not self._changed: return - formatversions.remove_versions(cfg_file) + component_version.remove_footer(cfg_file) self.write_config_file_versions(cfg_versions) diff --git a/python/vyos/nat.py b/python/vyos/nat.py new file mode 100644 index 000000000..8a311045a --- /dev/null +++ b/python/vyos/nat.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from vyos.template import is_ip_network +from vyos.util import dict_search_args +from vyos.template import bracketize_ipv6 + + +def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): + output = [] + ip_prefix = 'ip6' if ipv6 else 'ip' + log_prefix = ('DST' if nat_type == 'destination' else 'SRC') + f'-NAT-{rule_id}' + log_suffix = '' + + if ipv6: + log_prefix = log_prefix.replace("NAT-", "NAT66-") + + ignore_type_addr = False + translation_str = '' + + if 'inbound_interface' in rule_conf: + ifname = rule_conf['inbound_interface'] + if ifname != 'any': + output.append(f'iifname "{ifname}"') + + if 'outbound_interface' in rule_conf: + ifname = rule_conf['outbound_interface'] + if ifname != 'any': + output.append(f'oifname "{ifname}"') + + if 'protocol' in rule_conf and rule_conf['protocol'] != 'all': + protocol = rule_conf['protocol'] + if protocol == 'tcp_udp': + protocol = '{ tcp, udp }' + output.append(f'meta l4proto {protocol}') + + if 'exclude' in rule_conf: + translation_str = 'return' + log_suffix = '-EXCL' + elif 'translation' in rule_conf: + translation_prefix = nat_type[:1] + translation_output = [f'{translation_prefix}nat'] + addr = dict_search_args(rule_conf, 'translation', 'address') + port = dict_search_args(rule_conf, 'translation', 'port') + + if addr and is_ip_network(addr): + if not ipv6: + map_addr = dict_search_args(rule_conf, nat_type, 'address') + translation_output.append(f'{ip_prefix} prefix to {ip_prefix} {translation_prefix}addr map {{ {map_addr} : {addr} }}') + ignore_type_addr = True + else: + translation_output.append(f'prefix to {addr}') + elif addr == 'masquerade': + if port: + addr = f'{addr} to ' + translation_output = [addr] + log_suffix = '-MASQ' + else: + translation_output.append('to') + if addr: + addr = bracketize_ipv6(addr) + translation_output.append(addr) + + options = [] + addr_mapping = dict_search_args(rule_conf, 'translation', 'options', 'address_mapping') + port_mapping = dict_search_args(rule_conf, 'translation', 'options', 'port_mapping') + if addr_mapping == 'persistent': + options.append('persistent') + if port_mapping and port_mapping != 'none': + options.append(port_mapping) + + translation_str = " ".join(translation_output) + (f':{port}' if port else '') + + if options: + translation_str += f' {",".join(options)}' + + for target in ['source', 'destination']: + if target not in rule_conf: + continue + + side_conf = rule_conf[target] + prefix = target[:1] + + addr = dict_search_args(side_conf, 'address') + if addr and not (ignore_type_addr and target == nat_type): + operator = '' + if addr[:1] == '!': + operator = '!=' + addr = addr[1:] + output.append(f'{ip_prefix} {prefix}addr {operator} {addr}') + + addr_prefix = dict_search_args(side_conf, 'prefix') + if addr_prefix and ipv6: + operator = '' + if addr_prefix[:1] == '!': + operator = '!=' + addr_prefix = addr[1:] + output.append(f'ip6 {prefix}addr {operator} {addr_prefix}') + + port = dict_search_args(side_conf, 'port') + if port: + protocol = rule_conf['protocol'] + if protocol == 'tcp_udp': + protocol = 'th' + operator = '' + if port[:1] == '!': + operator = '!=' + port = port[1:] + output.append(f'{protocol} {prefix}port {operator} {{ {port} }}') + + if 'group' in side_conf: + group = side_conf['group'] + if 'address_group' in group and not (ignore_type_addr and target == nat_type): + group_name = group['address_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_prefix} {prefix}addr {operator} @A_{group_name}') + # Generate firewall group domain-group + elif 'domain_group' in group and not (ignore_type_addr and target == nat_type): + group_name = group['domain_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_prefix} {prefix}addr {operator} @D_{group_name}') + elif 'network_group' in group and not (ignore_type_addr and target == nat_type): + group_name = group['network_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_prefix} {prefix}addr {operator} @N_{group_name}') + if 'mac_group' in group: + group_name = group['mac_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'ether {prefix}addr {operator} @M_{group_name}') + if 'port_group' in group: + proto = rule_conf['protocol'] + group_name = group['port_group'] + + if proto == 'tcp_udp': + proto = 'th' + + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + + output.append(f'{proto} {prefix}port {operator} @P_{group_name}') + + output.append('counter') + + if 'log' in rule_conf: + output.append(f'log prefix "[{log_prefix}{log_suffix}]"') + + if translation_str: + output.append(translation_str) + + output.append(f'comment "{log_prefix}"') + + return " ".join(output) + +def parse_nat_static_rule(rule_conf, rule_id, nat_type): + output = [] + log_prefix = ('STATIC-DST' if nat_type == 'destination' else 'STATIC-SRC') + f'-NAT-{rule_id}' + log_suffix = '' + + ignore_type_addr = False + translation_str = '' + + if 'inbound_interface' in rule_conf: + ifname = rule_conf['inbound_interface'] + ifprefix = 'i' if nat_type == 'destination' else 'o' + if ifname != 'any': + output.append(f'{ifprefix}ifname "{ifname}"') + + if 'exclude' in rule_conf: + translation_str = 'return' + log_suffix = '-EXCL' + elif 'translation' in rule_conf: + translation_prefix = nat_type[:1] + translation_output = [f'{translation_prefix}nat'] + addr = dict_search_args(rule_conf, 'translation', 'address') + map_addr = dict_search_args(rule_conf, 'destination', 'address') + + if nat_type == 'source': + addr, map_addr = map_addr, addr # Swap + + if addr and is_ip_network(addr): + translation_output.append(f'ip prefix to ip {translation_prefix}addr map {{ {map_addr} : {addr} }}') + ignore_type_addr = True + elif addr: + translation_output.append(f'to {addr}') + + options = [] + addr_mapping = dict_search_args(rule_conf, 'translation', 'options', 'address_mapping') + port_mapping = dict_search_args(rule_conf, 'translation', 'options', 'port_mapping') + if addr_mapping == 'persistent': + options.append('persistent') + if port_mapping and port_mapping != 'none': + options.append(port_mapping) + + if options: + translation_output.append(",".join(options)) + + translation_str = " ".join(translation_output) + + prefix = nat_type[:1] + addr = dict_search_args(rule_conf, 'translation' if nat_type == 'source' else nat_type, 'address') + if addr and not ignore_type_addr: + output.append(f'ip {prefix}addr {addr}') + + output.append('counter') + + if translation_str: + output.append(translation_str) + + if 'log' in rule_conf: + output.append(f'log prefix "[{log_prefix}{log_suffix}]"') + + output.append(f'comment "{log_prefix}"') + + return " ".join(output) diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py new file mode 100644 index 000000000..5ff768859 --- /dev/null +++ b/python/vyos/opmode.py @@ -0,0 +1,224 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +import re +import sys +import typing +from humps import decamelize + + +class Error(Exception): + """ Any error that makes requested operation impossible to complete + for reasons unrelated to the user input or script logic. + """ + pass + +class UnconfiguredSubsystem(Error): + """ Requested operation is valid, but cannot be completed + because corresponding subsystem is not configured and running. + """ + pass + +class DataUnavailable(Error): + """ Requested operation is valid, but cannot be completed + because data for it is not available. + This error MAY be treated as temporary because such issues + are often caused by transient events such as service restarts. + """ + pass + +class PermissionDenied(Error): + """ Requested operation is valid, but the caller has no permission + to perform it. + """ + pass + +class IncorrectValue(Error): + """ Requested operation is valid, but an argument provided has an + incorrect value, preventing successful completion. + """ + pass + +class CommitInProgress(Error): + """ Requested operation is valid, but not possible at the time due + to a commit being in progress. + """ + pass + +class InternalError(Error): + """ Any situation when VyOS detects that it could not perform + an operation correctly due to logic errors in its own code + or errors in underlying software. + """ + pass + + +def _is_op_mode_function_name(name): + if re.match(r"^(show|clear|reset|restart)", name): + return True + else: + return False + +def _is_show(name): + if re.match(r"^show", name): + return True + else: + return False + +def _get_op_mode_functions(module): + from inspect import getmembers, isfunction + + # Get all functions in that module + funcs = getmembers(module, isfunction) + + # getmembers returns (name, func) tuples + funcs = list(filter(lambda ft: _is_op_mode_function_name(ft[0]), funcs)) + + funcs_dict = {} + for (name, thunk) in funcs: + funcs_dict[name] = thunk + + return funcs_dict + +def _is_optional_type(t): + # Optional[t] is internally an alias for Union[t, NoneType] + # and there's no easy way to get union members it seems + if (type(t) == typing._UnionGenericAlias): + if (len(t.__args__) == 2): + if t.__args__[1] == type(None): + return True + + return False + +def _get_arg_type(t): + """ Returns the type itself if it's a primitive type, + or the "real" type of typing.Optional + + Doesn't work with anything else at the moment! + """ + if _is_optional_type(t): + return t.__args__[0] + else: + return t + +def _normalize_field_name(name): + # Convert the name to string if it is not + # (in some cases they may be numbers) + name = str(name) + + # Replace all separators with underscores + name = re.sub(r'(\s|[\(\)\[\]\{\}\-\.\,:\"\'\`])+', '_', name) + + # Replace specific characters with textual descriptions + name = re.sub(r'@', '_at_', name) + name = re.sub(r'%', '_percentage_', name) + name = re.sub(r'~', '_tilde_', name) + + # Force all letters to lowercase + name = name.lower() + + # Remove leading and trailing underscores, if any + name = re.sub(r'(^(_+)(?=[^_])|_+$)', '', name) + + # Ensure there are only single underscores + name = re.sub(r'_+', '_', name) + + return name + +def _normalize_dict_field_names(old_dict): + new_dict = {} + + for key in old_dict: + new_key = _normalize_field_name(key) + new_dict[new_key] = _normalize_field_names(old_dict[key]) + + # Sanity check + if len(old_dict) != len(new_dict): + raise InternalError("Dictionary fields do not allow unique normalization") + else: + return new_dict + +def _normalize_field_names(value): + if isinstance(value, dict): + return _normalize_dict_field_names(value) + elif isinstance(value, list): + return list(map(lambda v: _normalize_field_names(v), value)) + else: + return value + +def run(module): + from argparse import ArgumentParser + + functions = _get_op_mode_functions(module) + + parser = ArgumentParser() + subparsers = parser.add_subparsers(dest="subcommand") + + for function_name in functions: + subparser = subparsers.add_parser(function_name, help=functions[function_name].__doc__) + + type_hints = typing.get_type_hints(functions[function_name]) + if 'return' in type_hints: + del type_hints['return'] + for opt in type_hints: + th = type_hints[opt] + + if _get_arg_type(th) == bool: + subparser.add_argument(f"--{opt}", action='store_true') + else: + if _is_optional_type(th): + subparser.add_argument(f"--{opt}", type=_get_arg_type(th), default=None) + else: + subparser.add_argument(f"--{opt}", type=_get_arg_type(th), required=True) + + # Get options as a dict rather than a namespace, + # so that we can modify it and pack for passing to functions + args = vars(parser.parse_args()) + + if not args["subcommand"]: + print("Subcommand required!") + parser.print_usage() + sys.exit(1) + + function_name = args["subcommand"] + func = functions[function_name] + + # Remove the subcommand from the arguments, + # it would cause an extra argument error when we pass the dict to a function + del args["subcommand"] + + # Show commands must always get the "raw" argument, + # but other commands (clear/reset/restart) should not, + # because they produce no output and it makes no sense for them. + if ("raw" not in args) and _is_show(function_name): + args["raw"] = False + + if re.match(r"^show", function_name): + # Show commands are slightly special: + # they may return human-formatted output + # or a raw dict that we need to serialize in JSON for printing + res = func(**args) + if not args["raw"]: + return res + else: + res = decamelize(res) + res = _normalize_field_names(res) + from json import dumps + return dumps(res, indent=4) + else: + # Other functions should not return anything, + # although they may print their own warnings or status messages + func(**args) + diff --git a/python/vyos/pki.py b/python/vyos/pki.py index fd91fc9bf..cd15e3878 100644 --- a/python/vyos/pki.py +++ b/python/vyos/pki.py @@ -332,6 +332,54 @@ def verify_certificate(cert, ca_cert): except InvalidSignature: return False +def verify_crl(crl, ca_cert): + # Verify CRL was signed by specified CA + if ca_cert.subject != crl.issuer: + return False + + ca_public_key = ca_cert.public_key() + try: + if isinstance(ca_public_key, rsa.RSAPublicKeyWithSerialization): + ca_public_key.verify( + crl.signature, + crl.tbs_certlist_bytes, + padding=padding.PKCS1v15(), + algorithm=crl.signature_hash_algorithm) + elif isinstance(ca_public_key, dsa.DSAPublicKeyWithSerialization): + ca_public_key.verify( + crl.signature, + crl.tbs_certlist_bytes, + algorithm=crl.signature_hash_algorithm) + elif isinstance(ca_public_key, ec.EllipticCurvePublicKeyWithSerialization): + ca_public_key.verify( + crl.signature, + crl.tbs_certlist_bytes, + signature_algorithm=ec.ECDSA(crl.signature_hash_algorithm)) + else: + return False # We cannot verify it + return True + except InvalidSignature: + return False + +def verify_ca_chain(sorted_names, pki_node): + if len(sorted_names) == 1: # Single cert, no chain + return True + + for name in sorted_names: + cert = load_certificate(pki_node[name]['certificate']) + verified = False + for ca_name in sorted_names: + if name == ca_name: + continue + ca_cert = load_certificate(pki_node[ca_name]['certificate']) + if verify_certificate(cert, ca_cert): + verified = True + break + if not verified and name != sorted_names[-1]: + # Only permit top-most certificate to fail verify (e.g. signed by public CA not explicitly in chain) + return False + return True + # Certificate chain def find_parent(cert, ca_certs): @@ -357,3 +405,16 @@ def find_chain(cert, ca_certs): chain.append(parent) return chain + +def sort_ca_chain(ca_names, pki_node): + def ca_cmp(ca_name1, ca_name2, pki_node): + cert1 = load_certificate(pki_node[ca_name1]['certificate']) + cert2 = load_certificate(pki_node[ca_name2]['certificate']) + + if verify_certificate(cert1, cert2): # cert1 is child of cert2 + return -1 + return 1 + + from functools import cmp_to_key + return sorted(ca_names, key=cmp_to_key(lambda cert1, cert2: ca_cmp(cert1, cert2, pki_node))) + diff --git a/python/vyos/systemversions.py b/python/vyos/systemversions.py deleted file mode 100644 index f2da76d4f..000000000 --- a/python/vyos/systemversions.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see <http://www.gnu.org/licenses/>. - -import os -import re -import sys -import vyos.defaults -from vyos.xml import component_version - -# legacy version, reading from the file names in -# /opt/vyatta/etc/config-migrate/current -def get_system_versions(): - """ - Get component versions from running system; critical failure if - unable to read migration directory. - """ - system_versions = {} - - try: - version_info = os.listdir(vyos.defaults.directories['current']) - except OSError as err: - print("OS error: {}".format(err)) - sys.exit(1) - - for info in version_info: - if re.match(r'[\w,-]+@\d+', info): - pair = info.split('@') - system_versions[pair[0]] = int(pair[1]) - - return system_versions - -# read from xml cache -def get_system_component_version(): - return component_version() diff --git a/python/vyos/template.py b/python/vyos/template.py index 132f5ddde..2a4135f9e 100644 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -1,4 +1,4 @@ -# Copyright 2019-2020 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -548,24 +548,35 @@ def nft_rule(rule_conf, fw_name, rule_id, ip_name='ip'): return parse_rule(rule_conf, fw_name, rule_id, ip_name) @register_filter('nft_default_rule') -def nft_default_rule(fw_conf, fw_name): +def nft_default_rule(fw_conf, fw_name, ipv6=False): output = ['counter'] - default_action = fw_conf.get('default_action', 'accept') + default_action = fw_conf['default_action'] if 'enable_default_log' in fw_conf: action_suffix = default_action[:1].upper() - output.append(f'log prefix "[{fw_name[:19]}-default-{action_suffix}] "') + output.append(f'log prefix "[{fw_name[:19]}-default-{action_suffix}]"') output.append(nft_action(default_action)) + if 'default_jump_target' in fw_conf: + target = fw_conf['default_jump_target'] + def_suffix = '6' if ipv6 else '' + output.append(f'NAME{def_suffix}_{target}') + output.append(f'comment "{fw_name} default-action {default_action}"') return " ".join(output) @register_filter('nft_state_policy') -def nft_state_policy(conf, state, ipv6=False): +def nft_state_policy(conf, state): out = [f'ct state {state}'] if 'log' in conf and 'enable' in conf['log']: - out.append('log') + log_state = state[:3].upper() + log_action = (conf['action'] if 'action' in conf else 'accept')[:1].upper() + out.append(f'log prefix "[STATE-POLICY-{log_state}-{log_action}]"') + + if 'log_level' in conf: + log_level = conf['log_level'] + out.append(f'level {log_level}') out.append('counter') @@ -590,6 +601,45 @@ def nft_intra_zone_action(zone_conf, ipv6=False): return f'jump {name_prefix}{name}' return 'return' +@register_filter('nft_nested_group') +def nft_nested_group(out_list, includes, groups, key): + if not vyos_defined(out_list): + out_list = [] + + def add_includes(name): + if key in groups[name]: + for item in groups[name][key]: + if item in out_list: + continue + out_list.append(item) + + if 'include' in groups[name]: + for name_inc in groups[name]['include']: + add_includes(name_inc) + + for name in includes: + add_includes(name) + return out_list + +@register_filter('nat_rule') +def nat_rule(rule_conf, rule_id, nat_type, ipv6=False): + from vyos.nat import parse_nat_rule + return parse_nat_rule(rule_conf, rule_id, nat_type, ipv6) + +@register_filter('nat_static_rule') +def nat_static_rule(rule_conf, rule_id, nat_type): + from vyos.nat import parse_nat_static_rule + return parse_nat_static_rule(rule_conf, rule_id, nat_type) + +@register_filter('range_to_regex') +def range_to_regex(num_range): + from vyos.range_regex import range_to_regex + if '-' not in num_range: + return num_range + + regex = range_to_regex(num_range) + return f'({regex})' + @register_test('vyos_defined') def vyos_defined(value, test_value=None, var_type=None): """ diff --git a/python/vyos/util.py b/python/vyos/util.py index 0d62fbfe9..6a828c0ac 100644 --- a/python/vyos/util.py +++ b/python/vyos/util.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2020-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -164,6 +164,27 @@ def cmd(command, flag='', shell=None, input=None, timeout=None, env=None, return decoded +def rc_cmd(command, flag='', shell=None, input=None, timeout=None, env=None, + stdout=PIPE, stderr=STDOUT, decode='utf-8'): + """ + A wrapper around popen, which returns the return code + of a command and stdout + + % rc_cmd('uname') + (0, 'Linux') + % rc_cmd('ip link show dev eth99') + (1, 'Device "eth99" does not exist.') + """ + out, code = popen( + command, flag, + stdout=stdout, stderr=stderr, + input=input, timeout=timeout, + env=env, shell=shell, + decode=decode, + ) + return code, out + + def call(command, flag='', shell=None, input=None, timeout=None, env=None, stdout=PIPE, stderr=PIPE, decode='utf-8'): """ @@ -197,7 +218,7 @@ def read_file(fname, defaultonfailure=None): return defaultonfailure raise e -def write_file(fname, data, defaultonfailure=None, user=None, group=None, mode=None): +def write_file(fname, data, defaultonfailure=None, user=None, group=None, mode=None, append=False): """ Write content of data to given fname, should defaultonfailure be not None, it is returned on failure to read. @@ -212,7 +233,7 @@ def write_file(fname, data, defaultonfailure=None, user=None, group=None, mode=N try: """ Write a file to string """ bytes = 0 - with open(fname, 'w') as f: + with open(fname, 'w' if not append else 'a') as f: bytes = f.write(data) chown(fname, user, group) chmod(fname, mode) @@ -450,6 +471,35 @@ def process_named_running(name): return p.pid return None +def is_list_equal(first: list, second: list) -> bool: + """ Check if 2 lists are equal and list not empty """ + if len(first) != len(second) or len(first) == 0: + return False + return sorted(first) == sorted(second) + +def is_listen_port_bind_service(port: int, service: str) -> bool: + """Check if listen port bound to expected program name + :param port: Bind port + :param service: Program name + :return: bool + + Example: + % is_listen_port_bind_service(443, 'nginx') + True + % is_listen_port_bind_service(443, 'ocservr-main') + False + """ + from psutil import net_connections as connections + from psutil import Process as process + for connection in connections(): + addr = connection.laddr + pid = connection.pid + pid_name = process(pid).name() + pid_port = addr.port + if service == pid_name and port == pid_port: + return True + return False + def seconds_to_human(s, separator=""): """ Converts number of seconds passed to a human-readable interval such as 1w4d18h35m59s @@ -489,13 +539,16 @@ def seconds_to_human(s, separator=""): return result -def bytes_to_human(bytes, initial_exponent=0): +def bytes_to_human(bytes, initial_exponent=0, precision=2): """ Converts a value in bytes to a human-readable size string like 640 KB The initial_exponent parameter is the exponent of 2, e.g. 10 (1024) for kilobytes, 20 (1024 * 1024) for megabytes. """ + if bytes == 0: + return "0 B" + from math import log2 bytes = bytes * (2**initial_exponent) @@ -521,9 +574,40 @@ def bytes_to_human(bytes, initial_exponent=0): # Add a new case when the first machine with petabyte RAM # hits the market. - size_string = "{0:.2f} {1}".format(value, suffix) + size_string = "{0:.{1}f} {2}".format(value, precision, suffix) return size_string +def human_to_bytes(value): + """ Converts a data amount with a unit suffix to bytes, like 2K to 2048 """ + + from re import match as re_match + + res = re_match(r'^\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]+)\s*$', value) + + if not res: + raise ValueError(f"'{value}' is not a valid data amount") + else: + amount = float(res.group(1)) + unit = res.group(2).lower() + + if unit == 'b': + res = amount + elif (unit == 'k') or (unit == 'kb'): + res = amount * 1024 + elif (unit == 'm') or (unit == 'mb'): + res = amount * 1024**2 + elif (unit == 'g') or (unit == 'gb'): + res = amount * 1024**3 + elif (unit == 't') or (unit == 'tb'): + res = amount * 1024**4 + else: + raise ValueError(f"Unsupported data unit '{unit}'") + + # There cannot be fractional bytes, so we convert them to integer. + # However, truncating causes problems with conversion back to human unit, + # so we round instead -- that seems to work well enough. + return round(res) + def get_cfg_group_id(): from grp import getgrnam from vyos.defaults import cfg_group @@ -779,6 +863,32 @@ def dict_search_recursive(dict_object, key, path=[]): for x in dict_search_recursive(j, key, new_path): yield x +def convert_data(data): + """Convert multiple types of data to types usable in CLI + + Args: + data (str | bytes | list | OrderedDict): input data + + Returns: + str | list | dict: converted data + """ + from collections import OrderedDict + + if isinstance(data, str): + return data + if isinstance(data, bytes): + return data.decode() + if isinstance(data, list): + list_tmp = [] + for item in data: + list_tmp.append(convert_data(item)) + return list_tmp + if isinstance(data, OrderedDict): + dict_tmp = {} + for key, value in data.items(): + dict_tmp[key] = convert_data(value) + return dict_tmp + def get_bridge_fdb(interface): """ Returns the forwarding database entries for a given interface """ if not os.path.exists(f'/sys/class/net/{interface}'): @@ -1029,3 +1139,18 @@ def sysctl_write(name, value): call(f'sysctl -wq {name}={value}') return True return False + +# approach follows a discussion in: +# https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case +def camel_to_snake_case(name: str) -> str: + pattern = r'\d+|[A-Z]?[a-z]+|\W|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)' + words = re.findall(pattern, name) + return '_'.join(map(str.lower, words)) + +def load_as_module(name: str, path: str): + import importlib.util + + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod diff --git a/python/vyos/validate.py b/python/vyos/validate.py index e005da0e4..a83193363 100644 --- a/python/vyos/validate.py +++ b/python/vyos/validate.py @@ -264,3 +264,22 @@ def has_address_configured(conf, intf): conf.set_level(old_level) return ret + +def has_vrf_configured(conf, intf): + """ + Checks if interface has a VRF configured. + + Returns True if interface has VRF configured, False if it doesn't. + """ + from vyos.ifconfig import Section + ret = False + + old_level = conf.get_level() + conf.set_level([]) + + tmp = ['interfaces', Section.get_config_path(intf), 'vrf'] + if conf.exists(tmp): + ret = True + + conf.set_level(old_level) + return ret diff --git a/python/vyos/version.py b/python/vyos/version.py index 871bb0f1b..fb706ad44 100644 --- a/python/vyos/version.py +++ b/python/vyos/version.py @@ -31,6 +31,7 @@ Example of the version data dict:: import os import json +import requests import vyos.defaults from vyos.util import read_file @@ -105,3 +106,41 @@ def get_full_version_data(fname=version_file): version_data['hardware_uuid'] = read_file(subsystem + '/product_uuid', 'Unknown') return version_data + +def get_remote_version(url): + """ + Get remote available JSON file from remote URL + An example of the image-version.json + + [ + { + "arch":"amd64", + "flavors":[ + "generic" + ], + "image":"vyos-rolling-latest.iso", + "latest":true, + "lts":false, + "release_date":"2022-09-06", + "release_train":"sagitta", + "url":"http://xxx/rolling/current/vyos-rolling-latest.iso", + "version":"vyos-1.4-rolling-202209060217" + } + ] + """ + headers = {} + try: + remote_data = requests.get(url=url, headers=headers) + remote_data.raise_for_status() + if remote_data.status_code != 200: + return False + return remote_data.json() + except requests.exceptions.HTTPError as errh: + print ("HTTP Error:", errh) + except requests.exceptions.ConnectionError as errc: + print ("Connecting error:", errc) + except requests.exceptions.Timeout as errt: + print ("Timeout error:", errt) + except requests.exceptions.RequestException as err: + print ("Unable to get remote data", err) + return False |