diff options
Diffstat (limited to 'python')
34 files changed, 1588 insertions, 395 deletions
diff --git a/python/vyos/base.py b/python/vyos/base.py index 4e23714e5..78067d5b2 100644 --- a/python/vyos/base.py +++ b/python/vyos/base.py @@ -1,4 +1,4 @@ -# Copyright 2018 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2018-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,6 +13,23 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. +from textwrap import fill + +class Warning(): + def __init__(self, message): + # Reformat the message and trim it to 72 characters in length + message = fill(message, width=72) + print(f'\nWARNING: {message}') + +class DeprecationWarning(): + def __init__(self, message): + # Reformat the message and trim it to 72 characters in length + message = fill(message, width=72) + print(f'\nDEPRECATION WARNING: {message}\n') class ConfigError(Exception): - pass + def __init__(self, message): + # Reformat the message and trim it to 72 characters in length + message = fill(message, width=72) + # Call the base class constructor with the parameters it needs + super().__init__(message) diff --git a/python/vyos/config.py b/python/vyos/config.py index a5c1ad122..287fd2ed1 100644 --- a/python/vyos/config.py +++ b/python/vyos/config.py @@ -142,31 +142,41 @@ class Config(object): def exists(self, path): """ - Checks if a node with given path exists in the running or proposed config + Checks if a node or value with given path exists in the proposed config. + + Args: + path (str): Configuration tree path Returns: - True if node exists, False otherwise + True if node or value exists in the proposed config, False otherwise Note: - This function cannot be used outside a configuration sessions. + This function should not be used outside of configuration sessions. In operational mode scripts, use ``exists_effective``. """ - if not self._session_config: + if self._session_config is None: return False + + # Assume the path is a node path first if self._session_config.exists(self._make_path(path)): return True else: + # If that check fails, it may mean the path has a value at the end. # libvyosconfig exists() works only for _nodes_, not _values_ - # libvyattacfg one also worked for values, so we emulate that case here + # libvyattacfg also worked for values, so we emulate that case here if isinstance(path, str): path = re.split(r'\s+', path) path_without_value = path[:-1] - path_str = " ".join(path_without_value) try: - value = self._session_config.return_value(self._make_path(path_str)) - return (value == path[-1]) + # return_values() is safe to use with single-value nodes, + # it simply returns a single-item list in that case. + values = self._session_config.return_values(self._make_path(path_without_value)) + + # If we got this far, the node does exist and has values, + # so we need to check if it has the value in question among its values. + return (path[-1] in values) except vyos.configtree.ConfigTreeError: - # node doesn't exist at all + # Even the parent node doesn't exist at all return False def session_changed(self): @@ -380,7 +390,7 @@ class Config(object): def exists_effective(self, path): """ - Check if a node exists in the running (effective) config + Checks if a node or value exists in the running (effective) config. Args: path (str): Configuration tree path @@ -392,10 +402,31 @@ class Config(object): This function is safe to use in operational mode. In configuration mode, it ignores uncommited changes. """ - if self._running_config: - return(self._running_config.exists(self._make_path(path))) + if self._running_config is None: + return False + + # Assume the path is a node path first + if self._running_config.exists(self._make_path(path)): + return True + else: + # If that check fails, it may mean the path has a value at the end. + # libvyosconfig exists() works only for _nodes_, not _values_ + # libvyattacfg also worked for values, so we emulate that case here + if isinstance(path, str): + path = re.split(r'\s+', path) + path_without_value = path[:-1] + try: + # return_values() is safe to use with single-value nodes, + # it simply returns a single-item list in that case. + values = self._running_config.return_values(self._make_path(path_without_value)) + + # If we got this far, the node does exist and has values, + # so we need to check if it has the value in question among its values. + return (path[-1] in values) + except vyos.configtree.ConfigTreeError: + # Even the parent node doesn't exist at all + return False - return False def return_effective_value(self, path, default=None): """ diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py index 425a2e416..04ddc10e9 100644 --- a/python/vyos/configdict.py +++ b/python/vyos/configdict.py @@ -1,4 +1,4 @@ -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -104,6 +104,11 @@ def list_diff(first, second): second = set(second) return [item for item in first if item not in second] +def is_node_changed(conf, path): + from vyos.configdiff import get_config_diff + D = get_config_diff(conf, key_mangling=('-', '_')) + return D.is_node_changed(path) + def leaf_node_changed(conf, path): """ Check if a leaf node was altered. If it has been altered - values has been @@ -114,9 +119,14 @@ def leaf_node_changed(conf, path): """ from vyos.configdiff import get_config_diff D = get_config_diff(conf, key_mangling=('-', '_')) - D.set_level(conf.get_level()) (new, old) = D.get_value_diff(path) if new != old: + if isinstance(old, dict): + # valueLess nodes return {} if node is deleted + return True + if old is None and isinstance(new, dict): + # valueLess nodes return {} if node was added + return True if old is None: return [] if isinstance(old, str): @@ -130,7 +140,7 @@ def leaf_node_changed(conf, path): return None -def node_changed(conf, path, key_mangling=None): +def node_changed(conf, path, key_mangling=None, recursive=False): """ Check if a leaf node was altered. If it has been altered - values has been changed, or it was added/removed, we will return the old value. If nothing @@ -138,12 +148,11 @@ def node_changed(conf, path, key_mangling=None): """ from vyos.configdiff import get_config_diff, Diff D = get_config_diff(conf, key_mangling) - D.set_level(conf.get_level()) # get_child_nodes() will return dict_keys(), mangle this into a list with PEP448 - keys = D.get_child_nodes_diff(path, expand_nodes=Diff.DELETE)['delete'].keys() + keys = D.get_child_nodes_diff(path, expand_nodes=Diff.DELETE, recursive=recursive)['delete'].keys() return list(keys) -def get_removed_vlans(conf, dict): +def get_removed_vlans(conf, path, dict): """ Common function to parse a dictionary retrieved via get_config_dict() and determine any added/removed VLAN interfaces - be it 802.1q or Q-in-Q. @@ -153,16 +162,17 @@ def get_removed_vlans(conf, dict): # Check vif, vif-s/vif-c VLAN interfaces for removal D = get_config_diff(conf, key_mangling=('-', '_')) D.set_level(conf.get_level()) + # get_child_nodes() will return dict_keys(), mangle this into a list with PEP448 - keys = D.get_child_nodes_diff(['vif'], expand_nodes=Diff.DELETE)['delete'].keys() + keys = D.get_child_nodes_diff(path + ['vif'], expand_nodes=Diff.DELETE)['delete'].keys() if keys: dict['vif_remove'] = [*keys] # get_child_nodes() will return dict_keys(), mangle this into a list with PEP448 - keys = D.get_child_nodes_diff(['vif-s'], expand_nodes=Diff.DELETE)['delete'].keys() + keys = D.get_child_nodes_diff(path + ['vif-s'], expand_nodes=Diff.DELETE)['delete'].keys() if keys: dict['vif_s_remove'] = [*keys] for vif in dict.get('vif_s', {}).keys(): - keys = D.get_child_nodes_diff(['vif-s', vif, 'vif-c'], expand_nodes=Diff.DELETE)['delete'].keys() + keys = D.get_child_nodes_diff(path + ['vif-s', vif, 'vif-c'], expand_nodes=Diff.DELETE)['delete'].keys() if keys: dict['vif_s'][vif]['vif_c_remove'] = [*keys] return dict @@ -196,7 +206,7 @@ def is_member(conf, interface, intftype=None): interface name -> Interface is a member of this interface False -> interface type cannot have members """ - ret_val = None + ret_val = {} intftypes = ['bonding', 'bridge'] if intftype not in intftypes + [None]: @@ -206,20 +216,15 @@ def is_member(conf, interface, intftype=None): intftype = intftypes if intftype == None else [intftype] - # set config level to root - old_level = conf.get_level() - conf.set_level([]) - for iftype in intftype: base = ['interfaces', iftype] for intf in conf.list_nodes(base): member = base + [intf, 'member', 'interface', interface] if conf.exists(member): tmp = conf.get_config_dict(member, key_mangling=('-', '_'), - get_first_key=True) - ret_val = {intf : tmp} + get_first_key=True, no_tag_node_value_mangle=True) + ret_val.update({intf : tmp}) - old_level = conf.set_level(old_level) return ret_val def is_mirror_intf(conf, interface, direction=None): @@ -241,8 +246,6 @@ def is_mirror_intf(conf, interface, direction=None): direction = directions if direction == None else [direction] ret_val = None - old_level = conf.get_level() - conf.set_level([]) base = ['interfaces'] for dir in direction: @@ -256,7 +259,6 @@ def is_mirror_intf(conf, interface, direction=None): get_first_key=True) ret_val = {intf : tmp} - old_level = conf.set_level(old_level) return ret_val def has_vlan_subinterface_configured(conf, intf): @@ -270,15 +272,11 @@ def has_vlan_subinterface_configured(conf, intf): from vyos.ifconfig import Section ret = False - old_level = conf.get_level() - conf.set_level([]) - intfpath = ['interfaces', Section.section(intf), intf] if ( conf.exists(intfpath + ['vif']) or conf.exists(intfpath + ['vif-s'])): ret = True - conf.set_level(old_level) return ret def is_source_interface(conf, interface, intftype=None): @@ -300,11 +298,6 @@ def is_source_interface(conf, interface, intftype=None): 'have a source-interface') intftype = intftypes if intftype == None else [intftype] - - # set config level to root - old_level = conf.get_level() - conf.set_level([]) - for it in intftype: base = ['interfaces', it] for intf in conf.list_nodes(base): @@ -313,43 +306,77 @@ def is_source_interface(conf, interface, intftype=None): ret_val = intf break - old_level = conf.set_level(old_level) return ret_val def get_dhcp_interfaces(conf, vrf=None): """ Common helper functions to retrieve all interfaces from current CLI sessions that have DHCP configured. """ - dhcp_interfaces = [] + dhcp_interfaces = {} dict = conf.get_config_dict(['interfaces'], get_first_key=True) if not dict: return dhcp_interfaces - def check_dhcp(config, ifname): - out = [] + def check_dhcp(config): + ifname = config['ifname'] + tmp = {} if 'address' in config and 'dhcp' in config['address']: + options = {} + if dict_search('dhcp_options.default_route_distance', config) != None: + options.update({'dhcp_options' : config['dhcp_options']}) if 'vrf' in config: - if vrf is config['vrf']: out.append(ifname) - else: out.append(ifname) - return out + if vrf is config['vrf']: tmp.update({ifname : options}) + else: tmp.update({ifname : options}) + + return tmp for section, interface in dict.items(): - for ifname, ifconfig in interface.items(): - tmp = check_dhcp(ifconfig, ifname) - dhcp_interfaces.extend(tmp) + for ifname in interface: + # always reset config level, as get_interface_dict() will alter it + conf.set_level([]) + # we already have a dict representation of the config from get_config_dict(), + # but with the extended information from get_interface_dict() we also + # get the DHCP client default-route-distance default option if not specified. + _, ifconfig = get_interface_dict(conf, ['interfaces', section], ifname) + + tmp = check_dhcp(ifconfig) + dhcp_interfaces.update(tmp) # check per VLAN interfaces for vif, vif_config in ifconfig.get('vif', {}).items(): - tmp = check_dhcp(vif_config, f'{ifname}.{vif}') - dhcp_interfaces.extend(tmp) + tmp = check_dhcp(vif_config) + dhcp_interfaces.update(tmp) # check QinQ VLAN interfaces - for vif_s, vif_s_config in ifconfig.get('vif-s', {}).items(): - tmp = check_dhcp(vif_s_config, f'{ifname}.{vif_s}') - dhcp_interfaces.extend(tmp) - for vif_c, vif_c_config in vif_s_config.get('vif-c', {}).items(): - tmp = check_dhcp(vif_c_config, f'{ifname}.{vif_s}.{vif_c}') - dhcp_interfaces.extend(tmp) + for vif_s, vif_s_config in ifconfig.get('vif_s', {}).items(): + tmp = check_dhcp(vif_s_config) + dhcp_interfaces.update(tmp) + for vif_c, vif_c_config in vif_s_config.get('vif_c', {}).items(): + tmp = check_dhcp(vif_c_config) + dhcp_interfaces.update(tmp) return dhcp_interfaces +def get_pppoe_interfaces(conf, vrf=None): + """ Common helper functions to retrieve all interfaces from current CLI + sessions that have DHCP configured. """ + pppoe_interfaces = {} + for ifname in conf.list_nodes(['interfaces', 'pppoe']): + # always reset config level, as get_interface_dict() will alter it + conf.set_level([]) + # we already have a dict representation of the config from get_config_dict(), + # but with the extended information from get_interface_dict() we also + # get the DHCP client default-route-distance default option if not specified. + ifconfig = get_interface_dict(conf, ['interfaces', 'pppoe'], ifname) + + options = {} + if 'default_route_distance' in ifconfig: + options.update({'default_route_distance' : ifconfig['default_route_distance']}) + if 'no_default_route' in ifconfig: + options.update({'no_default_route' : {}}) + if 'vrf' in ifconfig: + if vrf is ifconfig['vrf']: pppoe_interfaces.update({ifname : options}) + else: pppoe_interfaces.update({ifname : options}) + + return pppoe_interfaces + def get_interface_dict(config, base, ifname=''): """ Common utility function to retrieve and mangle the interfaces configuration @@ -359,7 +386,6 @@ def get_interface_dict(config, base, ifname=''): Return a dictionary with the necessary interface config keys. """ - if not ifname: from vyos import ConfigError # determine tagNode instance @@ -376,9 +402,8 @@ def get_interface_dict(config, base, ifname=''): for vif in ['vif', 'vif_s']: if vif in default_values: del default_values[vif] - # setup config level which is extracted in get_removed_vlans() - config.set_level(base + [ifname]) - dict = config.get_config_dict([], key_mangling=('-', '_'), get_first_key=True, + dict = config.get_config_dict(base + [ifname], key_mangling=('-', '_'), + get_first_key=True, no_tag_node_value_mangle=True) # Check if interface has been removed. We must use exists() as @@ -386,8 +411,8 @@ def get_interface_dict(config, base, ifname=''): # node like the following exists. # +macsec macsec1 { # +} - if not config.exists([]): - dict.update({'deleted' : ''}) + if not config.exists(base + [ifname]): + dict.update({'deleted' : {}}) # Add interface instance name into dictionary dict.update({'ifname': ifname}) @@ -405,10 +430,16 @@ def get_interface_dict(config, base, ifname=''): if 'deleted' not in dict: dict = dict_merge(default_values, dict) + # If interface does not request an IPv4 DHCP address there is no need + # to keep the dhcp-options key + if 'address' not in dict or 'dhcp' not in dict['address']: + if 'dhcp_options' in dict: + del dict['dhcp_options'] + # XXX: T2665: blend in proper DHCPv6-PD default values dict = T2665_set_dhcpv6pd_defaults(dict) - address = leaf_node_changed(config, ['address']) + address = leaf_node_changed(config, base + [ifname, 'address']) if address: dict.update({'address_old' : address}) # Check if we are a member of a bridge device @@ -423,6 +454,10 @@ def get_interface_dict(config, base, ifname=''): bond = is_member(config, ifname, 'bonding') if bond: dict.update({'is_bond_member' : bond}) + # Check if any DHCP options changed which require a client restat + dhcp = node_changed(config, ['dhcp-options'], recursive=True) + if dhcp: dict.update({'dhcp_options_changed' : ''}) + # Some interfaces come with a source_interface which must also not be part # of any other bond or bridge interface as it is exclusivly assigned as the # Kernels "lower" interface to this new "virtual/upper" interface. @@ -435,10 +470,10 @@ def get_interface_dict(config, base, ifname=''): tmp = is_member(config, dict['source_interface'], 'bonding') if tmp: dict.update({'source_interface_is_bond_member' : tmp}) - mac = leaf_node_changed(config, ['mac']) + mac = leaf_node_changed(config, base + [ifname, 'mac']) if mac: dict.update({'mac_old' : mac}) - eui64 = leaf_node_changed(config, ['ipv6', 'address', 'eui64']) + eui64 = leaf_node_changed(config, base + [ifname, 'ipv6', 'address', 'eui64']) if eui64: tmp = dict_search('ipv6.address', dict) if not tmp: @@ -450,6 +485,9 @@ def get_interface_dict(config, base, ifname=''): # identical for all types of VLAN interfaces as they all include the same # XML definitions which hold the defaults. for vif, vif_config in dict.get('vif', {}).items(): + # Add subinterface name to dictionary + dict['vif'][vif].update({'ifname' : f'{ifname}.{vif}'}) + default_vif_values = defaults(base + ['vif']) # XXX: T2665: When there is no DHCPv6-PD configuration given, we can safely # remove the default values from the dict. @@ -459,15 +497,31 @@ def get_interface_dict(config, base, ifname=''): # Only add defaults if interface is not about to be deleted - this is # to keep a cleaner config dict. if 'deleted' not in dict: - dict['vif'][vif] = dict_merge(default_vif_values, vif_config) + address = leaf_node_changed(config, base + [ifname, 'vif', vif, 'address']) + if address: dict['vif'][vif].update({'address_old' : address}) + + dict['vif'][vif] = dict_merge(default_vif_values, dict['vif'][vif]) # XXX: T2665: blend in proper DHCPv6-PD default values dict['vif'][vif] = T2665_set_dhcpv6pd_defaults(dict['vif'][vif]) + # If interface does not request an IPv4 DHCP address there is no need + # to keep the dhcp-options key + if 'address' not in dict['vif'][vif] or 'dhcp' not in dict['vif'][vif]['address']: + if 'dhcp_options' in dict['vif'][vif]: + del dict['vif'][vif]['dhcp_options'] + # Check if we are a member of a bridge device bridge = is_member(config, f'{ifname}.{vif}', 'bridge') if bridge: dict['vif'][vif].update({'is_bridge_member' : bridge}) + # Check if any DHCP options changed which require a client restat + dhcp = node_changed(config, ['vif', vif, 'dhcp-options'], recursive=True) + if dhcp: dict['vif'][vif].update({'dhcp_options_changed' : ''}) + for vif_s, vif_s_config in dict.get('vif_s', {}).items(): + # Add subinterface name to dictionary + dict['vif_s'][vif_s].update({'ifname' : f'{ifname}.{vif_s}'}) + default_vif_s_values = defaults(base + ['vif-s']) # XXX: T2665: we only wan't the vif-s defaults - do not care about vif-c if 'vif_c' in default_vif_s_values: del default_vif_s_values['vif_c'] @@ -480,15 +534,33 @@ def get_interface_dict(config, base, ifname=''): # Only add defaults if interface is not about to be deleted - this is # to keep a cleaner config dict. if 'deleted' not in dict: - dict['vif_s'][vif_s] = dict_merge(default_vif_s_values, vif_s_config) + address = leaf_node_changed(config, base + [ifname, 'vif-s', vif_s, 'address']) + if address: dict['vif_s'][vif_s].update({'address_old' : address}) + + dict['vif_s'][vif_s] = dict_merge(default_vif_s_values, + dict['vif_s'][vif_s]) # XXX: T2665: blend in proper DHCPv6-PD default values dict['vif_s'][vif_s] = T2665_set_dhcpv6pd_defaults(dict['vif_s'][vif_s]) + # If interface does not request an IPv4 DHCP address there is no need + # to keep the dhcp-options key + if 'address' not in dict['vif_s'][vif_s] or 'dhcp' not in \ + dict['vif_s'][vif_s]['address']: + if 'dhcp_options' in dict['vif_s'][vif_s]: + del dict['vif_s'][vif_s]['dhcp_options'] + # Check if we are a member of a bridge device bridge = is_member(config, f'{ifname}.{vif_s}', 'bridge') if bridge: dict['vif_s'][vif_s].update({'is_bridge_member' : bridge}) + # Check if any DHCP options changed which require a client restat + dhcp = node_changed(config, ['vif-s', vif_s, 'dhcp-options'], recursive=True) + if dhcp: dict['vif_s'][vif_s].update({'dhcp_options_changed' : ''}) + for vif_c, vif_c_config in vif_s_config.get('vif_c', {}).items(): + # Add subinterface name to dictionary + dict['vif_s'][vif_s]['vif_c'][vif_c].update({'ifname' : f'{ifname}.{vif_s}.{vif_c}'}) + default_vif_c_values = defaults(base + ['vif-s', 'vif-c']) # XXX: T2665: When there is no DHCPv6-PD configuration given, we can safely @@ -499,20 +571,35 @@ def get_interface_dict(config, base, ifname=''): # Only add defaults if interface is not about to be deleted - this is # to keep a cleaner config dict. if 'deleted' not in dict: + address = leaf_node_changed(config, base + [ifname, 'vif-s', vif_s, 'vif-c', vif_c, 'address']) + if address: dict['vif_s'][vif_s]['vif_c'][vif_c].update( + {'address_old' : address}) + dict['vif_s'][vif_s]['vif_c'][vif_c] = dict_merge( - default_vif_c_values, vif_c_config) + default_vif_c_values, dict['vif_s'][vif_s]['vif_c'][vif_c]) # XXX: T2665: blend in proper DHCPv6-PD default values dict['vif_s'][vif_s]['vif_c'][vif_c] = T2665_set_dhcpv6pd_defaults( dict['vif_s'][vif_s]['vif_c'][vif_c]) + # If interface does not request an IPv4 DHCP address there is no need + # to keep the dhcp-options key + if 'address' not in dict['vif_s'][vif_s]['vif_c'][vif_c] or 'dhcp' \ + not in dict['vif_s'][vif_s]['vif_c'][vif_c]['address']: + if 'dhcp_options' in dict['vif_s'][vif_s]['vif_c'][vif_c]: + del dict['vif_s'][vif_s]['vif_c'][vif_c]['dhcp_options'] + # Check if we are a member of a bridge device bridge = is_member(config, f'{ifname}.{vif_s}.{vif_c}', 'bridge') if bridge: dict['vif_s'][vif_s]['vif_c'][vif_c].update( {'is_bridge_member' : bridge}) + # Check if any DHCP options changed which require a client restat + dhcp = node_changed(config, ['vif-s', vif_s, 'vif-c', vif_c, 'dhcp-options'], recursive=True) + if dhcp: dict['vif_s'][vif_s]['vif_c'][vif_c].update({'dhcp_options_changed' : ''}) + # Check vif, vif-s/vif-c VLAN interfaces for removal - dict = get_removed_vlans(config, dict) - return dict + dict = get_removed_vlans(config, base + [ifname], dict) + return ifname, dict def get_vlan_ids(interface): """ @@ -534,7 +621,6 @@ def get_vlan_ids(interface): return vlan_ids - def get_accel_dict(config, base, chap_secrets): """ Common utility function to retrieve and mangle the Accel-PPP configuration diff --git a/python/vyos/configdiff.py b/python/vyos/configdiff.py index 0e41fbe27..9185575df 100644 --- a/python/vyos/configdiff.py +++ b/python/vyos/configdiff.py @@ -16,8 +16,11 @@ from enum import IntFlag, auto from vyos.config import Config +from vyos.configtree import DiffTree from vyos.configdict import dict_merge +from vyos.configdict import list_diff from vyos.util import get_sub_dict, mangle_dict_keys +from vyos.util import dict_search_args from vyos.xml import defaults class ConfigDiffError(Exception): @@ -36,6 +39,8 @@ class Diff(IntFlag): ADD = auto() STABLE = auto() +ALL = Diff.MERGE | Diff.DELETE | Diff.ADD | Diff.STABLE + requires_effective = [enum_to_key(Diff.DELETE)] target_defaults = [enum_to_key(Diff.MERGE)] @@ -73,19 +78,24 @@ def get_config_diff(config, key_mangling=None): isinstance(key_mangling[1], str)): raise ValueError("key_mangling must be a tuple of two strings") - return ConfigDiff(config, key_mangling) + diff_t = DiffTree(config._running_config, config._session_config) + + return ConfigDiff(config, key_mangling, diff_tree=diff_t) class ConfigDiff(object): """ The class of config changes as represented by comparison between the session config dict and the effective config dict. """ - def __init__(self, config, key_mangling=None): + def __init__(self, config, key_mangling=None, diff_tree=None): self._level = config.get_level() self._session_config_dict = config.get_cached_root_dict(effective=False) self._effective_config_dict = config.get_cached_root_dict(effective=True) self._key_mangling = key_mangling + self._diff_tree = diff_tree + self._diff_dict = diff_tree.dict if diff_tree else {} + # mirrored from Config; allow path arguments relative to level def _make_path(self, path): if isinstance(path, str): @@ -134,7 +144,45 @@ class ConfigDiff(object): self._key_mangling[1]) return config_dict - def get_child_nodes_diff(self, path=[], expand_nodes=Diff(0), no_defaults=False): + def is_node_changed(self, path=[]): + if self._diff_tree is None: + raise NotImplementedError("diff_tree class not available") + + if (self._diff_tree.add.exists(self._make_path(path)) or + self._diff_tree.sub.exists(self._make_path(path))): + return True + return False + + def get_child_nodes_diff_str(self, path=[]): + ret = {'add': {}, 'change': {}, 'delete': {}} + + diff = self.get_child_nodes_diff(path, + expand_nodes=Diff.ADD | Diff.DELETE | Diff.MERGE | Diff.STABLE, + no_defaults=True) + + def parse_dict(diff_dict, diff_type, prefix=[]): + for k, v in diff_dict.items(): + if isinstance(v, dict): + parse_dict(v, diff_type, prefix + [k]) + else: + path_str = ' '.join(prefix + [k]) + if diff_type == 'add' or diff_type == 'delete': + if isinstance(v, list): + v = ', '.join(v) + ret[diff_type][path_str] = v + elif diff_type == 'merge': + old_value = dict_search_args(diff['stable'], *prefix, k) + if old_value and old_value != v: + ret['change'][path_str] = [old_value, v] + + parse_dict(diff['merge'], 'merge') + parse_dict(diff['add'], 'add') + parse_dict(diff['delete'], 'delete') + + return ret + + def get_child_nodes_diff(self, path=[], expand_nodes=Diff(0), no_defaults=False, + recursive=False): """ Args: path (str|list): config path @@ -144,6 +192,8 @@ class ConfigDiff(object): value no_detaults=False: if expand_nodes & Diff.MERGE, do not merge default values to ret['merge'] + recursive: if true, use config_tree diff algorithm provided by + diff_tree class Returns: dict of lists, representing differences between session and effective config, under path @@ -154,6 +204,34 @@ class ConfigDiff(object): """ session_dict = get_sub_dict(self._session_config_dict, self._make_path(path), get_first_key=True) + + if recursive: + if self._diff_tree is None: + raise NotImplementedError("diff_tree class not available") + else: + add = get_sub_dict(self._diff_tree.dict, ['add'], get_first_key=True) + sub = get_sub_dict(self._diff_tree.dict, ['sub'], get_first_key=True) + inter = get_sub_dict(self._diff_tree.dict, ['inter'], get_first_key=True) + ret = {} + ret[enum_to_key(Diff.MERGE)] = session_dict + ret[enum_to_key(Diff.DELETE)] = get_sub_dict(sub, self._make_path(path), + get_first_key=True) + ret[enum_to_key(Diff.ADD)] = get_sub_dict(add, self._make_path(path), + get_first_key=True) + ret[enum_to_key(Diff.STABLE)] = get_sub_dict(inter, self._make_path(path), + get_first_key=True) + for e in Diff: + k = enum_to_key(e) + if not (e & expand_nodes): + ret[k] = list(ret[k]) + else: + if self._key_mangling: + ret[k] = self._mangle_dict_keys(ret[k]) + if k in target_defaults and not no_defaults: + default_values = defaults(self._make_path(path)) + ret[k] = dict_merge(default_values, ret[k]) + return ret + effective_dict = get_sub_dict(self._effective_config_dict, self._make_path(path), get_first_key=True) @@ -179,7 +257,8 @@ class ConfigDiff(object): return ret - def get_node_diff(self, path=[], expand_nodes=Diff(0), no_defaults=False): + def get_node_diff(self, path=[], expand_nodes=Diff(0), no_defaults=False, + recursive=False): """ Args: path (str|list): config path @@ -189,6 +268,8 @@ class ConfigDiff(object): value no_detaults=False: if expand_nodes & Diff.MERGE, do not merge default values to ret['merge'] + recursive: if true, use config_tree diff algorithm provided by + diff_tree class Returns: dict of lists, representing differences between session and effective config, at path @@ -198,6 +279,31 @@ class ConfigDiff(object): dict['stable'] = config values in both session and effective """ session_dict = get_sub_dict(self._session_config_dict, self._make_path(path)) + + if recursive: + if self._diff_tree is None: + raise NotImplementedError("diff_tree class not available") + else: + add = get_sub_dict(self._diff_tree.dict, ['add'], get_first_key=True) + sub = get_sub_dict(self._diff_tree.dict, ['sub'], get_first_key=True) + inter = get_sub_dict(self._diff_tree.dict, ['inter'], get_first_key=True) + ret = {} + ret[enum_to_key(Diff.MERGE)] = session_dict + ret[enum_to_key(Diff.DELETE)] = get_sub_dict(sub, self._make_path(path)) + ret[enum_to_key(Diff.ADD)] = get_sub_dict(add, self._make_path(path)) + ret[enum_to_key(Diff.STABLE)] = get_sub_dict(inter, self._make_path(path)) + for e in Diff: + k = enum_to_key(e) + if not (e & expand_nodes): + ret[k] = list(ret[k]) + else: + if self._key_mangling: + ret[k] = self._mangle_dict_keys(ret[k]) + if k in target_defaults and not no_defaults: + default_values = defaults(self._make_path(path)) + ret[k] = dict_merge(default_values, ret[k]) + return ret + effective_dict = get_sub_dict(self._effective_config_dict, self._make_path(path)) ret = _key_sets_from_dicts(session_dict, effective_dict) diff --git a/python/vyos/configquery.py b/python/vyos/configquery.py index 1cdcbcf39..5b097b312 100644 --- a/python/vyos/configquery.py +++ b/python/vyos/configquery.py @@ -18,16 +18,15 @@ A small library that allows querying existence or value(s) of config settings from op mode, and execution of arbitrary op mode commands. ''' -import re -import json -from copy import deepcopy +import os from subprocess import STDOUT -import vyos.util -import vyos.xml +from vyos.util import popen, boot_configuration_complete from vyos.config import Config -from vyos.configtree import ConfigTree -from vyos.configsource import ConfigSourceSession +from vyos.configsource import ConfigSourceSession, ConfigSourceString +from vyos.defaults import directories + +config_file = os.path.join(directories['config'], 'config.boot') class ConfigQueryError(Exception): pass @@ -58,21 +57,21 @@ class CliShellApiConfigQuery(GenericConfigQuery): def exists(self, path: list): cmd = ' '.join(path) - (_, err) = vyos.util.popen(f'cli-shell-api existsActive {cmd}') + (_, err) = popen(f'cli-shell-api existsActive {cmd}') if err: return False return True def value(self, path: list): cmd = ' '.join(path) - (out, err) = vyos.util.popen(f'cli-shell-api returnActiveValue {cmd}') + (out, err) = popen(f'cli-shell-api returnActiveValue {cmd}') if err: raise ConfigQueryError('No value for given path') return out def values(self, path: list): cmd = ' '.join(path) - (out, err) = vyos.util.popen(f'cli-shell-api returnActiveValues {cmd}') + (out, err) = popen(f'cli-shell-api returnActiveValues {cmd}') if err: raise ConfigQueryError('No values for given path') return out @@ -81,25 +80,36 @@ class ConfigTreeQuery(GenericConfigQuery): def __init__(self): super().__init__() - config_source = ConfigSourceSession() - self.configtree = Config(config_source=config_source) + if boot_configuration_complete(): + config_source = ConfigSourceSession() + self.config = Config(config_source=config_source) + else: + try: + with open(config_file) as f: + config_string = f.read() + except OSError as err: + raise ConfigQueryError('No config file available') from err + + config_source = ConfigSourceString(running_config_text=config_string, + session_config_text=config_string) + self.config = Config(config_source=config_source) def exists(self, path: list): - return self.configtree.exists(path) + return self.config.exists(path) def value(self, path: list): - return self.configtree.return_value(path) + return self.config.return_value(path) def values(self, path: list): - return self.configtree.return_values(path) + return self.config.return_values(path) def list_nodes(self, path: list): - return self.configtree.list_nodes(path) + return self.config.list_nodes(path) def get_config_dict(self, path=[], effective=False, key_mangling=None, get_first_key=False, no_multi_convert=False, no_tag_node_value_mangle=False): - return self.configtree.get_config_dict(path, effective=effective, + return self.config.get_config_dict(path, effective=effective, key_mangling=key_mangling, get_first_key=get_first_key, no_multi_convert=no_multi_convert, no_tag_node_value_mangle=no_tag_node_value_mangle) @@ -110,7 +120,7 @@ class VbashOpRun(GenericOpRun): def run(self, path: list, **kwargs): cmd = ' '.join(path) - (out, err) = vyos.util.popen(f'. /opt/vyatta/share/vyatta-op/functions/interpreter/vyatta-op-run; _vyatta_op_run {cmd}', stderr=STDOUT, **kwargs) + (out, err) = popen(f'/opt/vyatta/bin/vyatta-op-cmd-wrapper {cmd}', stderr=STDOUT, **kwargs) if err: raise ConfigQueryError(out) return out diff --git a/python/vyos/configsource.py b/python/vyos/configsource.py index b0981d25e..510b5b65a 100644 --- a/python/vyos/configsource.py +++ b/python/vyos/configsource.py @@ -19,6 +19,7 @@ import re import subprocess from vyos.configtree import ConfigTree +from vyos.util import boot_configuration_complete class VyOSError(Exception): """ @@ -117,7 +118,7 @@ class ConfigSourceSession(ConfigSource): # Running config can be obtained either from op or conf mode, it always succeeds # once the config system is initialized during boot; # before initialization, set to empty string - if os.path.isfile('/tmp/vyos-config-status'): + if boot_configuration_complete(): try: running_config_text = self._run([self._cli_shell_api, '--show-active-only', '--show-show-defaults', '--show-ignore-edit', 'showConfig']) except VyOSError: @@ -161,7 +162,7 @@ class ConfigSourceSession(ConfigSource): if p.returncode != 0: raise VyOSError() else: - return out.decode('ascii', 'ignore') + return out.decode() def set_level(self, path): """ diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index d8ffaca99..e9cdb69e4 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -17,6 +17,7 @@ import json from ctypes import cdll, c_char_p, c_void_p, c_int +LIBPATH = '/usr/lib/libvyosconfig.so.0' def escape_backslash(string: str) -> str: """Escape single backslashes in string that are not in escape sequence""" @@ -42,7 +43,9 @@ class ConfigTreeError(Exception): class ConfigTree(object): - def __init__(self, config_string, libpath='/usr/lib/libvyosconfig.so.0'): + def __init__(self, config_string=None, address=None, libpath=LIBPATH): + if config_string is None and address is None: + raise TypeError("ConfigTree() requires one of 'config_string' or 'address'") self.__config = None self.__lib = cdll.LoadLibrary(libpath) @@ -60,7 +63,7 @@ class ConfigTree(object): self.__to_string.restype = c_char_p self.__to_commands = self.__lib.to_commands - self.__to_commands.argtypes = [c_void_p] + self.__to_commands.argtypes = [c_void_p, c_char_p] self.__to_commands.restype = c_char_p self.__to_json = self.__lib.to_json @@ -123,18 +126,26 @@ class ConfigTree(object): self.__set_tag.argtypes = [c_void_p, c_char_p] self.__set_tag.restype = c_int + self.__get_subtree = self.__lib.get_subtree + self.__get_subtree.argtypes = [c_void_p, c_char_p] + self.__get_subtree.restype = c_void_p + self.__destroy = self.__lib.destroy self.__destroy.argtypes = [c_void_p] - config_section, version_section = extract_version(config_string) - config_section = escape_backslash(config_section) - config = self.__from_string(config_section.encode()) - if config is None: - msg = self.__get_error().decode() - raise ValueError("Failed to parse config: {0}".format(msg)) + if address is None: + config_section, version_section = extract_version(config_string) + config_section = escape_backslash(config_section) + config = self.__from_string(config_section.encode()) + if config is None: + msg = self.__get_error().decode() + raise ValueError("Failed to parse config: {0}".format(msg)) + else: + self.__config = config + self.__version = version_section else: - self.__config = config - self.__version = version_section + self.__config = address + self.__version = '' def __del__(self): if self.__config is not None: @@ -143,13 +154,16 @@ class ConfigTree(object): def __str__(self): return self.to_string() + def _get_config(self): + return self.__config + def to_string(self): config_string = self.__to_string(self.__config).decode() config_string = "{0}\n{1}".format(config_string, self.__version) return config_string - def to_commands(self): - return self.__to_commands(self.__config).decode() + def to_commands(self, op="set"): + return self.__to_commands(self.__config, op.encode()).decode() def to_json(self): return self.__to_json(self.__config).decode() @@ -281,3 +295,61 @@ class ConfigTree(object): else: raise ConfigTreeError("Path [{}] doesn't exist".format(path_str)) + def get_subtree(self, path, with_node=False): + check_path(path) + path_str = " ".join(map(str, path)).encode() + + res = self.__get_subtree(self.__config, path_str, with_node) + subt = ConfigTree(address=res) + return subt + +class DiffTree: + def __init__(self, left, right, path=[], libpath=LIBPATH): + if left is None: + left = ConfigTree(config_string='\n') + if right is None: + right = ConfigTree(config_string='\n') + if not (isinstance(left, ConfigTree) and isinstance(right, ConfigTree)): + raise TypeError("Arguments must be instances of ConfigTree") + if path: + if not left.exists(path): + raise ConfigTreeError(f"Path {path} doesn't exist in lhs tree") + if not right.exists(path): + raise ConfigTreeError(f"Path {path} doesn't exist in rhs tree") + + self.left = left + self.right = right + + self.__lib = cdll.LoadLibrary(libpath) + + self.__diff_tree = self.__lib.diff_tree + self.__diff_tree.argtypes = [c_char_p, c_void_p, c_void_p] + self.__diff_tree.restype = c_void_p + + self.__trim_tree = self.__lib.trim_tree + self.__trim_tree.argtypes = [c_void_p, c_void_p] + self.__trim_tree.restype = c_void_p + + check_path(path) + path_str = " ".join(map(str, path)).encode() + + res = self.__diff_tree(path_str, left._get_config(), right._get_config()) + + # full diff config_tree and python dict representation + self.full = ConfigTree(address=res) + self.dict = json.loads(self.full.to_json()) + + # config_tree sub-trees + self.add = self.full.get_subtree(['add']) + self.sub = self.full.get_subtree(['sub']) + self.inter = self.full.get_subtree(['inter']) + + # trim sub(-tract) tree to get delete tree for commands + ref = self.right.get_subtree(path, with_node=True) if path else self.right + res = self.__trim_tree(self.sub._get_config(), ref._get_config()) + self.delete = ConfigTree(address=res) + + def to_commands(self): + add = self.add.to_commands() + delete = self.delete.to_commands(op="delete") + return delete + "\n" + add diff --git a/python/vyos/configverify.py b/python/vyos/configverify.py index 365a28feb..438485d98 100644 --- a/python/vyos/configverify.py +++ b/python/vyos/configverify.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2020-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -173,24 +173,43 @@ def verify_eapol(config): if ca_cert_name not in config['pki']['ca']: raise ConfigError('Invalid CA certificate specified for EAPoL') - ca_cert = config['pki']['ca'][cert_name] + ca_cert = config['pki']['ca'][ca_cert_name] if 'certificate' not in ca_cert: raise ConfigError('Invalid CA certificate specified for EAPoL') -def verify_mirror(config): +def verify_mirror_redirect(config): """ Common helper function used by interface implementations to perform - recurring validation of mirror interface configuration. + recurring validation of mirror and redirect interface configuration via tc(8) It makes no sense to mirror traffic back at yourself! """ + import os + if {'mirror', 'redirect'} <= set(config): + raise ConfigError('Mirror and redirect can not be enabled at the same time!') + if 'mirror' in config: for direction, mirror_interface in config['mirror'].items(): + if not os.path.exists(f'/sys/class/net/{mirror_interface}'): + raise ConfigError(f'Requested mirror interface "{mirror_interface}" '\ + 'does not exist!') + if mirror_interface == config['ifname']: - raise ConfigError(f'Can not mirror "{direction}" traffic back ' \ + raise ConfigError(f'Can not mirror "{direction}" traffic back '\ 'the originating interface!') + if 'redirect' in config: + redirect_ifname = config['redirect'] + if not os.path.exists(f'/sys/class/net/{redirect_ifname}'): + raise ConfigError(f'Requested redirect interface "{redirect_ifname}" '\ + 'does not exist!') + + if ('mirror' in config or 'redirect' in config) and dict_search('traffic_policy.in', config) is not None: + # XXX: support combination of limiting and redirect/mirror - this is an + # artificial limitation + raise ConfigError('Can not use ingress policy together with mirror or redirect!') + def verify_authentication(config): """ Common helper function used by interface implementations to perform @@ -224,9 +243,10 @@ def verify_bridge_delete(config): when interface also is part of a bridge. """ if 'is_bridge_member' in config: - raise ConfigError( - 'Interface "{ifname}" cannot be deleted as it is a ' - 'member of bridge "{is_bridge_member}"!'.format(**config)) + interface = config['ifname'] + for bridge in config['is_bridge_member']: + raise ConfigError(f'Interface "{interface}" cannot be deleted as it ' + f'is a member of bridge "{bridge}"!') def verify_interface_exists(ifname): """ @@ -308,27 +328,37 @@ def verify_vlan_config(config): if duplicate: raise ConfigError(f'Duplicate VLAN id "{duplicate[0]}" used for vif and vif-s interfaces!') + parent_ifname = config['ifname'] # 802.1q VLANs - for vlan in config.get('vif', {}): - vlan = config['vif'][vlan] + for vlan_id in config.get('vif', {}): + vlan = config['vif'][vlan_id] + vlan['ifname'] = f'{parent_ifname}.{vlan_id}' + verify_dhcpv6(vlan) verify_address(vlan) verify_vrf(vlan) + verify_mirror_redirect(vlan) verify_mtu_parent(vlan, config) # 802.1ad (Q-in-Q) VLANs - for s_vlan in config.get('vif_s', {}): - s_vlan = config['vif_s'][s_vlan] + for s_vlan_id in config.get('vif_s', {}): + s_vlan = config['vif_s'][s_vlan_id] + s_vlan['ifname'] = f'{parent_ifname}.{s_vlan_id}' + verify_dhcpv6(s_vlan) verify_address(s_vlan) verify_vrf(s_vlan) + verify_mirror_redirect(s_vlan) verify_mtu_parent(s_vlan, config) - for c_vlan in s_vlan.get('vif_c', {}): - c_vlan = s_vlan['vif_c'][c_vlan] + for c_vlan_id in s_vlan.get('vif_c', {}): + c_vlan = s_vlan['vif_c'][c_vlan_id] + c_vlan['ifname'] = f'{parent_ifname}.{s_vlan_id}.{c_vlan_id}' + verify_dhcpv6(c_vlan) verify_address(c_vlan) verify_vrf(c_vlan) + verify_mirror_redirect(c_vlan) verify_mtu_parent(c_vlan, config) verify_mtu_parent(c_vlan, s_vlan) diff --git a/python/vyos/cpu.py b/python/vyos/cpu.py new file mode 100644 index 000000000..a0ef864be --- /dev/null +++ b/python/vyos/cpu.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +""" +Retrieves (or at least attempts to retrieve) the total number of real CPU cores +installed in a Linux system. + +The issue of core count is complicated by existence of SMT, e.g. Intel's Hyper Threading. +GNU nproc returns the number of LOGICAL cores, +which is 2x of the real cores if SMT is enabled. + +The idea is to find all physical CPUs and add up their core counts. +It has special cases for x86_64 and MAY work correctly on other architectures, +but nothing is certain. +""" + +import re + + +def _read_cpuinfo(): + with open('/proc/cpuinfo', 'r') as f: + return f.readlines() + +def _split_line(l): + l = l.strip() + parts = re.split(r'\s*:\s*', l) + return (parts[0], ":".join(parts[1:])) + +def _find_cpus(cpuinfo_lines): + # Make a dict because it's more convenient to work with later, + # when we need to find physicall distinct CPUs there. + cpus = {} + + cpu_number = 0 + + for l in cpuinfo_lines: + key, value = _split_line(l) + if key == 'processor': + cpu_number = value + cpus[cpu_number] = {} + else: + cpus[cpu_number][key] = value + + return cpus + +def _find_physical_cpus(): + cpus = _find_cpus(_read_cpuinfo()) + + phys_cpus = {} + + for num in cpus: + if 'physical id' in cpus[num]: + # On at least some architectures, CPUs in different sockets + # have different 'physical id' field, e.g. on x86_64. + phys_id = cpus[num]['physical id'] + if phys_id not in phys_cpus: + phys_cpus[phys_id] = cpus[num] + else: + # On other architectures, e.g. on ARM, there's no such field. + # We just assume they are different CPUs, + # whether single core ones or cores of physical CPUs. + phys_cpus[num] = cpu[num] + + return phys_cpus + +def get_cpus(): + """ Returns a list of /proc/cpuinfo entries that belong to different CPUs. + """ + cpus_dict = _find_physical_cpus() + return list(cpus_dict.values()) + +def get_core_count(): + """ Returns the total number of physical CPU cores + (even if Hyper-Threading or another SMT is enabled and has inflated + the number of cores in /proc/cpuinfo) + """ + physical_cpus = _find_physical_cpus() + + core_count = 0 + + for num in physical_cpus: + # Some architectures, e.g. x86_64, include a field for core count. + # Since we found unique physical CPU entries, we can sum their core counts. + if 'cpu cores' in physical_cpus[num]: + core_count += int(physical_cpus[num]['cpu cores']) + else: + core_count += 1 + + return core_count diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index 00b14a985..fcb6a7fbc 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -29,6 +29,8 @@ directories = { "vyos_udev_dir": "/run/udev/vyos" } +config_status = '/tmp/vyos-config-status' + cfg_group = 'vyattacfg' cfg_vintage = 'vyos' @@ -44,8 +46,10 @@ https_data = { api_data = { 'listen_address' : '127.0.0.1', 'port' : '8080', - 'strict' : 'false', - 'debug' : 'false', + 'socket' : False, + 'strict' : False, + 'gql' : False, + 'debug' : False, 'api_keys' : [ {"id": "testapp", "key": "qwerty"} ] } diff --git a/python/vyos/ethtool.py b/python/vyos/ethtool.py index e45b0f041..2b6012a73 100644 --- a/python/vyos/ethtool.py +++ b/python/vyos/ethtool.py @@ -1,4 +1,4 @@ -# Copyright 2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2021-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -18,6 +18,11 @@ import re from vyos.util import popen +# These drivers do not support using ethtool to change the speed, duplex, or +# flow control settings +_drivers_without_speed_duplex_flow = ['vmxnet3', 'virtio_net', 'xen_netfront', + 'iavf', 'ice', 'i40e', 'hv_netvsc'] + class Ethtool: """ Class is used to retrive and cache information about an ethernet adapter @@ -41,7 +46,7 @@ class Ethtool: # '100' : {'full': '', 'half': ''}, # '1000': {'full': ''} # } - _speed_duplex = { } + _speed_duplex = {'auto': {'auto': ''}} _ring_buffers = { } _ring_buffers_max = { } _driver_name = None @@ -188,7 +193,7 @@ class Ethtool: if duplex not in ['auto', 'full', 'half']: raise ValueError(f'Value "{duplex}" for duplex is invalid!') - if self.get_driver_name() in ['vmxnet3', 'virtio_net', 'xen_netfront']: + if self.get_driver_name() in _drivers_without_speed_duplex_flow: return False if speed in self._speed_duplex: @@ -198,7 +203,7 @@ class Ethtool: def check_flow_control(self): """ Check if the NIC supports flow-control """ - if self.get_driver_name() in ['vmxnet3', 'virtio_net', 'xen_netfront']: + if self.get_driver_name() in _drivers_without_speed_duplex_flow: return False return self._flow_control diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py new file mode 100644 index 000000000..04fd44173 --- /dev/null +++ b/python/vyos/firewall.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2021 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import re + +from vyos.util import cmd +from vyos.util import dict_search_args + +def find_nftables_rule(table, chain, rule_matches=[]): + # Find rule in table/chain that matches all criteria and return the handle + results = cmd(f'sudo nft -a list chain {table} {chain}').split("\n") + for line in results: + if all(rule_match in line for rule_match in rule_matches): + handle_search = re.search('handle (\d+)', line) + if handle_search: + return handle_search[1] + return None + +def remove_nftables_rule(table, chain, handle): + cmd(f'sudo nft delete rule {table} {chain} handle {handle}') + +# Functions below used by template generation + +def nft_action(vyos_action): + if vyos_action == 'accept': + return 'return' + return vyos_action + +def parse_rule(rule_conf, fw_name, rule_id, ip_name): + output = [] + def_suffix = '6' if ip_name == 'ip6' else '' + + if 'state' in rule_conf and rule_conf['state']: + states = ",".join([s for s, v in rule_conf['state'].items() if v == 'enable']) + + if states: + output.append(f'ct state {{{states}}}') + + if 'connection_status' in rule_conf and rule_conf['connection_status']: + status = rule_conf['connection_status'] + if status['nat'] == 'destination': + nat_status = '{dnat}' + output.append(f'ct status {nat_status}') + if status['nat'] == 'source': + nat_status = '{snat}' + output.append(f'ct status {nat_status}') + + if 'protocol' in rule_conf and rule_conf['protocol'] != 'all': + proto = rule_conf['protocol'] + operator = '' + if proto[0] == '!': + operator = '!=' + proto = proto[1:] + if proto == 'tcp_udp': + proto = '{tcp, udp}' + output.append(f'meta l4proto {operator} {proto}') + + for side in ['destination', 'source']: + if side in rule_conf: + prefix = side[0] + side_conf = rule_conf[side] + + if 'address' in side_conf: + suffix = side_conf['address'] + if suffix[0] == '!': + suffix = f'!= {suffix[1:]}' + output.append(f'{ip_name} {prefix}addr {suffix}') + + if 'mac_address' in side_conf: + suffix = side_conf["mac_address"] + if suffix[0] == '!': + suffix = f'!= {suffix[1:]}' + output.append(f'ether {prefix}addr {suffix}') + + if 'port' in side_conf: + proto = rule_conf['protocol'] + port = side_conf['port'].split(',') + + ports = [] + negated_ports = [] + + for p in port: + if p[0] == '!': + negated_ports.append(p[1:]) + else: + ports.append(p) + + if proto == 'tcp_udp': + proto = 'th' + + if ports: + ports_str = ','.join(ports) + output.append(f'{proto} {prefix}port {{{ports_str}}}') + + if negated_ports: + negated_ports_str = ','.join(negated_ports) + output.append(f'{proto} {prefix}port != {{{negated_ports_str}}}') + + if 'group' in side_conf: + group = side_conf['group'] + if 'address_group' in group: + group_name = group['address_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_name} {prefix}addr {operator} $A{def_suffix}_{group_name}') + elif 'network_group' in group: + group_name = group['network_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_name} {prefix}addr {operator} $N{def_suffix}_{group_name}') + if 'mac_group' in group: + group_name = group['mac_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'ether {prefix}addr {operator} $M_{group_name}') + if 'port_group' in group: + proto = rule_conf['protocol'] + group_name = group['port_group'] + + if proto == 'tcp_udp': + proto = 'th' + + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + + output.append(f'{proto} {prefix}port {operator} $P_{group_name}') + + if 'log' in rule_conf and rule_conf['log'] == 'enable': + action = rule_conf['action'] if 'action' in rule_conf else 'accept' + output.append(f'log prefix "[{fw_name[:19]}-{rule_id}-{action[:1].upper()}] "') + + if 'hop_limit' in rule_conf: + operators = {'eq': '==', 'gt': '>', 'lt': '<'} + for op, operator in operators.items(): + if op in rule_conf['hop_limit']: + value = rule_conf['hop_limit'][op] + output.append(f'ip6 hoplimit {operator} {value}') + + for icmp in ['icmp', 'icmpv6']: + if icmp in rule_conf: + if 'type_name' in rule_conf[icmp]: + output.append(icmp + ' type ' + rule_conf[icmp]['type_name']) + else: + if 'code' in rule_conf[icmp]: + output.append(icmp + ' code ' + rule_conf[icmp]['code']) + if 'type' in rule_conf[icmp]: + output.append(icmp + ' type ' + rule_conf[icmp]['type']) + + if 'ipsec' in rule_conf: + if 'match_ipsec' in rule_conf['ipsec']: + output.append('meta ipsec == 1') + if 'match_non_ipsec' in rule_conf['ipsec']: + output.append('meta ipsec == 0') + + if 'fragment' in rule_conf: + # Checking for fragmentation after priority -400 is not possible, + # so we use a priority -450 hook to set a mark + if 'match_frag' in rule_conf['fragment']: + output.append('meta mark 0xffff1') + if 'match_non_frag' in rule_conf['fragment']: + output.append('meta mark != 0xffff1') + + if 'limit' in rule_conf: + if 'rate' in rule_conf['limit']: + output.append(f'limit rate {rule_conf["limit"]["rate"]}') + if 'burst' in rule_conf['limit']: + output.append(f'burst {rule_conf["limit"]["burst"]} packets') + + if 'recent' in rule_conf: + count = rule_conf['recent']['count'] + time = rule_conf['recent']['time'] + output.append(f'add @RECENT{def_suffix}_{fw_name}_{rule_id} {{ {ip_name} saddr limit rate over {count}/{time} burst {count} packets }}') + + if 'time' in rule_conf: + output.append(parse_time(rule_conf['time'])) + + tcp_flags = dict_search_args(rule_conf, 'tcp', 'flags') + if tcp_flags: + output.append(parse_tcp_flags(tcp_flags)) + + output.append('counter') + + if 'set' in rule_conf: + output.append(parse_policy_set(rule_conf['set'], def_suffix)) + + if 'action' in rule_conf: + output.append(nft_action(rule_conf['action'])) + else: + output.append('return') + + output.append(f'comment "{fw_name}-{rule_id}"') + return " ".join(output) + +def parse_tcp_flags(flags): + include = [flag for flag in flags if flag != 'not'] + exclude = list(flags['not']) if 'not' in flags else [] + return f'tcp flags & ({"|".join(include + exclude)}) == {"|".join(include) if include else "0x0"}' + +def parse_time(time): + out = [] + if 'startdate' in time: + start = time['startdate'] + if 'T' not in start and 'starttime' in time: + start += f' {time["starttime"]}' + out.append(f'time >= "{start}"') + if 'starttime' in time and 'startdate' not in time: + out.append(f'hour >= "{time["starttime"]}"') + if 'stopdate' in time: + stop = time['stopdate'] + if 'T' not in stop and 'stoptime' in time: + stop += f' {time["stoptime"]}' + out.append(f'time < "{stop}"') + if 'stoptime' in time and 'stopdate' not in time: + out.append(f'hour < "{time["stoptime"]}"') + if 'weekdays' in time: + days = time['weekdays'].split(",") + out_days = [f'"{day}"' for day in days if day[0] != '!'] + out.append(f'day {{{",".join(out_days)}}}') + return " ".join(out) + +def parse_policy_set(set_conf, def_suffix): + out = [] + if 'dscp' in set_conf: + dscp = set_conf['dscp'] + out.append(f'ip{def_suffix} dscp set {dscp}') + if 'mark' in set_conf: + mark = set_conf['mark'] + out.append(f'meta mark set {mark}') + if 'table' in set_conf: + table = set_conf['table'] + if table == 'main': + table = '254' + mark = 0x7FFFFFFF - int(table) + out.append(f'meta mark set {mark}') + if 'tcp_mss' in set_conf: + mss = set_conf['tcp_mss'] + out.append(f'tcp option maxseg size set {mss}') + return " ".join(out) diff --git a/python/vyos/frr.py b/python/vyos/frr.py index df6849472..cbba19ab7 100644 --- a/python/vyos/frr.py +++ b/python/vyos/frr.py @@ -73,23 +73,25 @@ from vyos.util import cmd import logging from logging.handlers import SysLogHandler import os +import sys + LOG = logging.getLogger(__name__) +DEBUG = False -DEBUG = os.path.exists('/tmp/vyos.frr.debug') -if DEBUG: - LOG.setLevel(logging.DEBUG) - ch = SysLogHandler(address='/dev/log') - ch2 = logging.StreamHandler() - LOG.addHandler(ch) - LOG.addHandler(ch2) +ch = SysLogHandler(address='/dev/log') +ch2 = logging.StreamHandler(stream=sys.stdout) +LOG.addHandler(ch) +LOG.addHandler(ch2) _frr_daemons = ['zebra', 'bgpd', 'fabricd', 'isisd', 'ospf6d', 'ospfd', 'pbrd', - 'pimd', 'ripd', 'ripngd', 'sharpd', 'staticd', 'vrrpd', 'ldpd'] + 'pimd', 'ripd', 'ripngd', 'sharpd', 'staticd', 'vrrpd', 'ldpd', + 'bfdd'] path_vtysh = '/usr/bin/vtysh' path_frr_reload = '/usr/lib/frr/frr-reload.py' path_config = '/run/frr' +default_add_before = r'(ip prefix-list .*|route-map .*|line vty|end)' class FrrError(Exception): pass @@ -119,6 +121,12 @@ class ConfigSectionNotFound(FrrError): """ pass +def init_debugging(): + global DEBUG + + DEBUG = os.path.exists('/tmp/vyos.frr.debug') + if DEBUG: + LOG.setLevel(logging.DEBUG) def get_configuration(daemon=None, marked=False): """ Get current running FRR configuration @@ -214,13 +222,8 @@ def reload_configuration(config, daemon=None): def save_configuration(): - """Save FRR configuration to /run/frr/config/frr.conf - It save configuration on each commit. T3217 - """ - - cmd(f'{path_vtysh} -n -w') - - return + """ T3217: Save FRR configuration to /run/frr/config/frr.conf """ + return cmd(f'{path_vtysh} -n -w') def execute(command): @@ -427,6 +430,8 @@ class FRRConfig: Using this overwrites the current loaded config objects and replaces the original loaded config ''' + init_debugging() + self.imported_config = get_configuration(daemon=daemon) if daemon: LOG.debug(f'load_configuration: Configuration loaded from FRR daemon {daemon}') @@ -448,16 +453,37 @@ class FRRConfig: mark_configuration('\n'.join(self.config)) def commit_configuration(self, daemon=None): - '''Commit the current configuration to FRR - daemon: str with name of the FRR daemon to commit to or - None to use the consolidated config + ''' + Commit the current configuration to FRR daemon: str with name of the + FRR daemon to commit to or None to use the consolidated config. + + Configuration is automatically saved after apply ''' LOG.debug('commit_configuration: Commiting configuration') for i, e in enumerate(self.config): LOG.debug(f'commit_configuration: new_config {i:3} {e}') - reload_configuration('\n'.join(self.config), daemon=daemon) - def modify_section(self, start_pattern, replacement=[], stop_pattern=r'\S+', remove_stop_mark=False, count=0): + # https://github.com/FRRouting/frr/issues/10132 + # https://github.com/FRRouting/frr/issues/10133 + count = 0 + count_max = 5 + while count < count_max: + count += 1 + try: + reload_configuration('\n'.join(self.config), daemon=daemon) + break + except: + # we just need to re-try the commit of the configuration + # for the listed FRR issues above + pass + if count >= count_max: + raise ConfigurationNotValid(f'Config commit retry counter ({count_max}) exceeded') + + # Save configuration to /run/frr/config/frr.conf + save_configuration() + + + def modify_section(self, start_pattern, replacement='!', stop_pattern=r'\S+', remove_stop_mark=False, count=0): if isinstance(replacement, str): replacement = replacement.split('\n') elif not isinstance(replacement, list): diff --git a/python/vyos/hostsd_client.py b/python/vyos/hostsd_client.py index 303b6ea47..f31ef51cf 100644 --- a/python/vyos/hostsd_client.py +++ b/python/vyos/hostsd_client.py @@ -79,6 +79,18 @@ class Client(object): msg = {'type': 'forward_zones', 'op': 'get'} return self._communicate(msg) + def add_authoritative_zones(self, data): + msg = {'type': 'authoritative_zones', 'op': 'add', 'data': data} + self._communicate(msg) + + def delete_authoritative_zones(self, data): + msg = {'type': 'authoritative_zones', 'op': 'delete', 'data': data} + self._communicate(msg) + + def get_authoritative_zones(self): + msg = {'type': 'authoritative_zones', 'op': 'get'} + return self._communicate(msg) + def add_search_domains(self, data): msg = {'type': 'search_domains', 'op': 'add', 'data': data} self._communicate(msg) diff --git a/python/vyos/ifconfig/__init__.py b/python/vyos/ifconfig/__init__.py index 2d3e406ac..a37615c8f 100644 --- a/python/vyos/ifconfig/__init__.py +++ b/python/vyos/ifconfig/__init__.py @@ -26,6 +26,7 @@ from vyos.ifconfig.ethernet import EthernetIf from vyos.ifconfig.geneve import GeneveIf from vyos.ifconfig.loopback import LoopbackIf from vyos.ifconfig.macvlan import MACVLANIf +from vyos.ifconfig.input import InputIf from vyos.ifconfig.vxlan import VXLANIf from vyos.ifconfig.wireguard import WireGuardIf from vyos.ifconfig.vtun import VTunIf diff --git a/python/vyos/ifconfig/bridge.py b/python/vyos/ifconfig/bridge.py index 27073b266..ffd9c590f 100644 --- a/python/vyos/ifconfig/bridge.py +++ b/python/vyos/ifconfig/bridge.py @@ -298,7 +298,6 @@ class BridgeIf(Interface): tmp = dict_search('member.interface', config) if tmp: - for interface, interface_config in tmp.items(): # if interface does yet not exist bail out early and # add it later @@ -316,10 +315,13 @@ class BridgeIf(Interface): # enslave interface port to bridge self.add_port(interface) - # always set private-vlan/port isolation - tmp = dict_search('isolated', interface_config) - value = 'on' if (tmp != None) else 'off' - lower.set_port_isolation(value) + if not interface.startswith('wlan'): + # always set private-vlan/port isolation - this can not be + # done when lower link is a wifi link, as it will trigger: + # RTNETLINK answers: Operation not supported + tmp = dict_search('isolated', interface_config) + value = 'on' if (tmp != None) else 'off' + lower.set_port_isolation(value) # set bridge port path cost if 'cost' in interface_config: diff --git a/python/vyos/ifconfig/ethernet.py b/python/vyos/ifconfig/ethernet.py index 9d54dc78e..1280fc238 100644 --- a/python/vyos/ifconfig/ethernet.py +++ b/python/vyos/ifconfig/ethernet.py @@ -170,11 +170,18 @@ class EthernetIf(Interface): return cmd = f'ethtool --change {ifname}' - if speed == 'auto' or duplex == 'auto': - cmd += ' autoneg on' - else: - cmd += f' speed {speed} duplex {duplex} autoneg off' - return self._cmd(cmd) + try: + if speed == 'auto' or duplex == 'auto': + cmd += ' autoneg on' + else: + cmd += f' speed {speed} duplex {duplex} autoneg off' + return self._cmd(cmd) + except PermissionError: + # Some NICs do not tell that they don't suppport settings speed/duplex, + # but they do not actually support it either. + # In that case it's probably better to ignore the error + # than end up with a broken config. + print('Warning: could not set speed/duplex settings: operation not permitted!') def set_gro(self, state): """ diff --git a/python/vyos/ifconfig/geneve.py b/python/vyos/ifconfig/geneve.py index 7cb3968df..276c34cd7 100644 --- a/python/vyos/ifconfig/geneve.py +++ b/python/vyos/ifconfig/geneve.py @@ -42,7 +42,7 @@ class GeneveIf(Interface): # arguments used by iproute2. For more information please refer to: # - https://man7.org/linux/man-pages/man8/ip-link.8.html mapping = { - 'parameters.ip.dont_fragment': 'df set', + 'parameters.ip.df' : 'df', 'parameters.ip.tos' : 'tos', 'parameters.ip.ttl' : 'ttl', 'parameters.ipv6.flowlabel' : 'flowlabel', diff --git a/python/vyos/ifconfig/interface.py b/python/vyos/ifconfig/interface.py index 58d130ef6..22441d1d2 100755 --- a/python/vyos/ifconfig/interface.py +++ b/python/vyos/ifconfig/interface.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,7 +13,6 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. -from netifaces import interfaces import os import re import json @@ -37,6 +36,7 @@ from vyos.util import mac2eui64 from vyos.util import dict_search from vyos.util import read_file from vyos.util import get_interface_config +from vyos.util import get_interface_namespace from vyos.util import is_systemd_service_active from vyos.template import is_ipv4 from vyos.template import is_ipv6 @@ -135,6 +135,9 @@ class Interface(Control): 'validate': assert_mtu, 'shellcmd': 'ip link set dev {ifname} mtu {value}', }, + 'netns': { + 'shellcmd': 'ip link set dev {ifname} netns {value}', + }, 'vrf': { 'convert': lambda v: f'master {v}' if v else 'nomaster', 'shellcmd': 'ip link set dev {ifname} {value}', @@ -512,6 +515,35 @@ class Interface(Control): if prev_state == 'up': self.set_admin_state('up') + def del_netns(self, netns): + """ + Remove interface from given NETNS. + """ + + # If NETNS does not exist then there is nothing to delete + if not os.path.exists(f'/run/netns/{netns}'): + return None + + # As a PoC we only allow 'dummy' interfaces + if 'dum' not in self.ifname: + return None + + # Check if interface realy exists in namespace + if get_interface_namespace(self.ifname) != None: + self._cmd(f'ip netns exec {get_interface_namespace(self.ifname)} ip link del dev {self.ifname}') + return + + def set_netns(self, netns): + """ + Add interface from given NETNS. + + Example: + >>> from vyos.ifconfig import Interface + >>> Interface('dum0').set_netns('foo') + """ + + self.set_interface('netns', netns) + def set_vrf(self, vrf): """ Add/Remove interface from given VRF instance. @@ -544,6 +576,15 @@ class Interface(Control): return None return self.set_interface('arp_cache_tmo', tmo) + def _cleanup_mss_rules(self, table, ifname): + commands = [] + results = self._cmd(f'nft -a list chain {table} VYOS_TCP_MSS').split("\n") + for line in results: + if f'oifname "{ifname}"' in line: + handle_search = re.search('handle (\d+)', line) + if handle_search: + self._cmd(f'nft delete rule {table} VYOS_TCP_MSS handle {handle_search[1]}') + def set_tcp_ipv4_mss(self, mss): """ Set IPv4 TCP MSS value advertised when TCP SYN packets leave this @@ -555,22 +596,14 @@ class Interface(Control): >>> from vyos.ifconfig import Interface >>> Interface('eth0').set_tcp_ipv4_mss(1340) """ - iptables_bin = 'iptables' - base_options = f'-A FORWARD -o {self.ifname} -p tcp -m tcp --tcp-flags SYN,RST SYN' - out = self._cmd(f'{iptables_bin}-save -t mangle') - for line in out.splitlines(): - if line.startswith(base_options): - # remove OLD MSS mangling configuration - line = line.replace('-A FORWARD', '-D FORWARD') - self._cmd(f'{iptables_bin} -t mangle {line}') - - cmd_mss = f'{iptables_bin} -t mangle {base_options} --jump TCPMSS' + self._cleanup_mss_rules('raw', self.ifname) + nft_prefix = 'nft add rule raw VYOS_TCP_MSS' + base_cmd = f'oifname "{self.ifname}" tcp flags & (syn|rst) == syn' if mss == 'clamp-mss-to-pmtu': - self._cmd(f'{cmd_mss} --clamp-mss-to-pmtu') + self._cmd(f"{nft_prefix} '{base_cmd} tcp option maxseg size set rt mtu'") elif int(mss) > 0: - # probably add option to clamp only if bigger: low_mss = str(int(mss) + 1) - self._cmd(f'{cmd_mss} -m tcpmss --mss {low_mss}:65535 --set-mss {mss}') + self._cmd(f"{nft_prefix} '{base_cmd} tcp option maxseg size {low_mss}-65535 tcp option maxseg size set {mss}'") def set_tcp_ipv6_mss(self, mss): """ @@ -583,22 +616,14 @@ class Interface(Control): >>> from vyos.ifconfig import Interface >>> Interface('eth0').set_tcp_mss(1320) """ - iptables_bin = 'ip6tables' - base_options = f'-A FORWARD -o {self.ifname} -p tcp -m tcp --tcp-flags SYN,RST SYN' - out = self._cmd(f'{iptables_bin}-save -t mangle') - for line in out.splitlines(): - if line.startswith(base_options): - # remove OLD MSS mangling configuration - line = line.replace('-A FORWARD', '-D FORWARD') - self._cmd(f'{iptables_bin} -t mangle {line}') - - cmd_mss = f'{iptables_bin} -t mangle {base_options} --jump TCPMSS' + self._cleanup_mss_rules('ip6 raw', self.ifname) + nft_prefix = 'nft add rule ip6 raw VYOS_TCP_MSS' + base_cmd = f'oifname "{self.ifname}" tcp flags & (syn|rst) == syn' if mss == 'clamp-mss-to-pmtu': - self._cmd(f'{cmd_mss} --clamp-mss-to-pmtu') + self._cmd(f"{nft_prefix} '{base_cmd} tcp option maxseg size set rt mtu'") elif int(mss) > 0: - # probably add option to clamp only if bigger: low_mss = str(int(mss) + 1) - self._cmd(f'{cmd_mss} -m tcpmss --mss {low_mss}:65535 --set-mss {mss}') + self._cmd(f"{nft_prefix} '{base_cmd} tcp option maxseg size {low_mss}-65535 tcp option maxseg size set {mss}'") def set_arp_filter(self, arp_filter): """ @@ -711,8 +736,9 @@ class Interface(Control): if all_rp_filter == 1: global_setting = 'strict' elif all_rp_filter == 2: global_setting = 'loose' - print(f'WARNING: Global source-validation is set to "{global_setting}\n"' \ - 'this overrides per interface setting!') + from vyos.base import Warning + Warning(f'Global source-validation is set to "{global_setting} '\ + f'this overrides per interface setting!') tmp = self.get_interface('rp_filter') if int(tmp) == value: @@ -1054,16 +1080,17 @@ class Interface(Control): if addr in self._addr: return False - addr_is_v4 = is_ipv4(addr) - # add to interface if addr == 'dhcp': self.set_dhcp(True) elif addr == 'dhcpv6': self.set_dhcpv6(True) elif not is_intf_addr_assigned(self.ifname, addr): - self._cmd(f'ip addr add "{addr}" ' - f'{"brd + " if addr_is_v4 else ""}dev "{self.ifname}"') + tmp = f'ip addr add {addr} dev {self.ifname}' + # Add broadcast address for IPv4 + if is_ipv4(addr): tmp += ' brd +' + + self._cmd(tmp) else: return False @@ -1202,12 +1229,11 @@ class Interface(Control): options_file = f'{config_base}_{ifname}.options' pid_file = f'{config_base}_{ifname}.pid' lease_file = f'{config_base}_{ifname}.leases' - - # Stop client with old config files to get the right IF_METRIC. systemd_service = f'dhclient@{ifname}.service' - if is_systemd_service_active(systemd_service): - self._cmd(f'systemctl stop {systemd_service}') + # 'up' check is mandatory b/c even if the interface is A/D, as soon as + # the DHCP client is started the interface will be placed in u/u state. + # This is not what we intended to do when disabling an interface. if enable and 'disable' not in self._config: if dict_search('dhcp_options.host_name', self._config) == None: # read configured system hostname. @@ -1218,16 +1244,19 @@ class Interface(Control): tmp = {'dhcp_options' : { 'host_name' : hostname}} self._config = dict_merge(tmp, self._config) - render(options_file, 'dhcp-client/daemon-options.tmpl', - self._config) - render(config_file, 'dhcp-client/ipv4.tmpl', - self._config) + render(options_file, 'dhcp-client/daemon-options.j2', self._config) + render(config_file, 'dhcp-client/ipv4.j2', self._config) - # 'up' check is mandatory b/c even if the interface is A/D, as soon as - # the DHCP client is started the interface will be placed in u/u state. - # This is not what we intended to do when disabling an interface. - return self._cmd(f'systemctl restart {systemd_service}') + # When the DHCP client is restarted a brief outage will occur, as + # the old lease is released a new one is acquired (T4203). We will + # only restart DHCP client if it's option changed, or if it's not + # running, but it should be running (e.g. on system startup) + if 'dhcp_options_changed' in self._config or not is_systemd_service_active(systemd_service): + return self._cmd(f'systemctl restart {systemd_service}') + return None else: + if is_systemd_service_active(systemd_service): + self._cmd(f'systemctl stop {systemd_service}') # cleanup old config files for file in [config_file, options_file, pid_file, lease_file]: if os.path.isfile(file): @@ -1246,8 +1275,7 @@ class Interface(Control): systemd_service = f'dhcp6c@{ifname}.service' if enable and 'disable' not in self._config: - render(config_file, 'dhcp-client/ipv6.tmpl', - self._config) + render(config_file, 'dhcp-client/ipv6.j2', self._config) # We must ignore any return codes. This is required to enable # DHCPv6-PD for interfaces which are yet not up and running. @@ -1258,48 +1286,57 @@ class Interface(Control): if os.path.isfile(config_file): os.remove(config_file) - def set_mirror(self): + def set_mirror_redirect(self): # Please refer to the document for details # - https://man7.org/linux/man-pages/man8/tc.8.html # - https://man7.org/linux/man-pages/man8/tc-mirred.8.html # Depening if we are the source or the target interface of the port # mirror we need to setup some variables. source_if = self._config['ifname'] - config = self._config.get('mirror', None) + mirror_config = None + if 'mirror' in self._config: + mirror_config = self._config['mirror'] if 'is_mirror_intf' in self._config: source_if = next(iter(self._config['is_mirror_intf'])) - config = self._config['is_mirror_intf'][source_if].get('mirror', None) - - # Check configuration stored by old perl code before delete T3782 - if not 'redirect' in self._config: - # Please do not clear the 'set $? = 0 '. It's meant to force a return of 0 - # Remove existing mirroring rules - delete_tc_cmd = f'tc qdisc del dev {source_if} handle ffff: ingress 2> /dev/null;' - delete_tc_cmd += f'tc qdisc del dev {source_if} handle 1: root prio 2> /dev/null;' - delete_tc_cmd += 'set $?=0' - self._popen(delete_tc_cmd) - - # Bail out early if nothing needs to be configured - if not config: - return - - for direction, mirror_if in config.items(): - if mirror_if not in interfaces(): - continue - - if direction == 'ingress': - handle = 'ffff: ingress' - parent = 'ffff:' - elif direction == 'egress': - handle = '1: root prio' - parent = '1:' - - # Mirror egress traffic - mirror_cmd = f'tc qdisc add dev {source_if} handle {handle}; ' - # Export the mirrored traffic to the interface - mirror_cmd += f'tc filter add dev {source_if} parent {parent} protocol all prio 10 u32 match u32 0 0 flowid 1:1 action mirred egress mirror dev {mirror_if}' - self._popen(mirror_cmd) + mirror_config = self._config['is_mirror_intf'][source_if].get('mirror', None) + + redirect_config = None + + # clear existing ingess - ignore errors (e.g. "Error: Cannot find specified + # qdisc on specified device") - we simply cleanup all stuff here + self._popen(f'tc qdisc del dev {source_if} parent ffff: 2>/dev/null'); + self._popen(f'tc qdisc del dev {source_if} parent 1: 2>/dev/null'); + + # Apply interface mirror policy + if mirror_config: + for direction, target_if in mirror_config.items(): + if direction == 'ingress': + handle = 'ffff: ingress' + parent = 'ffff:' + elif direction == 'egress': + handle = '1: root prio' + parent = '1:' + + # Mirror egress traffic + mirror_cmd = f'tc qdisc add dev {source_if} handle {handle}; ' + # Export the mirrored traffic to the interface + mirror_cmd += f'tc filter add dev {source_if} parent {parent} protocol '\ + f'all prio 10 u32 match u32 0 0 flowid 1:1 action mirred '\ + f'egress mirror dev {target_if}' + _, err = self._popen(mirror_cmd) + if err: print('tc qdisc(filter for mirror port failed') + + # Apply interface traffic redirection policy + elif 'redirect' in self._config: + _, err = self._popen(f'tc qdisc add dev {source_if} handle ffff: ingress') + if err: print(f'tc qdisc add for redirect failed!') + + target_if = self._config['redirect'] + _, err = self._popen(f'tc filter add dev {source_if} parent ffff: protocol '\ + f'all prio 10 u32 match u32 0 0 flowid 1:1 action mirred '\ + f'egress redirect dev {target_if}') + if err: print('tc filter add for redirect failed') def set_xdp(self, state): """ @@ -1353,6 +1390,16 @@ class Interface(Control): if mac: self.set_mac(mac) + # If interface is connected to NETNS we don't have to check all other + # settings like MTU/IPv6/sysctl values, etc. + # Since the interface is pushed onto a separate logical stack + # Configure NETNS + if dict_search('netns', config) != None: + self.set_netns(config.get('netns', '')) + return + else: + self.del_netns(config.get('netns', '')) + # Update interface description self.set_alias(config.get('description', '')) @@ -1388,9 +1435,6 @@ class Interface(Control): else: self.del_addr(addr) - for addr in new_addr: - self.add_addr(addr) - # start DHCPv6 client when only PD was configured if dhcpv6pd: self.set_dhcpv6(True) @@ -1405,16 +1449,15 @@ class Interface(Control): # checked before self.set_vrf(config.get('vrf', '')) + # Add this section after vrf T4331 + for addr in new_addr: + self.add_addr(addr) + # Configure MSS value for IPv4 TCP connections tmp = dict_search('ip.adjust_mss', config) value = tmp if (tmp != None) else '0' self.set_tcp_ipv4_mss(value) - # Configure MSS value for IPv6 TCP connections - tmp = dict_search('ipv6.adjust_mss', config) - value = tmp if (tmp != None) else '0' - self.set_tcp_ipv6_mss(value) - # Configure ARP cache timeout in milliseconds - has default value tmp = dict_search('ip.arp_cache_timeout', config) value = tmp if (tmp != None) else '30' @@ -1460,6 +1503,18 @@ class Interface(Control): value = tmp if (tmp != None) else '0' self.set_ipv4_source_validation(value) + # MTU - Maximum Transfer Unit has a default value. It must ALWAYS be set + # before mangling any IPv6 option. If MTU is less then 1280 IPv6 will be + # automatically disabled by the kernel. Also MTU must be increased before + # configuring any IPv6 address on the interface. + if 'mtu' in config: + self.set_mtu(config.get('mtu')) + + # Configure MSS value for IPv6 TCP connections + tmp = dict_search('ipv6.adjust_mss', config) + value = tmp if (tmp != None) else '0' + self.set_tcp_ipv6_mss(value) + # IPv6 forwarding tmp = dict_search('ipv6.disable_forwarding', config) value = '0' if (tmp != None) else '1' @@ -1482,10 +1537,6 @@ class Interface(Control): value = tmp if (tmp != None) else '1' self.set_ipv6_dad_messages(value) - # MTU - Maximum Transfer Unit - if 'mtu' in config: - self.set_mtu(config.get('mtu')) - # Delete old IPv6 EUI64 addresses before changing MAC for addr in (dict_search('ipv6.address.eui64_old', config) or []): self.del_ipv6_eui64_address(addr) @@ -1510,8 +1561,8 @@ class Interface(Control): # eXpress Data Path - highly experimental self.set_xdp('xdp' in config) - # configure port mirror - self.set_mirror() + # configure interface mirror or redirection target + self.set_mirror_redirect() # Enable/Disable of an interface must always be done at the end of the # derived class to make use of the ref-counting set_admin_state() @@ -1536,12 +1587,10 @@ class Interface(Control): tmp['source_interface'] = ifname tmp['vlan_id'] = vif_s_id - vif_s_ifname = f'{ifname}.{vif_s_id}' - vif_s_config['ifname'] = vif_s_ifname - # It is not possible to change the VLAN encapsulation protocol # "on-the-fly". For this "quirk" we need to actively delete and # re-create the VIF-S interface. + vif_s_ifname = f'{ifname}.{vif_s_id}' if self.exists(vif_s_ifname): cur_cfg = get_interface_config(vif_s_ifname) protocol = dict_search('linkinfo.info_data.protocol', cur_cfg).lower() @@ -1563,7 +1612,6 @@ class Interface(Control): tmp['vlan_id'] = vif_c_id vif_c_ifname = f'{vif_s_ifname}.{vif_c_id}' - vif_c_config['ifname'] = vif_c_ifname c_vlan = VLANIf(vif_c_ifname, **tmp) c_vlan.update(vif_c_config) @@ -1574,10 +1622,7 @@ class Interface(Control): # create/update 802.1q VLAN interfaces for vif_id, vif_config in config.get('vif', {}).items(): - vif_ifname = f'{ifname}.{vif_id}' - vif_config['ifname'] = vif_ifname - tmp = deepcopy(VLANIf.get_config()) tmp['source_interface'] = ifname tmp['vlan_id'] = vif_id @@ -1670,6 +1715,3 @@ class VLANIf(Interface): return None return super().set_admin_state(state) - - def set_mirror(self): - return diff --git a/python/vyos/ifconfig/loopback.py b/python/vyos/ifconfig/loopback.py index 192c12f5c..b3babfadc 100644 --- a/python/vyos/ifconfig/loopback.py +++ b/python/vyos/ifconfig/loopback.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -23,7 +23,6 @@ class LoopbackIf(Interface): """ _persistent_addresses = ['127.0.0.1/8', '::1/128'] iftype = 'loopback' - definition = { **Interface.definition, **{ @@ -32,6 +31,7 @@ class LoopbackIf(Interface): 'bridgeable': True, } } + def remove(self): """ Loopback interface can not be deleted from operating system. We can @@ -57,12 +57,14 @@ class LoopbackIf(Interface): interface setup code and provide a single point of entry when workin on any interface. """ - addr = config.get('address', []) + address = config.get('address', []) # We must ensure that the loopback addresses are never deleted from the system - addr += self._persistent_addresses + for tmp in self._persistent_addresses: + if tmp not in address: + address.append(tmp) # Update IP address entry in our dictionary - config.update({'address' : addr}) + config.update({'address' : address}) # call base class super().update(config) diff --git a/python/vyos/ifconfig/pppoe.py b/python/vyos/ifconfig/pppoe.py index 1d13264bf..63ffc8069 100644 --- a/python/vyos/ifconfig/pppoe.py +++ b/python/vyos/ifconfig/pppoe.py @@ -27,12 +27,13 @@ class PPPoEIf(Interface): }, } - def _remove_routes(self, vrf=''): + def _remove_routes(self, vrf=None): # Always delete default routes when interface is removed + vrf_cmd = '' if vrf: - vrf = f'-c "vrf {vrf}"' - self._cmd(f'vtysh -c "conf t" {vrf} -c "no ip route 0.0.0.0/0 {self.ifname} tag 210"') - self._cmd(f'vtysh -c "conf t" {vrf} -c "no ipv6 route ::/0 {self.ifname} tag 210"') + vrf_cmd = f'-c "vrf {vrf}"' + self._cmd(f'vtysh -c "conf t" {vrf_cmd} -c "no ip route 0.0.0.0/0 {self.ifname} tag 210"') + self._cmd(f'vtysh -c "conf t" {vrf_cmd} -c "no ipv6 route ::/0 {self.ifname} tag 210"') def remove(self): """ @@ -44,11 +45,11 @@ class PPPoEIf(Interface): >>> i = Interface('pppoe0') >>> i.remove() """ - + vrf = None tmp = get_interface_config(self.ifname) - vrf = '' if 'master' in tmp: - self._remove_routes(tmp['master']) + vrf = tmp['master'] + self._remove_routes(vrf) # remove bond master which places members in disabled state super().remove() @@ -84,10 +85,12 @@ class PPPoEIf(Interface): self._config = config # remove old routes from an e.g. old VRF assignment - vrf = '' - if 'vrf_old' in config: - vrf = config['vrf_old'] - self._remove_routes(vrf) + if 'shutdown_required': + vrf = None + tmp = get_interface_config(self.ifname) + if 'master' in tmp: + vrf = tmp['master'] + self._remove_routes(vrf) # DHCPv6 PD handling is a bit different on PPPoE interfaces, as we do # not require an 'address dhcpv6' CLI option as with other interfaces @@ -98,54 +101,15 @@ class PPPoEIf(Interface): super().update(config) - if 'default_route' not in config or config['default_route'] == 'none': - return - - # - # Set default routes pointing to pppoe interface - # - vrf = '' - sed_opt = '^ip route' - - install_v4 = True - install_v6 = True - # generate proper configuration string when VRFs are in use + vrf = '' if 'vrf' in config: tmp = config['vrf'] vrf = f'-c "vrf {tmp}"' - sed_opt = f'vrf {tmp}' - - if config['default_route'] == 'auto': - # only add route if there is no default route present - tmp = self._cmd(f'vtysh -c "show running-config staticd no-header" | sed -n "/{sed_opt}/,/!/p"') - for line in tmp.splitlines(): - line = line.lstrip() - if line.startswith('ip route 0.0.0.0/0'): - install_v4 = False - continue - - if 'ipv6' in config and line.startswith('ipv6 route ::/0'): - install_v6 = False - continue - - elif config['default_route'] == 'force': - # Force means that all static routes are replaced with the ones from this interface - tmp = self._cmd(f'vtysh -c "show running-config staticd no-header" | sed -n "/{sed_opt}/,/!/p"') - for line in tmp.splitlines(): - if self.ifname in line: - # It makes no sense to remove a route with our interface and the later re-add it. - # This will only make traffic disappear - which is a no-no! - continue - - line = line.lstrip() - if line.startswith('ip route 0.0.0.0/0'): - self._cmd(f'vtysh -c "conf t" {vrf} -c "no {line}"') - - if 'ipv6' in config and line.startswith('ipv6 route ::/0'): - self._cmd(f'vtysh -c "conf t" {vrf} -c "no {line}"') - - if install_v4: - self._cmd(f'vtysh -c "conf t" {vrf} -c "ip route 0.0.0.0/0 {self.ifname} tag 210"') - if install_v6 and 'ipv6' in config: - self._cmd(f'vtysh -c "conf t" {vrf} -c "ipv6 route ::/0 {self.ifname} tag 210"') + + if 'no_default_route' not in config: + # Set default route(s) pointing to PPPoE interface + distance = config['default_route_distance'] + self._cmd(f'vtysh -c "conf t" {vrf} -c "ip route 0.0.0.0/0 {self.ifname} tag 210 {distance}"') + if 'ipv6' in config: + self._cmd(f'vtysh -c "conf t" {vrf} -c "ipv6 route ::/0 {self.ifname} tag 210 {distance}"') diff --git a/python/vyos/ifconfig/section.py b/python/vyos/ifconfig/section.py index 0e4447b9e..91f667b65 100644 --- a/python/vyos/ifconfig/section.py +++ b/python/vyos/ifconfig/section.py @@ -52,12 +52,12 @@ class Section: name: name of the interface vlan: if vlan is True, do not stop at the vlan number """ - name = name.rstrip('0123456789') - name = name.rstrip('.') - if vlan: - name = name.rstrip('0123456789.') if vrrp: - name = name.rstrip('0123456789v') + name = re.sub(r'\d(\d|v|\.)*$', '', name) + elif vlan: + name = re.sub(r'\d(\d|\.)*$', '', name) + else: + name = re.sub(r'\d+$', '', name) return name @classmethod diff --git a/python/vyos/ifconfig/vxlan.py b/python/vyos/ifconfig/vxlan.py index d73fb47b8..5baff10a9 100644 --- a/python/vyos/ifconfig/vxlan.py +++ b/python/vyos/ifconfig/vxlan.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -54,18 +54,31 @@ class VXLANIf(Interface): # arguments used by iproute2. For more information please refer to: # - https://man7.org/linux/man-pages/man8/ip-link.8.html mapping = { - 'source_address' : 'local', - 'source_interface' : 'dev', - 'remote' : 'remote', 'group' : 'group', - 'parameters.ip.dont_fragment': 'df set', + 'external' : 'external', + 'gpe' : 'gpe', + 'parameters.ip.df' : 'df', 'parameters.ip.tos' : 'tos', 'parameters.ip.ttl' : 'ttl', 'parameters.ipv6.flowlabel' : 'flowlabel', 'parameters.nolearning' : 'nolearning', + 'remote' : 'remote', + 'source_address' : 'local', + 'source_interface' : 'dev', + 'vni' : 'id', } - cmd = 'ip link add {ifname} type {type} id {vni} dstport {port}' + # IPv6 flowlabels can only be used on IPv6 tunnels, thus we need to + # ensure that at least the first remote IP address is passed to the + # tunnel creation command. Subsequent tunnel remote addresses can later + # be added to the FDB + remote_list = None + if 'remote' in self.config: + # skip first element as this is already configured as remote + remote_list = self.config['remote'][1:] + self.config['remote'] = self.config['remote'][0] + + cmd = 'ip link add {ifname} type {type} dstport {port}' for vyos_key, iproute2_key in mapping.items(): # dict_search will return an empty dict "{}" for valueless nodes like # "parameters.nolearning" - thus we need to test the nodes existence @@ -79,3 +92,10 @@ class VXLANIf(Interface): self._cmd(cmd.format(**self.config)) # interface is always A/D down. It needs to be enabled explicitly self.set_admin_state('down') + + # VXLAN tunnel is always recreated on any change - see interfaces-vxlan.py + if remote_list: + for remote in remote_list: + cmd = f'bridge fdb append to 00:00:00:00:00:00 dst {remote} ' \ + 'port {port} dev {ifname}' + self._cmd(cmd.format(**self.config)) diff --git a/python/vyos/ifconfig/wireless.py b/python/vyos/ifconfig/wireless.py index 748b6e02d..88eaa772b 100644 --- a/python/vyos/ifconfig/wireless.py +++ b/python/vyos/ifconfig/wireless.py @@ -49,10 +49,10 @@ class WiFiIf(Interface): on any interface. """ # We can not call add_to_bridge() until wpa_supplicant is running, thus - # we will remove the key from the config dict and react to this specal - # case in thie derived class. + # we will remove the key from the config dict and react to this special + # case in this derived class. # re-add ourselves to any bridge we might have fallen out of - bridge_member = '' + bridge_member = None if 'is_bridge_member' in config: bridge_member = config['is_bridge_member'] del config['is_bridge_member'] diff --git a/python/vyos/migrator.py b/python/vyos/migrator.py index 4574bb6d1..a2e0daabd 100644 --- a/python/vyos/migrator.py +++ b/python/vyos/migrator.py @@ -195,7 +195,7 @@ class Migrator(object): # This will force calling all migration scripts: cfg_versions = {} - sys_versions = systemversions.get_system_versions() + sys_versions = systemversions.get_system_component_version() # save system component versions in json file for easy reference self.save_json_record(sys_versions) diff --git a/python/vyos/pki.py b/python/vyos/pki.py index 68ad73bf2..0b916eaae 100644 --- a/python/vyos/pki.py +++ b/python/vyos/pki.py @@ -331,3 +331,29 @@ def verify_certificate(cert, ca_cert): return True except InvalidSignature: return False + +# Certificate chain + +def find_parent(cert, ca_certs): + for ca_cert in ca_certs: + if verify_certificate(cert, ca_cert): + return ca_cert + return None + +def find_chain(cert, ca_certs): + remaining = ca_certs.copy() + chain = [cert] + + while remaining: + parent = find_parent(chain[-1], remaining) + if parent is None: + # No parent in the list of remaining certificates or there's a circular dependency + break + elif parent == chain[-1]: + # Self-signed: must be root CA (end of chain) + break + else: + remaining.remove(parent) + chain.append(parent) + + return chain diff --git a/python/vyos/range_regex.py b/python/vyos/range_regex.py new file mode 100644 index 000000000..a8190d140 --- /dev/null +++ b/python/vyos/range_regex.py @@ -0,0 +1,142 @@ +'''Copyright (c) 2013, Dmitry Voronin +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +''' +import math + +# coding=utf8 + +# Split range to ranges that has its unique pattern. +# Example for 12-345: +# +# 12- 19: 1[2-9] +# 20- 99: [2-9]\d +# 100-299: [1-2]\d{2} +# 300-339: 3[0-3]\d +# 340-345: 34[0-5] + +def range_to_regex(inpt_range): + if isinstance(inpt_range, str): + range_list = inpt_range.split('-') + # Check input arguments + if len(range_list) == 2: + # The first element in range must be higher then the second + if int(range_list[0]) < int(range_list[1]): + return regex_for_range(int(range_list[0]), int(range_list[1])) + + return None + +def bounded_regex_for_range(min_, max_): + return r'\b({})\b'.format(regex_for_range(min_, max_)) + +def regex_for_range(min_, max_): + """ + > regex_for_range(12, 345) + '1[2-9]|[2-9]\d|[1-2]\d{2}|3[0-3]\d|34[0-5]' + """ + positive_subpatterns = [] + negative_subpatterns = [] + + if min_ < 0: + min__ = 1 + if max_ < 0: + min__ = abs(max_) + max__ = abs(min_) + + negative_subpatterns = split_to_patterns(min__, max__) + min_ = 0 + + if max_ >= 0: + positive_subpatterns = split_to_patterns(min_, max_) + + negative_only_subpatterns = ['-' + val for val in negative_subpatterns if val not in positive_subpatterns] + positive_only_subpatterns = [val for val in positive_subpatterns if val not in negative_subpatterns] + intersected_subpatterns = ['-?' + val for val in negative_subpatterns if val in positive_subpatterns] + + subpatterns = negative_only_subpatterns + intersected_subpatterns + positive_only_subpatterns + return '|'.join(subpatterns) + + +def split_to_patterns(min_, max_): + subpatterns = [] + + start = min_ + for stop in split_to_ranges(min_, max_): + subpatterns.append(range_to_pattern(start, stop)) + start = stop + 1 + + return subpatterns + + +def split_to_ranges(min_, max_): + stops = {max_} + + nines_count = 1 + stop = fill_by_nines(min_, nines_count) + while min_ <= stop < max_: + stops.add(stop) + + nines_count += 1 + stop = fill_by_nines(min_, nines_count) + + zeros_count = 1 + stop = fill_by_zeros(max_ + 1, zeros_count) - 1 + while min_ < stop <= max_: + stops.add(stop) + + zeros_count += 1 + stop = fill_by_zeros(max_ + 1, zeros_count) - 1 + + stops = list(stops) + stops.sort() + + return stops + + +def fill_by_nines(integer, nines_count): + return int(str(integer)[:-nines_count] + '9' * nines_count) + + +def fill_by_zeros(integer, zeros_count): + return integer - integer % 10 ** zeros_count + + +def range_to_pattern(start, stop): + pattern = '' + any_digit_count = 0 + + for start_digit, stop_digit in zip(str(start), str(stop)): + if start_digit == stop_digit: + pattern += start_digit + elif start_digit != '0' or stop_digit != '9': + pattern += '[{}-{}]'.format(start_digit, stop_digit) + else: + any_digit_count += 1 + + if any_digit_count: + pattern += r'\d' + + if any_digit_count > 1: + pattern += '{{{}}}'.format(any_digit_count) + + return pattern
\ No newline at end of file diff --git a/python/vyos/remote.py b/python/vyos/remote.py index 2419f8873..66044fa52 100644 --- a/python/vyos/remote.py +++ b/python/vyos/remote.py @@ -74,20 +74,6 @@ class SourceAdapter(HTTPAdapter): num_pools=connections, maxsize=maxsize, block=block, source_address=self._source_pair) -class WrappedFile: - def __init__(self, obj, size=None, chunk_size=CHUNK_SIZE): - self._obj = obj - self._progress = size and make_incremental_progressbar(chunk_size / size) - def read(self, size=-1): - if self._progress: - next(self._progress) - self._obj.read(size) - def write(self, size=-1): - if self._progress: - next(self._progress) - self._obj.write(size) - def __getattr__(self, attr): - return getattr(self._obj, attr) def check_storage(path, size): """ @@ -97,8 +83,7 @@ def check_storage(path, size): directory = path if os.path.isdir(path) else (os.path.dirname(os.path.expanduser(path)) or os.getcwd()) # `size` can be None or 0 to indicate unknown size. if not size: - print_error('Warning: Cannot determine size of remote file.') - print_error('Bravely continuing regardless.') + print_error('Warning: Cannot determine size of remote file. Bravely continuing regardless.') return if size < 1024 * 1024: @@ -236,9 +221,13 @@ class HttpC: # Not only would it potentially mess up with the progress bar but # `shutil.copyfileobj(request.raw, file)` does not handle automatic decoding. s.headers.update({'Accept-Encoding': 'identity'}) - with s.head(self.urlstring) as r: + with s.head(self.urlstring, allow_redirects=True) as r: # Abort early if the destination is inaccessible. r.raise_for_status() + # If the request got redirected, keep the last URL we ended up with. + final_urlstring = r.url + if r.history and self.progressbar: + print_error('Redirecting to ' + final_urlstring) # Check for the prospective file size. try: size = int(r.headers['Content-Length']) @@ -247,7 +236,7 @@ class HttpC: size = None if self.check_space: check_storage(location, size) - with s.get(self.urlstring, stream=True) as r, open(location, 'wb') as f: + with s.get(final_urlstring, stream=True) as r, open(location, 'wb') as f: if self.progressbar and size: progress = make_incremental_progressbar(CHUNK_SIZE / size) next(progress) @@ -259,10 +248,9 @@ class HttpC: shutil.copyfileobj(r.raw, f) def upload(self, location: str): - size = os.path.getsize(location) if self.progressbar else None - # Keep in mind that `data` can be a file-like or iterable object. - with self._establish() as s, file(location, 'rb') as f: - s.post(self.urlstring, data=WrappedFile(f, size)) + # Does not yet support progressbars. + with self._establish() as s, open(location, 'rb') as f: + s.post(self.urlstring, data=f, allow_redirects=True) class TftpC: @@ -324,15 +312,16 @@ def friendly_download(local_path, urlstring, source_host='', source_port=0): print_error('Downloading...') download(local_path, urlstring, True, True, source_host, source_port) except KeyboardInterrupt: - print_error('Download aborted by user.') + print_error('\nDownload aborted by user.') sys.exit(1) except: import traceback + print_error(f'Failed to download {urlstring}.') # There are a myriad different reasons a download could fail. # SSH errors, FTP errors, I/O errors, HTTP errors (403, 404...) # We omit the scary stack trace but print the error nevertheless. - print_error(f'Failed to download {urlstring}.') - traceback.print_exception(*sys.exc_info()[:2], None) + exc_type, exc_value, exc_traceback = sys.exc_info() + traceback.print_exception(exc_type, exc_value, None, 0, None, False) sys.exit(1) else: print_error('Download complete.') diff --git a/python/vyos/systemversions.py b/python/vyos/systemversions.py index 9b3f4f413..f2da76d4f 100644 --- a/python/vyos/systemversions.py +++ b/python/vyos/systemversions.py @@ -17,7 +17,10 @@ import os import re import sys import vyos.defaults +from vyos.xml import component_version +# legacy version, reading from the file names in +# /opt/vyatta/etc/config-migrate/current def get_system_versions(): """ Get component versions from running system; critical failure if @@ -37,3 +40,7 @@ def get_system_versions(): system_versions[pair[0]] = int(pair[1]) return system_versions + +# read from xml cache +def get_system_component_version(): + return component_version() diff --git a/python/vyos/template.py b/python/vyos/template.py index b32cafe74..132f5ddde 100644 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -18,14 +18,16 @@ import os from jinja2 import Environment from jinja2 import FileSystemLoader - +from jinja2 import ChainableUndefined from vyos.defaults import directories from vyos.util import chmod from vyos.util import chown +from vyos.util import dict_search_args from vyos.util import makedir # Holds template filters registered via register_filter() _FILTERS = {} +_TESTS = {} # reuse Environments with identical settings to improve performance @functools.lru_cache(maxsize=2) @@ -41,8 +43,10 @@ def _get_environment(location=None): cache_size=100, loader=loc_loader, trim_blocks=True, + undefined=ChainableUndefined, ) env.filters.update(_FILTERS) + env.tests.update(_TESTS) return env @@ -66,6 +70,26 @@ def register_filter(name, func=None): _FILTERS[name] = func return func +def register_test(name, func=None): + """Register a function to be available as test in templates under given name. + + It can also be used as a decorator, see below in this module for examples. + + :raise RuntimeError: + when trying to register a test after a template has been rendered already + :raise ValueError: when trying to register a name which was taken already + """ + if func is None: + return functools.partial(register_test, name) + if _get_environment.cache_info().currsize: + raise RuntimeError( + "Tests can only be registered before rendering the first template" + ) + if name in _TESTS: + raise ValueError(f"A test with name {name!r} was registered already") + _TESTS[name] = func + return func + def render_to_string(template, content, formater=None, location=None): """Render a template from the template directory, raise on any errors. @@ -126,6 +150,14 @@ def render( ################################## # Custom template filters follow # ################################## +@register_filter('force_to_list') +def force_to_list(value): + """ Convert scalars to single-item lists and leave lists untouched """ + if isinstance(value, list): + return value + else: + return [value] + @register_filter('ip_from_cidr') def ip_from_cidr(prefix): """ Take an IPv4/IPv6 CIDR host and strip cidr mask. @@ -151,6 +183,16 @@ def bracketize_ipv6(address): return f'[{address}]' return address +@register_filter('dot_colon_to_dash') +def dot_colon_to_dash(text): + """ Replace dot and colon to dash for string + Example: + 192.0.2.1 => 192-0-2-1, 2001:db8::1 => 2001-db8--1 + """ + text = text.replace(":", "-") + text = text.replace(".", "-") + return text + @register_filter('netmask_from_cidr') def netmask_from_cidr(prefix): """ Take CIDR prefix and convert the prefix length to a "subnet mask". @@ -479,3 +521,123 @@ def get_openvpn_ncp_ciphers(ciphers): else: out.append(cipher) return ':'.join(out).upper() + +@register_filter('snmp_auth_oid') +def snmp_auth_oid(type): + if type not in ['md5', 'sha', 'aes', 'des', 'none']: + raise ValueError() + + OIDs = { + 'md5' : '.1.3.6.1.6.3.10.1.1.2', + 'sha' : '.1.3.6.1.6.3.10.1.1.3', + 'aes' : '.1.3.6.1.6.3.10.1.2.4', + 'des' : '.1.3.6.1.6.3.10.1.2.2', + 'none': '.1.3.6.1.6.3.10.1.2.1' + } + return OIDs[type] + +@register_filter('nft_action') +def nft_action(vyos_action): + if vyos_action == 'accept': + return 'return' + return vyos_action + +@register_filter('nft_rule') +def nft_rule(rule_conf, fw_name, rule_id, ip_name='ip'): + from vyos.firewall import parse_rule + return parse_rule(rule_conf, fw_name, rule_id, ip_name) + +@register_filter('nft_default_rule') +def nft_default_rule(fw_conf, fw_name): + output = ['counter'] + default_action = fw_conf.get('default_action', 'accept') + + if 'enable_default_log' in fw_conf: + action_suffix = default_action[:1].upper() + output.append(f'log prefix "[{fw_name[:19]}-default-{action_suffix}] "') + + output.append(nft_action(default_action)) + output.append(f'comment "{fw_name} default-action {default_action}"') + return " ".join(output) + +@register_filter('nft_state_policy') +def nft_state_policy(conf, state, ipv6=False): + out = [f'ct state {state}'] + + if 'log' in conf and 'enable' in conf['log']: + out.append('log') + + out.append('counter') + + if 'action' in conf: + out.append(conf['action']) + + return " ".join(out) + +@register_filter('nft_intra_zone_action') +def nft_intra_zone_action(zone_conf, ipv6=False): + if 'intra_zone_filtering' in zone_conf: + intra_zone = zone_conf['intra_zone_filtering'] + fw_name = 'ipv6_name' if ipv6 else 'name' + name_prefix = 'NAME6_' if ipv6 else 'NAME_' + + if 'action' in intra_zone: + if intra_zone['action'] == 'accept': + return 'return' + return intra_zone['action'] + elif dict_search_args(intra_zone, 'firewall', fw_name): + name = dict_search_args(intra_zone, 'firewall', fw_name) + return f'jump {name_prefix}{name}' + return 'return' + +@register_test('vyos_defined') +def vyos_defined(value, test_value=None, var_type=None): + """ + Jinja2 plugin to test if a variable is defined and not none - vyos_defined + will test value if defined and is not none and return true or false. + + If test_value is supplied, the value must also pass == test_value to return true. + If var_type is supplied, the value must also be of the specified class/type + + Examples: + 1. Test if var is defined and not none: + {% if foo is vyos_defined %} + ... + {% endif %} + + 2. Test if variable is defined, not none and has value "something" + {% if bar is vyos_defined("something") %} + ... + {% endif %} + + Parameters + ---------- + value : any + Value to test from ansible + test_value : any, optional + Value to test in addition of defined and not none, by default None + var_type : ['float', 'int', 'str', 'list', 'dict', 'tuple', 'bool'], optional + Type or Class to test for + + Returns + ------- + boolean + True if variable matches criteria, False in other cases. + + Implementation inspired and re-used from https://github.com/aristanetworks/ansible-avd/ + """ + + from jinja2 import Undefined + + if isinstance(value, Undefined) or value is None: + # Invalid value - return false + return False + elif test_value is not None and value != test_value: + # Valid value but not matching the optional argument + return False + elif str(var_type).lower() in ['float', 'int', 'str', 'list', 'dict', 'tuple', 'bool'] and str(var_type).lower() != type(value).__name__: + # Invalid class - return false + return False + else: + # Valid value and is matching optional argument if provided - return true + return True diff --git a/python/vyos/util.py b/python/vyos/util.py index d8e83ab8d..de55e108b 100644 --- a/python/vyos/util.py +++ b/python/vyos/util.py @@ -774,6 +774,14 @@ def dict_search_recursive(dict_object, key): for x in dict_search_recursive(j, key): yield x +def get_bridge_fdb(interface): + """ Returns the forwarding database entries for a given interface """ + if not os.path.exists(f'/sys/class/net/{interface}'): + return None + from json import loads + tmp = loads(cmd(f'bridge -j fdb show dev {interface}')) + return tmp + def get_interface_config(interface): """ Returns the used encapsulation protocol for given interface. If interface does not exist, None is returned. @@ -794,6 +802,24 @@ def get_interface_address(interface): tmp = loads(cmd(f'ip -d -j addr show {interface}'))[0] return tmp +def get_interface_namespace(iface): + """ + Returns wich netns the interface belongs to + """ + from json import loads + # Check if netns exist + tmp = loads(cmd(f'ip --json netns ls')) + if len(tmp) == 0: + return None + + for ns in tmp: + namespace = f'{ns["name"]}' + # Search interface in each netns + data = loads(cmd(f'ip netns exec {namespace} ip -j link show')) + for compare in data: + if iface == compare["ifname"]: + return namespace + def get_all_vrfs(): """ Return a dictionary of all system wide known VRF instances """ from json import loads @@ -934,14 +960,23 @@ def install_into_config(conf, config_paths, override_prompt=True): return None count = 0 + failed = [] for path in config_paths: if override_prompt and conf.exists(path) and not conf.is_multi(path): if not ask_yes_no(f'Config node "{node}" already exists. Do you want to overwrite it?'): continue - cmd(f'/opt/vyatta/sbin/my_set {path}') - count += 1 + try: + cmd(f'/opt/vyatta/sbin/my_set {path}') + count += 1 + except: + failed.append(path) + + if failed: + print(f'Failed to install {len(failed)} value(s). Commands to manually install:') + for path in failed: + print(f'set {path}') if count > 0: print(f'{count} value(s) installed. Use "compare" to see the pending changes, and "commit" to apply.') @@ -954,6 +989,11 @@ def is_wwan_connected(interface): if not interface.startswith('wwan'): raise ValueError(f'Specified interface "{interface}" is not a WWAN interface') + # ModemManager is required for connection(s) - if service is not running, + # there won't be any connection at all! + if not is_systemd_service_active('ModemManager.service'): + return False + modem = interface.lstrip('wwan') tmp = cmd(f'mmcli --modem {modem} --output-json') @@ -961,3 +1001,26 @@ def is_wwan_connected(interface): # return True/False if interface is in connected state return dict_search('modem.generic.state', tmp) == 'connected' + +def boot_configuration_complete() -> bool: + """ Check if the boot config loader has completed + """ + from vyos.defaults import config_status + + if os.path.isfile(config_status): + return True + return False + +def sysctl_read(name): + """ Read and return current value of sysctl() option """ + tmp = cmd(f'sysctl {name}') + return tmp.split()[-1] + +def sysctl_write(name, value): + """ Change value via sysctl() - return True if changed, False otherwise """ + tmp = cmd(f'sysctl {name}') + # last list index contains the actual value - only write if value differs + if sysctl_read(name) != str(value): + call(f'sysctl -wq {name}={value}') + return True + return False diff --git a/python/vyos/validate.py b/python/vyos/validate.py index 0dad2a6cb..e005da0e4 100644 --- a/python/vyos/validate.py +++ b/python/vyos/validate.py @@ -43,19 +43,14 @@ def _are_same_ip(one, two): s_two = AF_INET if is_ipv4(two) else AF_INET6 return inet_pton(f_one, one) == inet_pton(f_one, two) -def is_intf_addr_assigned(intf, addr): - if '/' in addr: - ip,mask = addr.split('/') - return _is_intf_addr_assigned(intf, ip, mask) - return _is_intf_addr_assigned(intf, addr) - -def _is_intf_addr_assigned(intf, address, netmask=None): +def is_intf_addr_assigned(intf, address) -> bool: """ Verify if the given IPv4/IPv6 address is assigned to specific interface. It can check both a single IP address (e.g. 192.0.2.1 or a assigned CIDR address 192.0.2.1/24. """ from vyos.template import is_ipv4 + from netifaces import ifaddresses from netifaces import AF_INET from netifaces import AF_INET6 @@ -76,6 +71,9 @@ def _is_intf_addr_assigned(intf, address, netmask=None): addr_type = AF_INET if is_ipv4(address) else AF_INET6 # Check every IP address on this interface for a match + netmask = None + if '/' in address: + address, netmask = address.split('/') for ip in ifaces.get(addr_type,[]): # ip can have the interface name in the 'addr' field, we need to remove it # {'addr': 'fe80::a00:27ff:fec5:f821%eth2', 'netmask': 'ffff:ffff:ffff:ffff::'} @@ -99,14 +97,20 @@ def _is_intf_addr_assigned(intf, address, netmask=None): return False -def is_addr_assigned(addr): - """ - Verify if the given IPv4/IPv6 address is assigned to any interface - """ +def is_addr_assigned(ip_address, vrf=None) -> bool: + """ Verify if the given IPv4/IPv6 address is assigned to any interfac """ from netifaces import interfaces - for intf in interfaces(): - tmp = is_intf_addr_assigned(intf, addr) - if tmp == True: + from vyos.util import get_interface_config + from vyos.util import dict_search + for interface in interfaces(): + # Check if interface belongs to the requested VRF, if this is not the + # case there is no need to proceed with this data set - continue loop + # with next element + tmp = get_interface_config(interface) + if dict_search('master', tmp) != vrf: + continue + + if is_intf_addr_assigned(interface, ip_address): return True return False diff --git a/python/vyos/xml/__init__.py b/python/vyos/xml/__init__.py index e0eacb2d1..6db446a40 100644 --- a/python/vyos/xml/__init__.py +++ b/python/vyos/xml/__init__.py @@ -46,8 +46,8 @@ def is_tag(lpath): def is_leaf(lpath, flat=True): return load_configuration().is_leaf(lpath, flat) -def component_versions(): - return load_configuration().component_versions() +def component_version(): + return load_configuration().component_version() def defaults(lpath, flat=False): return load_configuration().defaults(lpath, flat) diff --git a/python/vyos/xml/definition.py b/python/vyos/xml/definition.py index 5e0d5282c..bc3892b42 100644 --- a/python/vyos/xml/definition.py +++ b/python/vyos/xml/definition.py @@ -249,10 +249,11 @@ class XML(dict): # @lru_cache(maxsize=100) # XXX: need to use cachetool instead - for later - def component_versions(self) -> dict: - sort_component = sorted(self[kw.component_version].items(), - key = lambda kv: kv[0]) - return dict(sort_component) + def component_version(self) -> dict: + d = {} + for k in sorted(self[kw.component_version]): + d[k] = int(self[kw.component_version][k]) + return d def defaults(self, lpath, flat): d = self[kw.default] |