diff options
Diffstat (limited to 'python/vyos')
30 files changed, 1869 insertions, 1116 deletions
diff --git a/python/vyos/config.py b/python/vyos/config.py index 780b48a7b..de79a3654 100644 --- a/python/vyos/config.py +++ b/python/vyos/config.py @@ -63,22 +63,14 @@ In operational mode, all functions return values from the running config. """ -import os import re import json -import subprocess +from copy import deepcopy +import vyos.xml import vyos.util import vyos.configtree - -class VyOSError(Exception): - """ - Raised on config access errors, most commonly if the type of a config tree node - in the system does not match the type of operation. - - """ - pass - +from vyos.configsource import ConfigSource, ConfigSourceSession class Config(object): """ @@ -88,49 +80,18 @@ class Config(object): the only state it keeps is relative *config path* for convenient access to config subtrees. """ - def __init__(self, session_env=None): - self._cli_shell_api = "/bin/cli-shell-api" - self._level = [] - if session_env: - self.__session_env = session_env - else: - self.__session_env = None - - # Running config can be obtained either from op or conf mode, it always succeeds - # once the config system is initialized during boot; - # before initialization, set to empty string - if os.path.isfile('/tmp/vyos-config-status'): - try: - running_config_text = self._run([self._cli_shell_api, '--show-active-only', '--show-show-defaults', '--show-ignore-edit', 'showConfig']) - except VyOSError: - running_config_text = '' - else: - running_config_text = '' - - # Session config ("active") only exists in conf mode. - # In op mode, we'll just use the same running config for both active and session configs. - if self.in_session(): - try: - session_config_text = self._run([self._cli_shell_api, '--show-working-only', '--show-show-defaults', '--show-ignore-edit', 'showConfig']) - except VyOSError: - session_config_text = '' + def __init__(self, session_env=None, config_source=None): + if config_source is None: + self._config_source = ConfigSourceSession(session_env) else: - session_config_text = running_config_text + if not isinstance(config_source, ConfigSource): + raise TypeError("config_source not of type ConfigSource") + self._config_source = config_source - if running_config_text: - self._running_config = vyos.configtree.ConfigTree(running_config_text) - else: - self._running_config = None - - if session_config_text: - self._session_config = vyos.configtree.ConfigTree(session_config_text) - else: - self._session_config = None - - def _make_command(self, op, path): - args = path.split() - cmd = [self._cli_shell_api, op] + args - return cmd + self._level = [] + self._dict_cache = {} + (self._running_config, + self._session_config) = self._config_source.get_configtree_tuple() def _make_path(self, path): # Backwards-compatibility stuff: original implementation used string paths @@ -146,19 +107,6 @@ class Config(object): raise TypeError("Path must be a whitespace-separated string or a list") return (self._level + path) - def _run(self, cmd): - if self.__session_env: - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=self.__session_env) - else: - p = subprocess.Popen(cmd, stdout=subprocess.PIPE) - out = p.stdout.read() - p.wait() - p.communicate() - if p.returncode != 0: - raise VyOSError() - else: - return out.decode('ascii') - def set_level(self, path): """ Set the *edit level*, that is, a relative config tree path. @@ -226,22 +174,14 @@ class Config(object): Returns: True if the config session has uncommited changes, False otherwise. """ - try: - self._run(self._make_command('sessionChanged', '')) - return True - except VyOSError: - return False + return self._config_source.session_changed() def in_session(self): """ Returns: True if called from a configuration session, False otherwise. """ - try: - self._run(self._make_command('inSession', '')) - return True - except VyOSError: - return False + return self._config_source.in_session() def show_config(self, path=[], default=None, effective=False): """ @@ -252,72 +192,64 @@ class Config(object): Returns: str: working configuration """ + return self._config_source.show_config(path, default, effective) - # show_config should be independent of CLI edit level. - # Set the CLI edit environment to the top level, and - # restore original on exit. - save_env = self.__session_env - - env_str = self._run(self._make_command('getEditResetEnv', '')) - env_list = re.findall(r'([A-Z_]+)=\'([^;\s]+)\'', env_str) - root_env = os.environ - for k, v in env_list: - root_env[k] = v - - self.__session_env = root_env + def get_cached_root_dict(self, effective=False): + cached = self._dict_cache.get(effective, {}) + if cached: + return cached - # FIXUP: by default, showConfig will give you a diff - # if there are uncommitted changes. - # The config parser obviously cannot work with diffs, - # so we need to supress diff production using appropriate - # options for getting either running (active) - # or proposed (working) config. if effective: - path = ['--show-active-only'] + path + config = self._running_config else: - path = ['--show-working-only'] + path - - if isinstance(path, list): - path = " ".join(path) - try: - out = self._run(self._make_command('showConfig', path)) - self.__session_env = save_env - return out - except VyOSError: - self.__session_env = save_env - return(default) + config = self._session_config + + if config: + config_dict = json.loads(config.to_json()) + else: + config_dict = {} - def get_config_dict(self, path=[], effective=False, key_mangling=None, get_first_key=False): + self._dict_cache[effective] = config_dict + + return config_dict + + def get_config_dict(self, path=[], effective=False, key_mangling=None, + get_first_key=False, no_multi_convert=False): """ Args: path (str list): Configuration tree path, can be empty effective=False: effective or session config key_mangling=None: mangle dict keys according to regex and replacement get_first_key=False: if k = path[:-1], return sub-dict d[k] instead of {k: d[k]} + no_multi_convert=False: if convert, return single value of multi node as list Returns: a dict representation of the config under path """ - config_dict = {} + lpath = self._make_path(path) + root_dict = self.get_cached_root_dict(effective) + conf_dict = vyos.util.get_sub_dict(root_dict, lpath, get_first_key) - if effective: - if self._running_config: - config_dict = json.loads((self._running_config).to_json()) - else: - if self._session_config: - config_dict = json.loads((self._session_config).to_json()) + if not key_mangling and no_multi_convert: + return deepcopy(conf_dict) - config_dict = vyos.util.get_sub_dict(config_dict, self._make_path(path), get_first_key) + xmlpath = lpath if get_first_key else lpath[:-1] - if key_mangling: - if not (isinstance(key_mangling, tuple) and \ - (len(key_mangling) == 2) and \ - isinstance(key_mangling[0], str) and \ - isinstance(key_mangling[1], str)): - raise ValueError("key_mangling must be a tuple of two strings") - else: - config_dict = vyos.util.mangle_dict_keys(config_dict, key_mangling[0], key_mangling[1]) + if not key_mangling: + conf_dict = vyos.xml.multi_to_list(xmlpath, conf_dict) + return conf_dict - return config_dict + if no_multi_convert is False: + conf_dict = vyos.xml.multi_to_list(xmlpath, conf_dict) + + if not (isinstance(key_mangling, tuple) and \ + (len(key_mangling) == 2) and \ + isinstance(key_mangling[0], str) and \ + isinstance(key_mangling[1], str)): + raise ValueError("key_mangling must be a tuple of two strings") + + conf_dict = vyos.util.mangle_dict_keys(conf_dict, key_mangling[0], key_mangling[1]) + + return conf_dict def is_multi(self, path): """ @@ -330,12 +262,8 @@ class Config(object): Note: It also returns False if node doesn't exist. """ - try: - path = " ".join(self._level) + " " + path - self._run(self._make_command('isMulti', path)) - return True - except VyOSError: - return False + self._config_source.set_level(self.get_level) + return self._config_source.is_multi(path) def is_tag(self, path): """ @@ -348,12 +276,8 @@ class Config(object): Note: It also returns False if node doesn't exist. """ - try: - path = " ".join(self._level) + " " + path - self._run(self._make_command('isTag', path)) - return True - except VyOSError: - return False + self._config_source.set_level(self.get_level) + return self._config_source.is_tag(path) def is_leaf(self, path): """ @@ -366,12 +290,8 @@ class Config(object): Note: It also returns False if node doesn't exist. """ - try: - path = " ".join(self._level) + " " + path - self._run(self._make_command('isLeaf', path)) - return True - except VyOSError: - return False + self._config_source.set_level(self.get_level) + return self._config_source.is_leaf(path) def return_value(self, path, default=None): """ @@ -532,9 +452,6 @@ class Config(object): Returns: str list: child node names - - Raises: - VyOSError: if the node is not a tag node """ if self._running_config: try: diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py index 0dc7578d8..e8c0aa5b3 100644 --- a/python/vyos/configdict.py +++ b/python/vyos/configdict.py @@ -15,15 +15,13 @@ """ A library for retrieving value dicts from VyOS configs in a declarative fashion. - """ +import os from enum import Enum from copy import deepcopy from vyos import ConfigError -from vyos.validate import is_member -from vyos.util import ifname_from_config def retrieve_config(path_hash, base_path, config): """ @@ -94,7 +92,7 @@ def dict_merge(source, destination): tmp = deepcopy(destination) for key, value in source.items(): - if key not in tmp.keys(): + if key not in tmp: tmp[key] = value elif isinstance(source[key], dict): tmp[key] = dict_merge(source[key], tmp[key]) @@ -102,397 +100,212 @@ def dict_merge(source, destination): return tmp def list_diff(first, second): - """ - Diff two dictionaries and return only unique items - """ + """ Diff two dictionaries and return only unique items """ second = set(second) return [item for item in first if item not in second] - -def get_ethertype(ethertype_val): - if ethertype_val == '0x88A8': - return '802.1ad' - elif ethertype_val == '0x8100': - return '802.1q' - else: - raise ConfigError('invalid ethertype "{}"'.format(ethertype_val)) - -dhcpv6_pd_default_data = { - 'dhcpv6_prm_only': False, - 'dhcpv6_temporary': False, - 'dhcpv6_pd_length': '', - 'dhcpv6_pd_interfaces': [] -} - -interface_default_data = { - **dhcpv6_pd_default_data, - 'address': [], - 'address_remove': [], - 'description': '', - 'dhcp_client_id': '', - 'dhcp_hostname': '', - 'dhcp_vendor_class_id': '', - 'disable': False, - 'disable_link_detect': 1, - 'ip_disable_arp_filter': 1, - 'ip_enable_arp_accept': 0, - 'ip_enable_arp_announce': 0, - 'ip_enable_arp_ignore': 0, - 'ip_proxy_arp': 0, - 'ipv6_accept_ra': 1, - 'ipv6_autoconf': 0, - 'ipv6_eui64_prefix': [], - 'ipv6_eui64_prefix_remove': [], - 'ipv6_forwarding': 1, - 'ipv6_dup_addr_detect': 1, - 'is_bridge_member': False, - 'mac': '', - 'mtu': 1500, - 'vrf': '' -} - -vlan_default = { - **interface_default_data, - 'egress_qos': '', - 'egress_qos_changed': False, - 'ingress_qos': '', - 'ingress_qos_changed': False, - 'vif_c': {}, - 'vif_c_remove': [] -} - -# see: https://docs.python.org/3/library/enum.html#functional-api -disable = Enum('disable','none was now both') - -def disable_state(conf, check=[3,5,7]): +def T2665_default_dict_cleanup(dict): + """ Cleanup default keys for tag nodes https://phabricator.vyos.net/T2665. """ + # Cleanup + for vif in ['vif', 'vif_s']: + if vif in dict: + for key in ['ip', 'mtu', 'dhcpv6_options']: + if key in dict[vif]: + del dict[vif][key] + + # cleanup VIF-S defaults + if 'vif_c' in dict[vif]: + for key in ['ip', 'mtu', 'dhcpv6_options']: + if key in dict[vif]['vif_c']: + del dict[vif]['vif_c'][key] + # If there is no vif-c defined and we just cleaned the default + # keys - we can clean the entire vif-c dict as it's useless + if not dict[vif]['vif_c']: + del dict[vif]['vif_c'] + + # If there is no real vif/vif-s defined and we just cleaned the default + # keys - we can clean the entire vif dict as it's useless + if not dict[vif]: + del dict[vif] + + if 'dhcpv6_options' in dict and 'pd' in dict['dhcpv6_options']: + if 'length' in dict['dhcpv6_options']['pd']: + del dict['dhcpv6_options']['pd']['length'] + + # delete empty dicts + if 'dhcpv6_options' in dict: + if 'pd' in dict['dhcpv6_options']: + # test if 'pd' is an empty node so we can remove it + if not dict['dhcpv6_options']['pd']: + del dict['dhcpv6_options']['pd'] + + # test if 'dhcpv6_options' is an empty node so we can remove it + if not dict['dhcpv6_options']: + del dict['dhcpv6_options'] + + return dict + +def leaf_node_changed(conf, path): """ - return if and how a particual section of the configuration is has disable'd - using "disable" including if it was disabled by one of its parent. - - check: a list of the level we should check, here 7,5 and 3 - interfaces ethernet eth1 vif-s 1 vif-c 2 disable - interfaces ethernet eth1 vif 1 disable - interfaces ethernet eth1 disable - - it returns an enum (none, was, now, both) + Check if a leaf node was altered. If it has been altered - values has been + changed, or it was added/removed, we will return the old value. If nothing + has been changed, None is returned """ - - # save where we are in the config - current_level = conf.get_level() - - # logic to figure out if the interface (or one of it parent is disabled) - eff_disable = False - act_disable = False - - levels = check[:] - working_level = current_level[:] - - while levels: - position = len(working_level) - if not position: - break - if position not in levels: - working_level = working_level[:-1] - continue - - levels.remove(position) - conf.set_level(working_level) - working_level = working_level[:-1] - - eff_disable = eff_disable or conf.exists_effective('disable') - act_disable = act_disable or conf.exists('disable') - - conf.set_level(current_level) - - # how the disabling changed - if eff_disable and act_disable: - return disable.both - if eff_disable and not eff_disable: - return disable.was - if not eff_disable and act_disable: - return disable.now - return disable.none - - -def intf_to_dict(conf, default): - from vyos.ifconfig import Interface - + from vyos.configdiff import get_config_diff + D = get_config_diff(conf, key_mangling=('-', '_')) + D.set_level(conf.get_level()) + (new, old) = D.get_value_diff(path) + if new != old: + if isinstance(old, str): + return old + elif isinstance(old, list): + if isinstance(new, str): + new = [new] + elif isinstance(new, type(None)): + new = [] + return list_diff(old, new) + + return None + +def node_changed(conf, path): """ - Common used function which will extract VLAN related information from config - and represent the result as Python dictionary. - - Function call's itself recursively if a vif-s/vif-c pair is detected. + Check if a leaf node was altered. If it has been altered - values has been + changed, or it was added/removed, we will return the old value. If nothing + has been changed, None is returned """ + from vyos.configdiff import get_config_diff, Diff + D = get_config_diff(conf, key_mangling=('-', '_')) + D.set_level(conf.get_level()) + # get_child_nodes() will return dict_keys(), mangle this into a list with PEP448 + keys = D.get_child_nodes_diff(path, expand_nodes=Diff.DELETE)['delete'].keys() + return list(keys) + +def get_removed_vlans(conf, dict): + """ + Common function to parse a dictionary retrieved via get_config_dict() and + determine any added/removed VLAN interfaces - be it 802.1q or Q-in-Q. + """ + from vyos.configdiff import get_config_diff, Diff - intf = deepcopy(default) - intf['intf'] = ifname_from_config(conf) - - current_vif_list = conf.list_nodes(['vif']) - previous_vif_list = conf.list_effective_nodes(['vif']) - - # set the vif to be deleted - for vif in previous_vif_list: - if vif not in current_vif_list: - intf['vif_remove'].append(vif) - - # retrieve interface description - if conf.exists(['description']): - intf['description'] = conf.return_value(['description']) - - # get DHCP client identifier - if conf.exists(['dhcp-options', 'client-id']): - intf['dhcp_client_id'] = conf.return_value(['dhcp-options', 'client-id']) - - # DHCP client host name (overrides the system host name) - if conf.exists(['dhcp-options', 'host-name']): - intf['dhcp_hostname'] = conf.return_value(['dhcp-options', 'host-name']) - - # DHCP client vendor identifier - if conf.exists(['dhcp-options', 'vendor-class-id']): - intf['dhcp_vendor_class_id'] = conf.return_value( - ['dhcp-options', 'vendor-class-id']) + # Check vif, vif-s/vif-c VLAN interfaces for removal + D = get_config_diff(conf, key_mangling=('-', '_')) + D.set_level(conf.get_level()) + # get_child_nodes() will return dict_keys(), mangle this into a list with PEP448 + keys = D.get_child_nodes_diff(['vif'], expand_nodes=Diff.DELETE)['delete'].keys() + if keys: + dict.update({'vif_remove': [*keys]}) - # DHCPv6 only acquire config parameters, no address - if conf.exists(['dhcpv6-options', 'parameters-only']): - intf['dhcpv6_prm_only'] = True + # get_child_nodes() will return dict_keys(), mangle this into a list with PEP448 + keys = D.get_child_nodes_diff(['vif-s'], expand_nodes=Diff.DELETE)['delete'].keys() + if keys: + dict.update({'vif_s_remove': [*keys]}) - # DHCPv6 prefix delegation (RFC3633) - current_level = conf.get_level() - if conf.exists(['dhcpv6-options', 'prefix-delegation']): - dhcpv6_pd_path = current_level + ['dhcpv6-options', 'prefix-delegation'] - conf.set_level(dhcpv6_pd_path) + for vif in dict.get('vif_s', {}).keys(): + keys = D.get_child_nodes_diff(['vif-s', vif, 'vif-c'], expand_nodes=Diff.DELETE)['delete'].keys() + if keys: + dict.update({'vif_s': { vif : {'vif_c_remove': [*keys]}}}) - # retriebe DHCPv6-PD prefix helper length as some ISPs only hand out a - # /64 by default (https://phabricator.vyos.net/T2506) - if conf.exists(['length']): - intf['dhcpv6_pd_length'] = conf.return_value(['length']) + return dict - for interface in conf.list_nodes(['interface']): - conf.set_level(dhcpv6_pd_path + ['interface', interface]) - pd = { - 'ifname': interface, - 'sla_id': '', - 'sla_len': '', - 'if_id': '' - } - if conf.exists(['sla-id']): - pd['sla_id'] = conf.return_value(['sla-id']) +def dict_add_dhcpv6pd_defaults(defaults, config_dict): + # Implant default dictionary for DHCPv6-PD instances + if 'dhcpv6_options' in config_dict and 'pd' in config_dict['dhcpv6_options']: + for pd, pd_config in config_dict['dhcpv6_options']['pd'].items(): + config_dict['dhcpv6_options']['pd'][pd] = dict_merge( + defaults, pd_config) - if conf.exists(['sla-len']): - pd['sla_len'] = conf.return_value(['sla-len']) + return config_dict - if conf.exists(['address']): - pd['if_id'] = conf.return_value(['address']) - - intf['dhcpv6_pd_interfaces'].append(pd) - - # re-set config level - conf.set_level(current_level) - - # DHCPv6 temporary IPv6 address - if conf.exists(['dhcpv6-options', 'temporary']): - intf['dhcpv6_temporary'] = True - - # ignore link state changes - if conf.exists(['disable-link-detect']): - intf['disable_link_detect'] = 2 - - # ARP filter configuration - if conf.exists(['ip', 'disable-arp-filter']): - intf['ip_disable_arp_filter'] = 0 - - # ARP enable accept - if conf.exists(['ip', 'enable-arp-accept']): - intf['ip_enable_arp_accept'] = 1 - - # ARP enable announce - if conf.exists(['ip', 'enable-arp-announce']): - intf['ip_enable_arp_announce'] = 1 - - # ARP enable ignore - if conf.exists(['ip', 'enable-arp-ignore']): - intf['ip_enable_arp_ignore'] = 1 - - # Enable Proxy ARP - if conf.exists(['ip', 'enable-proxy-arp']): - intf['ip_proxy_arp'] = 1 - - # Enable acquisition of IPv6 address using stateless autoconfig (SLAAC) - if conf.exists(['ipv6', 'address', 'autoconf']): - intf['ipv6_autoconf'] = 1 - - # Disable IPv6 forwarding on this interface - if conf.exists(['ipv6', 'disable-forwarding']): - intf['ipv6_forwarding'] = 0 - - # check if interface is member of a bridge - intf['is_bridge_member'] = is_member(conf, intf['intf'], 'bridge') - - # IPv6 Duplicate Address Detection (DAD) tries - if conf.exists(['ipv6', 'dup-addr-detect-transmits']): - intf['ipv6_dup_addr_detect'] = int( - conf.return_value(['ipv6', 'dup-addr-detect-transmits'])) - - # Media Access Control (MAC) address - if conf.exists(['mac']): - intf['mac'] = conf.return_value(['mac']) - - # Maximum Transmission Unit (MTU) - if conf.exists(['mtu']): - intf['mtu'] = int(conf.return_value(['mtu'])) - - # retrieve VRF instance - if conf.exists(['vrf']): - intf['vrf'] = conf.return_value(['vrf']) - - # egress QoS - if conf.exists(['egress-qos']): - intf['egress_qos'] = conf.return_value(['egress-qos']) - - # egress changes QoS require VLAN interface recreation - if conf.return_effective_value(['egress-qos']): - if intf['egress_qos'] != conf.return_effective_value(['egress-qos']): - intf['egress_qos_changed'] = True - - # ingress QoS - if conf.exists(['ingress-qos']): - intf['ingress_qos'] = conf.return_value(['ingress-qos']) - - # ingress changes QoS require VLAN interface recreation - if conf.return_effective_value(['ingress-qos']): - if intf['ingress_qos'] != conf.return_effective_value(['ingress-qos']): - intf['ingress_qos_changed'] = True - - # Get the interface addresses - intf['address'] = conf.return_values(['address']) - - # addresses to remove - difference between effective and working config - intf['address_remove'] = list_diff( - conf.return_effective_values(['address']), intf['address']) - - # Get prefixes for IPv6 addressing based on MAC address (EUI-64) - intf['ipv6_eui64_prefix'] = conf.return_values(['ipv6', 'address', 'eui64']) - - # EUI64 to remove - difference between effective and working config - intf['ipv6_eui64_prefix_remove'] = list_diff( - conf.return_effective_values(['ipv6', 'address', 'eui64']), - intf['ipv6_eui64_prefix']) - - # Determine if the interface should be disabled - disabled = disable_state(conf) - if disabled == disable.both: - # was and is still disabled - intf['disable'] = True - elif disabled == disable.now: - # it is now disable but was not before - intf['disable'] = True - elif disabled == disable.was: - # it was disable but not anymore - intf['disable'] = False - else: - # normal change - intf['disable'] = False - - # Remove the default link-local address if no-default-link-local is set, - # if member of a bridge or if disabled (it may not have a MAC if it's down) - if ( conf.exists(['ipv6', 'address', 'no-default-link-local']) - or intf.get('is_bridge_member') or intf['disable'] ): - intf['ipv6_eui64_prefix_remove'].append('fe80::/64') - else: - # add the link-local by default to make IPv6 work - intf['ipv6_eui64_prefix'].append('fe80::/64') - - # If MAC has changed, remove and re-add all IPv6 EUI64 addresses - try: - interface = Interface(intf['intf'], create=False) - if intf['mac'] and intf['mac'] != interface.get_mac(): - intf['ipv6_eui64_prefix_remove'] += intf['ipv6_eui64_prefix'] - except Exception: - # If the interface does not exist, it could not have changed - pass - - # to make IPv6 SLAAC and DHCPv6 work with forwarding=1, - # accept_ra must be 2 - if intf['ipv6_autoconf'] or 'dhcpv6' in intf['address']: - intf['ipv6_accept_ra'] = 2 - - return intf, disable - - - -def add_to_dict(conf, disabled, ifdict, section, key): +def get_interface_dict(config, base, ifname=''): """ - parse a section of vif/vif-s/vif-c and add them to the dict - follow the convention to: - * use the "key" for what to add - * use the "key" what what to remove - - conf: is the Config() already at the level we need to parse - disabled: is a disable enum so we know how to handle to data - intf: if the interface dictionary - section: is the section name to parse (vif/vif-s/vif-c) - key: is the dict key to use (vif/vifs/vifc) + Common utility function to retrieve and mandgle the interfaces available + in CLI configuration. All interfaces have a common base ground where the + value retrival is identical - so it can and should be reused + + Will return a dictionary with the necessary interface configuration """ + from vyos.util import vyos_dict_search + from vyos.validate import is_member + from vyos.xml import defaults + + if not ifname: + # determine tagNode instance + if 'VYOS_TAGNODE_VALUE' not in os.environ: + raise ConfigError('Interface (VYOS_TAGNODE_VALUE) not specified') + ifname = os.environ['VYOS_TAGNODE_VALUE'] + + # retrieve interface default values + default_values = defaults(base) + + # setup config level which is extracted in get_removed_vlans() + config.set_level(base + [ifname]) + dict = config.get_config_dict([], key_mangling=('-', '_'), get_first_key=True) + + # Check if interface has been removed + if dict == {}: + dict.update({'deleted' : ''}) + + # Add interface instance name into dictionary + dict.update({'ifname': ifname}) + + # We have gathered the dict representation of the CLI, but there are + # default options which we need to update into the dictionary + # retrived. + dict = dict_merge(default_values, dict) + + # Check if we are a member of a bridge device + bridge = is_member(config, ifname, 'bridge') + if bridge: + dict.update({'is_bridge_member' : bridge}) + + # Check if we are a member of a bond device + bond = is_member(config, ifname, 'bonding') + if bond: + dict.update({'is_bond_member' : bond}) + + mac = leaf_node_changed(config, ['mac']) + if mac: + dict.update({'mac_old' : mac}) + + eui64 = leaf_node_changed(config, ['ipv6', 'address', 'eui64']) + if eui64: + tmp = vyos_dict_search('ipv6.address', dict) + if not tmp: + dict.update({'ipv6': {'address': {'eui64_old': eui64}}}) + else: + dict['ipv6']['address'].update({'eui64_old': eui64}) + + # remove wrongly inserted values + dict = T2665_default_dict_cleanup(dict) + + # Implant default dictionary for DHCPv6-PD instances + default_pd_values = defaults(base + ['dhcpv6-options', 'pd']) + dict = dict_add_dhcpv6pd_defaults(default_pd_values, dict) + + # Implant default dictionary in vif/vif-s VLAN interfaces. Values are + # identical for all types of VLAN interfaces as they all include the same + # XML definitions which hold the defaults. + default_vif_values = defaults(base + ['vif']) + for vif, vif_config in dict.get('vif', {}).items(): + dict['vif'][vif] = dict_add_dhcpv6pd_defaults( + default_pd_values, vif_config) + dict['vif'][vif] = T2665_default_dict_cleanup( + dict_merge(default_vif_values, vif_config)) + + for vif_s, vif_s_config in dict.get('vif_s', {}).items(): + dict['vif_s'][vif_s] = dict_add_dhcpv6pd_defaults( + default_pd_values, vif_s_config) + dict['vif_s'][vif_s] = T2665_default_dict_cleanup( + dict_merge(default_vif_values, vif_s_config)) + for vif_c, vif_c_config in vif_s_config.get('vif_c', {}).items(): + dict['vif_s'][vif_s]['vif_c'][vif_c] = dict_add_dhcpv6pd_defaults( + default_pd_values, vif_c_config) + dict['vif_s'][vif_s]['vif_c'][vif_c] = T2665_default_dict_cleanup( + dict_merge(default_vif_values, vif_c_config)) + + # Check vif, vif-s/vif-c VLAN interfaces for removal + dict = get_removed_vlans(config, dict) + + return dict - if not conf.exists(section): - return ifdict - - effect = conf.list_effective_nodes(section) - active = conf.list_nodes(section) - - # the section to parse for vlan - sections = [] - - # determine which interfaces to add or remove based on disable state - if disabled == disable.both: - # was and is still disabled - ifdict[f'{key}_remove'] = [] - elif disabled == disable.now: - # it is now disable but was not before - ifdict[f'{key}_remove'] = effect - elif disabled == disable.was: - # it was disable but not anymore - ifdict[f'{key}_remove'] = [] - sections = active - else: - # normal change - # get interfaces (currently effective) - to determine which - # interface is no longer present and needs to be removed - ifdict[f'{key}_remove'] = list_diff(effect, active) - sections = active - - current_level = conf.get_level() - - # add each section, the key must already exists - for s in sections: - # set config level to vif interface - conf.set_level(current_level + [section, s]) - # add the vlan config as a key (vlan id) - value (config) pair - ifdict[key][s] = vlan_to_dict(conf) - - # re-set configuration level to leave things as found - conf.set_level(current_level) - - return ifdict - - -def vlan_to_dict(conf, default=vlan_default): - vlan, disabled = intf_to_dict(conf, default) - - # if this is a not within vif-s node, we are done - if conf.get_level()[-2] != 'vif-s': - return vlan - - # ethertype is mandatory on vif-s nodes and only exists here! - # ethertype uses a default of 0x88A8 - tmp = '0x88A8' - if conf.exists('ethertype'): - tmp = conf.return_value('ethertype') - vlan['ethertype'] = get_ethertype(tmp) - - # check if there is a Q-in-Q vlan customer interface - # and call this function recursively - add_to_dict(conf, disable, vlan, 'vif-c', 'vif_c') - - return vlan diff --git a/python/vyos/configdiff.py b/python/vyos/configdiff.py new file mode 100644 index 000000000..0e41fbe27 --- /dev/null +++ b/python/vyos/configdiff.py @@ -0,0 +1,249 @@ +# Copyright 2020 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +from enum import IntFlag, auto + +from vyos.config import Config +from vyos.configdict import dict_merge +from vyos.util import get_sub_dict, mangle_dict_keys +from vyos.xml import defaults + +class ConfigDiffError(Exception): + """ + Raised on config dict access errors, for example, calling get_value on + a non-leaf node. + """ + pass + +def enum_to_key(e): + return e.name.lower() + +class Diff(IntFlag): + MERGE = auto() + DELETE = auto() + ADD = auto() + STABLE = auto() + +requires_effective = [enum_to_key(Diff.DELETE)] +target_defaults = [enum_to_key(Diff.MERGE)] + +def _key_sets_from_dicts(session_dict, effective_dict): + session_keys = list(session_dict) + effective_keys = list(effective_dict) + + ret = {} + stable_keys = [k for k in session_keys if k in effective_keys] + + ret[enum_to_key(Diff.MERGE)] = session_keys + ret[enum_to_key(Diff.DELETE)] = [k for k in effective_keys if k not in stable_keys] + ret[enum_to_key(Diff.ADD)] = [k for k in session_keys if k not in stable_keys] + ret[enum_to_key(Diff.STABLE)] = stable_keys + + return ret + +def _dict_from_key_set(key_set, d): + # This will always be applied to a key_set obtained from a get_sub_dict, + # hence there is no possibility of KeyError, as get_sub_dict guarantees + # a return type of dict + ret = {k: d[k] for k in key_set} + + return ret + +def get_config_diff(config, key_mangling=None): + """ + Check type and return ConfigDiff instance. + """ + if not config or not isinstance(config, Config): + raise TypeError("argument must me a Config instance") + if key_mangling and not (isinstance(key_mangling, tuple) and \ + (len(key_mangling) == 2) and \ + isinstance(key_mangling[0], str) and \ + isinstance(key_mangling[1], str)): + raise ValueError("key_mangling must be a tuple of two strings") + + return ConfigDiff(config, key_mangling) + +class ConfigDiff(object): + """ + The class of config changes as represented by comparison between the + session config dict and the effective config dict. + """ + def __init__(self, config, key_mangling=None): + self._level = config.get_level() + self._session_config_dict = config.get_cached_root_dict(effective=False) + self._effective_config_dict = config.get_cached_root_dict(effective=True) + self._key_mangling = key_mangling + + # mirrored from Config; allow path arguments relative to level + def _make_path(self, path): + if isinstance(path, str): + path = path.split() + elif isinstance(path, list): + pass + else: + raise TypeError("Path must be a whitespace-separated string or a list") + + ret = self._level + path + return ret + + def set_level(self, path): + """ + Set the *edit level*, that is, a relative config dict path. + Once set, all operations will be relative to this path, + for example, after ``set_level("system")``, calling + ``get_value("name-server")`` is equivalent to calling + ``get_value("system name-server")`` without ``set_level``. + + Args: + path (str|list): relative config path + """ + if isinstance(path, str): + if path: + self._level = path.split() + else: + self._level = [] + elif isinstance(path, list): + self._level = path.copy() + else: + raise TypeError("Level path must be either a whitespace-separated string or a list") + + def get_level(self): + """ + Gets the current edit level. + + Returns: + str: current edit level + """ + ret = self._level.copy() + return ret + + def _mangle_dict_keys(self, config_dict): + config_dict = mangle_dict_keys(config_dict, self._key_mangling[0], + self._key_mangling[1]) + return config_dict + + def get_child_nodes_diff(self, path=[], expand_nodes=Diff(0), no_defaults=False): + """ + Args: + path (str|list): config path + expand_nodes=Diff(0): bit mask of enum indicating for which nodes + to provide full dict; for example, Diff.MERGE + will expand dict['merge'] into dict under + value + no_detaults=False: if expand_nodes & Diff.MERGE, do not merge default + values to ret['merge'] + + Returns: dict of lists, representing differences between session + and effective config, under path + dict['merge'] = session config values + dict['delete'] = effective config values, not in session + dict['add'] = session config values, not in effective + dict['stable'] = config values in both session and effective + """ + session_dict = get_sub_dict(self._session_config_dict, + self._make_path(path), get_first_key=True) + effective_dict = get_sub_dict(self._effective_config_dict, + self._make_path(path), get_first_key=True) + + ret = _key_sets_from_dicts(session_dict, effective_dict) + + if not expand_nodes: + return ret + + for e in Diff: + if expand_nodes & e: + k = enum_to_key(e) + if k in requires_effective: + ret[k] = _dict_from_key_set(ret[k], effective_dict) + else: + ret[k] = _dict_from_key_set(ret[k], session_dict) + + if self._key_mangling: + ret[k] = self._mangle_dict_keys(ret[k]) + + if k in target_defaults and not no_defaults: + default_values = defaults(self._make_path(path)) + ret[k] = dict_merge(default_values, ret[k]) + + return ret + + def get_node_diff(self, path=[], expand_nodes=Diff(0), no_defaults=False): + """ + Args: + path (str|list): config path + expand_nodes=Diff(0): bit mask of enum indicating for which nodes + to provide full dict; for example, Diff.MERGE + will expand dict['merge'] into dict under + value + no_detaults=False: if expand_nodes & Diff.MERGE, do not merge default + values to ret['merge'] + + Returns: dict of lists, representing differences between session + and effective config, at path + dict['merge'] = session config values + dict['delete'] = effective config values, not in session + dict['add'] = session config values, not in effective + dict['stable'] = config values in both session and effective + """ + session_dict = get_sub_dict(self._session_config_dict, self._make_path(path)) + effective_dict = get_sub_dict(self._effective_config_dict, self._make_path(path)) + + ret = _key_sets_from_dicts(session_dict, effective_dict) + + if not expand_nodes: + return ret + + for e in Diff: + if expand_nodes & e: + k = enum_to_key(e) + if k in requires_effective: + ret[k] = _dict_from_key_set(ret[k], effective_dict) + else: + ret[k] = _dict_from_key_set(ret[k], session_dict) + + if self._key_mangling: + ret[k] = self._mangle_dict_keys(ret[k]) + + if k in target_defaults and not no_defaults: + default_values = defaults(self._make_path(path)) + ret[k] = dict_merge(default_values, ret[k]) + + return ret + + def get_value_diff(self, path=[]): + """ + Args: + path (str|list): config path + + Returns: (new, old) tuple of values in session config/effective config + """ + # one should properly use is_leaf as check; for the moment we will + # deduce from type, which will not catch call on non-leaf node if None + new_value_dict = get_sub_dict(self._session_config_dict, self._make_path(path)) + old_value_dict = get_sub_dict(self._effective_config_dict, self._make_path(path)) + + new_value = None + old_value = None + if new_value_dict: + new_value = next(iter(new_value_dict.values())) + if old_value_dict: + old_value = next(iter(old_value_dict.values())) + + if new_value and isinstance(new_value, dict): + raise ConfigDiffError("get_value_changed called on non-leaf node") + if old_value and isinstance(old_value, dict): + raise ConfigDiffError("get_value_changed called on non-leaf node") + + return new_value, old_value diff --git a/python/vyos/configsession.py b/python/vyos/configsession.py index f2524b37e..6e4214360 100644 --- a/python/vyos/configsession.py +++ b/python/vyos/configsession.py @@ -17,6 +17,8 @@ import re import sys import subprocess +from vyos.util import call + CLI_SHELL_API = '/bin/cli-shell-api' SET = '/opt/vyatta/sbin/my_set' DELETE = '/opt/vyatta/sbin/my_delete' @@ -26,7 +28,7 @@ DISCARD = '/opt/vyatta/sbin/my_discard' SHOW_CONFIG = ['/bin/cli-shell-api', 'showConfig'] LOAD_CONFIG = ['/bin/cli-shell-api', 'loadFile'] SAVE_CONFIG = ['/opt/vyatta/sbin/vyatta-save-config.pl'] -INSTALL_IMAGE = ['/opt/vyatta/sbin/install-image'] +INSTALL_IMAGE = ['/opt/vyatta/sbin/install-image', '--url'] REMOVE_IMAGE = ['/opt/vyatta/bin/vyatta-boot-image.pl', '--del'] GENERATE = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'generate'] SHOW = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'show'] @@ -69,6 +71,11 @@ def inject_vyos_env(env): env['vyos_sbin_dir'] = '/usr/sbin' env['vyos_validators_dir'] = '/usr/libexec/vyos/validators' + # if running the vyos-configd daemon, inject the vyshim env var + ret = call('systemctl is-active --quiet vyos-configd.service') + if not ret: + env['vyshim'] = '/usr/sbin/vyshim' + return env diff --git a/python/vyos/configsource.py b/python/vyos/configsource.py new file mode 100644 index 000000000..50222e385 --- /dev/null +++ b/python/vyos/configsource.py @@ -0,0 +1,318 @@ + +# Copyright 2020 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +import os +import re +import subprocess + +from vyos.configtree import ConfigTree + +class VyOSError(Exception): + """ + Raised on config access errors. + """ + pass + +class ConfigSourceError(Exception): + ''' + Raised on error in ConfigSource subclass init. + ''' + pass + +class ConfigSource: + def __init__(self): + self._running_config: ConfigTree = None + self._session_config: ConfigTree = None + + def get_configtree_tuple(self): + return self._running_config, self._session_config + + def session_changed(self): + """ + Returns: + True if the config session has uncommited changes, False otherwise. + """ + raise NotImplementedError(f"function not available for {type(self)}") + + def in_session(self): + """ + Returns: + True if called from a configuration session, False otherwise. + """ + raise NotImplementedError(f"function not available for {type(self)}") + + def show_config(self, path=[], default=None, effective=False): + """ + Args: + path (str|list): Configuration tree path, or empty + default (str): Default value to return + + Returns: + str: working configuration + """ + raise NotImplementedError(f"function not available for {type(self)}") + + def is_multi(self, path): + """ + Args: + path (str): Configuration tree path + + Returns: + True if a node can have multiple values, False otherwise. + + Note: + It also returns False if node doesn't exist. + """ + raise NotImplementedError(f"function not available for {type(self)}") + + def is_tag(self, path): + """ + Args: + path (str): Configuration tree path + + Returns: + True if a node is a tag node, False otherwise. + + Note: + It also returns False if node doesn't exist. + """ + raise NotImplementedError(f"function not available for {type(self)}") + + def is_leaf(self, path): + """ + Args: + path (str): Configuration tree path + + Returns: + True if a node is a leaf node, False otherwise. + + Note: + It also returns False if node doesn't exist. + """ + raise NotImplementedError(f"function not available for {type(self)}") + +class ConfigSourceSession(ConfigSource): + def __init__(self, session_env=None): + super().__init__() + self._cli_shell_api = "/bin/cli-shell-api" + self._level = [] + if session_env: + self.__session_env = session_env + else: + self.__session_env = None + + # Running config can be obtained either from op or conf mode, it always succeeds + # once the config system is initialized during boot; + # before initialization, set to empty string + if os.path.isfile('/tmp/vyos-config-status'): + try: + running_config_text = self._run([self._cli_shell_api, '--show-active-only', '--show-show-defaults', '--show-ignore-edit', 'showConfig']) + except VyOSError: + running_config_text = '' + else: + running_config_text = '' + + # Session config ("active") only exists in conf mode. + # In op mode, we'll just use the same running config for both active and session configs. + if self.in_session(): + try: + session_config_text = self._run([self._cli_shell_api, '--show-working-only', '--show-show-defaults', '--show-ignore-edit', 'showConfig']) + except VyOSError: + session_config_text = '' + else: + session_config_text = running_config_text + + if running_config_text: + self._running_config = ConfigTree(running_config_text) + else: + self._running_config = None + + if session_config_text: + self._session_config = ConfigTree(session_config_text) + else: + self._session_config = None + + def _make_command(self, op, path): + args = path.split() + cmd = [self._cli_shell_api, op] + args + return cmd + + def _run(self, cmd): + if self.__session_env: + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=self.__session_env) + else: + p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + out = p.stdout.read() + p.wait() + p.communicate() + if p.returncode != 0: + raise VyOSError() + else: + return out.decode('ascii') + + def set_level(self, path): + """ + Set the *edit level*, that is, a relative config tree path. + Once set, all operations will be relative to this path, + for example, after ``set_level("system")``, calling + ``exists("name-server")`` is equivalent to calling + ``exists("system name-server"`` without ``set_level``. + + Args: + path (str|list): relative config path + """ + # Make sure there's always a space between default path (level) + # and path supplied as method argument + # XXX: for small strings in-place concatenation is not a problem + if isinstance(path, str): + if path: + self._level = re.split(r'\s+', path) + else: + self._level = [] + elif isinstance(path, list): + self._level = path.copy() + else: + raise TypeError("Level path must be either a whitespace-separated string or a list") + + def session_changed(self): + """ + Returns: + True if the config session has uncommited changes, False otherwise. + """ + try: + self._run(self._make_command('sessionChanged', '')) + return True + except VyOSError: + return False + + def in_session(self): + """ + Returns: + True if called from a configuration session, False otherwise. + """ + try: + self._run(self._make_command('inSession', '')) + return True + except VyOSError: + return False + + def show_config(self, path=[], default=None, effective=False): + """ + Args: + path (str|list): Configuration tree path, or empty + default (str): Default value to return + + Returns: + str: working configuration + """ + + # show_config should be independent of CLI edit level. + # Set the CLI edit environment to the top level, and + # restore original on exit. + save_env = self.__session_env + + env_str = self._run(self._make_command('getEditResetEnv', '')) + env_list = re.findall(r'([A-Z_]+)=\'([^;\s]+)\'', env_str) + root_env = os.environ + for k, v in env_list: + root_env[k] = v + + self.__session_env = root_env + + # FIXUP: by default, showConfig will give you a diff + # if there are uncommitted changes. + # The config parser obviously cannot work with diffs, + # so we need to supress diff production using appropriate + # options for getting either running (active) + # or proposed (working) config. + if effective: + path = ['--show-active-only'] + path + else: + path = ['--show-working-only'] + path + + if isinstance(path, list): + path = " ".join(path) + try: + out = self._run(self._make_command('showConfig', path)) + self.__session_env = save_env + return out + except VyOSError: + self.__session_env = save_env + return(default) + + def is_multi(self, path): + """ + Args: + path (str): Configuration tree path + + Returns: + True if a node can have multiple values, False otherwise. + + Note: + It also returns False if node doesn't exist. + """ + try: + path = " ".join(self._level) + " " + path + self._run(self._make_command('isMulti', path)) + return True + except VyOSError: + return False + + def is_tag(self, path): + """ + Args: + path (str): Configuration tree path + + Returns: + True if a node is a tag node, False otherwise. + + Note: + It also returns False if node doesn't exist. + """ + try: + path = " ".join(self._level) + " " + path + self._run(self._make_command('isTag', path)) + return True + except VyOSError: + return False + + def is_leaf(self, path): + """ + Args: + path (str): Configuration tree path + + Returns: + True if a node is a leaf node, False otherwise. + + Note: + It also returns False if node doesn't exist. + """ + try: + path = " ".join(self._level) + " " + path + self._run(self._make_command('isLeaf', path)) + return True + except VyOSError: + return False + +class ConfigSourceString(ConfigSource): + def __init__(self, running_config_text=None, session_config_text=None): + super().__init__() + + try: + self._running_config = ConfigTree(running_config_text) if running_config_text else None + self._session_config = ConfigTree(session_config_text) if session_config_text else None + except ValueError: + raise ConfigSourceError(f"Init error in {type(self)}") diff --git a/python/vyos/configverify.py b/python/vyos/configverify.py index 32129a048..7e1930878 100644 --- a/python/vyos/configverify.py +++ b/python/vyos/configverify.py @@ -29,11 +29,11 @@ def verify_vrf(config): recurring validation of VRF configuration. """ from netifaces import interfaces - if 'vrf' in config.keys(): + if 'vrf' in config: if config['vrf'] not in interfaces(): raise ConfigError('VRF "{vrf}" does not exist'.format(**config)) - if 'is_bridge_member' in config.keys(): + if 'is_bridge_member' in config: raise ConfigError( 'Interface "{ifname}" cannot be both a member of VRF "{vrf}" ' 'and bridge "{is_bridge_member}"!'.format(**config)) @@ -41,14 +41,14 @@ def verify_vrf(config): def verify_address(config): """ - Common helper function used by interface implementations to - perform recurring validation of IP address assignmenr - when interface also is part of a bridge. + Common helper function used by interface implementations to perform + recurring validation of IP address assignment when interface is part + of a bridge or bond. """ if {'is_bridge_member', 'address'} <= set(config): raise ConfigError( - f'Cannot assign address to interface "{ifname}" as it is a ' - f'member of bridge "{is_bridge_member}"!'.format(**config)) + 'Cannot assign address to interface "{ifname}" as it is a ' + 'member of bridge "{is_bridge_member}"!'.format(**config)) def verify_bridge_delete(config): @@ -57,11 +57,20 @@ def verify_bridge_delete(config): perform recurring validation of IP address assignmenr when interface also is part of a bridge. """ - if 'is_bridge_member' in config.keys(): + if 'is_bridge_member' in config: raise ConfigError( 'Interface "{ifname}" cannot be deleted as it is a ' 'member of bridge "{is_bridge_member}"!'.format(**config)) +def verify_interface_exists(config): + """ + Common helper function used by interface implementations to perform + recurring validation if an interface actually exists. + """ + from netifaces import interfaces + if not config['ifname'] in interfaces(): + raise ConfigError('Interface "{ifname}" does not exist!' + .format(**config)) def verify_source_interface(config): """ @@ -70,9 +79,61 @@ def verify_source_interface(config): required by e.g. peth/MACvlan, MACsec ... """ from netifaces import interfaces - if not 'source_interface' in config.keys(): + if 'source_interface' not in config: raise ConfigError('Physical source-interface required for ' 'interface "{ifname}"'.format(**config)) - if not config['source_interface'] in interfaces(): - raise ConfigError(f'Source interface {source_interface} does not ' - f'exist'.format(**config)) + if config['source_interface'] not in interfaces(): + raise ConfigError('Source interface {source_interface} does not ' + 'exist'.format(**config)) + +def verify_dhcpv6(config): + """ + Common helper function used by interface implementations to perform + recurring validation of DHCPv6 options which are mutually exclusive. + """ + if 'dhcpv6_options' in config: + from vyos.util import vyos_dict_search + + if {'parameters_only', 'temporary'} <= set(config['dhcpv6_options']): + raise ConfigError('DHCPv6 temporary and parameters-only options ' + 'are mutually exclusive!') + + # It is not allowed to have duplicate SLA-IDs as those identify an + # assigned IPv6 subnet from a delegated prefix + for pd in vyos_dict_search('dhcpv6_options.pd', config): + sla_ids = [] + for interface in vyos_dict_search(f'dhcpv6_options.pd.{pd}.interface', config): + sla_id = vyos_dict_search( + f'dhcpv6_options.pd.{pd}.interface.{interface}.sla_id', config) + sla_ids.append(sla_id) + + # Check for duplicates + duplicates = [x for n, x in enumerate(sla_ids) if x in sla_ids[:n]] + if duplicates: + raise ConfigError('Site-Level Aggregation Identifier (SLA-ID) ' + 'must be unique per prefix-delegation!') + +def verify_vlan_config(config): + """ + Common helper function used by interface implementations to perform + recurring validation of interface VLANs + """ + # 802.1q VLANs + for vlan in config.get('vif', {}): + vlan = config['vif'][vlan] + verify_dhcpv6(vlan) + verify_address(vlan) + verify_vrf(vlan) + + # 802.1ad (Q-in-Q) VLANs + for vlan in config.get('vif_s', {}): + vlan = config['vif_s'][vlan] + verify_dhcpv6(vlan) + verify_address(vlan) + verify_vrf(vlan) + + for vlan in config.get('vif_s', {}).get('vif_c', {}): + vlan = config['vif_c'][vlan] + verify_dhcpv6(vlan) + verify_address(vlan) + verify_vrf(vlan) diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index 3062ed31c..9921e3b5f 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -21,7 +21,8 @@ directories = { "current": "/opt/vyatta/etc/config-migrate/current", "migrate": "/opt/vyatta/etc/config-migrate/migrate", "log": "/var/log/vyatta", - "templates": "/usr/share/vyos/templates/" + "templates": "/usr/share/vyos/templates/", + "certbot": "/config/auth/letsencrypt" } cfg_group = 'vyattacfg' diff --git a/python/vyos/frr.py b/python/vyos/frr.py index e39b6a914..3fc75bbdf 100644 --- a/python/vyos/frr.py +++ b/python/vyos/frr.py @@ -281,7 +281,7 @@ def replace_section(config, replacement, from_re, to_re=r'!', before_re=r'line v startline and endline tags will be automatically added to the resulting from_re/to_re and before_re regex'es """ - return _replace_section(config, replacement, replace_re=rf'^{from_re}$.*?^{to_re}$', before_re=rf'^{before_re}$') + return _replace_section(config, replacement, replace_re=rf'^{from_re}$.*?^{to_re}$', before_re=rf'^({before_re})$') def remove_section(config, from_re, to_re='!'): diff --git a/python/vyos/ifconfig/__init__.py b/python/vyos/ifconfig/__init__.py index a7cdeadd1..9cd8d44c1 100644 --- a/python/vyos/ifconfig/__init__.py +++ b/python/vyos/ifconfig/__init__.py @@ -13,12 +13,10 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. - from vyos.ifconfig.section import Section from vyos.ifconfig.control import Control from vyos.ifconfig.interface import Interface from vyos.ifconfig.operational import Operational -from vyos.ifconfig.dhcp import DHCP from vyos.ifconfig.vrrp import VRRP from vyos.ifconfig.bond import BondIf diff --git a/python/vyos/ifconfig/bond.py b/python/vyos/ifconfig/bond.py index 47dd4ff34..64407401b 100644 --- a/python/vyos/ifconfig/bond.py +++ b/python/vyos/ifconfig/bond.py @@ -18,10 +18,11 @@ import os from vyos.ifconfig.interface import Interface from vyos.ifconfig.vlan import VLAN +from vyos.util import cmd +from vyos.util import vyos_dict_search from vyos.validate import assert_list from vyos.validate import assert_positive - @Interface.register @VLAN.enable class BondIf(Interface): @@ -179,7 +180,13 @@ class BondIf(Interface): >>> BondIf('bond0').get_arp_ip_target() '192.0.2.1' """ - return self.get_interface('bond_arp_ip_target') + # As this function might also be called from update() of a VLAN interface + # we must check if the bond_arp_ip_target retrieval worked or not - as this + # can not be set for a bond vif interface + try: + return self.get_interface('bond_arp_ip_target') + except FileNotFoundError: + return '' def set_arp_ip_target(self, target): """ @@ -209,11 +216,31 @@ class BondIf(Interface): >>> BondIf('bond0').add_port('eth0') >>> BondIf('bond0').add_port('eth1') """ - # An interface can only be added to a bond if it is in 'down' state. If - # interface is in 'up' state, the following Kernel error will be thrown: - # bond0: eth1 is up - this may be due to an out of date ifenslave. - Interface(interface).set_admin_state('down') - return self.set_interface('bond_add_port', f'+{interface}') + + # From drivers/net/bonding/bond_main.c: + # ... + # bond_set_slave_link_state(new_slave, + # BOND_LINK_UP, + # BOND_SLAVE_NOTIFY_NOW); + # ... + # + # The kernel will ALWAYS place new bond members in "up" state regardless + # what the CLI will tell us! + + # Physical interface must be in admin down state before they can be + # enslaved. If this is not the case an error will be shown: + # bond0: eth0 is up - this may be due to an out of date ifenslave + slave = Interface(interface) + slave_state = slave.get_admin_state() + if slave_state == 'up': + slave.set_admin_state('down') + + ret = self.set_interface('bond_add_port', f'+{interface}') + # The kernel will ALWAYS place new bond members in "up" state regardless + # what the LI is configured for - thus we place the interface in its + # desired state + slave.set_admin_state(slave_state) + return ret def del_port(self, interface): """ @@ -277,3 +304,80 @@ class BondIf(Interface): >>> BondIf('bond0').set_mode('802.3ad') """ return self.set_interface('bond_mode', mode) + + def update(self, config): + """ General helper function which works on a dictionary retrived by + get_config_dict(). It's main intention is to consolidate the scattered + interface setup code and provide a single point of entry when workin + on any interface. """ + + # use ref-counting function to place an interface into admin down state. + # set_admin_state_up() must be called the same amount of times else the + # interface won't come up. This can/should be used to prevent link flapping + # when changing interface parameters require the interface to be down. + # We will disable it once before reconfiguration and enable it afterwards. + if 'shutdown_required' in config: + self.set_admin_state('down') + + # call base class first + super().update(config) + + # ARP monitor targets need to be synchronized between sysfs and CLI. + # Unfortunately an address can't be send twice to sysfs as this will + # result in the following exception: OSError: [Errno 22] Invalid argument. + # + # We remove ALL addresses prior to adding new ones, this will remove + # addresses manually added by the user too - but as we are limited to 16 adresses + # from the kernel side this looks valid to me. We won't run into an error + # when a user added manual adresses which would result in having more + # then 16 adresses in total. + arp_tgt_addr = list(map(str, self.get_arp_ip_target().split())) + for addr in arp_tgt_addr: + self.set_arp_ip_target('-' + addr) + + # Add configured ARP target addresses + value = vyos_dict_search('arp_monitor.target', config) + if isinstance(value, str): + value = [value] + if value: + for addr in value: + self.set_arp_ip_target('+' + addr) + + # Bonding transmit hash policy + value = config.get('hash_policy') + if value: self.set_hash_policy(value) + + # Some interface options can only be changed if the interface is + # administratively down + if self.get_admin_state() == 'down': + # Delete bond member port(s) + for interface in self.get_slaves(): + self.del_port(interface) + + # Bonding policy/mode + value = config.get('mode') + if value: self.set_mode(value) + + # Add (enslave) interfaces to bond + value = vyos_dict_search('member.interface', config) + if value: + for interface in value: + # if we've come here we already verified the interface + # does not have an addresses configured so just flush + # any remaining ones + Interface(interface).flush_addrs() + self.add_port(interface) + + # Primary device interface - must be set after 'mode' + value = config.get('primary') + if value: self.set_primary(value) + + # Enable/Disable of an interface must always be done at the end of the + # derived class to make use of the ref-counting set_admin_state() + # function. We will only enable the interface if 'up' was called as + # often as 'down'. This is required by some interface implementations + # as certain parameters can only be changed when the interface is + # in admin-down state. This ensures the link does not flap during + # reconfiguration. + state = 'down' if 'disable' in config else 'up' + self.set_admin_state(state) diff --git a/python/vyos/ifconfig/bridge.py b/python/vyos/ifconfig/bridge.py index 44b92c1db..4c76fe996 100644 --- a/python/vyos/ifconfig/bridge.py +++ b/python/vyos/ifconfig/bridge.py @@ -13,12 +13,12 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. - from vyos.ifconfig.interface import Interface - +from vyos.ifconfig.stp import STP from vyos.validate import assert_boolean from vyos.validate import assert_positive - +from vyos.util import cmd +from vyos.util import vyos_dict_search @Interface.register class BridgeIf(Interface): @@ -187,3 +187,77 @@ class BridgeIf(Interface): >>> BridgeIf('br0').del_port('eth1') """ return self.set_interface('del_port', interface) + + def update(self, config): + """ General helper function which works on a dictionary retrived by + get_config_dict(). It's main intention is to consolidate the scattered + interface setup code and provide a single point of entry when workin + on any interface. """ + + # call base class first + super().update(config) + + # Set ageing time + value = config.get('aging') + self.set_ageing_time(value) + + # set bridge forward delay + value = config.get('forwarding_delay') + self.set_forward_delay(value) + + # set hello time + value = config.get('hello_time') + self.set_hello_time(value) + + # set max message age + value = config.get('max_age') + self.set_max_age(value) + + # set bridge priority + value = config.get('priority') + self.set_priority(value) + + # enable/disable spanning tree + value = '1' if 'stp' in config else '0' + self.set_stp(value) + + # enable or disable IGMP querier + tmp = vyos_dict_search('igmp.querier', config) + value = '1' if (tmp != None) else '0' + self.set_multicast_querier(value) + + # remove interface from bridge + tmp = vyos_dict_search('member.interface_remove', config) + if tmp: + for member in tmp: + self.del_port(member) + + STPBridgeIf = STP.enable(BridgeIf) + tmp = vyos_dict_search('member.interface', config) + if tmp: + for interface, interface_config in tmp.items(): + # if we've come here we already verified the interface + # does not have an addresses configured so just flush + # any remaining ones + Interface(interface).flush_addrs() + # enslave interface port to bridge + self.add_port(interface) + + tmp = STPBridgeIf(interface) + # set bridge port path cost + value = interface_config.get('cost') + tmp.set_path_cost(value) + + # set bridge port path priority + value = interface_config.get('priority') + tmp.set_path_priority(value) + + # Enable/Disable of an interface must always be done at the end of the + # derived class to make use of the ref-counting set_admin_state() + # function. We will only enable the interface if 'up' was called as + # often as 'down'. This is required by some interface implementations + # as certain parameters can only be changed when the interface is + # in admin-down state. This ensures the link does not flap during + # reconfiguration. + state = 'down' if 'disable' in config else 'up' + self.set_admin_state(state) diff --git a/python/vyos/ifconfig/dhcp.py b/python/vyos/ifconfig/dhcp.py deleted file mode 100644 index a8b9a2a87..000000000 --- a/python/vyos/ifconfig/dhcp.py +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright 2020 VyOS maintainers and contributors <maintainers@vyos.io> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. - -import os - -from vyos.dicts import FixedDict -from vyos.ifconfig.control import Control -from vyos.template import render - -class _DHCPv4 (Control): - def __init__(self, ifname): - super().__init__() - config_base = r'/var/lib/dhcp/dhclient_' - self.options = FixedDict(**{ - 'ifname': ifname, - 'hostname': '', - 'client_id': '', - 'vendor_class_id': '', - 'conf_file': config_base + f'{ifname}.conf', - 'options_file': config_base + f'{ifname}.options', - 'pid_file': config_base + f'{ifname}.pid', - 'lease_file': config_base + f'{ifname}.leases', - }) - - # replace dhcpv4/v6 with systemd.networkd? - def set(self): - """ - Configure interface as DHCP client. The dhclient binary is automatically - started in background! - - Example: - - >>> from vyos.ifconfig import Interface - >>> j = Interface('eth0') - >>> j.dhcp.v4.set() - """ - if not self.options['hostname']: - # read configured system hostname. - # maybe change to vyos hostd client ??? - with open('/etc/hostname', 'r') as f: - self.options['hostname'] = f.read().rstrip('\n') - - render(self.options['options_file'], 'dhcp-client/daemon-options.tmpl', self.options) - render(self.options['conf_file'], 'dhcp-client/ipv4.tmpl', self.options) - - return self._cmd('systemctl restart dhclient@{ifname}.service'.format(**self.options)) - - def delete(self): - """ - De-configure interface as DHCP clinet. All auto generated files like - pid, config and lease will be removed. - - Example: - - >>> from vyos.ifconfig import Interface - >>> j = Interface('eth0') - >>> j.dhcp.v4.delete() - """ - if not os.path.isfile(self.options['pid_file']): - self._debug_msg('No DHCP client PID found') - return None - - self._cmd('systemctl stop dhclient@{ifname}.service'.format(**self.options)) - - # cleanup old config files - for name in ('conf_file', 'options_file', 'pid_file', 'lease_file'): - if os.path.isfile(self.options[name]): - os.remove(self.options[name]) - -class _DHCPv6 (Control): - def __init__(self, ifname): - super().__init__() - self.options = FixedDict(**{ - 'ifname': ifname, - 'dhcpv6_prm_only': False, - 'dhcpv6_temporary': False, - 'dhcpv6_pd_interfaces': [], - 'dhcpv6_pd_length': '' - }) - self._conf_file = f'/run/dhcp6c/dhcp6c.{ifname}.conf' - - def set(self): - """ - Configure interface as DHCPv6 client. The dhclient binary is automatically - started in background! - - Example: - - >>> from vyos.ifconfig import Interface - >>> j = Interface('eth0') - >>> j.dhcp.v6.set() - """ - - # better save then sorry .. should be checked in interface script - # but if you missed it we are safe! - if self.options['dhcpv6_prm_only'] and self.options['dhcpv6_temporary']: - raise Exception( - 'DHCPv6 temporary and parameters-only options are mutually exclusive!') - - render(self._conf_file, 'dhcp-client/ipv6.tmpl', self.options, trim_blocks=True) - return self._cmd('systemctl restart dhcp6c@{ifname}.service'.format(**self.options)) - - def delete(self): - """ - De-configure interface as DHCPv6 clinet. All auto generated files like - pid, config and lease will be removed. - - Example: - - >>> from vyos.ifconfig import Interface - >>> j = Interface('eth0') - >>> j.dhcp.v6.delete() - """ - self._cmd('systemctl stop dhcp6c@{ifname}.service'.format(**self.options)) - - # cleanup old config files - if os.path.isfile(self._conf_file): - os.remove(self._conf_file) - - -class DHCP(object): - def __init__(self, ifname): - self.v4 = _DHCPv4(ifname) - self.v6 = _DHCPv6(ifname) diff --git a/python/vyos/ifconfig/dummy.py b/python/vyos/ifconfig/dummy.py index 404c490c7..43614cd1c 100644 --- a/python/vyos/ifconfig/dummy.py +++ b/python/vyos/ifconfig/dummy.py @@ -35,3 +35,22 @@ class DummyIf(Interface): 'prefixes': ['dum', ], }, } + + def update(self, config): + """ General helper function which works on a dictionary retrived by + get_config_dict(). It's main intention is to consolidate the scattered + interface setup code and provide a single point of entry when workin + on any interface. """ + + # call base class first + super().update(config) + + # Enable/Disable of an interface must always be done at the end of the + # derived class to make use of the ref-counting set_admin_state() + # function. We will only enable the interface if 'up' was called as + # often as 'down'. This is required by some interface implementations + # as certain parameters can only be changed when the interface is + # in admin-down state. This ensures the link does not flap during + # reconfiguration. + state = 'down' if 'disable' in config else 'up' + self.set_admin_state(state) diff --git a/python/vyos/ifconfig/ethernet.py b/python/vyos/ifconfig/ethernet.py index 5b18926c9..17c1bd64d 100644 --- a/python/vyos/ifconfig/ethernet.py +++ b/python/vyos/ifconfig/ethernet.py @@ -20,7 +20,7 @@ from vyos.ifconfig.interface import Interface from vyos.ifconfig.vlan import VLAN from vyos.validate import assert_list from vyos.util import run - +from vyos.util import vyos_dict_search @Interface.register @VLAN.enable @@ -252,3 +252,58 @@ class EthernetIf(Interface): >>> i.set_udp_offload('on') """ return self.set_interface('ufo', state) + + + def update(self, config): + """ General helper function which works on a dictionary retrived by + get_config_dict(). It's main intention is to consolidate the scattered + interface setup code and provide a single point of entry when workin + on any interface. """ + + # call base class first + super().update(config) + + # disable ethernet flow control (pause frames) + value = 'off' if 'disable_flow_control' in config.keys() else 'on' + self.set_flow_control(value) + + # GRO (generic receive offload) + tmp = vyos_dict_search('offload_options.generic_receive', config) + value = tmp if (tmp != None) else 'off' + self.set_gro(value) + + # GSO (generic segmentation offload) + tmp = vyos_dict_search('offload_options.generic_segmentation', config) + value = tmp if (tmp != None) else 'off' + self.set_gso(value) + + # scatter-gather option + tmp = vyos_dict_search('offload_options.scatter_gather', config) + value = tmp if (tmp != None) else 'off' + self.set_sg(value) + + # TSO (TCP segmentation offloading) + tmp = vyos_dict_search('offload_options.udp_fragmentation', config) + value = tmp if (tmp != None) else 'off' + self.set_tso(value) + + # UDP fragmentation offloading + tmp = vyos_dict_search('offload_options.udp_fragmentation', config) + value = tmp if (tmp != None) else 'off' + self.set_ufo(value) + + # Set physical interface speed and duplex + if {'speed', 'duplex'} <= set(config): + speed = config.get('speed') + duplex = config.get('duplex') + self.set_speed_duplex(speed, duplex) + + # Enable/Disable of an interface must always be done at the end of the + # derived class to make use of the ref-counting set_admin_state() + # function. We will only enable the interface if 'up' was called as + # often as 'down'. This is required by some interface implementations + # as certain parameters can only be changed when the interface is + # in admin-down state. This ensures the link does not flap during + # reconfiguration. + state = 'down' if 'disable' in config else 'up' + self.set_admin_state(state) diff --git a/python/vyos/ifconfig/geneve.py b/python/vyos/ifconfig/geneve.py index 145dc268c..dd0658668 100644 --- a/python/vyos/ifconfig/geneve.py +++ b/python/vyos/ifconfig/geneve.py @@ -64,3 +64,22 @@ class GeneveIf(Interface): >> dict = GeneveIf().get_config() """ return deepcopy(cls.default) + + def update(self, config): + """ General helper function which works on a dictionary retrived by + get_config_dict(). It's main intention is to consolidate the scattered + interface setup code and provide a single point of entry when workin + on any interface. """ + + # call base class first + super().update(config) + + # Enable/Disable of an interface must always be done at the end of the + # derived class to make use of the ref-counting set_admin_state() + # function. We will only enable the interface if 'up' was called as + # often as 'down'. This is required by some interface implementations + # as certain parameters can only be changed when the interface is + # in admin-down state. This ensures the link does not flap during + # reconfiguration. + state = 'down' if 'disable' in config else 'up' + self.set_admin_state(state) diff --git a/python/vyos/ifconfig/interface.py b/python/vyos/ifconfig/interface.py index 1819ffc82..ef2336c17 100644 --- a/python/vyos/ifconfig/interface.py +++ b/python/vyos/ifconfig/interface.py @@ -16,7 +16,10 @@ import os import re import json +import jmespath + from copy import deepcopy +from glob import glob from ipaddress import IPv4Network from ipaddress import IPv6Address @@ -28,7 +31,10 @@ from netifaces import AF_INET6 from vyos import ConfigError from vyos.configdict import list_diff +from vyos.configdict import dict_merge +from vyos.template import render from vyos.util import mac2eui64 +from vyos.util import vyos_dict_search from vyos.validate import is_ipv4 from vyos.validate import is_ipv6 from vyos.validate import is_intf_addr_assigned @@ -40,11 +46,17 @@ from vyos.validate import assert_positive from vyos.validate import assert_range from vyos.ifconfig.control import Control -from vyos.ifconfig.dhcp import DHCP from vyos.ifconfig.vrrp import VRRP from vyos.ifconfig.operational import Operational from vyos.ifconfig import Section +def get_ethertype(ethertype_val): + if ethertype_val == '0x88A8': + return '802.1ad' + elif ethertype_val == '0x8100': + return '802.1q' + else: + raise ConfigError('invalid ethertype "{}"'.format(ethertype_val)) class Interface(Control): # This is the class which will be used to create @@ -72,8 +84,12 @@ class Interface(Control): _command_get = { 'admin_state': { 'shellcmd': 'ip -json link show dev {ifname}', - 'format': lambda j: 'up' if 'UP' in json.loads(j)[0]['flags'] else 'down', - } + 'format': lambda j: 'up' if 'UP' in jmespath.search('[*].flags | [0]', json.loads(j)) else 'down', + }, + 'vlan_protocol': { + 'shellcmd': 'ip -json -details link show dev {ifname}', + 'format': lambda j: jmespath.search('[*].linkinfo.info_data.protocol | [0]', json.loads(j)), + }, } _command_set = { @@ -197,11 +213,11 @@ class Interface(Control): # make sure the ifname is the first argument and not from the dict self.config['ifname'] = ifname + self._admin_state_down_cnt = 0 # we must have updated config before initialising the Interface super().__init__(**kargs) self.ifname = ifname - self.dhcp = DHCP(ifname) if not self.exists(ifname): # Any instance of Interface, such as Interface('eth0') @@ -323,6 +339,10 @@ class Interface(Control): self.set_interface('mac', mac) + # Turn an interface to the 'up' state if it was changed to 'down' by this fucntion + if prev_state == 'up': + self.set_admin_state('up') + def set_vrf(self, vrf=''): """ Add/Remove interface from given VRF instance. @@ -539,6 +559,17 @@ class Interface(Control): """ self.set_interface('alias', ifalias) + def get_vlan_protocol(self): + """ + Retrieve VLAN protocol in use, this can be 802.1Q, 802.1ad or None + + Example: + >>> from vyos.ifconfig import Interface + >>> Interface('eth0.10').get_vlan_protocol() + '802.1Q' + """ + return self.get_interface('vlan_protocol') + def get_admin_state(self): """ Get interface administrative state. Function will return 'up' or 'down' @@ -560,7 +591,24 @@ class Interface(Control): >>> Interface('eth0').get_admin_state() 'down' """ - return self.set_interface('admin_state', state) + # A VLAN interface can only be placed in admin up state when + # the lower interface is up, too + if self.get_vlan_protocol(): + lower_interface = glob(f'/sys/class/net/{self.ifname}/lower*/flags')[0] + with open(lower_interface, 'r') as f: + flags = f.read() + # If parent is not up - bail out as we can not bring up the VLAN. + # Flags are defined in kernel source include/uapi/linux/if.h + if not int(flags, 16) & 1: + return None + + if state == 'up': + self._admin_state_down_cnt -= 1 + if self._admin_state_down_cnt < 1: + return self.set_interface('admin_state', state) + else: + self._admin_state_down_cnt += 1 + return self.set_interface('admin_state', state) def set_proxy_arp(self, enable): """ @@ -663,21 +711,24 @@ class Interface(Control): if addr in self._addr: return False + addr_is_v4 = is_ipv4(addr) + # we can't have both DHCP and static IPv4 addresses assigned for a in self._addr: if ( ( addr == 'dhcp' and a != 'dhcpv6' and is_ipv4(a) ) or - ( a == 'dhcp' and addr != 'dhcpv6' and is_ipv4(addr) ) ): + ( a == 'dhcp' and addr != 'dhcpv6' and addr_is_v4 ) ): raise ConfigError(( "Can't configure both static IPv4 and DHCP address " "on the same interface")) # add to interface if addr == 'dhcp': - self.dhcp.v4.set() + self.set_dhcp(True) elif addr == 'dhcpv6': - self.dhcp.v6.set() + self.set_dhcpv6(True) elif not is_intf_addr_assigned(self.ifname, addr): - self._cmd(f'ip addr add "{addr}" dev "{self.ifname}"') + self._cmd(f'ip addr add "{addr}" ' + f'{"brd + " if addr_is_v4 else ""}dev "{self.ifname}"') else: return False @@ -713,9 +764,9 @@ class Interface(Control): # remove from interface if addr == 'dhcp': - self.dhcp.v4.delete() + self.set_dhcp(False) elif addr == 'dhcpv6': - self.dhcp.v6.delete() + self.set_dhcpv6(False) elif is_intf_addr_assigned(self.ifname, addr): self._cmd(f'ip addr del "{addr}" dev "{self.ifname}"') else: @@ -734,8 +785,8 @@ class Interface(Control): Will raise an exception on error. """ # stop DHCP(v6) if running - self.dhcp.v4.delete() - self.dhcp.v6.delete() + self.set_dhcp(False) + self.set_dhcpv6(False) # flush all addresses self._cmd(f'ip addr flush dev "{self.ifname}"') @@ -759,23 +810,103 @@ class Interface(Control): return True + def set_dhcp(self, enable): + """ + Enable/Disable DHCP client on a given interface. + """ + if enable not in [True, False]: + raise ValueError() + + ifname = self.ifname + config_base = r'/var/lib/dhcp/dhclient' + config_file = f'{config_base}_{ifname}.conf' + options_file = f'{config_base}_{ifname}.options' + pid_file = f'{config_base}_{ifname}.pid' + lease_file = f'{config_base}_{ifname}.leases' + + if enable and 'disable' not in self._config: + if vyos_dict_search('dhcp_options.host_name', self._config) == None: + # read configured system hostname. + # maybe change to vyos hostd client ??? + hostname = 'vyos' + with open('/etc/hostname', 'r') as f: + hostname = f.read().rstrip('\n') + tmp = {'dhcp_options' : { 'host_name' : hostname}} + self._config = dict_merge(tmp, self._config) + + render(options_file, 'dhcp-client/daemon-options.tmpl', + self._config, trim_blocks=True) + render(config_file, 'dhcp-client/ipv4.tmpl', + self._config, trim_blocks=True) + + # 'up' check is mandatory b/c even if the interface is A/D, as soon as + # the DHCP client is started the interface will be placed in u/u state. + # This is not what we intended to do when disabling an interface. + return self._cmd(f'systemctl restart dhclient@{ifname}.service') + else: + self._cmd(f'systemctl stop dhclient@{ifname}.service') + + # cleanup old config files + for file in [config_file, options_file, pid_file, lease_file]: + if os.path.isfile(file): + os.remove(file) + + + def set_dhcpv6(self, enable): + """ + Enable/Disable DHCPv6 client on a given interface. + """ + if enable not in [True, False]: + raise ValueError() + + ifname = self.ifname + config_file = f'/run/dhcp6c/dhcp6c.{ifname}.conf' + + if enable and 'disable' not in self._config: + render(config_file, 'dhcp-client/ipv6.tmpl', + self._config, trim_blocks=True) + + # We must ignore any return codes. This is required to enable DHCPv6-PD + # for interfaces which are yet not up and running. + return self._popen(f'systemctl restart dhcp6c@{ifname}.service') + else: + self._popen(f'systemctl stop dhcp6c@{ifname}.service') + + if os.path.isfile(config_file): + os.remove(config_file) + + def update(self, config): """ General helper function which works on a dictionary retrived by get_config_dict(). It's main intention is to consolidate the scattered interface setup code and provide a single point of entry when workin on any interface. """ + # Cache the configuration - it will be reused inside e.g. DHCP handler + # XXX: maybe pass the option via __init__ in the future and rename this + # method to apply()? + self._config = config + # Update interface description - self.set_alias(config.get('description', None)) + self.set_alias(config.get('description', '')) + + # Ignore link state changes + value = '2' if 'disable_link_detect' in config else '1' + self.set_link_detect(value) # Configure assigned interface IP addresses. No longer # configured addresses will be removed first new_addr = config.get('address', []) - # XXX workaround for T2636, convert IP address string to a list - # with one element - if isinstance(new_addr, str): - new_addr = [new_addr] + # always ensure DHCP client is stopped (when not configured explicitly) + if 'dhcp' not in new_addr: + self.del_addr('dhcp') + + # always ensure DHCPv6 client is stopped (when not configured as client + # for IPv6 address or prefix delegation + dhcpv6pd = vyos_dict_search('dhcpv6_options.pd', config) + if 'dhcpv6' not in new_addr or dhcpv6pd == None: + self.del_addr('dhcpv6') # determine IP addresses which are assigned to the interface and build a # list of addresses which are no longer in the dict so they can be removed @@ -786,13 +917,144 @@ class Interface(Control): for addr in new_addr: self.add_addr(addr) + # start DHCPv6 client when only PD was configured + if dhcpv6pd != None: + self.set_dhcpv6(True) + # There are some items in the configuration which can only be applied # if this instance is not bound to a bridge. This should be checked # by the caller but better save then sorry! - if not config.get('is_bridge_member', False): - # Bind interface instance into VRF + if not any(k in ['is_bond_member', 'is_bridge_member'] for k in config): + # Bind interface to given VRF or unbind it if vrf node is not set. + # unbinding will call 'ip link set dev eth0 nomaster' which will + # also drop the interface out of a bridge or bond - thus this is + # checked before self.set_vrf(config.get('vrf', '')) - # Interface administrative state - state = 'down' if 'disable' in config.keys() else 'up' - self.set_admin_state(state) + # Configure ARP cache timeout in milliseconds - has default value + tmp = vyos_dict_search('ip.arp_cache_timeout', config) + value = tmp if (tmp != None) else '30' + self.set_arp_cache_tmo(value) + + # Configure ARP filter configuration + tmp = vyos_dict_search('ip.disable_arp_filter', config) + value = '0' if (tmp != None) else '1' + self.set_arp_filter(value) + + # Configure ARP accept + tmp = vyos_dict_search('ip.enable_arp_accept', config) + value = '1' if (tmp != None) else '0' + self.set_arp_accept(value) + + # Configure ARP announce + tmp = vyos_dict_search('ip.enable_arp_announce', config) + value = '1' if (tmp != None) else '0' + self.set_arp_announce(value) + + # Configure ARP ignore + tmp = vyos_dict_search('ip.enable_arp_ignore', config) + value = '1' if (tmp != None) else '0' + self.set_arp_ignore(value) + + # Enable proxy-arp on this interface + tmp = vyos_dict_search('ip.enable_proxy_arp', config) + value = '1' if (tmp != None) else '0' + self.set_proxy_arp(value) + + # Enable private VLAN proxy ARP on this interface + tmp = vyos_dict_search('ip.proxy_arp_pvlan', config) + value = '1' if (tmp != None) else '0' + self.set_proxy_arp_pvlan(value) + + # IPv6 forwarding + tmp = vyos_dict_search('ipv6.disable_forwarding', config) + value = '0' if (tmp != None) else '1' + self.set_ipv6_forwarding(value) + + # IPv6 router advertisements + tmp = vyos_dict_search('ipv6.address.autoconf', config) + value = '2' if (tmp != None) else '1' + if 'dhcpv6' in new_addr: + value = '2' + self.set_ipv6_accept_ra(value) + + # IPv6 address autoconfiguration + tmp = vyos_dict_search('ipv6.address.autoconf', config) + value = '1' if (tmp != None) else '0' + self.set_ipv6_autoconf(value) + + # IPv6 Duplicate Address Detection (DAD) tries + tmp = vyos_dict_search('ipv6.dup_addr_detect_transmits', config) + value = tmp if (tmp != None) else '1' + self.set_ipv6_dad_messages(value) + + # MTU - Maximum Transfer Unit + if 'mtu' in config: + self.set_mtu(config.get('mtu')) + + # Delete old IPv6 EUI64 addresses before changing MAC + tmp = vyos_dict_search('ipv6.address.eui64_old', config) + if tmp: + for addr in tmp: + self.del_ipv6_eui64_address(addr) + + # Change interface MAC address - re-set to real hardware address (hw-id) + # if custom mac is removed. Skip if bond member. + if 'is_bond_member' not in config: + mac = config.get('hw_id') + if 'mac' in config: + mac = config.get('mac') + if mac: + self.set_mac(mac) + + # Manage IPv6 link-local addresses + tmp = vyos_dict_search('ipv6.address.no_default_link_local', config) + # we must check explicitly for None type as if the key is set we will + # get an empty dict (<class 'dict'>) + if tmp is not None: + self.del_ipv6_eui64_address('fe80::/64') + else: + self.add_ipv6_eui64_address('fe80::/64') + + # Add IPv6 EUI-based addresses + tmp = vyos_dict_search('ipv6.address.eui64', config) + if tmp: + for addr in tmp: + self.add_ipv6_eui64_address(addr) + + # re-add ourselves to any bridge we might have fallen out of + if 'is_bridge_member' in config: + bridge = config.get('is_bridge_member') + self.add_to_bridge(bridge) + + # remove no longer required 802.1ad (Q-in-Q VLANs) + for vif_s_id in config.get('vif_s_remove', {}): + self.del_vlan(vif_s_id) + + # create/update 802.1ad (Q-in-Q VLANs) + ifname = config['ifname'] + for vif_s_id, vif_s in config.get('vif_s', {}).items(): + tmp=get_ethertype(vif_s.get('ethertype', '0x88A8')) + s_vlan = self.add_vlan(vif_s_id, ethertype=tmp) + vif_s['ifname'] = f'{ifname}.{vif_s_id}' + s_vlan.update(vif_s) + + # remove no longer required client VLAN (vif-c) + for vif_c_id in vif_s.get('vif_c_remove', {}): + s_vlan.del_vlan(vif_c_id) + + # create/update client VLAN (vif-c) interface + for vif_c_id, vif_c in vif_s.get('vif_c', {}).items(): + c_vlan = s_vlan.add_vlan(vif_c_id) + vif_c['ifname'] = f'{ifname}.{vif_s_id}.{vif_c_id}' + c_vlan.update(vif_c) + + # remove no longer required 802.1q VLAN interfaces + for vif_id in config.get('vif_remove', {}): + self.del_vlan(vif_id) + + # create/update 802.1q VLAN interfaces + for vif_id, vif in config.get('vif', {}).items(): + vlan = self.add_vlan(vif_id) + vif['ifname'] = f'{ifname}.{vif_id}' + vlan.update(vif) diff --git a/python/vyos/ifconfig/loopback.py b/python/vyos/ifconfig/loopback.py index 7ebd13b54..c70e1773f 100644 --- a/python/vyos/ifconfig/loopback.py +++ b/python/vyos/ifconfig/loopback.py @@ -64,16 +64,21 @@ class LoopbackIf(Interface): on any interface. """ addr = config.get('address', []) - # XXX workaround for T2636, convert IP address string to a list - # with one element - if isinstance(addr, str): - addr = [addr] - # We must ensure that the loopback addresses are never deleted from the system addr += self._persistent_addresses # Update IP address entry in our dictionary config.update({'address' : addr}) - # now call the regular function from within our base class + # call base class super().update(config) + + # Enable/Disable of an interface must always be done at the end of the + # derived class to make use of the ref-counting set_admin_state() + # function. We will only enable the interface if 'up' was called as + # often as 'down'. This is required by some interface implementations + # as certain parameters can only be changed when the interface is + # in admin-down state. This ensures the link does not flap during + # reconfiguration. + state = 'down' if 'disable' in config else 'up' + self.set_admin_state(state) diff --git a/python/vyos/ifconfig/macsec.py b/python/vyos/ifconfig/macsec.py index ea8c9807e..6f570d162 100644 --- a/python/vyos/ifconfig/macsec.py +++ b/python/vyos/ifconfig/macsec.py @@ -71,3 +71,22 @@ class MACsecIf(Interface): 'source_interface': '', } return config + + def update(self, config): + """ General helper function which works on a dictionary retrived by + get_config_dict(). It's main intention is to consolidate the scattered + interface setup code and provide a single point of entry when workin + on any interface. """ + + # call base class first + super().update(config) + + # Enable/Disable of an interface must always be done at the end of the + # derived class to make use of the ref-counting set_admin_state() + # function. We will only enable the interface if 'up' was called as + # often as 'down'. This is required by some interface implementations + # as certain parameters can only be changed when the interface is + # in admin-down state. This ensures the link does not flap during + # reconfiguration. + state = 'down' if 'disable' in config else 'up' + self.set_admin_state(state) diff --git a/python/vyos/ifconfig/macvlan.py b/python/vyos/ifconfig/macvlan.py index b5481f4a7..b068ce873 100644 --- a/python/vyos/ifconfig/macvlan.py +++ b/python/vyos/ifconfig/macvlan.py @@ -68,3 +68,22 @@ class MACVLANIf(Interface): >> dict = MACVLANIf().get_config() """ return deepcopy(cls.default) + + def update(self, config): + """ General helper function which works on a dictionary retrived by + get_config_dict(). It's main intention is to consolidate the scattered + interface setup code and provide a single point of entry when workin + on any interface. """ + + # call base class first + super().update(config) + + # Enable/Disable of an interface must always be done at the end of the + # derived class to make use of the ref-counting set_admin_state() + # function. We will only enable the interface if 'up' was called as + # often as 'down'. This is required by some interface implementations + # as certain parameters can only be changed when the interface is + # in admin-down state. This ensures the link does not flap during + # reconfiguration. + state = 'down' if 'disable' in config else 'up' + self.set_admin_state(state) diff --git a/python/vyos/ifconfig/vrrp.py b/python/vyos/ifconfig/vrrp.py index 5e6387881..01a7cc7ab 100644 --- a/python/vyos/ifconfig/vrrp.py +++ b/python/vyos/ifconfig/vrrp.py @@ -96,7 +96,7 @@ class VRRP(object): pid = util.read_file(cls.location['pid']) os.kill(int(pid), cls._signal[what]) - # shoud look for file size change ? + # should look for file size change? sleep(0.2) return util.read_file(fname) except FileNotFoundError: @@ -126,8 +126,8 @@ class VRRP(object): return disabled @classmethod - def format (cls, data): - headers = ["Name", "Interface", "VRID", "State", "Last Transition"] + def format(cls, data): + headers = ["Name", "Interface", "VRID", "State", "Priority", "Last Transition"] groups = [] data = json.loads(data) @@ -138,11 +138,12 @@ class VRRP(object): intf = data['ifp_ifname'] vrid = data['vrid'] state = cls.decode_state(data["state"]) + priority = data['effective_priority'] since = int(time() - float(data['last_transition'])) last = util.seconds_to_human(since) - groups.append([name, intf, vrid, state, last]) + groups.append([name, intf, vrid, state, priority, last]) # add to the active list disabled instances groups.extend(cls.disabled()) diff --git a/python/vyos/ifconfig/vxlan.py b/python/vyos/ifconfig/vxlan.py index 973b4ef05..18a500336 100644 --- a/python/vyos/ifconfig/vxlan.py +++ b/python/vyos/ifconfig/vxlan.py @@ -47,8 +47,8 @@ class VXLANIf(Interface): 'port': 8472, # The Linux implementation of VXLAN pre-dates # the IANA's selection of a standard destination port 'remote': '', - 'src_address': '', - 'src_interface': '', + 'source_address': '', + 'source_interface': '', 'vni': 0 } definition = { @@ -60,29 +60,29 @@ class VXLANIf(Interface): } } options = Interface.options + \ - ['group', 'remote', 'src_interface', 'port', 'vni', 'src_address'] + ['group', 'remote', 'source_interface', 'port', 'vni', 'source_address'] mapping = { 'ifname': 'add', 'vni': 'id', 'port': 'dstport', - 'src_address': 'local', - 'src_interface': 'dev', + 'source_address': 'local', + 'source_interface': 'dev', } def _create(self): cmdline = ['ifname', 'type', 'vni', 'port'] - if self.config['src_address']: - cmdline.append('src_address') + if self.config['source_address']: + cmdline.append('source_address') if self.config['remote']: cmdline.append('remote') - if self.config['group'] or self.config['src_interface']: - if self.config['group'] and self.config['src_interface']: + if self.config['group'] or self.config['source_interface']: + if self.config['group'] and self.config['source_interface']: cmdline.append('group') - cmdline.append('src_interface') + cmdline.append('source_interface') else: ifname = self.config['ifname'] raise ConfigError( @@ -109,3 +109,22 @@ class VXLANIf(Interface): >> dict = VXLANIf().get_config() """ return deepcopy(cls.default) + + def update(self, config): + """ General helper function which works on a dictionary retrived by + get_config_dict(). It's main intention is to consolidate the scattered + interface setup code and provide a single point of entry when workin + on any interface. """ + + # call base class first + super().update(config) + + # Enable/Disable of an interface must always be done at the end of the + # derived class to make use of the ref-counting set_admin_state() + # function. We will only enable the interface if 'up' was called as + # often as 'down'. This is required by some interface implementations + # as certain parameters can only be changed when the interface is + # in admin-down state. This ensures the link does not flap during + # reconfiguration. + state = 'down' if 'disable' in config else 'up' + self.set_admin_state(state) diff --git a/python/vyos/ifconfig/wireguard.py b/python/vyos/ifconfig/wireguard.py index 62ca57ca2..fad4ef282 100644 --- a/python/vyos/ifconfig/wireguard.py +++ b/python/vyos/ifconfig/wireguard.py @@ -24,7 +24,7 @@ from hurry.filesize import alternative from vyos.config import Config from vyos.ifconfig import Interface from vyos.ifconfig import Operational - +from vyos.validate import is_ipv6 class WireGuardOperational(Operational): def _dump(self): @@ -169,65 +169,79 @@ class WireGuardIf(Interface): ['port', 'private_key', 'pubkey', 'psk', 'allowed_ips', 'fwmark', 'endpoint', 'keepalive'] - """ - Wireguard interface class, contains a comnfig dictionary since - wireguard VPN is being comnfigured via the wg command rather than - writing the config into a file. Otherwise if a pre-shared key is used - (symetric enryption key), it would we exposed within multiple files. - Currently it's only within the config.boot if the config was saved. - - Example: - >>> from vyos.ifconfig import WireGuardIf as wg_if - >>> wg_intfc = wg_if("wg01") - >>> print (wg_intfc.wg_config) - {'private_key': None, 'keepalive': 0, 'endpoint': None, 'port': 0, - 'allowed_ips': [], 'pubkey': None, 'fwmark': 0, 'psk': '/dev/null'} - >>> wg_intfc.wg_config['keepalive'] = 100 - >>> print (wg_intfc.wg_config) - {'private_key': None, 'keepalive': 100, 'endpoint': None, 'port': 0, - 'allowed_ips': [], 'pubkey': None, 'fwmark': 0, 'psk': '/dev/null'} - """ - - def update(self): - if not self.config['private_key']: - raise ValueError("private key required") - else: - # fmask permission check? - pass - - cmd = 'wg set {ifname}'.format(**self.config) - cmd += ' listen-port {port}'.format(**self.config) - cmd += ' fwmark "{fwmark}" '.format(**self.config) - cmd += ' private-key {private_key}'.format(**self.config) - cmd += ' peer {pubkey}'.format(**self.config) - cmd += ' persistent-keepalive {keepalive}'.format(**self.config) - # allowed-ips must be properly quoted else the interface can't be properly - # created as the wg utility will tread multiple IP addresses as command - # parameters - cmd += ' allowed-ips "{}"'.format(','.join(self.config['allowed-ips'])) - - if self.config['endpoint']: - cmd += ' endpoint "{endpoint}"'.format(**self.config) - - psk_file = '' - if self.config['psk']: - psk_file = '/tmp/{ifname}.psk'.format(**self.config) - with open(psk_file, 'w') as f: - f.write(self.config['psk']) + def update(self, config): + """ General helper function which works on a dictionary retrived by + get_config_dict(). It's main intention is to consolidate the scattered + interface setup code and provide a single point of entry when workin + on any interface. """ + + # remove no longer associated peers first + if 'peer_remove' in config: + for tmp in config['peer_remove']: + peer = config['peer_remove'][tmp] + peer['ifname'] = config['ifname'] + + cmd = 'wg set {ifname} peer {pubkey} remove' + self._cmd(cmd.format(**peer)) + + # Wireguard base command is identical for every peer + base_cmd = 'wg set {ifname} private-key {private_key}' + if 'port' in config: + base_cmd += ' listen-port {port}' + if 'fwmark' in config: + base_cmd += ' fwmark {fwmark}' + + base_cmd = base_cmd.format(**config) + + for tmp in config['peer']: + peer = config['peer'][tmp] + + # start of with a fresh 'wg' command + cmd = base_cmd + ' peer {pubkey}' + + # If no PSK is given remove it by using /dev/null - passing keys via + # the shell (usually bash) is considered insecure, thus we use a file + no_psk_file = '/dev/null' + psk_file = no_psk_file + if 'preshared_key' in peer: + psk_file = '/tmp/tmp.wireguard.psk' + with open(psk_file, 'w') as f: + f.write(peer['preshared_key']) cmd += f' preshared-key {psk_file}' - self._cmd(cmd) - - # PSK key file is not required to be stored persistently as its backed by CLI - if os.path.exists(psk_file): - os.remove(psk_file) - - def remove_peer(self, peerkey): - """ - Remove a peer of an interface, peers are identified by their public key. - Giving it a readable name is a vyos feature, to remove a peer the pubkey - and the interface is needed, to remove the entry. - """ - cmd = "wg set {0} peer {1} remove".format( - self.config['ifname'], str(peerkey)) - return self._cmd(cmd) + # Persistent keepalive is optional + if 'persistent_keepalive'in peer: + cmd += ' persistent-keepalive {persistent_keepalive}' + + # Multiple allowed-ip ranges can be defined - ensure we are always + # dealing with a list + if isinstance(peer['allowed_ips'], str): + peer['allowed_ips'] = [peer['allowed_ips']] + cmd += ' allowed-ips ' + ','.join(peer['allowed_ips']) + + # Endpoint configuration is optional + if {'address', 'port'} <= set(peer): + if is_ipv6(config['address']): + cmd += ' endpoint [{address}]:{port}' + else: + cmd += ' endpoint {address}:{port}' + + self._cmd(cmd.format(**peer)) + + # PSK key file is not required to be stored persistently as its backed by CLI + if psk_file != no_psk_file and os.path.exists(psk_file): + os.remove(psk_file) + + # call base class + super().update(config) + + # Enable/Disable of an interface must always be done at the end of the + # derived class to make use of the ref-counting set_admin_state() + # function. We will only enable the interface if 'up' was called as + # often as 'down'. This is required by some interface implementations + # as certain parameters can only be changed when the interface is + # in admin-down state. This ensures the link does not flap during + # reconfiguration. + state = 'down' if 'disable' in config else 'up' + self.set_admin_state(state) + diff --git a/python/vyos/ifconfig/wireless.py b/python/vyos/ifconfig/wireless.py index 3122ac0a3..a50346ffa 100644 --- a/python/vyos/ifconfig/wireless.py +++ b/python/vyos/ifconfig/wireless.py @@ -71,6 +71,25 @@ class WiFiIf(Interface): return config + def update(self, config): + """ General helper function which works on a dictionary retrived by + get_config_dict(). It's main intention is to consolidate the scattered + interface setup code and provide a single point of entry when workin + on any interface. """ + + # call base class first + super().update(config) + + # Enable/Disable of an interface must always be done at the end of the + # derived class to make use of the ref-counting set_admin_state() + # function. We will only enable the interface if 'up' was called as + # often as 'down'. This is required by some interface implementations + # as certain parameters can only be changed when the interface is + # in admin-down state. This ensures the link does not flap during + # reconfiguration. + state = 'down' if 'disable' in config else 'up' + self.set_admin_state(state) + @Interface.register class WiFiModemIf(WiFiIf): diff --git a/python/vyos/ifconfig_vlan.py b/python/vyos/ifconfig_vlan.py deleted file mode 100644 index 442cb0db8..000000000 --- a/python/vyos/ifconfig_vlan.py +++ /dev/null @@ -1,245 +0,0 @@ -# Copyright 2019-2020 VyOS maintainers and contributors <maintainers@vyos.io> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. - -from netifaces import interfaces -from vyos import ConfigError - -def apply_all_vlans(intf, intfconfig): - """ - Function applies all VLANs to the passed interface. - - intf: object of Interface class - intfconfig: dict with interface configuration - """ - # remove no longer required service VLAN interfaces (vif-s) - for vif_s in intfconfig['vif_s_remove']: - intf.del_vlan(vif_s) - - # create service VLAN interfaces (vif-s) - for vif_s_id, vif_s in intfconfig['vif_s'].items(): - s_vlan = intf.add_vlan(vif_s_id, ethertype=vif_s['ethertype']) - apply_vlan_config(s_vlan, vif_s) - - # remove no longer required client VLAN interfaces (vif-c) - # on lower service VLAN interface - for vif_c in vif_s['vif_c_remove']: - s_vlan.del_vlan(vif_c) - - # create client VLAN interfaces (vif-c) - # on lower service VLAN interface - for vif_c_id, vif_c in vif_s['vif_c'].items(): - c_vlan = s_vlan.add_vlan(vif_c_id) - apply_vlan_config(c_vlan, vif_c) - - # remove no longer required VLAN interfaces (vif) - for vif in intfconfig['vif_remove']: - intf.del_vlan(vif) - - # create VLAN interfaces (vif) - for vif_id, vif in intfconfig['vif'].items(): - # QoS priority mapping can only be set during interface creation - # so we delete the interface first if required. - if vif['egress_qos_changed'] or vif['ingress_qos_changed']: - try: - # on system bootup the above condition is true but the interface - # does not exists, which throws an exception, but that's legal - intf.del_vlan(vif_id) - except: - pass - - vlan = intf.add_vlan(vif_id, ingress_qos=vif['ingress_qos'], egress_qos=vif['egress_qos']) - apply_vlan_config(vlan, vif) - - -def apply_vlan_config(vlan, config): - """ - Generic function to apply a VLAN configuration from a dictionary - to a VLAN interface - """ - - if not vlan.definition['vlan']: - raise TypeError() - - if config['dhcp_client_id']: - vlan.dhcp.v4.options['client_id'] = config['dhcp_client_id'] - - if config['dhcp_hostname']: - vlan.dhcp.v4.options['hostname'] = config['dhcp_hostname'] - - if config['dhcp_vendor_class_id']: - vlan.dhcp.v4.options['vendor_class_id'] = config['dhcp_vendor_class_id'] - - if config['dhcpv6_prm_only']: - vlan.dhcp.v6.options['dhcpv6_prm_only'] = True - - if config['dhcpv6_temporary']: - vlan.dhcp.v6.options['dhcpv6_temporary'] = True - - if config['dhcpv6_pd_length']: - vlan.dhcp.v6.options['dhcpv6_pd_length'] = config['dhcpv6_pd_length'] - - if config['dhcpv6_pd_interfaces']: - vlan.dhcp.v6.options['dhcpv6_pd_interfaces'] = config['dhcpv6_pd_interfaces'] - - # update interface description used e.g. within SNMP - vlan.set_alias(config['description']) - # ignore link state changes - vlan.set_link_detect(config['disable_link_detect']) - # configure ARP filter configuration - vlan.set_arp_filter(config['ip_disable_arp_filter']) - # configure ARP accept - vlan.set_arp_accept(config['ip_enable_arp_accept']) - # configure ARP announce - vlan.set_arp_announce(config['ip_enable_arp_announce']) - # configure ARP ignore - vlan.set_arp_ignore(config['ip_enable_arp_ignore']) - # configure Proxy ARP - vlan.set_proxy_arp(config['ip_proxy_arp']) - # IPv6 accept RA - vlan.set_ipv6_accept_ra(config['ipv6_accept_ra']) - # IPv6 address autoconfiguration - vlan.set_ipv6_autoconf(config['ipv6_autoconf']) - # IPv6 forwarding - vlan.set_ipv6_forwarding(config['ipv6_forwarding']) - # IPv6 Duplicate Address Detection (DAD) tries - vlan.set_ipv6_dad_messages(config['ipv6_dup_addr_detect']) - # Maximum Transmission Unit (MTU) - vlan.set_mtu(config['mtu']) - - # assign/remove VRF (ONLY when not a member of a bridge, - # otherwise 'nomaster' removes it from it) - if not config['is_bridge_member']: - vlan.set_vrf(config['vrf']) - - # Delete old IPv6 EUI64 addresses before changing MAC - for addr in config['ipv6_eui64_prefix_remove']: - vlan.del_ipv6_eui64_address(addr) - - # Change VLAN interface MAC address - if config['mac']: - vlan.set_mac(config['mac']) - - # Add IPv6 EUI-based addresses - for addr in config['ipv6_eui64_prefix']: - vlan.add_ipv6_eui64_address(addr) - - # enable/disable VLAN interface - if config['disable']: - vlan.set_admin_state('down') - else: - vlan.set_admin_state('up') - - # Configure interface address(es) - # - not longer required addresses get removed first - # - newly addresses will be added second - for addr in config['address_remove']: - vlan.del_addr(addr) - for addr in config['address']: - vlan.add_addr(addr) - - # re-add ourselves to any bridge we might have fallen out of - if config['is_bridge_member']: - vlan.add_to_bridge(config['is_bridge_member']) - -def verify_vlan_config(config): - """ - Generic function to verify VLAN config consistency. Instead of re- - implementing this function in multiple places use single source \o/ - """ - - # config['vif'] is a dict with ids as keys and config dicts as values - for vif in config['vif'].values(): - # DHCPv6 parameters-only and temporary address are mutually exclusive - if vif['dhcpv6_prm_only'] and vif['dhcpv6_temporary']: - raise ConfigError('DHCPv6 temporary and parameters-only options are mutually exclusive!') - - if ( vif['is_bridge_member'] - and ( vif['address'] - or vif['ipv6_eui64_prefix'] - or vif['ipv6_autoconf'] ) ): - raise ConfigError(( - f'Cannot assign address to vif interface {vif["intf"]} ' - f'which is a member of bridge {vif["is_bridge_member"]}')) - - if vif['vrf']: - if vif['vrf'] not in interfaces(): - raise ConfigError(f'VRF "{vif["vrf"]}" does not exist') - - if vif['is_bridge_member']: - raise ConfigError(( - f'vif {vif["intf"]} cannot be member of VRF {vif["vrf"]} ' - f'and bridge {vif["is_bridge_member"]} at the same time!')) - - # e.g. wireless interface has no vif_s support - # thus we bail out eraly. - if 'vif_s' not in config.keys(): - return - - # config['vif_s'] is a dict with ids as keys and config dicts as values - for vif_s_id, vif_s in config['vif_s'].items(): - for vif_id, vif in config['vif'].items(): - if vif_id == vif_s_id: - raise ConfigError(( - f'Cannot use identical ID on vif "{vif["intf"]}" ' - f'and vif-s "{vif_s["intf"]}"')) - - # DHCPv6 parameters-only and temporary address are mutually exclusive - if vif_s['dhcpv6_prm_only'] and vif_s['dhcpv6_temporary']: - raise ConfigError(( - 'DHCPv6 temporary and parameters-only options are mutually ' - 'exclusive!')) - - if ( vif_s['is_bridge_member'] - and ( vif_s['address'] - or vif_s['ipv6_eui64_prefix'] - or vif_s['ipv6_autoconf'] ) ): - raise ConfigError(( - f'Cannot assign address to vif-s interface {vif_s["intf"]} ' - f'which is a member of bridge {vif_s["is_bridge_member"]}')) - - if vif_s['vrf']: - if vif_s['vrf'] not in interfaces(): - raise ConfigError(f'VRF "{vif_s["vrf"]}" does not exist') - - if vif_s['is_bridge_member']: - raise ConfigError(( - f'vif-s {vif_s["intf"]} cannot be member of VRF {vif_s["vrf"]} ' - f'and bridge {vif_s["is_bridge_member"]} at the same time!')) - - # vif_c is a dict with ids as keys and config dicts as values - for vif_c in vif_s['vif_c'].values(): - # DHCPv6 parameters-only and temporary address are mutually exclusive - if vif_c['dhcpv6_prm_only'] and vif_c['dhcpv6_temporary']: - raise ConfigError(( - 'DHCPv6 temporary and parameters-only options are ' - 'mutually exclusive!')) - - if ( vif_c['is_bridge_member'] - and ( vif_c['address'] - or vif_c['ipv6_eui64_prefix'] - or vif_c['ipv6_autoconf'] ) ): - raise ConfigError(( - f'Cannot assign address to vif-c interface {vif_c["intf"]} ' - f'which is a member of bridge {vif_c["is_bridge_member"]}')) - - if vif_c['vrf']: - if vif_c['vrf'] not in interfaces(): - raise ConfigError(f'VRF "{vif_c["vrf"]}" does not exist') - - if vif_c['is_bridge_member']: - raise ConfigError(( - f'vif-c {vif_c["intf"]} cannot be member of VRF {vif_c["vrf"]} ' - f'and bridge {vif_c["is_bridge_member"]} at the same time!')) - diff --git a/python/vyos/template.py b/python/vyos/template.py index d9b0c749d..c88ab04a0 100644 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -1,4 +1,4 @@ -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2020 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -13,78 +13,131 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see <http://www.gnu.org/licenses/>. +import functools import os +from ipaddress import ip_network from jinja2 import Environment from jinja2 import FileSystemLoader from vyos.defaults import directories from vyos.util import chmod, chown, makedir -# reuse the same Environment to improve performance -_templates_env = { - False: Environment(loader=FileSystemLoader(directories['templates'])), - True: Environment(loader=FileSystemLoader(directories['templates']), trim_blocks=True), -} -_templates_mem = { - False: {}, - True: {}, -} -def vyos_address_from_cidr(text): - """ Take an IPv4/IPv6 CIDR prefix and convert the network to an "address". - Example: - 192.0.2.0/24 -> 192.0.2.0, 2001:db8::/48 -> 2001:db8:: - """ - from ipaddress import ip_network - return ip_network(text).network_address +# Holds template filters registered via register_filter() +_FILTERS = {} -def vyos_netmask_from_cidr(text): - """ Take an IPv4/IPv6 CIDR prefix and convert the prefix length to a "subnet mask". - Example: - 192.0.2.0/24 -> 255.255.255.0, 2001:db8::/48 -> ffff:ffff:ffff:: - """ - from ipaddress import ip_network - return ip_network(text).netmask -def render(destination, template, content, trim_blocks=False, formater=None, permission=None, user=None, group=None): +# reuse Environments with identical trim_blocks setting to improve performance +@functools.lru_cache(maxsize=2) +def _get_environment(trim_blocks): + env = Environment( + # Don't check if template files were modified upon re-rendering + auto_reload=False, + # Cache up to this number of templates for quick re-rendering + cache_size=100, + loader=FileSystemLoader(directories["templates"]), + trim_blocks=trim_blocks, + ) + env.filters.update(_FILTERS) + return env + + +def register_filter(name, func=None): + """Register a function to be available as filter in templates under given name. + + It can also be used as a decorator, see below in this module for examples. + + :raise RuntimeError: + when trying to register a filter after a template has been rendered already + :raise ValueError: when trying to register a name which was taken already """ - render a template from the template directory, it will raise on any errors - destination: the file where the rendered template must be saved - template: the path to the template relative to the template folder - content: the dictionary to use to render the template - - This classes cache the renderer, so rendering the same file multiple time - does not cause as too much overhead. If use everywhere, it could be changed - and load the template from python environement variables from an import - python module generated when the debian package is build - (recovering the load time and overhead caused by having the file out of the code) + if func is None: + return functools.partial(register_filter, name) + if _get_environment.cache_info().currsize: + raise RuntimeError( + "Filters can only be registered before rendering the first template" + ) + if name in _FILTERS: + raise ValueError(f"A filter with name {name!r} was registered already") + _FILTERS[name] = func + return func + + +def render_to_string(template, content, trim_blocks=False, formater=None): + """Render a template from the template directory, raise on any errors. + + :param template: the path to the template relative to the template folder + :param content: the dictionary of variables to put into rendering context + :param trim_blocks: controls the trim_blocks jinja2 feature + :param formater: + if given, it has to be a callable the rendered string is passed through + + The parsed template files are cached, so rendering the same file multiple times + does not cause as too much overhead. + If used everywhere, it could be changed to load the template from Python + environment variables from an importable Python module generated when the Debian + package is build (recovering the load time and overhead caused by having the + file out of the code). """ + template = _get_environment(bool(trim_blocks)).get_template(template) + rendered = template.render(content) + if formater is not None: + rendered = formater(rendered) + return rendered + + +def render( + destination, + template, + content, + trim_blocks=False, + formater=None, + permission=None, + user=None, + group=None, +): + """Render a template from the template directory to a file, raise on any errors. - # Create the directory if it does not exists + :param destination: path to the file to save the rendered template in + :param permission: permission bitmask to set for the output file + :param user: user to own the output file + :param group: group to own the output file + + All other parameters are as for :func:`render_to_string`. + """ + # Create the directory if it does not exist folder = os.path.dirname(destination) makedir(folder, user, group) - # Setup a renderer for the given template - # This is cached and re-used for performance - if template not in _templates_mem[trim_blocks]: - _env = _templates_env[trim_blocks] - _env.filters['address_from_cidr'] = vyos_address_from_cidr - _env.filters['netmask_from_cidr'] = vyos_netmask_from_cidr - _templates_mem[trim_blocks][template] = _env.get_template(template) + # As we are opening the file with 'w', we are performing the rendering before + # calling open() to not accidentally erase the file if rendering fails + rendered = render_to_string(template, content, trim_blocks, formater) - template = _templates_mem[trim_blocks][template] + # Write to file + with open(destination, "w") as file: + chmod(file.fileno(), permission) + chown(file.fileno(), user, group) + file.write(rendered) - # As we are opening the file with 'w', we are performing the rendering - # before calling open() to not accidentally erase the file if the - # templating fails - content = template.render(content) - if formater: - content = formater(content) +################################## +# Custom template filters follow # +################################## - # Write client config file - with open(destination, 'w') as f: - f.write(content) - chmod(destination, permission) - chown(destination, user, group) +@register_filter("address_from_cidr") +def vyos_address_from_cidr(text): + """ Take an IPv4/IPv6 CIDR prefix and convert the network to an "address". + Example: + 192.0.2.0/24 -> 192.0.2.0, 2001:db8::/48 -> 2001:db8:: + """ + return ip_network(text).network_address + + +@register_filter("netmask_from_cidr") +def vyos_netmask_from_cidr(text): + """ Take an IPv4/IPv6 CIDR prefix and convert the prefix length to a "subnet mask". + Example: + 192.0.2.0/24 -> 255.255.255.0, 2001:db8::/48 -> ffff:ffff:ffff:: + """ + return ip_network(text).netmask diff --git a/python/vyos/util.py b/python/vyos/util.py index 924df6b3a..84aa16791 100644 --- a/python/vyos/util.py +++ b/python/vyos/util.py @@ -240,9 +240,10 @@ def chown(path, user, group): if user is None or group is None: return False - if not os.path.exists(path): + # path may also be an open file descriptor + if not isinstance(path, int) and not os.path.exists(path): return False - + uid = getpwnam(user).pw_uid gid = getgrnam(group).gr_gid os.chown(path, uid, gid) @@ -250,7 +251,8 @@ def chown(path, user, group): def chmod(path, bitmask): - if not os.path.exists(path): + # path may also be an open file descriptor + if not isinstance(path, int) and not os.path.exists(path): return if bitmask is None: return @@ -261,28 +263,25 @@ def chmod_600(path): """ make file only read/writable by owner """ from stat import S_IRUSR, S_IWUSR - if os.path.exists(path): - bitmask = S_IRUSR | S_IWUSR - os.chmod(path, bitmask) + bitmask = S_IRUSR | S_IWUSR + chmod(path, bitmask) def chmod_750(path): """ make file/directory only executable to user and group """ from stat import S_IRUSR, S_IWUSR, S_IXUSR, S_IRGRP, S_IXGRP - if os.path.exists(path): - bitmask = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP - os.chmod(path, bitmask) + bitmask = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP + chmod(path, bitmask) def chmod_755(path): """ make file executable by all """ from stat import S_IRUSR, S_IWUSR, S_IXUSR, S_IRGRP, S_IXGRP, S_IROTH, S_IXOTH - if os.path.exists(path): - bitmask = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | \ - S_IROTH | S_IXOTH - os.chmod(path, bitmask) + bitmask = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | \ + S_IROTH | S_IXOTH + chmod(path, bitmask) def makedir(path, user=None, group=None): @@ -652,3 +651,38 @@ def get_bridge_member_config(conf, br, intf): conf.set_level(old_level) return memberconf + +def check_kmod(k_mod): + """ Common utility function to load required kernel modules on demand """ + if isinstance(k_mod, str): + k_mod = k_mod.split() + for module in k_mod: + if not os.path.exists(f'/sys/module/{module}'): + if call(f'modprobe {module}') != 0: + raise ConfigError(f'Loading Kernel module {module} failed') + +def find_device_file(device): + """ Recurively search /dev for the given device file and return its full path. + If no device file was found 'None' is returned """ + from fnmatch import fnmatch + + for root, dirs, files in os.walk('/dev'): + for basename in files: + if fnmatch(basename, device): + return os.path.join(root, basename) + + return None + +def vyos_dict_search(path, dict): + """ Traverse Python dictionary (dict) delimited by dot (.). + Return value of key if found, None otherwise. + + This is faster implementation then jmespath.search('foo.bar', dict)""" + parts = path.split('.') + inside = parts[:-1] + if not inside: + return dict[path] + c = dict + for p in parts[:-1]: + c = c.get(p, {}) + return c.get(parts[-1], None) diff --git a/python/vyos/validate.py b/python/vyos/validate.py index 9072c5817..ceeb6888a 100644 --- a/python/vyos/validate.py +++ b/python/vyos/validate.py @@ -19,6 +19,7 @@ import netifaces import ipaddress from vyos.util import cmd +from vyos import xml # Important note when you are adding new validation functions: # @@ -278,7 +279,6 @@ def is_member(conf, interface, intftype=None): False -> interface type cannot have members """ ret_val = None - if intftype not in ['bonding', 'bridge', None]: raise ValueError(( f'unknown interface type "{intftype}" or it cannot ' @@ -291,14 +291,14 @@ def is_member(conf, interface, intftype=None): conf.set_level([]) for it in intftype: - base = 'interfaces ' + it + base = ['interfaces', it] for intf in conf.list_nodes(base): - memberintf = f'{base} {intf} member interface' - if conf.is_tag(memberintf): + memberintf = base + [intf, 'member', 'interface'] + if xml.is_tag(memberintf): if interface in conf.list_nodes(memberintf): ret_val = intf break - elif conf.is_leaf(memberintf): + elif xml.is_leaf(memberintf): if ( conf.exists(memberintf) and interface in conf.return_values(memberintf) ): ret_val = intf diff --git a/python/vyos/xml/__init__.py b/python/vyos/xml/__init__.py index 6e0e73b1b..0ef0c85ce 100644 --- a/python/vyos/xml/__init__.py +++ b/python/vyos/xml/__init__.py @@ -35,10 +35,29 @@ def load_configuration(cache=[]): return xml +# def is_multi(lpath): +# return load_configuration().is_multi(lpath) + + +def is_tag(lpath): + return load_configuration().is_tag(lpath) + + +def is_leaf(lpath, flat=True): + return load_configuration().is_leaf(lpath, flat) + + def defaults(lpath, flat=False): return load_configuration().defaults(lpath, flat) +def multi_to_list(lpath, conf): + return load_configuration().multi_to_list(lpath, conf) + + if __name__ == '__main__': print(defaults(['service'], flat=True)) print(defaults(['service'], flat=False)) + + print(is_tag(["system", "login", "user", "vyos", "authentication", "public-keys"])) + print(is_tag(['protocols', 'static', 'multicast', 'route', '0.0.0.0/0', 'next-hop'])) diff --git a/python/vyos/xml/definition.py b/python/vyos/xml/definition.py index 5421007e0..a25fc50c5 100644 --- a/python/vyos/xml/definition.py +++ b/python/vyos/xml/definition.py @@ -126,10 +126,12 @@ class XML(dict): elif word: if data_node != kw.plainNode or len(passed) == 1: self.options = [_ for _ in self.tree if _.startswith(word)] + self.options.sort() else: self.options = [] else: self.options = named_options + self.options.sort() self.plain = not is_dataNode @@ -143,6 +145,7 @@ class XML(dict): self.word = '' if self.tree.get(kw.node,'') not in (kw.tagNode, kw.leafNode): self.options = [_ for _ in self.tree if not kw.found(_)] + self.options.sort() def checks(self, cmd): # as we move thought the named node twice @@ -248,9 +251,12 @@ class XML(dict): def defaults(self, lpath, flat): d = self[kw.default] for k in lpath: - d = d[k] + d = d.get(k, {}) if not flat: + # _flatten will make this conversion + d = self.multi_to_list(lpath, d) + r = {} for k in d: under = k.replace('-','_') @@ -278,6 +284,23 @@ class XML(dict): return _flatten(lpath, len(lpath), d) + def multi_to_list(self, lpath, conf): + r = {} + for k in conf: + # key mangling could also be done here + # it would prevent two parsing of the config tree + # under = k.replace('-','_') + under = k + fpath = lpath + [k] + if isinstance(conf[k],dict): + r[under] = self.multi_to_list(fpath, conf[k]) + continue + value = conf[k] + if self.is_multi(fpath) and not isinstance(value, list): + value = [value] + r[under] = value + return r + # from functools import lru_cache # @lru_cache(maxsize=100) # XXX: need to use cachetool instead - for later @@ -300,16 +323,28 @@ class XML(dict): return tree def _get(self, lpath, tag, with_tag=True): - return self._tree(lpath + [tag], with_tag) + tree = self._tree(lpath, with_tag) + if tree is None: + return None + return tree.get(tag, None) def is_multi(self, lpath, with_tag=True): - return self._get(lpath, kw.multi, with_tag) is True + tree = self._get(lpath, kw.multi, with_tag) + if tree is None: + return None + return tree is True def is_tag(self, lpath, with_tag=True): - return self._get(lpath, kw.node, with_tag) == kw.tagNode + tree = self._get(lpath, kw.node, with_tag) + if tree is None: + return None + return tree == kw.tagNode def is_leaf(self, lpath, with_tag=True): - return self._get(lpath, kw.node, with_tag) == kw.leafNode + tree = self._get(lpath, kw.node, with_tag) + if tree is None: + return None + return tree == kw.leafNode def exists(self, lpath, with_tag=True): return self._get(lpath, kw.node, with_tag) is not None diff --git a/python/vyos/xml/test_xml.py b/python/vyos/xml/test_xml.py index ac0620d99..ff55151d2 100644 --- a/python/vyos/xml/test_xml.py +++ b/python/vyos/xml/test_xml.py @@ -33,7 +33,7 @@ class TestSearch(TestCase): last = self.xml.traverse("") self.assertEqual(last, '') self.assertEqual(self.xml.inside, []) - self.assertEqual(self.xml.options, ['protocols', 'service', 'system', 'firewall', 'interfaces', 'vpn', 'nat', 'vrf', 'high-availability']) + self.assertEqual(self.xml.options, ['firewall', 'high-availability', 'interfaces', 'nat', 'protocols', 'service', 'system', 'vpn', 'vrf']) self.assertEqual(self.xml.filling, False) self.assertEqual(self.xml.word, last) self.assertEqual(self.xml.check, False) |