diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/vyos/config.py | 26 | ||||
| -rw-r--r-- | python/vyos/configdict.py | 5 | ||||
| -rw-r--r-- | python/vyos/configverify.py | 78 | ||||
| -rw-r--r-- | python/vyos/frr.py | 288 | ||||
| -rw-r--r-- | python/vyos/ifconfig/interface.py | 39 | ||||
| -rw-r--r-- | python/vyos/ifconfig/loopback.py | 25 | ||||
| -rw-r--r-- | python/vyos/template.py | 29 | ||||
| -rw-r--r-- | python/vyos/util.py | 40 | ||||
| -rw-r--r-- | python/vyos/xml/__init__.py | 11 | ||||
| -rw-r--r-- | python/vyos/xml/definition.py | 23 |
10 files changed, 536 insertions, 28 deletions
diff --git a/python/vyos/config.py b/python/vyos/config.py index 56353c322..780b48a7b 100644 --- a/python/vyos/config.py +++ b/python/vyos/config.py @@ -71,7 +71,6 @@ import subprocess import vyos.util import vyos.configtree - class VyOSError(Exception): """ Raised on config access errors, most commonly if the type of a config tree node @@ -288,17 +287,26 @@ class Config(object): self.__session_env = save_env return(default) - def get_config_dict(self, path=[], effective=False, key_mangling=None): + def get_config_dict(self, path=[], effective=False, key_mangling=None, get_first_key=False): """ - Args: path (str list): Configuration tree path, can be empty - Returns: a dict representation of the config + Args: + path (str list): Configuration tree path, can be empty + effective=False: effective or session config + key_mangling=None: mangle dict keys according to regex and replacement + get_first_key=False: if k = path[:-1], return sub-dict d[k] instead of {k: d[k]} + + Returns: a dict representation of the config under path """ - res = self.show_config(self._make_path(path), effective=effective) - if res: - config_tree = vyos.configtree.ConfigTree(res) - config_dict = json.loads(config_tree.to_json()) + config_dict = {} + + if effective: + if self._running_config: + config_dict = json.loads((self._running_config).to_json()) else: - config_dict = {} + if self._session_config: + config_dict = json.loads((self._session_config).to_json()) + + config_dict = vyos.util.get_sub_dict(config_dict, self._make_path(path), get_first_key) if key_mangling: if not (isinstance(key_mangling, tuple) and \ diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py index ce086872e..0dc7578d8 100644 --- a/python/vyos/configdict.py +++ b/python/vyos/configdict.py @@ -22,7 +22,6 @@ from enum import Enum from copy import deepcopy from vyos import ConfigError -from vyos.ifconfig import Interface from vyos.validate import is_member from vyos.util import ifname_from_config @@ -97,6 +96,8 @@ def dict_merge(source, destination): for key, value in source.items(): if key not in tmp.keys(): tmp[key] = value + elif isinstance(source[key], dict): + tmp[key] = dict_merge(source[key], tmp[key]) return tmp @@ -214,6 +215,8 @@ def disable_state(conf, check=[3,5,7]): def intf_to_dict(conf, default): + from vyos.ifconfig import Interface + """ Common used function which will extract VLAN related information from config and represent the result as Python dictionary. diff --git a/python/vyos/configverify.py b/python/vyos/configverify.py new file mode 100644 index 000000000..32129a048 --- /dev/null +++ b/python/vyos/configverify.py @@ -0,0 +1,78 @@ +# Copyright 2020 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +# The sole purpose of this module is to hold common functions used in +# all kinds of implementations to verify the CLI configuration. +# It is started by migrating the interfaces to the new get_config_dict() +# approach which will lead to a lot of code that can be reused. + +# NOTE: imports should be as local as possible to the function which +# makes use of it! + +from vyos import ConfigError + +def verify_vrf(config): + """ + Common helper function used by interface implementations to perform + recurring validation of VRF configuration. + """ + from netifaces import interfaces + if 'vrf' in config.keys(): + if config['vrf'] not in interfaces(): + raise ConfigError('VRF "{vrf}" does not exist'.format(**config)) + + if 'is_bridge_member' in config.keys(): + raise ConfigError( + 'Interface "{ifname}" cannot be both a member of VRF "{vrf}" ' + 'and bridge "{is_bridge_member}"!'.format(**config)) + + +def verify_address(config): + """ + Common helper function used by interface implementations to + perform recurring validation of IP address assignmenr + when interface also is part of a bridge. + """ + if {'is_bridge_member', 'address'} <= set(config): + raise ConfigError( + f'Cannot assign address to interface "{ifname}" as it is a ' + f'member of bridge "{is_bridge_member}"!'.format(**config)) + + +def verify_bridge_delete(config): + """ + Common helper function used by interface implementations to + perform recurring validation of IP address assignmenr + when interface also is part of a bridge. + """ + if 'is_bridge_member' in config.keys(): + raise ConfigError( + 'Interface "{ifname}" cannot be deleted as it is a ' + 'member of bridge "{is_bridge_member}"!'.format(**config)) + + +def verify_source_interface(config): + """ + Common helper function used by interface implementations to + perform recurring validation of the existence of a source-interface + required by e.g. peth/MACvlan, MACsec ... + """ + from netifaces import interfaces + if not 'source_interface' in config.keys(): + raise ConfigError('Physical source-interface required for ' + 'interface "{ifname}"'.format(**config)) + if not config['source_interface'] in interfaces(): + raise ConfigError(f'Source interface {source_interface} does not ' + f'exist'.format(**config)) diff --git a/python/vyos/frr.py b/python/vyos/frr.py new file mode 100644 index 000000000..e39b6a914 --- /dev/null +++ b/python/vyos/frr.py @@ -0,0 +1,288 @@ +# Copyright 2020 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +r""" +A Library for interracting with the FRR daemon suite. +It supports simple configuration manipulation and loading using the official tools +supplied with FRR (vtysh and frr-reload) + +All configuration management and manipulation is done using strings and regex. + + +Example Usage +##### + +# Reading configuration from frr: +``` +>>> original_config = get_configuration() +>>> repr(original_config) +'!\nfrr version 7.3.1\nfrr defaults traditional\nhostname debian\n...... +``` + + +# Modify a configuration section: +``` +>>> new_bgp_section = 'router bgp 65000\n neighbor 192.0.2.1 remote-as 65000\n' +>>> modified_config = replace_section(original_config, new_bgp_section, replace_re=r'router bgp \d+') +>>> repr(modified_config) +'............router bgp 65000\n neighbor 192.0.2.1 remote-as 65000\n...........' +``` + +Remove a configuration section: +``` +>>> modified_config = remove_section(original_config, r'router ospf') +``` + +Test the new configuration: +``` +>>> try: +>>> mark_configuration(modified configuration) +>>> except ConfigurationNotValid as e: +>>> print('resulting configuration is not valid') +>>> sys.exit(1) +``` + +Apply the new configuration: +``` +>>> try: +>>> replace_configuration(modified_config) +>>> except CommitError as e: +>>> print('Exception while commiting the supplied configuration') +>>> print(e) +>>> exit(1) +``` +""" + +import tempfile +import re +from vyos import util + +_frr_daemons = ['zebra', 'bgpd', 'fabricd', 'isisd', 'ospf6d', 'ospfd', 'pbrd', + 'pimd', 'ripd', 'ripngd', 'sharpd', 'staticd', 'vrrpd', 'ldpd'] + +path_vtysh = '/usr/bin/vtysh' +path_frr_reload = '/usr/lib/frr/frr-reload.py' + + +class FrrError(Exception): + pass + + +class ConfigurationNotValid(FrrError): + """ + The configuratioin supplied to vtysh is not valid + """ + pass + + +class CommitError(FrrError): + """ + Commiting the supplied configuration failed to commit by a unknown reason + see commit error and/or run mark_configuration on the specified configuration + to se error generated + + used by: reload_configuration() + """ + pass + + +class ConfigSectionNotFound(FrrError): + """ + Removal of configuration failed because it is not existing in the supplied configuration + """ + pass + + +def get_configuration(daemon=None, marked=False): + """ Get current running FRR configuration + daemon: Collect only configuration for the specified FRR daemon, + supplying daemon=None retrieves the complete configuration + marked: Mark the configuration with "end" tags + + return: string containing the running configuration from frr + + """ + if daemon and daemon not in _frr_daemons: + raise ValueError(f'The specified daemon type is not supported {repr(daemon)}') + + cmd = f"{path_vtysh} -c 'show run'" + if daemon: + cmd += f' -d {daemon}' + + output, code = util.popen(cmd, stderr=util.STDOUT) + if code: + raise OSError(code, output) + + config = output.replace('\r', '') + # Remove first header lines from FRR config + config = config.split("\n", 3)[-1] + # Mark the configuration with end tags + if marked: + config = mark_configuration(config) + + return config + + +def mark_configuration(config): + """ Add end marks and Test the configuration for syntax faults + If the configuration is valid a marked version of the configuration is returned, + or else it failes with a ConfigurationNotValid Exception + + config: The configuration string to mark/test + return: The marked configuration from FRR + """ + output, code = util.popen(f"{path_vtysh} -m -f -", stderr=util.STDOUT, input=config) + + if code == 2: + raise ConfigurationNotValid(str(output)) + elif code: + raise OSError(code, output) + + config = output.replace('\r', '') + return config + + +def reload_configuration(config, daemon=None): + """ Execute frr-reload with the new configuration + This will try to reapply the supplied configuration inside FRR. + The configuration needs to be a complete configuration from the integrated config or + from a daemon. + + config: The configuration to apply + daemon: Apply the conigutaion to the specified FRR daemon, + supplying daemon=None applies to the integrated configuration + return: None + """ + if daemon and daemon not in _frr_daemons: + raise ValueError(f'The specified daemon type is not supported {repr(daemon)}') + + f = tempfile.NamedTemporaryFile('w') + f.write(config) + f.flush() + + cmd = f'{path_frr_reload} --reload' + if daemon: + cmd += f' --daemon {daemon}' + cmd += f' {f.name}' + + output, code = util.popen(cmd, stderr=util.STDOUT) + f.close() + if code == 1: + raise CommitError(f'Configuration FRR failed while commiting code: {repr(output)}') + elif code: + raise OSError(code, output) + + return output + + +def execute(command): + """ Run commands inside vtysh + command: str containing commands to execute inside a vtysh session + """ + if not isinstance(command, str): + raise ValueError(f'command needs to be a string: {repr(command)}') + + cmd = f"{path_vtysh} -c '{command}'" + + output, code = util.popen(cmd, stderr=util.STDOUT) + if code: + raise OSError(code, output) + + config = output.replace('\r', '') + return config + + +def configure(lines, daemon=False): + """ run commands inside config mode vtysh + lines: list or str conaining commands to execute inside a configure session + only one command executed on each configure() + Executing commands inside a subcontext uses the list to describe the context + ex: ['router bgp 6500', 'neighbor 192.0.2.1 remote-as 65000'] + return: None + """ + if isinstance(lines, str): + lines = [lines] + elif not isinstance(lines, list): + raise ValueError('lines needs to be string or list of commands') + + if daemon and daemon not in _frr_daemons: + raise ValueError(f'The specified daemon type is not supported {repr(daemon)}') + + cmd = f'{path_vtysh}' + if daemon: + cmd += f' -d {daemon}' + + cmd += " -c 'configure terminal'" + for x in lines: + cmd += f" -c '{x}'" + + output, code = util.popen(cmd, stderr=util.STDOUT) + if code == 1: + raise ConfigurationNotValid(f'Configuration FRR failed: {repr(output)}') + elif code: + raise OSError(code, output) + + config = output.replace('\r', '') + return config + + +def _replace_section(config, replacement, replace_re, before_re): + r"""Replace a section of FRR config + config: full original configuration + replacement: replacement configuration section + replace_re: The regex to replace + example: ^router bgp \d+$.?*^!$ + this will replace everything between ^router bgp X$ and ^!$ + before_re: When replace_re is not existant, the config will be added before this tag + example: ^line vty$ + + return: modified configuration as a text file + """ + # Check if block is configured, remove the existing instance else add a new one + if re.findall(replace_re, config, flags=re.MULTILINE | re.DOTALL): + # Section is in the configration, replace it + return re.sub(replace_re, replacement, config, count=1, + flags=re.MULTILINE | re.DOTALL) + if before_re: + if not re.findall(before_re, config, flags=re.MULTILINE | re.DOTALL): + raise ConfigSectionNotFound(f"Config section {before_re} not found in config") + + # If no section is in the configuration, add it before the line vty line + return re.sub(before_re, rf'{replacement}\n\g<1>', config, count=1, + flags=re.MULTILINE | re.DOTALL) + + raise ConfigSectionNotFound(f"Config section {replacement} not found in config") + + +def replace_section(config, replacement, from_re, to_re=r'!', before_re=r'line vty'): + r"""Replace a section of FRR config + config: full original configuration + replacement: replacement configuration section + from_re: Regex for the start of section matching + example: 'router bgp \d+' + to_re: Regex for stop of section matching + default: '!' + example: '!' or 'end' + before_re: When from_re/to_re does not return a match, the config will + be added before this tag + default: ^line vty$ + + startline and endline tags will be automatically added to the resulting from_re/to_re and before_re regex'es + """ + return _replace_section(config, replacement, replace_re=rf'^{from_re}$.*?^{to_re}$', before_re=rf'^{before_re}$') + + +def remove_section(config, from_re, to_re='!'): + return _replace_section(config, '', replace_re=rf'^{from_re}$.*?^{to_re}$', before_re=None) diff --git a/python/vyos/ifconfig/interface.py b/python/vyos/ifconfig/interface.py index 2c2396440..1819ffc82 100644 --- a/python/vyos/ifconfig/interface.py +++ b/python/vyos/ifconfig/interface.py @@ -27,6 +27,7 @@ from netifaces import AF_INET from netifaces import AF_INET6 from vyos import ConfigError +from vyos.configdict import list_diff from vyos.util import mac2eui64 from vyos.validate import is_ipv4 from vyos.validate import is_ipv6 @@ -757,3 +758,41 @@ class Interface(Control): # TODO: port config (STP) return True + + def update(self, config): + """ General helper function which works on a dictionary retrived by + get_config_dict(). It's main intention is to consolidate the scattered + interface setup code and provide a single point of entry when workin + on any interface. """ + + # Update interface description + self.set_alias(config.get('description', None)) + + # Configure assigned interface IP addresses. No longer + # configured addresses will be removed first + new_addr = config.get('address', []) + + # XXX workaround for T2636, convert IP address string to a list + # with one element + if isinstance(new_addr, str): + new_addr = [new_addr] + + # determine IP addresses which are assigned to the interface and build a + # list of addresses which are no longer in the dict so they can be removed + cur_addr = self.get_addr() + for addr in list_diff(cur_addr, new_addr): + self.del_addr(addr) + + for addr in new_addr: + self.add_addr(addr) + + # There are some items in the configuration which can only be applied + # if this instance is not bound to a bridge. This should be checked + # by the caller but better save then sorry! + if not config.get('is_bridge_member', False): + # Bind interface instance into VRF + self.set_vrf(config.get('vrf', '')) + + # Interface administrative state + state = 'down' if 'disable' in config.keys() else 'up' + self.set_admin_state(state) diff --git a/python/vyos/ifconfig/loopback.py b/python/vyos/ifconfig/loopback.py index 8e4438662..7ebd13b54 100644 --- a/python/vyos/ifconfig/loopback.py +++ b/python/vyos/ifconfig/loopback.py @@ -23,7 +23,7 @@ class LoopbackIf(Interface): The loopback device is a special, virtual network interface that your router uses to communicate with itself. """ - + _persistent_addresses = ['127.0.0.1/8', '::1/128'] default = { 'type': 'loopback', } @@ -49,10 +49,31 @@ class LoopbackIf(Interface): """ # remove all assigned IP addresses from interface for addr in self.get_addr(): - if addr in ["127.0.0.1/8", "::1/128"]: + if addr in self._persistent_addresses: # Do not allow deletion of the default loopback addresses as # this will cause weird system behavior like snmp/ssh no longer # operating as expected, see https://phabricator.vyos.net/T2034. continue self.del_addr(addr) + + def update(self, config): + """ General helper function which works on a dictionary retrived by + get_config_dict(). It's main intention is to consolidate the scattered + interface setup code and provide a single point of entry when workin + on any interface. """ + + addr = config.get('address', []) + # XXX workaround for T2636, convert IP address string to a list + # with one element + if isinstance(addr, str): + addr = [addr] + + # We must ensure that the loopback addresses are never deleted from the system + addr += self._persistent_addresses + + # Update IP address entry in our dictionary + config.update({'address' : addr}) + + # now call the regular function from within our base class + super().update(config) diff --git a/python/vyos/template.py b/python/vyos/template.py index e4b253ed3..d9b0c749d 100644 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -17,11 +17,9 @@ import os from jinja2 import Environment from jinja2 import FileSystemLoader - from vyos.defaults import directories from vyos.util import chmod, chown, makedir - # reuse the same Environment to improve performance _templates_env = { False: Environment(loader=FileSystemLoader(directories['templates'])), @@ -32,6 +30,21 @@ _templates_mem = { True: {}, } +def vyos_address_from_cidr(text): + """ Take an IPv4/IPv6 CIDR prefix and convert the network to an "address". + Example: + 192.0.2.0/24 -> 192.0.2.0, 2001:db8::/48 -> 2001:db8:: + """ + from ipaddress import ip_network + return ip_network(text).network_address + +def vyos_netmask_from_cidr(text): + """ Take an IPv4/IPv6 CIDR prefix and convert the prefix length to a "subnet mask". + Example: + 192.0.2.0/24 -> 255.255.255.0, 2001:db8::/48 -> ffff:ffff:ffff:: + """ + from ipaddress import ip_network + return ip_network(text).netmask def render(destination, template, content, trim_blocks=False, formater=None, permission=None, user=None, group=None): """ @@ -42,8 +55,8 @@ def render(destination, template, content, trim_blocks=False, formater=None, per This classes cache the renderer, so rendering the same file multiple time does not cause as too much overhead. If use everywhere, it could be changed - and load the template from python environement variables from an import - python module generated when the debian package is build + and load the template from python environement variables from an import + python module generated when the debian package is build (recovering the load time and overhead caused by having the file out of the code) """ @@ -54,11 +67,15 @@ def render(destination, template, content, trim_blocks=False, formater=None, per # Setup a renderer for the given template # This is cached and re-used for performance if template not in _templates_mem[trim_blocks]: - _templates_mem[trim_blocks][template] = _templates_env[trim_blocks].get_template(template) + _env = _templates_env[trim_blocks] + _env.filters['address_from_cidr'] = vyos_address_from_cidr + _env.filters['netmask_from_cidr'] = vyos_netmask_from_cidr + _templates_mem[trim_blocks][template] = _env.get_template(template) + template = _templates_mem[trim_blocks][template] # As we are opening the file with 'w', we are performing the rendering - # before calling open() to not accidentally erase the file if the + # before calling open() to not accidentally erase the file if the # templating fails content = template.render(content) diff --git a/python/vyos/util.py b/python/vyos/util.py index 0ddc14963..924df6b3a 100644 --- a/python/vyos/util.py +++ b/python/vyos/util.py @@ -364,6 +364,46 @@ def mangle_dict_keys(data, regex, replacement): return new_dict +def _get_sub_dict(d, lpath): + k = lpath[0] + if k not in d.keys(): + return {} + c = {k: d[k]} + lpath = lpath[1:] + if not lpath: + return c + elif not isinstance(c[k], dict): + return {} + return _get_sub_dict(c[k], lpath) + +def get_sub_dict(source, lpath, get_first_key=False): + """ Returns the sub-dict of a nested dict, defined by path of keys. + + Args: + source (dict): Source dict to extract from + lpath (list[str]): sequence of keys + + Returns: source, if lpath is empty, else + {key : source[..]..[key]} for key the last element of lpath, if exists + {} otherwise + """ + if not isinstance(source, dict): + raise TypeError("source must be of type dict") + if not isinstance(lpath, list): + raise TypeError("path must be of type list") + if not lpath: + return source + + ret = _get_sub_dict(source, lpath) + + if get_first_key and lpath and ret: + tmp = next(iter(ret.values())) + if not isinstance(tmp, dict): + raise TypeError("Data under node is not of type dict") + ret = tmp + + return ret + def process_running(pid_file): """ Checks if a process with PID in pid_file is running """ from psutil import pid_exists diff --git a/python/vyos/xml/__init__.py b/python/vyos/xml/__init__.py index 52f5bfb38..6e0e73b1b 100644 --- a/python/vyos/xml/__init__.py +++ b/python/vyos/xml/__init__.py @@ -9,7 +9,7 @@ # See the GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License along with this library; -# if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA from vyos.xml import definition @@ -35,5 +35,10 @@ def load_configuration(cache=[]): return xml -def defaults(lpath): - return load_configuration().defaults(lpath) +def defaults(lpath, flat=False): + return load_configuration().defaults(lpath, flat) + + +if __name__ == '__main__': + print(defaults(['service'], flat=True)) + print(defaults(['service'], flat=False)) diff --git a/python/vyos/xml/definition.py b/python/vyos/xml/definition.py index a66170a18..5421007e0 100644 --- a/python/vyos/xml/definition.py +++ b/python/vyos/xml/definition.py @@ -11,7 +11,6 @@ # You should have received a copy of the GNU Lesser General Public License along with this library; # if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - from vyos.xml import kw # As we index by key, the name is first and then the data: @@ -246,28 +245,38 @@ class XML(dict): # @lru_cache(maxsize=100) # XXX: need to use cachetool instead - for later - def defaults(self, lpath): + def defaults(self, lpath, flat): d = self[kw.default] for k in lpath: d = d[k] - r = {} - def _flatten(inside, index, d, r): + if not flat: + r = {} + for k in d: + under = k.replace('-','_') + if isinstance(d[k],dict): + r[under] = self.defaults(lpath + [k], flat) + continue + r[under] = d[k] + return r + + def _flatten(inside, index, d): + r = {} local = inside[index:] prefix = '_'.join(_.replace('-','_') for _ in local) + '_' if local else '' for k in d: under = prefix + k.replace('-','_') level = inside + [k] if isinstance(d[k],dict): - _flatten(level, index, d[k], r) + r.update(_flatten(level, index, d[k])) continue if self.is_multi(level, with_tag=False): r[under] = [_.strip() for _ in d[k].split(',')] continue r[under] = d[k] + return r - _flatten(lpath, len(lpath), d, r) - return r + return _flatten(lpath, len(lpath), d) # from functools import lru_cache # @lru_cache(maxsize=100) |