diff options
| author | Christian Breunig <christian@breunig.cc> | 2023-08-09 09:45:40 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-08-09 09:45:40 +0200 |
| commit | 98ed11a7d6e35de49f57406a0dd9d4759541f2d4 (patch) | |
| tree | 3eb831987a33c1d86acd6668c034ee25188d21ef | |
| parent | 0e92ee262d8ec6ee88d7331f2cbffe8b6b689437 (diff) | |
| parent | 4c5afc9bd6d5acd6089d2b78c03daa65529eaec9 (diff) | |
| download | vyos-1x-98ed11a7d6e35de49f57406a0dd9d4759541f2d4.tar.gz vyos-1x-98ed11a7d6e35de49f57406a0dd9d4759541f2d4.zip | |
Merge pull request #2136 from jestabro/with-defaults
T5319: remove workarounds for incorrect defaults in config-mode scripts
56 files changed, 350 insertions, 1104 deletions
diff --git a/python/vyos/config.py b/python/vyos/config.py index 179f60c43..6fececd76 100644 --- a/python/vyos/config.py +++ b/python/vyos/config.py @@ -66,17 +66,31 @@ In operational mode, all functions return values from the running config. import re import json from copy import deepcopy +from typing import Union import vyos.configtree -from vyos.xml_ref import multi_to_list, from_source -from vyos.xml_ref import merge_defaults, relative_defaults -from vyos.utils.dict import get_sub_dict, mangle_dict_keys -from vyos.configsource import ConfigSource, ConfigSourceSession +from vyos.xml_ref import multi_to_list +from vyos.xml_ref import from_source +from vyos.xml_ref import ext_dict_merge +from vyos.xml_ref import relative_defaults +from vyos.utils.dict import get_sub_dict +from vyos.utils.dict import mangle_dict_keys +from vyos.configsource import ConfigSource +from vyos.configsource import ConfigSourceSession class ConfigDict(dict): _from_defaults = {} - def from_defaults(self, path: list[str]): + _dict_kwargs = {} + def from_defaults(self, path: list[str]) -> bool: return from_source(self._from_defaults, path) + @property + def kwargs(self) -> dict: + return self._dict_kwargs + +def config_dict_merge(src: dict, dest: Union[dict, ConfigDict]) -> ConfigDict: + if not isinstance(dest, ConfigDict): + dest = ConfigDict(dest) + return ext_dict_merge(src, dest) class Config(object): """ @@ -229,6 +243,13 @@ class Config(object): return config_dict + def verify_mangling(self, key_mangling): + if not (isinstance(key_mangling, tuple) and \ + (len(key_mangling) == 2) and \ + isinstance(key_mangling[0], str) and \ + isinstance(key_mangling[1], str)): + raise ValueError("key_mangling must be a tuple of two strings") + def get_config_dict(self, path=[], effective=False, key_mangling=None, get_first_key=False, no_multi_convert=False, no_tag_node_value_mangle=False, @@ -243,44 +264,37 @@ class Config(object): Returns: a dict representation of the config under path """ + kwargs = locals().copy() + del kwargs['self'] + del kwargs['no_multi_convert'] + del kwargs['with_defaults'] + del kwargs['with_recursive_defaults'] + lpath = self._make_path(path) root_dict = self.get_cached_root_dict(effective) conf_dict = get_sub_dict(root_dict, lpath, get_first_key=get_first_key) - if key_mangling is None and no_multi_convert and not (with_defaults or with_recursive_defaults): - return deepcopy(conf_dict) - rpath = lpath if get_first_key else lpath[:-1] if not no_multi_convert: conf_dict = multi_to_list(rpath, conf_dict) + if key_mangling is not None: + self.verify_mangling(key_mangling) + conf_dict = mangle_dict_keys(conf_dict, + key_mangling[0], key_mangling[1], + abs_path=rpath, + no_tag_node_value_mangle=no_tag_node_value_mangle) + if with_defaults or with_recursive_defaults: + defaults = self.get_config_defaults(**kwargs, + recursive=with_recursive_defaults) + conf_dict = config_dict_merge(defaults, conf_dict) + else: conf_dict = ConfigDict(conf_dict) - conf_dict = merge_defaults(lpath, conf_dict, - get_first_key=get_first_key, - recursive=with_recursive_defaults) - if key_mangling is None: - return conf_dict - - if not (isinstance(key_mangling, tuple) and \ - (len(key_mangling) == 2) and \ - isinstance(key_mangling[0], str) and \ - isinstance(key_mangling[1], str)): - raise ValueError("key_mangling must be a tuple of two strings") - - def mangle(obj): - return mangle_dict_keys(obj, key_mangling[0], key_mangling[1], - abs_path=rpath, - no_tag_node_value_mangle=no_tag_node_value_mangle) - - if isinstance(conf_dict, ConfigDict): - from_defaults = mangle(conf_dict._from_defaults) - conf_dict = mangle(conf_dict) - conf_dict._from_defaults = from_defaults - else: - conf_dict = mangle(conf_dict) + # save optional args for a call to get_config_defaults + setattr(conf_dict, '_dict_kwargs', kwargs) return conf_dict @@ -294,21 +308,29 @@ class Config(object): defaults = relative_defaults(lpath, conf_dict, get_first_key=get_first_key, recursive=recursive) - if key_mangling is None: - return defaults rpath = lpath if get_first_key else lpath[:-1] - if not (isinstance(key_mangling, tuple) and \ - (len(key_mangling) == 2) and \ - isinstance(key_mangling[0], str) and \ - isinstance(key_mangling[1], str)): - raise ValueError("key_mangling must be a tuple of two strings") - - defaults = mangle_dict_keys(defaults, key_mangling[0], key_mangling[1], abs_path=rpath, no_tag_node_value_mangle=no_tag_node_value_mangle) + if key_mangling is not None: + self.verify_mangling(key_mangling) + defaults = mangle_dict_keys(defaults, + key_mangling[0], key_mangling[1], + abs_path=rpath, + no_tag_node_value_mangle=no_tag_node_value_mangle) return defaults + def merge_defaults(self, config_dict: ConfigDict, recursive=False): + if not isinstance(config_dict, ConfigDict): + raise TypeError('argument is not of type ConfigDict') + if not config_dict.kwargs: + raise ValueError('argument missing metadata') + + args = config_dict.kwargs + d = self.get_config_defaults(**args, recursive=recursive) + config_dict = config_dict_merge(d, config_dict) + return config_dict + def is_multi(self, path): """ Args: diff --git a/python/vyos/xml_ref/__init__.py b/python/vyos/xml_ref/__init__.py index ad2130dca..02bbaffd8 100644 --- a/python/vyos/xml_ref/__init__.py +++ b/python/vyos/xml_ref/__init__.py @@ -13,8 +13,12 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see <http://www.gnu.org/licenses/>. +from typing import Optional, Union, TYPE_CHECKING from vyos.xml_ref import definition +if TYPE_CHECKING: + from vyos.config import ConfigDict + def load_reference(cache=[]): if cache: return cache[0] @@ -48,12 +52,12 @@ def is_leaf(path: list) -> bool: def cli_defined(path: list, node: str, non_local=False) -> bool: return load_reference().cli_defined(path, node, non_local=non_local) -def from_source(d: dict, path: list) -> bool: - return load_reference().from_source(d, path) - def component_version() -> dict: return load_reference().component_version() +def default_value(path: list) -> Optional[Union[str, list]]: + return load_reference().default_value(path) + def multi_to_list(rpath: list, conf: dict) -> dict: return load_reference().multi_to_list(rpath, conf) @@ -68,8 +72,8 @@ def relative_defaults(rpath: list, conf: dict, get_first_key=False, get_first_key=get_first_key, recursive=recursive) -def merge_defaults(path: list, conf: dict, get_first_key=False, - recursive=False) -> dict: - return load_reference().merge_defaults(path, conf, - get_first_key=get_first_key, - recursive=recursive) +def from_source(d: dict, path: list) -> bool: + return definition.from_source(d, path) + +def ext_dict_merge(source: dict, destination: Union[dict, 'ConfigDict']): + return definition.ext_dict_merge(source, destination) diff --git a/python/vyos/xml_ref/definition.py b/python/vyos/xml_ref/definition.py index d95d580e2..38e07f0a7 100644 --- a/python/vyos/xml_ref/definition.py +++ b/python/vyos/xml_ref/definition.py @@ -20,6 +20,45 @@ from typing import Optional, Union, Any, TYPE_CHECKING if TYPE_CHECKING: from vyos.config import ConfigDict +def set_source_recursive(o: Union[dict, str, list], b: bool): + d = {} + if not isinstance(o, dict): + d = {'_source': b} + else: + for k, v in o.items(): + d[k] = set_source_recursive(v, b) + d |= {'_source': b} + return d + +def source_dict_merge(src: dict, dest: dict): + from copy import deepcopy + dst = deepcopy(dest) + from_src = {} + + for key, value in src.items(): + if key not in dst: + dst[key] = value + from_src[key] = set_source_recursive(value, True) + elif isinstance(src[key], dict): + dst[key], f = source_dict_merge(src[key], dst[key]) + f |= {'_source': False} + from_src[key] = f + + return dst, from_src + +def ext_dict_merge(src: dict, dest: Union[dict, 'ConfigDict']): + d, f = source_dict_merge(src, dest) + if hasattr(d, '_from_defaults'): + setattr(d, '_from_defaults', f) + return d + +def from_source(d: dict, path: list) -> bool: + for key in path: + d = d[key] if key in d else {} + if not d or not isinstance(d, dict): + return False + return d.get('_source', False) + class Xml: def __init__(self): self.ref = {} @@ -153,6 +192,15 @@ class Xml: return default.split() return default + def default_value(self, path: list) -> Optional[Union[str, list]]: + d = self._get_ref_path(path) + default = self._get_default_value(d) + if default is None: + return None + if self._is_multi_node(d) or self._is_tag_node(d): + return default.split() + return default + def get_defaults(self, path: list, get_first_key=False, recursive=False) -> dict: """Return dict containing default values below path @@ -212,43 +260,6 @@ class Xml: return False return True - def _set_source_recursive(self, o: Union[dict, str, list], b: bool): - d = {} - if not isinstance(o, dict): - d = {'_source': b} - else: - for k, v in o.items(): - d[k] = self._set_source_recursive(v, b) - d |= {'_source': b} - return d - - # use local copy of function in module configdict, to avoid circular - # import - # - # extend dict_merge to keep track of keys only in source - def _dict_merge(self, source, destination): - from copy import deepcopy - dest = deepcopy(destination) - from_source = {} - - for key, value in source.items(): - if key not in dest: - dest[key] = value - from_source[key] = self._set_source_recursive(value, True) - elif isinstance(source[key], dict): - dest[key], f = self._dict_merge(source[key], dest[key]) - f |= {'_source': False} - from_source[key] = f - - return dest, from_source - - def from_source(self, d: dict, path: list) -> bool: - for key in path: - d = d[key] if key in d else {} - if not d or not isinstance(d, dict): - return False - return d.get('_source', False) - def _relative_defaults(self, rpath: list, conf: dict, recursive=False) -> dict: res: dict = {} res = self.get_defaults(rpath, recursive=recursive, @@ -289,17 +300,3 @@ class Xml: res = {} return res - - def merge_defaults(self, path: list, conf: Union[dict, 'ConfigDict'], - get_first_key=False, recursive=False) -> dict: - """Return config dict with defaults non-destructively merged - - This merges non-recursive defaults relative to the config dict. - """ - d = self.relative_defaults(path, conf, get_first_key=get_first_key, - recursive=recursive) - d, f = self._dict_merge(d, conf) - d = type(conf)(d) - if hasattr(d, '_from_defaults'): - setattr(d, '_from_defaults', f) - return d diff --git a/src/conf_mode/conntrack.py b/src/conf_mode/conntrack.py index 2a77540f7..9c43640a9 100755 --- a/src/conf_mode/conntrack.py +++ b/src/conf_mode/conntrack.py @@ -20,7 +20,6 @@ import re from sys import exit from vyos.config import Config -from vyos.configdict import dict_merge from vyos.firewall import find_nftables_rule from vyos.firewall import remove_nftables_rule from vyos.utils.process import process_named_running @@ -28,7 +27,6 @@ from vyos.utils.dict import dict_search from vyos.utils.process import cmd from vyos.utils.process import run from vyos.template import render -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -77,16 +75,8 @@ def get_config(config=None): base = ['system', 'conntrack'] conntrack = conf.get_config_dict(base, key_mangling=('-', '_'), - get_first_key=True) - - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - # XXX: T2665: we can not safely rely on the defaults() when there are - # tagNodes in place, it is better to blend in the defaults manually. - if 'timeout' in default_values and 'custom' in default_values['timeout']: - del default_values['timeout']['custom'] - conntrack = dict_merge(default_values, conntrack) + get_first_key=True, + with_recursive_defaults=True) return conntrack diff --git a/src/conf_mode/conntrack_sync.py b/src/conf_mode/conntrack_sync.py index 6a4d102f7..4fb2ce27f 100755 --- a/src/conf_mode/conntrack_sync.py +++ b/src/conf_mode/conntrack_sync.py @@ -18,7 +18,6 @@ import os from sys import exit from vyos.config import Config -from vyos.configdict import dict_merge from vyos.configverify import verify_interface_exists from vyos.utils.dict import dict_search from vyos.utils.process import process_named_running @@ -28,7 +27,6 @@ from vyos.utils.process import run from vyos.template import render from vyos.template import get_ipv4 from vyos.utils.network import is_addr_assigned -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -50,11 +48,7 @@ def get_config(config=None): return None conntrack = conf.get_config_dict(base, key_mangling=('-', '_'), - get_first_key=True) - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - conntrack = dict_merge(default_values, conntrack) + get_first_key=True, with_defaults=True) conntrack['hash_size'] = read_file('/sys/module/nf_conntrack/parameters/hashsize') conntrack['table_size'] = read_file('/proc/sys/net/netfilter/nf_conntrack_max') diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py index 3378aac63..ed7cc809c 100755 --- a/src/conf_mode/container.py +++ b/src/conf_mode/container.py @@ -37,7 +37,7 @@ from vyos.template import inc_ip from vyos.template import is_ipv4 from vyos.template import is_ipv6 from vyos.template import render -from vyos.xml import defaults +from vyos.xml_ref import default_value from vyos import ConfigError from vyos import airbag airbag.enable() @@ -66,58 +66,26 @@ def get_config(config=None): base = ['container'] container = conf.get_config_dict(base, key_mangling=('-', '_'), - get_first_key=True, no_tag_node_value_mangle=True) - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - # container base default values can not be merged here - remove and add them later - if 'name' in default_values: - del default_values['name'] - # registry will be handled below - if 'registry' in default_values: - del default_values['registry'] - container = dict_merge(default_values, container) - - # Merge per-container default values - if 'name' in container: - default_values = defaults(base + ['name']) - if 'port' in default_values: - del default_values['port'] - if 'volume' in default_values: - del default_values['volume'] - for name in container['name']: - container['name'][name] = dict_merge(default_values, container['name'][name]) - - # T5047: Any container related configuration changed? We only - # wan't to restart the required containers and not all of them ... - tmp = is_node_changed(conf, base + ['name', name]) - if tmp: - if 'container_restart' not in container: - container['container_restart'] = [name] - else: - container['container_restart'].append(name) - - # XXX: T2665: we can not safely rely on the defaults() when there are - # tagNodes in place, it is better to blend in the defaults manually. - if 'port' in container['name'][name]: - for port in container['name'][name]['port']: - default_values_port = defaults(base + ['name', 'port']) - container['name'][name]['port'][port] = dict_merge( - default_values_port, container['name'][name]['port'][port]) - # XXX: T2665: we can not safely rely on the defaults() when there are - # tagNodes in place, it is better to blend in the defaults manually. - if 'volume' in container['name'][name]: - for volume in container['name'][name]['volume']: - default_values_volume = defaults(base + ['name', 'volume']) - container['name'][name]['volume'][volume] = dict_merge( - default_values_volume, container['name'][name]['volume'][volume]) + no_tag_node_value_mangle=True, + get_first_key=True, + with_recursive_defaults=True) + + for name in container.get('name', []): + # T5047: Any container related configuration changed? We only + # wan't to restart the required containers and not all of them ... + tmp = is_node_changed(conf, base + ['name', name]) + if tmp: + if 'container_restart' not in container: + container['container_restart'] = [name] + else: + container['container_restart'].append(name) # registry is a tagNode with default values - merge the list from # default_values['registry'] into the tagNode variables if 'registry' not in container: container.update({'registry' : {}}) - default_values = defaults(base) - for registry in default_values['registry'].split(): + default_values = default_value(base + ['registry']) + for registry in default_values: tmp = {registry : {}} container['registry'] = dict_merge(tmp, container['registry']) diff --git a/src/conf_mode/dhcp_relay.py b/src/conf_mode/dhcp_relay.py index fd39bd9fe..37d708847 100755 --- a/src/conf_mode/dhcp_relay.py +++ b/src/conf_mode/dhcp_relay.py @@ -20,12 +20,10 @@ from sys import exit from vyos.base import Warning from vyos.config import Config -from vyos.configdict import dict_merge from vyos.template import render from vyos.base import Warning from vyos.utils.process import call from vyos.utils.dict import dict_search -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -41,11 +39,9 @@ def get_config(config=None): if not conf.exists(base): return None - relay = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - relay = dict_merge(default_values, relay) + relay = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True) return relay diff --git a/src/conf_mode/dhcp_server.py b/src/conf_mode/dhcp_server.py index 3ea708902..280057f04 100755 --- a/src/conf_mode/dhcp_server.py +++ b/src/conf_mode/dhcp_server.py @@ -23,14 +23,12 @@ from netaddr import IPRange from sys import exit from vyos.config import Config -from vyos.configdict import dict_merge from vyos.template import render from vyos.utils.dict import dict_search from vyos.utils.process import call from vyos.utils.process import run from vyos.utils.network import is_subnet_connected from vyos.utils.network import is_addr_assigned -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -109,19 +107,15 @@ def get_config(config=None): if not conf.exists(base): return None - dhcp = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) - # T2665: defaults include lease time per TAG node which need to be added to - # individual subnet definitions - default_values = defaults(base + ['shared-network-name', 'subnet']) + dhcp = conf.get_config_dict(base, key_mangling=('-', '_'), + no_tag_node_value_mangle=True, + get_first_key=True, + with_recursive_defaults=True) if 'shared_network_name' in dhcp: for network, network_config in dhcp['shared_network_name'].items(): if 'subnet' in network_config: for subnet, subnet_config in network_config['subnet'].items(): - if 'lease' not in subnet_config: - dhcp['shared_network_name'][network]['subnet'][subnet] = dict_merge( - default_values, dhcp['shared_network_name'][network]['subnet'][subnet]) - # If exclude IP addresses are defined we need to slice them out of # the defined ranges if {'exclude', 'range'} <= set(subnet_config): diff --git a/src/conf_mode/dhcpv6_relay.py b/src/conf_mode/dhcpv6_relay.py index d912611b3..6537ca3c2 100755 --- a/src/conf_mode/dhcpv6_relay.py +++ b/src/conf_mode/dhcpv6_relay.py @@ -19,14 +19,11 @@ import os from sys import exit from vyos.config import Config -from vyos.configdict import dict_merge from vyos.ifconfig import Interface from vyos.template import render from vyos.template import is_ipv6 from vyos.utils.process import call -from vyos.utils.dict import dict_search from vyos.utils.network import is_ipv6_link_local -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -42,11 +39,9 @@ def get_config(config=None): if not conf.exists(base): return None - relay = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - relay = dict_merge(default_values, relay) + relay = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True) return relay diff --git a/src/conf_mode/dns_dynamic.py b/src/conf_mode/dns_dynamic.py index 97d46148a..ab80defe8 100755 --- a/src/conf_mode/dns_dynamic.py +++ b/src/conf_mode/dns_dynamic.py @@ -19,10 +19,8 @@ import os from sys import exit from vyos.config import Config -from vyos.configdict import dict_merge from vyos.template import render from vyos.utils.process import call -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -50,8 +48,9 @@ def get_config(config=None): return None dyndns = conf.get_config_dict(base_level, key_mangling=('-', '_'), - get_first_key=True, no_tag_node_value_mangle=True, - with_defaults=True, with_recursive_defaults=True) + no_tag_node_value_mangle=True, + get_first_key=True, + with_recursive_defaults=True) dyndns['config_file'] = config_file return dyndns diff --git a/src/conf_mode/dns_forwarding.py b/src/conf_mode/dns_forwarding.py index 2d98bffe3..c186f47af 100755 --- a/src/conf_mode/dns_forwarding.py +++ b/src/conf_mode/dns_forwarding.py @@ -21,14 +21,12 @@ from sys import exit from glob import glob from vyos.config import Config -from vyos.configdict import dict_merge from vyos.hostsd_client import Client as hostsd_client from vyos.template import render from vyos.template import bracketize_ipv6 from vyos.utils.process import call from vyos.utils.permission import chown from vyos.utils.dict import dict_search -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag @@ -52,31 +50,10 @@ def get_config(config=None): if not conf.exists(base): return None - dns = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrieved. - default_values = defaults(base) - # T2665 due to how defaults under tag nodes work, we must clear these out before we merge - del default_values['authoritative_domain'] - del default_values['name_server'] - del default_values['domain']['name_server'] - dns = dict_merge(default_values, dns) - - # T2665: we cleared default values for tag node 'name_server' above. - # We now need to add them back back in a granular way. - if 'name_server' in dns: - default_values = defaults(base + ['name-server']) - for server in dns['name_server']: - dns['name_server'][server] = dict_merge(default_values, dns['name_server'][server]) - - # T2665: we cleared default values for tag node 'domain' above. - # We now need to add them back back in a granular way. - if 'domain' in dns: - default_values = defaults(base + ['domain', 'name-server']) - for domain in dns['domain'].keys(): - for server in dns['domain'][domain]['name_server']: - dns['domain'][domain]['name_server'][server] = dict_merge( - default_values, dns['domain'][domain]['name_server'][server]) + dns = conf.get_config_dict(base, key_mangling=('-', '_'), + no_tag_node_value_mangle=True, + get_first_key=True, + with_recursive_defaults=True) # some additions to the default dictionary if 'system' in dns: @@ -109,9 +86,6 @@ def get_config(config=None): rdata = recorddata[rtype][subnode] if rtype in [ 'a', 'aaaa' ]: - rdefaults = defaults(base + ['authoritative-domain', 'records', rtype]) # T2665 - rdata = dict_merge(rdefaults, rdata) - if not 'address' in rdata: dns['authoritative_zone_errors'].append(f'{subnode}.{node}: at least one address is required') continue @@ -127,9 +101,6 @@ def get_config(config=None): 'value': address }) elif rtype in ['cname', 'ptr', 'ns']: - rdefaults = defaults(base + ['authoritative-domain', 'records', rtype]) # T2665 - rdata = dict_merge(rdefaults, rdata) - if not 'target' in rdata: dns['authoritative_zone_errors'].append(f'{subnode}.{node}: target is required') continue @@ -141,18 +112,12 @@ def get_config(config=None): 'value': '{}.'.format(rdata['target']) }) elif rtype == 'mx': - rdefaults = defaults(base + ['authoritative-domain', 'records', rtype]) # T2665 - del rdefaults['server'] - rdata = dict_merge(rdefaults, rdata) - if not 'server' in rdata: dns['authoritative_zone_errors'].append(f'{subnode}.{node}: at least one server is required') continue for servername in rdata['server']: serverdata = rdata['server'][servername] - serverdefaults = defaults(base + ['authoritative-domain', 'records', rtype, 'server']) # T2665 - serverdata = dict_merge(serverdefaults, serverdata) zone['records'].append({ 'name': subnode, 'type': rtype.upper(), @@ -160,9 +125,6 @@ def get_config(config=None): 'value': '{} {}.'.format(serverdata['priority'], servername) }) elif rtype == 'txt': - rdefaults = defaults(base + ['authoritative-domain', 'records', rtype]) # T2665 - rdata = dict_merge(rdefaults, rdata) - if not 'value' in rdata: dns['authoritative_zone_errors'].append(f'{subnode}.{node}: at least one value is required') continue @@ -175,9 +137,6 @@ def get_config(config=None): 'value': "\"{}\"".format(value.replace("\"", "\\\"")) }) elif rtype == 'spf': - rdefaults = defaults(base + ['authoritative-domain', 'records', rtype]) # T2665 - rdata = dict_merge(rdefaults, rdata) - if not 'value' in rdata: dns['authoritative_zone_errors'].append(f'{subnode}.{node}: value is required') continue @@ -189,19 +148,12 @@ def get_config(config=None): 'value': '"{}"'.format(rdata['value'].replace("\"", "\\\"")) }) elif rtype == 'srv': - rdefaults = defaults(base + ['authoritative-domain', 'records', rtype]) # T2665 - del rdefaults['entry'] - rdata = dict_merge(rdefaults, rdata) - if not 'entry' in rdata: dns['authoritative_zone_errors'].append(f'{subnode}.{node}: at least one entry is required') continue for entryno in rdata['entry']: entrydata = rdata['entry'][entryno] - entrydefaults = defaults(base + ['authoritative-domain', 'records', rtype, 'entry']) # T2665 - entrydata = dict_merge(entrydefaults, entrydata) - if not 'hostname' in entrydata: dns['authoritative_zone_errors'].append(f'{subnode}.{node}: hostname is required for entry {entryno}') continue @@ -217,19 +169,12 @@ def get_config(config=None): 'value': '{} {} {} {}.'.format(entrydata['priority'], entrydata['weight'], entrydata['port'], entrydata['hostname']) }) elif rtype == 'naptr': - rdefaults = defaults(base + ['authoritative-domain', 'records', rtype]) # T2665 - del rdefaults['rule'] - rdata = dict_merge(rdefaults, rdata) - - if not 'rule' in rdata: dns['authoritative_zone_errors'].append(f'{subnode}.{node}: at least one rule is required') continue for ruleno in rdata['rule']: ruledata = rdata['rule'][ruleno] - ruledefaults = defaults(base + ['authoritative-domain', 'records', rtype, 'rule']) # T2665 - ruledata = dict_merge(ruledefaults, ruledata) flags = "" if 'lookup-srv' in ruledata: flags += "S" diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index 07166d457..7242e503a 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -23,7 +23,6 @@ from sys import exit from vyos.base import Warning from vyos.config import Config -from vyos.configdict import dict_merge from vyos.configdict import node_changed from vyos.configdiff import get_config_diff, Diff from vyos.configdep import set_dependents, call_dependents @@ -37,7 +36,6 @@ from vyos.utils.dict import dict_search_args from vyos.utils.dict import dict_search_recursive from vyos.utils.process import process_named_running from vyos.utils.process import rc_cmd -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -125,43 +123,13 @@ def get_config(config=None): conf = Config() base = ['firewall'] - firewall = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True, - no_tag_node_value_mangle=True) - - # We have gathered the dict representation of the CLI, but there are - # default options which we need to update into the dictionary retrived. - # XXX: T2665: we currently have no nice way for defaults under tag - # nodes, thus we load the defaults "by hand" - default_values = defaults(base) - for tmp in ['name', 'ipv6_name']: - if tmp in default_values: - del default_values[tmp] - - if 'zone' in default_values: - del default_values['zone'] - - firewall = dict_merge(default_values, firewall) - - # Merge in defaults for IPv4 ruleset - if 'name' in firewall: - default_values = defaults(base + ['name']) - for name in firewall['name']: - firewall['name'][name] = dict_merge(default_values, - firewall['name'][name]) - - # Merge in defaults for IPv6 ruleset - if 'ipv6_name' in firewall: - default_values = defaults(base + ['ipv6-name']) - for ipv6_name in firewall['ipv6_name']: - firewall['ipv6_name'][ipv6_name] = dict_merge(default_values, - firewall['ipv6_name'][ipv6_name]) - - if 'zone' in firewall: - default_values = defaults(base + ['zone']) - for zone in firewall['zone']: - firewall['zone'][zone] = dict_merge(default_values, firewall['zone'][zone]) + firewall = conf.get_config_dict(base, key_mangling=('-', '_'), + no_tag_node_value_mangle=True, + get_first_key=True, + with_recursive_defaults=True) firewall['group_resync'] = bool('group' in firewall or node_changed(conf, base + ['group'])) + if firewall['group_resync']: # Update nat and policy-route as firewall groups were updated set_dependents('group_resync', conf) diff --git a/src/conf_mode/flow_accounting_conf.py b/src/conf_mode/flow_accounting_conf.py index 372bb0da7..71acd69fa 100755 --- a/src/conf_mode/flow_accounting_conf.py +++ b/src/conf_mode/flow_accounting_conf.py @@ -22,14 +22,13 @@ from ipaddress import ip_address from vyos.base import Warning from vyos.config import Config -from vyos.configdict import dict_merge +from vyos.config import config_dict_merge from vyos.configverify import verify_vrf from vyos.ifconfig import Section from vyos.template import render from vyos.utils.process import call from vyos.utils.process import cmd from vyos.utils.network import is_addr_assigned -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -128,30 +127,19 @@ def get_config(config=None): flow_accounting = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) + # We have gathered the dict representation of the CLI, but there are + # default values which we need to conditionally update into the + # dictionary retrieved. + default_values = conf.get_config_defaults(**flow_accounting.kwargs, + recursive=True) - # delete individual flow type default - should only be added if user uses - # this feature + # delete individual flow type defaults - should only be added if user + # sets this feature for flow_type in ['sflow', 'netflow']: - if flow_type in default_values: + if flow_type not in flow_accounting and flow_type in default_values: del default_values[flow_type] - flow_accounting = dict_merge(default_values, flow_accounting) - for flow_type in ['sflow', 'netflow']: - if flow_type in flow_accounting: - default_values = defaults(base + [flow_type]) - # we need to merge individual server configurations - if 'server' in default_values: - del default_values['server'] - flow_accounting[flow_type] = dict_merge(default_values, flow_accounting[flow_type]) - - if 'server' in flow_accounting[flow_type]: - default_values = defaults(base + [flow_type, 'server']) - for server in flow_accounting[flow_type]['server']: - flow_accounting[flow_type]['server'][server] = dict_merge( - default_values,flow_accounting[flow_type]['server'][server]) + flow_accounting = config_dict_merge(default_values, flow_accounting) return flow_accounting diff --git a/src/conf_mode/http-api.py b/src/conf_mode/http-api.py index 7bdf448a3..793a90d88 100755 --- a/src/conf_mode/http-api.py +++ b/src/conf_mode/http-api.py @@ -24,12 +24,9 @@ from copy import deepcopy import vyos.defaults from vyos.config import Config -from vyos.configdict import dict_merge from vyos.configdep import set_dependents, call_dependents from vyos.template import render -from vyos.utils.process import cmd from vyos.utils.process import call -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -72,8 +69,9 @@ def get_config(config=None): return None api_dict = conf.get_config_dict(base, key_mangling=('-', '_'), - no_tag_node_value_mangle=True, - get_first_key=True) + no_tag_node_value_mangle=True, + get_first_key=True, + with_recursive_defaults=True) # One needs to 'flatten' the keys dict from the config into the # http-api.conf format for api_keys: @@ -93,8 +91,8 @@ def get_config(config=None): if 'api_keys' in api_dict: keys_added = True - if 'graphql' in api_dict: - api_dict = dict_merge(defaults(base), api_dict) + if api_dict.from_defaults(['graphql']): + del api_dict['graphql'] http_api.update(api_dict) diff --git a/src/conf_mode/igmp_proxy.py b/src/conf_mode/igmp_proxy.py index 4ec2f1835..40db417dd 100755 --- a/src/conf_mode/igmp_proxy.py +++ b/src/conf_mode/igmp_proxy.py @@ -21,11 +21,9 @@ from netifaces import interfaces from vyos.base import Warning from vyos.config import Config -from vyos.configdict import dict_merge from vyos.template import render from vyos.utils.process import call from vyos.utils.dict import dict_search -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -39,16 +37,9 @@ def get_config(config=None): conf = Config() base = ['protocols', 'igmp-proxy'] - igmp_proxy = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - - if 'interface' in igmp_proxy: - # T2665: we must add the tagNode defaults individually until this is - # moved to the base class - default_values = defaults(base + ['interface']) - for interface in igmp_proxy['interface']: - igmp_proxy['interface'][interface] = dict_merge(default_values, - igmp_proxy['interface'][interface]) - + igmp_proxy = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, + with_defaults=True) if conf.exists(['protocols', 'igmp']): igmp_proxy.update({'igmp_configured': ''}) diff --git a/src/conf_mode/interfaces-bridge.py b/src/conf_mode/interfaces-bridge.py index 1bdd61eca..c82f01e53 100755 --- a/src/conf_mode/interfaces-bridge.py +++ b/src/conf_mode/interfaces-bridge.py @@ -14,10 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import os - from sys import exit -from netifaces import interfaces from vyos.config import Config from vyos.configdict import get_interface_dict @@ -25,16 +22,13 @@ from vyos.configdict import node_changed from vyos.configdict import is_member from vyos.configdict import is_source_interface from vyos.configdict import has_vlan_subinterface_configured -from vyos.configdict import dict_merge from vyos.configverify import verify_dhcpv6 from vyos.configverify import verify_mirror_redirect from vyos.configverify import verify_vrf from vyos.ifconfig import BridgeIf from vyos.configdict import has_address_configured from vyos.configdict import has_vrf_configured -from vyos.xml import defaults -from vyos.utils.process import cmd from vyos.utils.dict import dict_search from vyos import ConfigError @@ -61,22 +55,8 @@ def get_config(config=None): else: bridge.update({'member' : {'interface_remove' : tmp }}) - if dict_search('member.interface', bridge) != None: - # XXX: T2665: we need a copy of the dict keys for iteration, else we will get: - # RuntimeError: dictionary changed size during iteration + if dict_search('member.interface', bridge) is not None: for interface in list(bridge['member']['interface']): - for key in ['cost', 'priority']: - if interface == key: - del bridge['member']['interface'][key] - continue - - # the default dictionary is not properly paged into the dict (see T2665) - # thus we will ammend it ourself - default_member_values = defaults(base + ['member', 'interface']) - for interface,interface_config in bridge['member']['interface'].items(): - bridge['member']['interface'][interface] = dict_merge( - default_member_values, bridge['member']['interface'][interface]) - # Check if member interface is already member of another bridge tmp = is_member(conf, interface, 'bridge') if tmp and bridge['ifname'] not in tmp: diff --git a/src/conf_mode/interfaces-wireless.py b/src/conf_mode/interfaces-wireless.py index 42326bea0..e49ad25ac 100755 --- a/src/conf_mode/interfaces-wireless.py +++ b/src/conf_mode/interfaces-wireless.py @@ -79,27 +79,9 @@ def get_config(config=None): ifname, wifi = get_interface_dict(conf, base) - # Cleanup "delete" default values when required user selectable values are - # not defined at all - tmp = conf.get_config_dict(base + [ifname], key_mangling=('-', '_'), - get_first_key=True) - if not (dict_search('security.wpa.passphrase', tmp) or - dict_search('security.wpa.radius', tmp)): - if 'deleted' not in wifi: - del wifi['security']['wpa'] - # if 'security' key is empty, drop it too - if len(wifi['security']) == 0: - del wifi['security'] - - # defaults include RADIUS server specifics per TAG node which need to be - # added to individual RADIUS servers instead - so we can simply delete them - if dict_search('security.wpa.radius.server.port', wifi) != None: - del wifi['security']['wpa']['radius']['server']['port'] - if not len(wifi['security']['wpa']['radius']['server']): - del wifi['security']['wpa']['radius'] - if not len(wifi['security']['wpa']): - del wifi['security']['wpa'] - if not len(wifi['security']): + if 'deleted' not in wifi: + # then get_interface_dict provides default keys + if wifi.from_defaults(['security']): # if not set by user del wifi['security'] if 'security' in wifi and 'wpa' in wifi['security']: @@ -120,14 +102,6 @@ def get_config(config=None): tmp = find_other_stations(conf, base, wifi['ifname']) if tmp: wifi['station_interfaces'] = tmp - # Add individual RADIUS server default values - if dict_search('security.wpa.radius.server', wifi): - default_values = defaults(base + ['security', 'wpa', 'radius', 'server']) - - for server in dict_search('security.wpa.radius.server', wifi): - wifi['security']['wpa']['radius']['server'][server] = dict_merge( - default_values, wifi['security']['wpa']['radius']['server'][server]) - return wifi def verify(wifi): diff --git a/src/conf_mode/lldp.py b/src/conf_mode/lldp.py index c8f341327..c2e87d171 100755 --- a/src/conf_mode/lldp.py +++ b/src/conf_mode/lldp.py @@ -20,13 +20,11 @@ from sys import exit from vyos.base import Warning from vyos.config import Config -from vyos.configdict import dict_merge from vyos.utils.network import is_addr_assigned from vyos.utils.network import is_loopback_addr from vyos.version import get_version_data from vyos.utils.process import call from vyos.utils.dict import dict_search -from vyos.xml import defaults from vyos.template import render from vyos import ConfigError from vyos import airbag @@ -46,7 +44,9 @@ def get_config(config=None): return {} lldp = conf.get_config_dict(base, key_mangling=('-', '_'), - get_first_key=True, no_tag_node_value_mangle=True) + no_tag_node_value_mangle=True, + get_first_key=True, + with_recursive_defaults=True) if conf.exists(['service', 'snmp']): lldp['system_snmp_enabled'] = '' @@ -54,27 +54,12 @@ def get_config(config=None): version_data = get_version_data() lldp['version'] = version_data['version'] - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - # location coordinates have a default value - if 'interface' in lldp: - for interface, interface_config in lldp['interface'].items(): - default_values = defaults(base + ['interface']) - if dict_search('location.coordinate_based', interface_config) == None: - # no location specified - no need to add defaults - del default_values['location']['coordinate_based']['datum'] - del default_values['location']['coordinate_based']['altitude'] - - # cleanup default_values dictionary from inner to outer - # this might feel overkill here, but it does support easy extension - # in the future with additional default values - if len(default_values['location']['coordinate_based']) == 0: - del default_values['location']['coordinate_based'] - if len(default_values['location']) == 0: - del default_values['location'] - - lldp['interface'][interface] = dict_merge(default_values, - lldp['interface'][interface]) + # prune location information if not set by user + for interface in lldp.get('interface', []): + if lldp.from_defaults(['interface', interface, 'location']): + del lldp['interface'][interface]['location'] + elif lldp.from_defaults(['interface', interface, 'location','coordinate_based']): + del lldp['interface'][interface]['location']['coordinate_based'] return lldp diff --git a/src/conf_mode/load-balancing-haproxy.py b/src/conf_mode/load-balancing-haproxy.py index 2fb0edf8e..8fe429653 100755 --- a/src/conf_mode/load-balancing-haproxy.py +++ b/src/conf_mode/load-balancing-haproxy.py @@ -20,14 +20,12 @@ from sys import exit from shutil import rmtree from vyos.config import Config -from vyos.configdict import dict_merge from vyos.utils.process import call from vyos.utils.network import check_port_availability from vyos.utils.network import is_listen_port_bind_service from vyos.pki import wrap_certificate from vyos.pki import wrap_private_key from vyos.template import render -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -54,18 +52,8 @@ def get_config(config=None): lb['pki'] = conf.get_config_dict(['pki'], key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - if 'backend' in default_values: - del default_values['backend'] if lb: - lb = dict_merge(default_values, lb) - - if 'backend' in lb: - for backend in lb['backend']: - default_balues_backend = defaults(base + ['backend']) - lb['backend'][backend] = dict_merge(default_balues_backend, lb['backend'][backend]) + lb = conf.merge_defaults(lb, recursive=True) return lb diff --git a/src/conf_mode/load-balancing-wan.py b/src/conf_mode/load-balancing-wan.py index 3533a5a04..ad9c80d72 100755 --- a/src/conf_mode/load-balancing-wan.py +++ b/src/conf_mode/load-balancing-wan.py @@ -21,10 +21,8 @@ from shutil import rmtree from vyos.base import Warning from vyos.config import Config -from vyos.configdict import dict_merge from vyos.utils.process import cmd from vyos.template import render -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -41,48 +39,15 @@ def get_config(config=None): conf = Config() base = ['load-balancing', 'wan'] - lb = conf.get_config_dict(base, + lb = conf.get_config_dict(base, key_mangling=('-', '_'), + no_tag_node_value_mangle=True, get_first_key=True, - key_mangling=('-', '_'), - no_tag_node_value_mangle=True) - - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - # lb base default values can not be merged here - remove and add them later - if 'interface_health' in default_values: - del default_values['interface_health'] - if 'rule' in default_values: - del default_values['rule'] - lb = dict_merge(default_values, lb) - - if 'interface_health' in lb: - for iface in lb.get('interface_health'): - default_values_iface = defaults(base + ['interface-health']) - if 'test' in default_values_iface: - del default_values_iface['test'] - lb['interface_health'][iface] = dict_merge( - default_values_iface, lb['interface_health'][iface]) - if 'test' in lb['interface_health'][iface]: - for node_test in lb['interface_health'][iface]['test']: - default_values_test = defaults(base + - ['interface-health', 'test']) - lb['interface_health'][iface]['test'][node_test] = dict_merge( - default_values_test, - lb['interface_health'][iface]['test'][node_test]) - - if 'rule' in lb: - for rule in lb.get('rule'): - default_values_rule = defaults(base + ['rule']) - if 'interface' in default_values_rule: - del default_values_rule['interface'] - lb['rule'][rule] = dict_merge(default_values_rule, lb['rule'][rule]) - if not conf.exists(base + ['rule', rule, 'limit']): - del lb['rule'][rule]['limit'] - if 'interface' in lb['rule'][rule]: - for iface in lb['rule'][rule]['interface']: - default_values_rule_iface = defaults(base + ['rule', 'interface']) - lb['rule'][rule]['interface'][iface] = dict_merge(default_values_rule_iface, lb['rule'][rule]['interface'][iface]) + with_recursive_defaults=True) + + # prune limit key if not set by user + for rule in lb.get('rule', []): + if lb.from_defaults(['rule', rule, 'limit']): + del lb['rule'][rule]['limit'] return lb diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py index 8e3a11ff4..f9d711b36 100755 --- a/src/conf_mode/nat.py +++ b/src/conf_mode/nat.py @@ -25,7 +25,6 @@ from netifaces import interfaces from vyos.base import Warning from vyos.config import Config -from vyos.configdict import dict_merge from vyos.template import render from vyos.template import is_ip_network from vyos.utils.kernel import check_kmod @@ -34,7 +33,6 @@ from vyos.utils.dict import dict_search_args from vyos.utils.process import cmd from vyos.utils.process import run from vyos.utils.network import is_addr_assigned -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag @@ -145,16 +143,9 @@ def get_config(config=None): conf = Config() base = ['nat'] - nat = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - - # T2665: we must add the tagNode defaults individually until this is - # moved to the base class - for direction in ['source', 'destination', 'static']: - if direction in nat: - default_values = defaults(base + [direction, 'rule']) - for rule in dict_search(f'{direction}.rule', nat) or []: - nat[direction]['rule'][rule] = dict_merge(default_values, - nat[direction]['rule'][rule]) + nat = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True) # read in current nftable (once) for further processing tmp = cmd('nft -j list table raw') diff --git a/src/conf_mode/nat66.py b/src/conf_mode/nat66.py index 25f625b84..4c12618bc 100755 --- a/src/conf_mode/nat66.py +++ b/src/conf_mode/nat66.py @@ -23,13 +23,11 @@ from netifaces import interfaces from vyos.base import Warning from vyos.config import Config -from vyos.configdict import dict_merge from vyos.template import render from vyos.utils.process import cmd from vyos.utils.kernel import check_kmod from vyos.utils.dict import dict_search from vyos.template import is_ipv6 -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -60,16 +58,6 @@ def get_config(config=None): base = ['nat66'] nat = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - # T2665: we must add the tagNode defaults individually until this is - # moved to the base class - for direction in ['source', 'destination']: - if direction in nat: - default_values = defaults(base + [direction, 'rule']) - if 'rule' in nat[direction]: - for rule in nat[direction]['rule']: - nat[direction]['rule'][rule] = dict_merge(default_values, - nat[direction]['rule'][rule]) - # read in current nftable (once) for further processing tmp = cmd('nft -j list table ip6 raw') nftable_json = json.loads(tmp) diff --git a/src/conf_mode/pki.py b/src/conf_mode/pki.py index eb8cb3940..34ba2fe69 100755 --- a/src/conf_mode/pki.py +++ b/src/conf_mode/pki.py @@ -18,7 +18,6 @@ from sys import exit from vyos.config import Config from vyos.configdep import set_dependents, call_dependents -from vyos.configdict import dict_merge from vyos.configdict import node_changed from vyos.pki import is_ca_certificate from vyos.pki import load_certificate @@ -28,7 +27,6 @@ from vyos.pki import load_crl from vyos.pki import load_dh_parameters from vyos.utils.dict import dict_search_args from vyos.utils.dict import dict_search_recursive -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -113,8 +111,7 @@ def get_config(config=None): # We only merge on the defaults of there is a configuration at all if conf.exists(base): - default_values = defaults(base) - pki = dict_merge(default_values, pki) + pki = conf.merge_defaults(pki, recursive=True) # We need to get the entire system configuration to verify that we are not # deleting a certificate that is still referenced somewhere! diff --git a/src/conf_mode/protocols_babel.py b/src/conf_mode/protocols_babel.py index f5ac56f65..104711b55 100755 --- a/src/conf_mode/protocols_babel.py +++ b/src/conf_mode/protocols_babel.py @@ -19,13 +19,13 @@ import os from sys import exit from vyos.config import Config +from vyos.config import config_dict_merge from vyos.configdict import dict_merge from vyos.configdict import node_changed from vyos.configverify import verify_common_route_maps from vyos.configverify import verify_access_list from vyos.configverify import verify_prefix_list from vyos.utils.dict import dict_search -from vyos.xml import defaults from vyos.template import render_to_string from vyos import ConfigError from vyos import frr @@ -38,7 +38,8 @@ def get_config(config=None): else: conf = Config() base = ['protocols', 'babel'] - babel = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) + babel = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True) # FRR has VRF support for different routing daemons. As interfaces belong # to VRFs - or the global VRF, we need to check for changed interfaces so @@ -54,15 +55,13 @@ def get_config(config=None): return babel # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) + # values which we need to update into the dictionary retrieved. + default_values = conf.get_config_defaults(base, key_mangling=('-', '_'), + get_first_key=True, + recursive=True) - # XXX: T2665: we currently have no nice way for defaults under tag nodes, - # clean them out and add them manually :( - del default_values['interface'] - - # merge in remaining default values - babel = dict_merge(default_values, babel) + # merge in default values + babel = config_dict_merge(default_values, babel) # We also need some additional information from the config, prefix-lists # and route-maps for instance. They will be used in verify(). diff --git a/src/conf_mode/protocols_bfd.py b/src/conf_mode/protocols_bfd.py index b8d2d65ee..dab784662 100755 --- a/src/conf_mode/protocols_bfd.py +++ b/src/conf_mode/protocols_bfd.py @@ -17,12 +17,10 @@ import os from vyos.config import Config -from vyos.configdict import dict_merge from vyos.configverify import verify_vrf from vyos.template import is_ipv6 from vyos.template import render_to_string from vyos.utils.network import is_ipv6_link_local -from vyos.xml import defaults from vyos import ConfigError from vyos import frr from vyos import airbag @@ -41,18 +39,7 @@ def get_config(config=None): if not conf.exists(base): return bfd - # We have gathered the dict representation of the CLI, but there are - # default options which we need to update into the dictionary retrived. - # XXX: T2665: we currently have no nice way for defaults under tag - # nodes, thus we load the defaults "by hand" - default_values = defaults(base + ['peer']) - if 'peer' in bfd: - for peer in bfd['peer']: - bfd['peer'][peer] = dict_merge(default_values, bfd['peer'][peer]) - - if 'profile' in bfd: - for profile in bfd['profile']: - bfd['profile'][profile] = dict_merge(default_values, bfd['profile'][profile]) + bfd = conf.merge_defaults(bfd, recursive=True) return bfd diff --git a/src/conf_mode/protocols_failover.py b/src/conf_mode/protocols_failover.py index faf56d741..e7e44db84 100755 --- a/src/conf_mode/protocols_failover.py +++ b/src/conf_mode/protocols_failover.py @@ -19,10 +19,8 @@ import json from pathlib import Path from vyos.config import Config -from vyos.configdict import dict_merge from vyos.template import render from vyos.utils.process import call -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag @@ -42,15 +40,12 @@ def get_config(config=None): conf = Config() base = ['protocols', 'failover'] - failover = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) + failover = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True) # Set default values only if we set config - if failover.get('route'): - for route, route_config in failover.get('route').items(): - for next_hop, next_hop_config in route_config.get('next_hop').items(): - default_values = defaults(base + ['route']) - failover['route'][route]['next_hop'][next_hop] = dict_merge( - default_values['next_hop'], failover['route'][route]['next_hop'][next_hop]) + if failover.get('route') is not None: + failover = conf.merge_defaults(failover, recursive=True) return failover diff --git a/src/conf_mode/protocols_isis.py b/src/conf_mode/protocols_isis.py index 4c637a99f..197f8131d 100755 --- a/src/conf_mode/protocols_isis.py +++ b/src/conf_mode/protocols_isis.py @@ -28,7 +28,6 @@ from vyos.ifconfig import Interface from vyos.utils.dict import dict_search from vyos.utils.network import get_interface_config from vyos.template import render_to_string -from vyos.xml import defaults from vyos import ConfigError from vyos import frr from vyos import airbag @@ -69,14 +68,8 @@ def get_config(config=None): isis.update({'deleted' : ''}) return isis - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - # XXX: Note that we can not call defaults(base), as defaults does not work - # on an instance of a tag node. As we use the exact same CLI definition for - # both the non-vrf and vrf version this is absolutely safe! - default_values = defaults(base_path) # merge in default values - isis = dict_merge(default_values, isis) + isis = conf.merge_defaults(isis, recursive=True) # We also need some additional information from the config, prefix-lists # and route-maps for instance. They will be used in verify(). diff --git a/src/conf_mode/protocols_ospf.py b/src/conf_mode/protocols_ospf.py index f2075d25b..4803ad2a1 100755 --- a/src/conf_mode/protocols_ospf.py +++ b/src/conf_mode/protocols_ospf.py @@ -20,6 +20,7 @@ from sys import exit from sys import argv from vyos.config import Config +from vyos.config import config_dict_merge from vyos.configdict import dict_merge from vyos.configdict import node_changed from vyos.configverify import verify_common_route_maps @@ -29,7 +30,6 @@ from vyos.configverify import verify_access_list from vyos.template import render_to_string from vyos.utils.dict import dict_search from vyos.utils.network import get_interface_config -from vyos.xml import defaults from vyos import ConfigError from vyos import frr from vyos import airbag @@ -72,10 +72,7 @@ def get_config(config=None): # We have gathered the dict representation of the CLI, but there are default # options which we need to update into the dictionary retrived. - # XXX: Note that we can not call defaults(base), as defaults does not work - # on an instance of a tag node. As we use the exact same CLI definition for - # both the non-vrf and vrf version this is absolutely safe! - default_values = defaults(base_path) + default_values = conf.get_config_defaults(**ospf.kwargs, recursive=True) # We have to cleanup the default dict, as default values could enable features # which are not explicitly enabled on the CLI. Example: default-information @@ -84,62 +81,27 @@ def get_config(config=None): # need to check this first and probably drop that key. if dict_search('default_information.originate', ospf) is None: del default_values['default_information'] - if dict_search('area.area_type.nssa', ospf) is None: - del default_values['area']['area_type']['nssa'] if 'mpls_te' not in ospf: del default_values['mpls_te'] if 'graceful_restart' not in ospf: del default_values['graceful_restart'] + for area_num in default_values.get('area', []): + if dict_search(f'area.{area_num}.area_type.nssa', ospf) is None: + del default_values['area'][area_num]['area_type']['nssa'] - for protocol in ['babel', 'bgp', 'connected', 'isis', 'kernel', 'rip', 'static', 'table']: - # table is a tagNode thus we need to clean out all occurances for the - # default values and load them in later individually - if protocol == 'table': - del default_values['redistribute']['table'] - continue + for protocol in ['babel', 'bgp', 'connected', 'isis', 'kernel', 'rip', 'static']: if dict_search(f'redistribute.{protocol}', ospf) is None: del default_values['redistribute'][protocol] - # XXX: T2665: we currently have no nice way for defaults under tag nodes, - # clean them out and add them manually :( - del default_values['neighbor'] - del default_values['area']['virtual_link'] - del default_values['interface'] - - # merge in remaining default values - ospf = dict_merge(default_values, ospf) - - if 'neighbor' in ospf: - default_values = defaults(base + ['neighbor']) - for neighbor in ospf['neighbor']: - ospf['neighbor'][neighbor] = dict_merge(default_values, ospf['neighbor'][neighbor]) + for interface in ospf.get('interface', []): + # We need to reload the defaults on every pass b/c of + # hello-multiplier dependency on dead-interval + # If hello-multiplier is set, we need to remove the default from + # dead-interval. + if 'hello_multiplier' in ospf['interface'][interface]: + del default_values['interface'][interface]['dead_interval'] - if 'area' in ospf: - default_values = defaults(base + ['area', 'virtual-link']) - for area, area_config in ospf['area'].items(): - if 'virtual_link' in area_config: - for virtual_link in area_config['virtual_link']: - ospf['area'][area]['virtual_link'][virtual_link] = dict_merge( - default_values, ospf['area'][area]['virtual_link'][virtual_link]) - - if 'interface' in ospf: - for interface in ospf['interface']: - # We need to reload the defaults on every pass b/c of - # hello-multiplier dependency on dead-interval - default_values = defaults(base + ['interface']) - # If hello-multiplier is set, we need to remove the default from - # dead-interval. - if 'hello_multiplier' in ospf['interface'][interface]: - del default_values['dead_interval'] - - ospf['interface'][interface] = dict_merge(default_values, - ospf['interface'][interface]) - - if 'redistribute' in ospf and 'table' in ospf['redistribute']: - default_values = defaults(base + ['redistribute', 'table']) - for table in ospf['redistribute']['table']: - ospf['redistribute']['table'][table] = dict_merge(default_values, - ospf['redistribute']['table'][table]) + ospf = config_dict_merge(default_values, ospf) # We also need some additional information from the config, prefix-lists # and route-maps for instance. They will be used in verify(). diff --git a/src/conf_mode/protocols_ospfv3.py b/src/conf_mode/protocols_ospfv3.py index fbea51f56..e68fa1fe6 100755 --- a/src/conf_mode/protocols_ospfv3.py +++ b/src/conf_mode/protocols_ospfv3.py @@ -20,6 +20,7 @@ from sys import exit from sys import argv from vyos.config import Config +from vyos.config import config_dict_merge from vyos.configdict import dict_merge from vyos.configdict import node_changed from vyos.configverify import verify_common_route_maps @@ -29,7 +30,6 @@ from vyos.template import render_to_string from vyos.ifconfig import Interface from vyos.utils.dict import dict_search from vyos.utils.network import get_interface_config -from vyos.xml import defaults from vyos import ConfigError from vyos import frr from vyos import airbag @@ -71,10 +71,8 @@ def get_config(config=None): # We have gathered the dict representation of the CLI, but there are default # options which we need to update into the dictionary retrived. - # XXX: Note that we can not call defaults(base), as defaults does not work - # on an instance of a tag node. As we use the exact same CLI definition for - # both the non-vrf and vrf version this is absolutely safe! - default_values = defaults(base_path) + default_values = conf.get_config_defaults(**ospfv3.kwargs, + recursive=True) # We have to cleanup the default dict, as default values could enable features # which are not explicitly enabled on the CLI. Example: default-information @@ -86,12 +84,10 @@ def get_config(config=None): if 'graceful_restart' not in ospfv3: del default_values['graceful_restart'] - # XXX: T2665: we currently have no nice way for defaults under tag nodes, - # clean them out and add them manually :( - del default_values['interface'] + default_values.pop('interface', {}) # merge in remaining default values - ospfv3 = dict_merge(default_values, ospfv3) + ospfv3 = config_dict_merge(default_values, ospfv3) # We also need some additional information from the config, prefix-lists # and route-maps for instance. They will be used in verify(). diff --git a/src/conf_mode/protocols_rip.py b/src/conf_mode/protocols_rip.py index 5661dc377..bd47dfd00 100755 --- a/src/conf_mode/protocols_rip.py +++ b/src/conf_mode/protocols_rip.py @@ -25,7 +25,6 @@ from vyos.configverify import verify_common_route_maps from vyos.configverify import verify_access_list from vyos.configverify import verify_prefix_list from vyos.utils.dict import dict_search -from vyos.xml import defaults from vyos.template import render_to_string from vyos import ConfigError from vyos import frr @@ -55,9 +54,7 @@ def get_config(config=None): # We have gathered the dict representation of the CLI, but there are default # options which we need to update into the dictionary retrived. - default_values = defaults(base) - # merge in remaining default values - rip = dict_merge(default_values, rip) + rip = conf.merge_defaults(rip, recursive=True) # We also need some additional information from the config, prefix-lists # and route-maps for instance. They will be used in verify(). diff --git a/src/conf_mode/protocols_ripng.py b/src/conf_mode/protocols_ripng.py index e3c904e33..dd1550033 100755 --- a/src/conf_mode/protocols_ripng.py +++ b/src/conf_mode/protocols_ripng.py @@ -24,7 +24,6 @@ from vyos.configverify import verify_common_route_maps from vyos.configverify import verify_access_list from vyos.configverify import verify_prefix_list from vyos.utils.dict import dict_search -from vyos.xml import defaults from vyos.template import render_to_string from vyos import ConfigError from vyos import frr @@ -45,9 +44,7 @@ def get_config(config=None): # We have gathered the dict representation of the CLI, but there are default # options which we need to update into the dictionary retrived. - default_values = defaults(base) - # merge in remaining default values - ripng = dict_merge(default_values, ripng) + ripng = conf.merge_defaults(ripng, recursive=True) # We also need some additional information from the config, prefix-lists # and route-maps for instance. They will be used in verify(). diff --git a/src/conf_mode/protocols_rpki.py b/src/conf_mode/protocols_rpki.py index 035b7db05..05e876f3b 100755 --- a/src/conf_mode/protocols_rpki.py +++ b/src/conf_mode/protocols_rpki.py @@ -19,10 +19,8 @@ import os from sys import exit from vyos.config import Config -from vyos.configdict import dict_merge from vyos.template import render_to_string from vyos.utils.dict import dict_search -from vyos.xml import defaults from vyos import ConfigError from vyos import frr from vyos import airbag @@ -43,8 +41,7 @@ def get_config(config=None): # We have gathered the dict representation of the CLI, but there are default # options which we need to update into the dictionary retrived. - default_values = defaults(base) - rpki = dict_merge(default_values, rpki) + rpki = conf.merge_defaults(rpki, recursive=True) return rpki diff --git a/src/conf_mode/qos.py b/src/conf_mode/qos.py index 53e9ff50d..5536adda2 100755 --- a/src/conf_mode/qos.py +++ b/src/conf_mode/qos.py @@ -38,7 +38,6 @@ from vyos.qos import TrafficShaper from vyos.qos import TrafficShaperHFSC from vyos.utils.process import call from vyos.utils.dict import dict_search_recursive -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -97,63 +96,32 @@ def get_config(config=None): type_node = path.split(" ")[1] # return only interface type node set_dependents(type_node, conf, ifname) - if 'policy' in qos: - for policy in qos['policy']: - # when calling defaults() we need to use the real CLI node, thus we - # need a hyphen - policy_hyphen = policy.replace('_', '-') - - if policy in ['random_detect']: - for rd_name, rd_config in qos['policy'][policy].items(): - # There are eight precedence levels - ensure all are present - # to be filled later down with the appropriate default values - default_precedence = {'precedence' : { '0' : {}, '1' : {}, '2' : {}, '3' : {}, - '4' : {}, '5' : {}, '6' : {}, '7' : {} }} - qos['policy']['random_detect'][rd_name] = dict_merge( - default_precedence, qos['policy']['random_detect'][rd_name]) - - for p_name, p_config in qos['policy'][policy].items(): - default_values = defaults(base + ['policy', policy_hyphen]) - - if policy in ['priority_queue']: - if 'default' not in p_config: - raise ConfigError(f'QoS policy {p_name} misses "default" class!') - - # XXX: T2665: we can not safely rely on the defaults() when there are - # tagNodes in place, it is better to blend in the defaults manually. - if 'class' in default_values: - del default_values['class'] - if 'precedence' in default_values: - del default_values['precedence'] - - qos['policy'][policy][p_name] = dict_merge( - default_values, qos['policy'][policy][p_name]) - - # class is another tag node which requires individual handling - if 'class' in p_config: - default_values = defaults(base + ['policy', policy_hyphen, 'class']) - for p_class in p_config['class']: - qos['policy'][policy][p_name]['class'][p_class] = dict_merge( - default_values, qos['policy'][policy][p_name]['class'][p_class]) - - if 'precedence' in p_config: - default_values = defaults(base + ['policy', policy_hyphen, 'precedence']) - # precedence values are a bit more complex as they are calculated - # under specific circumstances - thus we need to iterate two times. - # first blend in the defaults from XML / CLI - for precedence in p_config['precedence']: - qos['policy'][policy][p_name]['precedence'][precedence] = dict_merge( - default_values, qos['policy'][policy][p_name]['precedence'][precedence]) - # second calculate defaults based on actual dictionary - for precedence in p_config['precedence']: - max_thr = int(qos['policy'][policy][p_name]['precedence'][precedence]['maximum_threshold']) - if 'minimum_threshold' not in qos['policy'][policy][p_name]['precedence'][precedence]: - qos['policy'][policy][p_name]['precedence'][precedence]['minimum_threshold'] = str( - int((9 + int(precedence)) * max_thr) // 18); - - if 'queue_limit' not in qos['policy'][policy][p_name]['precedence'][precedence]: - qos['policy'][policy][p_name]['precedence'][precedence]['queue_limit'] = \ - str(int(4 * max_thr)) + for policy in qos.get('policy', []): + if policy in ['random_detect']: + for rd_name in list(qos['policy'][policy]): + # There are eight precedence levels - ensure all are present + # to be filled later down with the appropriate default values + default_precedence = {'precedence' : { '0' : {}, '1' : {}, '2' : {}, '3' : {}, + '4' : {}, '5' : {}, '6' : {}, '7' : {} }} + qos['policy']['random_detect'][rd_name] = dict_merge( + default_precedence, qos['policy']['random_detect'][rd_name]) + + qos = conf.merge_defaults(qos, recursive=True) + + for policy in qos.get('policy', []): + for p_name, p_config in qos['policy'][policy].items(): + if 'precedence' in p_config: + # precedence settings are a bit more complex as they are + # calculated under specific circumstances: + for precedence in p_config['precedence']: + max_thr = int(qos['policy'][policy][p_name]['precedence'][precedence]['maximum_threshold']) + if 'minimum_threshold' not in qos['policy'][policy][p_name]['precedence'][precedence]: + qos['policy'][policy][p_name]['precedence'][precedence]['minimum_threshold'] = str( + int((9 + int(precedence)) * max_thr) // 18); + + if 'queue_limit' not in qos['policy'][policy][p_name]['precedence'][precedence]: + qos['policy'][policy][p_name]['precedence'][precedence]['queue_limit'] = \ + str(int(4 * max_thr)) return qos @@ -202,7 +170,9 @@ def verify(qos): queue_lim = int(precedence_config['queue_limit']) if queue_lim < max_tr: raise ConfigError(f'Policy "{policy}" uses queue-limit "{queue_lim}" < max-threshold "{max_tr}"!') - + if policy_type in ['priority_queue']: + if 'default' not in policy_config: + raise ConfigError(f'Policy {policy} misses "default" class!') if 'default' in policy_config: if 'bandwidth' not in policy_config['default'] and policy_type not in ['priority_queue', 'round_robin']: raise ConfigError('Bandwidth not defined for default traffic!') diff --git a/src/conf_mode/salt-minion.py b/src/conf_mode/salt-minion.py index 3ff7880b2..a8fce8e01 100755 --- a/src/conf_mode/salt-minion.py +++ b/src/conf_mode/salt-minion.py @@ -22,12 +22,10 @@ from urllib3 import PoolManager from vyos.base import Warning from vyos.config import Config -from vyos.configdict import dict_merge from vyos.configverify import verify_interface_exists from vyos.template import render from vyos.utils.process import call from vyos.utils.permission import chown -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag @@ -55,8 +53,7 @@ def get_config(config=None): salt['id'] = gethostname() # We have gathered the dict representation of the CLI, but there are default # options which we need to update into the dictionary retrived. - default_values = defaults(base) - salt = dict_merge(default_values, salt) + salt = conf.merge_defaults(salt, recursive=True) if not conf.exists(base): return None diff --git a/src/conf_mode/service_config_sync.py b/src/conf_mode/service_config_sync.py index 5cde735a1..4b8a7f6ee 100755 --- a/src/conf_mode/service_config_sync.py +++ b/src/conf_mode/service_config_sync.py @@ -19,8 +19,6 @@ import json from pathlib import Path from vyos.config import Config -from vyos.configdict import dict_merge -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag @@ -42,12 +40,8 @@ def get_config(config=None): base = ['service', 'config-sync'] if not conf.exists(base): return None - config = conf.get_config_dict(base, - get_first_key=True, - no_tag_node_value_mangle=True) - - default_values = defaults(base) - config = dict_merge(default_values, config) + config = conf.get_config_dict(base, get_first_key=True, + with_recursive_defaults=True) return config diff --git a/src/conf_mode/service_console-server.py b/src/conf_mode/service_console-server.py index 7eb41ea87..b112add3f 100755 --- a/src/conf_mode/service_console-server.py +++ b/src/conf_mode/service_console-server.py @@ -20,10 +20,8 @@ from sys import exit from psutil import process_iter from vyos.config import Config -from vyos.configdict import dict_merge from vyos.template import render from vyos.utils.process import call -from vyos.xml import defaults from vyos import ConfigError config_file = '/run/conserver/conserver.cf' @@ -49,11 +47,7 @@ def get_config(config=None): # We have gathered the dict representation of the CLI, but there are default # options which we need to update into the dictionary retrived. - default_values = defaults(base + ['device']) - if 'device' in proxy: - for device in proxy['device']: - tmp = dict_merge(default_values, proxy['device'][device]) - proxy['device'][device] = tmp + proxy = conf.merge_defaults(proxy, recursive=True) return proxy diff --git a/src/conf_mode/service_ids_fastnetmon.py b/src/conf_mode/service_ids_fastnetmon.py index f6b80552b..276a71fcb 100755 --- a/src/conf_mode/service_ids_fastnetmon.py +++ b/src/conf_mode/service_ids_fastnetmon.py @@ -19,10 +19,8 @@ import os from sys import exit from vyos.config import Config -from vyos.configdict import dict_merge from vyos.template import render from vyos.utils.process import call -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -41,11 +39,9 @@ def get_config(config=None): if not conf.exists(base): return None - fastnetmon = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - fastnetmon = dict_merge(default_values, fastnetmon) + fastnetmon = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True) return fastnetmon diff --git a/src/conf_mode/service_monitoring_telegraf.py b/src/conf_mode/service_monitoring_telegraf.py index 0269bedd9..40eb13e23 100755 --- a/src/conf_mode/service_monitoring_telegraf.py +++ b/src/conf_mode/service_monitoring_telegraf.py @@ -22,7 +22,6 @@ from sys import exit from shutil import rmtree from vyos.config import Config -from vyos.configdict import dict_merge from vyos.configdict import is_node_changed from vyos.configverify import verify_vrf from vyos.ifconfig import Section @@ -30,7 +29,6 @@ from vyos.template import render from vyos.utils.process import call from vyos.utils.permission import chown from vyos.utils.process import cmd -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -83,8 +81,7 @@ def get_config(config=None): # We have gathered the dict representation of the CLI, but there are default # options which we need to update into the dictionary retrived. - default_values = defaults(base) - monitoring = dict_merge(default_values, monitoring) + monitoring = conf.merge_defaults(monitoring, recursive=True) monitoring['custom_scripts_dir'] = custom_scripts_dir monitoring['hostname'] = get_hostname() diff --git a/src/conf_mode/service_router-advert.py b/src/conf_mode/service_router-advert.py index fe33c43ea..dbb47de4e 100755 --- a/src/conf_mode/service_router-advert.py +++ b/src/conf_mode/service_router-advert.py @@ -19,10 +19,8 @@ import os from sys import exit from vyos.base import Warning from vyos.config import Config -from vyos.configdict import dict_merge from vyos.template import render from vyos.utils.process import call -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -35,40 +33,9 @@ def get_config(config=None): else: conf = Config() base = ['service', 'router-advert'] - rtradv = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_interface_values = defaults(base + ['interface']) - # we deal with prefix, route defaults later on - if 'prefix' in default_interface_values: - del default_interface_values['prefix'] - if 'route' in default_interface_values: - del default_interface_values['route'] - - default_prefix_values = defaults(base + ['interface', 'prefix']) - default_route_values = defaults(base + ['interface', 'route']) - - if 'interface' in rtradv: - for interface in rtradv['interface']: - rtradv['interface'][interface] = dict_merge( - default_interface_values, rtradv['interface'][interface]) - - if 'prefix' in rtradv['interface'][interface]: - for prefix in rtradv['interface'][interface]['prefix']: - rtradv['interface'][interface]['prefix'][prefix] = dict_merge( - default_prefix_values, rtradv['interface'][interface]['prefix'][prefix]) - - if 'route' in rtradv['interface'][interface]: - for route in rtradv['interface'][interface]['route']: - rtradv['interface'][interface]['route'][route] = dict_merge( - default_route_values, rtradv['interface'][interface]['route'][route]) - - if 'name_server' in rtradv['interface'][interface]: - # always use a list when dealing with nameservers - eases the template generation - if isinstance(rtradv['interface'][interface]['name_server'], str): - rtradv['interface'][interface]['name_server'] = [ - rtradv['interface'][interface]['name_server']] + rtradv = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True) return rtradv diff --git a/src/conf_mode/service_sla.py b/src/conf_mode/service_sla.py index 54b72e029..ba5e645f0 100755 --- a/src/conf_mode/service_sla.py +++ b/src/conf_mode/service_sla.py @@ -19,10 +19,8 @@ import os from sys import exit from vyos.config import Config -from vyos.configdict import dict_merge from vyos.template import render from vyos.utils.process import call -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -44,11 +42,9 @@ def get_config(config=None): if not conf.exists(base): return None - sla = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - sla = dict_merge(default_values, sla) + sla = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True) # Ignore default XML values if config doesn't exists # Delete key from dict diff --git a/src/conf_mode/service_upnp.py b/src/conf_mode/service_upnp.py index b37d502c2..cf26bf9ce 100755 --- a/src/conf_mode/service_upnp.py +++ b/src/conf_mode/service_upnp.py @@ -23,12 +23,10 @@ from ipaddress import IPv4Network from ipaddress import IPv6Network from vyos.config import Config -from vyos.configdict import dict_merge from vyos.utils.process import call from vyos.template import render from vyos.template import is_ipv4 from vyos.template import is_ipv6 -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -47,10 +45,7 @@ def get_config(config=None): if not upnpd: return None - if 'rule' in upnpd: - default_member_values = defaults(base + ['rule']) - for rule,rule_config in upnpd['rule'].items(): - upnpd['rule'][rule] = dict_merge(default_member_values, upnpd['rule'][rule]) + upnpd = conf.merge_defaults(upnpd, recursive=True) uuidgen = uuid.uuid1() upnpd.update({'uuid': uuidgen}) diff --git a/src/conf_mode/service_webproxy.py b/src/conf_mode/service_webproxy.py index db4066572..12ae4135e 100755 --- a/src/conf_mode/service_webproxy.py +++ b/src/conf_mode/service_webproxy.py @@ -20,14 +20,13 @@ from shutil import rmtree from sys import exit from vyos.config import Config -from vyos.configdict import dict_merge +from vyos.config import config_dict_merge from vyos.template import render from vyos.utils.process import call from vyos.utils.permission import chmod_755 from vyos.utils.dict import dict_search from vyos.utils.file import write_file from vyos.utils.network import is_addr_assigned -from vyos.xml import defaults from vyos.base import Warning from vyos import ConfigError from vyos import airbag @@ -125,7 +124,8 @@ def get_config(config=None): get_first_key=True) # We have gathered the dict representation of the CLI, but there are default # options which we need to update into the dictionary retrived. - default_values = defaults(base) + default_values = conf.get_config_defaults(**proxy.kwargs, + recursive=True) # if no authentication method is supplied, no need to add defaults if not dict_search('authentication.method', proxy): @@ -138,16 +138,7 @@ def get_config(config=None): proxy['squidguard_conf'] = squidguard_config_file proxy['squidguard_db_dir'] = squidguard_db_dir - # XXX: T2665: blend in proper cache-peer default values later - default_values.pop('cache_peer') - proxy = dict_merge(default_values, proxy) - - # XXX: T2665: blend in proper cache-peer default values - if 'cache_peer' in proxy: - default_values = defaults(base + ['cache-peer']) - for peer in proxy['cache_peer']: - proxy['cache_peer'][peer] = dict_merge(default_values, - proxy['cache_peer'][peer]) + proxy = config_dict_merge(default_values, proxy) return proxy diff --git a/src/conf_mode/snmp.py b/src/conf_mode/snmp.py index 4bf67f079..7882f8510 100755 --- a/src/conf_mode/snmp.py +++ b/src/conf_mode/snmp.py @@ -31,7 +31,6 @@ from vyos.utils.permission import chmod_755 from vyos.utils.dict import dict_search from vyos.utils.network import is_addr_assigned from vyos.version import get_version_data -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -70,26 +69,9 @@ def get_config(config=None): # We have gathered the dict representation of the CLI, but there are default # options which we need to update into the dictionary retrived. - default_values = defaults(base) - - # We can not merge defaults for tagNodes - those need to be blended in - # per tagNode instance - if 'listen_address' in default_values: - del default_values['listen_address'] - if 'community' in default_values: - del default_values['community'] - if 'trap_target' in default_values: - del default_values['trap_target'] - if 'v3' in default_values: - del default_values['v3'] - snmp = dict_merge(default_values, snmp) + snmp = conf.merge_defaults(snmp, recursive=True) if 'listen_address' in snmp: - default_values = defaults(base + ['listen-address']) - for address in snmp['listen_address']: - snmp['listen_address'][address] = dict_merge( - default_values, snmp['listen_address'][address]) - # Always listen on localhost if an explicit address has been configured # This is a safety measure to not end up with invalid listen addresses # that are not configured on this system. See https://vyos.dev/T850 @@ -101,41 +83,6 @@ def get_config(config=None): tmp = {'::1': {'port': '161'}} snmp['listen_address'] = dict_merge(tmp, snmp['listen_address']) - if 'community' in snmp: - default_values = defaults(base + ['community']) - if 'network' in default_values: - # convert multiple default networks to list - default_values['network'] = default_values['network'].split() - for community in snmp['community']: - snmp['community'][community] = dict_merge( - default_values, snmp['community'][community]) - - if 'trap_target' in snmp: - default_values = defaults(base + ['trap-target']) - for trap in snmp['trap_target']: - snmp['trap_target'][trap] = dict_merge( - default_values, snmp['trap_target'][trap]) - - if 'v3' in snmp: - default_values = defaults(base + ['v3']) - # tagNodes need to be merged in individually later on - for tmp in ['user', 'group', 'trap_target']: - del default_values[tmp] - snmp['v3'] = dict_merge(default_values, snmp['v3']) - - for user_group in ['user', 'group']: - if user_group in snmp['v3']: - default_values = defaults(base + ['v3', user_group]) - for tmp in snmp['v3'][user_group]: - snmp['v3'][user_group][tmp] = dict_merge( - default_values, snmp['v3'][user_group][tmp]) - - if 'trap_target' in snmp['v3']: - default_values = defaults(base + ['v3', 'trap-target']) - for trap in snmp['v3']['trap_target']: - snmp['v3']['trap_target'][trap] = dict_merge( - default_values, snmp['v3']['trap_target'][trap]) - return snmp def verify(snmp): diff --git a/src/conf_mode/ssh.py b/src/conf_mode/ssh.py index 3b63fcb7d..ee5e1eca2 100755 --- a/src/conf_mode/ssh.py +++ b/src/conf_mode/ssh.py @@ -21,12 +21,10 @@ from syslog import syslog from syslog import LOG_INFO from vyos.config import Config -from vyos.configdict import dict_merge from vyos.configdict import is_node_changed from vyos.configverify import verify_vrf from vyos.utils.process import call from vyos.template import render -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -57,8 +55,8 @@ def get_config(config=None): # We have gathered the dict representation of the CLI, but there are default # options which we need to update into the dictionary retrived. - default_values = defaults(base) - ssh = dict_merge(default_values, ssh) + ssh = conf.merge_defaults(ssh, recursive=True) + # pass config file path - used in override template ssh['config_file'] = config_file diff --git a/src/conf_mode/system-ip.py b/src/conf_mode/system-ip.py index cca996e4f..63dff0e36 100755 --- a/src/conf_mode/system-ip.py +++ b/src/conf_mode/system-ip.py @@ -24,7 +24,6 @@ from vyos.utils.process import call from vyos.utils.dict import dict_search from vyos.utils.file import write_file from vyos.utils.system import sysctl_write -from vyos.xml import defaults from vyos import ConfigError from vyos import frr from vyos import airbag @@ -37,11 +36,9 @@ def get_config(config=None): conf = Config() base = ['system', 'ip'] - opt = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - opt = dict_merge(default_values, opt) + opt = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True) # When working with FRR we need to know the corresponding address-family opt['afi'] = 'ip' diff --git a/src/conf_mode/system-ipv6.py b/src/conf_mode/system-ipv6.py index 22210c27a..8a4df11fa 100755 --- a/src/conf_mode/system-ipv6.py +++ b/src/conf_mode/system-ipv6.py @@ -24,7 +24,6 @@ from vyos.template import render_to_string from vyos.utils.dict import dict_search from vyos.utils.system import sysctl_write from vyos.utils.file import write_file -from vyos.xml import defaults from vyos import ConfigError from vyos import frr from vyos import airbag @@ -37,12 +36,9 @@ def get_config(config=None): conf = Config() base = ['system', 'ipv6'] - opt = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - opt = dict_merge(default_values, opt) + opt = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True) # When working with FRR we need to know the corresponding address-family opt['afi'] = 'ipv6' diff --git a/src/conf_mode/system-login.py b/src/conf_mode/system-login.py index 82941e0c0..02c97afaa 100755 --- a/src/conf_mode/system-login.py +++ b/src/conf_mode/system-login.py @@ -24,7 +24,6 @@ from sys import exit from time import sleep from vyos.config import Config -from vyos.configdict import dict_merge from vyos.configverify import verify_vrf from vyos.defaults import directories from vyos.template import render @@ -35,7 +34,6 @@ from vyos.utils.process import call from vyos.utils.process import rc_cmd from vyos.utils.process import run from vyos.utils.process import DEVNULL -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -93,7 +91,9 @@ def get_config(config=None): conf = Config() base = ['system', 'login'] login = conf.get_config_dict(base, key_mangling=('-', '_'), - no_tag_node_value_mangle=True, get_first_key=True) + no_tag_node_value_mangle=True, + get_first_key=True, + with_recursive_defaults=True) # users no longer existing in the running configuration need to be deleted local_users = get_local_users() @@ -101,27 +101,9 @@ def get_config(config=None): if 'user' in login: cli_users = list(login['user']) - # XXX: T2665: we can not safely rely on the defaults() when there are - # tagNodes in place, it is better to blend in the defaults manually. - default_values = defaults(base + ['user']) - for user in login['user']: - login['user'][user] = dict_merge(default_values, login['user'][user]) - - # Add TACACS global defaults - if 'tacacs' in login: - default_values = defaults(base + ['tacacs']) - if 'server' in default_values: - del default_values['server'] - login['tacacs'] = dict_merge(default_values, login['tacacs']) - - # XXX: T2665: we can not safely rely on the defaults() when there are - # tagNodes in place, it is better to blend in the defaults manually. - for backend in ['radius', 'tacacs']: - default_values = defaults(base + [backend, 'server']) - for server in dict_search(f'{backend}.server', login) or []: - login[backend]['server'][server] = dict_merge(default_values, - login[backend]['server'][server]) - + # prune TACACS global defaults if not set by user + if login.from_defaults(['tacacs']): + del login['tacacs'] # create a list of all users, cli and users all_users = list(set(local_users + cli_users)) diff --git a/src/conf_mode/system-logs.py b/src/conf_mode/system-logs.py index 12145d641..8ad4875d4 100755 --- a/src/conf_mode/system-logs.py +++ b/src/conf_mode/system-logs.py @@ -19,11 +19,9 @@ from sys import exit from vyos import ConfigError from vyos import airbag from vyos.config import Config -from vyos.configdict import dict_merge from vyos.logger import syslog from vyos.template import render from vyos.utils.dict import dict_search -from vyos.xml import defaults airbag.enable() # path to logrotate configs @@ -38,11 +36,9 @@ def get_config(config=None): conf = Config() base = ['system', 'logs'] - default_values = defaults(base) - logs_config = conf.get_config_dict(base, - key_mangling=('-', '_'), - get_first_key=True) - logs_config = dict_merge(default_values, logs_config) + logs_config = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True) return logs_config diff --git a/src/conf_mode/system-option.py b/src/conf_mode/system-option.py index 917013651..d92121b3d 100755 --- a/src/conf_mode/system-option.py +++ b/src/conf_mode/system-option.py @@ -21,14 +21,12 @@ from sys import exit from time import sleep from vyos.config import Config -from vyos.configdict import dict_merge from vyos.configverify import verify_source_interface from vyos.template import render from vyos.utils.process import cmd from vyos.utils.process import is_systemd_service_running from vyos.utils.network import is_addr_assigned from vyos.utils.network import is_intf_addr_assigned -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -48,12 +46,9 @@ def get_config(config=None): else: conf = Config() base = ['system', 'option'] - options = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - options = dict_merge(default_values, options) + options = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True) return options diff --git a/src/conf_mode/system-syslog.py b/src/conf_mode/system-syslog.py index 19c87bcee..07fbb0734 100755 --- a/src/conf_mode/system-syslog.py +++ b/src/conf_mode/system-syslog.py @@ -19,12 +19,10 @@ import os from sys import exit from vyos.config import Config -from vyos.configdict import dict_merge from vyos.configdict import is_node_changed from vyos.configverify import verify_vrf from vyos.utils.process import call from vyos.template import render -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -50,43 +48,9 @@ def get_config(config=None): tmp = is_node_changed(conf, base + ['vrf']) if tmp: syslog.update({'restart_required': {}}) - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - # XXX: some syslog default values can not be merged here (originating from - # a tagNode - remove and add them later per individual tagNode instance - if 'console' in default_values: - del default_values['console'] - for entity in ['global', 'user', 'host', 'file']: - if entity in default_values: - del default_values[entity] - - syslog = dict_merge(default_values, syslog) - - # XXX: add defaults for "console" tree - if 'console' in syslog and 'facility' in syslog['console']: - default_values = defaults(base + ['console', 'facility']) - for facility in syslog['console']['facility']: - syslog['console']['facility'][facility] = dict_merge(default_values, - syslog['console']['facility'][facility]) - - # XXX: add defaults for "host" tree - for syslog_type in ['host', 'user', 'file']: - # Bail out early if there is nothing to do - if syslog_type not in syslog: - continue - - default_values_host = defaults(base + [syslog_type]) - if 'facility' in default_values_host: - del default_values_host['facility'] - - for tmp, tmp_config in syslog[syslog_type].items(): - syslog[syslog_type][tmp] = dict_merge(default_values_host, syslog[syslog_type][tmp]) - if 'facility' in tmp_config: - default_values_facility = defaults(base + [syslog_type, 'facility']) - for facility in tmp_config['facility']: - syslog[syslog_type][tmp]['facility'][facility] = dict_merge(default_values_facility, - syslog[syslog_type][tmp]['facility'][facility]) + syslog = conf.merge_defaults(syslog, recursive=True) + if syslog.from_defaults(['global']): + del syslog['global'] return syslog diff --git a/src/conf_mode/system_console.py b/src/conf_mode/system_console.py index 87d587959..ebf9a113b 100755 --- a/src/conf_mode/system_console.py +++ b/src/conf_mode/system_console.py @@ -19,12 +19,10 @@ import re from pathlib import Path from vyos.config import Config -from vyos.configdict import dict_merge from vyos.utils.process import call from vyos.utils.file import read_file from vyos.utils.file import write_file from vyos.template import render -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -45,16 +43,12 @@ def get_config(config=None): if 'device' not in console: return console - # convert CLI values to system values - default_values = defaults(base + ['device']) for device, device_config in console['device'].items(): if 'speed' not in device_config and device.startswith('hvc'): # XEN console has a different default console speed console['device'][device]['speed'] = 38400 - else: - # Merge in XML defaults - the proper way to do it - console['device'][device] = dict_merge(default_values, - console['device'][device]) + + console = conf.merge_defaults(console, recursive=True) return console diff --git a/src/conf_mode/system_sflow.py b/src/conf_mode/system_sflow.py index eae869a6d..2df1bbb7a 100755 --- a/src/conf_mode/system_sflow.py +++ b/src/conf_mode/system_sflow.py @@ -19,11 +19,9 @@ import os from sys import exit from vyos.config import Config -from vyos.configdict import dict_merge from vyos.template import render from vyos.utils.process import call from vyos.utils.network import is_addr_assigned -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -42,26 +40,9 @@ def get_config(config=None): if not conf.exists(base): return None - sflow = conf.get_config_dict(base, - key_mangling=('-', '_'), - get_first_key=True) - - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - - sflow = dict_merge(default_values, sflow) - - # Ignore default XML values if config doesn't exists - # Delete key from dict - if 'port' in sflow['server']: - del sflow['server']['port'] - - # Set default values per server - if 'server' in sflow: - for server in sflow['server']: - default_values = defaults(base + ['server']) - sflow['server'][server] = dict_merge(default_values, sflow['server'][server]) + sflow = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True) return sflow diff --git a/src/conf_mode/tftp_server.py b/src/conf_mode/tftp_server.py index 32882fc12..3ad346e2e 100755 --- a/src/conf_mode/tftp_server.py +++ b/src/conf_mode/tftp_server.py @@ -24,14 +24,12 @@ from sys import exit from vyos.base import Warning from vyos.config import Config -from vyos.configdict import dict_merge from vyos.configverify import verify_vrf from vyos.template import render from vyos.template import is_ipv4 from vyos.utils.process import call from vyos.utils.permission import chmod_755 from vyos.utils.network import is_addr_assigned -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -48,11 +46,9 @@ def get_config(config=None): if not conf.exists(base): return None - tftpd = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - tftpd = dict_merge(default_values, tftpd) + tftpd = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True) return tftpd def verify(tftpd): diff --git a/src/conf_mode/vpn_ipsec.py b/src/conf_mode/vpn_ipsec.py index 9a27a44bf..fa271cbdb 100755 --- a/src/conf_mode/vpn_ipsec.py +++ b/src/conf_mode/vpn_ipsec.py @@ -27,7 +27,6 @@ from vyos.base import Warning from vyos.config import Config from vyos.configdict import leaf_node_changed from vyos.configverify import verify_interface_exists -from vyos.configdict import dict_merge from vyos.defaults import directories from vyos.ifconfig import Interface from vyos.pki import encode_public_key @@ -45,7 +44,6 @@ from vyos.utils.dict import dict_search from vyos.utils.dict import dict_search_args from vyos.utils.process import call from vyos.utils.process import run -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag airbag.enable() @@ -84,88 +82,23 @@ def get_config(config=None): # retrieve common dictionary keys ipsec = conf.get_config_dict(base, key_mangling=('-', '_'), - get_first_key=True, no_tag_node_value_mangle=True) - - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - # XXX: T2665: we must safely remove default values for tag nodes, those are - # added in a more fine grained way later on - del default_values['esp_group'] - del default_values['ike_group'] - del default_values['remote_access'] - del default_values['site_to_site'] - ipsec = dict_merge(default_values, ipsec) - - if 'esp_group' in ipsec: - default_values = defaults(base + ['esp-group']) - for group in ipsec['esp_group']: - ipsec['esp_group'][group] = dict_merge(default_values, - ipsec['esp_group'][group]) - if 'ike_group' in ipsec: - default_values = defaults(base + ['ike-group']) - # proposal is a tag node which may come with individual defaults per node - if 'proposal' in default_values: - del default_values['proposal'] - - for group in ipsec['ike_group']: - ipsec['ike_group'][group] = dict_merge(default_values, - ipsec['ike_group'][group]) - - if 'proposal' in ipsec['ike_group'][group]: - default_values = defaults(base + ['ike-group', 'proposal']) - for proposal in ipsec['ike_group'][group]['proposal']: - ipsec['ike_group'][group]['proposal'][proposal] = dict_merge(default_values, - ipsec['ike_group'][group]['proposal'][proposal]) - - # XXX: T2665: we can not safely rely on the defaults() when there are - # tagNodes in place, it is better to blend in the defaults manually. - if dict_search('remote_access.connection', ipsec): - default_values = defaults(base + ['remote-access', 'connection']) - for rw in ipsec['remote_access']['connection']: - ipsec['remote_access']['connection'][rw] = dict_merge(default_values, - ipsec['remote_access']['connection'][rw]) - - # XXX: T2665: we can not safely rely on the defaults() when there are - # tagNodes in place, it is better to blend in the defaults manually. - if dict_search('remote_access.radius.server', ipsec): - # Fist handle the "base" stuff like RADIUS timeout - default_values = defaults(base + ['remote-access', 'radius']) - if 'server' in default_values: - del default_values['server'] - ipsec['remote_access']['radius'] = dict_merge(default_values, - ipsec['remote_access']['radius']) - - # Take care about individual RADIUS servers implemented as tagNodes - this - # requires special treatment - default_values = defaults(base + ['remote-access', 'radius', 'server']) - for server in ipsec['remote_access']['radius']['server']: - ipsec['remote_access']['radius']['server'][server] = dict_merge(default_values, - ipsec['remote_access']['radius']['server'][server]) - - # XXX: T2665: we can not safely rely on the defaults() when there are - # tagNodes in place, it is better to blend in the defaults manually. - if dict_search('site_to_site.peer', ipsec): - default_values = defaults(base + ['site-to-site', 'peer']) - for peer in ipsec['site_to_site']['peer']: - ipsec['site_to_site']['peer'][peer] = dict_merge(default_values, - ipsec['site_to_site']['peer'][peer]) + no_tag_node_value_mangle=True, + get_first_key=True, + with_recursive_defaults=True) ipsec['dhcp_no_address'] = {} ipsec['install_routes'] = 'no' if conf.exists(base + ["options", "disable-route-autoinstall"]) else default_install_routes ipsec['interface_change'] = leaf_node_changed(conf, base + ['interface']) ipsec['nhrp_exists'] = conf.exists(['protocols', 'nhrp', 'tunnel']) ipsec['pki'] = conf.get_config_dict(['pki'], key_mangling=('-', '_'), - get_first_key=True, - no_tag_node_value_mangle=True) + no_tag_node_value_mangle=True, + get_first_key=True) tmp = conf.get_config_dict(l2tp_base, key_mangling=('-', '_'), - get_first_key=True, - no_tag_node_value_mangle=True) + no_tag_node_value_mangle=True, + get_first_key=True) if tmp: - ipsec['l2tp'] = tmp - l2tp_defaults = defaults(l2tp_base) - ipsec['l2tp'] = dict_merge(l2tp_defaults, ipsec['l2tp']) + ipsec['l2tp'] = conf.merge_defaults(tmp, recursive=True) ipsec['l2tp_outside_address'] = conf.return_value(['vpn', 'l2tp', 'remote-access', 'outside-address']) ipsec['l2tp_ike_default'] = 'aes256-sha1-modp1024,3des-sha1-modp1024' ipsec['l2tp_esp_default'] = 'aes256-sha1,3des-sha1' diff --git a/src/conf_mode/vpn_openconnect.py b/src/conf_mode/vpn_openconnect.py index e82862fa3..a039172c4 100755 --- a/src/conf_mode/vpn_openconnect.py +++ b/src/conf_mode/vpn_openconnect.py @@ -19,7 +19,6 @@ from sys import exit from vyos.base import Warning from vyos.config import Config -from vyos.configdict import dict_merge from vyos.pki import wrap_certificate from vyos.pki import wrap_private_key from vyos.template import render @@ -28,7 +27,6 @@ from vyos.utils.network import check_port_availability from vyos.utils.process import is_systemd_service_running from vyos.utils.network import is_listen_port_bind_service from vyos.utils.dict import dict_search -from vyos.xml import defaults from vyos import ConfigError from passlib.hash import sha512_crypt from time import sleep @@ -47,66 +45,6 @@ radius_servers = cfg_dir + '/radius_servers' def get_hash(password): return sha512_crypt.hash(password) - - -def _default_dict_cleanup(origin: dict, default_values: dict) -> dict: - """ - https://vyos.dev/T2665 - Clear unnecessary key values in merged config by dict_merge function - :param origin: config - :type origin: dict - :param default_values: default values - :type default_values: dict - :return: merged dict - :rtype: dict - """ - if 'mode' in origin["authentication"] and "local" in \ - origin["authentication"]["mode"]: - del origin['authentication']['local_users']['username']['otp'] - if not origin["authentication"]["local_users"]["username"]: - raise ConfigError( - 'Openconnect authentication mode local requires at least one user') - default_ocserv_usr_values = \ - default_values['authentication']['local_users']['username']['otp'] - for user, params in origin['authentication']['local_users'][ - 'username'].items(): - # Not every configuration requires OTP settings - if origin['authentication']['local_users']['username'][user].get( - 'otp'): - origin['authentication']['local_users']['username'][user][ - 'otp'] = dict_merge(default_ocserv_usr_values, - origin['authentication'][ - 'local_users']['username'][user][ - 'otp']) - - if 'mode' in origin["authentication"] and "radius" in \ - origin["authentication"]["mode"]: - del origin['authentication']['radius']['server']['port'] - if not origin["authentication"]['radius']['server']: - raise ConfigError( - 'Openconnect authentication mode radius requires at least one RADIUS server') - default_values_radius_port = \ - default_values['authentication']['radius']['server']['port'] - for server, params in origin['authentication']['radius'][ - 'server'].items(): - if 'port' not in params: - params['port'] = default_values_radius_port - - if 'mode' in origin["accounting"] and "radius" in \ - origin["accounting"]["mode"]: - del origin['accounting']['radius']['server']['port'] - if not origin["accounting"]['radius']['server']: - raise ConfigError( - 'Openconnect accounting mode radius requires at least one RADIUS server') - default_values_radius_port = \ - default_values['accounting']['radius']['server']['port'] - for server, params in origin['accounting']['radius'][ - 'server'].items(): - if 'port' not in params: - params['port'] = default_values_radius_port - return origin - - def get_config(config=None): if config: conf = config @@ -116,16 +54,14 @@ def get_config(config=None): if not conf.exists(base): return None - ocserv = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - ocserv = dict_merge(default_values, ocserv) - # workaround a "know limitation" - https://vyos.dev/T2665 - ocserv = _default_dict_cleanup(ocserv, default_values) + ocserv = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True) + if ocserv: ocserv['pki'] = conf.get_config_dict(['pki'], key_mangling=('-', '_'), - get_first_key=True, no_tag_node_value_mangle=True) + no_tag_node_value_mangle=True, + get_first_key=True) return ocserv @@ -142,6 +78,8 @@ def verify(ocserv): # Check accounting if "accounting" in ocserv: if "mode" in ocserv["accounting"] and "radius" in ocserv["accounting"]["mode"]: + if not origin["accounting"]['radius']['server']: + raise ConfigError('Openconnect accounting mode radius requires at least one RADIUS server') if "authentication" not in ocserv or "mode" not in ocserv["authentication"]: raise ConfigError('Accounting depends on OpenConnect authentication configuration') elif "radius" not in ocserv["authentication"]["mode"]: @@ -150,9 +88,13 @@ def verify(ocserv): # Check authentication if "authentication" in ocserv: if "mode" in ocserv["authentication"]: - if "local" in ocserv["authentication"]["mode"]: - if "radius" in ocserv["authentication"]["mode"]: + if ("local" in ocserv["authentication"]["mode"] and + "radius" in ocserv["authentication"]["mode"]): raise ConfigError('OpenConnect authentication modes are mutually-exclusive, remove either local or radius from your configuration') + if "radius" in ocserv["authentication"]["mode"]: + if not ocserv["authentication"]['radius']['server']: + raise ConfigError('Openconnect authentication mode radius requires at least one RADIUS server') + if "local" in ocserv["authentication"]["mode"]: if not ocserv["authentication"]["local_users"]: raise ConfigError('openconnect mode local required at least one user') if not ocserv["authentication"]["local_users"]["username"]: diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py index 80ce1e8e3..82c2f236e 100755 --- a/src/conf_mode/vpp.py +++ b/src/conf_mode/vpp.py @@ -22,7 +22,6 @@ from re import search as re_search, MULTILINE as re_M from vyos.config import Config from vyos.configdep import set_dependents, call_dependents -from vyos.configdict import dict_merge from vyos.configdict import node_changed from vyos.ifconfig import Section from vyos.utils.boot import boot_configuration_complete @@ -31,7 +30,6 @@ from vyos.utils.process import rc_cmd from vyos.utils.system import sysctl_read from vyos.utils.system import sysctl_apply from vyos.template import render -from vyos.xml import defaults from vyos import ConfigError from vyos import airbag @@ -94,28 +92,18 @@ def get_config(config=None): if not conf.exists(base): return {'removed_ifaces': removed_ifaces} - config = conf.get_config_dict(base, + config = conf.get_config_dict(base, key_mangling=('-', '_'), + no_tag_node_value_mangle=True, get_first_key=True, - key_mangling=('-', '_'), - no_tag_node_value_mangle=True) - - # We have gathered the dict representation of the CLI, but there are default - # options which we need to update into the dictionary retrived. - default_values = defaults(base) - if 'interface' in default_values: - del default_values['interface'] - config = dict_merge(default_values, config) + with_recursive_defaults=True) if 'interface' in config: for iface, iface_config in config['interface'].items(): - default_values_iface = defaults(base + ['interface']) - config['interface'][iface] = dict_merge(default_values_iface, config['interface'][iface]) # add an interface to a list of interfaces that need # to be reinitialized after the commit set_dependents('ethernet', conf, iface) - # Get PCI address auto - for iface, iface_config in config['interface'].items(): + # Get PCI address auto if iface_config['pci'] == 'auto': config['interface'][iface]['pci'] = _get_pci_address_by_interface(iface) |