From 1a18d5a59d14877984fe00d6257ba8bd5d5036d4 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 16 May 2023 21:18:34 -0500 Subject: config: T5228: simplify mangle_dict_keys --- python/vyos/utils/dict.py | 43 +++++++++++++++++++------------------------ 1 file changed, 19 insertions(+), 24 deletions(-) (limited to 'python') diff --git a/python/vyos/utils/dict.py b/python/vyos/utils/dict.py index 7c93deef6..8fb519a5c 100644 --- a/python/vyos/utils/dict.py +++ b/python/vyos/utils/dict.py @@ -65,7 +65,7 @@ def colon_separated_to_dict(data_string, uniquekeys=False): return data -def _mangle_dict_keys(data, regex, replacement, abs_path=[], no_tag_node_value_mangle=False, mod=0): +def mangle_dict_keys(data, regex, replacement, abs_path=None, no_tag_node_value_mangle=False): """ Mangles dict keys according to a regex and replacement character. Some libraries like Jinja2 do not like certain characters in dict keys. This function can be used for replacing all offending characters @@ -73,44 +73,39 @@ def _mangle_dict_keys(data, regex, replacement, abs_path=[], no_tag_node_value_m Args: data (dict): Original dict to mangle + regex, replacement (str): arguments to re.sub(regex, replacement, ...) + abs_path (list): if data is a config dict and no_tag_node_value_mangle is True + then abs_path should be the absolute config path to the first + keys of data, non-inclusive + no_tag_node_value_mangle (bool): do not mangle keys of tag node values Returns: dict """ - from vyos.xml import is_tag - - new_dict = {} + import re + from vyos.xml_ref import is_tag_value - for key in data.keys(): - save_mod = mod - save_path = abs_path[:] + if abs_path is None: + abs_path = [] - abs_path.append(key) + new_dict = {} - if not is_tag(abs_path): - new_key = re.sub(regex, replacement, key) + for k in data.keys(): + if no_tag_node_value_mangle and is_tag_value(abs_path + [k]): + new_key = k else: - if mod%2: - new_key = key - else: - new_key = re.sub(regex, replacement, key) - if no_tag_node_value_mangle: - mod += 1 + new_key = re.sub(regex, replacement, k) - value = data[key] + value = data[k] if isinstance(value, dict): - new_dict[new_key] = _mangle_dict_keys(value, regex, replacement, abs_path=abs_path, mod=mod, no_tag_node_value_mangle=no_tag_node_value_mangle) + new_dict[new_key] = mangle_dict_keys(value, regex, replacement, + abs_path=abs_path + [k], + no_tag_node_value_mangle=no_tag_node_value_mangle) else: new_dict[new_key] = value - mod = save_mod - abs_path = save_path[:] - return new_dict -def mangle_dict_keys(data, regex, replacement, abs_path=[], no_tag_node_value_mangle=False): - return _mangle_dict_keys(data, regex, replacement, abs_path=abs_path, no_tag_node_value_mangle=no_tag_node_value_mangle, mod=0) - def _get_sub_dict(d, lpath): k = lpath[0] if k not in d.keys(): -- cgit v1.2.3 From f2f6b963b755ca5da3321e84738bfec1d08fb1ea Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 2 Jun 2023 23:29:47 -0500 Subject: xml: T5218: fix error and simplify logic in recursive option --- python/vyos/xml_ref/__init__.py | 7 ++- python/vyos/xml_ref/definition.py | 103 ++++++++++++++++++++------------------ 2 files changed, 60 insertions(+), 50 deletions(-) (limited to 'python') diff --git a/python/vyos/xml_ref/__init__.py b/python/vyos/xml_ref/__init__.py index ae5184746..d3fb4ab07 100644 --- a/python/vyos/xml_ref/__init__.py +++ b/python/vyos/xml_ref/__init__.py @@ -62,5 +62,8 @@ def get_config_defaults(rpath: list, conf: dict, get_first_key=False, get_first_key=get_first_key, recursive=recursive) -def merge_defaults(path: list, conf: dict) -> dict: - return load_reference().merge_defaults(path, conf) +def merge_defaults(path: list, conf: dict, get_first_key=False, + recursive=False) -> dict: + return load_reference().merge_defaults(path, conf, + get_first_key=get_first_key, + recursive=recursive) diff --git a/python/vyos/xml_ref/definition.py b/python/vyos/xml_ref/definition.py index 429331577..970dd915f 100644 --- a/python/vyos/xml_ref/definition.py +++ b/python/vyos/xml_ref/definition.py @@ -13,7 +13,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see . -from typing import Union, Any +from typing import Optional, Union, Any from vyos.configdict import dict_merge class Xml: @@ -116,9 +116,17 @@ class Xml: return res - def _get_default_value(self, node: dict): + def _get_default_value(self, node: dict) -> Optional[str]: return self._get_ref_node_data(node, "default_value") + def _get_default(self, node: dict) -> Optional[Union[str, list]]: + default = self._get_default_value(node) + if default is None: + return None + if self._is_multi_node(node) and not isinstance(default, list): + return [default] + return default + def get_defaults(self, path: list, get_first_key=False, recursive=False) -> dict: """Return dict containing default values below path @@ -128,18 +136,23 @@ class Xml: 'relative_defaults' """ res: dict = {} + if self.is_tag(path): + return res + d = self._get_ref_path(path) + + if self._is_leaf_node(d): + default_value = self._get_default(d) + if default_value is not None: + return {path[-1]: default_value} if path else {} + for k in list(d): if k in ('node_data', 'component_version') : continue - d_k = d[k] - if self._is_leaf_node(d_k): - default_value = self._get_default_value(d_k) + if self._is_leaf_node(d[k]): + default_value = self._get_default(d[k]) if default_value is not None: - pos = default_value - if self._is_multi_node(d_k) and not isinstance(pos, list): - pos = [pos] - res |= {k: pos} + res |= {k: default_value} elif self.is_tag(path + [k]): # tag node defaults are used as suggestion, not default value; # should this change, append to path and continue if recursive @@ -150,8 +163,6 @@ class Xml: res |= pos if res: if get_first_key or not path: - if not isinstance(res, dict): - raise TypeError("Cannot get_first_key as data under node is not of type dict") return res return {path[-1]: res} @@ -163,7 +174,7 @@ class Xml: return [next(iter(c.keys()))] if c else [] try: tmp = step(conf) - if self.is_tag_value(path + tmp): + if tmp and self.is_tag_value(path + tmp): c = conf[tmp[0]] if not isinstance(c, dict): raise ValueError @@ -175,57 +186,53 @@ class Xml: return False return True - def relative_defaults(self, rpath: list, conf: dict, get_first_key=False, + def _relative_defaults(self, rpath: list, conf: dict, recursive=False) -> dict: + res: dict = {} + res = self.get_defaults(rpath, recursive=recursive, + get_first_key=True) + for k in list(conf): + if isinstance(conf[k], dict): + step = self._relative_defaults(rpath + [k], conf=conf[k], + recursive=recursive) + res |= step + + if res: + return {rpath[-1]: res} if rpath else res + + return {} + + def relative_defaults(self, path: list, conf: dict, get_first_key=False, recursive=False) -> dict: """Return dict containing defaults along paths of a config dict """ if not conf: - return self.get_defaults(rpath, get_first_key=get_first_key, + return self.get_defaults(path, get_first_key=get_first_key, recursive=recursive) - if rpath and rpath[-1] in list(conf): - conf = conf[rpath[-1]] - if not isinstance(conf, dict): - raise TypeError('conf at path is not of type dict') + if path and path[-1] in list(conf): + conf = conf[path[-1]] + conf = {} if not isinstance(conf, dict) else conf - if not self._well_defined(rpath, conf): + if not self._well_defined(path, conf): print('path to config dict does not define full config paths') return {} - res: dict = {} - for k in list(conf): - pos = self.get_defaults(rpath + [k], recursive=recursive) - res |= pos - - if isinstance(conf[k], dict): - step = self.relative_defaults(rpath + [k], conf=conf[k], - recursive=recursive) - res |= step + res = self._relative_defaults(path, conf, recursive=recursive) - if res: - if get_first_key: - return res - return {rpath[-1]: res} if rpath else res + if get_first_key and path: + if res.values(): + res = next(iter(res.values())) + else: + res = {} - return {} + return res - def merge_defaults(self, path: list, conf: dict) -> dict: + def merge_defaults(self, path: list, conf: dict, get_first_key=False, + recursive=False) -> dict: """Return config dict with defaults non-destructively merged This merges non-recursive defaults relative to the config dict. """ - if path[-1] in list(conf): - config = conf[path[-1]] - if not isinstance(config, dict): - raise TypeError('conf at path is not of type dict') - shift = False - else: - config = conf - shift = True - - if not self._well_defined(path, config): - print('path to config dict does not define config paths; conf returned unchanged') - return conf - - d = self.relative_defaults(path, conf=config, get_first_key=shift) + d = self.relative_defaults(path, conf, get_first_key=get_first_key, + recursive=recursive) d = dict_merge(d, conf) return d -- cgit v1.2.3 From d57b7d327cef20dc607550bda1c5632c3d24a719 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 2 Jun 2023 23:29:56 -0500 Subject: config: T5228: use local _dict_merge to avoid circular import --- python/vyos/xml_ref/definition.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'python') diff --git a/python/vyos/xml_ref/definition.py b/python/vyos/xml_ref/definition.py index 970dd915f..92d069f05 100644 --- a/python/vyos/xml_ref/definition.py +++ b/python/vyos/xml_ref/definition.py @@ -14,7 +14,6 @@ # along with this library. If not, see . from typing import Optional, Union, Any -from vyos.configdict import dict_merge class Xml: def __init__(self): @@ -186,6 +185,20 @@ class Xml: return False return True + # use local copy of function in module configdict, to avoid circular + # import + def _dict_merge(self, source, destination): + from copy import deepcopy + tmp = deepcopy(destination) + + for key, value in source.items(): + if key not in tmp: + tmp[key] = value + elif isinstance(source[key], dict): + tmp[key] = self._dict_merge(source[key], tmp[key]) + + return tmp + def _relative_defaults(self, rpath: list, conf: dict, recursive=False) -> dict: res: dict = {} res = self.get_defaults(rpath, recursive=recursive, @@ -234,5 +247,5 @@ class Xml: """ d = self.relative_defaults(path, conf, get_first_key=get_first_key, recursive=recursive) - d = dict_merge(d, conf) + d = self._dict_merge(d, conf) return d -- cgit v1.2.3 From bf1fd02453b9b82938656ac2037a61fd45ba96a2 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 2 Jun 2023 23:31:21 -0500 Subject: config: T5228: add arg with_defaults to get_config_dict --- python/vyos/config.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) (limited to 'python') diff --git a/python/vyos/config.py b/python/vyos/config.py index 287fd2ed1..1b9e61651 100644 --- a/python/vyos/config.py +++ b/python/vyos/config.py @@ -1,4 +1,4 @@ -# Copyright 2017, 2019 VyOS maintainers and contributors +# Copyright 2017, 2019-2023 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -67,9 +67,9 @@ import re import json from copy import deepcopy -import vyos.xml -import vyos.util import vyos.configtree +from vyos.xml_ref import multi_to_list, merge_defaults +from vyos.utils.dict import get_sub_dict, mangle_dict_keys from vyos.configsource import ConfigSource, ConfigSourceSession class Config(object): @@ -225,7 +225,8 @@ class Config(object): def get_config_dict(self, path=[], effective=False, key_mangling=None, get_first_key=False, no_multi_convert=False, - no_tag_node_value_mangle=False): + no_tag_node_value_mangle=False, + with_defaults=False, with_recursive_defaults=False): """ Args: path (str list): Configuration tree path, can be empty @@ -238,19 +239,23 @@ class Config(object): """ lpath = self._make_path(path) root_dict = self.get_cached_root_dict(effective) - conf_dict = vyos.util.get_sub_dict(root_dict, lpath, get_first_key) + conf_dict = get_sub_dict(root_dict, lpath, get_first_key) - if not key_mangling and no_multi_convert: + if key_mangling is None and no_multi_convert and not with_defaults: return deepcopy(conf_dict) - xmlpath = lpath if get_first_key else lpath[:-1] + rpath = lpath if get_first_key else lpath[:-1] - if not key_mangling: - conf_dict = vyos.xml.multi_to_list(xmlpath, conf_dict) - return conf_dict + if not no_multi_convert: + conf_dict = multi_to_list(rpath, conf_dict) + + if with_defaults or with_recursive_defaults: + conf_dict = merge_defaults(lpath, conf_dict, + get_first_key=get_first_key, + recursive=with_recursive_defaults) - if no_multi_convert is False: - conf_dict = vyos.xml.multi_to_list(xmlpath, conf_dict) + if key_mangling is None: + return conf_dict if not (isinstance(key_mangling, tuple) and \ (len(key_mangling) == 2) and \ @@ -258,7 +263,7 @@ class Config(object): isinstance(key_mangling[1], str)): raise ValueError("key_mangling must be a tuple of two strings") - conf_dict = vyos.util.mangle_dict_keys(conf_dict, key_mangling[0], key_mangling[1], abs_path=xmlpath, no_tag_node_value_mangle=no_tag_node_value_mangle) + conf_dict = mangle_dict_keys(conf_dict, key_mangling[0], key_mangling[1], abs_path=rpath, no_tag_node_value_mangle=no_tag_node_value_mangle) return conf_dict -- cgit v1.2.3 From d19e7e5ce633f42f2a084ebd17e8c1e3dfef6b03 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 2 Jun 2023 23:35:00 -0500 Subject: config: T5228: add get_config_defaults options to match get_config_dict For those cases not covered by automatic merging of defaults in get_config_dict(..., with_defaults=True), get_config_defaults should take arguments consistent with those of get_config_dict, for ease of merging results. --- python/vyos/config.py | 27 ++++++++++++++++++++++++++- python/vyos/xml_ref/__init__.py | 6 +++--- 2 files changed, 29 insertions(+), 4 deletions(-) (limited to 'python') diff --git a/python/vyos/config.py b/python/vyos/config.py index 1b9e61651..c3bb68373 100644 --- a/python/vyos/config.py +++ b/python/vyos/config.py @@ -68,7 +68,7 @@ import json from copy import deepcopy import vyos.configtree -from vyos.xml_ref import multi_to_list, merge_defaults +from vyos.xml_ref import multi_to_list, merge_defaults, relative_defaults from vyos.utils.dict import get_sub_dict, mangle_dict_keys from vyos.configsource import ConfigSource, ConfigSourceSession @@ -267,6 +267,31 @@ class Config(object): return conf_dict + def get_config_defaults(self, path=[], effective=False, key_mangling=None, + no_tag_node_value_mangle=False, get_first_key=False, + recursive=False) -> dict: + lpath = self._make_path(path) + root_dict = self.get_cached_root_dict(effective) + conf_dict = get_sub_dict(root_dict, lpath, get_first_key) + + defaults = relative_defaults(lpath, conf_dict, + get_first_key=get_first_key, + recursive=recursive) + if key_mangling is None: + return defaults + + rpath = lpath if get_first_key else lpath[:-1] + + if not (isinstance(key_mangling, tuple) and \ + (len(key_mangling) == 2) and \ + isinstance(key_mangling[0], str) and \ + isinstance(key_mangling[1], str)): + raise ValueError("key_mangling must be a tuple of two strings") + + defaults = mangle_dict_keys(defaults, key_mangling[0], key_mangling[1], abs_path=rpath, no_tag_node_value_mangle=no_tag_node_value_mangle) + + return defaults + def is_multi(self, path): """ Args: diff --git a/python/vyos/xml_ref/__init__.py b/python/vyos/xml_ref/__init__.py index d3fb4ab07..53ca6ed98 100644 --- a/python/vyos/xml_ref/__init__.py +++ b/python/vyos/xml_ref/__init__.py @@ -55,10 +55,10 @@ def get_defaults(path: list, get_first_key=False, recursive=False) -> dict: return load_reference().get_defaults(path, get_first_key=get_first_key, recursive=recursive) -def get_config_defaults(rpath: list, conf: dict, get_first_key=False, - recursive=False) -> dict: +def relative_defaults(rpath: list, conf: dict, get_first_key=False, + recursive=False) -> dict: - return load_reference().relative_defaults(rpath, conf=conf, + return load_reference().relative_defaults(rpath, conf, get_first_key=get_first_key, recursive=recursive) -- cgit v1.2.3