From b0b5654a1f99489e93ed255065818860c92045e5 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 8 Feb 2022 21:13:46 -0600 Subject: configtree: T4235: encapsulate config tree diff function (cherry picked from commit 2ee94418ce24429dbf6a52c2a327ed08a1935958) --- python/vyos/configtree.py | 65 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 13 deletions(-) (limited to 'python/vyos') diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index d8ffaca99..866f24e47 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -15,8 +15,9 @@ import re import json -from ctypes import cdll, c_char_p, c_void_p, c_int +from ctypes import cdll, c_char_p, c_void_p, c_int, POINTER +LIBPATH = '/usr/lib/libvyosconfig.so.0' def escape_backslash(string: str) -> str: """Escape single backslashes in string that are not in escape sequence""" @@ -42,7 +43,9 @@ class ConfigTreeError(Exception): class ConfigTree(object): - def __init__(self, config_string, libpath='/usr/lib/libvyosconfig.so.0'): + def __init__(self, config_string=None, address=None, libpath=LIBPATH): + if config_string is None and address is None: + raise TypeError("ConfigTree() requires one of 'config_string' or 'address'") self.__config = None self.__lib = cdll.LoadLibrary(libpath) @@ -60,7 +63,7 @@ class ConfigTree(object): self.__to_string.restype = c_char_p self.__to_commands = self.__lib.to_commands - self.__to_commands.argtypes = [c_void_p] + self.__to_commands.argtypes = [c_void_p, c_char_p] self.__to_commands.restype = c_char_p self.__to_json = self.__lib.to_json @@ -126,15 +129,19 @@ class ConfigTree(object): self.__destroy = self.__lib.destroy self.__destroy.argtypes = [c_void_p] - config_section, version_section = extract_version(config_string) - config_section = escape_backslash(config_section) - config = self.__from_string(config_section.encode()) - if config is None: - msg = self.__get_error().decode() - raise ValueError("Failed to parse config: {0}".format(msg)) + if address is None: + config_section, version_section = extract_version(config_string) + config_section = escape_backslash(config_section) + config = self.__from_string(config_section.encode()) + if config is None: + msg = self.__get_error().decode() + raise ValueError("Failed to parse config: {0}".format(msg)) + else: + self.__config = config + self.__version = version_section else: - self.__config = config - self.__version = version_section + self.__config = address + self.__version = '' def __del__(self): if self.__config is not None: @@ -143,13 +150,16 @@ class ConfigTree(object): def __str__(self): return self.to_string() + def _get_config(self): + return self.__config + def to_string(self): config_string = self.__to_string(self.__config).decode() config_string = "{0}\n{1}".format(config_string, self.__version) return config_string - def to_commands(self): - return self.__to_commands(self.__config).decode() + def to_commands(self, op="set"): + return self.__to_commands(self.__config, op.encode()).decode() def to_json(self): return self.__to_json(self.__config).decode() @@ -281,3 +291,32 @@ class ConfigTree(object): else: raise ConfigTreeError("Path [{}] doesn't exist".format(path_str)) +class Diff: + def __init__(self, left, right, path=[], libpath=LIBPATH): + if not (isinstance(left, ConfigTree) and isinstance(right, ConfigTree)): + raise TypeError("Arguments must be instances of ConfigTree") + if path: + if not left.exists(path): + raise ConfigTreeError(f"Path {path} doesn't exist in lhs tree") + if not right.exists(path): + raise ConfigTreeError(f"Path {path} doesn't exist in rhs tree") + self.left = left + self.right = right + + check_path(path) + path_str = " ".join(map(str, path)).encode() + df = cdll.LoadLibrary(libpath).diffs + df.restype = POINTER(c_void_p * 3) + res = list(df(path_str, left._get_config(), right._get_config()).contents) + self._diff = {'add': ConfigTree(address=res[0]), + 'del': ConfigTree(address=res[1]), + 'int': ConfigTree(address=res[2]) } + + self.add = self._diff['add'] + self.delete = self._diff['del'] + self.inter = self._diff['int'] + + def to_commands(self): + add = self.add.to_commands() + delete = self.delete.to_commands(op="delete") + return delete + "\n" + add -- cgit v1.2.3 From 5d299e9f9bfbcacec15834805dc276b5594bdabb Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 25 Feb 2022 16:05:03 -0600 Subject: configtree: T4235: add utility get_subtree (cherry picked from commit ff7e2cd622cf3679cd9265b2cb766395a1830f50) --- python/vyos/configtree.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'python/vyos') diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index 866f24e47..0ec15cf40 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -126,6 +126,10 @@ class ConfigTree(object): self.__set_tag.argtypes = [c_void_p, c_char_p] self.__set_tag.restype = c_int + self.__get_subtree = self.__lib.get_subtree + self.__get_subtree.argtypes = [c_void_p, c_char_p] + self.__get_subtree.restype = c_void_p + self.__destroy = self.__lib.destroy self.__destroy.argtypes = [c_void_p] @@ -291,6 +295,14 @@ class ConfigTree(object): else: raise ConfigTreeError("Path [{}] doesn't exist".format(path_str)) + def get_subtree(self, path, with_node=False): + check_path(path) + path_str = " ".join(map(str, path)).encode() + + res = self.__get_subtree(self.__config, path_str, with_node) + subt = ConfigTree(address=res) + return subt + class Diff: def __init__(self, left, right, path=[], libpath=LIBPATH): if not (isinstance(left, ConfigTree) and isinstance(right, ConfigTree)): -- cgit v1.2.3 From 4102ff9c7ca96f6056864a9560e3634fb3d8e34d Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 25 Feb 2022 16:10:12 -0600 Subject: configtree: T4235: simplification of diff_tree class The return value of diff_tree is now a single config_tree, with initial children of names: ["add", "delete", "inter"] containing the config sub-trees of added paths; deleted paths; and intersection, respectively. The simplifies dumping to json, and checking existence of paths, hence, of node changes. (cherry picked from commit 3d28528ff84b4e874faf80028709bd08b2956933) --- python/vyos/configtree.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) (limited to 'python/vyos') diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index 0ec15cf40..f5e697137 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -15,7 +15,7 @@ import re import json -from ctypes import cdll, c_char_p, c_void_p, c_int, POINTER +from ctypes import cdll, c_char_p, c_void_p, c_int LIBPATH = '/usr/lib/libvyosconfig.so.0' @@ -303,7 +303,7 @@ class ConfigTree(object): subt = ConfigTree(address=res) return subt -class Diff: +class DiffTree: def __init__(self, left, right, path=[], libpath=LIBPATH): if not (isinstance(left, ConfigTree) and isinstance(right, ConfigTree)): raise TypeError("Arguments must be instances of ConfigTree") @@ -312,21 +312,28 @@ class Diff: raise ConfigTreeError(f"Path {path} doesn't exist in lhs tree") if not right.exists(path): raise ConfigTreeError(f"Path {path} doesn't exist in rhs tree") + self.left = left self.right = right + self.__lib = cdll.LoadLibrary(libpath) + + self.__diff_tree = self.__lib.diff_tree + self.__diff_tree.argtypes = [c_char_p, c_void_p, c_void_p] + self.__diff_tree.restype = c_void_p + check_path(path) path_str = " ".join(map(str, path)).encode() - df = cdll.LoadLibrary(libpath).diffs - df.restype = POINTER(c_void_p * 3) - res = list(df(path_str, left._get_config(), right._get_config()).contents) - self._diff = {'add': ConfigTree(address=res[0]), - 'del': ConfigTree(address=res[1]), - 'int': ConfigTree(address=res[2]) } - - self.add = self._diff['add'] - self.delete = self._diff['del'] - self.inter = self._diff['int'] + res = self.__diff_tree(path_str, left._get_config(), right._get_config()) + + # full diff config_tree and python dict representation + self.full = ConfigTree(address=res) + self.dict = json.loads(self.full.to_json()) + + # config_tree sub-trees + self.add = self.full.get_subtree(['add']) + self.delete = self.full.get_subtree(['delete']) + self.inter = self.full.get_subtree(['inter']) def to_commands(self): add = self.add.to_commands() -- cgit v1.2.3 From 5b1231a0511095aa1b82a8c7bb1e119e3576e228 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 25 Feb 2022 16:12:29 -0600 Subject: configtree: T4235: allow empty arguments (cherry picked from commit 4625fd41f99ddf77c104a657cd90a1ddf5449dd8) --- python/vyos/configtree.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'python/vyos') diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index f5e697137..5ba829a4c 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -305,6 +305,10 @@ class ConfigTree(object): class DiffTree: def __init__(self, left, right, path=[], libpath=LIBPATH): + if left is None: + left = ConfigTree(config_string='\n') + if right is None: + right = ConfigTree(config_string='\n') if not (isinstance(left, ConfigTree) and isinstance(right, ConfigTree)): raise TypeError("Arguments must be instances of ConfigTree") if path: -- cgit v1.2.3 From a2bbde8f17d268ddaa4c0986230dc5ac5682b019 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 27 Feb 2022 10:05:40 -0600 Subject: configtree: T4235: distinguish sub(-tract) tree from delete tree The DiffTree class maintains both the 'sub'(-tract) configtree, containing all paths in the LHS of the comparison that are not in the RHS, and the 'delete' configtree: the delete tree is the minimal subtree containing only the first node of a path not present in the RHS. It is the delete tree that is needed to produce 'delete' commands for config mode, whereas the 'sub' tree contains full information, needed for recursively detecting changes to a node. (cherry picked from commit 193cbd15ba39a41614c63b997e6a62254589158a) --- python/vyos/configtree.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'python/vyos') diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index 5ba829a4c..e9cdb69e4 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -326,8 +326,13 @@ class DiffTree: self.__diff_tree.argtypes = [c_char_p, c_void_p, c_void_p] self.__diff_tree.restype = c_void_p + self.__trim_tree = self.__lib.trim_tree + self.__trim_tree.argtypes = [c_void_p, c_void_p] + self.__trim_tree.restype = c_void_p + check_path(path) path_str = " ".join(map(str, path)).encode() + res = self.__diff_tree(path_str, left._get_config(), right._get_config()) # full diff config_tree and python dict representation @@ -336,9 +341,14 @@ class DiffTree: # config_tree sub-trees self.add = self.full.get_subtree(['add']) - self.delete = self.full.get_subtree(['delete']) + self.sub = self.full.get_subtree(['sub']) self.inter = self.full.get_subtree(['inter']) + # trim sub(-tract) tree to get delete tree for commands + ref = self.right.get_subtree(path, with_node=True) if path else self.right + res = self.__trim_tree(self.sub._get_config(), ref._get_config()) + self.delete = ConfigTree(address=res) + def to_commands(self): add = self.add.to_commands() delete = self.delete.to_commands(op="delete") -- cgit v1.2.3 From d42e080510bda0db0e849daf4a3b065ff83ae82f Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 2 Mar 2022 07:59:47 -0600 Subject: configdiff: T4260: add support for diff_tree class Add support for the configtree diff algorithm. A new function ConfigDiff().is_node_changed(path) -> bool is added to recursively detect changes in the tree below the node at path; existing functions take the keyword argument 'recursive: bool' to apply the algorithm in place of the existing, non-recursive, comparison. (cherry picked from commit e5d04b20be0ef270d20f1d5ac9203b3a03649135) --- python/vyos/configdiff.py | 84 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 80 insertions(+), 4 deletions(-) (limited to 'python/vyos') diff --git a/python/vyos/configdiff.py b/python/vyos/configdiff.py index 0e41fbe27..81932e6d0 100644 --- a/python/vyos/configdiff.py +++ b/python/vyos/configdiff.py @@ -16,6 +16,7 @@ from enum import IntFlag, auto from vyos.config import Config +from vyos.configtree import DiffTree from vyos.configdict import dict_merge from vyos.util import get_sub_dict, mangle_dict_keys from vyos.xml import defaults @@ -36,6 +37,8 @@ class Diff(IntFlag): ADD = auto() STABLE = auto() +ALL = Diff.MERGE | Diff.DELETE | Diff.ADD | Diff.STABLE + requires_effective = [enum_to_key(Diff.DELETE)] target_defaults = [enum_to_key(Diff.MERGE)] @@ -73,19 +76,24 @@ def get_config_diff(config, key_mangling=None): isinstance(key_mangling[1], str)): raise ValueError("key_mangling must be a tuple of two strings") - return ConfigDiff(config, key_mangling) + diff_t = DiffTree(config._running_config, config._session_config) + + return ConfigDiff(config, key_mangling, diff_tree=diff_t) class ConfigDiff(object): """ The class of config changes as represented by comparison between the session config dict and the effective config dict. """ - def __init__(self, config, key_mangling=None): + def __init__(self, config, key_mangling=None, diff_tree=None): self._level = config.get_level() self._session_config_dict = config.get_cached_root_dict(effective=False) self._effective_config_dict = config.get_cached_root_dict(effective=True) self._key_mangling = key_mangling + self._diff_tree = diff_tree + self._diff_dict = diff_tree.dict if diff_tree else {} + # mirrored from Config; allow path arguments relative to level def _make_path(self, path): if isinstance(path, str): @@ -134,7 +142,17 @@ class ConfigDiff(object): self._key_mangling[1]) return config_dict - def get_child_nodes_diff(self, path=[], expand_nodes=Diff(0), no_defaults=False): + def is_node_changed(self, path=[]): + if self._diff_tree is None: + raise NotImplementedError("diff_tree class not available") + + if (self._diff_tree.add.exists(self._make_path(path)) or + self._diff_tree.sub.exists(self._make_path(path))): + return True + return False + + def get_child_nodes_diff(self, path=[], expand_nodes=Diff(0), no_defaults=False, + recursive=False): """ Args: path (str|list): config path @@ -144,6 +162,8 @@ class ConfigDiff(object): value no_detaults=False: if expand_nodes & Diff.MERGE, do not merge default values to ret['merge'] + recursive: if true, use config_tree diff algorithm provided by + diff_tree class Returns: dict of lists, representing differences between session and effective config, under path @@ -154,6 +174,34 @@ class ConfigDiff(object): """ session_dict = get_sub_dict(self._session_config_dict, self._make_path(path), get_first_key=True) + + if recursive: + if self._diff_tree is None: + raise NotImplementedError("diff_tree class not available") + else: + add = get_sub_dict(self._diff_tree.dict, ['add'], get_first_key=True) + sub = get_sub_dict(self._diff_tree.dict, ['sub'], get_first_key=True) + inter = get_sub_dict(self._diff_tree.dict, ['inter'], get_first_key=True) + ret = {} + ret[enum_to_key(Diff.MERGE)] = session_dict + ret[enum_to_key(Diff.DELETE)] = get_sub_dict(sub, self._make_path(path), + get_first_key=True) + ret[enum_to_key(Diff.ADD)] = get_sub_dict(add, self._make_path(path), + get_first_key=True) + ret[enum_to_key(Diff.STABLE)] = get_sub_dict(inter, self._make_path(path), + get_first_key=True) + for e in Diff: + k = enum_to_key(e) + if not (e & expand_nodes): + ret[k] = list(ret[k]) + else: + if self._key_mangling: + ret[k] = self._mangle_dict_keys(ret[k]) + if k in target_defaults and not no_defaults: + default_values = defaults(self._make_path(path)) + ret[k] = dict_merge(default_values, ret[k]) + return ret + effective_dict = get_sub_dict(self._effective_config_dict, self._make_path(path), get_first_key=True) @@ -179,7 +227,8 @@ class ConfigDiff(object): return ret - def get_node_diff(self, path=[], expand_nodes=Diff(0), no_defaults=False): + def get_node_diff(self, path=[], expand_nodes=Diff(0), no_defaults=False, + recursive=False): """ Args: path (str|list): config path @@ -189,6 +238,8 @@ class ConfigDiff(object): value no_detaults=False: if expand_nodes & Diff.MERGE, do not merge default values to ret['merge'] + recursive: if true, use config_tree diff algorithm provided by + diff_tree class Returns: dict of lists, representing differences between session and effective config, at path @@ -198,6 +249,31 @@ class ConfigDiff(object): dict['stable'] = config values in both session and effective """ session_dict = get_sub_dict(self._session_config_dict, self._make_path(path)) + + if recursive: + if self._diff_tree is None: + raise NotImplementedError("diff_tree class not available") + else: + add = get_sub_dict(self._diff_tree.dict, ['add'], get_first_key=True) + sub = get_sub_dict(self._diff_tree.dict, ['sub'], get_first_key=True) + inter = get_sub_dict(self._diff_tree.dict, ['inter'], get_first_key=True) + ret = {} + ret[enum_to_key(Diff.MERGE)] = session_dict + ret[enum_to_key(Diff.DELETE)] = get_sub_dict(sub, self._make_path(path)) + ret[enum_to_key(Diff.ADD)] = get_sub_dict(add, self._make_path(path)) + ret[enum_to_key(Diff.STABLE)] = get_sub_dict(inter, self._make_path(path)) + for e in Diff: + k = enum_to_key(e) + if not (e & expand_nodes): + ret[k] = list(ret[k]) + else: + if self._key_mangling: + ret[k] = self._mangle_dict_keys(ret[k]) + if k in target_defaults and not no_defaults: + default_values = defaults(self._make_path(path)) + ret[k] = dict_merge(default_values, ret[k]) + return ret + effective_dict = get_sub_dict(self._effective_config_dict, self._make_path(path)) ret = _key_sets_from_dicts(session_dict, effective_dict) -- cgit v1.2.3