From e8bfd3c7dcf59d9df74de85291bec2ee0f6c2950 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 10 May 2023 16:01:59 -0500 Subject: configtree: T5194: add function reference_tree_to_json --- python/vyos/configtree.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index 9308bdde4..19b9838d4 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -373,6 +373,19 @@ def union(left, right, libpath=LIBPATH): return tree +def reference_tree_to_json(from_dir, to_file, libpath=LIBPATH): + __lib = cdll.LoadLibrary(libpath) + __reference_tree_to_json = __lib.reference_tree_to_json + __reference_tree_to_json.argtypes = [c_char_p, c_char_p] + __get_error = __lib.get_error + __get_error.argtypes = [] + __get_error.restype = c_char_p + + res = __reference_tree_to_json(from_dir.encode(), to_file.encode()) + if res == 1: + msg = __get_error().decode() + raise ConfigTreeError(msg) + class DiffTree: def __init__(self, left, right, path=[], libpath=LIBPATH): if left is None: -- cgit v1.2.3 From ec4f1851058cc7c57ec460f3e81146bf318e2cc4 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 10 May 2023 16:03:46 -0500 Subject: xml: T5218: generate xml cache from reference_tree --- Makefile | 2 + python/vyos/xml_ref/generate_cache.py | 94 +++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100755 python/vyos/xml_ref/generate_cache.py diff --git a/Makefile b/Makefile index 4400cbfdc..37499c178 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,8 @@ interface_definitions: $(config_xml_obj) $(CURDIR)/scripts/override-default $(BUILD_DIR)/interface-definitions + $(CURDIR)/python/vyos/xml_ref/generate_cache.py --xml-dir $(BUILD_DIR)/interface-definitions + find $(BUILD_DIR)/interface-definitions -type f -name "*.xml" | xargs -I {} $(CURDIR)/scripts/build-command-templates {} $(CURDIR)/schema/interface_definition.rng $(TMPL_DIR) || exit 1 # XXX: delete top level node.def's that now live in other packages diff --git a/python/vyos/xml_ref/generate_cache.py b/python/vyos/xml_ref/generate_cache.py new file mode 100755 index 000000000..792c6eea7 --- /dev/null +++ b/python/vyos/xml_ref/generate_cache.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# + +import sys +import json +import argparse +from os.path import join +from os.path import abspath +from os.path import dirname +from xmltodict import parse + +_here = dirname(__file__) + +sys.path.append(join(_here, '..')) +from configtree import reference_tree_to_json, ConfigTreeError + +xml_cache = abspath(join(_here, 'cache.py')) +xml_cache_json = 'xml_cache.json' +xml_tmp = join('/tmp', xml_cache_json) + +node_data_fields = ("node_type", "multi", "valueless", "default_value") + +def trim_node_data(cache: dict): + for k in list(cache): + if k == "node_data": + for l in list(cache[k]): + if l not in node_data_fields: + del cache[k][l] + else: + if isinstance(cache[k], dict): + trim_node_data(cache[k]) + +def main(): + parser = argparse.ArgumentParser(description='generate and save dict from xml defintions') + parser.add_argument('--xml-dir', type=str, required=True, + help='transcluded xml interface-definition directory') + parser.add_argument('--save-json-dir', type=str, + help='directory to save json cache if needed') + args = parser.parse_args() + + xml_dir = abspath(args.xml_dir) + save_dir = abspath(args.save_json_dir) if args.save_json_dir else None + + try: + reference_tree_to_json(xml_dir, xml_tmp) + except ConfigTreeError as e: + print(e) + sys.exit(1) + + with open(xml_tmp) as f: + d = json.loads(f.read()) + + trim_node_data(d) + + if save_dir is not None: + save_file = join(save_dir, xml_cache_json) + with open(save_file, 'w') as f: + f.write(json.dumps(d)) + + syntax_version = join(xml_dir, 'xml-component-version.xml') + with open(syntax_version) as f: + content = f.read() + + parsed = parse(content) + converted = parsed['interfaceDefinition']['syntaxVersion'] + version = {} + for i in converted: + tmp = {i['@component']: i['@version']} + version |= tmp + + version = {"component_version": version} + + d |= version + + with open(xml_cache, 'w') as f: + f.write(f'reference = {str(d)}') + +if __name__ == '__main__': + main() -- cgit v1.2.3 From 222f459d82ec8bed2f4e4c89e7ea81249bc4ebe4 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 10 May 2023 16:05:00 -0500 Subject: xml: T5218: add operations on xml cache --- python/vyos/xml_ref/__init__.py | 58 ++++++++++ python/vyos/xml_ref/definition.py | 234 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 292 insertions(+) create mode 100644 python/vyos/xml_ref/__init__.py create mode 100644 python/vyos/xml_ref/definition.py diff --git a/python/vyos/xml_ref/__init__.py b/python/vyos/xml_ref/__init__.py new file mode 100644 index 000000000..66525865c --- /dev/null +++ b/python/vyos/xml_ref/__init__.py @@ -0,0 +1,58 @@ +# Copyright 2023 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see . + +from vyos.xml_ref import definition + +def load_reference(cache=[]): + if cache: + return cache[0] + + xml = definition.Xml() + + try: + from vyos.xml_ref.cache import reference + xml.define(reference) + cache.append(xml) + except Exception: + raise ImportError('no xml reference cache !!') + + return xml + +def is_tag(path: list) -> bool: + return load_reference().is_tag(path) + +def is_tag_value(path: list) -> bool: + return load_reference().is_tag_value(path) + +def is_multi(path: list) -> bool: + return load_reference().is_multi(path) + +def is_valueless(path: list) -> bool: + return load_reference().is_valueless(path) + +def is_leaf(path: list) -> bool: + return load_reference().is_leaf(path) + +def component_version() -> dict: + return load_reference().component_version() + +def multi_to_list(rpath: list, conf: dict) -> dict: + return load_reference().multi_to_list(rpath, conf) + +def get_defaults(path: list, get_first_key=False) -> dict: + return load_reference().get_defaults(path, get_first_key=get_first_key) + +def merge_defaults(path: list, conf: dict) -> dict: + return load_reference().merge_defaults(path, conf) diff --git a/python/vyos/xml_ref/definition.py b/python/vyos/xml_ref/definition.py new file mode 100644 index 000000000..64a33e4d0 --- /dev/null +++ b/python/vyos/xml_ref/definition.py @@ -0,0 +1,234 @@ +# Copyright 2023 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see . + +from typing import Union, Optional, Any +from vyos.configdict import dict_merge + +class Xml: + def __init__(self): + self.ref = {} + + def define(self, ref: dict): + self.ref = ref + + def _get_ref_node_data(self, node: dict, data: str) -> Union[bool, str]: + res = node.get('node_data', {}) + if not res: + raise ValueError("non-existent node data") + if data not in res: + raise ValueError("non-existent data field") + + return res.get(data) + + def _get_ref_path(self, path: list) -> dict: + ref_path = path.copy() + d = self.ref + while ref_path and d: + d = d.get(ref_path[0], {}) + ref_path.pop(0) + if self._is_tag_node(d) and ref_path: + ref_path.pop(0) + + return d + + def _is_tag_node(self, node: dict) -> bool: + res = self._get_ref_node_data(node, 'node_type') + return res == 'tag' + + def is_tag(self, path: list) -> bool: + ref_path = path.copy() + d = self.ref + while ref_path and d: + d = d.get(ref_path[0], {}) + ref_path.pop(0) + if self._is_tag_node(d) and ref_path: + if len(ref_path) == 1: + return False + ref_path.pop(0) + + return self._is_tag_node(d) + + def is_tag_value(self, path: list) -> bool: + if len(path) < 2: + return False + + return self.is_tag(path[:-1]) + + def _is_multi_node(self, node: dict) -> bool: + b = self._get_ref_node_data(node, 'multi') + assert isinstance(b, bool) + return b + + def is_multi(self, path: list) -> bool: + d = self._get_ref_path(path) + return self._is_multi_node(d) + + def _is_valueless_node(self, node: dict) -> bool: + b = self._get_ref_node_data(node, 'valueless') + assert isinstance(b, bool) + return b + + def is_valueless(self, path: list) -> bool: + d = self._get_ref_path(path) + return self._is_valueless_node(d) + + def _is_leaf_node(self, node: dict) -> bool: + res = self._get_ref_node_data(node, 'node_type') + return res == 'leaf' + + def is_leaf(self, path: list) -> bool: + d = self._get_ref_path(path) + return self._is_leaf_node(d) + + def component_version(self) -> dict: + d = {} + for k, v in self.ref['component_version']: + d[k] = int(v) + return d + + def multi_to_list(self, rpath: list, conf: dict) -> dict: + if rpath and rpath[-1] in list(conf): + raise ValueError('rpath should be disjoint from conf keys') + + res = {} + + for k in list(conf): + d = self._get_ref_path(rpath + [k]) + if self._is_leaf_node(d): + if self._is_multi_node(d) and not isinstance(conf[k], list): + res[k] = [conf[k]] + else: + res[k] = conf[k] + else: + res[k] = self.multi_to_list(rpath + [k], conf[k]) + + return res + + def _get_default_value(self, node: dict): + return self._get_ref_node_data(node, "default_value") + + def get_defaults(self, path: list, get_first_key=False) -> dict: + """Return dict containing default values below path + + Note that descent below path will not proceed beyond an encountered + tag node, as no tag node value is known. For a default dict relative + to an existing config dict containing tag node values, see function: + 'relative_defaults' + """ + res: Any = {} + d = self._get_ref_path(path) + if self._is_leaf_node(d): + default_value = self._get_default_value(d) + if default_value is not None: + res = default_value + if self._is_multi_node(d) and not isinstance(res, list): + res = [res] + elif self.is_tag(path): + # tag node defaults are used as suggestion, not default value; + # should this change, append to path and continue recursion + pass + else: + for k in list(d): + if k in ('node_data', 'component_version') : + continue + pos = self.get_defaults(path + [k]) + res |= pos + if res: + if get_first_key or not path: + if not isinstance(res, dict): + raise TypeError("Cannot get_first_key as data under node is not of type dict") + return res + return {path[-1]: res} + + return {} + + def _relative_defaults(self, rpath: list, conf: Optional[dict] = None, + get_first_key=False) -> dict: + # Return dict containing defaults along paths of a config dict + + # Note that for conf argument {} or None, this function returns the + # same result as 'get_defaults' above; for clarity, the functions + # are individually defined. + + if conf is None: + conf = {} + if rpath and rpath[-1] in list(conf): + raise ValueError('rpath should be disjoint from conf keys') + res: Any = {} + d = self._get_ref_path(rpath) + if self._is_leaf_node(d): + default_value = self._get_default_value(d) + if default_value is not None: + res = default_value + if self._is_multi_node(d) and not isinstance(res, list): + res = [res] + elif self.is_tag(rpath): + for k in list(conf): + pos = self._relative_defaults(rpath + [k], conf[k]) + res |= pos + else: + for k in list(d): + if k in ('node_data', 'component_version') : + continue + pos = self._relative_defaults(rpath + [k], conf.get(k, {})) + res |= pos + if res: + if get_first_key or not rpath: + if not isinstance(res, dict): + raise TypeError("Cannot get_first_key as data under node is not of type dict") + return res + return {rpath[-1]: res} + + return {} + + def _defines_config_path(self, path: list, conf: dict) -> bool: + # test disjoint path + conf for sensible config paths + def walk(c): + return [next(iter(c.keys()))] if c else [] + try: + tmp = walk(conf) + if self.is_tag_value(path + tmp): + c = conf[tmp[0]] + if not isinstance(c, dict): + raise ValueError + tmp = tmp + walk(c) + self._get_ref_path(path + tmp) + else: + self._get_ref_path(path + tmp) + except ValueError: + return False + return True + + def merge_defaults(self, path: list, conf: dict) -> dict: + """Return config dict with defaults non-destructively merged + """ + if not path: + path = [next(iter(conf.keys()))] + if path[-1] in list(conf): + config = conf[path[-1]] + if not isinstance(config, dict): + raise ValueError('conf at path is not of type dict') + first = False + else: + config = conf + first = True + + if not self._defines_config_path(path, config): + print('path + conf do not define config paths; conf returned unchanged') + return conf + + d = self._relative_defaults(path, conf=config, get_first_key=first) + d = dict_merge(d, conf) + return d -- cgit v1.2.3 From d958f00dda8bcf15720d32c610f1e2fb5909a53a Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 19 May 2023 14:16:21 -0500 Subject: xml: T5218: return defaults only for child leaf-nodes, unless recursive Operations get_defaults and get_config_defaults return default values only for nodes with parent in the config dict (get_config_defaults) or at the path (get_defaults). To include default values of decendent nodes, set option recursive=True. --- python/vyos/xml_ref/__init__.py | 12 +++- python/vyos/xml_ref/definition.py | 139 +++++++++++++++++++------------------- 2 files changed, 78 insertions(+), 73 deletions(-) diff --git a/python/vyos/xml_ref/__init__.py b/python/vyos/xml_ref/__init__.py index 66525865c..ae5184746 100644 --- a/python/vyos/xml_ref/__init__.py +++ b/python/vyos/xml_ref/__init__.py @@ -51,8 +51,16 @@ def component_version() -> dict: def multi_to_list(rpath: list, conf: dict) -> dict: return load_reference().multi_to_list(rpath, conf) -def get_defaults(path: list, get_first_key=False) -> dict: - return load_reference().get_defaults(path, get_first_key=get_first_key) +def get_defaults(path: list, get_first_key=False, recursive=False) -> dict: + return load_reference().get_defaults(path, get_first_key=get_first_key, + recursive=recursive) + +def get_config_defaults(rpath: list, conf: dict, get_first_key=False, + recursive=False) -> dict: + + return load_reference().relative_defaults(rpath, conf=conf, + get_first_key=get_first_key, + recursive=recursive) def merge_defaults(path: list, conf: dict) -> dict: return load_reference().merge_defaults(path, conf) diff --git a/python/vyos/xml_ref/definition.py b/python/vyos/xml_ref/definition.py index 64a33e4d0..429331577 100644 --- a/python/vyos/xml_ref/definition.py +++ b/python/vyos/xml_ref/definition.py @@ -13,7 +13,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see . -from typing import Union, Optional, Any +from typing import Union, Any from vyos.configdict import dict_merge class Xml: @@ -102,7 +102,7 @@ class Xml: if rpath and rpath[-1] in list(conf): raise ValueError('rpath should be disjoint from conf keys') - res = {} + res: Any = {} for k in list(conf): d = self._get_ref_path(rpath + [k]) @@ -119,7 +119,7 @@ class Xml: def _get_default_value(self, node: dict): return self._get_ref_node_data(node, "default_value") - def get_defaults(self, path: list, get_first_key=False) -> dict: + def get_defaults(self, path: list, get_first_key=False, recursive=False) -> dict: """Return dict containing default values below path Note that descent below path will not proceed beyond an encountered @@ -127,24 +127,27 @@ class Xml: to an existing config dict containing tag node values, see function: 'relative_defaults' """ - res: Any = {} + res: dict = {} d = self._get_ref_path(path) - if self._is_leaf_node(d): - default_value = self._get_default_value(d) - if default_value is not None: - res = default_value - if self._is_multi_node(d) and not isinstance(res, list): - res = [res] - elif self.is_tag(path): - # tag node defaults are used as suggestion, not default value; - # should this change, append to path and continue recursion - pass - else: - for k in list(d): - if k in ('node_data', 'component_version') : - continue - pos = self.get_defaults(path + [k]) - res |= pos + for k in list(d): + if k in ('node_data', 'component_version') : + continue + d_k = d[k] + if self._is_leaf_node(d_k): + default_value = self._get_default_value(d_k) + if default_value is not None: + pos = default_value + if self._is_multi_node(d_k) and not isinstance(pos, list): + pos = [pos] + res |= {k: pos} + elif self.is_tag(path + [k]): + # tag node defaults are used as suggestion, not default value; + # should this change, append to path and continue if recursive + pass + else: + if recursive: + pos = self.get_defaults(path + [k], recursive=True) + res |= pos if res: if get_first_key or not path: if not isinstance(res, dict): @@ -154,56 +157,17 @@ class Xml: return {} - def _relative_defaults(self, rpath: list, conf: Optional[dict] = None, - get_first_key=False) -> dict: - # Return dict containing defaults along paths of a config dict - - # Note that for conf argument {} or None, this function returns the - # same result as 'get_defaults' above; for clarity, the functions - # are individually defined. - - if conf is None: - conf = {} - if rpath and rpath[-1] in list(conf): - raise ValueError('rpath should be disjoint from conf keys') - res: Any = {} - d = self._get_ref_path(rpath) - if self._is_leaf_node(d): - default_value = self._get_default_value(d) - if default_value is not None: - res = default_value - if self._is_multi_node(d) and not isinstance(res, list): - res = [res] - elif self.is_tag(rpath): - for k in list(conf): - pos = self._relative_defaults(rpath + [k], conf[k]) - res |= pos - else: - for k in list(d): - if k in ('node_data', 'component_version') : - continue - pos = self._relative_defaults(rpath + [k], conf.get(k, {})) - res |= pos - if res: - if get_first_key or not rpath: - if not isinstance(res, dict): - raise TypeError("Cannot get_first_key as data under node is not of type dict") - return res - return {rpath[-1]: res} - - return {} - - def _defines_config_path(self, path: list, conf: dict) -> bool: + def _well_defined(self, path: list, conf: dict) -> bool: # test disjoint path + conf for sensible config paths - def walk(c): + def step(c): return [next(iter(c.keys()))] if c else [] try: - tmp = walk(conf) + tmp = step(conf) if self.is_tag_value(path + tmp): c = conf[tmp[0]] if not isinstance(c, dict): raise ValueError - tmp = tmp + walk(c) + tmp = tmp + step(c) self._get_ref_path(path + tmp) else: self._get_ref_path(path + tmp) @@ -211,24 +175,57 @@ class Xml: return False return True + def relative_defaults(self, rpath: list, conf: dict, get_first_key=False, + recursive=False) -> dict: + """Return dict containing defaults along paths of a config dict + """ + if not conf: + return self.get_defaults(rpath, get_first_key=get_first_key, + recursive=recursive) + if rpath and rpath[-1] in list(conf): + conf = conf[rpath[-1]] + if not isinstance(conf, dict): + raise TypeError('conf at path is not of type dict') + + if not self._well_defined(rpath, conf): + print('path to config dict does not define full config paths') + return {} + + res: dict = {} + for k in list(conf): + pos = self.get_defaults(rpath + [k], recursive=recursive) + res |= pos + + if isinstance(conf[k], dict): + step = self.relative_defaults(rpath + [k], conf=conf[k], + recursive=recursive) + res |= step + + if res: + if get_first_key: + return res + return {rpath[-1]: res} if rpath else res + + return {} + def merge_defaults(self, path: list, conf: dict) -> dict: """Return config dict with defaults non-destructively merged + + This merges non-recursive defaults relative to the config dict. """ - if not path: - path = [next(iter(conf.keys()))] if path[-1] in list(conf): config = conf[path[-1]] if not isinstance(config, dict): - raise ValueError('conf at path is not of type dict') - first = False + raise TypeError('conf at path is not of type dict') + shift = False else: config = conf - first = True + shift = True - if not self._defines_config_path(path, config): - print('path + conf do not define config paths; conf returned unchanged') + if not self._well_defined(path, config): + print('path to config dict does not define config paths; conf returned unchanged') return conf - d = self._relative_defaults(path, conf=config, get_first_key=first) + d = self.relative_defaults(path, conf=config, get_first_key=shift) d = dict_merge(d, conf) return d -- cgit v1.2.3