From e7c69ead85af485b2b881f24b929a4bcc779a4ce Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 11 Aug 2021 11:06:39 -0500 Subject: xml: T3732: simplifications for merging defaultValue and default-less nodes --- scripts/override-default | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'scripts') diff --git a/scripts/override-default b/scripts/override-default index c8a0ff1da..228aca30f 100755 --- a/scripts/override-default +++ b/scripts/override-default @@ -62,26 +62,26 @@ def override_element(l: list): def collect_and_override(dir_name): """ - Collect elements with defaultValue tag into dictionary indexed by tuple - of (name: str, ancestor path: str). + Collect elements with defaultValue tag into dictionary indexed by name + attributes of ancestor path. """ for fname in glob.glob(f'{dir_name}/*.xml'): tree = etree.parse(fname) root = tree.getroot() defv = {} - xpath_str = f'//defaultValue' + xpath_str = '//defaultValue' xp = tree.xpath(xpath_str) for element in xp: ap = element.xpath('ancestor::*[@name]') ap_name = [el.get("name") for el in ap] - ap_path_str = ' '.join(ap_name[:-1]) - defv.setdefault((ap_name[-1], ap_path_str), []).append(element) + ap_path_str = ' '.join(ap_name) + defv.setdefault(ap_path_str, []).append(element) for k, v in defv.items(): if len(v) > 1: - logger.info(f"overridding default in {k[0]}, path '{k[1]}'") + logger.info(f"overridding default in path '{k}'") override_element(v) revised_str = etree.tostring(root, encoding='unicode', pretty_print=True) -- cgit v1.2.3 From ec6dc94a5144ea4ac75ba6fac4d85b4f66337ae2 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 11 Aug 2021 11:54:28 -0500 Subject: xml: T3732: merge leafNode with defaultValue with leafNode(s) of same path --- scripts/override-default | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'scripts') diff --git a/scripts/override-default b/scripts/override-default index 228aca30f..0c49087c8 100755 --- a/scripts/override-default +++ b/scripts/override-default @@ -27,6 +27,7 @@ import sys import glob import logging +from copy import deepcopy from lxml import etree debug = False @@ -60,6 +61,28 @@ def override_element(l: list): for el in parents: el.getparent().remove(el) +def merge_remaining(l: list, elementtree): + """ + Merge (now) single leaf node containing 'defaultValue' with leaf nodes + of same path and no 'defaultValue'. + """ + for p in l: + p = p.split() + path_str = f'/interfaceDefinition/*' + path_list = [] + for i in range(len(p)): + path_list.append(f'[@name="{p[i]}"]') + path_str += '/children/*'.join(path_list) + rp = elementtree.xpath(path_str) + if len(rp) > 1: + for el in rp[1:]: + # in practice there will only be one child of the path, + # either defaultValue or Properties, since + # override_element() has already run + for child in el: + rp[0].append(deepcopy(child)) + el.getparent().remove(el) + def collect_and_override(dir_name): """ Collect elements with defaultValue tag into dictionary indexed by name @@ -84,6 +107,9 @@ def collect_and_override(dir_name): logger.info(f"overridding default in path '{k}'") override_element(v) + to_merge = list(defv) + merge_remaining(to_merge, tree) + revised_str = etree.tostring(root, encoding='unicode', pretty_print=True) with open(f'{fname}', 'w') as f: -- cgit v1.2.3 From 9ac5e1c8479049ae63747db36f4a0ef13ce6bc01 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 19 Aug 2021 11:41:00 -0500 Subject: Revert "xml: T1962: Add script to process syntaxVersion tags during build" This reverts commit 0ecc2c26f7ac939e4e23c14f5027ac7592c25761. --- Makefile | 7 +----- scripts/build-component-versions | 47 ---------------------------------------- 2 files changed, 1 insertion(+), 53 deletions(-) delete mode 100755 scripts/build-component-versions (limited to 'scripts') diff --git a/Makefile b/Makefile index c8f6ac7c9..0aa35d17d 100644 --- a/Makefile +++ b/Makefile @@ -63,11 +63,6 @@ op_mode_definitions: $(op_xml_obj) # could mask help strings or mandatory priority statements #find $(OP_TMPL_DIR) -name node.def -type f -empty -exec false {} + || sh -c 'echo "There are empty node.def files! Check your interface definitions." && exit 1' -.PHONY: component_versions -.ONESHELL: -component_versions: interface_definitions - $(CURDIR)/scripts/build-component-versions $(BUILD_DIR)/interface-definitions $(DATA_DIR) - .PHONY: vyshim vyshim: $(MAKE) -C $(SHIM_DIR) @@ -77,7 +72,7 @@ vyxdp: $(MAKE) -C $(XDP_DIR) .PHONY: all -all: clean interface_definitions op_mode_definitions component_versions vyshim +all: clean interface_definitions op_mode_definitions vyshim .PHONY: clean clean: diff --git a/scripts/build-component-versions b/scripts/build-component-versions deleted file mode 100755 index 5362dbdd4..000000000 --- a/scripts/build-component-versions +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -import argparse -import json - -from lxml import etree as ET - -parser = argparse.ArgumentParser() -parser.add_argument('INPUT_DIR', type=str, - help="Directory containing XML interface definition files") -parser.add_argument('OUTPUT_DIR', type=str, - help="Output directory for JSON file") - -args = parser.parse_args() - -input_dir = args.INPUT_DIR -output_dir = args.OUTPUT_DIR - -version_dict = {} - -for filename in os.listdir(input_dir): - filepath = os.path.join(input_dir, filename) - print(filepath) - try: - xml = ET.parse(filepath) - except Exception as e: - print("Failed to load interface definition file {0}".format(filename)) - print(e) - sys.exit(1) - - root = xml.getroot() - version_data = root.iterfind("syntaxVersion") - for ver in version_data: - component = ver.get("component") - version = int(ver.get("version")) - - v = version_dict.get(component) - if v is None: - version_dict[component] = version - elif version > v: - version_dict[component] = version - -out_file = os.path.join(output_dir, 'component-versions.json') -with open(out_file, 'w') as f: - json.dump(version_dict, f, indent=4, sort_keys=True) -- cgit v1.2.3 From 252bc820b0d130d8d81b5711586eca41287abdca Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sun, 22 Aug 2021 19:48:28 +0200 Subject: scripts: op-mode: use Python 'f'ormat strings on debug messages --- scripts/build-command-op-templates | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/build-command-op-templates b/scripts/build-command-op-templates index c285ee594..d0e5833cc 100755 --- a/scripts/build-command-op-templates +++ b/scripts/build-command-op-templates @@ -54,7 +54,7 @@ debug = args.debug try: xml = ET.parse(input_file) except Exception as e: - print("Failed to load interface definition file {0}".format(input_file)) + print(f"Failed to load interface definition file {input_file}") print(e) sys.exit(1) @@ -64,15 +64,15 @@ try: if not validator.validate(xml): print(validator.error_log) - print("Interface definition file {0} does not match the schema!".format(input_file)) + print(f"Interface definition file {input_file} does not match the schema!") sys.exit(1) except Exception as e: - print("Failed to load the XML schema {0}".format(schema_file)) + print(f"Failed to load the XML schema {schema_file}") print(e) sys.exit(1) if not os.access(output_dir, os.W_OK): - print("The output directory {0} is not writeable".format(output_dir)) + print(f"The output directory {output_dir} is not writeable") sys.exit(1) ## If we got this far, everything must be ok and we can convert the file @@ -160,14 +160,14 @@ def process_node(n, tmpl_dir): my_tmpl_dir.append(name) if debug: - print("Name of the node: {};\n Created directory: ".format(name), end="") + print(f"Name of the node: {name};\n Created directory: ", end="") os.makedirs(make_path(my_tmpl_dir), exist_ok=True) props = get_properties(props_elem) if node_type == "node": if debug: - print("Processing node {}".format(name)) + print(f"Processing node {name}") nodedef_path = os.path.join(make_path(my_tmpl_dir), "node.def") # Only create the "node.def" file if it exists but is empty, or if it @@ -180,9 +180,10 @@ def process_node(n, tmpl_dir): inner_nodes = children.iterfind("*") for inner_n in inner_nodes: process_node(inner_n, my_tmpl_dir) + if node_type == "tagNode": if debug: - print("Processing tag node {}".format(name)) + print(f"Processing tagNode {name}") os.makedirs(make_path(my_tmpl_dir), exist_ok=True) @@ -211,7 +212,7 @@ def process_node(n, tmpl_dir): else: # This is a leaf node if debug: - print("Processing leaf node {}".format(name)) + print(f"Processing leaf node {name}") with open(os.path.join(make_path(my_tmpl_dir), "node.def"), "w") as f: f.write(make_node_def(props, command)) -- cgit v1.2.3 From 17b5ac143c9128ac3e187d8d8167dd8fe6cbca7d Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sun, 22 Aug 2021 20:26:36 +0200 Subject: T3165: op-mode: prevent override of populated node.def file with empty content This is an extension to commit b4fdcebe ("T3165: prevent override of populated node.def file with empty content") which implemented the same thing for the configuration mode commands. --- scripts/build-command-op-templates | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) (limited to 'scripts') diff --git a/scripts/build-command-op-templates b/scripts/build-command-op-templates index d0e5833cc..a4d6d1d08 100755 --- a/scripts/build-command-op-templates +++ b/scripts/build-command-op-templates @@ -165,11 +165,11 @@ def process_node(n, tmpl_dir): props = get_properties(props_elem) + nodedef_path = os.path.join(make_path(my_tmpl_dir), "node.def") if node_type == "node": if debug: print(f"Processing node {name}") - nodedef_path = os.path.join(make_path(my_tmpl_dir), "node.def") # Only create the "node.def" file if it exists but is empty, or if it # does not exist at all. if not os.path.exists(nodedef_path) or os.path.getsize(nodedef_path) == 0: @@ -187,13 +187,11 @@ def process_node(n, tmpl_dir): os.makedirs(make_path(my_tmpl_dir), exist_ok=True) - nodedef_path = os.path.join(make_path(my_tmpl_dir), "node.def") - if not os.path.exists(nodedef_path): + # Only create the "node.def" file if it exists but is empty, or if it + # does not exist at all. + if not os.path.exists(nodedef_path) or os.path.getsize(nodedef_path) == 0: with open(nodedef_path, "w") as f: f.write('help: {0}\n'.format(props['help'])) - else: - # Something has already generated this file - pass # Create the inner node.tag part my_tmpl_dir.append("node.tag") @@ -202,8 +200,12 @@ def process_node(n, tmpl_dir): print("Created path for the tagNode: {}".format(make_path(my_tmpl_dir)), end="") # Not sure if we want partially defined tag nodes, write the file unconditionally - with open(os.path.join(make_path(my_tmpl_dir), "node.def"), "w") as f: - f.write(make_node_def(props, command)) + nodedef_path = os.path.join(make_path(my_tmpl_dir), "node.def") + # Only create the "node.def" file if it exists but is empty, or if it + # does not exist at all. + if not os.path.exists(nodedef_path) or os.path.getsize(nodedef_path) == 0: + with open(nodedef_path, "w") as f: + f.write(make_node_def(props, command)) if children is not None: inner_nodes = children.iterfind("*") @@ -214,9 +216,9 @@ def process_node(n, tmpl_dir): if debug: print(f"Processing leaf node {name}") - with open(os.path.join(make_path(my_tmpl_dir), "node.def"), "w") as f: - f.write(make_node_def(props, command)) - + if not os.path.exists(nodedef_path) or os.path.getsize(nodedef_path) == 0: + with open(nodedef_path, "w") as f: + f.write(make_node_def(props, command)) root = xml.getroot() -- cgit v1.2.3 From 7623e37c918c65418d8dfc521f976bb91f0594c0 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Tue, 7 Sep 2021 11:41:12 +0200 Subject: scripts: op-mode: T3807: bugfix node.def generator process_node() processes the XML tree in a fixed order, "node" before "tagNode" before "leafNode". If the generator created a "node.def" file, it can no longer be overwritten - else we would have some stale "node.def" files with an empty help string (T2555). Without the fixed order this would resulted in a case where we get a node and a tagNode with the same name, e.g. "show interfaces ethernet" and "show interfaces ethernet eth0" that the node implementation was not callable from the CLI, rendering this command useless (T3807). This can be fixed by forcing the "node", "tagNode", "leafNode" order by sorting the input XML file automatically (sorting from https://stackoverflow.com/a/46128043) thus adding no additional overhead to the user. --- scripts/build-command-op-templates | 57 +++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 16 deletions(-) (limited to 'scripts') diff --git a/scripts/build-command-op-templates b/scripts/build-command-op-templates index a4d6d1d08..d4515b8db 100755 --- a/scripts/build-command-op-templates +++ b/scripts/build-command-op-templates @@ -29,13 +29,10 @@ import functools from lxml import etree as ET # Defaults - validator_dir = "/opt/vyatta/libexec/validators" default_constraint_err_msg = "Invalid value" - ## Get arguments - parser = argparse.ArgumentParser(description='Converts new-style XML interface definitions to old-style command templates') parser.add_argument('--debug', help='Enable debug information output', action='store_true') parser.add_argument('INPUT_FILE', type=str, help="XML interface definition file") @@ -50,7 +47,6 @@ output_dir = args.OUTPUT_DIR debug = args.debug ## Load and validate the inputs - try: xml = ET.parse(input_file) except Exception as e: @@ -76,7 +72,6 @@ if not os.access(output_dir, os.W_OK): sys.exit(1) ## If we got this far, everything must be ok and we can convert the file - def make_path(l): path = functools.reduce(os.path.join, l) if debug: @@ -125,21 +120,14 @@ def get_properties(p): def make_node_def(props, command): # XXX: replace with a template processor if it grows # out of control - node_def = "" if "help" in props: node_def += "help: {0}\n".format(props["help"]) - - if "comp_help" in props: node_def += "allowed: {0}\n".format(props["comp_help"]) - - if command is not None: node_def += "run: {0}\n".format(command.text) - - if debug: print("The contents of the node.def file:\n", node_def) @@ -152,7 +140,6 @@ def process_node(n, tmpl_dir): props_elem = n.find("properties") children = n.find("children") command = n.find("command") - name = n.get("name") node_type = n.tag @@ -180,8 +167,7 @@ def process_node(n, tmpl_dir): inner_nodes = children.iterfind("*") for inner_n in inner_nodes: process_node(inner_n, my_tmpl_dir) - - if node_type == "tagNode": + elif node_type == "tagNode": if debug: print(f"Processing tagNode {name}") @@ -211,7 +197,7 @@ def process_node(n, tmpl_dir): inner_nodes = children.iterfind("*") for inner_n in inner_nodes: process_node(inner_n, my_tmpl_dir) - else: + elif node_type == "leafNode": # This is a leaf node if debug: print(f"Processing leaf node {name}") @@ -219,9 +205,48 @@ def process_node(n, tmpl_dir): if not os.path.exists(nodedef_path) or os.path.getsize(nodedef_path) == 0: with open(nodedef_path, "w") as f: f.write(make_node_def(props, command)) + else: + print(f"Unknown node_type: {node_type}") + + +def get_node_key(node, attr=None): + """ Return the sorting key of an xml node using tag and attributes """ + if attr is None: + return '%s' % node.tag + ':'.join([node.get(attr) + for attr in sorted(node.attrib)]) + if attr in node.attrib: + return '%s:%s' % (node.tag, node.get(attr)) + return '%s' % node.tag + + +def sort_children(node, attr=None): + """ Sort children along tag and given attribute. if attr is None, sort + along all attributes """ + if not isinstance(node.tag, str): # PYTHON 2: use basestring instead + # not a TAG, it is comment or DATA + # no need to sort + return + # sort child along attr + node[:] = sorted(node, key=lambda child: get_node_key(child, attr)) + # and recurse + for child in node: + sort_children(child, attr) root = xml.getroot() +# process_node() processes the XML tree in a fixed order, "node" before "tagNode" +# before "leafNode". If the generator created a "node.def" file, it can no longer +# be overwritten - else we would have some stale "node.def" files with an empty +# help string (T2555). Without the fixed order this would resulted in a case +# where we get a node and a tagNode with the same name, e.g. "show interfaces +# ethernet" and "show interfaces ethernet eth0" that the node implementation +# was not callable from the CLI, rendering this command useless (T3807). +# +# This can be fixed by forcing the "node", "tagNode", "leafNode" order by sorting +# the input XML file automatically (sorting from https://stackoverflow.com/a/46128043) +# thus adding no additional overhead to the user. +sort_children(root, 'name') + nodes = root.iterfind("*") for n in nodes: process_node(n, [output_dir]) -- cgit v1.2.3