Merge branch 'current' into 2fa

3 files changed, 94 insertions, 87 deletions

diff --git a/scripts/build-command-op-templates b/scripts/build-command-op-templates
index c285ee594..d4515b8db 100755
--- a/scripts/build-command-op-templates
+++ b/scripts/build-command-op-templates
@@ -29,13 +29,10 @@ import functools
 from lxml import etree as ET
 
 # Defaults
-
 validator_dir = "/opt/vyatta/libexec/validators"
 default_constraint_err_msg = "Invalid value"
 
-
 ## Get arguments
-
 parser = argparse.ArgumentParser(description='Converts new-style XML interface definitions to old-style command templates')
 parser.add_argument('--debug', help='Enable debug information output', action='store_true')
 parser.add_argument('INPUT_FILE', type=str, help="XML interface definition file")
@@ -50,11 +47,10 @@ output_dir = args.OUTPUT_DIR
 debug = args.debug
 
 ## Load and validate the inputs
-
 try:
     xml = ET.parse(input_file)
 except Exception as e:
-    print("Failed to load interface definition file {0}".format(input_file))
+    print(f"Failed to load interface definition file {input_file}")
     print(e)
     sys.exit(1)
 
@@ -64,19 +60,18 @@ try:
 
     if not validator.validate(xml):
         print(validator.error_log)
-        print("Interface definition file {0} does not match the schema!".format(input_file))
+        print(f"Interface definition file {input_file} does not match the schema!")
         sys.exit(1)
 except Exception as e:
-    print("Failed to load the XML schema {0}".format(schema_file))
+    print(f"Failed to load the XML schema {schema_file}")
     print(e)
     sys.exit(1)
 
 if not os.access(output_dir, os.W_OK):
-    print("The output directory {0} is not writeable".format(output_dir))
+    print(f"The output directory {output_dir} is not writeable")
     sys.exit(1)
 
 ## If we got this far, everything must be ok and we can convert the file
-
 def make_path(l):
     path = functools.reduce(os.path.join, l)
     if debug:
@@ -125,21 +120,14 @@ def get_properties(p):
 def make_node_def(props, command):
     # XXX: replace with a template processor if it grows
     #      out of control
-
     node_def = ""
 
     if "help" in props:
         node_def += "help: {0}\n".format(props["help"])
-
-
     if "comp_help" in props:
         node_def += "allowed: {0}\n".format(props["comp_help"])
-
-
     if command is not None:
         node_def += "run: {0}\n".format(command.text)
-
-
     if debug:
         print("The contents of the node.def file:\n", node_def)
 
@@ -152,7 +140,6 @@ def process_node(n, tmpl_dir):
     props_elem = n.find("properties")
     children = n.find("children")
     command = n.find("command")
-
     name = n.get("name")
 
     node_type = n.tag
@@ -160,16 +147,16 @@ def process_node(n, tmpl_dir):
     my_tmpl_dir.append(name)
 
     if debug:
-        print("Name of the node: {};\n Created directory: ".format(name), end="")
+        print(f"Name of the node: {name};\n Created directory: ", end="")
     os.makedirs(make_path(my_tmpl_dir), exist_ok=True)
 
     props = get_properties(props_elem)
 
+    nodedef_path = os.path.join(make_path(my_tmpl_dir), "node.def")
     if node_type == "node":
         if debug:
-          print("Processing node {}".format(name))
+          print(f"Processing node {name}")
 
-        nodedef_path = os.path.join(make_path(my_tmpl_dir), "node.def")
         # Only create the "node.def" file if it exists but is empty, or if it
         # does not exist at all.
         if not os.path.exists(nodedef_path) or os.path.getsize(nodedef_path) == 0:
@@ -180,19 +167,17 @@ def process_node(n, tmpl_dir):
             inner_nodes = children.iterfind("*")
             for inner_n in inner_nodes:
                 process_node(inner_n, my_tmpl_dir)
-    if node_type == "tagNode":
+    elif node_type == "tagNode":
         if debug:
-          print("Processing tag node {}".format(name))
+          print(f"Processing tagNode {name}")
 
         os.makedirs(make_path(my_tmpl_dir), exist_ok=True)
 
-        nodedef_path = os.path.join(make_path(my_tmpl_dir), "node.def")
-        if not os.path.exists(nodedef_path):
+        # Only create the "node.def" file if it exists but is empty, or if it
+        # does not exist at all.
+        if not os.path.exists(nodedef_path) or os.path.getsize(nodedef_path) == 0:
             with open(nodedef_path, "w") as f:
                 f.write('help: {0}\n'.format(props['help']))
-        else:
-            # Something has already generated this file
-            pass
 
         # Create the inner node.tag part
         my_tmpl_dir.append("node.tag")
@@ -201,24 +186,67 @@ def process_node(n, tmpl_dir):
             print("Created path for the tagNode: {}".format(make_path(my_tmpl_dir)), end="")
 
         # Not sure if we want partially defined tag nodes, write the file unconditionally
-        with open(os.path.join(make_path(my_tmpl_dir), "node.def"), "w") as f:
-            f.write(make_node_def(props, command))
+        nodedef_path = os.path.join(make_path(my_tmpl_dir), "node.def")
+        # Only create the "node.def" file if it exists but is empty, or if it
+        # does not exist at all.
+        if not os.path.exists(nodedef_path) or os.path.getsize(nodedef_path) == 0:
+            with open(nodedef_path, "w") as f:
+                f.write(make_node_def(props, command))
 
         if children is not None:
             inner_nodes = children.iterfind("*")
             for inner_n in inner_nodes:
                 process_node(inner_n, my_tmpl_dir)
-    else:
+    elif node_type == "leafNode":
         # This is a leaf node
         if debug:
-            print("Processing leaf node {}".format(name))
-
-        with open(os.path.join(make_path(my_tmpl_dir), "node.def"), "w") as f:
-            f.write(make_node_def(props, command))
+            print(f"Processing leaf node {name}")
 
+        if not os.path.exists(nodedef_path) or os.path.getsize(nodedef_path) == 0:
+            with open(nodedef_path, "w") as f:
+                f.write(make_node_def(props, command))
+    else:
+        print(f"Unknown node_type: {node_type}")
+
+
+def get_node_key(node, attr=None):
+    """ Return the sorting key of an xml node using tag and attributes """
+    if attr is None:
+        return '%s' % node.tag + ':'.join([node.get(attr)
+                                        for attr in sorted(node.attrib)])
+    if attr in node.attrib:
+        return '%s:%s' % (node.tag, node.get(attr))
+    return '%s' % node.tag
+
+
+def sort_children(node, attr=None):
+    """ Sort children along tag and given attribute. if attr is None, sort
+    along all attributes """
+    if not isinstance(node.tag, str):  # PYTHON 2: use basestring instead
+        # not a TAG, it is comment or DATA
+        # no need to sort
+        return
+    # sort child along attr
+    node[:] = sorted(node, key=lambda child: get_node_key(child, attr))
+    # and recurse
+    for child in node:
+        sort_children(child, attr)
 
 root = xml.getroot()
 
+# process_node() processes the XML tree in a fixed order, "node" before "tagNode"
+# before "leafNode". If the generator created a "node.def" file, it can no longer
+# be overwritten - else we would have some stale "node.def" files with an empty
+# help string (T2555). Without the fixed order this would resulted in a case
+# where we get a node and a tagNode with the same name, e.g. "show interfaces
+# ethernet" and "show interfaces ethernet eth0" that the node implementation
+# was not callable from the CLI, rendering this command useless (T3807).
+#
+# This can be fixed by forcing the "node", "tagNode", "leafNode" order by sorting
+# the input XML file automatically (sorting from https://stackoverflow.com/a/46128043)
+# thus adding no additional overhead to the user.
+sort_children(root, 'name')
+
 nodes = root.iterfind("*")
 for n in nodes:
     process_node(n, [output_dir])
diff --git a/scripts/build-component-versions b/scripts/build-component-versions
deleted file mode 100755
index 5362dbdd4..000000000
--- a/scripts/build-component-versions
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-import os
-import argparse
-import json
-
-from lxml import etree as ET
-
-parser = argparse.ArgumentParser()
-parser.add_argument('INPUT_DIR', type=str,
-                    help="Directory containing XML interface definition files")
-parser.add_argument('OUTPUT_DIR', type=str,
-                    help="Output directory for JSON file")
-
-args = parser.parse_args()
-
-input_dir = args.INPUT_DIR
-output_dir = args.OUTPUT_DIR
-
-version_dict = {}
-
-for filename in os.listdir(input_dir):
-    filepath = os.path.join(input_dir, filename)
-    print(filepath)
-    try:
-        xml = ET.parse(filepath)
-    except Exception as e:
-        print("Failed to load interface definition file {0}".format(filename))
-        print(e)
-        sys.exit(1)
-
-    root = xml.getroot()
-    version_data = root.iterfind("syntaxVersion")
-    for ver in version_data:
-        component = ver.get("component")
-        version = int(ver.get("version"))
-
-        v = version_dict.get(component)
-        if v is None:
-            version_dict[component] = version
-        elif version > v:
-            version_dict[component] = version
-
-out_file = os.path.join(output_dir, 'component-versions.json')
-with open(out_file, 'w') as f:
-    json.dump(version_dict, f, indent=4, sort_keys=True)
diff --git a/scripts/override-default b/scripts/override-default
index c8a0ff1da..0c49087c8 100755
--- a/scripts/override-default
+++ b/scripts/override-default
@@ -27,6 +27,7 @@
 import sys
 import glob
 import logging
+from copy import deepcopy
 from lxml import etree
 
 debug = False
@@ -60,30 +61,55 @@ def override_element(l: list):
     for el in parents:
         el.getparent().remove(el)
 
+def merge_remaining(l: list, elementtree):
+    """
+    Merge (now) single leaf node containing 'defaultValue' with leaf nodes
+    of same path and no 'defaultValue'.
+    """
+    for p in l:
+        p = p.split()
+        path_str = f'/interfaceDefinition/*'
+        path_list = []
+        for i in range(len(p)):
+            path_list.append(f'[@name="{p[i]}"]')
+        path_str += '/children/*'.join(path_list)
+        rp = elementtree.xpath(path_str)
+        if len(rp) > 1:
+            for el in rp[1:]:
+                # in practice there will only be one child of the path,
+                # either defaultValue or Properties, since
+                # override_element() has already run
+                for child in el:
+                    rp[0].append(deepcopy(child))
+                el.getparent().remove(el)
+
 def collect_and_override(dir_name):
     """
-    Collect elements with defaultValue tag into dictionary indexed by tuple
-    of (name: str, ancestor path: str).
+    Collect elements with defaultValue tag into dictionary indexed by name
+    attributes of ancestor path.
     """
     for fname in glob.glob(f'{dir_name}/*.xml'):
         tree = etree.parse(fname)
         root = tree.getroot()
         defv = {}
 
-        xpath_str = f'//defaultValue'
+        xpath_str = '//defaultValue'
         xp = tree.xpath(xpath_str)
 
         for element in xp:
             ap = element.xpath('ancestor::*[@name]')
             ap_name = [el.get("name") for el in ap]
-            ap_path_str = ' '.join(ap_name[:-1])
-            defv.setdefault((ap_name[-1], ap_path_str), []).append(element)
+            ap_path_str = ' '.join(ap_name)
+            defv.setdefault(ap_path_str, []).append(element)
 
         for k, v in defv.items():
             if len(v) > 1:
-                logger.info(f"overridding default in {k[0]}, path '{k[1]}'")
+                logger.info(f"overridding default in path '{k}'")
                 override_element(v)
 
+        to_merge = list(defv)
+        merge_remaining(to_merge, tree)
+
         revised_str = etree.tostring(root, encoding='unicode', pretty_print=True)
 
         with open(f'{fname}', 'w') as f: