Merge pull request #1 from vyos/master

Update fork

1 files changed, 382 insertions, 0 deletions

diff --git a/docs/_ext/testcoverage.py b/docs/_ext/testcoverage.py
new file mode 100644
index 00000000..39028912
--- /dev/null
+++ b/docs/_ext/testcoverage.py
@@ -0,0 +1,382 @@
+'''
+generate json with all commands from xml for vyos documentation coverage
+
+'''
+
+
+import sys
+import os
+import json
+import re
+import logging
+
+from io import BytesIO
+from lxml import etree as ET
+import shutil
+
+default_constraint_err_msg = "Invalid value"
+validator_dir = ""
+
+
+input_data = [
+    {
+        "kind": "cfgcmd",
+        "input_dir": "_include/vyos-1x/interface-definitions/",
+        "schema_file": "_include/vyos-1x/schema/interface_definition.rng",
+        "files": []
+    },
+    {
+        "kind": "opcmd",
+        "input_dir": "_include/vyos-1x/op-mode-definitions/",
+        "schema_file": "_include/vyos-1x/schema/op-mode-definition.rng",
+        "files": []
+    }
+]
+
+node_data = {
+    'cfgcmd': {},
+    'opcmd': {},
+}
+
+def get_properties(p):
+    props = {}
+    props['valueless'] = False
+
+    try:
+        if p.find("valueless") is not None:
+            props['valueless'] = True
+    except:
+        pass
+
+    if p is None:
+        return props
+
+    # Get the help string
+    try:
+        props["help"] = p.find("help").text
+    except:
+        pass
+
+    # Get value help strings
+    try:
+        vhe = p.findall("valueHelp")
+        vh = []
+        for v in vhe:
+            vh.append( (v.find("format").text, v.find("description").text) )
+        props["val_help"] = vh
+    except:
+        props["val_help"] = []
+
+    # Get the constraint statements
+    error_msg = default_constraint_err_msg
+    # Get the error message if it's there
+    try:
+        error_msg = p.find("constraintErrorMessage").text
+    except:
+        pass
+    
+
+    vce = p.find("constraint")
+    vc = []
+    if vce is not None:
+        # The old backend doesn't support multiple validators in OR mode
+        # so we emulate it
+
+        regexes = []
+        regex_elements = vce.findall("regex")
+        if regex_elements is not None:
+            regexes = list(map(lambda e: e.text.strip(), regex_elements))
+        if "" in regexes:
+            print("Warning: empty regex, node will be accepting any value")
+
+        validator_elements = vce.findall("validator")
+        validators = []
+        if validator_elements is not None:
+            for v in validator_elements:
+                v_name = os.path.join(validator_dir, v.get("name"))
+
+                # XXX: lxml returns None for empty arguments
+                v_argument = None
+                try:
+                    v_argument = v.get("argument")
+                except:
+                    pass
+                if v_argument is None:
+                    v_argument = ""
+
+                validators.append("{0} {1}".format(v_name, v_argument))
+
+
+        regex_args = " ".join(map(lambda s: "--regex \\\'{0}\\\'".format(s), regexes))
+        validator_args = " ".join(map(lambda s: "--exec \\\"{0}\\\"".format(s), validators))
+        validator_script = '${vyos_libexec_dir}/validate-value.py'
+        validator_string = "exec \"{0} {1} {2} --value \\\'$VAR(@)\\\'\"; \"{3}\"".format(validator_script, regex_args, validator_args, error_msg)
+
+        props["constraint"] = validator_string
+
+    # Get the completion help strings
+    try:
+        che = p.findall("completionHelp")
+        ch = ""
+        for c in che:
+            scripts = c.findall("script")
+            paths = c.findall("path")
+            lists = c.findall("list")
+
+            # Current backend doesn't support multiple allowed: tags
+            # so we get to emulate it
+            comp_exprs = []
+            for i in lists:
+                comp_exprs.append("echo \"{0}\"".format(i.text))
+            for i in paths:
+                comp_exprs.append("/bin/cli-shell-api listNodes {0}".format(i.text))
+            for i in scripts:
+                comp_exprs.append("sh -c \"{0}\"".format(i.text))
+            comp_help = " && ".join(comp_exprs)
+            props["comp_help"] = comp_help
+    except:
+        props["comp_help"] = []
+
+    # Get priority
+    try:
+        props["priority"] = p.find("priority").text
+    except:
+        pass
+
+    # Get "multi"
+    if p.find("multi") is not None:
+        props["multi"] = True
+
+    # Get "valueless"
+    if p.find("valueless") is not None:
+        props["valueless"] = True
+
+    return props
+
+def process_node(n, f):
+
+    props_elem = n.find("properties")
+    children = n.find("children")
+    command = n.find("command")
+    children_nodes = []
+    owner = n.get("owner")
+    node_type = n.tag
+
+    name = n.get("name")
+    props = get_properties(props_elem)
+
+    if node_type != "node":
+        if "valueless" not in props.keys():
+            props["type"] = "txt"
+    if node_type == "tagNode":
+        props["tag"] = "True"
+    
+    if node_type == "node" and children is not None:
+        inner_nodes = children.iterfind("*")
+        index_child = 0
+        for inner_n in inner_nodes:
+            children_nodes.append(process_node(inner_n, f))
+            index_child = index_child + 1
+
+    if node_type == "tagNode" and children is not None:
+        inner_nodes = children.iterfind("*")
+        index_child = 0
+        for inner_n in inner_nodes:
+            children_nodes.append(process_node(inner_n, f))
+            index_child = index_child + 1
+    else:
+        # This is a leaf node
+        pass
+    
+    if command is not None:
+        test_command = True
+    else:
+        test_command = False
+    node = {
+        'name': name,
+        'type': node_type,
+        'children': children_nodes,
+        'props': props,
+        'command': test_command,
+        'filename': f
+    }
+    return node
+
+
+
+def create_commands(data, parent_list=[], level=0):
+    result = []
+    command = {
+        'name': [],
+        'help': None,
+        'tag_help': [],
+        'level': level,
+        'no_childs': False,
+        'filename': None
+    }
+    command['filename'] = data['filename']
+    command['name'].extend(parent_list)
+    command['name'].append(data['name'])
+
+    if data['type'] == 'tagNode':
+        command['name'].append("<" + data['name'] + ">")
+
+    if 'val_help' in data['props'].keys():
+        for val_help in data['props']['val_help']:
+            command['tag_help'].append(val_help)
+    
+    if len(data['children']) == 0:
+        command['no_childs'] = True
+    
+    if data['command']:
+        command['no_childs'] = True
+    
+    try:
+        help_text = data['props']['help']
+        command['help'] = re.sub(r"[\n\t]*", "", help_text)
+        
+    except:
+        command['help'] = ""
+    
+    command['valueless'] = data['props']['valueless']
+    
+    if 'children' in data.keys():
+        children_bool = True
+        for child in data['children']:
+            result.extend(create_commands(child, command['name'], level + 1))
+    
+    if command['no_childs']:
+        result.append(command)
+    
+
+
+    return result
+
+
+def include_file(line, input_dir):
+    string = ""
+    if "#include <include" in line.strip():
+        include_filename = line.strip().split('<')[1][:-1]
+        with open(input_dir + include_filename) as ifp:
+            iline = ifp.readline()
+            while iline:
+                string = string + include_file(iline.strip(), input_dir)
+                iline = ifp.readline()
+    else:
+        string = line
+    return string
+
+
+def get_working_commands():
+    for entry in input_data:
+        for (dirpath, dirnames, filenames) in os.walk(entry['input_dir']):
+            entry['files'].extend(filenames)
+            break
+
+        for f in entry['files']:
+
+            string = ""
+            with open(entry['input_dir'] + f) as fp:
+                line = fp.readline()
+                while line:                
+                    string = string + include_file(line.strip(), entry['input_dir'])
+                    line = fp.readline()
+
+            try:
+                xml = ET.parse(BytesIO(bytes(string, 'utf-8')))
+            except Exception as e:
+                print("Failed to load interface definition file {0}".format(f))
+                print(e)
+                sys.exit(1)
+
+            override_defaults(xml)
+            
+            try:
+                relaxng_xml = ET.parse(entry['schema_file'])
+                validator = ET.RelaxNG(relaxng_xml)
+
+                if not validator.validate(xml):
+                    print(validator.error_log)
+                    print("Interface definition file {0} does not match the schema!".format(f))
+                    sys.exit(1)
+            except Exception as e:
+                print("Failed to load the XML schema {0}".format(entry['schema_file']))
+                print(e)
+                sys.exit(1)
+
+            root = xml.getroot()
+            nodes = root.iterfind("*")
+            for n in nodes:
+                node_data[entry['kind']][f] = process_node(n, f)
+
+    # build config tree and sort
+
+    config_tree_new = {
+        'cfgcmd': {},
+        'opcmd': {},
+    }
+
+    for kind in node_data:
+        for entry in node_data[kind]:
+            node_0 = node_data[kind][entry]['name']
+            
+            if node_0 not in config_tree_new[kind].keys():
+                config_tree_new[kind][node_0] = {
+                    'name': node_0,
+                    'type': node_data[kind][entry]['type'],
+                    'props': node_data[kind][entry]['props'],
+                    'children': [],
+                    'command': node_data[kind][entry]['command'],
+                    'filename': node_data[kind][entry]['filename'],
+                }
+            config_tree_new[kind][node_0]['children'].extend(node_data[kind][entry]['children'])
+    
+    result = {
+        'cfgcmd': [],
+        'opcmd': [],
+    }
+    for kind in  config_tree_new:
+        for e in config_tree_new[kind]:
+            result[kind].extend(create_commands(config_tree_new[kind][e]))
+    
+    for cmd in result['cfgcmd']:
+        cmd['cmd'] = " ".join(cmd['name'])
+    for cmd in result['opcmd']:
+        cmd['cmd'] = " ".join(cmd['name'])
+    return result
+
+def override_defaults(xml):
+    root = xml.getroot()
+    defv = {}
+
+    xpath_str = f'//defaultValue'
+    xp = xml.xpath(xpath_str)
+
+    for element in xp:
+        ap = element.xpath('ancestor::*[@name]')
+        defv.setdefault((ap[-1].get("name"), str(ap[:-1])), []).append(element)
+
+    for k, v in defv.items():
+        if len(v) > 1:
+            override_element(v)
+
+def override_element(l: list):
+    if len(l) < 2:
+        return
+
+    # assemble list of leafNodes of overriding defaultValues, for later removal
+    parents = []
+    for el in l[1:]:
+        parents.append(el.getparent())
+
+    # replace element with final override
+    l[0].getparent().replace(l[0], l[-1])
+
+    # remove all but overridden element
+    for el in parents:
+        el.getparent().remove(el)
+
+if __name__ == "__main__":
+    res = get_working_commands()
+    print(json.dumps(res))
+    #print(res['cfgcmd'][0])