''' generate json with all commands from xml for vyos documentation coverage ''' import sys import os import json import re import logging from io import BytesIO from lxml import etree as ET import shutil default_constraint_err_msg = "Invalid value" validator_dir = "" input_data = [ { "kind": "cfgcmd", "input_dir": "_include/vyos-1x/interface-definitions/", "schema_file": "_include/vyos-1x/schema/interface_definition.rng", "files": [] }, { "kind": "opcmd", "input_dir": "_include/vyos-1x/op-mode-definitions/", "schema_file": "_include/vyos-1x/schema/op-mode-definition.rng", "files": [] } ] node_data = { 'cfgcmd': {}, 'opcmd': {}, } def get_properties(p): props = {} props['valueless'] = False try: if p.find("valueless") is not None: props['valueless'] = True except: pass if p is None: return props # Get the help string try: props["help"] = p.find("help").text except: pass # Get value help strings try: vhe = p.findall("valueHelp") vh = [] for v in vhe: vh.append( (v.find("format").text, v.find("description").text) ) props["val_help"] = vh except: props["val_help"] = [] # Get the constraint statements error_msg = default_constraint_err_msg # Get the error message if it's there try: error_msg = p.find("constraintErrorMessage").text except: pass vce = p.find("constraint") vc = [] if vce is not None: # The old backend doesn't support multiple validators in OR mode # so we emulate it regexes = [] regex_elements = vce.findall("regex") if regex_elements is not None: regexes = list(map(lambda e: e.text.strip(), regex_elements)) if "" in regexes: print("Warning: empty regex, node will be accepting any value") validator_elements = vce.findall("validator") validators = [] if validator_elements is not None: for v in validator_elements: v_name = os.path.join(validator_dir, v.get("name")) # XXX: lxml returns None for empty arguments v_argument = None try: v_argument = v.get("argument") except: pass if v_argument is None: v_argument = "" validators.append("{0} {1}".format(v_name, v_argument)) regex_args = " ".join(map(lambda s: "--regex \\\'{0}\\\'".format(s), regexes)) validator_args = " ".join(map(lambda s: "--exec \\\"{0}\\\"".format(s), validators)) validator_script = '${vyos_libexec_dir}/validate-value.py' validator_string = "exec \"{0} {1} {2} --value \\\'$VAR(@)\\\'\"; \"{3}\"".format(validator_script, regex_args, validator_args, error_msg) props["constraint"] = validator_string # Get the completion help strings try: che = p.findall("completionHelp") ch = "" for c in che: scripts = c.findall("script") paths = c.findall("path") lists = c.findall("list") # Current backend doesn't support multiple allowed: tags # so we get to emulate it comp_exprs = [] for i in lists: comp_exprs.append("echo \"{0}\"".format(i.text)) for i in paths: comp_exprs.append("/bin/cli-shell-api listNodes {0}".format(i.text)) for i in scripts: comp_exprs.append("sh -c \"{0}\"".format(i.text)) comp_help = " && ".join(comp_exprs) props["comp_help"] = comp_help except: props["comp_help"] = [] # Get priority try: props["priority"] = p.find("priority").text except: pass # Get "multi" if p.find("multi") is not None: props["multi"] = True # Get "valueless" if p.find("valueless") is not None: props["valueless"] = True return props def process_node(n, f): props_elem = n.find("properties") children = n.find("children") command = n.find("command") children_nodes = [] owner = n.get("owner") node_type = n.tag name = n.get("name") props = get_properties(props_elem) if node_type != "node": if "valueless" not in props.keys(): props["type"] = "txt" if node_type == "tagNode": props["tag"] = "True" if node_type == "node" and children is not None: inner_nodes = children.iterfind("*") index_child = 0 for inner_n in inner_nodes: children_nodes.append(process_node(inner_n, f)) index_child = index_child + 1 if node_type == "tagNode" and children is not None: inner_nodes = children.iterfind("*") index_child = 0 for inner_n in inner_nodes: children_nodes.append(process_node(inner_n, f)) index_child = index_child + 1 else: # This is a leaf node pass if command is not None: test_command = True else: test_command = False node = { 'name': name, 'type': node_type, 'children': children_nodes, 'props': props, 'command': test_command, 'filename': f } return node def create_commands(data, parent_list=[], level=0): result = [] command = { 'name': [], 'help': None, 'tag_help': [], 'level': level, 'no_childs': False, 'filename': None } command['filename'] = data['filename'] command['name'].extend(parent_list) command['name'].append(data['name']) if data['type'] == 'tagNode': command['name'].append("<" + data['name'] + ">") if 'val_help' in data['props'].keys(): for val_help in data['props']['val_help']: command['tag_help'].append(val_help) if len(data['children']) == 0: command['no_childs'] = True if data['command']: command['no_childs'] = True try: help_text = data['props']['help'] command['help'] = re.sub(r"[\n\t]*", "", help_text) except: command['help'] = "" command['valueless'] = data['props']['valueless'] if 'children' in data.keys(): children_bool = True for child in data['children']: result.extend(create_commands(child, command['name'], level + 1)) if command['no_childs']: result.append(command) return result def include_file(line, input_dir): string = "" if "#include <include" in line.strip(): include_filename = line.strip().split('<')[1][:-1] with open(input_dir + include_filename) as ifp: iline = ifp.readline() while iline: string = string + include_file(iline.strip(), input_dir) iline = ifp.readline() else: string = line return string def get_working_commands(): for entry in input_data: for (dirpath, dirnames, filenames) in os.walk(entry['input_dir']): entry['files'].extend(filenames) break for f in entry['files']: string = "" with open(entry['input_dir'] + f) as fp: line = fp.readline() while line: string = string + include_file(line.strip(), entry['input_dir']) line = fp.readline() try: xml = ET.parse(BytesIO(bytes(string, 'utf-8'))) except Exception as e: print("Failed to load interface definition file {0}".format(f)) print(e) sys.exit(1) try: relaxng_xml = ET.parse(entry['schema_file']) validator = ET.RelaxNG(relaxng_xml) if not validator.validate(xml): print(validator.error_log) print("Interface definition file {0} does not match the schema!".format(f)) sys.exit(1) except Exception as e: print("Failed to load the XML schema {0}".format(entry['schema_file'])) print(e) sys.exit(1) root = xml.getroot() nodes = root.iterfind("*") for n in nodes: node_data[entry['kind']][f] = process_node(n, f) # build config tree and sort config_tree_new = { 'cfgcmd': {}, 'opcmd': {}, } for kind in node_data: for entry in node_data[kind]: node_0 = node_data[kind][entry]['name'] if node_0 not in config_tree_new[kind].keys(): config_tree_new[kind][node_0] = { 'name': node_0, 'type': node_data[kind][entry]['type'], 'props': node_data[kind][entry]['props'], 'children': [], 'command': node_data[kind][entry]['command'], 'filename': node_data[kind][entry]['filename'], } config_tree_new[kind][node_0]['children'].extend(node_data[kind][entry]['children']) result = { 'cfgcmd': [], 'opcmd': [], } for kind in config_tree_new: for e in config_tree_new[kind]: result[kind].extend(create_commands(config_tree_new[kind][e])) for cmd in result['cfgcmd']: cmd['cmd'] = " ".join(cmd['name']) for cmd in result['opcmd']: cmd['cmd'] = " ".join(cmd['name']) return result if __name__ == "__main__": res = get_working_commands() print(json.dumps(res)) #print(res['cfgcmd'][0])