import re import json import os from datetime import datetime from unittest import defaultTestLoader from docutils import io, nodes, utils, statemachine from docutils.parsers.rst.roles import set_classes from docutils.parsers.rst import Directive, directives, states from sphinx.util.docutils import SphinxDirective from testcoverage import get_working_commands, get_vyos_commands from sphinx.util import logging logger = logging.getLogger(__name__) def setup(app): app.add_config_value( 'vyos_phabricator_url', 'https://vyos.dev/', 'html' ) app.add_config_value( 'vyos_working_commands', get_working_commands(), #{"cfgcmd": [], "opcmd": []}, 'html' ) app.add_config_value( 'vyos_commands', get_vyos_commands(), 'html' ) app.add_config_value( 'vyos_coverage', { 'cfgcmd': [0,len(app.config.vyos_working_commands['cfgcmd'])], 'opcmd': [0,len(app.config.vyos_working_commands['opcmd'])] }, 'html' ) app.add_role('vytask', vytask_role) app.add_role('cfgcmd', cmd_role) app.add_role('opcmd', cmd_role) app.add_node( inlinecmd, html=(inlinecmd.visit_span, inlinecmd.depart_span), latex=(inlinecmd.visit_tex, inlinecmd.depart_tex), text=(inlinecmd.visit_span, inlinecmd.depart_span) ) app.add_node( CmdDiv, html=(CmdDiv.visit_div, CmdDiv.depart_div), latex=(CmdDiv.visit_tex, CmdDiv.depart_tex), text=(CmdDiv.visit_div, CmdDiv.depart_div) ) app.add_node( CmdBody, html=(CmdBody.visit_div, CmdBody.depart_div), latex=(CmdBody.visit_tex, CmdBody.depart_tex), text=(CmdBody.visit_div, CmdBody.depart_div) ) app.add_node( CmdHeader, html=(CmdHeader.visit_div, CmdHeader.depart_div), latex=(CmdHeader.tex, CmdHeader.tex), text=(CmdHeader.visit_div, CmdHeader.depart_div) ) app.add_node(CfgcmdList) app.add_node(CfgcmdListCoverage) app.add_directive('cfgcmdlist', CfgcmdlistDirective) app.add_node(OpcmdList) app.add_node(OpcmdListCoverage) app.add_directive('opcmdlist', OpcmdlistDirective) app.add_directive('cfgcmd', CfgCmdDirective) app.add_directive('opcmd', OpCmdDirective) app.add_directive('cmdinclude', CfgInclude) app.add_directive('cmdincludemd', CmdInclude) app.connect('doctree-resolved', process_cmd_nodes) app.connect('doctree-read', handle_document_meta_data) class CfgcmdList(nodes.General, nodes.Element): pass class OpcmdList(nodes.General, nodes.Element): pass class CfgcmdListCoverage(nodes.General, nodes.Element): pass class OpcmdListCoverage(nodes.General, nodes.Element): pass class CmdHeader(nodes.General, nodes.Element): @staticmethod def visit_div(self, node): self.body.append(self.starttag(node, 'div')) @staticmethod def depart_div(self, node=None): # self.body.append('\n') self.body.append('' % ( 'Permalink to this Command')) @staticmethod def tex(self, node=None): pass class CmdDiv(nodes.General, nodes.Element): @staticmethod def visit_div(self, node): self.body.append(self.starttag(node, 'div')) @staticmethod def depart_div(self, node=None): self.body.append('\n') @staticmethod def tex(self, node=None): pass @staticmethod def visit_tex(self, node=None): self.body.append('\n\n\\begin{changemargin}{0cm}{0cm}\n') @staticmethod def depart_tex(self, node=None): self.body.append('\n\\end{changemargin}\n\n') class CmdBody(nodes.General, nodes.Element): @staticmethod def visit_div(self, node): self.body.append(self.starttag(node, 'div')) @staticmethod def depart_div(self, node=None): self.body.append('\n') @staticmethod def visit_tex(self, node=None): self.body.append('\n\n\\begin{changemargin}{0.5cm}{0.5cm}\n') @staticmethod def depart_tex(self, node=None): self.body.append('\n\\end{changemargin}\n\n') @staticmethod def tex(self, node=None): pass class inlinecmd(nodes.inline): @staticmethod def visit_span(self, node): self.body.append(self.starttag(node, 'span')) @staticmethod def depart_span(self, node=None): self.body.append('\n') @staticmethod def visit_tex(self, node=None): self.body.append(r'\sphinxbfcode{\sphinxupquote{') #self.literal_whitespace += 1 @staticmethod def depart_tex(self, node=None): self.body.append(r'}}') #self.literal_whitespace -= 1 class CfgInclude(SphinxDirective): required_arguments = 1 optional_arguments = 0 final_argument_whitespace = True option_spec = { 'var0': str, 'var1': str, 'var2': str, 'var3': str, 'var4': str, 'var5': str, 'var6': str, 'var7': str, 'var8': str, 'var9': str } standard_include_path = os.path.join(os.path.dirname(states.__file__), 'include') def run(self): ### Copy from include directive docutils """Include a file as part of the content of this reST file.""" rel_filename, filename = self.env.relfn2path(self.arguments[0]) self.arguments[0] = filename self.env.note_included(filename) if not self.state.document.settings.file_insertion_enabled: raise self.warning('"%s" directive disabled.' % self.name) source = self.state_machine.input_lines.source( self.lineno - self.state_machine.input_offset - 1) source_dir = os.path.dirname(os.path.abspath(source)) path = directives.path(self.arguments[0]) if path.startswith('<') and path.endswith('>'): path = os.path.join(self.standard_include_path, path[1:-1]) path = os.path.normpath(os.path.join(source_dir, path)) path = utils.relative_path(None, path) path = nodes.reprunicode(path) encoding = self.options.get( 'encoding', self.state.document.settings.input_encoding) e_handler=self.state.document.settings.input_encoding_error_handler tab_width = self.options.get( 'tab-width', self.state.document.settings.tab_width) try: self.state.document.settings.record_dependencies.add(path) include_file = io.FileInput(source_path=path, encoding=encoding, error_handler=e_handler) except UnicodeEncodeError: raise self.severe(u'Problems with "%s" directive path:\n' 'Cannot encode input file path "%s" ' '(wrong locale?).' % (self.name, SafeString(path))) except IOError as error: raise self.severe(u'Problems with "%s" directive path:\n%s.' % (self.name, error)) startline = self.options.get('start-line', None) endline = self.options.get('end-line', None) try: if startline or (endline is not None): lines = include_file.readlines() rawtext = ''.join(lines[startline:endline]) else: rawtext = include_file.read() except UnicodeError: raise self.severe(u'Problem with "%s" directive:\n%s' % (self.name, ErrorString(error))) # start-after/end-before: no restrictions on newlines in match-text, # and no restrictions on matching inside lines vs. line boundaries after_text = self.options.get('start-after', None) if after_text: # skip content in rawtext before *and incl.* a matching text after_index = rawtext.find(after_text) if after_index < 0: raise self.severe('Problem with "start-after" option of "%s" ' 'directive:\nText not found.' % self.name) rawtext = rawtext[after_index + len(after_text):] before_text = self.options.get('end-before', None) if before_text: # skip content in rawtext after *and incl.* a matching text before_index = rawtext.find(before_text) if before_index < 0: raise self.severe('Problem with "end-before" option of "%s" ' 'directive:\nText not found.' % self.name) rawtext = rawtext[:before_index] include_lines = statemachine.string2lines(rawtext, tab_width, convert_whitespace=True) if 'literal' in self.options: # Convert tabs to spaces, if `tab_width` is positive. if tab_width >= 0: text = rawtext.expandtabs(tab_width) else: text = rawtext literal_block = nodes.literal_block(rawtext, source=path, classes=self.options.get('class', [])) literal_block.line = 1 self.add_name(literal_block) if 'number-lines' in self.options: try: startline = int(self.options['number-lines'] or 1) except ValueError: raise self.error(':number-lines: with non-integer ' 'start value') endline = startline + len(include_lines) if text.endswith('\n'): text = text[:-1] tokens = NumberLines([([], text)], startline, endline) for classes, value in tokens: if classes: literal_block += nodes.inline(value, value, classes=classes) else: literal_block += nodes.Text(value, value) else: literal_block += nodes.Text(text, text) return [literal_block] if 'code' in self.options: self.options['source'] = path codeblock = CodeBlock(self.name, [self.options.pop('code')], # arguments self.options, include_lines, # content self.lineno, self.content_offset, self.block_text, self.state, self.state_machine) return codeblock.run() new_include_lines = [] for line in include_lines: for i in range(10): value = self.options.get(f'var{i}','') if value == '': line = re.sub('\s?{{\s?var' + str(i) + '\s?}}',value,line) else: line = re.sub('{{\s?var' + str(i) + '\s?}}',value,line) new_include_lines.append(line) self.state_machine.insert_input(new_include_lines, path) return [] class CmdInclude(SphinxDirective): ''' 2nd CMDInclude only for Markdown, just the migration process ''' has_content = False required_arguments = 1 optional_arguments = 0 option_spec = { 'var0': str, 'var1': str, 'var2': str, 'var3': str, 'var4': str, 'var5': str, 'var6': str, 'var7': str, 'var8': str, 'var9': str } def run(self): include_file = self.env.relfn2path(self.arguments[0]) f = open(include_file[1], "r") file_content = f.readlines() f.close() new_include_lines = [] for line in file_content: for i in range(10): value = self.options.get(f'var{i}','') if value == '': line = re.sub('\s?{{\s?var' + str(i) + '\s?}}',value,line) else: line = re.sub('{{\s?var' + str(i) + '\s?}}',value,line) new_include_lines.append(line) self.state._renderer.nested_render_text(''.join(new_include_lines), self.lineno) return [] class CfgcmdlistDirective(Directive): has_content = False required_arguments = 0 option_spec = { 'show-coverage': directives.flag } def run(self): cfglist = CfgcmdList() cfglist['coverage'] = False if 'show-coverage' in self.options: cfglist['coverage'] = True return [cfglist] class OpcmdlistDirective(Directive): has_content = False required_arguments = 0 option_spec = { 'show-coverage': directives.flag } def run(self): oplist = OpcmdList() oplist['coverage'] = False if 'show-coverage' in self.options: oplist['coverage'] = True return [oplist] def get_default_value(title_text, config, cfgmode): title_text = strip_cmd(title_text) for cmd in config.vyos_working_commands[cfgmode]: cmd_joined = ' '.join(cmd['name']) cmd_striped = strip_cmd(cmd_joined) if "table-size" in cmd['name']: pass #print(cmd) #print(cmd_striped) #print(title_text) #print() if cmd_striped == title_text: if cmd['defaultvalue']: return cmd['defaultvalue'] return None class CmdDirective(SphinxDirective): has_content = True custom_class = '' def run(self): title_list = [] content_list = [] title_text = '' content_text = '' defaultvalue = None has_body = False cfgmode = self.custom_class + "cmd" try: if '' in self.content: index = self.content.index('') title_list = self.content[0:index] content_list = self.content[index + 1:] title_text = ' '.join(title_list) content_text = content_text + '\n'.join(content_list) has_body = True else: title_list = self.content title_text = ' '.join(title_list) except Exception as e: print("error", e) # render defaultvalue if os.getenv('VYOS_DEFAULT') or ':defaultvalue:' in title_text: value = get_default_value(title_list, self.config, cfgmode) title_text = title_text.replace(":defaultvalue:", '') if value: defaultvalue = f"default: {value}\n" anchor_id = nodes.make_id(self.custom_class + "cmd-" + title_text) target = nodes.target(ids=[anchor_id]) panel_name = 'cmd-{}'.format(self.custom_class) panel_element = CmdDiv() panel_element['classes'] += ['cmd', panel_name] heading_element = CmdHeader(title_text) title_nodes, messages = self.state.inline_text(title_text, self.lineno) title = inlinecmd(title_text, '', *title_nodes) target['classes'] += [] title['classes'] += [cfgmode] heading_element.append(target) heading_element.append(title) heading_element['classes'] += [self.custom_class + 'cmd-heading'] panel_element.append(heading_element) if defaultvalue: defaultvalue_element = nodes.paragraph(text=defaultvalue) defaultvalue_element['classes'] = ["defaultvalue"] panel_element.append(defaultvalue_element) append_list = { 'docname': self.env.docname, 'cmdnode': title.deepcopy(), 'cmd': title_text, 'target': target, } if cfgmode == 'opcmd': if not hasattr(self.env, "vyos_opcmd"): self.env.vyos_opcmd = [] self.env.vyos_opcmd.append(append_list) if cfgmode == 'cfgcmd': if not hasattr(self.env, "vyos_cfgcmd"): self.env.vyos_cfgcmd = [] self.env.vyos_cfgcmd.append(append_list) if has_body: body_element = CmdBody(content_text) self.state.nested_parse( content_list, self.content_offset, body_element ) body_element['classes'] += [self.custom_class + 'cmd-body'] panel_element.append(body_element) return [panel_element] class OpCmdDirective(CmdDirective): custom_class = 'op' class CfgCmdDirective(CmdDirective): custom_class = 'cfg' def strip_cmd(cmd, debug=False): # find all [...] and also nested [...] # regex and str.find() had problems with nested [...] appearance = 0 cmd_new = "" for c in cmd: if c == "[": appearance = appearance + 1 if appearance == 0: cmd_new = f"{cmd_new}{c}" if c == "]": appearance = appearance - 1 # only if all [..] will be delete if appearance > 0 there is a syntax error if appearance == 0: cmd = cmd_new # same for <...> appearance = 0 cmd_new = "" for c in cmd: if c == "<": appearance = appearance + 1 if appearance == 0: cmd_new = f"{cmd_new}{c}" if c == ">": appearance = appearance - 1 # only if all <..> will be delete if appearance > 0 there is a syntax error if appearance == 0: cmd = cmd_new if debug: print("") print(cmd) cmd = re.sub('^set','',cmd) if debug: print(cmd) cmd = cmd.replace('|','') if debug: print(cmd) cmd = re.sub('\s+','',cmd) cmd = cmd.replace(':defaultvalue:','') if debug: print(cmd) print("") return cmd def build_row(app, fromdocname, rowdata): row = nodes.row() for cell in rowdata: entry = nodes.entry() row += entry if isinstance(cell, list): for item in cell: if isinstance(item, dict): entry += process_cmd_node(app, item, fromdocname, '') else: entry += nodes.paragraph(text=item) elif isinstance(cell, bool): if cell: entry += nodes.paragraph(text="✔") entry['classes'] = ['coverage-ok'] else: entry += nodes.paragraph(text="✕") entry['classes'] = ['coverage-fail'] else: entry += nodes.paragraph(text=cell) return row def process_coverage(app, fromdocname, doccmd, xmlcmd, vyoscmd, cli_type): coverage_list = {} strip_true_list = [] for cmd in doccmd: coverage_item = { 'doccmd': None, 'xmlcmd': None, 'vyoscmd': None, 'doccmd_item': None, 'xmlcmd_item': None, 'vyoscmd_item': None, 'indocs': False, 'inxml': False, 'invyos': False, 'xmlfilename': None } coverage_item['doccmd'] = cmd['cmd'] coverage_item['doccmd_item'] = cmd coverage_item['indocs'] = True coverage_list[strip_cmd(cmd['cmd'])] = dict(coverage_item) for cmd in xmlcmd: strip = strip_cmd(cmd['cmd']) if strip not in coverage_list.keys(): coverage_item = { 'doccmd': None, 'xmlcmd': None, 'vyoscmd': None, 'doccmd_item': None, 'xmlcmd_item': None, 'vyoscmd_item': None, 'indocs': False, 'inxml': False, 'invyos': False, 'xmlfilename': None } coverage_item['xmlcmd'] = cmd['cmd'] coverage_item['xmlcmd_item'] = cmd coverage_item['inxml'] = True coverage_item['xmlfilename'] = cmd['filename'] coverage_list[strip] = dict(coverage_item) else: coverage_list[strip]['xmlcmd'] = cmd['cmd'] coverage_list[strip]['xmlcmd_item'] = cmd coverage_list[strip]['inxml'] = True coverage_list[strip]['xmlfilename'] = cmd['filename'] for item in vyoscmd[cli_type]: cmd = ' '.join(item['cmd']) strip = strip_cmd(cmd) if strip not in coverage_list.keys(): coverage_item = { 'doccmd': None, 'xmlcmd': None, 'vyoscmd': None, 'doccmd_item': None, 'xmlcmd_item': None, 'vyoscmd_item': None, 'indocs': False, 'inxml': False, 'invyos': False, 'xmlfilename': None } coverage_item['vyoscmd'] = cmd coverage_item['invyos'] = True coverage_list[strip] = dict(coverage_item) else: coverage_list[strip]['vyoscmd'] = cmd coverage_list[strip]['invyos'] = True if coverage_list[strip]['indocs'] and coverage_list[strip]['inxml']: strip_true_list.append(strip) strip_true_list = list(set(strip_true_list)) # to find syntax errors in cfg or cmd commands #for k in coverage_list.keys(): # if ("[" in k) or ("]" in k) or ("<" in k) or (">" in k) or ("|" in k): # print(coverage_list[k]) table = nodes.table() tgroup = nodes.tgroup(cols=4) table += tgroup header = (f'Status {len(strip_true_list)}/{len(coverage_list)}', 'Documentation', 'XML', f'in VyOS {vyoscmd["os"]}') colwidths = (5, 33 , 33, 33) table = nodes.table() tgroup = nodes.tgroup(cols=len(header)) table += tgroup for colwidth in colwidths: tgroup += nodes.colspec(colwidth=colwidth) thead = nodes.thead() tgroup += thead thead += build_row(app, fromdocname, header) tbody = nodes.tbody() tgroup += tbody for entry in sorted(coverage_list): doc_cmd_text = [] doc_xml_text = [] doc_vyos_text = [] if coverage_list[entry]['indocs']: doc_cmd_text.append(coverage_list[entry]['doccmd_item']) else: doc_cmd_text.append('not yet documented') if coverage_list[entry]['inxml']: doc_xml_text.append(str(coverage_list[entry]['xmlfilename']) + ":") doc_xml_text.append(coverage_list[entry]['xmlcmd']) else: doc_xml_text.append('Nothing found in XML Definitions') if coverage_list[entry]['invyos']: doc_vyos_text.append(coverage_list[entry]['vyoscmd']) else: doc_vyos_text.append('Nothing found in VyOS') if not coverage_list[entry]['indocs'] or not coverage_list[entry]['inxml'] or not coverage_list[entry]['invyos']: status = False else: status = True tbody += build_row(app, fromdocname, ( status, doc_cmd_text, doc_xml_text, doc_vyos_text ) ) table['ids'] = [f'table-{cli_type}'] return table def process_cmd_node(app, cmd, fromdocname, cli_type): para = nodes.paragraph() newnode = nodes.reference('', '') innernode = cmd['cmdnode'] newnode['refdocname'] = cmd['docname'] newnode['refuri'] = app.builder.get_relative_uri( fromdocname, cmd['docname']) newnode['refuri'] += '#' + cmd['target']['refid'] newnode['classes'] += ['cmdlink'] newnode.append(innernode) para += newnode return para def process_cmd_nodes(app, doctree, fromdocname): try: env = app.builder.env for node in doctree.traverse(CfgcmdList): content = [] if node.attributes['coverage']: node.replace_self( process_coverage( app, fromdocname, env.vyos_cfgcmd, app.config.vyos_working_commands['cfgcmd'], app.config.vyos_commands, 'cfgcmd' ) ) else: for cmd in sorted(env.vyos_cfgcmd, key=lambda i: i['cmd']): content.append(process_cmd_node(app, cmd, fromdocname, 'cfgcmd')) node.replace_self(content) for node in doctree.traverse(OpcmdList): content = [] if node.attributes['coverage']: node.replace_self( process_coverage( app, fromdocname, env.vyos_opcmd, app.config.vyos_working_commands['opcmd'], app.config.vyos_commands, 'opcmd' ) ) else: for cmd in sorted(env.vyos_opcmd, key=lambda i: i['cmd']): content.append(process_cmd_node(app, cmd, fromdocname, 'opcmd')) node.replace_self(content) except Exception as inst: print(inst) def vytask_role(name, rawtext, text, lineno, inliner, options={}, content=[]): app = inliner.document.settings.env.app base = app.config.vyos_phabricator_url ref = base + str(text) set_classes(options) node = nodes.reference( rawtext, utils.unescape(str(text)), refuri=ref, **options) return [node], [] def cmd_role(name, rawtext, text, lineno, inliner, options={}, content=[]): node = nodes.literal(text, text) return [node], [] def handle_document_meta_data(app, document): docname = app.env.docname lastproofread = app.env.metadata[docname].get('lastproofread', False) if lastproofread: try: lastproofread_time = datetime.strptime(lastproofread, '%Y-%m-%d') delta = datetime.now() - lastproofread_time if delta.days > 365: logger.warning(f'{delta.days} days since last proofread {app.env.doc2path(docname)}') except Exception as e: logger.warning(f'lastproofread meta data error in {app.env.doc2path(docname)}: {e}') else: pass #logger.warning(f'lastproofread meta data missing in {app.env.doc2path(docname)}')