Merge pull request #1 from vyos/master

Update fork

1 files changed, 381 insertions, 28 deletions

diff --git a/docs/_ext/vyos.py b/docs/_ext/vyos.py
index 4ee87d63..46ebae36 100644
--- a/docs/_ext/vyos.py
+++ b/docs/_ext/vyos.py
@@ -1,21 +1,42 @@
-from docutils import nodes, utils
+import re
+import json
+import os
+from docutils import io, nodes, utils, statemachine
 from docutils.parsers.rst.roles import set_classes
-from docutils.parsers.rst import Directive
+from docutils.parsers.rst import Directive, directives, states
+
 from sphinx.util.docutils import SphinxDirective
 
+from testcoverage import get_working_commands
+
 
 def setup(app):
 
     app.add_config_value(
         'vyos_phabricator_url',
-        'https://phabricator.vyos.net/', ''
+        'https://phabricator.vyos.net/',
+        'html'
+    )
+
+    app.add_config_value(
+        'vyos_working_commands',
+        get_working_commands(),
+        #{"cfgcmd": [], "opcmd": []},
+        'html'
+    )
+    app.add_config_value(
+        'vyos_coverage',
+        {
+            'cfgcmd': [0,len(app.config.vyos_working_commands['cfgcmd'])],
+            'opcmd': [0,len(app.config.vyos_working_commands['opcmd'])]
+        },
+        'html'
     )
+
     app.add_role('vytask', vytask_role)
     app.add_role('cfgcmd', cmd_role)
     app.add_role('opcmd', cmd_role)
 
-    print(app.config.vyos_phabricator_url)
-
     app.add_node(
         inlinecmd,
         html=(inlinecmd.visit_span, inlinecmd.depart_span),
@@ -42,24 +63,29 @@ def setup(app):
         text=(CmdHeader.visit_div, CmdHeader.depart_div)
     )
     app.add_node(CfgcmdList)
+    app.add_node(CfgcmdListCoverage)
     app.add_directive('cfgcmdlist', CfgcmdlistDirective)
 
     app.add_node(OpcmdList)
+    app.add_node(OpcmdListCoverage)
     app.add_directive('opcmdlist', OpcmdlistDirective)
 
     app.add_directive('cfgcmd', CfgCmdDirective)
     app.add_directive('opcmd', OpCmdDirective)
+    app.add_directive('cmdinclude', CfgInclude)
     app.connect('doctree-resolved', process_cmd_nodes)
 
-
 class CfgcmdList(nodes.General, nodes.Element):
     pass
 
-
 class OpcmdList(nodes.General, nodes.Element):
     pass
 
-import json
+class CfgcmdListCoverage(nodes.General, nodes.Element):
+    pass
+
+class OpcmdListCoverage(nodes.General, nodes.Element):
+    pass
 
 class CmdHeader(nodes.General, nodes.Element):
 
@@ -148,16 +174,177 @@ class inlinecmd(nodes.inline):
         #self.literal_whitespace -= 1
 
 
+class CfgInclude(SphinxDirective):
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = True
+    option_spec = {
+        'var0': str,
+        'var1': str,
+        'var2': str,
+        'var3': str,
+        'var4': str,
+        'var5': str,
+        'var6': str,
+        'var7': str,
+        'var8': str,
+        'var9': str
+    }
+    standard_include_path = os.path.join(os.path.dirname(states.__file__),
+                                         'include')
+
+    def run(self):
+        ### Copy from include directive docutils 
+        """Include a file as part of the content of this reST file."""
+        rel_filename, filename = self.env.relfn2path(self.arguments[0])
+        self.arguments[0] = filename
+        self.env.note_included(filename)
+        if not self.state.document.settings.file_insertion_enabled:
+            raise self.warning('"%s" directive disabled.' % self.name)
+        source = self.state_machine.input_lines.source(
+            self.lineno - self.state_machine.input_offset - 1)
+        source_dir = os.path.dirname(os.path.abspath(source))
+        path = directives.path(self.arguments[0])
+        if path.startswith('<') and path.endswith('>'):
+            path = os.path.join(self.standard_include_path, path[1:-1])
+        path = os.path.normpath(os.path.join(source_dir, path))
+        path = utils.relative_path(None, path)
+        path = nodes.reprunicode(path)
+        encoding = self.options.get(
+            'encoding', self.state.document.settings.input_encoding)
+        e_handler=self.state.document.settings.input_encoding_error_handler
+        tab_width = self.options.get(
+            'tab-width', self.state.document.settings.tab_width)
+        try:
+            self.state.document.settings.record_dependencies.add(path)
+            include_file = io.FileInput(source_path=path,
+                                        encoding=encoding,
+                                        error_handler=e_handler)
+        except UnicodeEncodeError:
+            raise self.severe(u'Problems with "%s" directive path:\n'
+                              'Cannot encode input file path "%s" '
+                              '(wrong locale?).' %
+                              (self.name, SafeString(path)))
+        except IOError as error:
+            raise self.severe(u'Problems with "%s" directive path:\n%s.' %
+                      (self.name, error))
+        startline = self.options.get('start-line', None)
+        endline = self.options.get('end-line', None)
+        try:
+            if startline or (endline is not None):
+                lines = include_file.readlines()
+                rawtext = ''.join(lines[startline:endline])
+            else:
+                rawtext = include_file.read()
+        except UnicodeError:
+            raise self.severe(u'Problem with "%s" directive:\n%s' %
+                              (self.name, ErrorString(error)))
+        # start-after/end-before: no restrictions on newlines in match-text,
+        # and no restrictions on matching inside lines vs. line boundaries
+        after_text = self.options.get('start-after', None)
+        if after_text:
+            # skip content in rawtext before *and incl.* a matching text
+            after_index = rawtext.find(after_text)
+            if after_index < 0:
+                raise self.severe('Problem with "start-after" option of "%s" '
+                                  'directive:\nText not found.' % self.name)
+            rawtext = rawtext[after_index + len(after_text):]
+        before_text = self.options.get('end-before', None)
+        if before_text:
+            # skip content in rawtext after *and incl.* a matching text
+            before_index = rawtext.find(before_text)
+            if before_index < 0:
+                raise self.severe('Problem with "end-before" option of "%s" '
+                                  'directive:\nText not found.' % self.name)
+            rawtext = rawtext[:before_index]
+
+        include_lines = statemachine.string2lines(rawtext, tab_width,
+                                                  convert_whitespace=True)
+        if 'literal' in self.options:
+            # Convert tabs to spaces, if `tab_width` is positive.
+            if tab_width >= 0:
+                text = rawtext.expandtabs(tab_width)
+            else:
+                text = rawtext
+            literal_block = nodes.literal_block(rawtext, source=path,
+                                    classes=self.options.get('class', []))
+            literal_block.line = 1
+            self.add_name(literal_block)
+            if 'number-lines' in self.options:
+                try:
+                    startline = int(self.options['number-lines'] or 1)
+                except ValueError:
+                    raise self.error(':number-lines: with non-integer '
+                                     'start value')
+                endline = startline + len(include_lines)
+                if text.endswith('\n'):
+                    text = text[:-1]
+                tokens = NumberLines([([], text)], startline, endline)
+                for classes, value in tokens:
+                    if classes:
+                        literal_block += nodes.inline(value, value,
+                                                      classes=classes)
+                    else:
+                        literal_block += nodes.Text(value, value)
+            else:
+                literal_block += nodes.Text(text, text)
+            return [literal_block]
+        if 'code' in self.options:
+            self.options['source'] = path
+            codeblock = CodeBlock(self.name,
+                                  [self.options.pop('code')], # arguments
+                                  self.options,
+                                  include_lines, # content
+                                  self.lineno,
+                                  self.content_offset,
+                                  self.block_text,
+                                  self.state,
+                                  self.state_machine)
+            return codeblock.run()
+        
+        new_include_lines = []
+        for line in include_lines:
+            for i in range(10):
+                value = self.options.get(f'var{i}','')
+                if value == '':
+                    line = re.sub('\s?{{\s?var' + str(i) + '\s?}}',value,line)
+                else:
+                    line = re.sub('{{\s?var' + str(i) + '\s?}}',value,line)
+            new_include_lines.append(line)
+        self.state_machine.insert_input(new_include_lines, path)
+        return []
+
+
 class CfgcmdlistDirective(Directive):
+    has_content = False
+    required_arguments = 0
+    option_spec = {
+        'show-coverage': directives.flag
+    }
 
     def run(self):
-        return [CfgcmdList('')]
+        cfglist = CfgcmdList()
+        cfglist['coverage'] = False
+        if 'show-coverage' in self.options:
+            cfglist['coverage'] = True
+        return [cfglist]
 
 
 class OpcmdlistDirective(Directive):
+    has_content = False
+    required_arguments = 0
+    option_spec = {
+        'show-coverage': directives.flag
+    }
 
     def run(self):
-        return [OpcmdList('')]
+        oplist = OpcmdList()
+        oplist['coverage'] = False
+        if 'show-coverage' in self.options:
+            oplist['coverage'] = True
+            
+        return [oplist]
+
 
 
 class CmdDirective(SphinxDirective):
@@ -165,7 +352,8 @@ class CmdDirective(SphinxDirective):
     has_content = True
     custom_class = ''
 
-    def run(self):
+    def run(self):        
+
         title_list = []
         content_list = []
         title_text = ''
@@ -243,7 +431,148 @@ class CfgCmdDirective(CmdDirective):
     custom_class = 'cfg'
 
 
-def process_cmd_node(app, cmd, fromdocname):
+def strip_cmd(cmd, debug=False):
+    if debug:
+        print("")
+        print(cmd)
+    cmd = re.sub('set','',cmd)
+    if debug:
+        print(cmd)
+    #while " | " in cmd:
+    cmd = re.sub('\s+\|\s+','',cmd)
+    if debug:
+        print(cmd)
+    cmd = re.sub('<\S*>','',cmd)
+    if debug:
+        print(cmd)
+    cmd = re.sub('\[\S\]','',cmd)
+    if debug:
+        print(cmd)
+    cmd = re.sub('\s+','',cmd)
+    if debug:
+        print(cmd)
+        print("")
+    return cmd
+
+def build_row(app, fromdocname, rowdata):   
+    row = nodes.row()
+    for cell in rowdata:
+        entry = nodes.entry()
+        row += entry
+        if isinstance(cell, list):
+            for item in cell:
+                if isinstance(item, dict):
+                    entry += process_cmd_node(app, item, fromdocname, '')
+                else:
+                    entry += nodes.paragraph(text=item)
+        elif isinstance(cell, bool):
+            if cell:
+                entry += nodes.paragraph(text="")
+                entry['classes'] = ['coverage-ok']
+            else:
+                entry += nodes.paragraph(text="")
+                entry['classes'] = ['coverage-fail']
+        else:
+            entry += nodes.paragraph(text=cell)
+    return row
+
+
+
+def process_coverage(app, fromdocname, doccmd, xmlcmd, cli_type):
+    coverage_list = {}
+    int_docs = 0
+    int_xml = 0
+    for cmd in doccmd:
+        coverage_item = {
+            'doccmd': None,
+            'xmlcmd': None,
+            'doccmd_item': None,
+            'xmlcmd_item': None,
+            'indocs': False,
+            'inxml': False,
+            'xmlfilename': None
+        }
+        coverage_item['doccmd'] = cmd['cmd']
+        coverage_item['doccmd_item'] = cmd
+        coverage_item['indocs'] = True
+        int_docs += 1
+
+        coverage_list[strip_cmd(cmd['cmd'])] = dict(coverage_item)
+
+    
+    #print(coverage_list.keys())
+    
+    for cmd in xmlcmd:
+        
+        strip = strip_cmd(cmd['cmd'])
+        if strip not in coverage_list.keys():
+            coverage_item = {
+                'doccmd': None,
+                'xmlcmd': None,
+                'doccmd_item': None,
+                'xmlcmd_item': None,
+                'indocs': False,
+                'inxml': False,
+                'xmlfilename': None
+            }
+            coverage_item['xmlcmd'] = cmd['cmd']
+            coverage_item['xmlcmd_item'] = cmd
+            coverage_item['inxml'] = True
+            coverage_item['xmlfilename'] = cmd['filename']
+            int_xml += 1
+            coverage_list[strip] = dict(coverage_item)
+        else:
+            coverage_list[strip]['xmlcmd'] = cmd['cmd']
+            coverage_list[strip]['xmlcmd_item'] = cmd
+            coverage_list[strip]['inxml'] = True
+            coverage_list[strip]['xmlfilename'] = cmd['filename']
+            int_xml += 1
+
+
+    
+
+    table = nodes.table()
+    tgroup = nodes.tgroup(cols=3)
+    table += tgroup
+
+    header = (f'{int_docs}/{len(coverage_list)} in Docs', f'{int_xml}/{len(coverage_list)} in XML', 'Command')
+    colwidths = (1, 1, 8)
+    table = nodes.table()
+    tgroup = nodes.tgroup(cols=len(header))
+    table += tgroup
+    for colwidth in colwidths:
+        tgroup += nodes.colspec(colwidth=colwidth)
+    thead = nodes.thead()
+    tgroup += thead
+    thead += build_row(app, fromdocname, header)
+    tbody = nodes.tbody()
+    tgroup += tbody
+    for entry in sorted(coverage_list):
+        body_text_list = []
+        if coverage_list[entry]['indocs']:
+            body_text_list.append(coverage_list[entry]['doccmd_item'])
+        else:
+            body_text_list.append('Not documented yet')
+
+        if coverage_list[entry]['inxml']:
+            body_text_list.append("------------------")
+            body_text_list.append(str(coverage_list[entry]['xmlfilename']) + ":")
+            body_text_list.append(coverage_list[entry]['xmlcmd'])
+        else:
+            body_text_list.append('Nothing found in XML Definitions')
+
+            
+        tbody += build_row(app, fromdocname, 
+            (
+                coverage_list[entry]['indocs'],
+                coverage_list[entry]['inxml'],
+                body_text_list
+            )
+        )
+
+    return table
+
+def process_cmd_node(app, cmd, fromdocname, cli_type):
     para = nodes.paragraph()
     newnode = nodes.reference('', '')
     innernode = cmd['cmdnode']
@@ -258,21 +587,45 @@ def process_cmd_node(app, cmd, fromdocname):
 
 
 def process_cmd_nodes(app, doctree, fromdocname):
-    env = app.builder.env
-
-    for node in doctree.traverse(CfgcmdList):
-        content = []
-
-        for cmd in sorted(env.vyos_cfgcmd, key=lambda i: i['cmd']):
-            content.append(process_cmd_node(app, cmd, fromdocname))
-        node.replace_self(content)
-
-    for node in doctree.traverse(OpcmdList):
-        content = []
-
-        for cmd in sorted(env.vyos_opcmd, key=lambda i: i['cmd']):
-            content.append(process_cmd_node(app, cmd, fromdocname))
-        node.replace_self(content)
+    try:
+        env = app.builder.env
+        
+        for node in doctree.traverse(CfgcmdList):
+            content = []
+            if node.attributes['coverage']:
+                node.replace_self(
+                    process_coverage(
+                        app,
+                        fromdocname,
+                        env.vyos_cfgcmd,
+                        app.config.vyos_working_commands['cfgcmd'],
+                        'cfgcmd'
+                        )
+                    )
+            else:
+                for cmd in sorted(env.vyos_cfgcmd, key=lambda i: i['cmd']):
+                    content.append(process_cmd_node(app, cmd, fromdocname, 'cfgcmd'))                
+                node.replace_self(content)
+            
+        for node in doctree.traverse(OpcmdList):
+            content = []
+            if node.attributes['coverage']:
+                node.replace_self(
+                    process_coverage(
+                        app,
+                        fromdocname,
+                        env.vyos_opcmd,
+                        app.config.vyos_working_commands['opcmd'],
+                        'opcmd'
+                        )
+                    )
+            else:
+                for cmd in sorted(env.vyos_opcmd, key=lambda i: i['cmd']):
+                    content.append(process_cmd_node(app, cmd, fromdocname, 'opcmd'))
+                node.replace_self(content)
+
+    except Exception as inst:
+        print(inst)
 
 
 def vytask_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
@@ -287,4 +640,4 @@ def vytask_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
 
 def cmd_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
     node = nodes.literal(text, text)
-    return [node], []
+    return [node], []
\ No newline at end of file