T6315: added scripts for pr check

3 files changed, 241 insertions, 0 deletions

diff --git a/scripts/check-pr-title-and-commit-messages.py b/scripts/check-pr-title-and-commit-messages.py
new file mode 100644
index 0000000..001f6cf
--- /dev/null
+++ b/scripts/check-pr-title-and-commit-messages.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+import re
+import sys
+import time
+
+import requests
+
+# Use the same regex for PR title and commit messages for now
+title_regex = r'^(([a-zA-Z\-_.]+:\s)?)T\d+:\s+[^\s]+.*'
+commit_regex = title_regex
+
+def check_pr_title(title):
+    if not re.match(title_regex, title):
+        print("PR title '{}' does not match the required format!".format(title))
+        print("Valid title example: T99999: make IPsec secure")
+        sys.exit(1)
+
+def check_commit_message(title):
+    if not re.match(commit_regex, title):
+        print("Commit title '{}' does not match the required format!".format(title))
+        print("Valid title example: T99999: make IPsec secure")
+        sys.exit(1)
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print("Please specify pull request URL!")
+        sys.exit(1)
+
+    # There seems to be a race condition that causes this scripts to receive
+    # an incomplete PR object that is missing certain fields,
+    # which causes temporary CI failures that require re-running the script
+    #
+    # It's probably better to add a small delay to prevent that
+    time.sleep(5)
+
+    # Get the pull request object
+    pr = requests.get(sys.argv[1]).json()
+    if "title" not in pr:
+        print("The PR object does not have a title field!")
+        print("Did not receive a valid pull request object, please check the URL!")
+        sys.exit(1)
+
+    check_pr_title(pr["title"])
+
+    # Get the list of commits
+    commits = requests.get(pr["commits_url"]).json()
+    for c in commits:
+        # Retrieve every individual commit and check its title
+        co = requests.get(c["url"]).json()
+        check_commit_message(co["commit"]["message"])
diff --git a/scripts/override-default b/scripts/override-default
new file mode 100644
index 0000000..5058e79
--- /dev/null
+++ b/scripts/override-default
@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+#
+# override-default: preprocessor for XML interface definitions to interpret
+# redundant entries (relative to path) with tag 'defaultValue' as an override
+# directive. Must be called before build-command-templates, as the schema
+# disallows redundancy.
+#
+# Copyright (C) 2021 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+# Use lxml xpath capability to find multiple elements with tag defaultValue
+# relative to path; replace and remove to override the value.
+
+import sys
+import glob
+import logging
+from copy import deepcopy
+from lxml import etree
+
+debug = False
+
+logger = logging.getLogger(__name__)
+logs_handler = logging.StreamHandler()
+logger.addHandler(logs_handler)
+
+if debug:
+    logger.setLevel(logging.DEBUG)
+else:
+    logger.setLevel(logging.INFO)
+
+def clear_empty_path(el):
+    # on the odd chance of interleaved comments
+    tmp = [l for l in el if isinstance(l.tag, str)]
+    if not tmp:
+        p = el.getparent()
+        p.remove(el)
+        clear_empty_path(p)
+
+def override_element(l: list):
+    """
+    Allow multiple override elements; use the final one (in document order).
+    """
+    if len(l) < 2:
+        logger.debug("passing list of single element to override_element")
+        return
+
+    # assemble list of leafNodes of overriding defaultValues, for later removal
+    parents = []
+    for el in l[1:]:
+        parents.append(el.getparent())
+
+    # replace element with final override
+    l[0].getparent().replace(l[0], l[-1])
+
+    # remove all but overridden element
+    for el in parents:
+        tmp = el.getparent()
+        tmp.remove(el)
+        clear_empty_path(tmp)
+
+def merge_remaining(l: list, elementtree):
+    """
+    Merge (now) single leaf node containing 'defaultValue' with leaf nodes
+    of same path and no 'defaultValue'.
+    """
+    for p in l:
+        p = p.split()
+        path_str = f'/interfaceDefinition/*'
+        path_list = []
+        for i in range(len(p)):
+            path_list.append(f'[@name="{p[i]}"]')
+        path_str += '/children/*'.join(path_list)
+        rp = elementtree.xpath(path_str)
+        if len(rp) > 1:
+            for el in rp[1:]:
+                # in practice there will only be one child of the path,
+                # either defaultValue or Properties, since
+                # override_element() has already run
+                for child in el:
+                    rp[0].append(deepcopy(child))
+                tmp = el.getparent()
+                tmp.remove(el)
+                clear_empty_path(tmp)
+
+def collect_and_override(dir_name):
+    """
+    Collect elements with defaultValue tag into dictionary indexed by name
+    attributes of ancestor path.
+    """
+    for fname in glob.glob(f'{dir_name}/*.xml'):
+        tree = etree.parse(fname)
+        root = tree.getroot()
+        defv = {}
+
+        xpath_str = '//defaultValue'
+        xp = tree.xpath(xpath_str)
+
+        for element in xp:
+            ap = element.xpath('ancestor::*[@name]')
+            ap_name = [el.get("name") for el in ap]
+            ap_path_str = ' '.join(ap_name)
+            defv.setdefault(ap_path_str, []).append(element)
+
+        for k, v in defv.items():
+            if len(v) > 1:
+                logger.info(f"overriding default in path '{k}'")
+                override_element(v)
+
+        to_merge = list(defv)
+        merge_remaining(to_merge, tree)
+
+        revised_str = etree.tostring(root, encoding='unicode', pretty_print=True)
+
+        with open(f'{fname}', 'w') as f:
+            f.write(revised_str)
+
+def main():
+    if len(sys.argv) < 2:
+        logger.critical('Must specify XML directory!')
+        sys.exit(1)
+
+    dir_name = sys.argv[1]
+
+    collect_and_override(dir_name)
+
+if __name__ == '__main__':
+    main()
diff --git a/scripts/transclude-template b/scripts/transclude-template
new file mode 100644
index 0000000..5c6668a
--- /dev/null
+++ b/scripts/transclude-template
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+#
+# transclude-template: preprocessor for XML interface definitions to
+# interpret #include statements to include nested XML fragments and
+# snippets in documents.
+#
+# Copyright (C) 2021 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import os
+import re
+import sys
+
+regexp = re.compile(r'^ *#include <(.+)>$')
+
+def parse_file(filename):
+    lines = ""
+    with open(filename, 'r') as f:
+        while True:
+            line = f.readline()
+            if line:
+                result = regexp.match(line)
+                if result:
+                    lines += parse_file(os.path.join(directory, result.group(1)))
+                else:
+                    lines += line
+            else:
+                return lines
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print('Must specify XML file!', file=sys.stderr)
+        sys.exit(1)
+    filename = sys.argv[1]
+    directory = os.path.dirname(os.path.abspath(filename))
+    print(parse_file(filename))
+