T6732: added same as vyos 1x

6 files changed, 1134 insertions, 0 deletions

diff --git a/scripts/build-command-op-templates b/scripts/build-command-op-templates
new file mode 100644
index 0000000..d203fdc
--- /dev/null
+++ b/scripts/build-command-op-templates
@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+#
+#    build-command-template: converts new style command definitions in XML
+#      to the old style (bunch of dirs and node.def's) command templates
+#
+#    Copyright (C) 2017-2024 VyOS maintainers <maintainers@vyos.net>
+#
+#    This library is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU Lesser General Public
+#    License as published by the Free Software Foundation; either
+#    version 2.1 of the License, or (at your option) any later version.
+#
+#    This library is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#    Lesser General Public License for more details.
+#
+#    You should have received a copy of the GNU Lesser General Public
+#    License along with this library; if not, write to the Free Software
+#    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
+#    USA
+
+import re
+import sys
+import os
+import argparse
+import copy
+import functools
+
+from lxml import etree as ET
+from textwrap import fill
+
+# Defaults
+validator_dir = "/opt/vyatta/libexec/validators"
+default_constraint_err_msg = "Invalid value"
+
+## Get arguments
+parser = argparse.ArgumentParser(description='Converts new-style XML interface definitions to old-style command templates')
+parser.add_argument('--debug', help='Enable debug information output', action='store_true')
+parser.add_argument('INPUT_FILE', type=str, help="XML interface definition file")
+parser.add_argument('SCHEMA_FILE', type=str, help="RelaxNG schema file")
+parser.add_argument('OUTPUT_DIR', type=str, help="Output directory")
+
+args = parser.parse_args()
+
+input_file = args.INPUT_FILE
+schema_file = args.SCHEMA_FILE
+output_dir = args.OUTPUT_DIR
+debug = args.debug
+
+## Load and validate the inputs
+try:
+    xml = ET.parse(input_file)
+except Exception as e:
+    print(f"Failed to load interface definition file {input_file}")
+    print(e)
+    sys.exit(1)
+
+try:
+    relaxng_xml = ET.parse(schema_file)
+    validator = ET.RelaxNG(relaxng_xml)
+
+    if not validator.validate(xml):
+        print(validator.error_log)
+        print(f"Interface definition file {input_file} does not match the schema!")
+        sys.exit(1)
+except Exception as e:
+    print(f"Failed to load the XML schema {schema_file}")
+    print(e)
+    sys.exit(1)
+
+if not os.access(output_dir, os.W_OK):
+    print(f"The output directory {output_dir} is not writeable")
+    sys.exit(1)
+
+## If we got this far, everything must be ok and we can convert the file
+def make_path(l):
+    path = functools.reduce(os.path.join, l)
+    if debug:
+        print(path)
+    return path
+
+def get_properties(p):
+    props = {}
+
+    if p is None:
+        return props
+
+    # Get the help string
+    try:
+        props["help"] = p.find("help").text
+    except:
+        props["help"] = "No help available"
+
+
+    # Get the completion help strings
+    try:
+        che = p.findall("completionHelp")
+        ch = ""
+        for c in che:
+            scripts = c.findall("script")
+            paths = c.findall("path")
+            lists = c.findall("list")
+            comptype = c.find("imagePath")
+
+            # Current backend doesn't support multiple allowed: tags
+            # so we get to emulate it
+            comp_exprs = []
+            for i in lists:
+                comp_exprs.append("echo \"{0}\"".format(i.text))
+            for i in paths:
+                path = re.sub(r'\s+', '/', i.text)
+                comp_exprs.append("ls /opt/vyatta/config/active/{0} 2>/dev/null".format(path))
+            for i in scripts:
+                comp_exprs.append("{0}".format(i.text))
+            if comptype is not None:
+                props["comp_type"] = "imagefiles"
+                comp_exprs.append("echo -n \"<imagefiles>\"")
+            comp_help = " && ".join(comp_exprs)
+            props["comp_help"] = comp_help
+
+    except:
+        props["comp_help"] = []
+
+    return props
+
+
+def make_node_def(props, command):
+    # XXX: replace with a template processor if it grows
+    #      out of control
+    node_def = ""
+
+    if "help" in props:
+        help = props["help"]
+        help = fill(help, width=64, subsequent_indent='\t\t\t')
+        node_def += f'help: {help}\n'
+    if "comp_type" in props:
+        node_def += f'comptype: {props["comp_type"]}\n'
+    if "comp_help" in props:
+        node_def += f'allowed: {props["comp_help"]}\n'
+    if command is not None:
+        node_def += f'run: {command.text}\n'
+    if debug:
+        print('Contents of the node.def file:\n', node_def)
+
+    return node_def
+
+def process_node(n, tmpl_dir):
+    # Avoid mangling the path from the outer call
+    my_tmpl_dir = copy.copy(tmpl_dir)
+
+    props_elem = n.find("properties")
+    children = n.find("children")
+    command = n.find("command")
+    name = n.get("name")
+
+    node_type = n.tag
+
+    my_tmpl_dir.append(name)
+
+    if debug:
+        print(f"Name of the node: {name};\n Created directory: ", end="")
+    os.makedirs(make_path(my_tmpl_dir), exist_ok=True)
+
+    props = get_properties(props_elem)
+
+    nodedef_path = os.path.join(make_path(my_tmpl_dir), "node.def")
+    if node_type == "node":
+        if debug:
+          print(f"Processing node {name}")
+
+        # Only create the "node.def" file if it exists but is empty, or if it
+        # does not exist at all.
+        if not os.path.exists(nodedef_path) or os.path.getsize(nodedef_path) == 0:
+            with open(nodedef_path, "w") as f:
+                f.write(make_node_def(props, command))
+
+        if children is not None:
+            inner_nodes = children.iterfind("*")
+            for inner_n in inner_nodes:
+                process_node(inner_n, my_tmpl_dir)
+    elif node_type == "tagNode":
+        if debug:
+          print(f"Processing tagNode {name}")
+
+        os.makedirs(make_path(my_tmpl_dir), exist_ok=True)
+
+        # Only create the "node.def" file if it exists but is empty, or if it
+        # does not exist at all.
+        if not os.path.exists(nodedef_path) or os.path.getsize(nodedef_path) == 0:
+            with open(nodedef_path, "w") as f:
+                f.write('help: {0}\n'.format(props['help']))
+
+        # Create the inner node.tag part
+        my_tmpl_dir.append("node.tag")
+        os.makedirs(make_path(my_tmpl_dir), exist_ok=True)
+        if debug:
+            print("Created path for the tagNode: {}".format(make_path(my_tmpl_dir)), end="")
+
+        # Not sure if we want partially defined tag nodes, write the file unconditionally
+        nodedef_path = os.path.join(make_path(my_tmpl_dir), "node.def")
+        # Only create the "node.def" file if it exists but is empty, or if it
+        # does not exist at all.
+        if not os.path.exists(nodedef_path) or os.path.getsize(nodedef_path) == 0:
+            with open(nodedef_path, "w") as f:
+                f.write(make_node_def(props, command))
+
+        if children is not None:
+            inner_nodes = children.iterfind("*")
+            for inner_n in inner_nodes:
+                process_node(inner_n, my_tmpl_dir)
+    elif node_type == "leafNode":
+        # This is a leaf node
+        if debug:
+            print(f"Processing leaf node {name}")
+
+        if not os.path.exists(nodedef_path) or os.path.getsize(nodedef_path) == 0:
+            with open(nodedef_path, "w") as f:
+                f.write(make_node_def(props, command))
+    else:
+        print(f"Unknown node_type: {node_type}")
+
+
+def get_node_key(node, attr=None):
+    """ Return the sorting key of an xml node using tag and attributes """
+    if attr is None:
+        return '%s' % node.tag + ':'.join([node.get(attr)
+                                        for attr in sorted(node.attrib)])
+    if attr in node.attrib:
+        return '%s:%s' % (node.tag, node.get(attr))
+    return '%s' % node.tag
+
+
+def sort_children(node, attr=None):
+    """ Sort children along tag and given attribute. if attr is None, sort
+    along all attributes """
+    if not isinstance(node.tag, str):  # PYTHON 2: use basestring instead
+        # not a TAG, it is comment or DATA
+        # no need to sort
+        return
+    # sort child along attr
+    node[:] = sorted(node, key=lambda child: get_node_key(child, attr))
+    # and recurse
+    for child in node:
+        sort_children(child, attr)
+
+root = xml.getroot()
+
+# process_node() processes the XML tree in a fixed order, "node" before "tagNode"
+# before "leafNode". If the generator created a "node.def" file, it can no longer
+# be overwritten - else we would have some stale "node.def" files with an empty
+# help string (T2555). Without the fixed order this would resulted in a case
+# where we get a node and a tagNode with the same name, e.g. "show interfaces
+# ethernet" and "show interfaces ethernet eth0" that the node implementation
+# was not callable from the CLI, rendering this command useless (T3807).
+#
+# This can be fixed by forcing the "node", "tagNode", "leafNode" order by sorting
+# the input XML file automatically (sorting from https://stackoverflow.com/a/46128043)
+# thus adding no additional overhead to the user.
+sort_children(root, 'name')
+
+nodes = root.iterfind("*")
+for n in nodes:
+    process_node(n, [output_dir])
diff --git a/scripts/build-command-templates b/scripts/build-command-templates
new file mode 100644
index 0000000..36929ab
--- /dev/null
+++ b/scripts/build-command-templates
@@ -0,0 +1,348 @@
+#!/usr/bin/env python3
+#
+#    build-command-template: converts new style command definitions in XML
+#      to the old style (bunch of dirs and node.def's) command templates
+#
+#    Copyright (C) 2017 VyOS maintainers <maintainers@vyos.net>
+#
+#    This library is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU Lesser General Public
+#    License as published by the Free Software Foundation; either
+#    version 2.1 of the License, or (at your option) any later version.
+#
+#    This library is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#    Lesser General Public License for more details.
+#
+#    You should have received a copy of the GNU Lesser General Public
+#    License along with this library; if not, write to the Free Software
+#    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
+#    USA
+
+import sys
+import os
+import argparse
+import copy
+import functools
+
+from lxml import etree as ET
+from textwrap import fill
+
+# Defaults
+
+#validator_dir = "/usr/libexec/vyos/validators"
+validator_dir = "${vyos_validators_dir}"
+default_constraint_err_msg = "Invalid value"
+
+
+## Get arguments
+
+parser = argparse.ArgumentParser(description='Converts new-style XML interface definitions to old-style command templates')
+parser.add_argument('--debug', help='Enable debug information output', action='store_true')
+parser.add_argument('INPUT_FILE', type=str, help="XML interface definition file")
+parser.add_argument('SCHEMA_FILE', type=str, help="RelaxNG schema file")
+parser.add_argument('OUTPUT_DIR', type=str, help="Output directory")
+
+args = parser.parse_args()
+
+input_file = args.INPUT_FILE
+schema_file = args.SCHEMA_FILE
+output_dir = args.OUTPUT_DIR
+debug = args.debug
+
+#debug = True
+
+## Load and validate the inputs
+
+try:
+    xml = ET.parse(input_file)
+except Exception as e:
+    print("Failed to load interface definition file {0}".format(input_file))
+    print(e)
+    sys.exit(1)
+
+try:
+    relaxng_xml = ET.parse(schema_file)
+    validator = ET.RelaxNG(relaxng_xml)
+
+    if not validator.validate(xml):
+        print(validator.error_log)
+        print("Interface definition file {0} does not match the schema!".format(input_file))
+        sys.exit(1)
+except Exception as e:
+    print("Failed to load the XML schema {0}".format(schema_file))
+    print(e)
+    sys.exit(1)
+
+if not os.access(output_dir, os.W_OK):
+    print("The output directory {0} is not writeable".format(output_dir))
+    sys.exit(1)
+
+## If we got this far, everything must be ok and we can convert the file
+
+def make_path(l):
+    path = functools.reduce(os.path.join, l)
+    if debug:
+        print(path)
+    return path
+
+def collect_validators(ve):
+    regexes = []
+    regex_elements = ve.findall("regex")
+    if regex_elements is not None:
+        regexes = list(map(lambda e: e.text.strip().replace('\\','\\\\'), regex_elements))
+    if "" in regexes:
+        print("Warning: empty regex, node will be accepting any value")
+
+    validator_elements = ve.findall("validator")
+    validators = []
+    if validator_elements is not None:
+        for v in validator_elements:
+            v_name = os.path.join(validator_dir, v.get("name"))
+
+            # XXX: lxml returns None for empty arguments
+            v_argument = None
+            try:
+                v_argument = v.get("argument")
+            except:
+                pass
+            if v_argument is None:
+                v_argument = ""
+
+            validators.append("{0} {1}".format(v_name, v_argument))
+
+
+    regex_args = " ".join(map(lambda s: "--regex \\\'{0}\\\'".format(s), regexes))
+    validator_args = " ".join(map(lambda s: "--exec \\\"{0}\\\"".format(s), validators))
+
+    return regex_args + " " + validator_args
+
+def get_properties(p, default=None):
+    props = {}
+
+    if p is None:
+        return props
+
+    # Get the help string
+    try:
+        help = p.find("help").text
+        if default != None:
+            # DNS forwarding for instance has multiple defaults - specified as whitespace separated list
+            tmp = ', '.join(default.text.split())
+            help += f' (default: {tmp})'
+        help = fill(help, width=64, subsequent_indent='\t\t\t')
+        props["help"] = help
+    except:
+        pass
+
+    # Get value help strings
+    try:
+        vhe = p.findall("valueHelp")
+        vh = []
+        for v in vhe:
+            format = v.find("format").text
+            description = v.find("description").text
+            if default != None and default.text == format:
+                description += f' (default)'
+            # Is no description was specified, keep it empty
+            if not description: description = ''
+            vh.append( (format, description) )
+        props["val_help"] = vh
+    except:
+        props["val_help"] = []
+
+    # Get the constraint and constraintGroup statements
+
+    error_msg = default_constraint_err_msg
+    # Get the error message if it's there
+    try:
+        error_msg = p.find("constraintErrorMessage").text
+    except:
+        pass
+
+    vce = p.find("constraint")
+
+    distinct_validator_string = ""
+    if vce is not None:
+        # The old backend doesn't support multiple validators in OR mode
+        # so we emulate it
+
+        distinct_validator_string = collect_validators(vce)
+
+    vcge = p.findall("constraintGroup")
+
+    group_validator_string = ""
+    if len(vcge):
+        for vcg in vcge:
+            group_validator_string = group_validator_string + " --grp " + collect_validators(vcg)
+
+    if vce is not None or len(vcge):
+        validator_script = '${vyos_libexec_dir}/validate-value'
+        validator_string = "exec \"{0} {1} {2} --value \\\'$VAR(@)\\\'\"; \"{3}\"".format(validator_script, distinct_validator_string, group_validator_string, error_msg)
+
+        props["constraint"] = validator_string
+
+    # Get the completion help strings
+    try:
+        che = p.findall("completionHelp")
+        ch = ""
+        for c in che:
+            scripts = c.findall("script")
+            paths = c.findall("path")
+            lists = c.findall("list")
+
+            # Current backend doesn't support multiple allowed: tags
+            # so we get to emulate it
+            comp_exprs = []
+            for i in lists:
+                comp_exprs.append(f'echo "{i.text}"')
+            for i in paths:
+                comp_exprs.append(f'/bin/cli-shell-api listNodes {i.text}')
+            for i in scripts:
+                comp_exprs.append(f'sh -c "{i.text}"')
+            comp_help = ' && echo " " && '.join(comp_exprs)
+            props["comp_help"] = comp_help
+    except:
+        props["comp_help"] = []
+
+    # Get priority
+    try:
+        props["priority"] = p.find("priority").text
+    except:
+        pass
+
+    # Get "multi"
+    if p.find("multi") is not None:
+        props["multi"] = True
+
+    # Get "valueless"
+    if p.find("valueless") is not None:
+        props["valueless"] = True
+
+    return props
+
+def make_node_def(props):
+    # XXX: replace with a template processor if it grows
+    #      out of control
+
+    node_def = ""
+
+    if "tag" in props:
+        node_def += "tag:\n"
+
+    if "multi" in props:
+        node_def += "multi:\n"
+
+    if "type" in props:
+        # Will always be txt in practice if it's set
+        node_def += "type: {0}\n".format(props["type"])
+
+    if "priority" in props:
+        node_def += "priority: {0}\n".format(props["priority"])
+
+    if "help" in props:
+        node_def += "help: {0}\n".format(props["help"])
+
+    if "val_help" in props:
+        for v in props["val_help"]:
+            node_def += "val_help: {0}; {1}\n".format(v[0], v[1])
+
+    if "comp_help" in props:
+        node_def += "allowed: {0}\n".format(props["comp_help"])
+
+    if "constraint" in props:
+        node_def += "syntax:expression: {0}\n".format(props["constraint"])
+
+    shim = '${vyshim}'
+
+    if "owner" in props:
+        if "tag" in props:
+            node_def += "end: sudo sh -c \"{1} VYOS_TAGNODE_VALUE='$VAR(@)' {0}\"\n".format(props["owner"], shim)
+        else:
+            node_def += "end: sudo sh -c \"{1} {0}\"\n".format(props["owner"], shim)
+
+    if debug:
+        print("The contents of the node.def file:\n", node_def)
+
+    return node_def
+
+def process_node(n, tmpl_dir):
+    # Avoid mangling the path from the outer call
+    my_tmpl_dir = copy.copy(tmpl_dir)
+
+    props_elem = n.find("properties")
+    children = n.find("children")
+
+    name = n.get("name")
+    owner = n.get("owner")
+    node_type = n.tag
+
+    my_tmpl_dir.append(name)
+
+    if debug:
+        print("Name of the node: {0}. Created directory: {1}\n".format(name, "/".join(my_tmpl_dir)), end="")
+    os.makedirs(make_path(my_tmpl_dir), exist_ok=True)
+
+    props = get_properties(props_elem, n.find("defaultValue"))
+    if owner:
+        props["owner"] = owner
+        # <priority> tag is mandatory if the parent node has an owner
+        if "priority" not in props:
+            raise ValueError(
+                f"<priority> tag should be set for the node <{name}> path '{' '.join(my_tmpl_dir[1:])}'"
+            )
+
+    # Type should not be set for non-tag, non-leaf nodes
+    # For non-valueless leaf nodes, set the type to txt: to make them have some type,
+    # actual value validation is handled by constraints translated to syntax:expression:
+    if node_type != "node":
+        if "valueless" not in props.keys():
+            props["type"] = "txt"
+    if node_type == "tagNode":
+        props["tag"] = "True"
+
+    if node_type != "leafNode":
+        if "multi" in props:
+            raise ValueError("<multi/> tag is only allowed in <leafNode>")
+        if "valueless" in props:
+            raise ValueError("<valueless/> is only allowed in <leafNode>")
+
+    nodedef_path = os.path.join(make_path(my_tmpl_dir), "node.def")
+
+    # Only create the "node.def" file if it exists but is empty, or if it does
+    # not exist at all. An empty node.def file could be generated by XML paths
+    # that derive from one another bot having a common base structure like
+    # "protocols static"
+    if not os.path.exists(nodedef_path) or os.path.getsize(nodedef_path) == 0:
+        with open(nodedef_path, "w") as f:
+            f.write(make_node_def(props))
+
+    if node_type == "node":
+        inner_nodes = children.iterfind("*")
+        for inner_n in inner_nodes:
+            process_node(inner_n, my_tmpl_dir)
+    if node_type == "tagNode":
+        my_tmpl_dir.append("node.tag")
+        if debug:
+            print("Created path for the tagNode:", end="")
+        os.makedirs(make_path(my_tmpl_dir), exist_ok=True)
+        inner_nodes = children.iterfind("*")
+        for inner_n in inner_nodes:
+            process_node(inner_n, my_tmpl_dir)
+    else:
+        # This is a leaf node
+        pass
+
+
+root = xml.getroot()
+
+nodes = root.iterfind("*")
+for n in nodes:
+    if n.tag == "syntaxVersion":
+        continue
+    try:
+        process_node(n, [output_dir])
+    except ValueError as e:
+        print(e)
+        sys.exit(1)
diff --git a/scripts/generate-configd-include-json.py b/scripts/generate-configd-include-json.py
new file mode 100644
index 0000000..b4b627f
--- /dev/null
+++ b/scripts/generate-configd-include-json.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+# Copyright (C) 2024 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+from jinja2 import Template
+
+conf_scripts = 'src/conf_mode'
+configd_include = 'data/configd-include.json'
+
+configd_template = Template("""[
+{% for file in files %}
+"{{ file }}"{{ "," if not loop.last else "" }}
+{% endfor %}
+]
+""", trim_blocks=True)
+
+files = [f for f in os.listdir(conf_scripts) if os.path.isfile(f'{conf_scripts}/{f}')]
+files = sorted(files)
+
+tmp = {'files' : files}
+with open(configd_include, 'w') as f:
+    f.write(configd_template.render(tmp))
diff --git a/scripts/override-default b/scripts/override-default
new file mode 100644
index 0000000..5058e79
--- /dev/null
+++ b/scripts/override-default
@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+#
+# override-default: preprocessor for XML interface definitions to interpret
+# redundant entries (relative to path) with tag 'defaultValue' as an override
+# directive. Must be called before build-command-templates, as the schema
+# disallows redundancy.
+#
+# Copyright (C) 2021 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+# Use lxml xpath capability to find multiple elements with tag defaultValue
+# relative to path; replace and remove to override the value.
+
+import sys
+import glob
+import logging
+from copy import deepcopy
+from lxml import etree
+
+debug = False
+
+logger = logging.getLogger(__name__)
+logs_handler = logging.StreamHandler()
+logger.addHandler(logs_handler)
+
+if debug:
+    logger.setLevel(logging.DEBUG)
+else:
+    logger.setLevel(logging.INFO)
+
+def clear_empty_path(el):
+    # on the odd chance of interleaved comments
+    tmp = [l for l in el if isinstance(l.tag, str)]
+    if not tmp:
+        p = el.getparent()
+        p.remove(el)
+        clear_empty_path(p)
+
+def override_element(l: list):
+    """
+    Allow multiple override elements; use the final one (in document order).
+    """
+    if len(l) < 2:
+        logger.debug("passing list of single element to override_element")
+        return
+
+    # assemble list of leafNodes of overriding defaultValues, for later removal
+    parents = []
+    for el in l[1:]:
+        parents.append(el.getparent())
+
+    # replace element with final override
+    l[0].getparent().replace(l[0], l[-1])
+
+    # remove all but overridden element
+    for el in parents:
+        tmp = el.getparent()
+        tmp.remove(el)
+        clear_empty_path(tmp)
+
+def merge_remaining(l: list, elementtree):
+    """
+    Merge (now) single leaf node containing 'defaultValue' with leaf nodes
+    of same path and no 'defaultValue'.
+    """
+    for p in l:
+        p = p.split()
+        path_str = f'/interfaceDefinition/*'
+        path_list = []
+        for i in range(len(p)):
+            path_list.append(f'[@name="{p[i]}"]')
+        path_str += '/children/*'.join(path_list)
+        rp = elementtree.xpath(path_str)
+        if len(rp) > 1:
+            for el in rp[1:]:
+                # in practice there will only be one child of the path,
+                # either defaultValue or Properties, since
+                # override_element() has already run
+                for child in el:
+                    rp[0].append(deepcopy(child))
+                tmp = el.getparent()
+                tmp.remove(el)
+                clear_empty_path(tmp)
+
+def collect_and_override(dir_name):
+    """
+    Collect elements with defaultValue tag into dictionary indexed by name
+    attributes of ancestor path.
+    """
+    for fname in glob.glob(f'{dir_name}/*.xml'):
+        tree = etree.parse(fname)
+        root = tree.getroot()
+        defv = {}
+
+        xpath_str = '//defaultValue'
+        xp = tree.xpath(xpath_str)
+
+        for element in xp:
+            ap = element.xpath('ancestor::*[@name]')
+            ap_name = [el.get("name") for el in ap]
+            ap_path_str = ' '.join(ap_name)
+            defv.setdefault(ap_path_str, []).append(element)
+
+        for k, v in defv.items():
+            if len(v) > 1:
+                logger.info(f"overriding default in path '{k}'")
+                override_element(v)
+
+        to_merge = list(defv)
+        merge_remaining(to_merge, tree)
+
+        revised_str = etree.tostring(root, encoding='unicode', pretty_print=True)
+
+        with open(f'{fname}', 'w') as f:
+            f.write(revised_str)
+
+def main():
+    if len(sys.argv) < 2:
+        logger.critical('Must specify XML directory!')
+        sys.exit(1)
+
+    dir_name = sys.argv[1]
+
+    collect_and_override(dir_name)
+
+if __name__ == '__main__':
+    main()
diff --git a/scripts/transclude-template b/scripts/transclude-template
new file mode 100644
index 0000000..5c6668a
--- /dev/null
+++ b/scripts/transclude-template
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+#
+# transclude-template: preprocessor for XML interface definitions to
+# interpret #include statements to include nested XML fragments and
+# snippets in documents.
+#
+# Copyright (C) 2021 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import os
+import re
+import sys
+
+regexp = re.compile(r'^ *#include <(.+)>$')
+
+def parse_file(filename):
+    lines = ""
+    with open(filename, 'r') as f:
+        while True:
+            line = f.readline()
+            if line:
+                result = regexp.match(line)
+                if result:
+                    lines += parse_file(os.path.join(directory, result.group(1)))
+                else:
+                    lines += line
+            else:
+                return lines
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print('Must specify XML file!', file=sys.stderr)
+        sys.exit(1)
+    filename = sys.argv[1]
+    directory = os.path.dirname(os.path.abspath(filename))
+    print(parse_file(filename))
+
diff --git a/scripts/update-configd-include-file b/scripts/update-configd-include-file
new file mode 100644
index 0000000..ca23408
--- /dev/null
+++ b/scripts/update-configd-include-file
@@ -0,0 +1,298 @@
+#!/usr/bin/env python3
+###
+# A simple script for safely editing configd-include.json, the list of
+# scripts which are off-loaded to be run by the daemon.
+# Usage:
+# update-configd-include-file --add script1.py script2.py ...
+#                             --remove scriptA.py scriptB.py ...
+#
+# Additionally, it offers optional sanity checks by examining the signatures
+# of functions and placement of Config instance for consistency with configd
+# requirements.
+# Usage:
+# update-configd-include-file --check-current
+#   to check the current include list
+# update-configd-include-file --check-file
+#   to check arbitrary conf_mode scripts
+#
+# Note that this feature is the basis for the configd smoketest, but it is of
+# limited use in this script, as it requires an environment that has all script
+# (python) dependencies installed (e.g. installed image) so that the script may
+# be imported for introspection. Nonetheless, for testing and development, it has
+# its uses.
+
+import os
+import sys
+import json
+import argparse
+import datetime
+import importlib.util
+from inspect import signature, getsource
+
+from vyos.defaults import directories
+from vyos.version import get_version
+from vyos.utils.process import cmd
+
+# Defaults
+
+installed_image = False
+
+include_file = 'configd-include.json'
+build_relative_include_file = '../data/configd-include.json'
+dirname = os.path.dirname(__file__)
+
+build_location_include_file = os.path.join(dirname, build_relative_include_file)
+image_location_include_file = os.path.join(directories['data'], include_file)
+
+build_relative_conf_dir = '../src/conf_mode'
+
+build_location_conf_dir = os.path.join(dirname, build_relative_conf_dir)
+image_location_conf_dir = directories['conf_mode']
+
+# Get arguments
+
+parser = argparse.ArgumentParser(description='Add or remove scripts from the list of scripts to be run be daemon')
+parser.add_argument('--add', nargs='*', default=[],
+                    help='scripts to add to configd include list')
+parser.add_argument('--remove', nargs='*', default=[],
+                    help='scripts to remove from configd include list')
+parser.add_argument('--show-diff', action='store_true',
+                    help='show list of conf_mode scripts not in include list')
+parser.add_argument('--check-file', nargs='*', default=[],
+                    help='check files for suitability to run under daemon')
+parser.add_argument('--check-current', action="store_true",
+                    help='check current include list for suitability to run under daemon')
+
+args = vars(parser.parse_args())
+
+# Check if we are running within installed image; since this script is not
+# part of the distribution, there is no need to check if live cd
+if get_version():
+    installed_image = True
+
+if installed_image:
+    include_file = image_location_include_file
+    conf_dir = image_location_conf_dir
+else:
+    include_file = build_location_include_file
+    conf_dir = build_location_conf_dir
+
+# Utilities for checking function signature and body
+def import_script(s: str):
+    """
+    A compact form of the import code in vyos-configd
+    """
+    path = os.path.join(conf_dir, s)
+    if not os.path.exists(path):
+        print(f"script {s} is not in conf_mode directory")
+        return None
+
+    name = os.path.splitext(s)[0].replace('-', '_')
+
+    spec = importlib.util.spec_from_file_location(name, path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    return module
+
+funcs = { 'get_config': False,
+          'verify':     False,
+          'generate':   False,
+          'apply':      False
+        }
+
+def check_signatures(s: str) -> bool:
+    """
+    Basic sanity check: script standard functions should all take one
+    argument, including get_config(config=None).
+    """
+    funcd = dict(funcs)
+    for i in list(funcd):
+        m = import_script(s)
+        f = getattr(m, i, None)
+        if not f:
+            funcd[i] = True
+            continue
+        sig = signature(f)
+        params = sig.parameters
+        if len(params) != 1:
+            continue
+        if i == 'get_config':
+            for p in params.values():
+                funcd[i] = True if (p.default is None) else False
+        else:
+            funcd[i] = True
+
+    res = True
+
+    for k, v in funcd.items():
+        if v is False:
+            if k == 'get_config':
+                print(f"function '{k}' will need the standard modification")
+            else:
+                print(f"function '{k}' in script '{s}' has wrong signature")
+            res = False
+
+    return res
+
+def check_instance_per_function(s: str) -> bool:
+    """
+    The standard function 'get_config' should have one instantiation of Config;
+    all other standard functions, zero.
+    """
+    funcd = dict(funcs)
+    for i in list(funcd):
+        m = import_script(s)
+        f = getattr(m, i, None)
+        if not f:
+            funcd[i] = True
+            continue
+        str_f = getsource(f)
+        n = str_f.count('Config()')
+        if n == 1 and i == 'get_config':
+            funcd[i] = True
+        if n == 0 and i != 'get_config':
+            funcd[i] = True
+
+    res = True
+
+    for k, v in funcd.items():
+        if v is False:
+            fi = 'zero' if k == 'get_config' else 'non-zero'
+            print(f"function '{k}' in script '{s}' has {fi} instances of Config")
+            res = False
+
+    return res
+
+def check_instance_total(s: str) -> bool:
+    """
+    A script should have at most one instantiation of Config.
+    """
+    m = import_script(s)
+    str_m = getsource(m)
+    n = str_m.count('Config()')
+    if n != 1:
+        print(f"instance of Config outside of 'get_config' in script '{s}'")
+        return False
+
+    return True
+
+def check_config_modification(s: str) -> bool:
+    """
+    Modification to the session config from within a script is necessary in
+    certain cases, but the script should then run as stand-alone.
+    """
+    m = import_script(s)
+    str_m = getsource(m)
+    n = str_m.count('my_set')
+    if n != 0:
+        print(f"modification of config within script")
+        return False
+
+    return True
+
+def check_viability(s: str) -> bool:
+    """
+    Check existence, and if on installed image, signatures, instances of
+    Config, and modification of session config
+    """
+    path = os.path.join(conf_dir, s)
+    if not os.path.exists(path):
+        print(f"script {s} is not in conf_mode directory")
+        return False
+
+    if not installed_image:
+        if args['check_file'] or args['check_current']:
+            print(f"In order to check script viability for offload, run this script on installed image")
+        return True
+
+    r1 = check_signatures(s)
+    r2 = check_instance_per_function(s)
+    r3 = check_instance_total(s)
+    r4 = check_config_modification(s)
+
+    if not r1 or not r2 or not r3 or not r4:
+        return False
+
+    return True
+
+def check_file(s: str) -> bool:
+    if not check_viability(s):
+        return False
+    return True
+
+def check_files(l: list) -> int:
+    check_list = l[:]
+    res = 0
+    for s in check_list:
+        if not check_file(s):
+            res = 1
+    return res
+
+# Status
+
+def show_diff(l: list):
+    print(conf_dir)
+    (_, _, filenames) = next(iter(os.walk(conf_dir)))
+    filenames.sort()
+    res = [i for i in filenames if i not in l]
+    print(res)
+
+# Read configd-include.json and add/remove/check/show scripts
+
+with open(include_file, 'r') as f:
+    try:
+        include_list = json.load(f)
+    except OSError as e:
+        print(f"configd include file error: {e}")
+        sys.exit(1)
+    except json.JSONDecodeError as e:
+        print(f"JSON load error: {e}")
+        sys.exit(1)
+
+if args['show_diff']:
+    show_diff(include_list)
+    sys.exit(0)
+
+if args['check_file']:
+    l = args['check_file']
+    ret = check_files(l)
+    if not ret:
+        print('pass')
+    sys.exit(ret)
+
+if args['check_current']:
+    ret = check_files(include_list)
+    if not ret:
+        print('pass')
+    sys.exit(ret)
+
+add_list = args['add']
+# drop redundencies
+add_list = [i for i in add_list if i not in include_list]
+# prune entries that don't pass check
+add_list = [i for i in add_list if check_file(i)]
+
+remove_list = args['remove']
+
+if not add_list and not remove_list:
+    sys.exit(0)
+
+separator = '.'
+backup_file_name = separator.join([include_file,
+    '{0:%Y-%m-%d-%H%M%S}'.format(datetime.datetime.now()), 'bak'])
+
+cmd(f'cp -p {include_file} {backup_file_name}')
+
+if add_list:
+    include_list.extend(add_list)
+    include_list.sort()
+if remove_list:
+    include_list = [i for i in include_list if i not in remove_list]
+
+with open(include_file, 'w') as f:
+    try:
+        json.dump(include_list, f, indent=0)
+    except OSError as e:
+        print(f"error writing configd include file: {e}")
+        sys.exit(1)