xml: T5403: add support for supplemental xml cache

5 files changed, 105 insertions, 21 deletions

diff --git a/.gitignore b/.gitignore
index e766a2c27..7707e94ca 100644
--- a/.gitignore
+++ b/.gitignore
@@ -144,3 +144,4 @@ debian/*.substvars
 data/component-versions.json
 # vyos-1x XML cache
 python/vyos/xml_ref/cache.py
+python/vyos/xml_ref/pkg_cache/*_cache.py
diff --git a/debian/vyos-1x.postinst b/debian/vyos-1x.postinst
index 93e7ced9b..b1bd23ff2 100644
--- a/debian/vyos-1x.postinst
+++ b/debian/vyos-1x.postinst
@@ -180,6 +180,9 @@ systemctl enable vyos-config-cloud-init.service
 # Generate API GraphQL schema
 /usr/libexec/vyos/services/api/graphql/generate/generate_schema.py
 
+# Update XML cache
+python3 /usr/lib/python3/dist-packages/vyos/xml_ref/update_cache.py
+
 # T1797: disable VPP support for rolling release, should be used by developers
 # only (in the initial phase). If you wan't to enable VPP use the below command
 # on your VyOS installation:
diff --git a/python/vyos/xml_ref/generate_cache.py b/python/vyos/xml_ref/generate_cache.py
index 792c6eea7..6f08486a8 100755
--- a/python/vyos/xml_ref/generate_cache.py
+++ b/python/vyos/xml_ref/generate_cache.py
@@ -18,10 +18,14 @@
 
 import sys
 import json
-import argparse
+from argparse import ArgumentParser
+from argparse import ArgumentTypeError
+from os import getcwd
+from os import makedirs
 from os.path import join
 from os.path import abspath
 from os.path import dirname
+from os.path import basename
 from xmltodict import parse
 
 _here = dirname(__file__)
@@ -29,9 +33,10 @@ _here = dirname(__file__)
 sys.path.append(join(_here, '..'))
 from configtree import reference_tree_to_json, ConfigTreeError
 
-xml_cache = abspath(join(_here, 'cache.py'))
 xml_cache_json = 'xml_cache.json'
 xml_tmp = join('/tmp', xml_cache_json)
+pkg_cache = abspath(join(_here, 'pkg_cache'))
+ref_cache = abspath(join(_here, 'cache.py'))
 
 node_data_fields = ("node_type", "multi", "valueless", "default_value")
 
@@ -45,16 +50,29 @@ def trim_node_data(cache: dict):
             if isinstance(cache[k], dict):
                 trim_node_data(cache[k])
 
+def non_trivial(s):
+    if not s:
+        raise ArgumentTypeError("Argument must be non empty string")
+    if s == 'vyos-1x' and basename(getcwd()) != 'vyos-1x':
+        # builds outside of vyos-1x must specify package name
+        raise ArgumentTypeError("Specify package name")
+    return s
+
 def main():
-    parser = argparse.ArgumentParser(description='generate and save dict from xml defintions')
+    parser = ArgumentParser(description='generate and save dict from xml defintions')
     parser.add_argument('--xml-dir', type=str, required=True,
                         help='transcluded xml interface-definition directory')
-    parser.add_argument('--save-json-dir', type=str,
-                        help='directory to save json cache if needed')
-    args = parser.parse_args()
-
-    xml_dir = abspath(args.xml_dir)
-    save_dir = abspath(args.save_json_dir) if args.save_json_dir else None
+    parser.add_argument('--package-name', type=non_trivial, default='vyos-1x',
+                        help='name of current package')
+    parser.add_argument('--output-path', help='path to generated cache')
+    args = vars(parser.parse_args())
+
+    xml_dir = abspath(args['xml_dir'])
+    pkg_name = args['package_name'].replace('-','_')
+    cache_name = pkg_name + '_cache.py'
+    out_path = args['output_path']
+    path = out_path if out_path is not None else pkg_cache
+    xml_cache = abspath(join(path, cache_name))
 
     try:
         reference_tree_to_json(xml_dir, xml_tmp)
@@ -67,21 +85,30 @@ def main():
 
     trim_node_data(d)
 
-    if save_dir is not None:
-        save_file = join(save_dir, xml_cache_json)
-        with open(save_file, 'w') as f:
-            f.write(json.dumps(d))
-
     syntax_version = join(xml_dir, 'xml-component-version.xml')
-    with open(syntax_version) as f:
-        content = f.read()
+    try:
+        with open(syntax_version) as f:
+            component = f.read()
+    except FileNotFoundError:
+        if pkg_name != 'vyos_1x':
+            component = ''
+        else:
+            print("\nWARNING: missing xml-component-version.xml\n")
+            sys.exit(1)
 
-    parsed = parse(content)
-    converted = parsed['interfaceDefinition']['syntaxVersion']
+    if component:
+        parsed = parse(component)
+    else:
+        parsed = None
     version = {}
-    for i in converted:
-        tmp = {i['@component']: i['@version']}
-        version |= tmp
+    # addon package definitions may have empty (== 0) version info
+    if parsed is not None and parsed['interfaceDefinition'] is not None:
+        converted = parsed['interfaceDefinition']['syntaxVersion']
+        if not isinstance(converted, list):
+            converted = [converted]
+        for i in converted:
+            tmp = {i['@component']: i['@version']}
+            version |= tmp
 
     version = {"component_version": version}
 
@@ -90,5 +117,7 @@ def main():
     with open(xml_cache, 'w') as f:
         f.write(f'reference = {str(d)}')
 
+    print(cache_name)
+
 if __name__ == '__main__':
     main()
diff --git a/python/vyos/xml_ref/pkg_cache/__init__.py b/python/vyos/xml_ref/pkg_cache/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/python/vyos/xml_ref/pkg_cache/__init__.py
diff --git a/python/vyos/xml_ref/update_cache.py b/python/vyos/xml_ref/update_cache.py
new file mode 100755
index 000000000..0842bcbe9
--- /dev/null
+++ b/python/vyos/xml_ref/update_cache.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2023 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+import os
+from copy import deepcopy
+from generate_cache import pkg_cache
+from generate_cache import ref_cache
+
+def dict_merge(source, destination):
+    dest = deepcopy(destination)
+
+    for key, value in source.items():
+        if key not in dest:
+            dest[key] = value
+        elif isinstance(source[key], dict):
+            dest[key] = dict_merge(source[key], dest[key])
+
+    return dest
+
+def main():
+    res = {}
+    cache_dir = os.path.basename(pkg_cache)
+    for mod in os.listdir(pkg_cache):
+        mod = os.path.splitext(mod)[0]
+        if not mod.endswith('_cache'):
+            continue
+        d = getattr(__import__(f'{cache_dir}.{mod}', fromlist=[mod]), 'reference')
+        if mod == 'vyos_1x_cache':
+            res = dict_merge(res, d)
+        else:
+            res = dict_merge(d, res)
+
+    with open(ref_cache, 'w') as f:
+        f.write(f'reference = {str(res)}')
+
+if __name__ == '__main__':
+    main()