From c67db6bc118bba219f9cc41a3a0192f772e795e8 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sat, 22 Jul 2023 20:52:17 +0200 Subject: Revert "Debian: T4974: add openvpn-dco dependency" This reverts commit 9f7b51370732606611253e2e6a16692bf706659b. --- debian/control | 1 - 1 file changed, 1 deletion(-) (limited to 'debian') diff --git a/debian/control b/debian/control index 7880bd317..8e9aaa702 100644 --- a/debian/control +++ b/debian/control @@ -116,7 +116,6 @@ Depends: openvpn, openvpn-auth-ldap, openvpn-auth-radius, - openvpn-dco, openvpn-otp, owamp-client, owamp-server, -- cgit v1.2.3 From 223a6c5cd63fbcf7ff265b2721d0cd9f0aafb369 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 26 Jul 2023 22:24:16 -0500 Subject: xml: T5403: add support for supplemental xml cache --- .gitignore | 1 + debian/vyos-1x.postinst | 3 ++ python/vyos/xml_ref/generate_cache.py | 71 ++++++++++++++++++++++--------- python/vyos/xml_ref/pkg_cache/__init__.py | 0 python/vyos/xml_ref/update_cache.py | 51 ++++++++++++++++++++++ 5 files changed, 105 insertions(+), 21 deletions(-) create mode 100644 python/vyos/xml_ref/pkg_cache/__init__.py create mode 100755 python/vyos/xml_ref/update_cache.py (limited to 'debian') diff --git a/.gitignore b/.gitignore index e766a2c27..7707e94ca 100644 --- a/.gitignore +++ b/.gitignore @@ -144,3 +144,4 @@ debian/*.substvars data/component-versions.json # vyos-1x XML cache python/vyos/xml_ref/cache.py +python/vyos/xml_ref/pkg_cache/*_cache.py diff --git a/debian/vyos-1x.postinst b/debian/vyos-1x.postinst index 93e7ced9b..b1bd23ff2 100644 --- a/debian/vyos-1x.postinst +++ b/debian/vyos-1x.postinst @@ -180,6 +180,9 @@ systemctl enable vyos-config-cloud-init.service # Generate API GraphQL schema /usr/libexec/vyos/services/api/graphql/generate/generate_schema.py +# Update XML cache +python3 /usr/lib/python3/dist-packages/vyos/xml_ref/update_cache.py + # T1797: disable VPP support for rolling release, should be used by developers # only (in the initial phase). If you wan't to enable VPP use the below command # on your VyOS installation: diff --git a/python/vyos/xml_ref/generate_cache.py b/python/vyos/xml_ref/generate_cache.py index 792c6eea7..6f08486a8 100755 --- a/python/vyos/xml_ref/generate_cache.py +++ b/python/vyos/xml_ref/generate_cache.py @@ -18,10 +18,14 @@ import sys import json -import argparse +from argparse import ArgumentParser +from argparse import ArgumentTypeError +from os import getcwd +from os import makedirs from os.path import join from os.path import abspath from os.path import dirname +from os.path import basename from xmltodict import parse _here = dirname(__file__) @@ -29,9 +33,10 @@ _here = dirname(__file__) sys.path.append(join(_here, '..')) from configtree import reference_tree_to_json, ConfigTreeError -xml_cache = abspath(join(_here, 'cache.py')) xml_cache_json = 'xml_cache.json' xml_tmp = join('/tmp', xml_cache_json) +pkg_cache = abspath(join(_here, 'pkg_cache')) +ref_cache = abspath(join(_here, 'cache.py')) node_data_fields = ("node_type", "multi", "valueless", "default_value") @@ -45,16 +50,29 @@ def trim_node_data(cache: dict): if isinstance(cache[k], dict): trim_node_data(cache[k]) +def non_trivial(s): + if not s: + raise ArgumentTypeError("Argument must be non empty string") + if s == 'vyos-1x' and basename(getcwd()) != 'vyos-1x': + # builds outside of vyos-1x must specify package name + raise ArgumentTypeError("Specify package name") + return s + def main(): - parser = argparse.ArgumentParser(description='generate and save dict from xml defintions') + parser = ArgumentParser(description='generate and save dict from xml defintions') parser.add_argument('--xml-dir', type=str, required=True, help='transcluded xml interface-definition directory') - parser.add_argument('--save-json-dir', type=str, - help='directory to save json cache if needed') - args = parser.parse_args() - - xml_dir = abspath(args.xml_dir) - save_dir = abspath(args.save_json_dir) if args.save_json_dir else None + parser.add_argument('--package-name', type=non_trivial, default='vyos-1x', + help='name of current package') + parser.add_argument('--output-path', help='path to generated cache') + args = vars(parser.parse_args()) + + xml_dir = abspath(args['xml_dir']) + pkg_name = args['package_name'].replace('-','_') + cache_name = pkg_name + '_cache.py' + out_path = args['output_path'] + path = out_path if out_path is not None else pkg_cache + xml_cache = abspath(join(path, cache_name)) try: reference_tree_to_json(xml_dir, xml_tmp) @@ -67,21 +85,30 @@ def main(): trim_node_data(d) - if save_dir is not None: - save_file = join(save_dir, xml_cache_json) - with open(save_file, 'w') as f: - f.write(json.dumps(d)) - syntax_version = join(xml_dir, 'xml-component-version.xml') - with open(syntax_version) as f: - content = f.read() + try: + with open(syntax_version) as f: + component = f.read() + except FileNotFoundError: + if pkg_name != 'vyos_1x': + component = '' + else: + print("\nWARNING: missing xml-component-version.xml\n") + sys.exit(1) - parsed = parse(content) - converted = parsed['interfaceDefinition']['syntaxVersion'] + if component: + parsed = parse(component) + else: + parsed = None version = {} - for i in converted: - tmp = {i['@component']: i['@version']} - version |= tmp + # addon package definitions may have empty (== 0) version info + if parsed is not None and parsed['interfaceDefinition'] is not None: + converted = parsed['interfaceDefinition']['syntaxVersion'] + if not isinstance(converted, list): + converted = [converted] + for i in converted: + tmp = {i['@component']: i['@version']} + version |= tmp version = {"component_version": version} @@ -90,5 +117,7 @@ def main(): with open(xml_cache, 'w') as f: f.write(f'reference = {str(d)}') + print(cache_name) + if __name__ == '__main__': main() diff --git a/python/vyos/xml_ref/pkg_cache/__init__.py b/python/vyos/xml_ref/pkg_cache/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/vyos/xml_ref/update_cache.py b/python/vyos/xml_ref/update_cache.py new file mode 100755 index 000000000..0842bcbe9 --- /dev/null +++ b/python/vyos/xml_ref/update_cache.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +import os +from copy import deepcopy +from generate_cache import pkg_cache +from generate_cache import ref_cache + +def dict_merge(source, destination): + dest = deepcopy(destination) + + for key, value in source.items(): + if key not in dest: + dest[key] = value + elif isinstance(source[key], dict): + dest[key] = dict_merge(source[key], dest[key]) + + return dest + +def main(): + res = {} + cache_dir = os.path.basename(pkg_cache) + for mod in os.listdir(pkg_cache): + mod = os.path.splitext(mod)[0] + if not mod.endswith('_cache'): + continue + d = getattr(__import__(f'{cache_dir}.{mod}', fromlist=[mod]), 'reference') + if mod == 'vyos_1x_cache': + res = dict_merge(res, d) + else: + res = dict_merge(d, res) + + with open(ref_cache, 'w') as f: + f.write(f'reference = {str(res)}') + +if __name__ == '__main__': + main() -- cgit v1.2.3 From 399edb32eb68ce5fa3189ff83f09b307c6e9222d Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sat, 29 Jul 2023 21:10:36 +0200 Subject: vpp: T1797: change dependency to amd64 builds only --- debian/control | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'debian') diff --git a/debian/control b/debian/control index 8e9aaa702..772edb540 100644 --- a/debian/control +++ b/debian/control @@ -91,7 +91,7 @@ Depends: libqmi-utils, libstrongswan-extra-plugins (>=5.9), libstrongswan-standard-plugins (>=5.9), - libvppinfra, + libvppinfra [amd64], libvyosconfig0, linux-cpupower, lldpd, @@ -145,7 +145,7 @@ Depends: python3-tabulate, python3-vici (>= 5.7.2), python3-voluptuous, - python3-vpp-api, + python3-vpp-api [amd64], python3-xmltodict, python3-zmq, qrencode, @@ -180,9 +180,9 @@ Depends: uidmap, usb-modeswitch, usbutils, - vpp, - vpp-plugin-core, - vpp-plugin-dpdk, + vpp [amd64], + vpp-plugin-core [amd64], + vpp-plugin-dpdk [amd64], vyatta-bash, vyatta-cfg, vyos-http-api-tools, -- cgit v1.2.3 From d59c9c35c03799000e91b4c1e12e9e8c02256aeb Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sat, 29 Jul 2023 15:04:39 -0500 Subject: xml: T5403: fix installation of xml cache --- .gitignore | 1 + Makefile | 5 ++++- debian/rules | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) (limited to 'debian') diff --git a/.gitignore b/.gitignore index 7707e94ca..d781beead 100644 --- a/.gitignore +++ b/.gitignore @@ -109,6 +109,7 @@ ENV/ templates-cfg/* templates-op/* tests/templates/* +xml_cache/* # Debian packaging debian/files diff --git a/Makefile b/Makefile index fe17ce994..ecd610bae 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ OP_TMPL_DIR := templates-op BUILD_DIR := build DATA_DIR := data SHIM_DIR := src/shim +CACHE_DIR := xml_cache LIBS := -lzmq CFLAGS := BUILD_ARCH := $(shell dpkg-architecture -q DEB_BUILD_ARCH) @@ -23,10 +24,11 @@ op_xml_obj = $(op_xml_src:.xml.in=.xml) .ONESHELL: interface_definitions: $(config_xml_obj) mkdir -p $(TMPL_DIR) + mkdir -p $(CACHE_DIR) $(CURDIR)/scripts/override-default $(BUILD_DIR)/interface-definitions - $(CURDIR)/python/vyos/xml_ref/generate_cache.py --xml-dir $(BUILD_DIR)/interface-definitions + $(CURDIR)/python/vyos/xml_ref/generate_cache.py --xml-dir $(BUILD_DIR)/interface-definitions --output-path $(CACHE_DIR) find $(BUILD_DIR)/interface-definitions -type f -name "*.xml" | xargs -I {} $(CURDIR)/scripts/build-command-templates {} $(CURDIR)/schema/interface_definition.rng $(TMPL_DIR) || exit 1 @@ -96,6 +98,7 @@ clean: rm -rf $(BUILD_DIR) rm -rf $(TMPL_DIR) rm -rf $(OP_TMPL_DIR) + rm -rf $(CACHE_DIR) $(MAKE) -C $(SHIM_DIR) clean .PHONY: test diff --git a/debian/rules b/debian/rules index 9ada2bf87..39185f1e6 100755 --- a/debian/rules +++ b/debian/rules @@ -9,6 +9,7 @@ VYOS_CFG_TMPL_DIR := opt/vyatta/share/vyatta-cfg/templates VYOS_OP_TMPL_DIR := opt/vyatta/share/vyatta-op/templates VYOS_MIBS_DIR := usr/share/snmp/mibs VYOS_LOCALUI_DIR := srv/localui +VYOS_XML_CACHE_DIR := python/vyos/xml_ref/pkg_cache MIGRATION_SCRIPTS_DIR := opt/vyatta/etc/config-migrate/migrate SYSTEM_SCRIPTS_DIR := usr/libexec/vyos/system @@ -35,6 +36,8 @@ override_dh_auto_install: # convert the XML to dictionaries env PYTHONPATH=python python3 python/vyos/xml/generate.py + cp xml_cache/vyos_1x_cache.py python/vyos/xml_ref/pkg_cache + cd python; python3 setup.py install --install-layout=deb --root ../$(DIR); cd .. # Install scripts -- cgit v1.2.3