From dbe09d4c3dac302ad7b24cdd4f2b1ed123436442 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 31 Jan 2025 15:48:33 -0600 Subject: T6946: add wrapper for show_commit_data and test function --- python/vyos/config_mgmt.py | 10 +++++----- python/vyos/configtree.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) (limited to 'python') diff --git a/python/vyos/config_mgmt.py b/python/vyos/config_mgmt.py index 1c2b70fdf..dd8910afb 100644 --- a/python/vyos/config_mgmt.py +++ b/python/vyos/config_mgmt.py @@ -287,7 +287,7 @@ Proceed ?""" # commits under commit-confirm are not added to revision list unless # confirmed, hence a soft revert is to revision 0 - revert_ct = self._get_config_tree_revision(0) + revert_ct = self.get_config_tree_revision(0) message = '[commit-confirm] Reverting to previous config now' os.system('wall -n ' + message) @@ -351,7 +351,7 @@ Proceed ?""" ) return msg, 1 - rollback_ct = self._get_config_tree_revision(rev) + rollback_ct = self.get_config_tree_revision(rev) try: load(rollback_ct, switch='explicit') print('Rollback diff has been applied.') @@ -382,7 +382,7 @@ Proceed ?""" if rev1 is not None: if not self._check_revision_number(rev1): return f'Invalid revision number {rev1}', 1 - ct1 = self._get_config_tree_revision(rev1) + ct1 = self.get_config_tree_revision(rev1) ct2 = self.working_config msg = f'No changes between working and revision {rev1} configurations.\n' if rev2 is not None: @@ -390,7 +390,7 @@ Proceed ?""" return f'Invalid revision number {rev2}', 1 # compare older to newer ct2 = ct1 - ct1 = self._get_config_tree_revision(rev2) + ct1 = self.get_config_tree_revision(rev2) msg = f'No changes between revisions {rev2} and {rev1} configurations.\n' out = '' @@ -575,7 +575,7 @@ Proceed ?""" r = f.read().decode() return r - def _get_config_tree_revision(self, rev: int): + def get_config_tree_revision(self, rev: int): c = self._get_file_revision(rev) return ConfigTree(c) diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index 4ad0620a5..2b8930882 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -488,6 +488,22 @@ def mask_inclusive(left, right, libpath=LIBPATH): return tree +def show_commit_data(active_tree, proposed_tree, libpath=LIBPATH): + if not ( + isinstance(active_tree, ConfigTree) and isinstance(proposed_tree, ConfigTree) + ): + raise TypeError('Arguments must be instances of ConfigTree') + + __lib = cdll.LoadLibrary(libpath) + __show_commit_data = __lib.show_commit_data + __show_commit_data.argtypes = [c_void_p, c_void_p] + __show_commit_data.restype = c_char_p + + res = __show_commit_data(active_tree._get_config(), proposed_tree._get_config()) + + return res.decode() + + def reference_tree_to_json(from_dir, to_file, internal_cache='', libpath=LIBPATH): try: __lib = cdll.LoadLibrary(libpath) -- cgit v1.2.3 From defc2b665940a8fa14dcc1753dc40bdee0f52a64 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 5 Feb 2025 15:30:50 -0600 Subject: T7121: add configtree read/write to internal representation --- python/vyos/configtree.py | 47 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 6 deletions(-) (limited to 'python') diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index 2b8930882..c1db0782b 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -66,9 +66,14 @@ class ConfigTreeError(Exception): class ConfigTree(object): - def __init__(self, config_string=None, address=None, libpath=LIBPATH): - if config_string is None and address is None: - raise TypeError("ConfigTree() requires one of 'config_string' or 'address'") + def __init__( + self, config_string=None, address=None, internal=None, libpath=LIBPATH + ): + if config_string is None and address is None and internal is None: + raise TypeError( + "ConfigTree() requires one of 'config_string', 'address', or 'internal'" + ) + self.__config = None self.__lib = cdll.LoadLibrary(libpath) @@ -89,6 +94,13 @@ class ConfigTree(object): self.__to_commands.argtypes = [c_void_p, c_char_p] self.__to_commands.restype = c_char_p + self.__read_internal = self.__lib.read_internal + self.__read_internal.argtypes = [c_char_p] + self.__read_internal.restype = c_void_p + + self.__write_internal = self.__lib.write_internal + self.__write_internal.argtypes = [c_void_p, c_char_p] + self.__to_json = self.__lib.to_json self.__to_json.argtypes = [c_void_p] self.__to_json.restype = c_char_p @@ -168,7 +180,21 @@ class ConfigTree(object): self.__destroy = self.__lib.destroy self.__destroy.argtypes = [c_void_p] - if address is None: + self.__equal = self.__lib.equal + self.__equal.argtypes = [c_void_p, c_void_p] + self.__equal.restype = c_bool + + if address is not None: + self.__config = address + self.__version = '' + elif internal is not None: + config = self.__read_internal(internal.encode()) + if config is None: + msg = self.__get_error().decode() + raise ValueError('Failed to read internal rep: {0}'.format(msg)) + else: + self.__config = config + elif config_string is not None: config_section, version_section = extract_version(config_string) config_section = escape_backslash(config_section) config = self.__from_string(config_section.encode()) @@ -179,8 +205,9 @@ class ConfigTree(object): self.__config = config self.__version = version_section else: - self.__config = address - self.__version = '' + raise TypeError( + "ConfigTree() requires one of 'config_string', 'address', or 'internal'" + ) self.__migration = os.environ.get('VYOS_MIGRATION') if self.__migration: @@ -190,6 +217,11 @@ class ConfigTree(object): if self.__config is not None: self.__destroy(self.__config) + def __eq__(self, other): + if isinstance(other, ConfigTree): + return self.__equal(self._get_config(), other._get_config()) + return False + def __str__(self): return self.to_string() @@ -199,6 +231,9 @@ class ConfigTree(object): def get_version_string(self): return self.__version + def write_cache(self, file_name): + self.__write_internal(self._get_config(), file_name) + def to_string(self, ordered_values=False, no_version=False): config_string = self.__to_string(self.__config, ordered_values).decode() config_string = unescape_backslash(config_string) -- cgit v1.2.3 From 33394e41516a7a7591629e913c7f981e2494e80a Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 16 Mar 2025 21:04:21 -0500 Subject: T7121: add Config init from internal cache The internal cache is used as a faster replacement to parsing the active and proposed configs on initialization of a commit session. --- python/vyos/configsource.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'python') diff --git a/python/vyos/configsource.py b/python/vyos/configsource.py index 59e5ac8a1..65cef5333 100644 --- a/python/vyos/configsource.py +++ b/python/vyos/configsource.py @@ -319,3 +319,13 @@ class ConfigSourceString(ConfigSource): self._session_config = ConfigTree(session_config_text) if session_config_text else None except ValueError: raise ConfigSourceError(f"Init error in {type(self)}") + +class ConfigSourceCache(ConfigSource): + def __init__(self, running_config_cache=None, session_config_cache=None): + super().__init__() + + try: + self._running_config = ConfigTree(internal=running_config_cache) if running_config_cache else None + self._session_config = ConfigTree(internal=session_config_cache) if session_config_cache else None + except ValueError: + raise ConfigSourceError(f"Init error in {type(self)}") -- cgit v1.2.3 From 3078c7c976627a735df9175fb28b2d33b77d1cae Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 16 Mar 2025 21:05:02 -0500 Subject: T7121: generate Python protobuf files at build --- .gitignore | 3 +++ python/setup.py | 38 ++++++++++++++++++++++++++++++++++++++ python/vyos/defaults.py | 3 ++- python/vyos/proto/__init__.py | 0 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 python/vyos/proto/__init__.py (limited to 'python') diff --git a/.gitignore b/.gitignore index d1bfc91d7..27ed8000f 100644 --- a/.gitignore +++ b/.gitignore @@ -151,6 +151,9 @@ data/reftree.cache # autogenerated vyos-configd JSON definition data/configd-include.json +# autogenerated vyos-commitd protobuf files +python/vyos/proto/*pb2.py + # We do not use pip Pipfile Pipfile.lock diff --git a/python/setup.py b/python/setup.py index 2d614e724..96dc211f7 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,5 +1,11 @@ import os +import sys +import subprocess from setuptools import setup +from setuptools.command.build_py import build_py + +sys.path.append('./vyos') +from defaults import directories def packages(directory): return [ @@ -8,6 +14,35 @@ def packages(directory): if os.path.isfile(os.path.join(_[0], '__init__.py')) ] + +class GenerateProto(build_py): + ver = os.environ.get('OCAML_VERSION') + if ver: + proto_path = f'/opt/opam/{ver}/share/vyconf' + else: + proto_path = directories['proto_path'] + + def run(self): + # find all .proto files in vyconf proto_path + proto_files = [] + for _, _, files in os.walk(self.proto_path): + for file in files: + if file.endswith('.proto'): + proto_files.append(file) + + # compile each .proto file to Python + for proto_file in proto_files: + subprocess.check_call( + [ + 'protoc', + '--python_out=vyos/proto', + f'--proto_path={self.proto_path}/', + proto_file, + ] + ) + + build_py.run(self) + setup( name = "vyos", version = "1.3.0", @@ -29,4 +64,7 @@ setup( "config-mgmt = vyos.config_mgmt:run", ], }, + cmdclass={ + 'build_py': GenerateProto, + }, ) diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index 86194cd55..2600cf4fb 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -38,7 +38,8 @@ directories = { 'vyos_configdir' : '/opt/vyatta/config', 'completion_dir' : f'{base_dir}/completion', 'ca_certificates' : '/usr/local/share/ca-certificates/vyos', - 'ppp_nexthop_dir' : '/run/ppp_nexthop' + 'ppp_nexthop_dir' : '/run/ppp_nexthop', + 'proto_path': '/usr/share/vyos/vyconf' } systemd_services = { diff --git a/python/vyos/proto/__init__.py b/python/vyos/proto/__init__.py new file mode 100644 index 000000000..e69de29bb -- cgit v1.2.3 From b3427a6aa693728b69b32430a521c8cc9cc5c015 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 16 Mar 2025 21:07:25 -0500 Subject: T7121: add defaults entry for vyconfd.conf The vyconfd configuration file contains socket name, canonical directories, and file names shared with vyos-commitd. --- python/vyos/defaults.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'python') diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index 2600cf4fb..2b08ff68e 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -39,7 +39,7 @@ directories = { 'completion_dir' : f'{base_dir}/completion', 'ca_certificates' : '/usr/local/share/ca-certificates/vyos', 'ppp_nexthop_dir' : '/run/ppp_nexthop', - 'proto_path': '/usr/share/vyos/vyconf' + 'proto_path' : '/usr/share/vyos/vyconf' } systemd_services = { @@ -70,3 +70,5 @@ rt_symbolic_names = { rt_global_vrf = rt_symbolic_names['main'] rt_global_table = rt_symbolic_names['main'] + +vyconfd_conf = '/etc/vyos/vyconfd.conf' -- cgit v1.2.3 From b6df26f89df776bb63e79203c06744f9bed65c72 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 16 Mar 2025 21:10:07 -0500 Subject: T7121: add test_commit wrapper and test script --- python/vyos/configtree.py | 13 ++++++++++++ src/helpers/test_commit.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100755 src/helpers/test_commit.py (limited to 'python') diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index c1db0782b..83954327c 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -539,6 +539,19 @@ def show_commit_data(active_tree, proposed_tree, libpath=LIBPATH): return res.decode() +def test_commit(active_tree, proposed_tree, libpath=LIBPATH): + if not ( + isinstance(active_tree, ConfigTree) and isinstance(proposed_tree, ConfigTree) + ): + raise TypeError('Arguments must be instances of ConfigTree') + + __lib = cdll.LoadLibrary(libpath) + __test_commit = __lib.test_commit + __test_commit.argtypes = [c_void_p, c_void_p] + + __test_commit(active_tree._get_config(), proposed_tree._get_config()) + + def reference_tree_to_json(from_dir, to_file, internal_cache='', libpath=LIBPATH): try: __lib = cdll.LoadLibrary(libpath) diff --git a/src/helpers/test_commit.py b/src/helpers/test_commit.py new file mode 100755 index 000000000..00a413687 --- /dev/null +++ b/src/helpers/test_commit.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2025 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# This script is used to test execution of the commit algorithm by vyos-commitd + +from pathlib import Path +from argparse import ArgumentParser +from datetime import datetime + +from vyos.configtree import ConfigTree +from vyos.configtree import test_commit + + +parser = ArgumentParser( + description='Execute commit priority queue' +) +parser.add_argument( + '--active-config', help='Path to the active configuration file', required=True +) +parser.add_argument( + '--proposed-config', help='Path to the proposed configuration file', required=True +) +args = parser.parse_args() + +active_arg = args.active_config +proposed_arg = args.proposed_config + +active = ConfigTree(Path(active_arg).read_text()) +proposed = ConfigTree(Path(proposed_arg).read_text()) + + +time_begin_commit = datetime.now() +test_commit(active, proposed) +time_end_commit = datetime.now() +print(f'commit time: {time_end_commit - time_begin_commit}') -- cgit v1.2.3