From 05db4cdef55acc1a5869dabaab60264074c30c7d Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Mon, 23 Jun 2025 13:54:05 -0500 Subject: T7499: update vyos-merge-config.py script to use tree merge function --- src/helpers/vyos-merge-config.py | 128 +++++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 67 deletions(-) (limited to 'src/helpers/vyos-merge-config.py') diff --git a/src/helpers/vyos-merge-config.py b/src/helpers/vyos-merge-config.py index 79b17a261..197b35bfa 100755 --- a/src/helpers/vyos-merge-config.py +++ b/src/helpers/vyos-merge-config.py @@ -2,10 +2,6 @@ # Copyright VyOS maintainers and contributors # -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -15,94 +11,92 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . +import os import sys +import shlex import tempfile -import vyos.defaults -import vyos.remote +import argparse +from vyos.defaults import directories +from vyos.remote import get_remote_config from vyos.config import Config from vyos.configtree import ConfigTree +from vyos.configtree import mask_inclusive +from vyos.configtree import merge from vyos.migrate import ConfigMigrate from vyos.migrate import ConfigMigrateError -from vyos.utils.process import cmd -from vyos.utils.process import DEVNULL +from vyos.load_config import load_explicit -if (len(sys.argv) < 2): - print("Need config file name to merge.") - print("Usage: merge [config path]") - sys.exit(0) -file_name = sys.argv[1] +parser = argparse.ArgumentParser() +parser.add_argument('config_file', help='config file to merge from') +parser.add_argument( + '--destructive', action='store_true', help='replace values with those of merge file' +) +parser.add_argument('--paths', nargs='+', help='only merge from listed paths') +parser.add_argument( + '--migrate', action='store_true', help='migrate config file before merge' +) -configdir = vyos.defaults.directories['config'] +args = parser.parse_args() + +file_name = args.config_file +paths = [shlex.split(s) for s in args.paths] if args.paths else [] + +configdir = directories['config'] protocols = ['scp', 'sftp', 'http', 'https', 'ftp', 'tftp'] -if any(x in file_name for x in protocols): - config_file = vyos.remote.get_remote_config(file_name) - if not config_file: - sys.exit("No config file by that name.") +if any(file_name.startswith(f'{x}://') for x in protocols): + file_path = get_remote_config(file_name) + if not file_path: + sys.exit(f'No such file {file_name}') else: - canonical_path = "{0}/{1}".format(configdir, file_name) - first_err = None - try: - with open(canonical_path, 'r') as f: - config_file = f.read() - except Exception as err: - first_err = err - try: - with open(file_name, 'r') as f: - config_file = f.read() - except Exception as err: - print(first_err) - print(err) - sys.exit(1) - -with tempfile.NamedTemporaryFile() as file_to_migrate: - with open(file_to_migrate.name, 'w') as fd: - fd.write(config_file) - - config_migrate = ConfigMigrate(file_to_migrate.name) + if os.path.isfile(file_name): + file_path = file_name + else: + file_path = os.path.join(configdir, file_name) + if not os.path.isfile(file_path): + sys.exit(f'No such file {file_name}') + +if args.migrate: + migrate = ConfigMigrate(file_path) try: - config_migrate.run() + migrate.run() except ConfigMigrateError as e: sys.exit(e) -merge_config_tree = ConfigTree(config_file) +with open(file_path) as f: + merge_str = f.read() -effective_config = Config() -effective_config_tree = effective_config._running_config +merge_ct = ConfigTree(merge_str) -effective_cmds = effective_config_tree.to_commands() -merge_cmds = merge_config_tree.to_commands() +if paths: + mask = ConfigTree('') + for p in paths: + mask.set(p) -effective_cmd_list = effective_cmds.splitlines() -merge_cmd_list = merge_cmds.splitlines() + merge_ct = mask_inclusive(merge_ct, mask) -effective_cmd_set = set(effective_cmd_list) -add_cmds = [ cmd for cmd in merge_cmd_list if cmd not in effective_cmd_set ] +config = Config() +session_ct = config.get_config_tree() -path = None -if (len(sys.argv) > 2): - path = sys.argv[2:] - if (not effective_config_tree.exists(path) and not - merge_config_tree.exists(path)): - print("path {} does not exist in either effective or merge" - " config; will use root.".format(path)) - path = None - else: - path = " ".join(path) +merge_res = merge(session_ct, merge_ct, destructive=args.destructive) -if path: - add_cmds = [ cmd for cmd in add_cmds if path in cmd ] +if config.vyconf_session is not None: + with tempfile.NamedTemporaryFile() as merged_file: + with open(merged_file, 'w') as f: + f.write(merge_res.to_string()) + + out, err = config.vyconf_session.load_config(merged_file) + if err: + sys.exit(out) + print(out) +else: + load_explicit(merge_res) -for add in add_cmds: - try: - cmd(f'/opt/vyatta/sbin/my_{add}', shell=True, stderr=DEVNULL) - except OSError as err: - print(err) -if effective_config.session_changed(): +if config.session_changed(): print("Merge complete. Use 'commit' to make changes effective.") else: - print("No configuration changes to commit.") + print('No configuration changes to commit.') -- cgit v1.2.3 From 500c150bf049421b74dd6bc6a3e55c3e2600cc62 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Mon, 23 Jun 2025 14:46:33 -0500 Subject: T7499: load from internal representation to avoid re-parsing --- python/vyos/vyconf_session.py | 8 ++++++-- src/helpers/vyos-merge-config.py | 7 +++---- 2 files changed, 9 insertions(+), 6 deletions(-) (limited to 'src/helpers/vyos-merge-config.py') diff --git a/python/vyos/vyconf_session.py b/python/vyos/vyconf_session.py index 4a2e6e393..3f3f3957b 100644 --- a/python/vyos/vyconf_session.py +++ b/python/vyos/vyconf_session.py @@ -163,7 +163,9 @@ class VyconfSession: @raise_exception @config_mode - def load_config(self, file: str, migrate: bool = False) -> tuple[str, int]: + def load_config( + self, file: str, migrate: bool = False, cached: bool = False + ) -> tuple[str, int]: # pylint: disable=consider-using-with if migrate: tmp = tempfile.NamedTemporaryFile() @@ -178,7 +180,9 @@ class VyconfSession: else: tmp = '' - out = vyconf_client.send_request('load', token=self.__token, location=file) + out = vyconf_client.send_request( + 'load', token=self.__token, location=file, cached=cached + ) if tmp: tmp.close() diff --git a/src/helpers/vyos-merge-config.py b/src/helpers/vyos-merge-config.py index 197b35bfa..9ac4a1aed 100755 --- a/src/helpers/vyos-merge-config.py +++ b/src/helpers/vyos-merge-config.py @@ -84,11 +84,10 @@ session_ct = config.get_config_tree() merge_res = merge(session_ct, merge_ct, destructive=args.destructive) if config.vyconf_session is not None: - with tempfile.NamedTemporaryFile() as merged_file: - with open(merged_file, 'w') as f: - f.write(merge_res.to_string()) + with tempfile.NamedTemporaryFile() as merged_cache: + merge_res.write_cache(merged_cache.name) - out, err = config.vyconf_session.load_config(merged_file) + out, err = config.vyconf_session.load_config(merged_cache.name, cached=True) if err: sys.exit(out) print(out) -- cgit v1.2.3 From 816834bcad0a536930dd1bc4a35ef6b8537f3522 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 24 Jun 2025 07:11:41 -0500 Subject: T7499: use direct request to vyconfd to avoid re-validating --- python/vyos/vyconf_session.py | 27 +++++++++++++++++++++++++++ src/helpers/vyos-merge-config.py | 25 ++++++++++++------------- 2 files changed, 39 insertions(+), 13 deletions(-) (limited to 'src/helpers/vyos-merge-config.py') diff --git a/python/vyos/vyconf_session.py b/python/vyos/vyconf_session.py index 3f3f3957b..b42266793 100644 --- a/python/vyos/vyconf_session.py +++ b/python/vyos/vyconf_session.py @@ -188,6 +188,33 @@ class VyconfSession: return self.output(out), out.status + @raise_exception + @config_mode + def merge_config( + self, file: str, migrate: bool = False, destructive: bool = False + ) -> tuple[str, int]: + # pylint: disable=consider-using-with + if migrate: + tmp = tempfile.NamedTemporaryFile() + shutil.copy2(file, tmp.name) + config_migrate = ConfigMigrate(tmp.name) + try: + config_migrate.run() + except ConfigMigrateError as e: + tmp.close() + return repr(e), 1 + file = tmp.name + else: + tmp = '' + + out = vyconf_client.send_request( + 'merge', token=self.__token, location=file, destructive=destructive + ) + if tmp: + tmp.close() + + return self.output(out), out.status + @raise_exception def save_config(self, file: str, append_version: bool = False) -> tuple[str, int]: out = vyconf_client.send_request('save', token=self.__token, location=file) diff --git a/src/helpers/vyos-merge-config.py b/src/helpers/vyos-merge-config.py index 9ac4a1aed..4879b0e9c 100755 --- a/src/helpers/vyos-merge-config.py +++ b/src/helpers/vyos-merge-config.py @@ -14,7 +14,6 @@ import os import sys import shlex -import tempfile import argparse from vyos.defaults import directories @@ -52,8 +51,9 @@ if any(file_name.startswith(f'{x}://') for x in protocols): if not file_path: sys.exit(f'No such file {file_name}') else: - if os.path.isfile(file_name): - file_path = file_name + full_path = os.path.realpath(file_name) + if os.path.isfile(full_path): + file_path = full_path else: file_path = os.path.join(configdir, file_name) if not os.path.isfile(file_path): @@ -79,19 +79,18 @@ if paths: merge_ct = mask_inclusive(merge_ct, mask) config = Config() -session_ct = config.get_config_tree() - -merge_res = merge(session_ct, merge_ct, destructive=args.destructive) if config.vyconf_session is not None: - with tempfile.NamedTemporaryFile() as merged_cache: - merge_res.write_cache(merged_cache.name) - - out, err = config.vyconf_session.load_config(merged_cache.name, cached=True) - if err: - sys.exit(out) - print(out) + out, err = config.vyconf_session.merge_config( + file_path, destructive=args.destructive + ) + if err: + sys.exit(out) + print(out) else: + session_ct = config.get_config_tree() + merge_res = merge(session_ct, merge_ct, destructive=args.destructive) + load_explicit(merge_res) -- cgit v1.2.3 From 04a714fbf22c4accd9253f55826b0eefcd9f88fa Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 29 Jun 2025 17:11:44 -0500 Subject: T7499: add utility to download/uncompress config file, for load/merge --- python/vyos/remote.py | 44 ++++++++++++++++++ src/helpers/vyos-load-config.py | 99 +++++++++++++++------------------------- src/helpers/vyos-merge-config.py | 47 +++++++++---------- 3 files changed, 104 insertions(+), 86 deletions(-) (limited to 'src/helpers/vyos-merge-config.py') diff --git a/python/vyos/remote.py b/python/vyos/remote.py index f6ab5c3f9..b73f486c0 100644 --- a/python/vyos/remote.py +++ b/python/vyos/remote.py @@ -22,6 +22,7 @@ import stat import sys import tempfile import urllib.parse +import gzip from contextlib import contextmanager from pathlib import Path @@ -44,6 +45,7 @@ from vyos.utils.misc import begin from vyos.utils.process import cmd, rc_cmd from vyos.version import get_version from vyos.base import Warning +from vyos.defaults import directories CHUNK_SIZE = 8192 @@ -478,3 +480,45 @@ def get_remote_config(urlstring, source_host='', source_port=0): return f.read() finally: os.remove(temp) + + +def get_config_file(file_in: str, file_out: str, source_host='', source_port=0): + protocols = ['scp', 'sftp', 'http', 'https', 'ftp', 'tftp'] + config_dir = directories['config'] + + with tempfile.NamedTemporaryFile() as tmp_file: + if any(file_in.startswith(f'{x}://') for x in protocols): + try: + download( + tmp_file.name, + file_in, + check_space=True, + source_host='', + source_port=0, + raise_error=True, + ) + except Exception as e: + return e + file_name = tmp_file.name + else: + full_path = os.path.realpath(file_in) + if os.path.isfile(full_path): + file_in = full_path + else: + file_in = os.path.join(config_dir, file_in) + if not os.path.isfile(file_in): + return ValueError(f'No such file {file_in}') + + file_name = file_in + + if file_in.endswith('.gz'): + try: + with gzip.open(file_name, 'rb') as f_in: + with open(file_out, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + except Exception as e: + return e + else: + shutil.copyfile(file_name, file_out) + + return None diff --git a/src/helpers/vyos-load-config.py b/src/helpers/vyos-load-config.py index cd6bff0d4..01a6a88dc 100755 --- a/src/helpers/vyos-load-config.py +++ b/src/helpers/vyos-load-config.py @@ -16,84 +16,57 @@ # # -"""Load config file from within config session. -Config file specified by URI or path (without scheme prefix). -Example: load https://somewhere.net/some.config - or - load /tmp/some.config -""" - import os import sys -import gzip +import argparse import tempfile -import vyos.defaults -import vyos.remote -from vyos.configsource import ConfigSourceSession, VyOSError + +from vyos.remote import get_config_file +from vyos.config import Config from vyos.migrate import ConfigMigrate from vyos.migrate import ConfigMigrateError +from vyos.load_config import load as load_config -class LoadConfig(ConfigSourceSession): - """A subclass for calling 'loadFile'. - This does not belong in configsource.py, and only has a single caller. - """ - def load_config(self, path): - return self._run(['/bin/cli-shell-api','loadFile',path]) - -file_name = sys.argv[1] if len(sys.argv) > 1 else 'config.boot' -configdir = vyos.defaults.directories['config'] -protocols = ['scp', 'sftp', 'http', 'https', 'ftp', 'tftp'] -def get_local_config(filename): - if os.path.isfile(filename): - fname = filename - elif os.path.isfile(os.path.join(configdir, filename)): - fname = os.path.join(configdir, filename) - else: - sys.exit(f"No such file '{filename}'") +parser = argparse.ArgumentParser() +parser.add_argument('config_file', help='config file to load') +parser.add_argument( + '--migrate', action='store_true', help='migrate config file before merge' +) - if fname.endswith('.gz'): - with gzip.open(fname, 'rb') as f: - try: - config_str = f.read().decode() - except OSError as e: - sys.exit(e) - else: - with open(fname, 'r') as f: - try: - config_str = f.read() - except OSError as e: - sys.exit(e) +args = parser.parse_args() - return config_str - -if any(file_name.startswith(f'{x}://') for x in protocols): - config_string = vyos.remote.get_remote_config(file_name) - if not config_string: - sys.exit(f"No such config file at '{file_name}'") -else: - config_string = get_local_config(file_name) +file_name = args.config_file -config = LoadConfig() +# pylint: disable=consider-using-with +file_path = tempfile.NamedTemporaryFile(delete=False).name +err = get_config_file(file_name, file_path) +if err: + os.remove(file_path) + sys.exit(err) -print(f"Loading configuration from '{file_name}'") +if args.migrate: + migrate = ConfigMigrate(file_path) + try: + migrate.run() + except ConfigMigrateError as e: + os.remove(file_path) + sys.exit(e) -with tempfile.NamedTemporaryFile() as fp: - with open(fp.name, 'w') as fd: - fd.write(config_string) +config = Config() - config_migrate = ConfigMigrate(fp.name) - try: - config_migrate.run() - except ConfigMigrateError as err: - sys.exit(err) +if config.vyconf_session is not None: + out, err = config.vyconf_session.load_config(file_path) + if err: + os.remove(file_path) + sys.exit(out) + print(out) +else: + load_config(file_path) - try: - config.load_config(fp.name) - except VyOSError as err: - sys.exit(err) +os.remove(file_path) if config.session_changed(): print("Load complete. Use 'commit' to make changes effective.") else: - print("No configuration changes to commit.") + print('No configuration changes to commit.') diff --git a/src/helpers/vyos-merge-config.py b/src/helpers/vyos-merge-config.py index 4879b0e9c..e8a696eb5 100755 --- a/src/helpers/vyos-merge-config.py +++ b/src/helpers/vyos-merge-config.py @@ -2,22 +2,27 @@ # Copyright VyOS maintainers and contributors # +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. # -# This library is distributed in the hope that it will be useful, +# This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# # -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see . import os import sys import shlex import argparse +import tempfile -from vyos.defaults import directories -from vyos.remote import get_remote_config +from vyos.remote import get_config_file from vyos.config import Config from vyos.configtree import ConfigTree from vyos.configtree import mask_inclusive @@ -42,28 +47,19 @@ args = parser.parse_args() file_name = args.config_file paths = [shlex.split(s) for s in args.paths] if args.paths else [] -configdir = directories['config'] - -protocols = ['scp', 'sftp', 'http', 'https', 'ftp', 'tftp'] - -if any(file_name.startswith(f'{x}://') for x in protocols): - file_path = get_remote_config(file_name) - if not file_path: - sys.exit(f'No such file {file_name}') -else: - full_path = os.path.realpath(file_name) - if os.path.isfile(full_path): - file_path = full_path - else: - file_path = os.path.join(configdir, file_name) - if not os.path.isfile(file_path): - sys.exit(f'No such file {file_name}') +# pylint: disable=consider-using-with +file_path = tempfile.NamedTemporaryFile(delete=False).name +err = get_config_file(file_name, file_path) +if err: + os.remove(file_path) + sys.exit(err) if args.migrate: migrate = ConfigMigrate(file_path) try: migrate.run() except ConfigMigrateError as e: + os.remove(file_path) sys.exit(e) with open(file_path) as f: @@ -78,6 +74,9 @@ if paths: merge_ct = mask_inclusive(merge_ct, mask) +with open(file_path, 'w') as f: + f.write(merge_ct.to_string()) + config = Config() if config.vyconf_session is not None: @@ -85,6 +84,7 @@ if config.vyconf_session is not None: file_path, destructive=args.destructive ) if err: + os.remove(file_path) sys.exit(out) print(out) else: @@ -93,6 +93,7 @@ else: load_explicit(merge_res) +os.remove(file_path) if config.session_changed(): print("Merge complete. Use 'commit' to make changes effective.") -- cgit v1.2.3