From 6c80440b0711b8f1e0a713df4f097ab1902163cd Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sun, 30 Aug 2026 13:40:39 +0200 Subject: vyos: T9261: defer heavy imports out of the Config/ConfigSession path Importing vyos.config dragged in the whole HTTP and crypto stack. vyos.version imported requests and urllib3 for get_remote_version(), plus vyos.system.image for a single call in get_full_version_data() - and that reaches vyos.system.grub and vyos.template, so every consumer paid for jinja2 as well. vyconf_session imported vyos.remote, and therefore paramiko and cryptography, for a get_config_file() used in two methods. All of these move to their call sites. vyos/utils/__init__.py eagerly imported all 18 submodules, so 'from vyos.utils.dict import get_sub_dict' also loaded auth, network and process. It now resolves them on first attribute access via a module-level __getattr__. No caller relied on the eager import; the common 'from vyos.utils. import ' form is unaffected. import vyos.config 129.6 -> 47.7 ms (-63%, 2.7x), 575 -> 194 modules import vyos.ifconfig 205.0 -> 149.7 ms (-27%, 1.4x) This is per process, so it is paid again by every conf-mode script not handled in-daemon by vyos-configd, every op-mode script, and every smoketest. --- python/vyos/utils/__init__.py | 58 +++++++++++++++++++++++++++++-------------- python/vyos/version.py | 11 +++++--- python/vyos/vyconf_session.py | 11 +++++++- 3 files changed, 58 insertions(+), 22 deletions(-) (limited to 'python') diff --git a/python/vyos/utils/__init__.py b/python/vyos/utils/__init__.py index 280cde17f..2d934b75f 100644 --- a/python/vyos/utils/__init__.py +++ b/python/vyos/utils/__init__.py @@ -13,21 +13,43 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . -from vyos.utils import assertion -from vyos.utils import auth -from vyos.utils import boot -from vyos.utils import commit -from vyos.utils import configfs -from vyos.utils import convert -from vyos.utils import cpu -from vyos.utils import dict -from vyos.utils import file -from vyos.utils import io -from vyos.utils import kernel -from vyos.utils import list -from vyos.utils import locking -from vyos.utils import misc -from vyos.utils import network -from vyos.utils import permission -from vyos.utils import process -from vyos.utils import system +import importlib + +__all__ = [ + 'assertion', + 'auth', + 'boot', + 'commit', + 'configfs', + 'convert', + 'cpu', + 'dict', + 'file', + 'io', + 'kernel', + 'list', + 'locking', + 'misc', + 'network', + 'permission', + 'process', + 'system', +] + + +def __getattr__(name): + """Import submodules on first access (PEP 562). + + Importing them eagerly pulled the whole set into every consumer of + 'from vyos.utils. import ', which is a measurable cost on the + conf-mode script path where each script is a fresh interpreter. + """ + if name not in __all__: + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + mod = importlib.import_module(f'{__name__}.{name}') + globals()[name] = mod + return mod + + +def __dir__(): + return sorted(list(globals()) + __all__) diff --git a/python/vyos/version.py b/python/vyos/version.py index 4ba44f166..4288a26ff 100644 --- a/python/vyos/version.py +++ b/python/vyos/version.py @@ -31,10 +31,7 @@ Example of the version data dict:: import os -import requests import vyos.defaults -from vyos.system.image import is_live_boot -from urllib3.util import retry from vyos.utils.file import read_file from vyos.utils.file import read_json @@ -70,6 +67,11 @@ def get_version(fname=version_file): def get_full_version_data(fname=version_file): + # Imported here rather than at module scope: vyos.system.image pulls in + # vyos.system.grub -> vyos.template -> jinja2, which every consumer of + # vyos.version would otherwise pay for on import. + from vyos.system.image import is_live_boot + version_data = get_version_data(fname) # Get system architecture (well, kernel architecture rather) @@ -126,6 +128,9 @@ def get_remote_version(url): ] """ + import requests + from urllib3.util import retry + headers = {} session = requests.Session() diff --git a/python/vyos/vyconf_session.py b/python/vyos/vyconf_session.py index 91e51338c..932c87964 100644 --- a/python/vyos/vyconf_session.py +++ b/python/vyos/vyconf_session.py @@ -31,7 +31,6 @@ from vyos.proto.vyconf_proto import Errnum from vyos.utils.commit import acquire_commit_lock_file from vyos.utils.commit import release_commit_lock_file from vyos.utils.commit import call_commit_hooks -from vyos.remote import get_config_file class VyconfSessionError(Exception): @@ -246,6 +245,11 @@ class VyconfSession: self, file_name: str, migrate: bool = False, cached: bool = False ) -> tuple[str, int]: # pylint: disable=consider-using-with + # Imported here rather than at module scope: vyos.remote pulls in + # paramiko and requests, which every consumer of vyos.config would + # otherwise pay for on import. + from vyos.remote import get_config_file + file_path = tempfile.NamedTemporaryFile(delete=False).name err = get_config_file(file_name, file_path) if err: @@ -275,6 +279,11 @@ class VyconfSession: self, file_name: str, migrate: bool = False, destructive: bool = False ) -> tuple[str, int]: # pylint: disable=consider-using-with + # Imported here rather than at module scope: vyos.remote pulls in + # paramiko and requests, which every consumer of vyos.config would + # otherwise pay for on import. + from vyos.remote import get_config_file + file_path = tempfile.NamedTemporaryFile(delete=False).name err = get_config_file(file_name, file_path) if err: -- cgit v1.2.3 From de81311ce99e1ce8776ac809c336726ad68d6751 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sat, 29 Aug 2026 09:04:05 +0000 Subject: configtree: T9261: bind libvyosconfig once instead of per ConfigTree Every ConfigTree re-opened libvyosconfig with cdll.LoadLibrary and re-declared all 31 function prototypes inline in __init__. DiffTree and the twelve module-level helpers repeated the pattern per call, and since a DiffTree builds six further trees internally, a single one performed seven dlopens and roughly 200 argtypes/restype assignments. referencetree.py had its own copy. Declare the prototypes once in a _PROTOTYPES table and hand out an lru_cache'd wrapper keyed on libpath. Symbols are resolved on first use rather than up front, because not every build of the library exports every entry point and the previous code only failed on symbols actually called. referencetree.py shares the cache and picks up LIBPATH from configtree, so it now honours the /tmp/libvyosconfig/_build development path instead of hardcoding the install path. ConfigTree() 72.3 -> 6.3 us (-91%, 11.5x) DiffTree() 397.3 -> 34.7 us (-91%, 11.4x) Two prototypes were wrong and are fixed by declaring them in one place. validate_tree_filter had no restype, so ctypes defaulted to c_int and truncated the returned tree pointer to 32 bits before ConfigTree(address=...) saw it; harmless while the heap stays below 4 GB, corrupt above it. get_subtree declared two argtypes but is called with three, leaving with_node unchecked. --- python/vyos/configtree.py | 463 +++++++++++++++++-------------------------- python/vyos/referencetree.py | 42 +--- 2 files changed, 190 insertions(+), 315 deletions(-) (limited to 'python') diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index 6791b1c29..90e889692 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -18,6 +18,7 @@ import json import logging from ctypes import cdll, c_char_p, c_void_p, c_int, c_bool +from functools import lru_cache from typing import TYPE_CHECKING from pathlib import Path @@ -31,6 +32,105 @@ BUILD_PATH = '/tmp/libvyosconfig/_build/libvyosconfig.so' INSTALL_PATH = '/usr/lib/libvyosconfig.so.0' LIBPATH = BUILD_PATH if os.path.isfile(BUILD_PATH) else INSTALL_PATH +# C symbol -> (argtypes, restype). A restype of None means "leave the ctypes +# default (c_int)", which is what these entry points relied on before. +_PROTOTYPES = { + 'from_string': ([c_char_p], c_void_p), + 'get_error': ([], c_char_p), + 'to_string': ([c_void_p, c_bool], c_char_p), + 'to_commands': ([c_void_p, c_char_p], c_char_p), + 'read_internal': ([c_char_p], c_void_p), + 'write_internal': ([c_void_p, c_char_p], None), + 'read_internal_string': ([c_char_p], c_void_p), + 'write_internal_string': ([c_void_p], c_char_p), + 'to_json': ([c_void_p], c_char_p), + 'to_json_ast': ([c_void_p], c_char_p), + 'create_node': ([c_void_p, c_char_p], c_int), + 'set_add_value': ([c_void_p, c_char_p, c_char_p], c_int), + 'delete_value': ([c_void_p, c_char_p, c_char_p], c_int), + 'delete_node': ([c_void_p, c_char_p], c_int), + 'rename_node': ([c_void_p, c_char_p, c_char_p], c_int), + 'copy_node': ([c_void_p, c_char_p, c_char_p], c_int), + 'set_replace_value': ([c_void_p, c_char_p, c_char_p], c_int), + 'set_valueless': ([c_void_p, c_char_p], c_int), + 'exists': ([c_void_p, c_char_p], c_int), + 'value_exists': ([c_void_p, c_char_p, c_char_p], c_int), + 'list_nodes': ([c_void_p, c_char_p], c_char_p), + 'return_value': ([c_void_p, c_char_p], c_char_p), + 'return_values': ([c_void_p, c_char_p], c_char_p), + 'is_tag': ([c_void_p, c_char_p], c_int), + 'set_tag': ([c_void_p, c_char_p, c_bool], c_int), + 'is_leaf': ([c_void_p, c_char_p], c_bool), + 'set_leaf': ([c_void_p, c_char_p, c_bool], c_int), + # third argument 'with_node' was passed unchecked before + 'get_subtree': ([c_void_p, c_char_p, c_bool], c_void_p), + 'destroy': ([c_void_p], None), + 'equal': ([c_void_p, c_void_p], c_bool), + 'config_dict': ( + [c_void_p, c_void_p, c_void_p, c_char_p, c_bool, c_bool], + c_char_p, + ), + 'diff_tree': ([c_char_p, c_void_p, c_void_p], c_void_p), + 'diff_compare': ([c_bool, c_char_p, c_void_p, c_void_p], c_char_p), + 'diff_show': ([c_void_p, c_void_p, c_void_p, c_char_p], c_char_p), + 'tree_union': ([c_void_p, c_void_p], c_void_p), + 'tree_merge': ([c_bool, c_void_p, c_void_p], c_void_p), + 'mask_inclusive': ([c_void_p, c_void_p], c_void_p), + 'mask_exclusive': ([c_void_p, c_void_p], c_void_p), + 'subtree_from_partial': ([c_void_p, c_void_p, c_void_p, c_char_p], c_void_p), + 'reference_tree_to_json': ([c_char_p, c_char_p, c_char_p], None), + 'merge_reference_tree_cache': ([c_char_p, c_char_p, c_char_p], None), + 'interface_definitions_to_cache': ([c_char_p, c_char_p], None), + 'reference_tree_cache_to_json': ([c_char_p, c_char_p], None), + # returns a tree pointer; without an explicit restype ctypes defaulted to + # c_int and truncated it to 32 bits + 'validate_tree_filter': ([c_void_p, c_char_p, c_char_p], c_void_p), + # vyos.referencetree + 'read_internal_string_reference_tree': ([c_char_p], c_void_p), + 'write_internal_reference_tree': ([c_void_p, c_char_p], None), + 'to_json_reference_tree': ([c_void_p], c_char_p), +} + + +class _Lib: + """Lazily resolving, prototype-declaring wrapper around libvyosconfig. + + A symbol is looked up and given its argtypes/restype on first use, then + cached as an instance attribute so __getattr__ is bypassed from then on. + Resolution stays lazy on purpose: not every build of the library exports + every entry point, and the previous per-instance binding code only failed + on symbols that were actually used. + """ + + def __init__(self, libpath): + self.__dict__['_lib'] = cdll.LoadLibrary(libpath) + + def __getattr__(self, name): + try: + argtypes, restype = _PROTOTYPES[name] + except KeyError: + raise AttributeError( + f'{name} is not a known libvyosconfig entry point' + ) from None + func = getattr(self._lib, name) + func.argtypes = argtypes + if restype is not None: + func.restype = restype + setattr(self, name, func) + return func + + +@lru_cache(maxsize=None) +def get_lib(libpath=LIBPATH): + """Load libvyosconfig once per process. + + Every ConfigTree/DiffTree used to re-open the library and re-assign all + argtypes/restypes in its constructor; the result is cached here instead. + Loading stays lazy so that importing this module does not require the + shared object to be present. + """ + return _Lib(libpath) + def replace_backslash(s, search, replace): """Modify quoted strings containing backslashes not of escape sequences""" @@ -95,137 +195,7 @@ class ConfigTree: arg = [(k, v) for (k, v) in zip(arg_names, args) if v is not None][0] self.__config = None - self.__lib = cdll.LoadLibrary(libpath) - - # Import functions - self.__from_string = self.__lib.from_string - self.__from_string.argtypes = [c_char_p] - self.__from_string.restype = c_void_p - - self.__get_error = self.__lib.get_error - self.__get_error.argtypes = [] - self.__get_error.restype = c_char_p - - self.__to_string = self.__lib.to_string - self.__to_string.argtypes = [c_void_p, c_bool] - self.__to_string.restype = c_char_p - - self.__to_commands = self.__lib.to_commands - self.__to_commands.argtypes = [c_void_p, c_char_p] - self.__to_commands.restype = c_char_p - - self.__read_internal = self.__lib.read_internal - self.__read_internal.argtypes = [c_char_p] - self.__read_internal.restype = c_void_p - - self.__write_internal = self.__lib.write_internal - self.__write_internal.argtypes = [c_void_p, c_char_p] - - self.__read_internal_string = self.__lib.read_internal_string - self.__read_internal_string.argtypes = [c_char_p] - self.__read_internal_string.restype = c_void_p - - self.__write_internal_string = self.__lib.write_internal_string - self.__write_internal_string.argtypes = [c_void_p] - self.__write_internal_string.restype = c_char_p - - self.__to_json = self.__lib.to_json - self.__to_json.argtypes = [c_void_p] - self.__to_json.restype = c_char_p - - self.__to_json_ast = self.__lib.to_json_ast - self.__to_json_ast.argtypes = [c_void_p] - self.__to_json_ast.restype = c_char_p - - self.__create_node = self.__lib.create_node - self.__create_node.argtypes = [c_void_p, c_char_p] - self.__create_node.restype = c_int - - self.__set_add_value = self.__lib.set_add_value - self.__set_add_value.argtypes = [c_void_p, c_char_p, c_char_p] - self.__set_add_value.restype = c_int - - self.__delete_value = self.__lib.delete_value - self.__delete_value.argtypes = [c_void_p, c_char_p, c_char_p] - self.__delete_value.restype = c_int - - self.__delete = self.__lib.delete_node - self.__delete.argtypes = [c_void_p, c_char_p] - self.__delete.restype = c_int - - self.__rename = self.__lib.rename_node - self.__rename.argtypes = [c_void_p, c_char_p, c_char_p] - self.__rename.restype = c_int - - self.__copy = self.__lib.copy_node - self.__copy.argtypes = [c_void_p, c_char_p, c_char_p] - self.__copy.restype = c_int - - self.__set_replace_value = self.__lib.set_replace_value - self.__set_replace_value.argtypes = [c_void_p, c_char_p, c_char_p] - self.__set_replace_value.restype = c_int - - self.__set_valueless = self.__lib.set_valueless - self.__set_valueless.argtypes = [c_void_p, c_char_p] - self.__set_valueless.restype = c_int - - self.__exists = self.__lib.exists - self.__exists.argtypes = [c_void_p, c_char_p] - self.__exists.restype = c_int - - self.__value_exists = self.__lib.value_exists - self.__value_exists.argtypes = [c_void_p, c_char_p, c_char_p] - self.__value_exists.restype = c_int - - self.__list_nodes = self.__lib.list_nodes - self.__list_nodes.argtypes = [c_void_p, c_char_p] - self.__list_nodes.restype = c_char_p - - self.__return_value = self.__lib.return_value - self.__return_value.argtypes = [c_void_p, c_char_p] - self.__return_value.restype = c_char_p - - self.__return_values = self.__lib.return_values - self.__return_values.argtypes = [c_void_p, c_char_p] - self.__return_values.restype = c_char_p - - self.__is_tag = self.__lib.is_tag - self.__is_tag.argtypes = [c_void_p, c_char_p] - self.__is_tag.restype = c_int - - self.__set_tag = self.__lib.set_tag - self.__set_tag.argtypes = [c_void_p, c_char_p, c_bool] - self.__set_tag.restype = c_int - - self.__is_leaf = self.__lib.is_leaf - self.__is_leaf.argtypes = [c_void_p, c_char_p] - self.__is_leaf.restype = c_bool - - self.__set_leaf = self.__lib.set_leaf - self.__set_leaf.argtypes = [c_void_p, c_char_p, c_bool] - self.__set_leaf.restype = c_int - - self.__get_subtree = self.__lib.get_subtree - self.__get_subtree.argtypes = [c_void_p, c_char_p] - self.__get_subtree.restype = c_void_p - - self.__destroy = self.__lib.destroy - self.__destroy.argtypes = [c_void_p] - - self.__equal = self.__lib.equal - self.__equal.argtypes = [c_void_p, c_void_p] - self.__equal.restype = c_bool - - self.__config_dict = self.__lib.config_dict - self.__config_dict.argtypes = [ - c_void_p, - c_void_p, - c_void_p, - c_char_p, - c_bool, - c_bool, - ] - self.__config_dict.restype = c_char_p + self.__lib = get_lib(libpath) match arg: case ('address', address): @@ -236,18 +206,18 @@ class ConfigTree: cache_string = Path(internal).read_bytes() except OSError as e: raise ValueError(f'Failed to read internal cache file: {e}') - config = self.__read_internal_string(cache_string) + config = self.__lib.read_internal_string(cache_string) if config is None: - msg = self.__get_error().decode() + msg = self.__lib.get_error().decode() raise ValueError( f'Failed to read internal representation from file {internal}: {msg}' ) self.__config = config self.__version = '' case ('internal_string', internal_string): - config = self.__read_internal_string(internal_string.encode()) + config = self.__lib.read_internal_string(internal_string.encode()) if config is None: - msg = self.__get_error().decode() + msg = self.__lib.get_error().decode() raise ValueError( f'Failed to read internal representation from string: {msg}' ) @@ -256,9 +226,9 @@ class ConfigTree: case ('config_string', config_string): config_section, version_section = extract_version(config_string) config_section = escape_backslash(config_section) - config = self.__from_string(config_section.encode()) + config = self.__lib.from_string(config_section.encode()) if config is None: - msg = self.__get_error().decode() + msg = self.__lib.get_error().decode() raise ValueError(f'Failed to parse config: {msg}') self.__config = config self.__version = version_section @@ -286,11 +256,11 @@ class ConfigTree: def __del__(self): if self.__config is not None: - self.__destroy(self.__config) + self.__lib.destroy(self.__config) def __eq__(self, other): if isinstance(other, ConfigTree): - return self.__equal(self.get_tree(), other.get_tree()) + return self.__lib.equal(self.get_tree(), other.get_tree()) return False def __str__(self): @@ -303,18 +273,18 @@ class ConfigTree: return self.__version def write_cache(self, file_name): - cache_string = self.__write_internal_string(self.get_tree()) + cache_string = self.__lib.write_internal_string(self.get_tree()) try: Path(file_name).write_bytes(cache_string) except OSError as e: raise ValueError(f'Failed to write internal cache file: {e}') def write_internal_string(self) -> str: - res = self.__write_internal_string(self.get_tree()) + res = self.__lib.write_internal_string(self.get_tree()) return res.decode() def to_string(self, ordered_values=False, no_version=False): - config_string = self.__to_string(self.__config, ordered_values).decode() + config_string = self.__lib.to_string(self.__config, ordered_values).decode() config_string = unescape_backslash(config_string) if no_version: return config_string @@ -322,23 +292,23 @@ class ConfigTree: return config_string def to_commands(self, op='set'): - commands = self.__to_commands(self.__config, op.encode()).decode() + commands = self.__lib.to_commands(self.__config, op.encode()).decode() commands = unescape_backslash(commands) return commands def to_json(self): - return self.__to_json(self.__config).decode() + return self.__lib.to_json(self.__config).decode() def to_json_ast(self): - return self.__to_json_ast(self.__config).decode() + return self.__lib.to_json_ast(self.__config).decode() def create_node(self, path): check_path(path) path_str = ' '.join(map(str, path)).encode() - res = self.__create_node(self.__config, path_str) + res = self.__lib.create_node(self.__config, path_str) if res != 0: - msg = self.__get_error().decode() + msg = self.__lib.get_error().decode() raise ConfigTreeError(f'{msg}: {path}') def set(self, path, value=None, replace=True): @@ -354,17 +324,17 @@ class ConfigTree: path_str = ' '.join(map(str, path)).encode() if value is None: - res = self.__set_valueless(self.__config, path_str) + res = self.__lib.set_valueless(self.__config, path_str) else: if replace: - res = self.__set_replace_value( + res = self.__lib.set_replace_value( self.__config, path_str, str(value).encode() ) else: - res = self.__set_add_value(self.__config, path_str, str(value).encode()) + res = self.__lib.set_add_value(self.__config, path_str, str(value).encode()) if res != 0: - msg = self.__get_error().decode() + msg = self.__lib.get_error().decode() raise ConfigTreeError( f'{msg}: path "{path}" value "{value}" replace "{replace}"' ) @@ -378,9 +348,9 @@ class ConfigTree: check_path(path) path_str = ' '.join(map(str, path)).encode() - res = self.__delete(self.__config, path_str) + res = self.__lib.delete_node(self.__config, path_str) if res != 0: - msg = self.__get_error().decode() + msg = self.__lib.get_error().decode() raise ConfigTreeError(f'{msg}: path "{path}"') if self.__migration: @@ -390,9 +360,9 @@ class ConfigTree: check_path(path) path_str = ' '.join(map(str, path)).encode() - res = self.__delete_value(self.__config, path_str, value.encode()) + res = self.__lib.delete_value(self.__config, path_str, value.encode()) if res != 0: - msg = self.__get_error().decode() + msg = self.__lib.get_error().decode() raise ConfigTreeError(f'{msg}: path "{path}" value "{value}"') if self.__migration: @@ -408,9 +378,9 @@ class ConfigTree: if self.exists(new_path): raise ConfigTreeError(f'Name {new_name} already exists') - res = self.__rename(self.__config, path_str, newname_str) + res = self.__lib.rename_node(self.__config, path_str, newname_str) if res != 0: - msg = self.__get_error().decode() + msg = self.__lib.get_error().decode() raise ConfigTreeError(f'{msg}: {path}') if self.__migration: @@ -427,9 +397,9 @@ class ConfigTree: # Check if a node with intended new name already exists if self.exists(new_path): raise ConfigTreeError() - res = self.__copy(self.__config, oldpath_str, newpath_str) + res = self.__lib.copy_node(self.__config, oldpath_str, newpath_str) if res != 0: - msg = self.__get_error().decode() + msg = self.__lib.get_error().decode() raise ConfigTreeError(msg) if self.__migration: @@ -441,21 +411,21 @@ class ConfigTree: check_path(path) path_str = ' '.join(map(str, path)).encode() - res = self.__exists(self.__config, path_str) + res = self.__lib.exists(self.__config, path_str) return bool(res) def value_exists(self, path, value): check_path(path) path_str = ' '.join(map(str, path)).encode() - res = self.__value_exists(self.__config, path_str, value.encode()) + res = self.__lib.value_exists(self.__config, path_str, value.encode()) return bool(res) def list_nodes(self, path, path_must_exist=True): check_path(path) path_str = ' '.join(map(str, path)).encode() - res_json = self.__list_nodes(self.__config, path_str).decode() + res_json = self.__lib.list_nodes(self.__config, path_str).decode() res = json.loads(res_json) if res is None: @@ -468,7 +438,7 @@ class ConfigTree: check_path(path) path_str = ' '.join(map(str, path)).encode() - res_json = self.__return_value(self.__config, path_str).decode() + res_json = self.__lib.return_value(self.__config, path_str).decode() res = json.loads(res_json) if res is None: @@ -479,7 +449,7 @@ class ConfigTree: check_path(path) path_str = ' '.join(map(str, path)).encode() - res_json = self.__return_values(self.__config, path_str).decode() + res_json = self.__lib.return_values(self.__config, path_str).decode() res = json.loads(res_json) if res is None: @@ -490,41 +460,41 @@ class ConfigTree: check_path(path) path_str = ' '.join(map(str, path)).encode() - res = self.__is_tag(self.__config, path_str) + res = self.__lib.is_tag(self.__config, path_str) return bool(res) def set_tag(self, path, value=True): check_path(path) path_str = ' '.join(map(str, path)).encode() - res = self.__set_tag(self.__config, path_str, value) + res = self.__lib.set_tag(self.__config, path_str, value) if res == 0: return True - msg = self.__get_error().decode() + msg = self.__lib.get_error().decode() raise ConfigTreeError(f'{msg}: {path}') def is_leaf(self, path): check_path(path) path_str = ' '.join(map(str, path)).encode() - res = self.__is_leaf(self.__config, path_str) + res = self.__lib.is_leaf(self.__config, path_str) return bool(res) def set_leaf(self, path, value): check_path(path) path_str = ' '.join(map(str, path)).encode() - res = self.__set_leaf(self.__config, path_str, value) + res = self.__lib.set_leaf(self.__config, path_str, value) if res == 0: return True - msg = self.__get_error().decode() + msg = self.__lib.get_error().decode() raise ConfigTreeError(f'{msg}: {path}') def get_subtree(self, path, with_node=False): check_path(path) path_str = ' '.join(map(str, path)).encode() - res = self.__get_subtree(self.__config, path_str, with_node) + res = self.__lib.get_subtree(self.__config, path_str, with_node) subt = ConfigTree(address=res) return subt @@ -534,7 +504,7 @@ class ConfigTree: check_path(path) path_str = ' '.join(map(str, path)).encode() - res_json = self.__config_dict( + res_json = self.__lib.config_dict( self.__config, ref_tree.get_tree(), mask.get_tree(), @@ -561,18 +531,12 @@ def diff_compare(left, right, path=None, commands=False, libpath=LIBPATH): check_path(path) path_str = ' '.join(map(str, path)).encode() - __lib = cdll.LoadLibrary(libpath) - __diff_compare = __lib.diff_compare - __diff_compare.argtypes = [c_bool, c_char_p, c_void_p, c_void_p] - __diff_compare.restype = c_char_p - __get_error = __lib.get_error - __get_error.argtypes = [] - __get_error.restype = c_char_p + lib = get_lib(libpath) - res = __diff_compare(commands, path_str, left.get_tree(), right.get_tree()) + res = lib.diff_compare(commands, path_str, left.get_tree(), right.get_tree()) res = res.decode() if res == '#1@': - msg = __get_error().decode() + msg = lib.get_error().decode() raise ConfigTreeError(msg) res = unescape_backslash(res) @@ -587,18 +551,12 @@ def diff_show(rt, left, right, path=None, libpath=LIBPATH): check_path(path) path_str = ' '.join(map(str, path)).encode() - __lib = cdll.LoadLibrary(libpath) - __diff_show = __lib.diff_show - __diff_show.argtypes = [c_void_p, c_void_p, c_void_p, c_char_p] - __diff_show.restype = c_char_p - __get_error = __lib.get_error - __get_error.argtypes = [] - __get_error.restype = c_char_p + lib = get_lib(libpath) - res = __diff_show(rt.get_tree(), left.get_tree(), right.get_tree(), path_str) + res = lib.diff_show(rt.get_tree(), left.get_tree(), right.get_tree(), path_str) res = res.decode() if res == '#1@': - msg = __get_error().decode() + msg = lib.get_error().decode() raise ConfigTreeError(msg) res = unescape_backslash(res) @@ -613,15 +571,9 @@ def union(left, right, libpath=LIBPATH): if not (isinstance(left, ConfigTree) and isinstance(right, ConfigTree)): raise TypeError('Arguments must be instances of ConfigTree') - __lib = cdll.LoadLibrary(libpath) - __tree_union = __lib.tree_union - __tree_union.argtypes = [c_void_p, c_void_p] - __tree_union.restype = c_void_p - __get_error = __lib.get_error - __get_error.argtypes = [] - __get_error.restype = c_char_p + lib = get_lib(libpath) - res = __tree_union(left.get_tree(), right.get_tree()) + res = lib.tree_union(left.get_tree(), right.get_tree()) tree = ConfigTree(address=res) return tree @@ -635,15 +587,9 @@ def merge(left, right, destructive=False, libpath=LIBPATH): if not (isinstance(left, ConfigTree) and isinstance(right, ConfigTree)): raise TypeError('Arguments must be instances of ConfigTree') - __lib = cdll.LoadLibrary(libpath) - __tree_merge = __lib.tree_merge - __tree_merge.argtypes = [c_bool, c_void_p, c_void_p] - __tree_merge.restype = c_void_p - __get_error = __lib.get_error - __get_error.argtypes = [] - __get_error.restype = c_char_p + lib = get_lib(libpath) - res = __tree_merge(destructive, left.get_tree(), right.get_tree()) + res = lib.tree_merge(destructive, left.get_tree(), right.get_tree()) tree = ConfigTree(address=res) return tree @@ -655,19 +601,13 @@ def mask_inclusive(left, right, libpath=LIBPATH): raise TypeError('Arguments must be instances of ConfigTree') try: - __lib = cdll.LoadLibrary(libpath) - __mask_inclusive = __lib.mask_inclusive - __mask_inclusive.argtypes = [c_void_p, c_void_p] - __mask_inclusive.restype = c_void_p - __get_error = __lib.get_error - __get_error.argtypes = [] - __get_error.restype = c_char_p - - res = __mask_inclusive(left.get_tree(), right.get_tree()) + lib = get_lib(libpath) + + res = lib.mask_inclusive(left.get_tree(), right.get_tree()) except Exception as e: raise ConfigTreeError(e) if not res: - msg = __get_error().decode() + msg = lib.get_error().decode() raise ConfigTreeError(msg) tree = ConfigTree(address=res) @@ -681,19 +621,13 @@ def mask_exclusive(left, right, libpath=LIBPATH): raise TypeError('Arguments must be instances of ConfigTree') try: - __lib = cdll.LoadLibrary(libpath) - __mask_exclusive = __lib.mask_exclusive - __mask_exclusive.argtypes = [c_void_p, c_void_p] - __mask_exclusive.restype = c_void_p - __get_error = __lib.get_error - __get_error.argtypes = [] - __get_error.restype = c_char_p - - res = __mask_exclusive(left.get_tree(), right.get_tree()) + lib = get_lib(libpath) + + res = lib.mask_exclusive(left.get_tree(), right.get_tree()) except Exception as e: raise ConfigTreeError(e) if not res: - msg = __get_error().decode() + msg = lib.get_error().decode() raise ConfigTreeError(msg) tree = ConfigTree(address=res) @@ -741,15 +675,9 @@ def subtree_from_partial( path_str = ' '.join(map(str, path)).encode() try: - __lib = cdll.LoadLibrary(libpath) - __subtree_from_partial = __lib.subtree_from_partial - __subtree_from_partial.argtypes = [c_void_p, c_void_p, c_void_p, c_char_p] - __subtree_from_partial.restype = c_void_p - __get_error = __lib.get_error - __get_error.argtypes = [] - __get_error.restype = c_char_p - - res = __subtree_from_partial( + lib = get_lib(libpath) + + res = lib.subtree_from_partial( reference_tree.get_tree(), config_tree.get_tree(), start.get_tree(), @@ -758,7 +686,7 @@ def subtree_from_partial( except Exception as e: raise ConfigTreeError(e) if not res: - msg = __get_error().decode() + msg = lib.get_error().decode() raise ConfigTreeError(msg) tree = ConfigTree(address=res) @@ -769,72 +697,52 @@ def subtree_from_partial( def reference_tree_to_json(from_dir, to_file, internal_cache='', libpath=LIBPATH): # pylint: disable=raise-missing-from try: - __lib = cdll.LoadLibrary(libpath) - __reference_tree_to_json = __lib.reference_tree_to_json - __reference_tree_to_json.argtypes = [c_char_p, c_char_p, c_char_p] - __get_error = __lib.get_error - __get_error.argtypes = [] - __get_error.restype = c_char_p - res = __reference_tree_to_json( + lib = get_lib(libpath) + res = lib.reference_tree_to_json( internal_cache.encode(), from_dir.encode(), to_file.encode() ) except Exception as e: raise ConfigTreeError(e) if res == 1: - msg = __get_error().decode() + msg = lib.get_error().decode() raise ConfigTreeError(msg) def merge_reference_tree_cache(cache_dir, primary_name, result_name, libpath=LIBPATH): # pylint: disable=raise-missing-from try: - __lib = cdll.LoadLibrary(libpath) - __merge_reference_tree_cache = __lib.merge_reference_tree_cache - __merge_reference_tree_cache.argtypes = [c_char_p, c_char_p, c_char_p] - __get_error = __lib.get_error - __get_error.argtypes = [] - __get_error.restype = c_char_p - res = __merge_reference_tree_cache( + lib = get_lib(libpath) + res = lib.merge_reference_tree_cache( cache_dir.encode(), primary_name.encode(), result_name.encode() ) except Exception as e: raise ConfigTreeError(e) if res == 1: - msg = __get_error().decode() + msg = lib.get_error().decode() raise ConfigTreeError(msg) def interface_definitions_to_cache(from_dir, cache_path, libpath=LIBPATH): # pylint: disable=raise-missing-from try: - __lib = cdll.LoadLibrary(libpath) - __interface_definitions_to_cache = __lib.interface_definitions_to_cache - __interface_definitions_to_cache.argtypes = [c_char_p, c_char_p] - __get_error = __lib.get_error - __get_error.argtypes = [] - __get_error.restype = c_char_p - res = __interface_definitions_to_cache(from_dir.encode(), cache_path.encode()) + lib = get_lib(libpath) + res = lib.interface_definitions_to_cache(from_dir.encode(), cache_path.encode()) except Exception as e: raise ConfigTreeError(e) if res == 1: - msg = __get_error().decode() + msg = lib.get_error().decode() raise ConfigTreeError(msg) def reference_tree_cache_to_json(cache_path, render_file, libpath=LIBPATH): # pylint: disable=raise-missing-from try: - __lib = cdll.LoadLibrary(libpath) - __reference_tree_cache_to_json = __lib.reference_tree_cache_to_json - __reference_tree_cache_to_json.argtypes = [c_char_p, c_char_p] - __get_error = __lib.get_error - __get_error.argtypes = [] - __get_error.restype = c_char_p - res = __reference_tree_cache_to_json(cache_path.encode(), render_file.encode()) + lib = get_lib(libpath) + res = lib.reference_tree_cache_to_json(cache_path.encode(), render_file.encode()) except Exception as e: raise ConfigTreeError(e) if res == 1: - msg = __get_error().decode() + msg = lib.get_error().decode() raise ConfigTreeError(msg) @@ -847,19 +755,14 @@ def validate_tree_filter( ): # pylint: disable=raise-missing-from try: - __lib = cdll.LoadLibrary(libpath) - __validate_tree_filter = __lib.validate_tree_filter - __validate_tree_filter.argtypes = [c_void_p, c_char_p, c_char_p] - __get_error = __lib.get_error - __get_error.argtypes = [] - __get_error.restype = c_char_p - res = __validate_tree_filter( + lib = get_lib(libpath) + res = lib.validate_tree_filter( config_tree.get_tree(), cache_path.encode(), validator_dir.encode() ) except Exception as e: raise ConfigTreeError(e) - msg = __get_error().decode() + msg = lib.get_error().decode() tree = ConfigTree(address=res) return tree, msg @@ -896,16 +799,12 @@ class DiffTree: self.left = left self.right = right - self.__lib = cdll.LoadLibrary(libpath) - - self.__diff_tree = self.__lib.diff_tree - self.__diff_tree.argtypes = [c_char_p, c_void_p, c_void_p] - self.__diff_tree.restype = c_void_p + self.__lib = get_lib(libpath) check_path(path) path_str = ' '.join(map(str, path)).encode() - res = self.__diff_tree(path_str, left.get_tree(), right.get_tree()) + res = self.__lib.diff_tree(path_str, left.get_tree(), right.get_tree()) # full diff config_tree and python dict representation self.full = ConfigTree(address=res) diff --git a/python/vyos/referencetree.py b/python/vyos/referencetree.py index b535615dd..c2cb654b4 100644 --- a/python/vyos/referencetree.py +++ b/python/vyos/referencetree.py @@ -13,12 +13,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see . -from ctypes import cdll, c_char_p, c_void_p, c_bool from pathlib import Path from vyos.defaults import reference_tree_cache - -LIBPATH = '/usr/lib/libvyosconfig.so.0' +from vyos.configtree import LIBPATH +from vyos.configtree import get_lib class ReferenceTreeError(Exception): @@ -29,49 +28,26 @@ class ReferenceTree: # pylint: disable=too-many-instance-attributes,raise-missing-from def __init__(self, cache_file=reference_tree_cache, libpath=LIBPATH): self.__pointer = None - self.__lib = cdll.LoadLibrary(libpath) - - # Import functions - self.__get_error = self.__lib.get_error - self.__get_error.argtypes = [] - self.__get_error.restype = c_char_p - - self.__read_internal_string = self.__lib.read_internal_string_reference_tree - self.__read_internal_string.argtypes = [c_char_p] - self.__read_internal_string.restype = c_void_p - - self.__write_internal = self.__lib.write_internal_reference_tree - self.__write_internal.argtypes = [c_void_p, c_char_p] - - self.__to_json = self.__lib.to_json_reference_tree - self.__to_json.argtypes = [c_void_p] - self.__to_json.restype = c_char_p - - self.__destroy = self.__lib.destroy - self.__destroy.argtypes = [c_void_p] - - self.__equal = self.__lib.equal - self.__equal.argtypes = [c_void_p, c_void_p] - self.__equal.restype = c_bool + self.__lib = get_lib(libpath) try: cache_string = Path(cache_file).read_bytes() except OSError as e: raise ValueError(f'Failed to read cache_file: {e}') - pointer = self.__read_internal_string(cache_string) + pointer = self.__lib.read_internal_string_reference_tree(cache_string) if pointer is None: - msg = self.__get_error().decode() + msg = self.__lib.get_error().decode() raise ValueError(f'Failed to read internal rep: {msg}') self.__pointer = pointer def __del__(self): if self.__pointer is not None: - self.__destroy(self.__pointer) + self.__lib.destroy(self.__pointer) def __eq__(self, other): if isinstance(other, ReferenceTree): - return self.__equal(self.get_tree(), other.get_tree()) + return self.__lib.equal(self.get_tree(), other.get_tree()) return False def __str__(self): @@ -81,7 +57,7 @@ class ReferenceTree: return self.__pointer def write_cache(self, file_name): - self.__write_internal(self.get_tree(), file_name.encode()) + self.__lib.write_internal_reference_tree(self.get_tree(), file_name.encode()) def to_json(self): - return self.__to_json(self.__pointer).decode() + return self.__lib.to_json_reference_tree(self.__pointer).decode() -- cgit v1.2.3 From 662ad8e5a93298f8d4fe813bd615cf00d74cf4c4 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sat, 29 Aug 2026 09:29:51 +0000 Subject: vyos: T9261: defer more heavy imports off the CLI hot paths Follow-up sweep to f696b2d65. Every conf-mode script not handled in-daemon by vyos-configd, every op-mode script and every Python validator is a fresh interpreter, so module-scope imports of heavy dependencies are paid over and over - which is what makes the CLI feel sluggish on CPUs with little cache. configsource.py imported vyos.vyconf_session, and with it google.protobuf, for a class that is only instantiated when the vyconf backend is active. opmode.py imported ConfigTreeQuery for a single decorator most op-mode scripts never use. template.py imported jinja2 at module scope although Environment is only touched in the lru_cache'd _get_environment(), while most callers want just the is_ipv4/is_ipv6 predicates. ethtool.py pulled pyroute2 via vyos.netlink for two calls in Ethtool.__init__. ifconfig/interface.py pulled the whole cryptography stack via vyos.pki for set_eapol() alone. Also deferred: netaddr, jmespath, tabulate, hurry.filesize, vyos.ifconfig in qos, and cracklib - the last of which also opens its dictionary database on import. import vyos.opmode 34.0 -> 4.6 ms (-86%), 206 -> 71 modules import vyos.ifconfig 132.9 -> 30.8 ms (-77%), 618 -> 193 modules import vyos.config 40.7 -> 20.6 ms (-49%), 194 -> 130 modules import vyos.qos 134.1 -> 12.5 ms (-91%), 631 -> 124 modules import vyos.template 18.8 -> 6.3 ms (-66%), 153 -> 89 modules import vyos.firewall 22.2 -> 12.2 ms (-45%), 168 -> 114 modules Deferring textwrap in base.py and logging in configtree.py was measured too and made no difference; both were left alone. --- python/vyos/configsource.py | 11 ++++++-- python/vyos/ethtool.py | 8 ++++-- python/vyos/ifconfig/ethernet.py | 6 ++++- python/vyos/ifconfig/interface.py | 53 +++++++++++++++++++++++++------------ python/vyos/ifconfig/operational.py | 3 ++- python/vyos/ifconfig/vrrp.py | 3 ++- python/vyos/ifconfig/wireguard.py | 13 ++++++--- python/vyos/opmode.py | 11 +++++--- python/vyos/qos/base.py | 9 +++++-- python/vyos/template.py | 10 ++++--- python/vyos/utils/auth.py | 13 ++++++--- 11 files changed, 100 insertions(+), 40 deletions(-) (limited to 'python') diff --git a/python/vyos/configsource.py b/python/vyos/configsource.py index b54f6283a..ce9b9a62a 100644 --- a/python/vyos/configsource.py +++ b/python/vyos/configsource.py @@ -21,8 +21,6 @@ from typing import Union from vyos.configtree import ConfigTree from vyos.utils.boot import boot_configuration_complete -from vyos.vyconf_session import VyconfSession -from vyos.vyconf_session import VyconfSessionError from vyos.defaults import directories from vyos.xml_ref import is_tag from vyos.xml_ref import is_leaf @@ -332,6 +330,13 @@ class ConfigSourceVyconfSession(ConfigSource): pid = os.environ.get('SESSION_PID', '') self.pid = int(pid) if pid else os.getppid() + # Imported here rather than at module scope: vyos.vyconf_session pulls + # in vyos.proto.vyconf_client and with it google.protobuf, which every + # consumer of vyos.config would otherwise pay for on import even when + # the vyconf backend is not in use. + from vyos.vyconf_session import VyconfSession + from vyos.vyconf_session import VyconfSessionError + self._vyconf_session = VyconfSession(pid=self.pid) try: out = self._vyconf_session.get_config() @@ -366,6 +371,8 @@ class ConfigSourceVyconfSession(ConfigSource): Returns: True if the config session has uncommitted changes, False otherwise. """ + from vyos.vyconf_session import VyconfSessionError + try: return self._vyconf_session.session_changed() except VyconfSessionError: diff --git a/python/vyos/ethtool.py b/python/vyos/ethtool.py index cbc13a0e9..66dc6bb89 100644 --- a/python/vyos/ethtool.py +++ b/python/vyos/ethtool.py @@ -19,8 +19,6 @@ import contextlib from json import loads from vyos.utils.network import interface_exists from vyos.utils.process import popen -from vyos.netlink import coalesce -from vyos.netlink import timestamp # These drivers do not support using ethtool to change the speed, duplex, or # flow control settings @@ -127,6 +125,12 @@ class Ethtool: if not bool(err): self._channels = out.lower() + # Imported here rather than at module scope: vyos.netlink.* imports + # pyroute2, which is by far the largest single cost on the vyos.ifconfig + # import path. Only code that actually constructs an Ethtool needs it. + from vyos.netlink import coalesce + from vyos.netlink import timestamp + # Get information about NIC coalesce settings with contextlib.suppress(coalesce.CoalesceError, coalesce.GeneralNetlinkError): self._coalesce = coalesce.get_coalesce(ifname) diff --git a/python/vyos/ifconfig/ethernet.py b/python/vyos/ifconfig/ethernet.py index 84c002550..bb5119f77 100644 --- a/python/vyos/ifconfig/ethernet.py +++ b/python/vyos/ifconfig/ethernet.py @@ -19,7 +19,6 @@ from glob import glob from vyos.base import Warning from vyos.ethtool import Ethtool -from vyos.netlink import coalesce from vyos.ifconfig import Section from vyos.ifconfig.interface import Interface from vyos.utils.dict import dict_search @@ -512,6 +511,11 @@ class EthernetIf(Interface): if not params: return None + # Imported here rather than at module scope: vyos.netlink.coalesce + # imports pyroute2, which every consumer of vyos.ifconfig would + # otherwise pay for on import. + from vyos.netlink import coalesce + # Override boolean parameters to true if they exist and supported by NIC driver for boolean_param in coalesce.get_all_params(boolean=True): supported = self.ethtool.check_coalesce(boolean_param) diff --git a/python/vyos/ifconfig/interface.py b/python/vyos/ifconfig/interface.py index e62812d31..3a5efec8c 100644 --- a/python/vyos/ifconfig/interface.py +++ b/python/vyos/ifconfig/interface.py @@ -16,7 +16,6 @@ import os import re import json -import jmespath from copy import deepcopy from glob import glob @@ -26,17 +25,11 @@ from ipaddress import IPv6Interface from netifaces import ifaddresses # pylint: disable = no-name-in-module from socket import AF_INET from socket import AF_INET6 -from netaddr import EUI -from netaddr import mac_unix_expanded from vyos.configdict import list_diff from vyos.configdict import dict_merge from vyos.configdict import get_vlan_ids from vyos.defaults import directories -from vyos.pki import find_chain -from vyos.pki import encode_certificate -from vyos.pki import load_certificate -from vyos.pki import wrap_private_key from vyos.template import is_ipv4 from vyos.template import is_ipv6 from vyos.template import render @@ -68,6 +61,18 @@ from vyos.ifconfig import Section link_local_prefix = 'fe80::/64' +def _jmes_search(expression, data): + """jmespath.search with a deferred import. + + The lookups below live in lambdas evaluated at call time, so importing + jmespath at module scope would put it on the vyos.ifconfig import path for + no reason. + """ + import jmespath + + return jmespath.search(expression, data) + + class Interface(Control): # This is the class which will be used to create # self.operational, it allows subclasses, such as @@ -92,39 +97,39 @@ class Interface(Control): _command_get = { 'admin_state': { 'shellcmd': 'ip -json link show dev {ifname}', - 'format': lambda j: 'up' if 'UP' in jmespath.search('[*].flags | [0]', json.loads(j)) else 'down', + 'format': lambda j: 'up' if 'UP' in _jmes_search('[*].flags | [0]', json.loads(j)) else 'down', }, 'alias': { 'shellcmd': 'ip -json -detail link list dev {ifname}', - 'format': lambda j: jmespath.search('[*].ifalias | [0]', json.loads(j)) or '', + 'format': lambda j: _jmes_search('[*].ifalias | [0]', json.loads(j)) or '', }, 'ifindex': { 'shellcmd': 'ip -json -detail link list dev {ifname}', - 'format': lambda j: jmespath.search('[*].ifindex | [0]', json.loads(j)) or '', + 'format': lambda j: _jmes_search('[*].ifindex | [0]', json.loads(j)) or '', }, 'mac': { 'shellcmd': 'ip -json -detail link list dev {ifname}', - 'format': lambda j: jmespath.search('[*].address | [0]', json.loads(j)), + 'format': lambda j: _jmes_search('[*].address | [0]', json.loads(j)), }, 'min_mtu': { 'shellcmd': 'ip -json -detail link list dev {ifname}', - 'format': lambda j: jmespath.search('[*].min_mtu | [0]', json.loads(j)), + 'format': lambda j: _jmes_search('[*].min_mtu | [0]', json.loads(j)), }, 'max_mtu': { 'shellcmd': 'ip -json -detail link list dev {ifname}', - 'format': lambda j: jmespath.search('[*].max_mtu | [0]', json.loads(j)), + 'format': lambda j: _jmes_search('[*].max_mtu | [0]', json.loads(j)), }, 'mtu': { 'shellcmd': 'ip -json -detail link list dev {ifname}', - 'format': lambda j: jmespath.search('[*].mtu | [0]', json.loads(j)), + 'format': lambda j: _jmes_search('[*].mtu | [0]', json.loads(j)), }, 'oper_state': { 'shellcmd': 'ip -json -detail link list dev {ifname}', - 'format': lambda j: jmespath.search('[*].operstate | [0]', json.loads(j)), + 'format': lambda j: _jmes_search('[*].operstate | [0]', json.loads(j)), }, 'vrf': { 'shellcmd': 'ip -json -detail link list dev {ifname}', - 'format': lambda j: jmespath.search('[?linkinfo.info_slave_kind == `vrf`].master | [0]', json.loads(j)), + 'format': lambda j: _jmes_search('[?linkinfo.info_slave_kind == `vrf`].master | [0]', json.loads(j)), }, } @@ -552,6 +557,11 @@ class Interface(Control): sha.update(cpu_id.encode()) sha.update(first_mac.encode()) sha.update(self.ifname.encode()) + # Imported here rather than at module scope: netaddr is only needed + # by this one method. + from netaddr import EUI + from netaddr import mac_unix_expanded + # take the most significant 48 bits from the SHA256 string tmp = sha.hexdigest()[:12] # Convert pseudo random string into EUI format which now represents a @@ -1709,13 +1719,22 @@ class Interface(Control): def set_eapol(self) -> None: """ Take care about EAPoL supplicant daemon """ - # XXX: wpa_supplicant works on the source interface cfg_dir = '/run/wpa_supplicant' wpa_supplicant_conf = f'{cfg_dir}/{self.ifname}.conf' eapol_action='stop' if 'eapol' in self.config: + # Imported here rather than at module scope: vyos.pki imports the + # whole cryptography stack, which every consumer of vyos.ifconfig + # would otherwise pay for on import. set_eapol() itself runs on + # every ethernet/bond update, so keep it inside this guard too. + # vyos/config.py and vyos/configverify.py defer vyos.pki the same way. + from vyos.pki import find_chain + from vyos.pki import encode_certificate + from vyos.pki import load_certificate + from vyos.pki import wrap_private_key + # The default is a fallback to hw_id which is not present for any interface # other then an ethernet interface. Thus we emulate hw_id by reading back the # Kernel assigned MAC address diff --git a/python/vyos/ifconfig/operational.py b/python/vyos/ifconfig/operational.py index caca40b3d..f513b9157 100644 --- a/python/vyos/ifconfig/operational.py +++ b/python/vyos/ifconfig/operational.py @@ -18,7 +18,6 @@ import os from time import time from datetime import datetime from functools import reduce -from tabulate import tabulate from vyos.ifconfig import Control @@ -169,6 +168,8 @@ class Operational(Control): tabs.append([f'{rtx.upper()}:', ] + [_ for _ in self._stat_names[rtx]]) tabs.append(['', ] + [stats[_] for _ in self._stats_dir[rtx]]) + from tabulate import tabulate + s = tabulate( tabs, stralign="right", diff --git a/python/vyos/ifconfig/vrrp.py b/python/vyos/ifconfig/vrrp.py index 4949fe571..8c0c473d0 100644 --- a/python/vyos/ifconfig/vrrp.py +++ b/python/vyos/ifconfig/vrrp.py @@ -18,7 +18,6 @@ import json import signal from time import time -from tabulate import tabulate from vyos.configquery import ConfigTreeQuery from vyos.utils.convert import seconds_to_human @@ -166,4 +165,6 @@ class VRRP(object): # add to the active list disabled instances groups.extend(cls.disabled()) + from tabulate import tabulate + return tabulate(groups, headers) diff --git a/python/vyos/ifconfig/wireguard.py b/python/vyos/ifconfig/wireguard.py index 4609aff25..8801ff43c 100644 --- a/python/vyos/ifconfig/wireguard.py +++ b/python/vyos/ifconfig/wireguard.py @@ -19,11 +19,7 @@ import time from datetime import timedelta from tempfile import NamedTemporaryFile -from hurry.filesize import size -from hurry.filesize import alternative - from vyos.base import Warning -from vyos.configquery import ConfigTreeQuery from vyos.ifconfig import Interface from vyos.ifconfig import Operational from vyos.template import is_ipv6 @@ -84,6 +80,11 @@ class WireGuardOperational(Operational): return output def show_interface(self): + # Imported here rather than at module scope: display-only helpers should + # not be on the vyos.ifconfig import path. + from hurry.filesize import size + from hurry.filesize import alternative + from vyos.config import Config c = Config() @@ -183,6 +184,10 @@ class WireGuardOperational(Operational): return output def reset_peer(self, peer_name=None, public_key=None): + # Imported here rather than at module scope: vyos.configquery pulls in + # the whole vyos.config tree. + from vyos.configquery import ConfigTreeQuery + c = ConfigTreeQuery() tmp = c.get_config_dict(['interfaces', 'wireguard', self.ifname], effective=True, get_first_key=True, diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index 421054f6e..6c3158ee6 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -17,8 +17,6 @@ import re import sys import typing -from humps import decamelize -from vyos.configquery import ConfigTreeQuery class Error(Exception): """Any error that makes requested operation impossible to complete @@ -319,9 +317,11 @@ def run(module): f'Bare literal is not an acceptable raw output, must be a list or an object.\ The output was:{res}' ) + from humps import decamelize + from json import dumps + res = decamelize(res) res = _normalize_field_names(res) - from json import dumps return dumps(res, indent=4) else: @@ -336,6 +336,11 @@ def verify_cli_exists(cli_path: list, error_msg: str=''): @wraps(func) def _wrapper(*args, **kwargs): + # Imported here rather than at module scope: vyos.configquery pulls + # in the whole vyos.config tree, and most op-mode scripts import + # vyos.opmode without ever using this decorator. + from vyos.configquery import ConfigTreeQuery + config = ConfigTreeQuery() interface = kwargs.get('intf_name') diff --git a/python/vyos/qos/base.py b/python/vyos/qos/base.py index 65aeb552e..651d3f1e1 100644 --- a/python/vyos/qos/base.py +++ b/python/vyos/qos/base.py @@ -14,10 +14,8 @@ # License along with this library. If not, see . import os -import jmespath from vyos.base import Warning -from vyos.ifconfig import Interface from vyos.utils.process import cmdl from vyos.utils.dict import dict_search from vyos.utils.file import read_file @@ -233,6 +231,13 @@ class QoSBase: def update(self, config, direction, priority=None): """ method must be called from derived class after it has completed qdisc setup """ + # Imported here rather than at module scope: pulling vyos.ifconfig in + # at import time is what made vyos.qos one of the most expensive + # modules in the tree. + import jmespath + + from vyos.ifconfig import Interface + if self._debug: import pprint pprint.pprint(config) diff --git a/python/vyos/template.py b/python/vyos/template.py index dbbc4df84..83805a73c 100755 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -16,9 +16,6 @@ import functools import os -from jinja2 import Environment -from jinja2 import FileSystemLoader -from jinja2 import ChainableUndefined from vyos.defaults import directories from vyos.utils.dict import dict_search_args from vyos.utils.file import makedir @@ -41,6 +38,13 @@ _CLEVER_FUNCTIONS = {} # reuse Environments with identical settings to improve performance @functools.lru_cache(maxsize=2) def _get_environment(location=None): + # Imported here rather than at module scope: most importers of this module + # want only the pure-Python predicates (is_ipv4, is_ipv6, ...) and never + # render a template, so they should not pay for jinja2. This function is + # lru_cache'd, so the import happens at most once per process. + from jinja2 import Environment + from jinja2 import FileSystemLoader + from jinja2 import ChainableUndefined from os import getenv if location is None: diff --git a/python/vyos/utils/auth.py b/python/vyos/utils/auth.py index e85dc9984..3950de649 100644 --- a/python/vyos/utils/auth.py +++ b/python/vyos/utils/auth.py @@ -13,7 +13,6 @@ # You should have received a copy of the GNU Lesser General Public License along with this library; # if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import cracklib import math import re import string @@ -76,15 +75,21 @@ def calculate_entropy(charset: str, passwd: str) -> float: def evaluate_strength(passwd: str) -> dict[str, str]: """ Evaluates password strength and returns a check result dict """ - charset = (cracklib.ASCII_UPPERCASE + cracklib.ASCII_LOWERCASE + - string.punctuation + string.digits) - result = { 'strength': '', 'error': '', } try: + # Imported here rather than at module scope: importing cracklib opens + # its dictionary database, so this module stays importable without it. + # Kept inside the try so that a missing or broken package is reported + # as EPasswdStrength.ERROR instead of escaping this function. + import cracklib + + charset = (cracklib.ASCII_UPPERCASE + cracklib.ASCII_LOWERCASE + + string.punctuation + string.digits) + cracklib.FascistCheck(passwd) except ValueError as e: # The password is vulnerable to dictionary attack no matter the entropy -- cgit v1.2.3