Merge pull request #3373 from jestabro/sagitta-configdep-redundancy

T5660: Remove redundant calls to config dependency scripts

3 files changed, 80 insertions, 11 deletions

diff --git a/python/vyos/configdep.py b/python/vyos/configdep.py
index 8a28811eb..73bd9ea96 100644
--- a/python/vyos/configdep.py
+++ b/python/vyos/configdep.py
@@ -1,4 +1,4 @@
-# Copyright 2023 VyOS maintainers and contributors <maintainers@vyos.io>
+# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -33,7 +33,14 @@ if typing.TYPE_CHECKING:
 dependency_dir = os.path.join(directories['data'],
                               'config-mode-dependencies')
 
-dependent_func: dict[str, list[typing.Callable]] = {}
+local_dependent_func: dict[str, list[typing.Callable]] = {}
+
+DEBUG = False
+FORCE_LOCAL = False
+
+def debug_print(s: str):
+    if DEBUG:
+        print(s)
 
 def canon_name(name: str) -> str:
     return os.path.splitext(name)[0].replace('-', '_')
@@ -45,6 +52,26 @@ def canon_name_of_path(path: str) -> str:
 def caller_name() -> str:
     return stack()[2].filename
 
+def name_of(f: typing.Callable) -> str:
+    return f.__name__
+
+def names_of(l: list[typing.Callable]) -> list[str]:
+    return [name_of(f) for f in l]
+
+def remove_redundant(l: list[typing.Callable]) -> list[typing.Callable]:
+    names = set()
+    for e in reversed(l):
+        _ = l.remove(e) if name_of(e) in names else names.add(name_of(e))
+
+def append_uniq(l: list[typing.Callable], e: typing.Callable):
+    """Append an element, removing earlier occurrences
+
+    The list of dependencies is generally short and traversing the list on
+    each append is preferable to the cost of redundant script invocation.
+    """
+    l.append(e)
+    remove_redundant(l)
+
 def read_dependency_dict(dependency_dir: str = dependency_dir) -> dict:
     res = {}
     for dep_file in os.listdir(dependency_dir):
@@ -95,18 +122,39 @@ def set_dependents(case: str, config: 'Config',
                    tagnode: typing.Optional[str] = None):
     d = get_dependency_dict(config)
     k = canon_name_of_path(caller_name())
-    l = dependent_func.setdefault(k, [])
+    tag_ext = f'_{tagnode}' if tagnode is not None else ''
+    if hasattr(config, 'dependent_func') and not FORCE_LOCAL:
+        dependent_func = getattr(config, 'dependent_func')
+        l = dependent_func.setdefault('vyos_configd', [])
+    else:
+        dependent_func = local_dependent_func
+        l = dependent_func.setdefault(k, [])
     for target in d[k][case]:
         func = def_closure(target, config, tagnode)
-        l.append(func)
+        func.__name__ = f'{target}{tag_ext}'
+        append_uniq(l, func)
+    debug_print(f'set_dependents: caller {k}, dependents {names_of(l)}')
 
-def call_dependents():
+def call_dependents(dependent_func: dict = None):
     k = canon_name_of_path(caller_name())
-    l = dependent_func.get(k, [])
+    if dependent_func is None or FORCE_LOCAL:
+        dependent_func = local_dependent_func
+        l = dependent_func.get(k, [])
+    else:
+        l = dependent_func.get('vyos_configd', [])
+    debug_print(f'call_dependents: caller {k}, dependents {names_of(l)}')
     while l:
         f = l.pop(0)
+        debug_print(f'calling: {f.__name__}')
         f()
 
+def called_as_dependent() -> bool:
+    st = stack()[1:]
+    for f in st:
+        if f.filename == __file__:
+            return True
+    return False
+
 def graph_from_dependency_dict(d: dict) -> dict:
     g = {}
     for k in list(d):
diff --git a/src/services/vyos-configd b/src/services/vyos-configd
index 355182b26..c89c486e5 100755
--- a/src/services/vyos-configd
+++ b/src/services/vyos-configd
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2020-2023 VyOS maintainers and contributors
+# Copyright (C) 2020-2024 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -19,6 +19,7 @@ import sys
 import grp
 import re
 import json
+import typing
 import logging
 import signal
 import importlib.util
@@ -29,6 +30,7 @@ from vyos.defaults import directories
 from vyos.utils.boot import boot_configuration_complete
 from vyos.configsource import ConfigSourceString
 from vyos.configsource import ConfigSourceError
+from vyos.configdep import call_dependents
 from vyos.config import Config
 from vyos import ConfigError
 
@@ -198,10 +200,12 @@ def initialization(socket):
         return None
 
     config = Config(config_source=configsource)
+    dependent_func: dict[str, list[typing.Callable]] = {}
+    setattr(config, 'dependent_func', dependent_func)
 
     return config
 
-def process_node_data(config, data) -> int:
+def process_node_data(config, data, last: bool = False) -> int:
     if not config:
         logger.critical(f"Empty config")
         return R_ERROR_DAEMON
@@ -223,11 +227,18 @@ def process_node_data(config, data) -> int:
     args.insert(0, f'{script_name}.py')
 
     if script_name not in include_set:
+        # call dependents now if last element of prio queue is run
+        # independent of configd
+        if last:
+            call_dependents(dependent_func=config.dependent_func)
         return R_PASS
 
     with stdout_redirected(session_out, session_mode):
         result = run_script(conf_mode_scripts[script_name], config, args)
 
+    if last and result == R_SUCCESS:
+        call_dependents(dependent_func=config.dependent_func)
+
     return result
 
 def remove_if_file(f: str):
@@ -281,7 +292,9 @@ if __name__ == '__main__':
             socket.send(resp.encode())
             config = initialization(socket)
         elif message["type"] == "node":
-            res = process_node_data(config, message["data"])
+            if message["last"]:
+                logger.debug(f'final element of priority queue')
+            res = process_node_data(config, message["data"], message["last"])
             response = res.to_bytes(1, byteorder=sys.byteorder)
             logger.debug(f"Sending response {res}")
             socket.send(response)
diff --git a/src/shim/vyshim.c b/src/shim/vyshim.c
index cae8b6152..41723e7a4 100644
--- a/src/shim/vyshim.c
+++ b/src/shim/vyshim.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020 VyOS maintainers and contributors
+ * Copyright (C) 2020-2024 VyOS maintainers and contributors
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 or later as
@@ -49,6 +49,7 @@
 #define GET_SESSION "cli-shell-api --show-working-only --show-show-defaults --show-ignore-edit showConfig"
 
 #define COMMIT_MARKER "/var/tmp/initial_in_commit"
+#define QUEUE_MARKER "/var/tmp/last_in_queue"
 
 enum {
     SUCCESS =      1 << 0,
@@ -77,6 +78,7 @@ int main(int argc, char* argv[])
 
     int ex_index;
     int init_timeout = 0;
+    int last = 0;
 
     debug_print("Connecting to vyos-configd ...\n");
     zmq_connect(requester, SOCKET_PATH);
@@ -101,10 +103,16 @@ int main(int argc, char* argv[])
         return ret;
     }
 
+    if (access(QUEUE_MARKER, F_OK) != -1) {
+        last = 1;
+        remove(QUEUE_MARKER);
+    }
+
     char error_code[1];
     debug_print("Sending node data ...\n");
-    char *string_node_data_msg = mkjson(MKJSON_OBJ, 2,
+    char *string_node_data_msg = mkjson(MKJSON_OBJ, 3,
                                         MKJSON_STRING, "type", "node",
+                                        MKJSON_BOOL, "last", last,
                                         MKJSON_STRING, "data", &string_node_data[0]);
 
     zmq_send(requester, string_node_data_msg, strlen(string_node_data_msg), 0);