Merge pull request #5271 from jestabro/config-sync-diff-exclude

T8980: update config sync diff for exclude mask

5 files changed, 152 insertions, 88 deletions

diff --git a/python/vyos/config_mgmt.py b/python/vyos/config_mgmt.py
index b3efdeac9..06cfb3d78 100644
--- a/python/vyos/config_mgmt.py
+++ b/python/vyos/config_mgmt.py
@@ -38,7 +38,6 @@ from vyos.configtree import ConfigTreeError
 from vyos.configsession import ConfigSession
 from vyos.configsession import ConfigSessionError
 from vyos.configtree import diff_compare
-from vyos.configtree import DiffTree
 from vyos.load_config import load
 from vyos.load_config import LoadConfigError
 from vyos.defaults import directories
@@ -452,80 +451,6 @@ Proceed ?"""
 
         return self.compare(commands=cmnds, rev1=r1, rev2=r2)
 
-    def _format_remote_diff(self, diff_tree: DiffTree, path: list, commands: bool):
-        add_tree = diff_tree.add
-        del_tree = diff_tree.delete
-        command_prefix = ' '.join(path)
-
-        result_lines = []
-        if commands:
-            # Process the deleted elements into command format and filter based on prefix (path)
-            for line in del_tree.to_commands(op='delete').splitlines():
-                if line.startswith(f'delete {command_prefix}'):
-                    result_lines.append(line)
-
-            # Process the added elements into command format and filter based on prefix (path)
-            for line in add_tree.to_commands(op='set').splitlines():
-                if line.startswith(f'set {command_prefix}'):
-                    result_lines.append(line)
-        else:
-            with_node = len(path) > 1
-            # Retrieve subtrees for the specified path from both added and deleted trees
-            del_tree = del_tree.get_subtree(path, with_node=with_node)
-            add_tree = add_tree.get_subtree(path, with_node=with_node)
-
-            # Convert the subtrees to string lines for further processing
-            del_tree_lines = str(del_tree).splitlines()
-            add_tree_lines = str(add_tree).splitlines()
-
-            # Format the lines with a prefix ('-', '+') and filter out empty lines
-            del_lines = [f'- {l}' for l in del_tree_lines if l.strip()]
-            add_lines = [f'+ {l}' for l in add_tree_lines if l.strip()]
-
-            if del_lines or add_lines:
-                # Adjust command prefix if a node is present in the path
-                command_prefix = ' '.join(path[:-1]) if with_node else command_prefix
-                if command_prefix:
-                    result_lines.append(f'[{command_prefix}]')
-
-                # Combine both deleted and added lines and process them
-                result_lines.extend(del_lines + add_lines)
-
-        # Join the result lines into a single string, excluding empty lines
-        return '\n'.join((line for line in result_lines if line.strip()))
-
-    def remote_compare(
-        self,
-        source: str,
-        remote_tree: ConfigTree,
-        path: Optional[list] = None,
-        commands: bool = False,
-    ) -> str:
-        """
-        Compares a local configuration tree with a remote
-        configuration tree based on the specified source ('running', 'candidate', 'saved').
-        """
-        path = path or []
-
-        # Determine the correct local configuration tree based on the 'source' parameter
-        if source == 'running':
-            local_tree = self.active_config
-        elif source == 'candidate':
-            local_tree = self.working_config
-        elif source == 'saved':
-            local_tree = self._get_saved_config_tree()
-        else:
-            raise ConfigMgmtError(
-                'Invalid source, must be one of: running, candidate, saved'
-            )
-
-        try:
-            diff_tree = DiffTree(remote_tree, local_tree)
-        except ConfigTreeError as e:
-            raise ConfigMgmtError(e) from e
-
-        return self._format_remote_diff(diff_tree, path, commands)
-
     # Initialization and post-commit hooks for conf-mode
     #
     def initialize_revision(self):
diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py
index 9c5d69ad1..24b3252fd 100644
--- a/python/vyos/configtree.py
+++ b/python/vyos/configtree.py
@@ -19,6 +19,8 @@ import logging
 
 from ctypes import cdll, c_char_p, c_void_p, c_int, c_bool
 from typing import TYPE_CHECKING
+from pathlib import Path
+
 
 # https://peps.python.org/pep-0484/#forward-references
 # for type 'ConfigDict'
@@ -265,6 +267,19 @@ class ConfigTree:
         if self.__migration:
             self.migration_log = logging.getLogger('vyos.migrate')
 
+    @classmethod
+    def load_file(cls, location):
+        # pylint: disable=raise-missing-from
+        config_file = Path(location)
+        if not config_file.exists():
+            raise ConfigTreeError(f'Missing config file at {location}')
+        try:
+            config_tree = cls(config_file.read_text())
+        except (OSError, ValueError, TypeError) as e:
+            raise ConfigTreeError(f'Corrupted config file at {location}: {e}')
+
+        return config_tree
+
     def __del__(self):
         if self.__config is not None:
             self.__destroy(self.__config)
diff --git a/python/vyos/derivedtree.py b/python/vyos/derivedtree.py
index dc46b8eb3..61e1a4d94 100644
--- a/python/vyos/derivedtree.py
+++ b/python/vyos/derivedtree.py
@@ -18,6 +18,7 @@ from vyos.referencetree import ReferenceTree
 from vyos.configtree import ConfigTree
 from vyos.configtree import ConfigTreeError
 from vyos.configtree import subtree_from_partial
+from vyos.configtree import mask_exclusive
 
 
 class DerivedTreeError(Exception):
@@ -29,7 +30,7 @@ def subtree_from_list_of_partial_paths(
     paths: list[list[str]],
     accumulator: ConfigTree = None,
     reference_tree: ReferenceTree = None,
-):
+) -> ConfigTree:
     """Return the union of subtrees of the ConfigTree argument matching each
     of the 'partial' paths. A partial path is one that may or may not
     contain intervening tag node values, in which case it will match for all
@@ -61,3 +62,15 @@ def subtree_from_list_of_partial_paths(
         raise DerivedTreeError(f'Nonsensical paths: {errors}')
 
     return accumulator
+
+
+def apply_exclusion_list(
+    config_tree: ConfigTree, exclusion_list: list[list[str]]
+) -> ConfigTree:
+    mask_ex = subtree_from_list_of_partial_paths(config_tree, exclusion_list)
+    try:
+        masked = mask_exclusive(config_tree, mask_ex)
+    except ConfigTreeError as e:
+        raise DerivedTreeError(str(e)) from e
+
+    return masked
diff --git a/python/vyos/utils/config.py b/python/vyos/utils/config.py
index 1032e22dc..004d64bbd 100644
--- a/python/vyos/utils/config.py
+++ b/python/vyos/utils/config.py
@@ -75,6 +75,18 @@ def write_saved_value(path: list, value=None, config_path: str=config_file):
 
     write_file(config_path, ct.to_string())
 
+
+def get_saved_config_tree() -> 'ConfigTree':
+    # pylint: disable=import-outside-toplevel
+    """Return config tree of saved config.
+
+    Raises ConfigTreeError.
+    """
+    from vyos.configtree import ConfigTree
+
+    return ConfigTree.load_file(config_file)
+
+
 def flag(l: list) -> list:
     res = [l[0:i] for i,_ in enumerate(l, start=1)]
     return res
diff --git a/src/op_mode/config_sync.py b/src/op_mode/config_sync.py
index d6eff7cd2..8d473cc70 100644
--- a/src/op_mode/config_sync.py
+++ b/src/op_mode/config_sync.py
@@ -23,10 +23,14 @@ from vyos import http_api_client as http
 from vyos.utils.file import read_json
 from vyos.utils.dict import dict_to_paths
 from vyos.utils.list import list_contains_sublist
+from vyos.utils.config import get_saved_config_tree
 from vyos.configtree import ConfigTree
 from vyos.configtree import ConfigTreeError
-from vyos.config_mgmt import ConfigMgmt
-from vyos.config_mgmt import ConfigMgmtError
+from vyos.configtree import DiffTree
+from vyos.config import Config
+from vyos.derivedtree import apply_exclusion_list
+from vyos.derivedtree import DerivedTreeError
+from vyos.defaults import config_sync_exclusion_list
 
 CONFIG_FILE = Path('/run/config_sync_conf.conf')
 
@@ -60,6 +64,7 @@ def _load_config_sync_sections() -> list:
 
 
 def _load_config_sync_settings() -> dict:
+    # pylint: disable=use-dict-literal
     """Load remote API settings from config-sync service runtime JSON file"""
 
     cfg = _read_json_config()
@@ -82,9 +87,12 @@ class ConfigSyncDiffManager:
         api_settings = _load_config_sync_settings()
         self._client = http.ApiClient(http.ApiClientConfig(**api_settings))
 
-        self._config_mgmt = ConfigMgmt()
+        self._config = Config()
+        self.running_config = self._config.get_config_tree(effective=True)
+        self.session_config = self._config.get_config_tree(effective=False)
 
     def _get_remote_config_tree(self, section_path: list = None) -> ConfigTree:
+        # pylint: disable=redefined-outer-name
         """
         Retrieve remote config (or subtree) as ConfigTree via HTTPS API.
 
@@ -116,6 +124,100 @@ class ConfigSyncDiffManager:
         except ConfigTreeError as e:
             raise opmode.InternalError(f'Unable to build remote ConfigTree: {e}') from e
 
+    def _get_exclude_list(self) -> list[list[str]]:
+        # add as instance method, for future access to CLI settings from self.Config
+        exclude_file = Path(config_sync_exclusion_list)
+        if not exclude_file.exists():
+            raise opmode.InternalError('Config-sync exclusion list file not available')
+
+        return read_json(exclude_file, defaultonfailure=[])
+
+    def _format_remote_diff(self, diff_tree: DiffTree, path: list, commands: bool):
+        add_tree = diff_tree.add
+        del_tree = diff_tree.delete
+        command_prefix = ' '.join(path)
+
+        result_lines = []
+        if commands:
+            # Process the deleted elements into command format and filter based on prefix (path)
+            for line in del_tree.to_commands(op='delete').splitlines():
+                if line.startswith(f'delete {command_prefix}'):
+                    result_lines.append(line)
+
+            # Process the added elements into command format and filter based on prefix (path)
+            for line in add_tree.to_commands(op='set').splitlines():
+                if line.startswith(f'set {command_prefix}'):
+                    result_lines.append(line)
+        else:
+            with_node = len(path) > 1
+            # Retrieve subtrees for the specified path from both added and deleted trees
+            del_tree = del_tree.get_subtree(path, with_node=with_node)
+            add_tree = add_tree.get_subtree(path, with_node=with_node)
+
+            # Convert the subtrees to string lines for further processing
+            del_tree_lines = str(del_tree).splitlines()
+            add_tree_lines = str(add_tree).splitlines()
+
+            # Format the lines with a prefix ('-', '+') and filter out empty lines
+            del_lines = [f'- {l}' for l in del_tree_lines if l.strip()]  # noqa: E741
+            add_lines = [f'+ {l}' for l in add_tree_lines if l.strip()]  # noqa: E741
+
+            if del_lines or add_lines:
+                # Adjust command prefix if a node is present in the path
+                command_prefix = ' '.join(path[:-1]) if with_node else command_prefix
+                if command_prefix:
+                    result_lines.append(f'[{command_prefix}]')
+
+                # Combine both deleted and added lines and process them
+                result_lines.extend(del_lines + add_lines)
+
+        # Join the result lines into a single string, excluding empty lines
+        return '\n'.join((line for line in result_lines if line.strip()))
+
+    def remote_compare(
+        self,
+        source: str,
+        remote_tree: ConfigTree,
+        path: typing.Optional[list] = None,
+        commands: bool = False,
+    ) -> str:
+        # pylint: disable=redefined-outer-name
+        """
+        Compares a local configuration tree with a remote
+        configuration tree based on the specified source ('running', 'candidate', 'saved').
+        """
+        path = path or []
+
+        # Determine the correct local configuration tree based on the 'source' parameter
+        if source == 'running':
+            local_tree = self.running_config
+        elif source == 'candidate':
+            local_tree = self.session_config
+        elif source == 'saved':
+            try:
+                local_tree = get_saved_config_tree()
+            except ConfigTreeError as e:
+                raise opmode.InternalError(str(e)) from e
+        else:
+            raise opmode.IncorrectValue(
+                'Invalid source, must be one of: running, candidate, saved'
+            )
+
+        exclude_list = self._get_exclude_list()
+
+        try:
+            masked_local = apply_exclusion_list(local_tree, exclude_list)
+            masked_remote = apply_exclusion_list(remote_tree, exclude_list)
+        except DerivedTreeError as e:
+            raise opmode.InternalError(str(e)) from e
+
+        try:
+            diff_tree = DiffTree(masked_remote, masked_local)
+        except ConfigTreeError as e:
+            raise opmode.InternalError(str(e)) from e
+
+        return self._format_remote_diff(diff_tree, path, commands)
+
     def get_sync_diff(
         self,
         source: str,
@@ -127,15 +229,12 @@ class ConfigSyncDiffManager:
         results = []
         remote_tree = self._get_remote_config_tree()
         for section_path in sections:
-            try:
-                result = self._config_mgmt.remote_compare(
-                    source,
-                    remote_tree,
-                    path=section_path,
-                    commands=commands,
-                )
-            except ConfigMgmtError as e:
-                raise opmode.InternalError(str(e)) from e
+            result = self.remote_compare(
+                source,
+                remote_tree,
+                path=section_path,
+                commands=commands,
+            )
 
             result = result.strip()
             if result: