Merge pull request #3404 from jestabro/sagitta-config-sync-backport

config-sync: T6185: T6146: combined backport of config-sync extensions and priority data

3 files changed, 105 insertions, 41 deletions

diff --git a/src/helpers/priority.py b/src/helpers/priority.py
new file mode 100755
index 000000000..04186104c
--- /dev/null
+++ b/src/helpers/priority.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2024 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import sys
+from argparse import ArgumentParser
+from tabulate import tabulate
+
+from vyos.priority import get_priority_data
+
+if __name__ == '__main__':
+    parser = ArgumentParser()
+    parser.add_argument('--legacy-format', action='store_true',
+                        help="format output for comparison with legacy 'priority.pl'")
+    args = parser.parse_args()
+
+    prio_list = get_priority_data()
+    if args.legacy_format:
+        for p in prio_list:
+            print(f'{p[2]} {"/".join(p[0])}')
+        sys.exit(0)
+
+    l = []
+    for p in prio_list:
+        l.append((p[2], p[1], p[0]))
+    headers = ['priority', 'owner', 'path']
+    out = tabulate(l, headers, numalign='right')
+    print(out)
diff --git a/src/helpers/vyos_config_sync.py b/src/helpers/vyos_config_sync.py
index 7eec3f4f3..0604b2837 100755
--- a/src/helpers/vyos_config_sync.py
+++ b/src/helpers/vyos_config_sync.py
@@ -21,9 +21,11 @@ import json
 import requests
 import urllib3
 import logging
-from typing import Optional, List, Dict, Any
+from typing import Optional, List, Tuple, Dict, Any
 
 from vyos.config import Config
+from vyos.configtree import ConfigTree
+from vyos.configtree import mask_inclusive
 from vyos.template import bracketize_ipv6
 
 
@@ -61,39 +63,45 @@ def post_request(url: str,
 
 
 
-def retrieve_config(section: Optional[List[str]] = None) -> Optional[Dict[str, Any]]:
+def retrieve_config(sections: List[list[str]]) -> Tuple[Dict[str, Any], Dict[str, Any]]:
     """Retrieves the configuration from the local server.
 
     Args:
-        section: List[str]: The section of the configuration to retrieve.
-        Default is None.
+        sections: List[list[str]]: The list of sections of the configuration
+        to retrieve, given as list of paths.
 
     Returns:
-        Optional[Dict[str, Any]]: The retrieved configuration as a
-        dictionary, or None if an error occurred.
+        Tuple[Dict[str, Any],Dict[str,Any]]: The tuple (mask, config) where:
+            - mask: The tree of paths of sections, as a dictionary.
+            - config: The subtree of masked config data, as a dictionary.
     """
-    if section is None:
-        section = []
 
-    conf = Config()
-    config = conf.get_config_dict(section, get_first_key=True)
-    if config:
-        return config
-    return None
+    mask = ConfigTree('')
+    for section in sections:
+        mask.set(section)
+    mask_dict = json.loads(mask.to_json())
+
+    config = Config()
+    config_tree = config.get_config_tree()
+    masked = mask_inclusive(config_tree, mask)
+    config_dict = json.loads(masked.to_json())
 
+    return mask_dict, config_dict
 
 def set_remote_config(
         address: str,
         key: str,
-        commands: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+        op: str,
+        mask: Dict[str, Any],
+        config: Dict[str, Any]) -> Optional[Dict[str, Any]]:
     """Loads the VyOS configuration in JSON format to a remote host.
 
     Args:
         address (str): The address of the remote host.
         key (str): The key to use for loading the configuration.
-        commands (list): List of set/load commands for request, given as:
-                         [{'op': str, 'path': list[str], 'section': dict},
-                         ...]
+        op (str): The operation to perform (set or load).
+        mask (dict): The dict of paths in sections.
+        config (dict): The dict of masked config data.
 
     Returns:
         Optional[Dict[str, Any]]: The response from the remote host as a
@@ -107,7 +115,9 @@ def set_remote_config(
 
     url = f'https://{address}/configure-section'
     data = json.dumps({
-        'commands': commands,
+        'op': op,
+        'mask': mask,
+        'config': config,
         'key': key
     })
 
@@ -140,23 +150,15 @@ def config_sync(secondary_address: str,
     )
 
     # Sync sections ("nat", "firewall", etc)
-    commands = []
-    for section in sections:
-        config_json = retrieve_config(section=section)
-        # Check if config path deesn't exist, for example "set nat"
-        # we set empty value for config_json data
-        # As we cannot send to the remote host section "nat None" config
-        if not config_json:
-            config_json = {}
-        logger.debug(
-            f"Retrieved config for section '{section}': {config_json}")
-
-        d = {'op': mode, 'path': section, 'section': config_json}
-        commands.append(d)
+    mask_dict, config_dict = retrieve_config(sections)
+    logger.debug(
+        f"Retrieved config for sections '{sections}': {config_dict}")
 
     set_config = set_remote_config(address=secondary_address,
                                    key=secondary_key,
-                                   commands=commands)
+                                   op=mode,
+                                   mask=mask_dict,
+                                   config=config_dict)
 
     logger.debug(f"Set config for sections '{sections}': {set_config}")
 
diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server
index 77870a84c..ecbf6fcf9 100755
--- a/src/services/vyos-http-api-server
+++ b/src/services/vyos-http-api-server
@@ -140,6 +140,14 @@ class ConfigSectionModel(ApiModel, BaseConfigSectionModel):
 class ConfigSectionListModel(ApiModel):
     commands: List[BaseConfigSectionModel]
 
+class BaseConfigSectionTreeModel(BaseModel):
+    op: StrictStr
+    mask: Dict
+    config: Dict
+
+class ConfigSectionTreeModel(ApiModel, BaseConfigSectionTreeModel):
+    pass
+
 class RetrieveModel(ApiModel):
     op: StrictStr
     path: List[StrictStr]
@@ -374,7 +382,7 @@ class MultipartRequest(Request):
                         self.form_err = (400,
                         f"Malformed command '{c}': missing 'op' field")
                     if endpoint not in ('/config-file', '/container-image',
-                                        '/image'):
+                                        '/image', '/configure-section'):
                         if 'path' not in c:
                             self.form_err = (400,
                             f"Malformed command '{c}': missing 'path' field")
@@ -392,12 +400,9 @@ class MultipartRequest(Request):
                             self.form_err = (400,
                             f"Malformed command '{c}': 'value' field must be a string")
                     if endpoint in ('/configure-section'):
-                        if 'section' not in c:
-                            self.form_err = (400,
-                            f"Malformed command '{c}': missing 'section' field")
-                        elif not isinstance(c['section'], dict):
+                        if 'section' not in c and 'config' not in c:
                             self.form_err = (400,
-                            f"Malformed command '{c}': 'section' field must be JSON of dict")
+                            f"Malformed command '{c}': missing 'section' or 'config' field")
 
                 if 'key' not in forms and 'key' not in merge:
                     self.form_err = (401, "Valid API key is required")
@@ -455,7 +460,8 @@ def call_commit(s: ConfigSession):
             logger.warning(f"ConfigSessionError: {e}")
 
 def _configure_op(data: Union[ConfigureModel, ConfigureListModel,
-                              ConfigSectionModel, ConfigSectionListModel],
+                              ConfigSectionModel, ConfigSectionListModel,
+                              ConfigSectionTreeModel],
                   request: Request, background_tasks: BackgroundTasks):
     session = app.state.vyos_session
     env = session.get_session_env()
@@ -481,7 +487,8 @@ def _configure_op(data: Union[ConfigureModel, ConfigureListModel,
     try:
         for c in data:
             op = c.op
-            path = c.path
+            if not isinstance(c, BaseConfigSectionTreeModel):
+                path = c.path
 
             if isinstance(c, BaseConfigureModel):
                 if c.value:
@@ -495,6 +502,10 @@ def _configure_op(data: Union[ConfigureModel, ConfigureListModel,
             elif isinstance(c, BaseConfigSectionModel):
                 section = c.section
 
+            elif isinstance(c, BaseConfigSectionTreeModel):
+                mask = c.mask
+                config = c.config
+
             if isinstance(c, BaseConfigureModel):
                 if op == 'set':
                     session.set(path, value=value)
@@ -514,6 +525,14 @@ def _configure_op(data: Union[ConfigureModel, ConfigureListModel,
                     session.load_section(path, section)
                 else:
                     raise ConfigSessionError(f"'{op}' is not a valid operation")
+
+            elif isinstance(c, BaseConfigSectionTreeModel):
+                if op == 'set':
+                    session.set_section_tree(config)
+                elif op == 'load':
+                    session.load_section_tree(mask, config)
+                else:
+                    raise ConfigSessionError(f"'{op}' is not a valid operation")
         # end for
         config = Config(session_env=env)
         d = get_config_diff(config)
@@ -554,7 +573,8 @@ def configure_op(data: Union[ConfigureModel,
 
 @app.post('/configure-section')
 def configure_section_op(data: Union[ConfigSectionModel,
-                                     ConfigSectionListModel],
+                                     ConfigSectionListModel,
+                                     ConfigSectionTreeModel],
                                request: Request, background_tasks: BackgroundTasks):
     return _configure_op(data, request, background_tasks)