From b83b9c4492f8af8228363d2dad2e748d621e4da7 Mon Sep 17 00:00:00 2001
From: John Estabrook <jestabro@vyos.io>
Date: Wed, 20 Mar 2024 10:54:23 -0500
Subject: xml: T6146: add utils and helper to provide priority data
(cherry picked from commit e915900bfec8d24276afb73599c94ab93f3c24ee)
---
python/vyos/priority.py | 75 +++++++++++++++++++++++++++++++++++
python/vyos/xml_ref/generate_cache.py | 3 +-
src/helpers/priority.py | 42 ++++++++++++++++++++
3 files changed, 119 insertions(+), 1 deletion(-)
create mode 100644 python/vyos/priority.py
create mode 100755 src/helpers/priority.py
diff --git a/python/vyos/priority.py b/python/vyos/priority.py
new file mode 100644
index 000000000..ab4e6d411
--- /dev/null
+++ b/python/vyos/priority.py
@@ -0,0 +1,75 @@
+# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+from pathlib import Path
+from typing import List
+
+from vyos.xml_ref import load_reference
+from vyos.base import Warning as Warn
+
+def priority_data(d: dict) -> list:
+ def func(d, path, res, hier):
+ for k,v in d.items():
+ if not 'node_data' in v:
+ continue
+ subpath = path + [k]
+ hier_prio = hier
+ data = v.get('node_data')
+ o = data.get('owner')
+ p = data.get('priority')
+ # a few interface-definitions have priority preceding owner
+ # attribute, instead of within properties; pass in descent
+ if p is not None and o is None:
+ hier_prio = p
+ if o is not None and p is None:
+ p = hier_prio
+ if o is not None and p is not None:
+ o = Path(o.split()[0]).name
+ p = int(p)
+ res.append((subpath, o, p))
+ if isinstance(v, dict):
+ func(v, subpath, res, hier_prio)
+ return res
+ ret = func(d, [], [], 0)
+ ret = sorted(ret, key=lambda x: x[0])
+ ret = sorted(ret, key=lambda x: x[2])
+ return ret
+
+def get_priority_data() -> list:
+ xml = load_reference()
+ return priority_data(xml.ref)
+
+def priority_sort(sections: List[list[str]] = None,
+ owners: List[str] = None,
+ reverse=False) -> List:
+ if sections is not None:
+ index = 0
+ collection: List = sections
+ elif owners is not None:
+ index = 1
+ collection = owners
+ else:
+ raise ValueError('one of sections or owners is required')
+
+ l = get_priority_data()
+ m = [item for item in l if item[index] in collection]
+ n = sorted(m, key=lambda x: x[2], reverse=reverse)
+ o = [item[index] for item in n]
+ # sections are unhashable; use comprehension
+ missed = [j for j in collection if j not in o]
+ if missed:
+ Warn(f'No priority available for elements {missed}')
+
+ return o
diff --git a/python/vyos/xml_ref/generate_cache.py b/python/vyos/xml_ref/generate_cache.py
index 892c9defb..5f3f84dee 100755
--- a/python/vyos/xml_ref/generate_cache.py
+++ b/python/vyos/xml_ref/generate_cache.py
@@ -33,7 +33,8 @@ xml_tmp = join('/tmp', xml_cache_json)
pkg_cache = abspath(join(_here, 'pkg_cache'))
ref_cache = abspath(join(_here, 'cache.py'))
-node_data_fields = ("node_type", "multi", "valueless", "default_value")
+node_data_fields = ("node_type", "multi", "valueless", "default_value",
+ "owner", "priority")
def trim_node_data(cache: dict):
for k in list(cache):
diff --git a/src/helpers/priority.py b/src/helpers/priority.py
new file mode 100755
index 000000000..04186104c
--- /dev/null
+++ b/src/helpers/priority.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2024 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import sys
+from argparse import ArgumentParser
+from tabulate import tabulate
+
+from vyos.priority import get_priority_data
+
+if __name__ == '__main__':
+ parser = ArgumentParser()
+ parser.add_argument('--legacy-format', action='store_true',
+ help="format output for comparison with legacy 'priority.pl'")
+ args = parser.parse_args()
+
+ prio_list = get_priority_data()
+ if args.legacy_format:
+ for p in prio_list:
+ print(f'{p[2]} {"/".join(p[0])}')
+ sys.exit(0)
+
+ l = []
+ for p in prio_list:
+ l.append((p[2], p[1], p[0]))
+ headers = ['priority', 'owner', 'path']
+ out = tabulate(l, headers, numalign='right')
+ print(out)
--
cgit v1.2.3
From 841747cbb4e9252fed60d4a6c6227b41c4986e84 Mon Sep 17 00:00:00 2001
From: John Estabrook <jestabro@vyos.io>
Date: Wed, 27 Mar 2024 20:44:35 -0500
Subject: configtree: T6180: add masking function mask_inclusive
(cherry picked from commit b2248b68afac795ad391b7203117d6d40a7ba6ed)
---
python/vyos/configtree.py | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py
index 423fe01ed..e4b282d72 100644
--- a/python/vyos/configtree.py
+++ b/python/vyos/configtree.py
@@ -401,6 +401,30 @@ def union(left, right, libpath=LIBPATH):
return tree
+def mask_inclusive(left, right, libpath=LIBPATH):
+ if not (isinstance(left, ConfigTree) and isinstance(right, ConfigTree)):
+ raise TypeError("Arguments must be instances of ConfigTree")
+
+ try:
+ __lib = cdll.LoadLibrary(libpath)
+ __mask_tree = __lib.mask_tree
+ __mask_tree.argtypes = [c_void_p, c_void_p]
+ __mask_tree.restype = c_void_p
+ __get_error = __lib.get_error
+ __get_error.argtypes = []
+ __get_error.restype = c_char_p
+
+ res = __mask_tree(left._get_config(), right._get_config())
+ except Exception as e:
+ raise ConfigTreeError(e)
+ if not res:
+ msg = __get_error().decode()
+ raise ConfigTreeError(msg)
+
+ tree = ConfigTree(address=res)
+
+ return tree
+
def reference_tree_to_json(from_dir, to_file, libpath=LIBPATH):
try:
__lib = cdll.LoadLibrary(libpath)
--
cgit v1.2.3
From bd8993540c6c1fc089943052aac51c9c318ffe6f Mon Sep 17 00:00:00 2001
From: John Estabrook <jestabro@vyos.io>
Date: Thu, 28 Mar 2024 14:34:20 -0500
Subject: config-sync: T6185: combine data for sections/configs in one command
Package path/section data in single command containing a tree (dict) of
section paths and the accompanying config data. This drops the call to
get_config_dict and the need for a list of commands in request.
(cherry picked from commit 30a530839cdbd934ea62369e385dc33fa50ab6de)
---
python/vyos/configsession.py | 19 +++++++++++
src/helpers/vyos_config_sync.py | 66 ++++++++++++++++++++-------------------
src/services/vyos-http-api-server | 38 ++++++++++++++++------
3 files changed, 82 insertions(+), 41 deletions(-)
diff --git a/python/vyos/configsession.py b/python/vyos/configsession.py
index 90842b749..ab7a631bb 100644
--- a/python/vyos/configsession.py
+++ b/python/vyos/configsession.py
@@ -176,6 +176,25 @@ class ConfigSession(object):
except (ValueError, ConfigSessionError) as e:
raise ConfigSessionError(e)
+ def set_section_tree(self, d: dict):
+ try:
+ if d:
+ for p in dict_to_paths(d):
+ self.set(p)
+ except (ValueError, ConfigSessionError) as e:
+ raise ConfigSessionError(e)
+
+ def load_section_tree(self, mask: dict, d: dict):
+ try:
+ if mask:
+ for p in dict_to_paths(mask):
+ self.delete(p)
+ if d:
+ for p in dict_to_paths(d):
+ self.set(p)
+ except (ValueError, ConfigSessionError) as e:
+ raise ConfigSessionError(e)
+
def comment(self, path, value=None):
if not value:
value = [""]
diff --git a/src/helpers/vyos_config_sync.py b/src/helpers/vyos_config_sync.py
index 7eec3f4f3..0604b2837 100755
--- a/src/helpers/vyos_config_sync.py
+++ b/src/helpers/vyos_config_sync.py
@@ -21,9 +21,11 @@ import json
import requests
import urllib3
import logging
-from typing import Optional, List, Dict, Any
+from typing import Optional, List, Tuple, Dict, Any
from vyos.config import Config
+from vyos.configtree import ConfigTree
+from vyos.configtree import mask_inclusive
from vyos.template import bracketize_ipv6
@@ -61,39 +63,45 @@ def post_request(url: str,
-def retrieve_config(section: Optional[List[str]] = None) -> Optional[Dict[str, Any]]:
+def retrieve_config(sections: List[list[str]]) -> Tuple[Dict[str, Any], Dict[str, Any]]:
"""Retrieves the configuration from the local server.
Args:
- section: List[str]: The section of the configuration to retrieve.
- Default is None.
+ sections: List[list[str]]: The list of sections of the configuration
+ to retrieve, given as list of paths.
Returns:
- Optional[Dict[str, Any]]: The retrieved configuration as a
- dictionary, or None if an error occurred.
+ Tuple[Dict[str, Any],Dict[str,Any]]: The tuple (mask, config) where:
+ - mask: The tree of paths of sections, as a dictionary.
+ - config: The subtree of masked config data, as a dictionary.
"""
- if section is None:
- section = []
- conf = Config()
- config = conf.get_config_dict(section, get_first_key=True)
- if config:
- return config
- return None
+ mask = ConfigTree('')
+ for section in sections:
+ mask.set(section)
+ mask_dict = json.loads(mask.to_json())
+
+ config = Config()
+ config_tree = config.get_config_tree()
+ masked = mask_inclusive(config_tree, mask)
+ config_dict = json.loads(masked.to_json())
+ return mask_dict, config_dict
def set_remote_config(
address: str,
key: str,
- commands: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ op: str,
+ mask: Dict[str, Any],
+ config: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""Loads the VyOS configuration in JSON format to a remote host.
Args:
address (str): The address of the remote host.
key (str): The key to use for loading the configuration.
- commands (list): List of set/load commands for request, given as:
- [{'op': str, 'path': list[str], 'section': dict},
- ...]
+ op (str): The operation to perform (set or load).
+ mask (dict): The dict of paths in sections.
+ config (dict): The dict of masked config data.
Returns:
Optional[Dict[str, Any]]: The response from the remote host as a
@@ -107,7 +115,9 @@ def set_remote_config(
url = f'https://{address}/configure-section'
data = json.dumps({
- 'commands': commands,
+ 'op': op,
+ 'mask': mask,
+ 'config': config,
'key': key
})
@@ -140,23 +150,15 @@ def config_sync(secondary_address: str,
)
# Sync sections ("nat", "firewall", etc)
- commands = []
- for section in sections:
- config_json = retrieve_config(section=section)
- # Check if config path deesn't exist, for example "set nat"
- # we set empty value for config_json data
- # As we cannot send to the remote host section "nat None" config
- if not config_json:
- config_json = {}
- logger.debug(
- f"Retrieved config for section '{section}': {config_json}")
-
- d = {'op': mode, 'path': section, 'section': config_json}
- commands.append(d)
+ mask_dict, config_dict = retrieve_config(sections)
+ logger.debug(
+ f"Retrieved config for sections '{sections}': {config_dict}")
set_config = set_remote_config(address=secondary_address,
key=secondary_key,
- commands=commands)
+ op=mode,
+ mask=mask_dict,
+ config=config_dict)
logger.debug(f"Set config for sections '{sections}': {set_config}")
diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server
index 77870a84c..ecbf6fcf9 100755
--- a/src/services/vyos-http-api-server
+++ b/src/services/vyos-http-api-server
@@ -140,6 +140,14 @@ class ConfigSectionModel(ApiModel, BaseConfigSectionModel):
class ConfigSectionListModel(ApiModel):
commands: List[BaseConfigSectionModel]
+class BaseConfigSectionTreeModel(BaseModel):
+ op: StrictStr
+ mask: Dict
+ config: Dict
+
+class ConfigSectionTreeModel(ApiModel, BaseConfigSectionTreeModel):
+ pass
+
class RetrieveModel(ApiModel):
op: StrictStr
path: List[StrictStr]
@@ -374,7 +382,7 @@ class MultipartRequest(Request):
self.form_err = (400,
f"Malformed command '{c}': missing 'op' field")
if endpoint not in ('/config-file', '/container-image',
- '/image'):
+ '/image', '/configure-section'):
if 'path' not in c:
self.form_err = (400,
f"Malformed command '{c}': missing 'path' field")
@@ -392,12 +400,9 @@ class MultipartRequest(Request):
self.form_err = (400,
f"Malformed command '{c}': 'value' field must be a string")
if endpoint in ('/configure-section'):
- if 'section' not in c:
- self.form_err = (400,
- f"Malformed command '{c}': missing 'section' field")
- elif not isinstance(c['section'], dict):
+ if 'section' not in c and 'config' not in c:
self.form_err = (400,
- f"Malformed command '{c}': 'section' field must be JSON of dict")
+ f"Malformed command '{c}': missing 'section' or 'config' field")
if 'key' not in forms and 'key' not in merge:
self.form_err = (401, "Valid API key is required")
@@ -455,7 +460,8 @@ def call_commit(s: ConfigSession):
logger.warning(f"ConfigSessionError: {e}")
def _configure_op(data: Union[ConfigureModel, ConfigureListModel,
- ConfigSectionModel, ConfigSectionListModel],
+ ConfigSectionModel, ConfigSectionListModel,
+ ConfigSectionTreeModel],
request: Request, background_tasks: BackgroundTasks):
session = app.state.vyos_session
env = session.get_session_env()
@@ -481,7 +487,8 @@ def _configure_op(data: Union[ConfigureModel, ConfigureListModel,
try:
for c in data:
op = c.op
- path = c.path
+ if not isinstance(c, BaseConfigSectionTreeModel):
+ path = c.path
if isinstance(c, BaseConfigureModel):
if c.value:
@@ -495,6 +502,10 @@ def _configure_op(data: Union[ConfigureModel, ConfigureListModel,
elif isinstance(c, BaseConfigSectionModel):
section = c.section
+ elif isinstance(c, BaseConfigSectionTreeModel):
+ mask = c.mask
+ config = c.config
+
if isinstance(c, BaseConfigureModel):
if op == 'set':
session.set(path, value=value)
@@ -514,6 +525,14 @@ def _configure_op(data: Union[ConfigureModel, ConfigureListModel,
session.load_section(path, section)
else:
raise ConfigSessionError(f"'{op}' is not a valid operation")
+
+ elif isinstance(c, BaseConfigSectionTreeModel):
+ if op == 'set':
+ session.set_section_tree(config)
+ elif op == 'load':
+ session.load_section_tree(mask, config)
+ else:
+ raise ConfigSessionError(f"'{op}' is not a valid operation")
# end for
config = Config(session_env=env)
d = get_config_diff(config)
@@ -554,7 +573,8 @@ def configure_op(data: Union[ConfigureModel,
@app.post('/configure-section')
def configure_section_op(data: Union[ConfigSectionModel,
- ConfigSectionListModel],
+ ConfigSectionListModel,
+ ConfigSectionTreeModel],
request: Request, background_tasks: BackgroundTasks):
return _configure_op(data, request, background_tasks)
--
cgit v1.2.3
From ee82253cda4e16ff04327b7235a8934497032ddc Mon Sep 17 00:00:00 2001
From: John Estabrook <jestabro@vyos.io>
Date: Thu, 28 Mar 2024 20:51:59 -0500
Subject: T6121: add section system time-zone
(cherry picked from commit b6c5e66cc44fdec21e6731d98a1065e2adf87b3b)
---
interface-definitions/service_config-sync.xml.in | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/interface-definitions/service_config-sync.xml.in b/interface-definitions/service_config-sync.xml.in
index cb51a33b1..e9ea9aa4b 100644
--- a/interface-definitions/service_config-sync.xml.in
+++ b/interface-definitions/service_config-sync.xml.in
@@ -495,6 +495,12 @@
<valueless/>
</properties>
</leafNode>
+ <leafNode name="time-zone">
+ <properties>
+ <help>Local time zone</help>
+ <valueless/>
+ </properties>
+ </leafNode>
</children>
</node>
<leafNode name="vpn">
--
cgit v1.2.3