13 files changed, 391 insertions, 16 deletions

diff --git a/src/conf_mode/config_mgmt.py b/src/conf_mode/config_mgmt.py
new file mode 100755
index 000000000..c681a8405
--- /dev/null
+++ b/src/conf_mode/config_mgmt.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2023 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+
+from vyos import ConfigError
+from vyos.config import Config
+from vyos.config_mgmt import ConfigMgmt
+from vyos.config_mgmt import commit_post_hook_dir, commit_hooks
+
+def get_config(config=None):
+    if config:
+        conf = config
+    else:
+        conf = Config()
+
+    base = ['system', 'config-management']
+    if not conf.exists(base):
+        return None
+
+    mgmt = ConfigMgmt(config=conf)
+
+    return mgmt
+
+def verify(_mgmt):
+    return
+
+def generate(mgmt):
+    if mgmt is None:
+        return
+
+    mgmt.initialize_revision()
+
+def apply(mgmt):
+    if mgmt is None:
+        return
+
+    locations = mgmt.locations
+    archive_target = os.path.join(commit_post_hook_dir,
+                               commit_hooks['commit_archive'])
+    if locations:
+        try:
+            os.symlink('/usr/bin/config-mgmt', archive_target)
+        except FileExistsError:
+            pass
+        except OSError as exc:
+            raise ConfigError from exc
+    else:
+        try:
+            os.unlink(archive_target)
+        except FileNotFoundError:
+            pass
+        except OSError as exc:
+            raise ConfigError from exc
+
+    revisions = mgmt.max_revisions
+    revision_target = os.path.join(commit_post_hook_dir,
+                               commit_hooks['commit_revision'])
+    if revisions > 0:
+        try:
+            os.symlink('/usr/bin/config-mgmt', revision_target)
+        except FileExistsError:
+            pass
+        except OSError as exc:
+            raise ConfigError from exc
+    else:
+        try:
+            os.unlink(revision_target)
+        except FileNotFoundError:
+            pass
+        except OSError as exc:
+            raise ConfigError from exc
+
+if __name__ == '__main__':
+    try:
+        c = get_config()
+        verify(c)
+        generate(c)
+        apply(c)
+    except ConfigError as e:
+        print(e)
+        sys.exit(1)
diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py
index 7567444db..08861053d 100755
--- a/src/conf_mode/container.py
+++ b/src/conf_mode/container.py
@@ -75,6 +75,8 @@ def get_config(config=None):
         default_values = defaults(base + ['name'])
         if 'port' in default_values:
             del default_values['port']
+        if 'volume' in default_values:
+            del default_values['volume']
         for name in container['name']:
             container['name'][name] = dict_merge(default_values, container['name'][name])
 
@@ -85,6 +87,13 @@ def get_config(config=None):
                     default_values = defaults(base + ['name', 'port'])
                     container['name'][name]['port'][port] = dict_merge(
                         default_values, container['name'][name]['port'][port])
+            # XXX: T2665: we can not safely rely on the defaults() when there are
+            # tagNodes in place, it is better to blend in the defaults manually.
+            if 'volume' in container['name'][name]:
+                for volume in container['name'][name]['volume']:
+                    default_values = defaults(base + ['name', 'volume'])
+                    container['name'][name]['volume'][volume] = dict_merge(
+                        default_values, container['name'][name]['volume'][volume])
 
     # Delete container network, delete containers
     tmp = node_changed(conf, base + ['network'])
@@ -245,7 +254,7 @@ def generate_run_arguments(name, container_config):
     env_opt = ''
     if 'environment' in container_config:
         for k, v in container_config['environment'].items():
-            env_opt += f" -e \"{k}={v['value']}\""
+            env_opt += f" --env \"{k}={v['value']}\""
 
     # Publish ports
     port = ''
@@ -255,7 +264,7 @@ def generate_run_arguments(name, container_config):
             protocol = container_config['port'][portmap]['protocol']
             sport = container_config['port'][portmap]['source']
             dport = container_config['port'][portmap]['destination']
-            port += f' -p {sport}:{dport}/{protocol}'
+            port += f' --publish {sport}:{dport}/{protocol}'
 
     # Bind volume
     volume = ''
@@ -263,7 +272,8 @@ def generate_run_arguments(name, container_config):
         for vol, vol_config in container_config['volume'].items():
             svol = vol_config['source']
             dvol = vol_config['destination']
-            volume += f' -v {svol}:{dvol}'
+            mode = vol_config['mode']
+            volume += f' --volume {svol}:{dvol}:{mode}'
 
     container_base_cmd = f'--detach --interactive --tty --replace {cap_add} ' \
                          f'--memory {memory}m --shm-size {shared_memory}m --memory-swap 0 --restart {restart} ' \
diff --git a/src/conf_mode/interfaces-pppoe.py b/src/conf_mode/interfaces-pppoe.py
index ee4defa0d..5f0b76f90 100755
--- a/src/conf_mode/interfaces-pppoe.py
+++ b/src/conf_mode/interfaces-pppoe.py
@@ -54,7 +54,8 @@ def get_config(config=None):
     # All parameters that can be changed on-the-fly (like interface description)
     # should not lead to a reconnect!
     for options in ['access-concentrator', 'connect-on-demand', 'service-name',
-                    'source-interface', 'vrf', 'no-default-route', 'authentication']:
+                    'source-interface', 'vrf', 'no-default-route',
+                    'authentication', 'host_uniq']:
         if is_node_changed(conf, base + [ifname, options]):
             pppoe.update({'shutdown_required': {}})
             # bail out early - no need to further process other nodes
diff --git a/src/conf_mode/snmp.py b/src/conf_mode/snmp.py
index 914ec245c..ab2ccf99e 100755
--- a/src/conf_mode/snmp.py
+++ b/src/conf_mode/snmp.py
@@ -166,6 +166,10 @@ def verify(snmp):
             if 'community' not in trap_config:
                 raise ConfigError(f'Trap target "{trap}" requires a community to be set!')
 
+    if 'oid_enable' in snmp:
+        Warning(f'Custom OIDs are enabled and may lead to system instability and high resource consumption')
+
+
     verify_vrf(snmp)
 
     # bail out early if SNMP v3 is not configured
diff --git a/src/migration-scripts/snmp/2-to-3 b/src/migration-scripts/snmp/2-to-3
new file mode 100755
index 000000000..5f8d9c88d
--- /dev/null
+++ b/src/migration-scripts/snmp/2-to-3
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2022 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# T4857: Implement FRR SNMP recomendations
+#  cli changes from:
+#  set service snmp oid-enable route-table
+#  To
+#  set service snmp oid-enable ip-forward
+
+import re
+
+from sys import argv
+from sys import exit
+
+from vyos.configtree import ConfigTree
+from vyos.ifconfig import Section
+
+if (len(argv) < 1):
+    print("Must specify file name!")
+    exit(1)
+
+file_name = argv[1]
+
+with open(file_name, 'r') as f:
+    config_file = f.read()
+
+base = ['service snmp']
+config = ConfigTree(config_file)
+
+if not config.exists(base):
+    # Nothing to do
+    exit(0)
+
+if config.exists(base + ['oid-enable']):
+    config.delete(base + ['oid-enable'])
+    config.set(base + ['oid-enable'], 'ip-forward')
+
+
+try:
+    with open(file_name, 'w') as f:
+        f.write(config.to_string())
+except OSError as e:
+    print("Failed to save the modified config: {}".format(e))
+    exit(1)
diff --git a/src/op_mode/config_mgmt.py b/src/op_mode/config_mgmt.py
new file mode 100755
index 000000000..66de26d1f
--- /dev/null
+++ b/src/op_mode/config_mgmt.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2023 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import typing
+
+import vyos.opmode
+from vyos.config_mgmt import ConfigMgmt
+
+def show_commit_diff(raw: bool, rev: int, rev2: typing.Optional[int],
+                     commands: bool):
+    config_mgmt = ConfigMgmt()
+    config_diff = config_mgmt.show_commit_diff(rev, rev2, commands)
+
+    if raw:
+        rev2 = (rev+1) if rev2 is None else rev2
+        if commands:
+            d = {f'config_command_diff_{rev2}_{rev}': config_diff}
+        else:
+            d = {f'config_file_diff_{rev2}_{rev}': config_diff}
+        return d
+
+    return config_diff
+
+def show_commit_file(raw: bool, rev: int):
+    config_mgmt = ConfigMgmt()
+    config_file = config_mgmt.show_commit_file(rev)
+
+    if raw:
+        d = {f'config_revision_{rev}': config_file}
+        return d
+
+    return config_file
+
+def show_commit_log(raw: bool):
+    config_mgmt = ConfigMgmt()
+
+    msg = ''
+    if config_mgmt.max_revisions == 0:
+        msg = ('commit-revisions is not configured;\n'
+               'commit log is empty or stale:\n\n')
+
+    data = config_mgmt.get_raw_log_data()
+    if raw:
+        return data
+
+    out = config_mgmt.format_log_data(data)
+    out = msg + out
+
+    return out
+
+def show_commit_log_brief(raw: bool):
+    # used internally for completion help for 'rollback'
+    # option 'raw' will return same as 'show_commit_log'
+    config_mgmt = ConfigMgmt()
+
+    data = config_mgmt.get_raw_log_data()
+    if raw:
+        return data
+
+    out = config_mgmt.format_log_data_brief(data)
+
+    return out
+
+if __name__ == '__main__':
+    try:
+        res = vyos.opmode.run(sys.modules[__name__])
+        if res:
+            print(res)
+    except (ValueError, vyos.opmode.Error) as e:
+        print(e)
+        sys.exit(1)
diff --git a/src/op_mode/generate_interfaces_debug_archive.py b/src/op_mode/generate_interfaces_debug_archive.py
new file mode 100644
index 000000000..f5767080a
--- /dev/null
+++ b/src/op_mode/generate_interfaces_debug_archive.py
@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2023 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from datetime import datetime
+from pathlib import Path
+from shutil import rmtree
+from socket import gethostname
+from sys import exit
+from tarfile import open as tar_open
+from vyos.util import rc_cmd
+import os
+
+# define a list of commands that needs to be executed
+
+CMD_LIST: list[str] = [
+    "journalctl -b -n 500",
+    "journalctl -b -k -n 500",
+    "ip -s l",
+    "cat /proc/interrupts",
+    "cat /proc/softirqs",
+    "top -b -d 1 -n 2 -1",
+    "netstat -l",             
+    "cat /proc/net/dev",
+    "cat /proc/net/softnet_stat",
+    "cat /proc/net/icmp",
+    "cat /proc/net/udp",
+    "cat /proc/net/tcp",
+    "cat /proc/net/netstat",
+    "sysctl net",
+    "timeout 10 tcpdump -c 500 -eni any port not 22"
+]
+
+CMD_INTERFACES_LIST: list[str] = [
+    "ethtool -i ",
+    "ethtool -S ",
+    "ethtool -g ",
+    "ethtool -c ",
+    "ethtool -a ",
+    "ethtool -k ",
+    "ethtool -i ",
+    "ethtool --phy-statistics "
+]
+
+# get intefaces info
+interfaces_list = os.popen('ls /sys/class/net/').read().split()
+
+# modify CMD_INTERFACES_LIST for all interfaces
+CMD_INTERFACES_LIST_MOD=[]
+for command_interface in interfaces_list:
+    for command_interfacev2 in CMD_INTERFACES_LIST:
+        CMD_INTERFACES_LIST_MOD.append (f'{command_interfacev2}{command_interface}')
+
+# execute a command and save the output to a file
+
+def save_stdout(command: str, file: Path) -> None:
+    rc, stdout = rc_cmd(command)
+    body: str = f'''### {command} ###
+Command: {command}
+Exit code: {rc}
+Stdout:
+{stdout}
+
+'''
+    with file.open(mode='a') as f:
+        f.write(body)
+
+# get local host name
+hostname: str = gethostname()
+# get current time
+time_now: str = datetime.now().isoformat(timespec='seconds')
+
+# define a temporary directory for logs and collected data
+tmp_dir: Path = Path(f'/tmp/drops-debug_{time_now}')
+# set file paths
+drops_file: Path = Path(f'{tmp_dir}/drops.txt')
+interfaces_file: Path = Path(f'{tmp_dir}/interfaces.txt')
+archive_file: str = f'/tmp/packet-drops-debug_{time_now}.tar.bz2'
+
+# create files
+tmp_dir.mkdir()
+drops_file.touch()
+interfaces_file.touch()
+
+try:
+    # execute all commands
+    for command in CMD_LIST:
+        save_stdout(command, drops_file)
+    for command_interface in CMD_INTERFACES_LIST_MOD:
+        save_stdout(command_interface, interfaces_file)
+
+    # create an archive
+    with tar_open(name=archive_file, mode='x:bz2') as tar_file:
+        tar_file.add(tmp_dir)
+
+    # inform user about success
+    print(f'Debug file is generated and located in {archive_file}')
+except Exception as err:
+    print(f'Error during generating a debug file: {err}')
+finally:
+    # cleanup
+    rmtree(tmp_dir)
+    exit()
diff --git a/src/op_mode/lldp.py b/src/op_mode/lldp.py
index dc2b1e0b5..1a1b94783 100755
--- a/src/op_mode/lldp.py
+++ b/src/op_mode/lldp.py
@@ -61,7 +61,14 @@ def _get_raw_data(interface=None, detail=False):
 
 def _get_formatted_output(raw_data):
     data_entries = []
-    for neighbor in dict_search('lldp.interface', raw_data):
+    tmp = dict_search('lldp.interface', raw_data)
+    if not tmp:
+        return None
+    # One can not always ensure that "interface" is of type list, add safeguard.
+    # E.G. Juniper Networks, Inc. ex2300-c-12t only has a dict, not a list of dicts
+    if isinstance(tmp, dict):
+        tmp = [tmp]
+    for neighbor in tmp:
         for local_if, values in neighbor.items():
             tmp = []
 
@@ -80,6 +87,10 @@ def _get_formatted_output(raw_data):
             # Capabilities
             cap = ''
             capabilities = jmespath.search('chassis.[*][0][0].capability', values)
+            # One can not always ensure that "capability" is of type list, add
+            # safeguard. E.G. Unify US-24-250W only has a dict, not a list of dicts
+            if isinstance(capabilities, dict):
+                capabilities = [capabilities]
             if capabilities:
                 for capability in capabilities:
                     if capability['enabled']:
diff --git a/src/services/api/graphql/generate/schema_from_op_mode.py b/src/services/api/graphql/generate/schema_from_op_mode.py
index fc63b0100..b320a529e 100755
--- a/src/services/api/graphql/generate/schema_from_op_mode.py
+++ b/src/services/api/graphql/generate/schema_from_op_mode.py
@@ -25,16 +25,17 @@ from inspect import signature, getmembers, isfunction, isclass, getmro
 from jinja2 import Template
 
 from vyos.defaults import directories
+from vyos.opmode import _is_op_mode_function_name as is_op_mode_function_name
 from vyos.util import load_as_module
 if __package__ is None or __package__ == '':
     sys.path.append("/usr/libexec/vyos/services/api")
-    from graphql.libs.op_mode import is_op_mode_function_name, is_show_function_name
+    from graphql.libs.op_mode import is_show_function_name
     from graphql.libs.op_mode import snake_to_pascal_case, map_type_name
     from vyos.config import Config
     from vyos.configdict import dict_merge
     from vyos.xml import defaults
 else:
-    from .. libs.op_mode import is_op_mode_function_name, is_show_function_name
+    from .. libs.op_mode import is_show_function_name
     from .. libs.op_mode import snake_to_pascal_case, map_type_name
     from .. import state
 
diff --git a/src/services/api/graphql/graphql/mutations.py b/src/services/api/graphql/graphql/mutations.py
index 87ea59c43..8254e22b1 100644
--- a/src/services/api/graphql/graphql/mutations.py
+++ b/src/services/api/graphql/graphql/mutations.py
@@ -15,7 +15,7 @@
 
 from importlib import import_module
 from typing import Any, Dict, Optional
-from ariadne import ObjectType, convert_kwargs_to_snake_case, convert_camel_case_to_snake
+from ariadne import ObjectType, convert_camel_case_to_snake
 from graphql import GraphQLResolveInfo
 from makefun import with_signature
 
@@ -45,7 +45,6 @@ def make_mutation_resolver(mutation_name, class_name, session_func):
     func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Optional[Dict]=None)'
 
     @mutation.field(mutation_name)
-    @convert_kwargs_to_snake_case
     @with_signature(func_sig, func_name=resolver_name)
     async def func_impl(*args, **kwargs):
         try:
diff --git a/src/services/api/graphql/graphql/queries.py b/src/services/api/graphql/graphql/queries.py
index 1ad586428..daccc19b2 100644
--- a/src/services/api/graphql/graphql/queries.py
+++ b/src/services/api/graphql/graphql/queries.py
@@ -15,7 +15,7 @@
 
 from importlib import import_module
 from typing import Any, Dict, Optional
-from ariadne import ObjectType, convert_kwargs_to_snake_case, convert_camel_case_to_snake
+from ariadne import ObjectType, convert_camel_case_to_snake
 from graphql import GraphQLResolveInfo
 from makefun import with_signature
 
@@ -45,7 +45,6 @@ def make_query_resolver(query_name, class_name, session_func):
     func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Optional[Dict]=None)'
 
     @query.field(query_name)
-    @convert_kwargs_to_snake_case
     @with_signature(func_sig, func_name=resolver_name)
     async def func_impl(*args, **kwargs):
         try:
diff --git a/src/services/api/graphql/libs/op_mode.py b/src/services/api/graphql/libs/op_mode.py
index c1eb493db..c553bbd67 100644
--- a/src/services/api/graphql/libs/op_mode.py
+++ b/src/services/api/graphql/libs/op_mode.py
@@ -29,11 +29,6 @@ def load_op_mode_as_module(name: str):
     name = os.path.splitext(name)[0].replace('-', '_')
     return load_as_module(name, path)
 
-def is_op_mode_function_name(name):
-    if re.match(r"^(show|clear|reset|restart|add|delete)", name):
-        return True
-    return False
-
 def is_show_function_name(name):
     if re.match(r"^show", name):
         return True
diff --git a/src/services/api/graphql/session/errors/op_mode_errors.py b/src/services/api/graphql/session/errors/op_mode_errors.py
index 4029fd0a1..a8a9ee426 100644
--- a/src/services/api/graphql/session/errors/op_mode_errors.py
+++ b/src/services/api/graphql/session/errors/op_mode_errors.py
@@ -4,6 +4,7 @@ op_mode_err_msg = {
     "UnconfiguredSubsystem": "subsystem is not configured or not running",
     "DataUnavailable": "data currently unavailable",
     "PermissionDenied": "client does not have permission",
+    "InsufficientResources": "insufficient system resources"
     "IncorrectValue": "argument value is incorrect",
     "UnsupportedOperation": "operation is not supported (yet)",
 }
@@ -11,6 +12,7 @@ op_mode_err_msg = {
 op_mode_err_code = {
     "UnconfiguredSubsystem": 2000,
     "DataUnavailable": 2001,
+    "InsufficientResources": 2002,
     "PermissionDenied": 1003,
     "IncorrectValue": 1002,
     "UnsupportedOperation": 1004,