12 files changed, 326 insertions, 33 deletions
diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py
index ded370a7a..14387cbbf 100755
--- a/src/conf_mode/container.py
+++ b/src/conf_mode/container.py
@@ -421,6 +421,10 @@ def generate(container):
                     'driver': 'host-local'
                 }
             }
+
+            if 'no_name_server' in network_config:
+                tmp['dns_enabled'] = False
+
             for prefix in network_config['prefix']:
                 net = {'subnet': prefix, 'gateway': inc_ip(prefix, 1)}
                 tmp['subnets'].append(net)
diff --git a/src/conf_mode/nat66.py b/src/conf_mode/nat66.py
index c44320f36..95dfae3a5 100755
--- a/src/conf_mode/nat66.py
+++ b/src/conf_mode/nat66.py
@@ -26,6 +26,7 @@ from vyos.utils.dict import dict_search
 from vyos.utils.kernel import check_kmod
 from vyos.utils.network import interface_exists
 from vyos.utils.process import cmd
+from vyos.utils.process import run
 from vyos.template import is_ipv6
 from vyos import ConfigError
 from vyos import airbag
@@ -48,6 +49,14 @@ def get_config(config=None):
 
     if not conf.exists(base):
         nat['deleted'] = ''
+        return nat
+
+    nat['firewall_group'] = conf.get_config_dict(['firewall', 'group'], key_mangling=('-', '_'), get_first_key=True,
+                                    no_tag_node_value_mangle=True)
+
+    # Remove dynamic firewall groups if present:
+    if 'dynamic_group' in nat['firewall_group']:
+        del nat['firewall_group']['dynamic_group']
 
     return nat
 
@@ -99,22 +108,33 @@ def verify(nat):
                         if not interface_exists(interface_name):
                             Warning(f'Interface "{interface_name}" for destination NAT66 rule "{rule}" does not exist!')
 
+            if 'destination' in config and 'group' in config['destination']:
+                if len({'address_group', 'network_group', 'domain_group'} & set(config['destination']['group'])) > 1:
+                    raise ConfigError('Only one address-group, network-group or domain-group can be specified')
+
     return None
 
 def generate(nat):
     if not os.path.exists(nftables_nat66_config):
         nat['first_install'] = True
 
-    render(nftables_nat66_config, 'firewall/nftables-nat66.j2', nat, permission=0o755)
+    render(nftables_nat66_config, 'firewall/nftables-nat66.j2', nat)
+
+    # dry-run newly generated configuration
+    tmp = run(f'nft --check --file {nftables_nat66_config}')
+    if tmp > 0:
+        raise ConfigError('Configuration file errors encountered!')
+
     return None
 
 def apply(nat):
-    if not nat:
-        return None
-
     check_kmod(k_mod)
 
     cmd(f'nft --file {nftables_nat66_config}')
+
+    if not nat or 'deleted' in nat:
+        os.unlink(nftables_nat66_config)
+
     call_dependents()
 
     return None
diff --git a/src/conf_mode/service_dns_forwarding.py b/src/conf_mode/service_dns_forwarding.py
index 70686534f..e3bdbc9f8 100755
--- a/src/conf_mode/service_dns_forwarding.py
+++ b/src/conf_mode/service_dns_forwarding.py
@@ -224,6 +224,18 @@ def get_config(config=None):
 
             dns['authoritative_zones'].append(zone)
 
+    if 'zone_cache' in dns:
+        # convert refresh interval to sec:
+        for _, zone_conf in dns['zone_cache'].items():
+            if 'options' in zone_conf \
+                    and 'refresh' in zone_conf['options']:
+
+                if 'on_reload' in zone_conf['options']['refresh']:
+                    interval = 0
+                else:
+                    interval = zone_conf['options']['refresh']['interval']
+                zone_conf['options']['refresh']['interval'] = interval
+
     return dns
 
 def verify(dns):
@@ -259,8 +271,16 @@ def verify(dns):
         if not 'system_name_server' in dns:
             print('Warning: No "system name-server" configured')
 
+    if 'zone_cache' in dns:
+        for name, conf in dns['zone_cache'].items():
+            if ('source' not in conf) \
+                    or ('url' in conf['source'] and 'axfr' in conf['source']):
+                raise ConfigError(f'Invalid configuration for zone "{name}": '
+                                  f'Please select one source type "url" or "axfr".')
+
     return None
 
+
 def generate(dns):
     # bail out early - looks like removal from running config
     if not dns:
diff --git a/src/conf_mode/system_option.py b/src/conf_mode/system_option.py
index d1647e3a1..52d0b7cda 100755
--- a/src/conf_mode/system_option.py
+++ b/src/conf_mode/system_option.py
@@ -85,6 +85,8 @@ def verify(options):
                 raise ConfigError('No interface with address "{address}" configured!')
 
         if 'source_interface' in config:
+            # verify_source_interface reuires key 'ifname'
+            config['ifname'] = config['source_interface']
             verify_source_interface(config)
             if 'source_address' in config:
                 address = config['source_address']
diff --git a/src/etc/dhcp/dhclient-enter-hooks.d/03-vyos-ipwrapper b/src/etc/dhcp/dhclient-enter-hooks.d/03-vyos-ipwrapper
index 5d879471d..2a1c5a7b2 100644
--- a/src/etc/dhcp/dhclient-enter-hooks.d/03-vyos-ipwrapper
+++ b/src/etc/dhcp/dhclient-enter-hooks.d/03-vyos-ipwrapper
@@ -72,6 +72,22 @@ function delroute () {
     fi
 }
 
+# try to communicate with vtysh
+function vtysh_conf () {
+    # perform 10 attempts with 1 second delay for retries
+    for i in {1..10} ; do
+        if vtysh  -c "conf t" -c "$1" ; then
+            logmsg info "Command was executed successfully via vtysh: \"$1\""
+            return 0
+        else
+            logmsg info "Failed to send command to vtysh, retrying in 1 second"
+            sleep 1
+        fi
+    done
+    logmsg error "Failed to execute command via vtysh after 10 attempts: \"$1\""
+    return 1
+}
+
 # replace ip command with this wrapper
 function ip () {
     # pass comand to system `ip` if this is not related to routes change
@@ -84,7 +100,7 @@ function ip () {
             delroute ${@:4}
             iptovtysh $@
             logmsg info "Sending command to vtysh"
-            vtysh -c "conf t" -c "$VTYSH_CMD"
+            vtysh_conf "$VTYSH_CMD"
         else
             # add ip route to kernel
             logmsg info "Modifying routes in kernel: \"/usr/sbin/ip $@\""
diff --git a/src/op_mode/monitor_bandwidth_test.sh b/src/op_mode/execute_bandwidth_test.sh
index a6ad0b42c..a6ad0b42c 100755
--- a/src/op_mode/monitor_bandwidth_test.sh
+++ b/src/op_mode/execute_bandwidth_test.sh
diff --git a/src/op_mode/execute_port-scan.py b/src/op_mode/execute_port-scan.py
new file mode 100644
index 000000000..bf17d0379
--- /dev/null
+++ b/src/op_mode/execute_port-scan.py
@@ -0,0 +1,155 @@
+#! /usr/bin/env python3
+#
+# Copyright (C) 2024 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+
+from vyos.utils.process import call
+
+
+options = {
+    'port': {
+        'cmd': '{command} -p {value}',
+        'type': '<1-65535> <list>',
+        'help': 'Scan specified ports.'
+    },
+    'tcp': {
+        'cmd': '{command} -sT',
+        'type': 'noarg',
+        'help': 'Use TCP scan.'
+    },
+    'udp': {
+        'cmd': '{command} -sU',
+        'type': 'noarg',
+        'help': 'Use UDP scan.'
+    },
+    'skip-ping': {
+        'cmd': '{command} -Pn',
+        'type': 'noarg',
+        'help': 'Skip the Nmap discovery stage altogether.'
+    },
+    'ipv6': {
+        'cmd': '{command} -6',
+        'type': 'noarg',
+        'help': 'Enable IPv6 scanning.'
+    },
+}
+
+nmap = 'sudo /usr/bin/nmap'
+
+
+class List(list):
+    def first(self):
+        return self.pop(0) if self else ''
+
+    def last(self):
+        return self.pop() if self else ''
+
+    def prepend(self, value):
+        self.insert(0, value)
+
+
+def completion_failure(option: str) -> None:
+    """
+    Shows failure message after TAB when option is wrong
+    :param option: failure option
+    :type str:
+    """
+    sys.stderr.write('\n\n Invalid option: {}\n\n'.format(option))
+    sys.stdout.write('<nocomps>')
+    sys.exit(1)
+
+
+def expansion_failure(option, completions):
+    reason = 'Ambiguous' if completions else 'Invalid'
+    sys.stderr.write(
+        '\n\n  {} command: {} [{}]\n\n'.format(reason, ' '.join(sys.argv),
+                                               option))
+    if completions:
+        sys.stderr.write('  Possible completions:\n   ')
+        sys.stderr.write('\n   '.join(completions))
+        sys.stderr.write('\n')
+    sys.stdout.write('<nocomps>')
+    sys.exit(1)
+
+
+def complete(prefix):
+    return [o for o in options if o.startswith(prefix)]
+
+
+def convert(command, args):
+    while args:
+        shortname = args.first()
+        longnames = complete(shortname)
+        if len(longnames) != 1:
+            expansion_failure(shortname, longnames)
+        longname = longnames[0]
+        if options[longname]['type'] == 'noarg':
+            command = options[longname]['cmd'].format(
+                command=command, value='')
+        elif not args:
+            sys.exit(f'port-scan: missing argument for {longname} option')
+        else:
+            command = options[longname]['cmd'].format(
+                command=command, value=args.first())
+    return command
+
+
+if __name__ == '__main__':
+    args = List(sys.argv[1:])
+    host = args.first()
+
+    if host == '--get-options-nested':
+        args.first()  # pop execute
+        args.first()  # pop port-scan
+        args.first()  # pop host
+        args.first()  # pop <host>
+        usedoptionslist = []
+        while args:
+            option = args.first()  # pop option
+            matched = complete(option)  # get option parameters
+            usedoptionslist.append(option)  # list of used options
+            # Select options
+            if not args:
+                # remove from Possible completions used options
+                for o in usedoptionslist:
+                    if o in matched:
+                        matched.remove(o)
+                if not matched:
+                    sys.stdout.write('<nocomps>')
+                else:
+                    sys.stdout.write(' '.join(matched))
+                sys.exit(0)
+
+            if len(matched) > 1:
+                sys.stdout.write(' '.join(matched))
+                sys.exit(0)
+            # If option doesn't have value
+            if matched:
+                if options[matched[0]]['type'] == 'noarg':
+                    continue
+            else:
+                # Unexpected option
+                completion_failure(option)
+
+            value = args.first()  # pop option's value
+            if not args:
+                matched = complete(option)
+                helplines = options[matched[0]]['type']
+                sys.stdout.write(helplines)
+                sys.exit(0)
+
+    command = convert(nmap, args)
+    call(f'{command} -T4 {host}')
diff --git a/src/op_mode/ntp.py b/src/op_mode/ntp.py
index e14cc46d0..6ec0fedcb 100644
--- a/src/op_mode/ntp.py
+++ b/src/op_mode/ntp.py
@@ -110,49 +110,62 @@ def _is_configured():
     if not config.exists("service ntp"):
         raise vyos.opmode.UnconfiguredSubsystem("NTP service is not enabled.")
 
+def _extend_command_vrf():
+    config = ConfigTreeQuery()
+    if config.exists('service ntp vrf'):
+        vrf = config.value('service ntp vrf')
+        return f'ip vrf exec {vrf} '
+    return ''
+
+
 def show_activity(raw: bool):
     _is_configured()
     command = f'chronyc'
 
     if raw:
-       command += f" -c activity"
-       return _get_raw_data(command)
+        command += f" -c activity"
+        return _get_raw_data(command)
     else:
-       command += f" activity"
-       return cmd(command)
+        command = _extend_command_vrf() + command
+        command += f" activity"
+        return cmd(command)
 
 def show_sources(raw: bool):
     _is_configured()
     command = f'chronyc'
 
     if raw:
-       command += f" -c sources"
-       return _get_raw_data(command)
+        command += f" -c sources"
+        return _get_raw_data(command)
     else:
-       command += f" sources -v"
-       return cmd(command)
+        command = _extend_command_vrf() + command
+        command += f" sources -v"
+        return cmd(command)
 
 def show_tracking(raw: bool):
     _is_configured()
     command = f'chronyc'
 
     if raw:
-       command += f" -c tracking"
-       return _get_raw_data(command)
+        command += f" -c tracking"
+        return _get_raw_data(command)
     else:
-       command += f" tracking"
-       return cmd(command)
+        command = _extend_command_vrf() + command
+        command += f" tracking"
+        return cmd(command)
 
 def show_sourcestats(raw: bool):
     _is_configured()
     command = f'chronyc'
 
     if raw:
-       command += f" -c sourcestats"
-       return _get_raw_data(command)
+        command += f" -c sourcestats"
+        return _get_raw_data(command)
     else:
-       command += f" sourcestats -v"
-       return cmd(command)
+        command = _extend_command_vrf() + command
+        command += f" sourcestats -v"
+        return cmd(command)
+
 
 if __name__ == '__main__':
     try:
diff --git a/src/op_mode/restart.py b/src/op_mode/restart.py
index 813d3a2b7..a83c8b9d8 100755
--- a/src/op_mode/restart.py
+++ b/src/op_mode/restart.py
@@ -25,11 +25,11 @@ from vyos.utils.commit import commit_in_progress
 config = ConfigTreeQuery()
 
 service_map = {
-    'dhcp' : {
+    'dhcp': {
         'systemd_service': 'kea-dhcp4-server',
         'path': ['service', 'dhcp-server'],
     },
-    'dhcpv6' : {
+    'dhcpv6': {
         'systemd_service': 'kea-dhcp6-server',
         'path': ['service', 'dhcpv6-server'],
     },
@@ -61,24 +61,40 @@ service_map = {
         'systemd_service': 'radvd',
         'path': ['service', 'router-advert'],
     },
-    'snmp' : {
+    'snmp': {
         'systemd_service': 'snmpd',
     },
-    'ssh' : {
+    'ssh': {
         'systemd_service': 'ssh',
     },
-    'suricata' : {
+    'suricata': {
         'systemd_service': 'suricata',
     },
-    'vrrp' : {
+    'vrrp': {
         'systemd_service': 'keepalived',
         'path': ['high-availability', 'vrrp'],
     },
-    'webproxy' : {
+    'webproxy': {
         'systemd_service': 'squid',
     },
 }
-services = typing.Literal['dhcp', 'dhcpv6', 'dns_dynamic', 'dns_forwarding', 'igmp_proxy', 'ipsec', 'mdns_repeater', 'reverse_proxy', 'router_advert', 'snmp', 'ssh', 'suricata' 'vrrp', 'webproxy']
+services = typing.Literal[
+    'dhcp',
+    'dhcpv6',
+    'dns_dynamic',
+    'dns_forwarding',
+    'igmp_proxy',
+    'ipsec',
+    'mdns_repeater',
+    'reverse_proxy',
+    'router_advert',
+    'snmp',
+    'ssh',
+    'suricata',
+    'vrrp',
+    'webproxy',
+]
+
 
 def _verify(func):
     """Decorator checks if DHCP(v6) config exists"""
@@ -102,13 +118,18 @@ def _verify(func):
 
         # Check if config does not exist
         if not config.exists(path):
-            raise vyos.opmode.UnconfiguredSubsystem(f'Service {human_name} is not configured!')
+            raise vyos.opmode.UnconfiguredSubsystem(
+                f'Service {human_name} is not configured!'
+            )
         if config.exists(path + ['disable']):
-            raise vyos.opmode.UnconfiguredSubsystem(f'Service {human_name} is disabled!')
+            raise vyos.opmode.UnconfiguredSubsystem(
+                f'Service {human_name} is disabled!'
+            )
         return func(*args, **kwargs)
 
     return _wrapper
 
+
 @_verify
 def restart_service(raw: bool, name: services, vrf: typing.Optional[str]):
     systemd_service = service_map[name]['systemd_service']
@@ -117,6 +138,7 @@ def restart_service(raw: bool, name: services, vrf: typing.Optional[str]):
     else:
         call(f'systemctl restart "{systemd_service}.service"')
 
+
 if __name__ == '__main__':
     try:
         res = vyos.opmode.run(sys.modules[__name__])
diff --git a/src/services/vyos-configd b/src/services/vyos-configd
index d797e90cf..3674d9627 100755
--- a/src/services/vyos-configd
+++ b/src/services/vyos-configd
@@ -30,6 +30,7 @@ from vyos.defaults import directories
 from vyos.utils.boot import boot_configuration_complete
 from vyos.configsource import ConfigSourceString
 from vyos.configsource import ConfigSourceError
+from vyos.configdiff import get_commit_scripts
 from vyos.config import Config
 from vyos import ConfigError
 
@@ -220,6 +221,12 @@ def initialization(socket):
     dependent_func: dict[str, list[typing.Callable]] = {}
     setattr(config, 'dependent_func', dependent_func)
 
+    commit_scripts = get_commit_scripts(config)
+    logger.debug(f'commit_scripts: {commit_scripts}')
+
+    scripts_called = []
+    setattr(config, 'scripts_called', scripts_called)
+
     return config
 
 def process_node_data(config, data, last: bool = False) -> int:
@@ -228,6 +235,7 @@ def process_node_data(config, data, last: bool = False) -> int:
         return R_ERROR_DAEMON
 
     script_name = None
+    os.environ['VYOS_TAGNODE_VALUE'] = ''
     args = []
     config.dependency_list.clear()
 
@@ -244,6 +252,12 @@ def process_node_data(config, data, last: bool = False) -> int:
         args = res.group(3).split()
     args.insert(0, f'{script_name}.py')
 
+    tag_value = os.getenv('VYOS_TAGNODE_VALUE', '')
+    tag_ext = f'_{tag_value}' if tag_value else ''
+    script_record = f'{script_name}{tag_ext}'
+    scripts_called = getattr(config, 'scripts_called', [])
+    scripts_called.append(script_record)
+
     if script_name not in include_set:
         return R_PASS
 
@@ -302,11 +316,12 @@ if __name__ == '__main__':
             socket.send(resp.encode())
             config = initialization(socket)
         elif message["type"] == "node":
-            if message["last"]:
-                logger.debug(f'final element of priority queue')
             res = process_node_data(config, message["data"], message["last"])
             response = res.to_bytes(1, byteorder=sys.byteorder)
             logger.debug(f"Sending response {res}")
             socket.send(response)
+            if message["last"] and config:
+                scripts_called = getattr(config, 'scripts_called', [])
+                logger.debug(f'scripts_called: {scripts_called}')
         else:
             logger.critical(f"Unexpected message: {message}")
diff --git a/src/systemd/podman.service b/src/systemd/podman.service
new file mode 100644
index 000000000..20a16304b
--- /dev/null
+++ b/src/systemd/podman.service
@@ -0,0 +1,16 @@
+[Unit]
+Description=Podman API Service
+Requires=podman.socket
+After=podman.socket
+Documentation=man:podman-system-service(1)
+StartLimitIntervalSec=0
+
+[Service]
+Delegate=true
+Type=exec
+KillMode=process
+Environment=LOGGING="--log-level=info"
+ExecStart=/usr/bin/podman $LOGGING system service
+
+[Install]
+WantedBy=default.target
diff --git a/src/systemd/podman.socket b/src/systemd/podman.socket
new file mode 100644
index 000000000..397058ee4
--- /dev/null
+++ b/src/systemd/podman.socket
@@ -0,0 +1,10 @@
+[Unit]
+Description=Podman API Socket
+Documentation=man:podman-system-service(1)
+
+[Socket]
+ListenStream=%t/podman/podman.sock
+SocketMode=0660
+
+[Install]
+WantedBy=sockets.target