16 files changed, 328 insertions, 97 deletions

diff --git a/src/conf_mode/conntrack.py b/src/conf_mode/conntrack.py
index 9c43640a9..a0de914bc 100755
--- a/src/conf_mode/conntrack.py
+++ b/src/conf_mode/conntrack.py
@@ -24,7 +24,9 @@ from vyos.firewall import find_nftables_rule
 from vyos.firewall import remove_nftables_rule
 from vyos.utils.process import process_named_running
 from vyos.utils.dict import dict_search
+from vyos.utils.dict import dict_search_args
 from vyos.utils.process import cmd
+from vyos.utils.process import rc_cmd
 from vyos.utils.process import run
 from vyos.template import render
 from vyos import ConfigError
@@ -62,6 +64,13 @@ module_map = {
      },
 }
 
+valid_groups = [
+    'address_group',
+    'domain_group',
+    'network_group',
+    'port_group'
+]
+
 def resync_conntrackd():
     tmp = run('/usr/libexec/vyos/conf_mode/conntrack_sync.py')
     if tmp > 0:
@@ -78,15 +87,53 @@ def get_config(config=None):
                                      get_first_key=True,
                                      with_recursive_defaults=True)
 
+    conntrack['firewall_group'] = conf.get_config_dict(['firewall', 'group'], key_mangling=('-', '_'),
+                                                 get_first_key=True,
+                                                 no_tag_node_value_mangle=True)
+
     return conntrack
 
 def verify(conntrack):
-    if dict_search('ignore.rule', conntrack) != None:
-        for rule, rule_config in conntrack['ignore']['rule'].items():
-            if dict_search('destination.port', rule_config) or \
-               dict_search('source.port', rule_config):
-               if 'protocol' not in rule_config or rule_config['protocol'] not in ['tcp', 'udp']:
-                   raise ConfigError(f'Port requires tcp or udp as protocol in rule {rule}')
+    for inet in ['ipv4', 'ipv6']:
+        if dict_search_args(conntrack, 'ignore', inet, 'rule') != None:
+            for rule, rule_config in conntrack['ignore'][inet]['rule'].items():
+                if dict_search('destination.port', rule_config) or \
+                   dict_search('destination.group.port_group', rule_config) or \
+                   dict_search('source.port', rule_config) or \
+                   dict_search('source.group.port_group', rule_config):
+                   if 'protocol' not in rule_config or rule_config['protocol'] not in ['tcp', 'udp']:
+                       raise ConfigError(f'Port requires tcp or udp as protocol in rule {rule}')
+
+                for side in ['destination', 'source']:
+                    if side in rule_config:
+                        side_conf = rule_config[side]
+
+                        if 'group' in side_conf:
+                            if len({'address_group', 'network_group', 'domain_group'} & set(side_conf['group'])) > 1:
+                                raise ConfigError('Only one address-group, network-group or domain-group can be specified')
+
+                            for group in valid_groups:
+                                if group in side_conf['group']:
+                                    group_name = side_conf['group'][group]
+                                    error_group = group.replace("_", "-")
+
+                                    if group in ['address_group', 'network_group', 'domain_group']:
+                                        if 'address' in side_conf:
+                                            raise ConfigError(f'{error_group} and address cannot both be defined')
+
+                                    if group_name and group_name[0] == '!':
+                                        group_name = group_name[1:]
+
+                                    if inet == 'ipv6':
+                                        group = f'ipv6_{group}'
+
+                                    group_obj = dict_search_args(conntrack['firewall_group'], group, group_name)
+
+                                    if group_obj is None:
+                                        raise ConfigError(f'Invalid {error_group} "{group_name}" on ignore rule')
+
+                                    if not group_obj:
+                                        Warning(f'{error_group} "{group_name}" has no members!')
 
     return None
 
@@ -94,26 +141,18 @@ def generate(conntrack):
     render(conntrack_config, 'conntrack/vyos_nf_conntrack.conf.j2', conntrack)
     render(sysctl_file, 'conntrack/sysctl.conf.j2', conntrack)
     render(nftables_ct_file, 'conntrack/nftables-ct.j2', conntrack)
-
-    # dry-run newly generated configuration
-    tmp = run(f'nft -c -f {nftables_ct_file}')
-    if tmp > 0:
-        if os.path.exists(nftables_ct_file):
-            os.unlink(nftables_ct_file)
-        raise ConfigError('Configuration file errors encountered!')
-
     return None
 
-def find_nftables_ct_rule(rule):
+def find_nftables_ct_rule(table, chain, rule):
     helper_search = re.search('ct helper set "(\w+)"', rule)
     if helper_search:
         rule = helper_search[1]
-    return find_nftables_rule('raw', 'VYOS_CT_HELPER', [rule])
+    return find_nftables_rule(table, chain, [rule])
 
-def find_remove_rule(rule):
-    handle = find_nftables_ct_rule(rule)
+def find_remove_rule(table, chain, rule):
+    handle = find_nftables_ct_rule(table, chain, rule)
     if handle:
-        remove_nftables_rule('raw', 'VYOS_CT_HELPER', handle)
+        remove_nftables_rule(table, chain, handle)
 
 def apply(conntrack):
     # Depending on the enable/disable state of the ALG (Application Layer Gateway)
@@ -127,18 +166,24 @@ def apply(conntrack):
                         cmd(f'rmmod {mod}')
             if 'nftables' in module_config:
                 for rule in module_config['nftables']:
-                    find_remove_rule(rule)
+                    find_remove_rule('raw', 'VYOS_CT_HELPER', rule)
+                    find_remove_rule('ip6 raw', 'VYOS_CT_HELPER', rule)
         else:
             if 'ko' in module_config:
                 for mod in module_config['ko']:
                     cmd(f'modprobe {mod}')
             if 'nftables' in module_config:
                 for rule in module_config['nftables']:
-                    if not find_nftables_ct_rule(rule):
-                        cmd(f'nft insert rule ip raw VYOS_CT_HELPER {rule}')
+                    if not find_nftables_ct_rule('raw', 'VYOS_CT_HELPER', rule):
+                        cmd(f'nft insert rule raw VYOS_CT_HELPER {rule}')
+
+                    if not find_nftables_ct_rule('ip6 raw', 'VYOS_CT_HELPER', rule):
+                        cmd(f'nft insert rule ip6 raw VYOS_CT_HELPER {rule}')
 
     # Load new nftables ruleset
-    cmd(f'nft -f {nftables_ct_file}')
+    install_result, output = rc_cmd(f'nft -f {nftables_ct_file}')
+    if install_result == 1:
+        raise ConfigError(f'Failed to apply configuration: {output}')
 
     if process_named_running('conntrackd'):
         # Reload conntrack-sync daemon to fetch new sysctl values
diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py
index 79b605ffb..daad9186e 100755
--- a/src/conf_mode/container.py
+++ b/src/conf_mode/container.py
@@ -178,6 +178,11 @@ def verify(container):
                     if 'value' not in cfg:
                         raise ConfigError(f'Environment variable {var} has no value assigned!')
 
+            if 'label' in container_config:
+                for var, cfg in container_config['label'].items():
+                    if 'value' not in cfg:
+                        raise ConfigError(f'Label variable {var} has no value assigned!')
+
             if 'volume' in container_config:
                 for volume, volume_config in container_config['volume'].items():
                     if 'source' not in volume_config:
@@ -268,6 +273,12 @@ def generate_run_arguments(name, container_config):
         for k, v in container_config['environment'].items():
             env_opt += f" --env \"{k}={v['value']}\""
 
+    # Check/set label options "--label foo=bar"
+    label = ''
+    if 'label' in container_config:
+        for k, v in container_config['label'].items():
+            label += f" --label \"{k}={v['value']}\""
+
     hostname = ''
     if 'host_name' in container_config:
         hostname = container_config['host_name']
@@ -303,7 +314,7 @@ def generate_run_arguments(name, container_config):
 
     container_base_cmd = f'--detach --interactive --tty --replace {cap_add} ' \
                          f'--memory {memory}m --shm-size {shared_memory}m --memory-swap 0 --restart {restart} ' \
-                         f'--name {name} {hostname} {device} {port} {volume} {env_opt}'
+                         f'--name {name} {hostname} {device} {port} {volume} {env_opt} {label}'
 
     entrypoint = ''
     if 'entrypoint' in container_config:
diff --git a/src/conf_mode/dhcp_server.py b/src/conf_mode/dhcp_server.py
index c4c72aae9..ac7d95632 100755
--- a/src/conf_mode/dhcp_server.py
+++ b/src/conf_mode/dhcp_server.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2018-2022 VyOS maintainers and contributors
+# Copyright (C) 2018-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -34,6 +34,7 @@ from vyos import airbag
 airbag.enable()
 
 config_file = '/run/dhcp-server/dhcpd.conf'
+systemd_override = r'/run/systemd/system/isc-dhcp-server.service.d/10-override.conf'
 
 def dhcp_slice_range(exclude_list, range_dict):
     """
@@ -295,6 +296,7 @@ def generate(dhcp):
     # render the "real" configuration
     render(config_file, 'dhcp-server/dhcpd.conf.j2', dhcp,
            formater=lambda _: _.replace(""", '"'))
+    render(systemd_override, 'dhcp-server/10-override.conf.j2', dhcp)
 
     # Clean up configuration test file
     if os.path.exists(tmp_file):
@@ -303,6 +305,7 @@ def generate(dhcp):
     return None
 
 def apply(dhcp):
+    call('systemctl daemon-reload')
     # bail out early - looks like removal from running config
     if not dhcp or 'disable' in dhcp:
         call('systemctl stop isc-dhcp-server.service')
diff --git a/src/conf_mode/dns_dynamic.py b/src/conf_mode/dns_dynamic.py
index ab80defe8..4b1aed742 100755
--- a/src/conf_mode/dns_dynamic.py
+++ b/src/conf_mode/dns_dynamic.py
@@ -104,7 +104,7 @@ def generate(dyndns):
     if not dyndns or 'address' not in dyndns:
         return None
 
-    render(config_file, 'dns-dynamic/ddclient.conf.j2', dyndns)
+    render(config_file, 'dns-dynamic/ddclient.conf.j2', dyndns, permission=0o600)
     render(systemd_override, 'dns-dynamic/override.conf.j2', dyndns)
     return None
 
diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py
index c3b1ee015..769cc598f 100755
--- a/src/conf_mode/firewall.py
+++ b/src/conf_mode/firewall.py
@@ -26,7 +26,7 @@ from vyos.config import Config
 from vyos.configdict import node_changed
 from vyos.configdiff import get_config_diff, Diff
 from vyos.configdep import set_dependents, call_dependents
-# from vyos.configverify import verify_interface_exists
+from vyos.configverify import verify_interface_exists
 from vyos.firewall import fqdn_config_parse
 from vyos.firewall import geoip_update
 from vyos.template import render
@@ -38,6 +38,7 @@ from vyos.utils.process import process_named_running
 from vyos.utils.process import rc_cmd
 from vyos import ConfigError
 from vyos import airbag
+
 airbag.enable()
 
 nat_conf_script = 'nat.py'
@@ -100,7 +101,7 @@ def geoip_updated(conf, firewall):
         elif (path[0] == 'ipv6'):
             set_name = f'GEOIP_CC6_{path[1]}_{path[2]}_{path[4]}'
             out['ipv6_name'].append(set_name)
-            
+
         updated = True
 
     if 'delete' in node_diff:
@@ -140,6 +141,14 @@ def get_config(config=None):
 
     fqdn_config_parse(firewall)
 
+    firewall['flowtable_enabled'] = False
+    flow_offload = dict_search_args(firewall, 'global_options', 'flow_offload')
+    if flow_offload and 'disable' not in flow_offload:
+        for offload_type in ('software', 'hardware'):
+            if dict_search_args(flow_offload, offload_type, 'interface'):
+                firewall['flowtable_enabled'] = True
+                break
+
     return firewall
 
 def verify_rule(firewall, rule_conf, ipv6):
@@ -327,6 +336,14 @@ def verify(firewall):
                         for rule_id, rule_conf in name_conf['rule'].items():
                             verify_rule(firewall, rule_conf, True)
 
+    # Verify flow offload options
+    flow_offload = dict_search_args(firewall, 'global_options', 'flow_offload')
+    for offload_type in ('software', 'hardware'):
+        interfaces = dict_search_args(flow_offload, offload_type, 'interface') or []
+        for interface in interfaces:
+            # nft will raise an error when adding a non-existent interface to a flowtable
+            verify_interface_exists(interface)
+
     return None
 
 def generate(firewall):
@@ -336,13 +353,15 @@ def generate(firewall):
     # Determine if conntrack is needed
     firewall['ipv4_conntrack_action'] = 'return'
     firewall['ipv6_conntrack_action'] = 'return'
-
-    for rules, path in dict_search_recursive(firewall, 'rule'):
-        if any(('state' in rule_conf or 'connection_status' in rule_conf) for rule_conf in rules.values()):
-            if path[0] == 'ipv4':
-                firewall['ipv4_conntrack_action'] = 'accept'
-            elif path[0] == 'ipv6':
-                firewall['ipv6_conntrack_action'] = 'accept'
+    if firewall['flowtable_enabled']:  # Netfilter's flowtable offload requires conntrack
+        firewall['ipv4_conntrack_action'] = 'accept'
+        firewall['ipv6_conntrack_action'] = 'accept'
+    else:  # Check if conntrack is needed by firewall rules
+        for proto in ('ipv4', 'ipv6'):
+            for rules, _ in dict_search_recursive(firewall.get(proto, {}), 'rule'):
+                if any(('state' in rule_conf or 'connection_status' in rule_conf) for rule_conf in rules.values()):
+                    firewall[f'{proto}_conntrack_action'] = 'accept'
+                    break
 
     render(nftables_conf, 'firewall/nftables.j2', firewall)
     return None
diff --git a/src/conf_mode/high-availability.py b/src/conf_mode/high-availability.py
index 626a3757e..70f43ab52 100755
--- a/src/conf_mode/high-availability.py
+++ b/src/conf_mode/high-availability.py
@@ -15,6 +15,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import os
+import time
+
 from sys import exit
 from ipaddress import ip_interface
 from ipaddress import IPv4Interface
@@ -22,15 +25,21 @@ from ipaddress import IPv6Interface
 
 from vyos.base import Warning
 from vyos.config import Config
+from vyos.configdict import leaf_node_changed
 from vyos.ifconfig.vrrp import VRRP
 from vyos.template import render
 from vyos.template import is_ipv4
 from vyos.template import is_ipv6
+from vyos.utils.network import is_ipv6_tentative
 from vyos.utils.process import call
 from vyos import ConfigError
 from vyos import airbag
 airbag.enable()
 
+
+systemd_override = r'/run/systemd/system/keepalived.service.d/10-override.conf'
+
+
 def get_config(config=None):
     if config:
         conf = config
@@ -50,6 +59,9 @@ def get_config(config=None):
     if conf.exists(conntrack_path):
         ha['conntrack_sync_group'] = conf.return_value(conntrack_path)
 
+    if leaf_node_changed(conf, base + ['vrrp', 'disable-snmp']):
+        ha.update({'restart_required': {}})
+
     return ha
 
 def verify(ha):
@@ -160,18 +172,38 @@ def verify(ha):
 
 def generate(ha):
     if not ha or 'disable' in ha:
+        if os.path.isfile(systemd_override):
+            os.unlink(systemd_override)
         return None
 
     render(VRRP.location['config'], 'high-availability/keepalived.conf.j2', ha)
+    render(systemd_override, 'high-availability/10-override.conf.j2', ha)
     return None
 
 def apply(ha):
     service_name = 'keepalived.service'
+    call('systemctl daemon-reload')
     if not ha or 'disable' in ha:
         call(f'systemctl stop {service_name}')
         return None
 
-    call(f'systemctl reload-or-restart {service_name}')
+    # Check if IPv6 address is tentative T5533
+    for group, group_config in ha.get('vrrp', {}).get('group', {}).items():
+        if 'hello_source_address' in group_config:
+            if is_ipv6(group_config['hello_source_address']):
+                ipv6_address = group_config['hello_source_address']
+                interface = group_config['interface']
+                checks = 20
+                interval = 0.1
+                for _ in range(checks):
+                    if is_ipv6_tentative(interface, ipv6_address):
+                        time.sleep(interval)
+
+    systemd_action = 'reload-or-restart'
+    if 'restart_required' in ha:
+        systemd_action = 'restart'
+
+    call(f'systemctl {systemd_action} {service_name}')
     return None
 
 if __name__ == '__main__':
diff --git a/src/conf_mode/interfaces-dummy.py b/src/conf_mode/interfaces-dummy.py
index e771581e1..db768b94d 100755
--- a/src/conf_mode/interfaces-dummy.py
+++ b/src/conf_mode/interfaces-dummy.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2019-2021 VyOS maintainers and contributors
+# Copyright (C) 2019-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -55,7 +55,7 @@ def generate(dummy):
     return None
 
 def apply(dummy):
-    d = DummyIf(dummy['ifname'])
+    d = DummyIf(**dummy)
 
     # Remove dummy interface
     if 'deleted' in dummy:
diff --git a/src/conf_mode/interfaces-ethernet.py b/src/conf_mode/interfaces-ethernet.py
index b015bba88..f3e65ad5e 100755
--- a/src/conf_mode/interfaces-ethernet.py
+++ b/src/conf_mode/interfaces-ethernet.py
@@ -186,14 +186,15 @@ def generate(ethernet):
 
         if 'ca_certificate' in ethernet['eapol']:
             ca_cert_file_path = os.path.join(cfg_dir, f'{ifname}_ca.pem')
-            ca_cert_name = ethernet['eapol']['ca_certificate']
-            pki_ca_cert = ethernet['pki']['ca'][ca_cert_name]
+            ca_chains = []
 
-            loaded_ca_cert = load_certificate(pki_ca_cert['certificate'])
-            ca_full_chain = find_chain(loaded_ca_cert, loaded_ca_certs)
+            for ca_cert_name in ethernet['eapol']['ca_certificate']:
+                pki_ca_cert = ethernet['pki']['ca'][ca_cert_name]
+                loaded_ca_cert = load_certificate(pki_ca_cert['certificate'])
+                ca_full_chain = find_chain(loaded_ca_cert, loaded_ca_certs)
+                ca_chains.append('\n'.join(encode_certificate(c) for c in ca_full_chain))
 
-            write_file(ca_cert_file_path,
-                       '\n'.join(encode_certificate(c) for c in ca_full_chain))
+            write_file(ca_cert_file_path, '\n'.join(ca_chains))
 
     return None
 
diff --git a/src/conf_mode/interfaces-openvpn.py b/src/conf_mode/interfaces-openvpn.py
index 1d0feb56f..9f4de990c 100755
--- a/src/conf_mode/interfaces-openvpn.py
+++ b/src/conf_mode/interfaces-openvpn.py
@@ -344,9 +344,6 @@ def verify(openvpn):
             if v6_subnets > 1:
                 raise ConfigError('Cannot specify more than 1 IPv6 server subnet')
 
-            if v6_subnets > 0 and v4_subnets == 0:
-                raise ConfigError('IPv6 server requires an IPv4 server subnet')
-
             for subnet in tmp:
                 if is_ipv4(subnet):
                     subnet = IPv4Network(subnet)
@@ -388,6 +385,10 @@ def verify(openvpn):
                         for v4PoolNet in v4PoolNets:
                             if IPv4Address(client['ip'][0]) in v4PoolNet:
                                 print(f'Warning: Client "{client["name"]}" IP {client["ip"][0]} is in server IP pool, it is not reserved for this client.')
+            # configuring a client_ip_pool will set 'server ... nopool' which is currently incompatible with 'server-ipv6' (probably to be fixed upstream)
+            for subnet in (dict_search('server.subnet', openvpn) or []):
+                if is_ipv6(subnet):
+                    raise ConfigError(f'Setting client-ip-pool is incompatible having an IPv6 server subnet.')
 
         for subnet in (dict_search('server.subnet', openvpn) or []):
             if is_ipv6(subnet):
diff --git a/src/conf_mode/interfaces-vxlan.py b/src/conf_mode/interfaces-vxlan.py
index a3b0867e0..05f68112a 100755
--- a/src/conf_mode/interfaces-vxlan.py
+++ b/src/conf_mode/interfaces-vxlan.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2019-2022 VyOS maintainers and contributors
+# Copyright (C) 2019-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -24,6 +24,7 @@ from vyos.config import Config
 from vyos.configdict import get_interface_dict
 from vyos.configdict import leaf_node_changed
 from vyos.configdict import is_node_changed
+from vyos.configdict import node_changed
 from vyos.configverify import verify_address
 from vyos.configverify import verify_bridge_delete
 from vyos.configverify import verify_mtu_ipv6
@@ -58,6 +59,9 @@ def get_config(config=None):
             vxlan.update({'rebuild_required': {}})
             break
 
+    tmp = node_changed(conf, base + [ifname, 'vlan-to-vni'], recursive=True)
+    if tmp: vxlan.update({'vlan_to_vni_removed': tmp})
+
     # We need to verify that no other VXLAN tunnel is configured when external
     # mode is in use - Linux Kernel limitation
     conf.set_level(base)
@@ -146,6 +150,20 @@ def verify(vxlan):
                     raise ConfigError(error_msg)
                 protocol = 'ipv4'
 
+    if 'vlan_to_vni' in vxlan:
+        if 'is_bridge_member' not in vxlan:
+            raise ConfigError('VLAN to VNI mapping requires that VXLAN interface '\
+                              'is member of a bridge interface!')
+
+        vnis_used = []
+        for vif, vif_config in vxlan['vlan_to_vni'].items():
+            if 'vni' not in vif_config:
+                raise ConfigError(f'Must define VNI for VLAN "{vif}"!')
+            vni = vif_config['vni']
+            if vni in vnis_used:
+                raise ConfigError(f'VNI "{vni}" is already assigned to a different VLAN!')
+            vnis_used.append(vni)
+
     verify_mtu_ipv6(vxlan)
     verify_address(vxlan)
     verify_bond_bridge_member(vxlan)
diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py
index 9da7fbe80..e37a7011c 100755
--- a/src/conf_mode/nat.py
+++ b/src/conf_mode/nat.py
@@ -112,7 +112,7 @@ def verify_rule(config, err_msg, groups_dict):
                         group_obj = dict_search_args(groups_dict, group, group_name)
 
                         if group_obj is None:
-                            raise ConfigError(f'Invalid {error_group} "{group_name}" on firewall rule')
+                            raise ConfigError(f'Invalid {error_group} "{group_name}" on nat rule')
 
                         if not group_obj:
                             Warning(f'{error_group} "{group_name}" has no members!')
@@ -195,11 +195,10 @@ def verify(nat):
     if dict_search('source.rule', nat):
         for rule, config in dict_search('source.rule', nat).items():
             err_msg = f'Source NAT configuration error in rule {rule}:'
-            if 'outbound_interface' not in config:
-                raise ConfigError(f'{err_msg} outbound-interface not specified')
 
-            if config['outbound_interface'] not in 'any' and config['outbound_interface'] not in interfaces():
-                Warning(f'rule "{rule}" interface "{config["outbound_interface"]}" does not exist on this system')
+            if 'outbound_interface' in config:
+                if config['outbound_interface'] not in 'any' and config['outbound_interface'] not in interfaces():
+                    Warning(f'rule "{rule}" interface "{config["outbound_interface"]}" does not exist on this system')
 
             if not dict_search('translation.address', config) and not dict_search('translation.port', config):
                 if 'exclude' not in config and 'backend' not in config['load_balance']:
@@ -218,11 +217,9 @@ def verify(nat):
         for rule, config in dict_search('destination.rule', nat).items():
             err_msg = f'Destination NAT configuration error in rule {rule}:'
 
-            if 'inbound_interface' not in config:
-                raise ConfigError(f'{err_msg}\n' \
-                                  'inbound-interface not specified')
-            elif config['inbound_interface'] not in 'any' and config['inbound_interface'] not in interfaces():
-                Warning(f'rule "{rule}" interface "{config["inbound_interface"]}" does not exist on this system')
+            if 'inbound_interface' in config:
+                if config['inbound_interface'] not in 'any' and config['inbound_interface'] not in interfaces():
+                    Warning(f'rule "{rule}" interface "{config["inbound_interface"]}" does not exist on this system')
 
             if not dict_search('translation.address', config) and not dict_search('translation.port', config) and 'redirect' not in config['translation']:
                 if 'exclude' not in config and 'backend' not in config['load_balance']:
diff --git a/src/conf_mode/protocols_igmp.py b/src/conf_mode/protocols_igmp.py
index f6097e282..435189025 100755
--- a/src/conf_mode/protocols_igmp.py
+++ b/src/conf_mode/protocols_igmp.py
@@ -102,7 +102,7 @@ def verify(igmp):
         # Check, is this multicast group
         for intfc in igmp['ifaces']:
             for gr_addr in igmp['ifaces'][intfc]['gr_join']:
-                if IPv4Address(gr_addr) < IPv4Address('224.0.0.0'):
+                if not IPv4Address(gr_addr).is_multicast:
                     raise ConfigError(gr_addr + " not a multicast group")
 
 def generate(igmp):
diff --git a/src/conf_mode/protocols_pim6.py b/src/conf_mode/protocols_pim6.py
new file mode 100755
index 000000000..6a1235ba5
--- /dev/null
+++ b/src/conf_mode/protocols_pim6.py
@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2023 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from ipaddress import IPv6Address
+from sys import exit
+from typing import Optional
+
+from vyos import ConfigError, airbag, frr
+from vyos.config import Config, ConfigDict
+from vyos.configdict import node_changed
+from vyos.configverify import verify_interface_exists
+from vyos.template import render_to_string
+
+airbag.enable()
+
+
+def get_config(config=None):
+    if config:
+        conf = config
+    else:
+        conf = Config()
+    base = ['protocols', 'pim6']
+    pim6 = conf.get_config_dict(base, key_mangling=('-', '_'),
+                                 get_first_key=True, with_recursive_defaults=True)
+
+    # FRR has VRF support for different routing daemons. As interfaces belong
+    # to VRFs - or the global VRF, we need to check for changed interfaces so
+    # that they will be properly rendered for the FRR config. Also this eases
+    # removal of interfaces from the running configuration.
+    interfaces_removed = node_changed(conf, base + ['interface'])
+    if interfaces_removed:
+        pim6['interface_removed'] = list(interfaces_removed)
+
+    return pim6
+
+
+def verify(pim6):
+    if pim6 is None:
+        return
+
+    for interface, interface_config in pim6.get('interface', {}).items():
+        verify_interface_exists(interface)
+        if 'mld' in interface_config:
+            mld = interface_config['mld']
+            for group in mld.get('join', {}).keys():
+                # Validate multicast group address
+                if not IPv6Address(group).is_multicast:
+                    raise ConfigError(f"{group} is not a multicast group")
+
+
+def generate(pim6):
+    if pim6 is None:
+        return
+
+    pim6['new_frr_config'] = render_to_string('frr/pim6d.frr.j2', pim6)
+
+
+def apply(pim6):
+    if pim6 is None:
+        return
+
+    pim6_daemon = 'pim6d'
+
+    # Save original configuration prior to starting any commit actions
+    frr_cfg = frr.FRRConfig()
+
+    frr_cfg.load_configuration(pim6_daemon)
+
+    for key in ['interface', 'interface_removed']:
+        if key not in pim6:
+            continue
+        for interface in pim6[key]:
+            frr_cfg.modify_section(
+                f'^interface {interface}', stop_pattern='^exit', remove_stop_mark=True)
+
+    if 'new_frr_config' in pim6:
+        frr_cfg.add_before(frr.default_add_before, pim6['new_frr_config'])
+    frr_cfg.commit_configuration(pim6_daemon)
+
+
+if __name__ == '__main__':
+    try:
+        c = get_config()
+        verify(c)
+        generate(c)
+        apply(c)
+    except ConfigError as e:
+        print(e)
+        exit(1)
diff --git a/src/conf_mode/system-ip.py b/src/conf_mode/system-ip.py
index 5e4e5ec28..7612e2c0d 100755
--- a/src/conf_mode/system-ip.py
+++ b/src/conf_mode/system-ip.py
@@ -20,10 +20,12 @@ from vyos.config import Config
 from vyos.configdict import dict_merge
 from vyos.configverify import verify_route_map
 from vyos.template import render_to_string
-from vyos.utils.process import call
 from vyos.utils.dict import dict_search
 from vyos.utils.file import write_file
+from vyos.utils.process import call
+from vyos.utils.process import is_systemd_service_active
 from vyos.utils.system import sysctl_write
+
 from vyos import ConfigError
 from vyos import frr
 from vyos import airbag
@@ -115,16 +117,20 @@ def apply(opt):
     value = '48' if (tmp is None) else tmp
     sysctl_write('net.ipv4.tcp_mtu_probe_floor', value)
 
-    zebra_daemon = 'zebra'
-    # Save original configuration prior to starting any commit actions
-    frr_cfg = frr.FRRConfig()
-
-    # The route-map used for the FIB (zebra) is part of the zebra daemon
-    frr_cfg.load_configuration(zebra_daemon)
-    frr_cfg.modify_section(r'ip protocol \w+ route-map [-a-zA-Z0-9.]+', stop_pattern='(\s|!)')
-    if 'frr_zebra_config' in opt:
-        frr_cfg.add_before(frr.default_add_before, opt['frr_zebra_config'])
-    frr_cfg.commit_configuration(zebra_daemon)
+    # During startup of vyos-router that brings up FRR, the service is not yet
+    # running when this script is called first. Skip this part and wait for initial
+    # commit of the configuration to trigger this statement
+    if is_systemd_service_active('frr.service'):
+        zebra_daemon = 'zebra'
+        # Save original configuration prior to starting any commit actions
+        frr_cfg = frr.FRRConfig()
+
+        # The route-map used for the FIB (zebra) is part of the zebra daemon
+        frr_cfg.load_configuration(zebra_daemon)
+        frr_cfg.modify_section(r'ip protocol \w+ route-map [-a-zA-Z0-9.]+', stop_pattern='(\s|!)')
+        if 'frr_zebra_config' in opt:
+            frr_cfg.add_before(frr.default_add_before, opt['frr_zebra_config'])
+        frr_cfg.commit_configuration(zebra_daemon)
 
 if __name__ == '__main__':
     try:
diff --git a/src/conf_mode/system-ipv6.py b/src/conf_mode/system-ipv6.py
index e40ed38e2..90a1a8087 100755
--- a/src/conf_mode/system-ipv6.py
+++ b/src/conf_mode/system-ipv6.py
@@ -22,8 +22,9 @@ from vyos.configdict import dict_merge
 from vyos.configverify import verify_route_map
 from vyos.template import render_to_string
 from vyos.utils.dict import dict_search
-from vyos.utils.system import sysctl_write
 from vyos.utils.file import write_file
+from vyos.utils.process import is_systemd_service_active
+from vyos.utils.system import sysctl_write
 from vyos import ConfigError
 from vyos import frr
 from vyos import airbag
@@ -93,16 +94,20 @@ def apply(opt):
             if name == 'accept_dad':
                 write_file(os.path.join(root, name), value)
 
-    zebra_daemon = 'zebra'
-    # Save original configuration prior to starting any commit actions
-    frr_cfg = frr.FRRConfig()
+    # During startup of vyos-router that brings up FRR, the service is not yet
+    # running when this script is called first. Skip this part and wait for initial
+    # commit of the configuration to trigger this statement
+    if is_systemd_service_active('frr.service'):
+        zebra_daemon = 'zebra'
+        # Save original configuration prior to starting any commit actions
+        frr_cfg = frr.FRRConfig()
 
-    # The route-map used for the FIB (zebra) is part of the zebra daemon
-    frr_cfg.load_configuration(zebra_daemon)
-    frr_cfg.modify_section(r'ipv6 protocol \w+ route-map [-a-zA-Z0-9.]+', stop_pattern='(\s|!)')
-    if 'frr_zebra_config' in opt:
-        frr_cfg.add_before(frr.default_add_before, opt['frr_zebra_config'])
-    frr_cfg.commit_configuration(zebra_daemon)
+        # The route-map used for the FIB (zebra) is part of the zebra daemon
+        frr_cfg.load_configuration(zebra_daemon)
+        frr_cfg.modify_section(r'ipv6 protocol \w+ route-map [-a-zA-Z0-9.]+', stop_pattern='(\s|!)')
+        if 'frr_zebra_config' in opt:
+            frr_cfg.add_before(frr.default_add_before, opt['frr_zebra_config'])
+        frr_cfg.commit_configuration(zebra_daemon)
 
 if __name__ == '__main__':
     try:
diff --git a/src/conf_mode/system_frr.py b/src/conf_mode/system_frr.py
index fb252238a..d8224b3c3 100755
--- a/src/conf_mode/system_frr.py
+++ b/src/conf_mode/system_frr.py
@@ -22,17 +22,14 @@ from vyos import airbag
 from vyos.config import Config
 from vyos.logger import syslog
 from vyos.template import render_to_string
+from vyos.utils.boot import boot_configuration_complete
 from vyos.utils.file import read_file
 from vyos.utils.file import write_file
-from vyos.utils.process import run
+from vyos.utils.process import call
 airbag.enable()
 
 # path to daemons config and config status files
 config_file = '/etc/frr/daemons'
-vyos_status_file = '/tmp/vyos-config-status'
-# path to watchfrr for FRR control
-watchfrr = '/usr/lib/frr/watchfrr.sh'
-
 
 def get_config(config=None):
     if config:
@@ -45,12 +42,10 @@ def get_config(config=None):
 
     return frr_config
 
-
 def verify(frr_config):
     # Nothing to verify here
     pass
 
-
 def generate(frr_config):
     # read daemons config file
     daemons_config_current = read_file(config_file)
@@ -62,25 +57,21 @@ def generate(frr_config):
         write_file(config_file, daemons_config_new)
         frr_config['config_file_changed'] = True
 
-
 def apply(frr_config):
-    # check if this is initial commit during boot or intiated by CLI
-    # if the file exists, this must be CLI commit
-    commit_type_cli = Path(vyos_status_file).exists()
     # display warning to user
-    if commit_type_cli and frr_config.get('config_file_changed'):
+    if boot_configuration_complete() and frr_config.get('config_file_changed'):
         # Since FRR restart is not safe thing, better to give
         # control over this to users
         print('''
         You need to reboot a router (preferred) or restart FRR
         to apply changes in modules settings
         ''')
-    # restart FRR automatically. DUring the initial boot this should be
-    # safe in most cases
-    if not commit_type_cli and frr_config.get('config_file_changed'):
-        syslog.warning('Restarting FRR to apply changes in modules')
-        run(f'{watchfrr} restart')
 
+    # restart FRR automatically
+    # During initial boot this should be safe in most cases
+    if not boot_configuration_complete() and frr_config.get('config_file_changed'):
+        syslog.warning('Restarting FRR to apply changes in modules')
+        call(f'systemctl restart frr.service')
 
 if __name__ == '__main__':
     try: