24 files changed, 169 insertions, 78 deletions

diff --git a/interface-definitions/policy.xml.in b/interface-definitions/policy.xml.in
index cc1de609d..e794c4b90 100644
--- a/interface-definitions/policy.xml.in
+++ b/interface-definitions/policy.xml.in
@@ -392,7 +392,7 @@
                     <description>Prefix to match against</description>
                   </valueHelp>
                   <constraint>
-                    <validator name="ip-prefix"/>
+                    <validator name="ipv4-prefix"/>
                   </constraint>
                 </properties>
               </leafNode>
diff --git a/op-mode-definitions/show-conntrack.xml.in b/op-mode-definitions/show-conntrack.xml.in
index 8d921e6a5..4cdcffcdb 100644
--- a/op-mode-definitions/show-conntrack.xml.in
+++ b/op-mode-definitions/show-conntrack.xml.in
@@ -7,6 +7,12 @@
           <help>Show conntrack tables entries</help>
         </properties>
         <children>
+          <node name="statistics">
+            <properties>
+              <help>Show conntrack statistics</help>
+            </properties>
+            <command>sudo ${vyos_op_scripts_dir}/conntrack.py show_statistics</command>
+          </node>
           <node name="table">
             <properties>
               <help>Show conntrack entries for table</help>
diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py
index 09ae73eac..6894fc4da 100644
--- a/python/vyos/defaults.py
+++ b/python/vyos/defaults.py
@@ -26,7 +26,7 @@ directories = {
   "templates": "/usr/share/vyos/templates/",
   "certbot": "/config/auth/letsencrypt",
   "api_schema": "/usr/libexec/vyos/services/api/graphql/graphql/schema/",
-  "api_templates": "/usr/libexec/vyos/services/api/graphql/recipes/templates/",
+  "api_templates": "/usr/libexec/vyos/services/api/graphql/session/templates/",
   "vyos_udev_dir": "/run/udev/vyos"
 }
 
diff --git a/python/vyos/ifconfig/bridge.py b/python/vyos/ifconfig/bridge.py
index 758967fbc..aa818bc5f 100644
--- a/python/vyos/ifconfig/bridge.py
+++ b/python/vyos/ifconfig/bridge.py
@@ -295,8 +295,24 @@ class BridgeIf(Interface):
                 self.del_port(member)
 
         # enable/disable Vlan Filter
-        vlan_filter = '1' if 'enable_vlan' in config else '0'
-        self.set_vlan_filter(vlan_filter)
+        tmp = '1' if 'enable_vlan' in config else '0'
+        self.set_vlan_filter(tmp)
+
+        # add VLAN interfaces to local 'parent' bridge to allow forwarding
+        if 'enable_vlan' in config:
+            for vlan in config.get('vif_remove', {}):
+                # Remove old VLANs from the bridge
+                cmd = f'bridge vlan del dev {self.ifname} vid {vlan} self'
+                self._cmd(cmd)
+
+            for vlan in config.get('vif', {}):
+                cmd = f'bridge vlan add dev {self.ifname} vid {vlan} self'
+                self._cmd(cmd)
+
+            # VLAN of bridge parent interface is always 1. VLAN 1 is the default
+            # VLAN for all unlabeled packets
+            cmd = f'bridge vlan add dev {self.ifname} vid 1 pvid untagged self'
+            self._cmd(cmd)
 
         tmp = dict_search('member.interface', config)
         if tmp:
diff --git a/smoketest/scripts/cli/test_interfaces_bridge.py b/smoketest/scripts/cli/test_interfaces_bridge.py
index 8f711af20..6d7af78eb 100755
--- a/smoketest/scripts/cli/test_interfaces_bridge.py
+++ b/smoketest/scripts/cli/test_interfaces_bridge.py
@@ -19,6 +19,7 @@ import json
 import unittest
 
 from base_interfaces_test import BasicInterfaceTest
+from copy import deepcopy
 from glob import glob
 from netifaces import interfaces
 
@@ -224,85 +225,78 @@ class BridgeInterfaceTest(BasicInterfaceTest.TestCase):
         super().test_vif_8021q_mtu_limits()
 
     def test_bridge_vlan_filter(self):
-        def _verify_members() -> None:
-            # check member interfaces are added on the bridge
-            for interface in self._interfaces:
-                bridge_members = []
-                for tmp in glob(f'/sys/class/net/{interface}/lower_*'):
-                    bridge_members.append(os.path.basename(tmp).replace('lower_', ''))
-
-                # We can not use assertListEqual() b/c the position of the interface
-                # names within the list is not fixed
-                self.assertEqual(len(self._members), len(bridge_members))
-                for member in self._members:
-                    self.assertIn(member, bridge_members)
-
-        def _check_vlan_filter() -> None:
-            for interface in self._interfaces:
-                tmp = cmd(f'bridge -j vlan show dev {interface}')
-                tmp = json.loads(tmp)
-                self.assertIsNotNone(tmp)
-
-                for interface_status in tmp:
-                    ifname = interface_status['ifname']
-                    for interface in self._members:
-                        vlan_success = 0;
-                        if interface == ifname:
-                            vlans_status = interface_status['vlans']
-                            for vlan_status in vlans_status:
-                                vlan_id = vlan_status['vlan']
-                                flag_num = 0
-                                if 'flags' in vlan_status:
-                                    flags = vlan_status['flags']
-                                    for flag in flags:
-                                        flag_num = flag_num +1
-                                if vlan_id == 2:
-                                    if flag_num == 0:
-                                        vlan_success = vlan_success + 1
-                                else:
-                                    for id in range(4,10):
-                                        if vlan_id == id:
-                                            if flag_num == 0:
-                                                vlan_success = vlan_success + 1
-                                    if vlan_id >= 101:
-                                        if flag_num == 2:
-                                            vlan_success = vlan_success + 1
-                            self.assertGreaterEqual(vlan_success, 7)
-
-        vif_vlan = 2
+        vifs = ['10', '20', '30', '40']
+        native_vlan = '20'
+
         # Add member interface to bridge and set VLAN filter
         for interface in self._interfaces:
             base = self._base_path + [interface]
             self.cli_set(base + ['enable-vlan'])
             self.cli_set(base + ['address', '192.0.2.1/24'])
-            self.cli_set(base + ['vif', str(vif_vlan), 'address', '192.0.3.1/24'])
-            self.cli_set(base + ['vif', str(vif_vlan), 'mtu', self._mtu])
 
-            vlan_id = 101
-            allowed_vlan = 2
-            allowed_vlan_range = '4-9'
-            # assign members to bridge interface
+            for vif in vifs:
+                self.cli_set(base + ['vif', vif, 'address', f'192.0.{vif}.1/24'])
+                self.cli_set(base + ['vif', vif, 'mtu', self._mtu])
+
             for member in self._members:
                 base_member = base + ['member', 'interface', member]
-                self.cli_set(base_member + ['allowed-vlan', str(allowed_vlan)])
-                self.cli_set(base_member + ['allowed-vlan', allowed_vlan_range])
-                self.cli_set(base_member + ['native-vlan', str(vlan_id)])
-                vlan_id += 1
+                self.cli_set(base_member + ['native-vlan', native_vlan])
+                for vif in vifs:
+                    self.cli_set(base_member + ['allowed-vlan', vif])
 
         # commit config
         self.cli_commit()
 
+        def _verify_members(interface, members) -> None:
+            # check member interfaces are added on the bridge
+            bridge_members = []
+            for tmp in glob(f'/sys/class/net/{interface}/lower_*'):
+                bridge_members.append(os.path.basename(tmp).replace('lower_', ''))
+
+            self.assertListEqual(sorted(members), sorted(bridge_members))
+
+        def _check_vlan_filter(interface, vifs) -> None:
+            configured_vlan_ids = []
+
+            bridge_json = cmd(f'bridge -j vlan show dev {interface}')
+            bridge_json = json.loads(bridge_json)
+            self.assertIsNotNone(bridge_json)
+
+            for tmp in bridge_json:
+                self.assertIn('vlans', tmp)
+
+                for vlan in tmp['vlans']:
+                    self.assertIn('vlan', vlan)
+                    configured_vlan_ids.append(str(vlan['vlan']))
+
+                    # Verify native VLAN ID has 'PVID' flag set on individual member ports
+                    if not interface.startswith('br') and str(vlan['vlan']) == native_vlan:
+                        self.assertIn('flags', vlan)
+                        self.assertIn('PVID', vlan['flags'])
+
+            self.assertListEqual(sorted(configured_vlan_ids), sorted(vifs))
+
         # Verify correct setting of VLAN filter function
         for interface in self._interfaces:
             tmp = read_file(f'/sys/class/net/{interface}/bridge/vlan_filtering')
             self.assertEqual(tmp, '1')
 
-        # Execute the program to obtain status information and verify proper
-        # VLAN filter setup
-        _check_vlan_filter()
+        # Obtain status information and verify proper VLAN filter setup.
+        # First check if all members are present, second check if all VLANs
+        # are assigned on the parend bridge interface, third verify all the
+        # VLANs are properly setup on the downstream "member" ports
+        for interface in self._interfaces:
+            # check member interfaces are added on the bridge
+            _verify_members(interface, self._members)
 
-        # check member interfaces are added on the bridge
-        _verify_members()
+            # Check if all VLAN ids are properly set up. Bridge interface always
+            # has native VLAN 1
+            tmp = deepcopy(vifs)
+            tmp.append('1')
+            _check_vlan_filter(interface, tmp)
+
+            for member in self._members:
+                _check_vlan_filter(member, vifs)
 
         # change member interface description to trigger config update,
         # VLANs must still exist (T4565)
@@ -313,12 +307,22 @@ class BridgeInterfaceTest(BasicInterfaceTest.TestCase):
         # commit config
         self.cli_commit()
 
-        # check member interfaces are added on the bridge
-        _verify_members()
+        # Obtain status information and verify proper VLAN filter setup.
+        # First check if all members are present, second check if all VLANs
+        # are assigned on the parend bridge interface, third verify all the
+        # VLANs are properly setup on the downstream "member" ports
+        for interface in self._interfaces:
+            # check member interfaces are added on the bridge
+            _verify_members(interface, self._members)
+
+            # Check if all VLAN ids are properly set up. Bridge interface always
+            # has native VLAN 1
+            tmp = deepcopy(vifs)
+            tmp.append('1')
+            _check_vlan_filter(interface, tmp)
 
-        # Execute the program to obtain status information and verify proper
-        # VLAN filter setup
-        _check_vlan_filter()
+            for member in self._members:
+                _check_vlan_filter(member, vifs)
 
         # delete all members
         for interface in self._interfaces:
@@ -337,7 +341,6 @@ class BridgeInterfaceTest(BasicInterfaceTest.TestCase):
             for member in self._members:
                 self.assertNotIn(member, bridge_members)
 
-
     def test_bridge_vif_members(self):
         # T2945: ensure that VIFs are not dropped from bridge
         vifs = ['300', '400']
diff --git a/src/conf_mode/https.py b/src/conf_mode/https.py
index 3057357fc..7cd7ea42e 100755
--- a/src/conf_mode/https.py
+++ b/src/conf_mode/https.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2019-2021 VyOS maintainers and contributors
+# Copyright (C) 2019-2022 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -29,6 +29,8 @@ from vyos.pki import wrap_certificate
 from vyos.pki import wrap_private_key
 from vyos.template import render
 from vyos.util import call
+from vyos.util import check_port_availability
+from vyos.util import is_listen_port_bind_service
 from vyos.util import write_file
 
 from vyos import airbag
@@ -107,6 +109,31 @@ def verify(https):
                 raise ConfigError("At least one 'virtual-host <id> server-name' "
                               "matching the 'certbot domain-name' is required.")
 
+    server_block_list = []
+
+    # organize by vhosts
+    vhost_dict = https.get('virtual-host', {})
+
+    if not vhost_dict:
+        # no specified virtual hosts (server blocks); use default
+        server_block_list.append(default_server_block)
+    else:
+        for vhost in list(vhost_dict):
+            server_block = deepcopy(default_server_block)
+            data = vhost_dict.get(vhost, {})
+            server_block['address'] = data.get('listen-address', '*')
+            server_block['port'] = data.get('listen-port', '443')
+            server_block_list.append(server_block)
+
+    for entry in server_block_list:
+        _address = entry.get('address')
+        _address = '0.0.0.0' if _address == '*' else _address
+        _port = entry.get('port')
+        proto = 'tcp'
+        if check_port_availability(_address, int(_port), proto) is not True and \
+                not is_listen_port_bind_service(int(_port), 'nginx'):
+            raise ConfigError(f'"{proto}" port "{_port}" is used by another service')
+
     verify_vrf(https)
     return None
 
diff --git a/src/op_mode/conntrack.py b/src/op_mode/conntrack.py
index 036226418..b27aa6060 100755
--- a/src/op_mode/conntrack.py
+++ b/src/op_mode/conntrack.py
@@ -51,6 +51,21 @@ def _get_raw_data(family):
     return _xml_to_dict(xml)
 
 
+def _get_raw_statistics():
+    entries = []
+    data = cmd('sudo conntrack -S')
+    data = data.replace('  \t', '').split('\n')
+    for entry in data:
+        entries.append(entry.split())
+    return entries
+
+
+def get_formatted_statistics(entries):
+    headers = ["CPU", "Found", "Invalid", "Insert", "Insert fail", "Drop", "Early drop", "Errors", "Search restart"]
+    output = tabulate(entries, headers, numalign="left")
+    return output
+
+
 def get_formatted_output(dict_data):
     """
     :param xml:
@@ -111,6 +126,14 @@ def show(raw: bool, family: str):
         return get_formatted_output(conntrack_data)
 
 
+def show_statistics(raw: bool):
+    conntrack_statistics = _get_raw_statistics()
+    if raw:
+        return conntrack_statistics
+    else:
+        return get_formatted_statistics(conntrack_statistics)
+
+
 if __name__ == '__main__':
     try:
         res = vyos.opmode.run(sys.modules[__name__])
diff --git a/src/services/api/graphql/graphql/mutations.py b/src/services/api/graphql/graphql/mutations.py
index 3e89fb239..c8ae0f516 100644
--- a/src/services/api/graphql/graphql/mutations.py
+++ b/src/services/api/graphql/graphql/mutations.py
@@ -21,7 +21,7 @@ from makefun import with_signature
 
 from .. import state
 from .. import key_auth
-from api.graphql.recipes.session import Session
+from api.graphql.session.session import Session
 
 mutation = ObjectType("Mutation")
 
@@ -71,7 +71,7 @@ def make_mutation_resolver(mutation_name, class_name, session_func):
 
             # one may override the session functions with a local subclass
             try:
-                mod = import_module(f'api.graphql.recipes.{func_base_name}')
+                mod = import_module(f'api.graphql.session.override.{func_base_name}')
                 klass = getattr(mod, class_name)
             except ImportError:
                 # otherwise, dynamically generate subclass to invoke subclass
diff --git a/src/services/api/graphql/graphql/queries.py b/src/services/api/graphql/graphql/queries.py
index f6544709e..921a66274 100644
--- a/src/services/api/graphql/graphql/queries.py
+++ b/src/services/api/graphql/graphql/queries.py
@@ -21,7 +21,7 @@ from makefun import with_signature
 
 from .. import state
 from .. import key_auth
-from api.graphql.recipes.session import Session
+from api.graphql.session.session import Session
 
 query = ObjectType("Query")
 
@@ -71,7 +71,7 @@ def make_query_resolver(query_name, class_name, session_func):
 
             # one may override the session functions with a local subclass
             try:
-                mod = import_module(f'api.graphql.recipes.{func_base_name}')
+                mod = import_module(f'api.graphql.session.override.{func_base_name}')
                 klass = getattr(mod, class_name)
             except ImportError:
                 # otherwise, dynamically generate subclass to invoke subclass
diff --git a/src/services/api/graphql/recipes/__init__.py b/src/services/api/graphql/session/__init__.py
index e69de29bb..e69de29bb 100644
--- a/src/services/api/graphql/recipes/__init__.py
+++ b/src/services/api/graphql/session/__init__.py
diff --git a/src/services/api/graphql/recipes/queries/system_status.py b/src/services/api/graphql/session/composite/system_status.py
index 8dadcc9f3..8dadcc9f3 100755
--- a/src/services/api/graphql/recipes/queries/system_status.py
+++ b/src/services/api/graphql/session/composite/system_status.py
diff --git a/src/services/api/graphql/recipes/remove_firewall_address_group_members.py b/src/services/api/graphql/session/override/remove_firewall_address_group_members.py
index b91932e14..b91932e14 100644
--- a/src/services/api/graphql/recipes/remove_firewall_address_group_members.py
+++ b/src/services/api/graphql/session/override/remove_firewall_address_group_members.py
diff --git a/src/services/api/graphql/recipes/session.py b/src/services/api/graphql/session/session.py
index ac185beb7..23bc7154c 100644
--- a/src/services/api/graphql/recipes/session.py
+++ b/src/services/api/graphql/session/session.py
@@ -149,7 +149,7 @@ class Session:
         return res
 
     def system_status(self):
-        import api.graphql.recipes.queries.system_status as system_status
+        import api.graphql.session.composite.system_status as system_status
 
         session = self._session
         data = self._data
diff --git a/src/services/api/graphql/recipes/templates/create_dhcp_server.tmpl b/src/services/api/graphql/session/templates/create_dhcp_server.tmpl
index 70de43183..70de43183 100644
--- a/src/services/api/graphql/recipes/templates/create_dhcp_server.tmpl
+++ b/src/services/api/graphql/session/templates/create_dhcp_server.tmpl
diff --git a/src/services/api/graphql/recipes/templates/create_firewall_address_group.tmpl b/src/services/api/graphql/session/templates/create_firewall_address_group.tmpl
index a890d0086..a890d0086 100644
--- a/src/services/api/graphql/recipes/templates/create_firewall_address_group.tmpl
+++ b/src/services/api/graphql/session/templates/create_firewall_address_group.tmpl
diff --git a/src/services/api/graphql/recipes/templates/create_firewall_address_ipv_6_group.tmpl b/src/services/api/graphql/session/templates/create_firewall_address_ipv_6_group.tmpl
index e9b660722..e9b660722 100644
--- a/src/services/api/graphql/recipes/templates/create_firewall_address_ipv_6_group.tmpl
+++ b/src/services/api/graphql/session/templates/create_firewall_address_ipv_6_group.tmpl
diff --git a/src/services/api/graphql/recipes/templates/create_interface_ethernet.tmpl b/src/services/api/graphql/session/templates/create_interface_ethernet.tmpl
index d9d7ed691..d9d7ed691 100644
--- a/src/services/api/graphql/recipes/templates/create_interface_ethernet.tmpl
+++ b/src/services/api/graphql/session/templates/create_interface_ethernet.tmpl
diff --git a/src/services/api/graphql/recipes/templates/remove_firewall_address_group_members.tmpl b/src/services/api/graphql/session/templates/remove_firewall_address_group_members.tmpl
index 458f3e5fc..458f3e5fc 100644
--- a/src/services/api/graphql/recipes/templates/remove_firewall_address_group_members.tmpl
+++ b/src/services/api/graphql/session/templates/remove_firewall_address_group_members.tmpl
diff --git a/src/services/api/graphql/recipes/templates/remove_firewall_address_ipv_6_group_members.tmpl b/src/services/api/graphql/session/templates/remove_firewall_address_ipv_6_group_members.tmpl
index 0efa0b226..0efa0b226 100644
--- a/src/services/api/graphql/recipes/templates/remove_firewall_address_ipv_6_group_members.tmpl
+++ b/src/services/api/graphql/session/templates/remove_firewall_address_ipv_6_group_members.tmpl
diff --git a/src/services/api/graphql/recipes/templates/update_firewall_address_group_members.tmpl b/src/services/api/graphql/session/templates/update_firewall_address_group_members.tmpl
index f56c61231..f56c61231 100644
--- a/src/services/api/graphql/recipes/templates/update_firewall_address_group_members.tmpl
+++ b/src/services/api/graphql/session/templates/update_firewall_address_group_members.tmpl
diff --git a/src/services/api/graphql/recipes/templates/update_firewall_address_ipv_6_group_members.tmpl b/src/services/api/graphql/session/templates/update_firewall_address_ipv_6_group_members.tmpl
index f98a5517c..f98a5517c 100644
--- a/src/services/api/graphql/recipes/templates/update_firewall_address_ipv_6_group_members.tmpl
+++ b/src/services/api/graphql/session/templates/update_firewall_address_ipv_6_group_members.tmpl
diff --git a/src/services/api/graphql/utils/schema_from_op_mode.py b/src/services/api/graphql/utils/schema_from_op_mode.py
index d27586747..f990aae52 100755
--- a/src/services/api/graphql/utils/schema_from_op_mode.py
+++ b/src/services/api/graphql/utils/schema_from_op_mode.py
@@ -25,7 +25,10 @@ from inspect import signature, getmembers, isfunction
 from jinja2 import Template
 
 from vyos.defaults import directories
-from . util import load_as_module, is_op_mode_function_name, is_show_function_name
+if __package__ is None or __package__ == '':
+    from util import load_as_module, is_op_mode_function_name, is_show_function_name
+else:
+    from . util import load_as_module, is_op_mode_function_name, is_show_function_name
 
 OP_MODE_PATH = directories['op_mode']
 SCHEMA_PATH = directories['api_schema']
diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server
index af8837e1e..190f3409d 100755
--- a/src/services/vyos-http-api-server
+++ b/src/services/vyos-http-api-server
@@ -678,6 +678,7 @@ if __name__ == '__main__':
         server_config = load_server_config()
     except Exception as err:
         logger.critical(f"Failed to load the HTTP API server config: {err}")
+        sys.exit(1)
 
     config_session = ConfigSession(os.getpid())
 
diff --git a/src/system/keepalived-fifo.py b/src/system/keepalived-fifo.py
index a8df232ae..a0fccd1d0 100755
--- a/src/system/keepalived-fifo.py
+++ b/src/system/keepalived-fifo.py
@@ -30,6 +30,7 @@ from vyos.ifconfig.vrrp import VRRP
 from vyos.configquery import ConfigTreeQuery
 from vyos.util import cmd
 from vyos.util import dict_search
+from vyos.util import commit_in_progress
 
 # configure logging
 logger = logging.getLogger(__name__)
@@ -63,6 +64,17 @@ class KeepalivedFifo:
 
     # load configuration
     def _config_load(self):
+        # For VRRP configuration to be read, the commit must be finished
+        count = 1
+        while commit_in_progress():
+            if ( count <= 40 ):
+                logger.debug(f'commit in progress try: {count}')
+            else:
+                logger.error(f'commit still in progress after {count} continuing anyway')
+                break
+            count += 1
+            time.sleep(0.5)
+
         try:
             base = ['high-availability', 'vrrp']
             conf = ConfigTreeQuery()