vxlan: T5753: add support for VNI filtering

In a service provider network a service provider typically supports multiple
bridge domains with overlapping vlans. One bridge domain per customer. Vlans in
each bridge domain are mapped to globally unique VXLAN VNI ranges assigned to
each customer.

Without the ability of VNI filtering, we can not provide VXLAN tunnels
with multiple tenants all requiring e.g. VLAN 10.

To Test:

set interfaces vxlan vxlan987 parameters external
set interfaces vxlan vxlan987 source-interface eth0
set interfaces vxlan vxlan987 parameters vni-filter
set interfaces vxlan vxlan987 vlan-to-vni 50 vni 10050
set interfaces vxlan vxlan987 vlan-to-vni 51 vni 10051
set interfaces vxlan vxlan987 vlan-to-vni 52 vni 10052
set interfaces vxlan vxlan987 vlan-to-vni 53 vni 10053
set interfaces vxlan vxlan987 vlan-to-vni 54 vni 10054
set interfaces vxlan vxlan987 vlan-to-vni 60 vni 10060
set interfaces vxlan vxlan987 vlan-to-vni 69 vni 10069
set interfaces bridge br0 member interface vxlan987

Add new op-mode command: show bridge vni

Interface    VNI
-----------  -----------
vxlan987     10050-10054
vxlan987     10060
vxlan987     10069

(cherry picked from commit 35f6033d21053fa420e837f157cd9377a4ccd26a)

7 files changed, 208 insertions, 12 deletions

diff --git a/interface-definitions/interfaces-vxlan.xml.in b/interface-definitions/interfaces-vxlan.xml.in
index f20743a65..53045d678 100644
--- a/interface-definitions/interfaces-vxlan.xml.in
+++ b/interface-definitions/interfaces-vxlan.xml.in
@@ -95,6 +95,12 @@
                   <valueless/>
                 </properties>
               </leafNode>
+              <leafNode name="vni-filter">
+                <properties>
+                  <help>Enable VNI filter support</help>
+                  <valueless/>
+                </properties>
+              </leafNode>
             </children>
           </node>
           #include <include/port-number.xml.i>
diff --git a/op-mode-definitions/show-bridge.xml.in b/op-mode-definitions/show-bridge.xml.in
index fad3f3418..5d8cc3847 100644
--- a/op-mode-definitions/show-bridge.xml.in
+++ b/op-mode-definitions/show-bridge.xml.in
@@ -21,6 +21,12 @@
               </leafNode>
             </children>
           </node>
+          <leafNode name="vni">
+            <properties>
+              <help>Virtual Network Identifier</help>
+            </properties>
+            <command>${vyos_op_scripts_dir}/bridge.py show_vni</command>
+          </leafNode>
         </children>
       </node>
       <leafNode name="bridge">
diff --git a/python/vyos/ifconfig/vxlan.py b/python/vyos/ifconfig/vxlan.py
index 8c5a0220e..23b6daa3a 100644
--- a/python/vyos/ifconfig/vxlan.py
+++ b/python/vyos/ifconfig/vxlan.py
@@ -22,6 +22,7 @@ from vyos.utils.assertion import assert_list
 from vyos.utils.dict import dict_search
 from vyos.utils.network import get_interface_config
 from vyos.utils.network import get_vxlan_vlan_tunnels
+from vyos.utils.network import get_vxlan_vni_filter
 
 @Interface.register
 class VXLANIf(Interface):
@@ -79,6 +80,7 @@ class VXLANIf(Interface):
             'parameters.ip.ttl'          : 'ttl',
             'parameters.ipv6.flowlabel'  : 'flowlabel',
             'parameters.nolearning'      : 'nolearning',
+            'parameters.vni_filter'      : 'vnifilter',
             'remote'                     : 'remote',
             'source_address'             : 'local',
             'source_interface'           : 'dev',
@@ -138,10 +140,14 @@ class VXLANIf(Interface):
         if not isinstance(state, bool):
             raise ValueError('Value out of range')
 
-        cur_vlan_ids = []
         if 'vlan_to_vni_removed' in self.config:
-            cur_vlan_ids = self.config['vlan_to_vni_removed']
-            for vlan in cur_vlan_ids:
+            cur_vni_filter = get_vxlan_vni_filter(self.ifname)
+            for vlan, vlan_config in self.config['vlan_to_vni_removed'].items():
+                # If VNI filtering is enabled, remove matching VNI filter
+                if dict_search('parameters.vni_filter', self.config) != None:
+                    vni = vlan_config['vni']
+                    if vni in cur_vni_filter:
+                        self._cmd(f'bridge vni delete dev {self.ifname} vni {vni}')
                 self._cmd(f'bridge vlan del dev {self.ifname} vid {vlan}')
 
         # Determine current OS Kernel vlan_tunnel setting - only adjust when needed
@@ -151,10 +157,9 @@ class VXLANIf(Interface):
         if cur_state != new_state:
             self.set_interface('vlan_tunnel', new_state)
 
-        # Determine current OS Kernel configured VLANs
-        os_configured_vlan_ids = get_vxlan_vlan_tunnels(self.ifname)
-
         if 'vlan_to_vni' in self.config:
+            # Determine current OS Kernel configured VLANs
+            os_configured_vlan_ids = get_vxlan_vlan_tunnels(self.ifname)
             add_vlan = list_diff(list(self.config['vlan_to_vni'].keys()), os_configured_vlan_ids)
 
             for vlan, vlan_config in self.config['vlan_to_vni'].items():
@@ -168,6 +173,10 @@ class VXLANIf(Interface):
                 self._cmd(f'bridge vlan add dev {self.ifname} vid {vlan}')
                 self._cmd(f'bridge vlan add dev {self.ifname} vid {vlan} tunnel_info id {vni}')
 
+                # If VNI filtering is enabled, install matching VNI filter
+                if dict_search('parameters.vni_filter', self.config) != None:
+                    self._cmd(f'bridge vni add dev {self.ifname} vni {vni}')
+
     def update(self, config):
         """ General helper function which works on a dictionary retrived by
         get_config_dict(). It's main intention is to consolidate the scattered
diff --git a/python/vyos/utils/network.py b/python/vyos/utils/network.py
index 3b5e146e1..4f7021509 100644
--- a/python/vyos/utils/network.py
+++ b/python/vyos/utils/network.py
@@ -504,3 +504,36 @@ def get_vxlan_vlan_tunnels(interface: str) -> list:
             os_configured_vlan_ids.append(str(vlanStart))
 
     return os_configured_vlan_ids
+
+def get_vxlan_vni_filter(interface: str) -> list:
+    """ Return a list of strings with VNIs configured in the Kernel"""
+    from json import loads
+    from vyos.utils.process import cmd
+
+    if not interface.startswith('vxlan'):
+        raise ValueError('Only applicable for VXLAN interfaces!')
+
+    # Determine current OS Kernel configured VNI filters in VXLAN interface
+    #
+    # $ bridge -j vni show dev vxlan1
+    # [{"ifname":"vxlan1","vnis":[{"vni":100},{"vni":200},{"vni":300,"vniEnd":399}]}]
+    #
+    # Example output: ['10010', '10020', '10021', '10022']
+    os_configured_vnis = []
+    tmp = loads(cmd(f'bridge --json vni show dev {interface}'))
+    if tmp:
+        for tunnel in tmp[0].get('vnis', {}):
+            vniStart = tunnel['vni']
+            if 'vniEnd' in tunnel:
+                vniEnd = tunnel['vniEnd']
+                # Build a real list for user VNIs
+                vni_list = list(range(vniStart, vniEnd +1))
+                # Convert list of integers to list or strings
+                os_configured_vnis.extend(map(str, vni_list))
+                # Proceed with next tunnel - this one is complete
+                continue
+
+            # Add single tunel id - not part of a range
+            os_configured_vnis.append(str(vniStart))
+
+    return os_configured_vnis
diff --git a/smoketest/scripts/cli/test_interfaces_vxlan.py b/smoketest/scripts/cli/test_interfaces_vxlan.py
index 17e4fc36f..be1fa9d7f 100755
--- a/smoketest/scripts/cli/test_interfaces_vxlan.py
+++ b/smoketest/scripts/cli/test_interfaces_vxlan.py
@@ -22,6 +22,7 @@ from vyos.utils.network import get_bridge_fdb
 from vyos.utils.network import get_interface_config
 from vyos.utils.network import interface_exists
 from vyos.utils.network import get_vxlan_vlan_tunnels
+from vyos.utils.network import get_vxlan_vni_filter
 from vyos.template import is_ipv6
 from base_interfaces_test import BasicInterfaceTest
 
@@ -151,6 +152,7 @@ class VXLANInterfaceTest(BasicInterfaceTest.TestCase):
         }
 
         self.cli_set(self._base_path + [interface, 'parameters', 'external'])
+        self.cli_set(self._base_path + [interface, 'parameters', 'vni-filter'])
         self.cli_set(self._base_path + [interface, 'source-interface', source_interface])
 
         for vlan, vni in vlan_to_vni.items():
@@ -222,5 +224,93 @@ class VXLANInterfaceTest(BasicInterfaceTest.TestCase):
 
         self.cli_delete(['interfaces', 'bridge', bridge])
 
+    def test_vxlan_vni_filter(self):
+        interfaces = ['vxlan987', 'vxlan986', 'vxlan985']
+        source_address = '192.0.2.77'
+
+        for interface in interfaces:
+            self.cli_set(self._base_path + [interface, 'parameters', 'external'])
+            self.cli_set(self._base_path + [interface, 'source-address', source_address])
+
+        # This must fail as there can only be one "external" VXLAN device unless "vni-filter" is defined
+        with self.assertRaises(ConfigSessionError):
+            self.cli_commit()
+
+        # Enable "vni-filter" on the first VXLAN interface
+        self.cli_set(self._base_path + [interfaces[0], 'parameters', 'vni-filter'])
+
+        # This must fail as if it's enabled on one VXLAN interface, it must be enabled on all
+        # VXLAN interfaces
+        with self.assertRaises(ConfigSessionError):
+            self.cli_commit()
+        for interface in interfaces:
+            self.cli_set(self._base_path + [interface, 'parameters', 'vni-filter'])
+
+        # commit configuration
+        self.cli_commit()
+
+        for interface in interfaces:
+            self.assertTrue(interface_exists(interface))
+
+            tmp = get_interface_config(interface)
+            self.assertTrue(tmp['linkinfo']['info_data']['vnifilter'])
+
+    def test_vxlan_vni_filter_add_remove(self):
+        interface = 'vxlan987'
+        source_address = '192.0.2.66'
+        bridge = 'br0'
+
+        self.cli_set(self._base_path + [interface, 'parameters', 'external'])
+        self.cli_set(self._base_path + [interface, 'source-address', source_address])
+        self.cli_set(self._base_path + [interface, 'parameters', 'vni-filter'])
+
+        # commit configuration
+        self.cli_commit()
+
+        # Check if VXLAN interface got created
+        self.assertTrue(interface_exists(interface))
+
+        # VNI filter configured?
+        tmp = get_interface_config(interface)
+        self.assertTrue(tmp['linkinfo']['info_data']['vnifilter'])
+
+        # Now create some VLAN mappings and VNI filter
+        vlan_to_vni = {
+            '50': '10050',
+            '51': '10051',
+            '52': '10052',
+            '53': '10053',
+            '54': '10054',
+            '60': '10060',
+            '69': '10069',
+        }
+        for vlan, vni in vlan_to_vni.items():
+            self.cli_set(self._base_path + [interface, 'vlan-to-vni', vlan, 'vni', vni])
+        # we need a bridge ...
+        self.cli_set(['interfaces', 'bridge', bridge, 'member', 'interface', interface])
+        # commit configuration
+        self.cli_commit()
+
+        # All VNIs configured?
+        tmp = get_vxlan_vni_filter(interface)
+        self.assertListEqual(list(vlan_to_vni.values()), tmp)
+
+        #
+        # Delete a VLAN mappings and check if all VNIs are properly set up
+        #
+        vlan_to_vni.popitem()
+        self.cli_delete(self._base_path + [interface, 'vlan-to-vni'])
+        for vlan, vni in vlan_to_vni.items():
+            self.cli_set(self._base_path + [interface, 'vlan-to-vni', vlan, 'vni', vni])
+
+        # commit configuration
+        self.cli_commit()
+
+        # All VNIs configured?
+        tmp = get_vxlan_vni_filter(interface)
+        self.assertListEqual(list(vlan_to_vni.values()), tmp)
+
+        self.cli_delete(['interfaces', 'bridge', bridge])
+
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/src/conf_mode/interfaces-vxlan.py b/src/conf_mode/interfaces-vxlan.py
index 6bf3227d5..4251e611b 100755
--- a/src/conf_mode/interfaces-vxlan.py
+++ b/src/conf_mode/interfaces-vxlan.py
@@ -60,8 +60,14 @@ def get_config(config=None):
             vxlan.update({'rebuild_required': {}})
             break
 
+    # When dealing with VNI filtering we need to know what VNI was actually removed,
+    # so build up a dict matching the vlan_to_vni structure but with removed values.
     tmp = node_changed(conf, base + [ifname, 'vlan-to-vni'], recursive=True)
-    if tmp: vxlan.update({'vlan_to_vni_removed': tmp})
+    if tmp:
+        vxlan.update({'vlan_to_vni_removed': {}})
+        for vlan in tmp:
+            vni = leaf_node_changed(conf, base + [ifname, 'vlan-to-vni', vlan, 'vni'])
+            vxlan['vlan_to_vni_removed'].update({vlan : {'vni' : vni[0]}})
 
     # We need to verify that no other VXLAN tunnel is configured when external
     # mode is in use - Linux Kernel limitation
@@ -98,14 +104,31 @@ def verify(vxlan):
     if 'vni' not in vxlan and dict_search('parameters.external', vxlan) == None:
         raise ConfigError('Must either configure VXLAN "vni" or use "external" CLI option!')
 
-    if dict_search('parameters.external', vxlan):
+    if dict_search('parameters.external', vxlan) != None:
         if 'vni' in vxlan:
             raise ConfigError('Can not specify both "external" and "VNI"!')
 
         if 'other_tunnels' in vxlan:
-            other_tunnels = ', '.join(vxlan['other_tunnels'])
-            raise ConfigError(f'Only one VXLAN tunnel is supported when "external" '\
-                              f'CLI option is used. Additional tunnels: {other_tunnels}')
+            # When multiple VXLAN interfaces are defined and "external" is used,
+            # all VXLAN interfaces need to have vni-filter enabled!
+            # See Linux Kernel commit f9c4bb0b245cee35ef66f75bf409c9573d934cf9
+            other_vni_filter = False
+            for tunnel, tunnel_config in vxlan['other_tunnels'].items():
+                if dict_search('parameters.vni_filter', tunnel_config) != None:
+                    other_vni_filter = True
+                    break
+            # eqivalent of the C foo ? 'a' : 'b' statement
+            vni_filter = True and (dict_search('parameters.vni_filter', vxlan) != None) or False
+            # If either one is enabled, so must be the other. Both can be off and both can be on
+            if (vni_filter and not other_vni_filter) or (not vni_filter and other_vni_filter):
+                raise ConfigError(f'Using multiple VXLAN interfaces with "external" '\
+                    'requires all VXLAN interfaces to have "vni-filter" configured!')
+
+            if not vni_filter and not other_vni_filter:
+                other_tunnels = ', '.join(vxlan['other_tunnels'])
+                raise ConfigError(f'Only one VXLAN tunnel is supported when "external" '\
+                                f'CLI option is used and "vni-filter" is unset. '\
+                                f'Additional tunnels: {other_tunnels}')
 
     if 'gpe' in vxlan and 'external' not in vxlan:
         raise ConfigError(f'VXLAN-GPE is only supported when "external" '\
@@ -165,7 +188,7 @@ def verify(vxlan):
                 raise ConfigError(f'VNI "{vni}" is already assigned to a different VLAN!')
             vnis_used.append(vni)
 
-    if dict_search('parameters.neighbor_suppress', vxlan):
+    if dict_search('parameters.neighbor_suppress', vxlan) != None:
         if 'is_bridge_member' not in vxlan:
             raise ConfigError('Neighbor suppression requires that VXLAN interface '\
                               'is member of a bridge interface!')
diff --git a/src/op_mode/bridge.py b/src/op_mode/bridge.py
index 185db4f20..412a4eba8 100755
--- a/src/op_mode/bridge.py
+++ b/src/op_mode/bridge.py
@@ -56,6 +56,13 @@ def _get_raw_data_vlan(tunnel:bool=False):
     data_dict = json.loads(json_data)
     return data_dict
 
+def _get_raw_data_vni() -> dict:
+    """
+    :returns dict
+    """
+    json_data = cmd(f'bridge --json vni show')
+    data_dict = json.loads(json_data)
+    return data_dict
 
 def _get_raw_data_fdb(bridge):
     """Get MAC-address for the bridge brX
@@ -165,6 +172,22 @@ def _get_formatted_output_vlan_tunnel(data):
     output = tabulate(data_entries, headers)
     return output
 
+def _get_formatted_output_vni(data):
+    data_entries = []
+    for entry in data:
+        interface = entry.get('ifname')
+        vlans = entry.get('vnis')
+        for vlan_entry in vlans:
+            vlan = vlan_entry.get('vni')
+            if vlan_entry.get('vniEnd'):
+                vlan_end = vlan_entry.get('vniEnd')
+                vlan = f'{vlan}-{vlan_end}'
+            data_entries.append([interface, vlan])
+
+    headers = ["Interface", "VNI"]
+    output = tabulate(data_entries, headers)
+    return output
+
 def _get_formatted_output_fdb(data):
     data_entries = []
     for entry in data:
@@ -228,6 +251,12 @@ def show_vlan(raw: bool, tunnel: typing.Optional[bool]):
         else:
             return _get_formatted_output_vlan(bridge_vlan)
 
+def show_vni(raw: bool):
+    bridge_vni = _get_raw_data_vni()
+    if raw:
+        return bridge_vni
+    else:
+        return _get_formatted_output_vni(bridge_vni)
 
 def show_fdb(raw: bool, interface: str):
     fdb_data = _get_raw_data_fdb(interface)