Merge branch 'netns' into current

* netns:
  smoketest: T5241: re-work netns assertions and provide common utility helper
  netns: T5241: simplify network namespace handling
  netns: T5241: improve get_interface_namespace() robustness
  netns: T5241: use common interface_exists() helper
  netns: T5241: provide is_netns_interface utility helper
  T5241: Support netns for veth and dummy interfaces

7 files changed, 158 insertions, 120 deletions

diff --git a/interface-definitions/interfaces-virtual-ethernet.xml.in b/interface-definitions/interfaces-virtual-ethernet.xml.in
index 1daa764d4..5f205f354 100644
--- a/interface-definitions/interfaces-virtual-ethernet.xml.in
+++ b/interface-definitions/interfaces-virtual-ethernet.xml.in
@@ -21,6 +21,7 @@
           #include <include/interface/dhcp-options.xml.i>
           #include <include/interface/dhcpv6-options.xml.i>
           #include <include/interface/disable.xml.i>
+          #include <include/interface/netns.xml.i>
           #include <include/interface/vif-s.xml.i>
           #include <include/interface/vif.xml.i>
           #include <include/interface/vrf.xml.i>
diff --git a/python/vyos/ifconfig/control.py b/python/vyos/ifconfig/control.py
index c8366cb58..7402da55a 100644
--- a/python/vyos/ifconfig/control.py
+++ b/python/vyos/ifconfig/control.py
@@ -1,4 +1,4 @@
-# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io>
+# Copyright 2019-2023 VyOS maintainers and contributors <maintainers@vyos.io>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -49,6 +49,18 @@ class Control(Section):
         return popen(command, self.debug)
 
     def _cmd(self, command):
+        import re
+        if 'netns' in self.config:
+            # This command must be executed from default netns 'ip link set dev X netns X'
+            # exclude set netns cmd from netns to avoid:
+            # failed to run command: ip netns exec ns01 ip link set dev veth20 netns ns01
+            pattern = r'ip link set dev (\S+) netns (\S+)'
+            matches = re.search(pattern, command)
+            if matches and matches.group(2) == self.config['netns']:
+                # Command already includes netns and matches desired namespace:
+                command = command
+            else:
+                command = f'ip netns exec {self.config["netns"]} {command}'
         return cmd(command, self.debug)
 
     def _get_command(self, config, name):
diff --git a/python/vyos/ifconfig/interface.py b/python/vyos/ifconfig/interface.py
index 41ce352ad..53256abb7 100644
--- a/python/vyos/ifconfig/interface.py
+++ b/python/vyos/ifconfig/interface.py
@@ -38,7 +38,9 @@ from vyos.utils.dict import dict_search
 from vyos.utils.file import read_file
 from vyos.utils.network import get_interface_config
 from vyos.utils.network import get_interface_namespace
+from vyos.utils.network import is_netns_interface
 from vyos.utils.process import is_systemd_service_active
+from vyos.utils.process import run
 from vyos.template import is_ipv4
 from vyos.template import is_ipv6
 from vyos.utils.network import is_intf_addr_assigned
@@ -138,9 +140,6 @@ class Interface(Control):
             'validate': assert_mtu,
             'shellcmd': 'ip link set dev {ifname} mtu {value}',
         },
-        'netns': {
-            'shellcmd': 'ip link set dev {ifname} netns {value}',
-        },
         'vrf': {
             'convert': lambda v: f'master {v}' if v else 'nomaster',
             'shellcmd': 'ip link set dev {ifname} {value}',
@@ -286,8 +285,11 @@ class Interface(Control):
     }
 
     @classmethod
-    def exists(cls, ifname):
-        return os.path.exists(f'/sys/class/net/{ifname}')
+    def exists(cls, ifname: str, netns: str=None) -> bool:
+        cmd = f'ip link show dev {ifname}'
+        if netns:
+           cmd = f'ip netns exec {netns} {cmd}'
+        return run(cmd) == 0
 
     @classmethod
     def get_config(cls):
@@ -355,7 +357,13 @@ class Interface(Control):
         self.vrrp = VRRP(ifname)
 
     def _create(self):
+        # Do not create interface that already exist or exists in netns
+        netns = self.config.get('netns', None)
+        if self.exists(f'{self.ifname}', netns=netns):
+            return
+
         cmd = 'ip link add dev {ifname} type {type}'.format(**self.config)
+        if 'netns' in self.config: cmd = f'ip netns exec {netns} {cmd}'
         self._cmd(cmd)
 
     def remove(self):
@@ -390,6 +398,9 @@ class Interface(Control):
         # after interface removal no other commands should be allowed
         # to be called and instead should raise an Exception:
         cmd = 'ip link del dev {ifname}'.format(**self.config)
+        # for delete we can't get data from self.config{'netns'}
+        netns = get_interface_namespace(self.ifname)
+        if netns: cmd = f'ip netns exec {netns} {cmd}'
         return self._cmd(cmd)
 
     def _set_vrf_ct_zone(self, vrf):
@@ -397,6 +408,10 @@ class Interface(Control):
         Add/Remove rules in nftables to associate traffic in VRF to an
         individual conntack zone
         """
+        # Don't allow for netns yet
+        if 'netns' in self.config:
+            return None
+
         if vrf:
             # Get routing table ID for VRF
             vrf_table_id = get_interface_config(vrf).get('linkinfo', {}).get(
@@ -540,36 +555,30 @@ class Interface(Control):
         if prev_state == 'up':
             self.set_admin_state('up')
 
-    def del_netns(self, netns):
-        """
-        Remove interface from given NETNS.
-        """
-
-        # If NETNS does not exist then there is nothing to delete
+    def del_netns(self, netns: str) -> bool:
+        """ Remove interface from given network namespace """
+        # If network namespace does not exist then there is nothing to delete
         if not os.path.exists(f'/run/netns/{netns}'):
-            return None
-
-        # As a PoC we only allow 'dummy' interfaces
-        if 'dum' not in self.ifname:
-            return None
+            return False
 
-        # Check if interface realy exists in namespace
-        if get_interface_namespace(self.ifname) != None:
-            self._cmd(f'ip netns exec {get_interface_namespace(self.ifname)} ip link del dev {self.ifname}')
-            return
+        # Check if interface exists in network namespace
+        if is_netns_interface(self.ifname, netns):
+            self._cmd(f'ip netns exec {netns} ip link del dev {self.ifname}')
+            return True
+        return False
 
-    def set_netns(self, netns):
+    def set_netns(self, netns: str) -> bool:
         """
-        Add interface from given NETNS.
+        Add interface from given network namespace
 
         Example:
         >>> from vyos.ifconfig import Interface
         >>> Interface('dum0').set_netns('foo')
         """
+        self._cmd(f'ip link set dev {self.ifname} netns {netns}')
+        return True
 
-        self.set_interface('netns', netns)
-
-    def set_vrf(self, vrf):
+    def set_vrf(self, vrf: str) -> bool:
         """
         Add/Remove interface from given VRF instance.
 
@@ -581,10 +590,11 @@ class Interface(Control):
 
         tmp = self.get_interface('vrf')
         if tmp == vrf:
-            return None
+            return False
 
         self.set_interface('vrf', vrf)
         self._set_vrf_ct_zone(vrf)
+        return True
 
     def set_arp_cache_tmo(self, tmo):
         """
@@ -621,6 +631,10 @@ class Interface(Control):
         >>> from vyos.ifconfig import Interface
         >>> Interface('eth0').set_tcp_ipv4_mss(1340)
         """
+        # Don't allow for netns yet
+        if 'netns' in self.config:
+            return None
+
         self._cleanup_mss_rules('raw', self.ifname)
         nft_prefix = 'nft add rule raw VYOS_TCP_MSS'
         base_cmd = f'oifname "{self.ifname}" tcp flags & (syn|rst) == syn'
@@ -641,6 +655,10 @@ class Interface(Control):
         >>> from vyos.ifconfig import Interface
         >>> Interface('eth0').set_tcp_mss(1320)
         """
+        # Don't allow for netns yet
+        if 'netns' in self.config:
+            return None
+
         self._cleanup_mss_rules('ip6 raw', self.ifname)
         nft_prefix = 'nft add rule ip6 raw VYOS_TCP_MSS'
         base_cmd = f'oifname "{self.ifname}" tcp flags & (syn|rst) == syn'
@@ -755,6 +773,10 @@ class Interface(Control):
 
         As per RFC3074.
         """
+        # Don't allow for netns yet
+        if 'netns' in self.config:
+            return None
+
         if value == 'strict':
             value = 1
         elif value == 'loose':
@@ -794,6 +816,10 @@ class Interface(Control):
         >>> from vyos.ifconfig import Interface
         >>> Interface('eth0').set_ipv6_source_validation('strict')
         """
+        # Don't allow for netns yet
+        if 'netns' in self.config:
+            return None
+
         self._cleanup_ipv6_source_validation_rules(self.ifname)
         nft_prefix = f'nft add rule ip6 raw vyos_rpfilter iifname "{self.ifname}"'
         if mode == 'strict':
@@ -1143,13 +1169,17 @@ class Interface(Control):
         if addr in self._addr:
             return False
 
+        # get interface network namespace if specified
+        netns = self.config.get('netns', None)
+
         # add to interface
         if addr == 'dhcp':
             self.set_dhcp(True)
         elif addr == 'dhcpv6':
             self.set_dhcpv6(True)
-        elif not is_intf_addr_assigned(self.ifname, addr):
-            tmp = f'ip addr add {addr} dev {self.ifname}'
+        elif not is_intf_addr_assigned(self.ifname, addr, netns=netns):
+            netns_cmd  = f'ip netns exec {netns}' if netns else ''
+            tmp = f'{netns_cmd} ip addr add {addr} dev {self.ifname}'
             # Add broadcast address for IPv4
             if is_ipv4(addr): tmp += ' brd +'
 
@@ -1189,13 +1219,17 @@ class Interface(Control):
         if not addr:
             raise ValueError()
 
+        # get interface network namespace if specified
+        netns = self.config.get('netns', None)
+
         # remove from interface
         if addr == 'dhcp':
             self.set_dhcp(False)
         elif addr == 'dhcpv6':
             self.set_dhcpv6(False)
-        elif is_intf_addr_assigned(self.ifname, addr):
-            self._cmd(f'ip addr del "{addr}" dev "{self.ifname}"')
+        elif is_intf_addr_assigned(self.ifname, addr, netns=netns):
+            netns_cmd  = f'ip netns exec {netns}' if netns else ''
+            self._cmd(f'{netns_cmd} ip addr del {addr} dev {self.ifname}')
         else:
             return False
 
@@ -1215,8 +1249,11 @@ class Interface(Control):
         self.set_dhcp(False)
         self.set_dhcpv6(False)
 
+        netns = get_interface_namespace(self.ifname)
+        netns_cmd = f'ip netns exec {netns}' if netns else ''
+        cmd = f'{netns_cmd} ip addr flush dev {self.ifname}'
         # flush all addresses
-        self._cmd(f'ip addr flush dev "{self.ifname}"')
+        self._cmd(cmd)
 
     def add_to_bridge(self, bridge_dict):
         """
@@ -1371,6 +1408,11 @@ class Interface(Control):
         #   - https://man7.org/linux/man-pages/man8/tc-mirred.8.html
         # Depening if we are the source or the target interface of the port
         # mirror we need to setup some variables.
+
+        # Don't allow for netns yet
+        if 'netns' in self.config:
+            return None
+
         source_if = self.config['ifname']
 
         mirror_config = None
@@ -1471,8 +1513,8 @@ class Interface(Control):
         # Since the interface is pushed onto a separate logical stack
         # Configure NETNS
         if dict_search('netns', config) != None:
-            self.set_netns(config.get('netns', ''))
-            return
+            if not is_netns_interface(self.ifname, self.config['netns']):
+                self.set_netns(config.get('netns', ''))
         else:
             self.del_netns(config.get('netns', ''))
 
diff --git a/python/vyos/utils/network.py b/python/vyos/utils/network.py
index bc6899e45..55ff29f0c 100644
--- a/python/vyos/utils/network.py
+++ b/python/vyos/utils/network.py
@@ -40,13 +40,19 @@ def interface_exists(interface) -> bool:
     import os
     return os.path.exists(f'/sys/class/net/{interface}')
 
-def interface_exists_in_netns(interface_name, netns):
+def is_netns_interface(interface, netns):
     from vyos.utils.process import rc_cmd
-    rc, out = rc_cmd(f'ip netns exec {netns} ip link show dev {interface_name}')
+    rc, out = rc_cmd(f'sudo ip netns exec {netns} ip link show dev {interface}')
     if rc == 0:
         return True
     return False
 
+def get_netns_all() -> list:
+    from json import loads
+    from vyos.utils.process import cmd
+    tmp = loads(cmd('ip --json netns ls'))
+    return [ netns['name'] for netns in tmp ]
+
 def get_vrf_members(vrf: str) -> list:
     """
     Get list of interface VRF members
@@ -78,8 +84,7 @@ def get_interface_config(interface):
     """ Returns the used encapsulation protocol for given interface.
         If interface does not exist, None is returned.
     """
-    import os
-    if not os.path.exists(f'/sys/class/net/{interface}'):
+    if not interface_exists(interface):
         return None
     from json import loads
     from vyos.utils.process import cmd
@@ -90,31 +95,30 @@ def get_interface_address(interface):
     """ Returns the used encapsulation protocol for given interface.
         If interface does not exist, None is returned.
     """
-    import os
-    if not os.path.exists(f'/sys/class/net/{interface}'):
+    if not interface_exists(interface):
         return None
     from json import loads
     from vyos.utils.process import cmd
     tmp = loads(cmd(f'ip --detail --json addr show dev {interface}'))[0]
     return tmp
 
-def get_interface_namespace(iface):
+def get_interface_namespace(interface: str):
     """
        Returns wich netns the interface belongs to
     """
     from json import loads
     from vyos.utils.process import cmd
-    # Check if netns exist
-    tmp = loads(cmd(f'ip --json netns ls'))
-    if len(tmp) == 0:
-        return None
 
-    for ns in tmp:
+    # Bail out early if netns does not exist
+    tmp = cmd(f'ip --json netns ls')
+    if not tmp: return None
+
+    for ns in loads(tmp):
         netns = f'{ns["name"]}'
         # Search interface in each netns
         data = loads(cmd(f'ip netns exec {netns} ip --json link show'))
         for tmp in data:
-            if iface == tmp["ifname"]:
+            if interface == tmp["ifname"]:
                 return netns
 
 def is_ipv6_tentative(iface: str, ipv6_address: str) -> bool:
@@ -172,8 +176,7 @@ def is_wwan_connected(interface):
 
 def get_bridge_fdb(interface):
     """ Returns the forwarding database entries for a given interface """
-    import os
-    if not os.path.exists(f'/sys/class/net/{interface}'):
+    if not interface_exists(interface):
         return None
     from json import loads
     from vyos.utils.process import cmd
@@ -305,57 +308,33 @@ def is_addr_assigned(ip_address, vrf=None) -> bool:
 
     return False
 
-def is_intf_addr_assigned(intf, address) -> bool:
+def is_intf_addr_assigned(ifname: str, addr: str, netns: str=None) -> bool:
     """
     Verify if the given IPv4/IPv6 address is assigned to specific interface.
     It can check both a single IP address (e.g. 192.0.2.1 or a assigned CIDR
     address 192.0.2.1/24.
     """
-    from vyos.template import is_ipv4
-
-    from netifaces import ifaddresses
-    from netifaces import AF_INET
-    from netifaces import AF_INET6
-
-    # check if the requested address type is configured at all
-    # {
-    # 17: [{'addr': '08:00:27:d9:5b:04', 'broadcast': 'ff:ff:ff:ff:ff:ff'}],
-    # 2:  [{'addr': '10.0.2.15', 'netmask': '255.255.255.0', 'broadcast': '10.0.2.255'}],
-    # 10: [{'addr': 'fe80::a00:27ff:fed9:5b04%eth0', 'netmask': 'ffff:ffff:ffff:ffff::'}]
-    # }
-    try:
-        addresses = ifaddresses(intf)
-    except ValueError as e:
-        print(e)
-        return False
-
-    # determine IP version (AF_INET or AF_INET6) depending on passed address
-    addr_type = AF_INET if is_ipv4(address) else AF_INET6
-
-    # Check every IP address on this interface for a match
-    netmask = None
-    if '/' in address:
-        address, netmask = address.split('/')
-    for ip in addresses.get(addr_type, []):
-        # ip can have the interface name in the 'addr' field, we need to remove it
-        # {'addr': 'fe80::a00:27ff:fec5:f821%eth2', 'netmask': 'ffff:ffff:ffff:ffff::'}
-        ip_addr = ip['addr'].split('%')[0]
-
-        if not _are_same_ip(address, ip_addr):
-            continue
-
-        # we do not have a netmask to compare against, they are the same
-        if not netmask:
-            return True
+    import json
+    import jmespath
 
-        prefixlen = ''
-        if is_ipv4(ip_addr):
-            prefixlen = sum([bin(int(_)).count('1') for _ in ip['netmask'].split('.')])
-        else:
-            prefixlen = sum([bin(int(_,16)).count('1') for _ in ip['netmask'].split('/')[0].split(':') if _])
+    from vyos.utils.process import rc_cmd
+    from ipaddress import ip_interface
 
-        if str(prefixlen) == netmask:
-            return True
+    netns_cmd = f'ip netns exec {netns}' if netns else ''
+    rc, out = rc_cmd(f'{netns_cmd} ip --json address show dev {ifname}')
+    if rc == 0:
+        json_out = json.loads(out)
+        addresses = jmespath.search("[].addr_info[].{family: family, address: local, prefixlen: prefixlen}", json_out)
+        for address_info in addresses:
+            family = address_info['family']
+            address = address_info['address']
+            prefixlen = address_info['prefixlen']
+            # Remove the interface name if present in the given address
+            if '%' in addr:
+                addr = addr.split('%')[0]
+            interface = ip_interface(f"{address}/{prefixlen}")
+            if ip_interface(addr) == interface or address == addr:
+                return True
 
     return False
 
diff --git a/python/vyos/utils/process.py b/python/vyos/utils/process.py
index e09c7d86d..c2ef98140 100644
--- a/python/vyos/utils/process.py
+++ b/python/vyos/utils/process.py
@@ -170,7 +170,7 @@ def rc_cmd(command, flag='', shell=None, input=None, timeout=None, env=None,
     (1, 'Device "eth99" does not exist.')
     """
     out, code = popen(
-        command, flag,
+        command.lstrip(), flag,
         stdout=stdout, stderr=stderr,
         input=input, timeout=timeout,
         env=env, shell=shell,
diff --git a/smoketest/scripts/cli/test_interfaces_netns.py b/smoketest/scripts/cli/test_netns.py
index b8bebb221..fd04dd520 100755
--- a/smoketest/scripts/cli/test_interfaces_netns.py
+++ b/smoketest/scripts/cli/test_netns.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021 VyOS maintainers and contributors
+# Copyright (C) 2021-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -16,7 +16,6 @@
 
 import unittest
 
-from netifaces import interfaces
 from base_vyostest_shim import VyOSUnitTestSHIM
 
 from vyos.configsession import ConfigSession
@@ -24,56 +23,61 @@ from vyos.configsession import ConfigSessionError
 from vyos.ifconfig import Interface
 from vyos.ifconfig import Section
 from vyos.utils.process import cmd
+from vyos.utils.network import is_netns_interface
+from vyos.utils.network import get_netns_all
 
 base_path = ['netns']
-namespaces = ['mgmt', 'front', 'back', 'ams-ix']
+interfaces = ['dum10', 'dum12', 'dum50']
 
-class NETNSTest(VyOSUnitTestSHIM.TestCase):
-    def setUp(self):
-        self._interfaces = ['dum10', 'dum12', 'dum50']
+class NetNSTest(VyOSUnitTestSHIM.TestCase):
+    def tearDown(self):
+        self.cli_delete(base_path)
+        # commit changes
+        self.cli_commit()
+
+        # There should be no network namespace remaining
+        tmp = cmd('ip netns ls')
+        self.assertFalse(tmp)
+
+        super(NetNSTest, self).tearDown()
 
-    def test_create_netns(self):
+    def test_netns_create(self):
+        namespaces = ['mgmt', 'front', 'back']
         for netns in namespaces:
-            base = base_path + ['name', netns]
-            self.cli_set(base)
+            self.cli_set(base_path + ['name', netns])
 
         # commit changes
         self.cli_commit()
 
-        netns_list = cmd('ip netns ls')
-
         # Verify NETNS configuration
         for netns in namespaces:
-            self.assertTrue(netns in netns_list)
-
+            self.assertIn(netns, get_netns_all())
 
-    def test_netns_assign_interface(self):
+    def test_netns_interface(self):
         netns = 'foo'
-        self.cli_set(['netns', 'name', netns])
+        self.cli_set(base_path + ['name', netns])
 
         # Set
-        for iface in self._interfaces:
+        for iface in interfaces:
             self.cli_set(['interfaces', 'dummy', iface, 'netns', netns])
 
         # commit changes
         self.cli_commit()
 
-        netns_iface_list = cmd(f'sudo ip netns exec {netns} ip link show')
-
-        for iface in self._interfaces:
-            self.assertTrue(iface in netns_iface_list)
+        for interface in interfaces:
+            self.assertTrue(is_netns_interface(interface, netns))
 
         # Delete
-        for iface in self._interfaces:
-            self.cli_delete(['interfaces', 'dummy', iface, 'netns', netns])
+        for interface in interfaces:
+            self.cli_delete(['interfaces', 'dummy', interface])
 
         # commit changes
         self.cli_commit()
 
         netns_iface_list = cmd(f'sudo ip netns exec {netns} ip link show')
 
-        for iface in self._interfaces:
-            self.assertNotIn(iface, netns_iface_list)
+        for interface in interfaces:
+            self.assertFalse(is_netns_interface(interface, netns))
 
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/src/conf_mode/interfaces-dummy.py b/src/conf_mode/interfaces-dummy.py
index e771581e1..db768b94d 100755
--- a/src/conf_mode/interfaces-dummy.py
+++ b/src/conf_mode/interfaces-dummy.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2019-2021 VyOS maintainers and contributors
+# Copyright (C) 2019-2023 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -55,7 +55,7 @@ def generate(dummy):
     return None
 
 def apply(dummy):
-    d = DummyIf(dummy['ifname'])
+    d = DummyIf(**dummy)
 
     # Remove dummy interface
     if 'deleted' in dummy: