ethernet: ifconfig: T2653: move to get_config_dict()

The current VyOS CLI parser code written in Python contains a ton of duplicates
which I can also hold myself accountable for - or maybe mainly me - depends on
the angle of judge.

While providing a new update() method in vyos.ifconfig.interfaces() this is
extended for ethernet based interfaces which also supports 802.1q, 802.1ad
VLANs. This commit migrates the existing codebase for an ethernet based
interfaces and implements the missing parts for VLANs. Adding or migrating other
interfaces (e.g. bridge or bond) will become much easier as they must reuse
the entire functionality - we now walk towards a single codepath.

Thanks for all who made this combined effort possible!

Signed-off-by: Christian Poessinger <christian@poessinger.com>

5 files changed, 298 insertions, 15 deletions

diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py
index 0dc7578d8..682caed8f 100644
--- a/python/vyos/configdict.py
+++ b/python/vyos/configdict.py
@@ -102,12 +102,35 @@ def dict_merge(source, destination):
     return tmp
 
 def list_diff(first, second):
-    """
-    Diff two dictionaries and return only unique items
-    """
+    """ Diff two dictionaries and return only unique items """
     second = set(second)
     return [item for item in first if item not in second]
 
+def T2665_default_dict_cleanup(dict):
+    """ Cleanup default keys for tag nodes https://phabricator.vyos.net/T2665. """
+    # Cleanup
+    for vif in ['vif', 'vif_s']:
+        if vif in dict.keys():
+            for key in ['ip', 'mtu']:
+                if key in dict[vif].keys():
+                    del dict[vif][key]
+
+            # cleanup VIF-S defaults
+            if 'vif_c' in dict[vif].keys():
+                for key in ['ip', 'mtu']:
+                    if key in dict[vif]['vif_c'].keys():
+                        del dict[vif]['vif_c'][key]
+                # If there is no vif-c defined and we just cleaned the default
+                # keys - we can clean the entire vif-c dict as it's useless
+                if not dict[vif]['vif_c']:
+                    del dict[vif]['vif_c']
+
+            # If there is no real vif/vif-s defined and we just cleaned the default
+            # keys - we can clean the entire vif dict as it's useless
+            if not dict[vif]:
+                del dict[vif]
+
+    return dict
 
 def get_ethertype(ethertype_val):
     if ethertype_val == '0x88A8':
diff --git a/python/vyos/configverify.py b/python/vyos/configverify.py
index 32129a048..36b10c956 100644
--- a/python/vyos/configverify.py
+++ b/python/vyos/configverify.py
@@ -41,14 +41,14 @@ def verify_vrf(config):
 
 def verify_address(config):
     """
-    Common helper function used by interface implementations to
-    perform recurring validation of IP address assignmenr
-    when interface also is part of a bridge.
+    Common helper function used by interface implementations to perform
+    recurring validation of IP address assignment when interface is part
+    of a bridge or bond.
     """
     if {'is_bridge_member', 'address'} <= set(config):
         raise ConfigError(
-            f'Cannot assign address to interface "{ifname}" as it is a '
-            f'member of bridge "{is_bridge_member}"!'.format(**config))
+            'Cannot assign address to interface "{ifname}" as it is a '
+            'member of bridge "{is_bridge_member}"!'.format(**config))
 
 
 def verify_bridge_delete(config):
@@ -62,6 +62,15 @@ def verify_bridge_delete(config):
             'Interface "{ifname}" cannot be deleted as it is a '
             'member of bridge "{is_bridge_member}"!'.format(**config))
 
+def verify_interface_exists(config):
+    """
+    Common helper function used by interface implementations to perform
+    recurring validation if an interface actually exists.
+    """
+    from netifaces import interfaces
+    if not config['ifname'] in interfaces():
+        raise ConfigError(f'Interface "{ifname}" does not exist!'
+                          .format(**config))
 
 def verify_source_interface(config):
     """
@@ -76,3 +85,37 @@ def verify_source_interface(config):
     if not config['source_interface'] in interfaces():
         raise ConfigError(f'Source interface {source_interface} does not '
                           f'exist'.format(**config))
+
+def verify_dhcpv6(config):
+    """
+    Common helper function used by interface implementations to perform
+    recurring validation of DHCPv6 options which are mutually exclusive.
+    """
+    if {'parameters_only', 'temporary'} <= set(config.get('dhcpv6_options', {})):
+        raise ConfigError('DHCPv6 temporary and parameters-only options '
+                          'are mutually exclusive!')
+
+def verify_vlan_config(config):
+    """
+    Common helper function used by interface implementations to perform
+    recurring validation of interface VLANs
+    """
+    # 802.1q VLANs
+    for vlan in config.get('vif', {}).keys():
+        vlan = config['vif'][vlan]
+        verify_dhcpv6(vlan)
+        verify_address(vlan)
+        verify_vrf(vlan)
+
+    # 802.1ad (Q-in-Q) VLANs
+    for vlan in config.get('vif_s', {}).keys():
+        vlan = config['vif_s'][vlan]
+        verify_dhcpv6(vlan)
+        verify_address(vlan)
+        verify_vrf(vlan)
+
+        for vlan in config.get('vif_s', {}).get('vif_c', {}).keys():
+            vlan = config['vif_c'][vlan]
+            verify_dhcpv6(vlan)
+            verify_address(vlan)
+            verify_vrf(vlan)
diff --git a/python/vyos/ifconfig/ethernet.py b/python/vyos/ifconfig/ethernet.py
index 5b18926c9..8a50a8699 100644
--- a/python/vyos/ifconfig/ethernet.py
+++ b/python/vyos/ifconfig/ethernet.py
@@ -15,13 +15,14 @@
 
 import os
 import re
+import jmespath
 
+from vyos.configdict import get_ethertype
 from vyos.ifconfig.interface import Interface
 from vyos.ifconfig.vlan import VLAN
 from vyos.validate import assert_list
 from vyos.util import run
 
-
 @Interface.register
 @VLAN.enable
 class EthernetIf(Interface):
@@ -252,3 +253,101 @@ class EthernetIf(Interface):
         >>> i.set_udp_offload('on')
         """
         return self.set_interface('ufo', state)
+
+
+    def update(self, config):
+        """ General helper function which works on a dictionary retrived by
+        get_config_dict(). It's main intention is to consolidate the scattered
+        interface setup code and provide a single point of entry when workin
+        on any interface. """
+
+        # now call the regular function from within our base class
+        super().update(config)
+
+        # disable ethernet flow control (pause frames)
+        value = 'off' if 'disable_flow_control' in config.keys() else 'on'
+        self.set_flow_control(value)
+
+        # GRO (generic receive offload)
+        tmp = jmespath.search('offload_options.generic_receive', config)
+        value = tmp if (tmp != None) else 'off'
+        self.set_gro(value)
+
+        # GSO (generic segmentation offload)
+        tmp = jmespath.search('offload_options.generic_segmentation', config)
+        value = tmp if (tmp != None) else 'off'
+        self.set_gso(value)
+
+        # scatter-gather option
+        tmp = jmespath.search('offload_options.scatter_gather', config)
+        value = tmp if (tmp != None) else 'off'
+        self.set_sg(value)
+
+        # TSO (TCP segmentation offloading)
+        tmp = jmespath.search('offload_options.udp_fragmentation', config)
+        value = tmp if (tmp != None) else 'off'
+        self.set_tso(value)
+
+        # UDP fragmentation offloading
+        tmp = jmespath.search('offload_options.udp_fragmentation', config)
+        value = tmp if (tmp != None) else 'off'
+        self.set_ufo(value)
+
+        # Set physical interface speed and duplex
+        if {'speed', 'duplex'} <= set(config):
+            speed = config.get('speed')
+            duplex = config.get('duplex')
+            self.set_speed_duplex(speed, duplex)
+
+        # Delete old IPv6 EUI64 addresses before changing MAC
+
+        # Change interface MAC address - re-set to real hardware address (hw-id)
+        # if custom mac is removed. Skip if bond member.
+        if 'is_bond_member' not in config:
+            mac = config.get('hw_id')
+            if 'mac' in config:
+                mac = config.get('mac')
+            if mac:
+                self.set_mac(mac)
+
+        # Add IPv6 EUI-based addresses
+        tmp = jmespath.search('ipv6.address.eui64', config)
+        if tmp:
+            # XXX: T2636 workaround: convert string to a list with one element
+            if isinstance(tmp, str):
+                tmp = [tmp]
+            for addr in tmp:
+                self.add_ipv6_eui64_address(addr)
+
+        # re-add ourselves to any bridge we might have fallen out of
+        if 'is_bridge_member' in config:
+            bridge = config.get('is_bridge_member')
+            self.add_to_bridge(bridge)
+
+        # remove no longer required 802.1ad (Q-in-Q VLANs)
+        for vif_s_id in config.get('vif_s_remove', {}):
+            self.del_vlan(vif_s_id)
+
+        # create/update 802.1ad (Q-in-Q VLANs)
+        for vif_s_id, vif_s in config.get('vif_s', {}).items():
+            tmp=get_ethertype(vif_s.get('ethertype', '0x88A8'))
+            s_vlan = self.add_vlan(vif_s_id, ethertype=tmp)
+            s_vlan.update(vif_s)
+
+            # remove no longer required client VLAN (vif-c)
+            for vif_c_id in vif_s.get('vif_c_remove', {}):
+                s_vlan.del_vlan(vif_c_id)
+
+            # create/update client VLAN (vif-c) interface
+            for vif_c_id, vif_c in vif_s.get('vif_c', {}).items():
+                c_vlan = s_vlan.add_vlan(vif_c_id)
+                c_vlan.update(vif_c)
+
+        # remove no longer required 802.1q VLAN interfaces
+        for vif_id in config.get('vif_remove', {}):
+            self.del_vlan(vif_id)
+
+        # create/update 802.1q VLAN interfaces
+        for vif_id, vif in config.get('vif', {}).items():
+            vlan = self.add_vlan(vif_id)
+            vlan.update(vif)
diff --git a/python/vyos/ifconfig/interface.py b/python/vyos/ifconfig/interface.py
index 8d7b247fc..689faa22b 100644
--- a/python/vyos/ifconfig/interface.py
+++ b/python/vyos/ifconfig/interface.py
@@ -16,6 +16,7 @@
 import os
 import re
 import json
+import jmespath
 from copy import deepcopy
 
 from ipaddress import IPv4Network
@@ -322,11 +323,11 @@ class Interface(Control):
             self.set_admin_state('down')
 
         self.set_interface('mac', mac)
-        
+
         # Turn an interface to the 'up' state if it was changed to 'down' by this fucntion
         if prev_state == 'up':
             self.set_admin_state('up')
-    
+
     def set_vrf(self, vrf=''):
         """
         Add/Remove interface from given VRF instance.
@@ -773,14 +774,17 @@ class Interface(Control):
         on any interface. """
 
         # Update interface description
-        self.set_alias(config.get('description', None))
+        self.set_alias(config.get('description', ''))
+
+        # Ignore link state changes
+        value = '2' if 'disable_link_detect' in config else '1'
+        self.set_link_detect(value)
 
         # Configure assigned interface IP addresses. No longer
         # configured addresses will be removed first
         new_addr = config.get('address', [])
 
-        # XXX workaround for T2636, convert IP address string to a list
-        # with one element
+        # XXX: T2636 workaround: convert string to a list with one element
         if isinstance(new_addr, str):
             new_addr = [new_addr]
 
@@ -800,6 +804,96 @@ class Interface(Control):
             # Bind interface instance into VRF
             self.set_vrf(config.get('vrf', ''))
 
+        # DHCP options
+        if 'dhcp_options' in config:
+            dhcp_options = config.get('dhcp_options')
+            if 'client_id' in dhcp_options:
+                self.dhcp.v4.options['client_id'] = dhcp_options.get('client_id')
+
+            if 'host_name' in dhcp_options:
+                self.dhcp.v4.options['hostname'] = dhcp_options.get('host_name')
+
+            if 'vendor_class_id' in dhcp_options:
+                self.dhcp.v4.options['vendor_class_id'] = dhcp_options.get('vendor_class_id')
+
+        # DHCPv6 options
+        if 'dhcpv6_options' in config:
+            dhcpv6_options = config.get('dhcpv6_options')
+            if 'parameters_only' in dhcpv6_options:
+                self.dhcp.v6.options['dhcpv6_prm_only'] = True
+
+            if 'temporary' in dhcpv6_options:
+                self.dhcp.v6.options['dhcpv6_temporary'] = True
+
+            if 'prefix_delegation' in dhcpv6_options:
+                prefix_delegation = dhcpv6_options.get('prefix_delegation')
+                if 'length' in prefix_delegation:
+                    self.dhcp.v6.options['dhcpv6_pd_length'] = prefix_delegation.get('length')
+
+                if 'interface' in prefix_delegation:
+                    self.dhcp.v6.options['dhcpv6_pd_interfaces'] = prefix_delegation.get('interface')
+
+        # Configure ARP cache timeout in milliseconds - has default value
+        tmp = jmespath.search('ip.arp_cache_timeout', config)
+        value = tmp if (tmp != None) else '30'
+        self.set_arp_cache_tmo(value)
+
+        # Configure ARP filter configuration
+        tmp = jmespath.search('ip.disable_arp_filter', config)
+        value = '0' if (tmp != None) else '1'
+        self.set_arp_filter(value)
+
+        # Configure ARP accept
+        tmp = jmespath.search('ip.enable_arp_accept', config)
+        value = '1' if (tmp != None) else '0'
+        self.set_arp_accept(value)
+
+        # Configure ARP announce
+        tmp = jmespath.search('ip.enable_arp_announce', config)
+        value = '1' if (tmp != None) else '0'
+        self.set_arp_announce(value)
+
+        # Configure ARP ignore
+        tmp = jmespath.search('ip.enable_arp_ignore', config)
+        value = '1' if (tmp != None) else '0'
+        self.set_arp_ignore(value)
+
+        # Enable proxy-arp on this interface
+        tmp = jmespath.search('ip.enable_proxy_arp', config)
+        value = '1' if (tmp != None) else '0'
+        self.set_proxy_arp(value)
+
+        # Enable private VLAN proxy ARP on this interface
+        tmp = jmespath.search('ip.proxy_arp_pvlan', config)
+        value = '1' if (tmp != None) else '0'
+        self.set_proxy_arp_pvlan(value)
+
+        # IPv6 forwarding
+        tmp = jmespath.search('ipv6.disable_forwarding', config)
+        value = '0' if (tmp != None) else '1'
+        self.set_ipv6_forwarding(value)
+
+        # IPv6 router advertisements
+        tmp = jmespath.search('ipv6.address.autoconf', config)
+        value = '2' if (tmp != None) else '1'
+        if 'dhcpv6' in new_addr:
+            value = '2'
+        self.set_ipv6_accept_ra(value)
+
+        # IPv6 address autoconfiguration
+        tmp = jmespath.search('ipv6.address.autoconf', config)
+        value = '1' if (tmp != None) else '0'
+        self.set_ipv6_autoconf(value)
+
+        # IPv6 Duplicate Address Detection (DAD) tries
+        tmp = jmespath.search('ipv6.dup_addr_detect_transmits', config)
+        value = tmp if (tmp != None) else '1'
+        self.set_ipv6_dad_messages(value)
+
+        # MTU - Maximum Transfer Unit
+        if 'mtu' in config:
+            self.set_mtu(config.get('mtu'))
+
         # Interface administrative state
-        state = 'down' if 'disable' in config.keys() else 'up'
+        state = 'down' if 'disable' in config else 'up'
         self.set_admin_state(state)
diff --git a/python/vyos/ifconfig_vlan.py b/python/vyos/ifconfig_vlan.py
index 442cb0db8..ecb6796fa 100644
--- a/python/vyos/ifconfig_vlan.py
+++ b/python/vyos/ifconfig_vlan.py
@@ -16,6 +16,30 @@
 from netifaces import interfaces
 from vyos import ConfigError
 
+def get_removed_vlans(conf, dict):
+    """
+    Common function to parse a dictionary retrieved via get_config_dict() and
+    determine any added/removed VLAN interfaces - be it 802.1q or Q-in-Q.
+    """
+    from vyos.configdiff import get_config_diff, Diff
+
+    # Check vif, vif-s/vif-c VLAN interfaces for removal
+    D = get_config_diff(conf, key_mangling=('-', '_'))
+    D.set_level(conf.get_level())
+    # get_child_nodes() will return dict_keys(), mangle this into a list with PEP448
+    keys = D.get_child_nodes_diff(['vif'], expand_nodes=Diff.DELETE)['delete'].keys()
+    dict['vif_remove'] = [*keys]
+
+    # get_child_nodes() will return dict_keys(), mangle this into a list with PEP448
+    keys = D.get_child_nodes_diff(['vif-s'], expand_nodes=Diff.DELETE)['delete'].keys()
+    dict['vif_s_remove'] = [*keys]
+
+    for vif in dict.get('vif_s', {}).keys():
+        keys = D.get_child_nodes_diff(['vif-s', vif, 'vif-c'], expand_nodes=Diff.DELETE)['delete'].keys()
+        dict['vif_s'][vif]['vif_c_remove'] = [*keys]
+
+    return dict
+
 def apply_all_vlans(intf, intfconfig):
     """
     Function applies all VLANs to the passed interface.