Merge branch 'interface-rewrite' of github.com:c-po/vyos-1x into current

* 'interface-rewrite' of github.com:c-po/vyos-1x:
  vyos.configverify: T2653: fix some formatting issues
  ifconfig: T2653: make ifname an optional argument to get_interface_dict()
  vyos.configdict: T2653: remove obsolete code from configdict and ifconfig_vlan
  wireless: ifconfig: T2653: move to get_config_dict()
  ifconfig: T2653: move get_ethertype() from configdict to interface
  vlan: ifconfig: T2653: move get_removed_vlans() to vyos.configdiff
  bonding: ifconfig: T2653: move to get_config_dict()
  ifconfig: T2653: move vlan configuration code to base class
  vyos.configdict: T2653: use dict_merge() over update()
  ifconfig: T2653: implement update() in derived classes for admin up/down
  vyos.configdict: T2653: add new reusable helper node_changed()
  geneve: ifconfig: T2653: move to get_config_dict()
  ifconfig: T2653: move bridge member check to base class
  interfaces: ifconfig: T2653: migrate to get_interface_dict() API
  pseudo-ethernet: ifconfig: T2653: move to get_config_dict()
  bridge: ifconfig: T2653: move to get_config_dict()
  vlan: ifconfig: T2653: only enable interface when lower interface is up
  ethernet: ifconfig: T2653: move to get_config_dict()
  ifconfig: T2653: set arp-cache-timeout default value of 30ms

1 files changed, 157 insertions, 381 deletions

diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py
index 0dc7578d8..126d6195a 100644
--- a/python/vyos/configdict.py
+++ b/python/vyos/configdict.py
@@ -15,15 +15,15 @@
 
 """
 A library for retrieving value dicts from VyOS configs in a declarative fashion.
-
 """
+import os
+import jmespath
 
 from enum import Enum
 from copy import deepcopy
 
 from vyos import ConfigError
 from vyos.validate import is_member
-from vyos.util import ifname_from_config
 
 def retrieve_config(path_hash, base_path, config):
     """
@@ -102,397 +102,173 @@ def dict_merge(source, destination):
     return tmp
 
 def list_diff(first, second):
-    """
-    Diff two dictionaries and return only unique items
-    """
+    """ Diff two dictionaries and return only unique items """
     second = set(second)
     return [item for item in first if item not in second]
 
-
-def get_ethertype(ethertype_val):
-    if ethertype_val == '0x88A8':
-        return '802.1ad'
-    elif ethertype_val == '0x8100':
-        return '802.1q'
-    else:
-        raise ConfigError('invalid ethertype "{}"'.format(ethertype_val))
-
-dhcpv6_pd_default_data = {
-    'dhcpv6_prm_only': False,
-    'dhcpv6_temporary': False,
-    'dhcpv6_pd_length': '',
-    'dhcpv6_pd_interfaces': []
-}
-
-interface_default_data = {
-    **dhcpv6_pd_default_data,
-    'address': [],
-    'address_remove': [],
-    'description': '',
-    'dhcp_client_id': '',
-    'dhcp_hostname': '',
-    'dhcp_vendor_class_id': '',
-    'disable': False,
-    'disable_link_detect': 1,
-    'ip_disable_arp_filter': 1,
-    'ip_enable_arp_accept': 0,
-    'ip_enable_arp_announce': 0,
-    'ip_enable_arp_ignore': 0,
-    'ip_proxy_arp': 0,
-    'ipv6_accept_ra': 1,
-    'ipv6_autoconf': 0,
-    'ipv6_eui64_prefix': [],
-    'ipv6_eui64_prefix_remove': [],
-    'ipv6_forwarding': 1,
-    'ipv6_dup_addr_detect': 1,
-    'is_bridge_member': False,
-    'mac': '',
-    'mtu': 1500,
-    'vrf': ''
-}
-
-vlan_default = {
-    **interface_default_data,
-    'egress_qos': '',
-    'egress_qos_changed': False,
-    'ingress_qos': '',
-    'ingress_qos_changed': False,
-    'vif_c': {},
-    'vif_c_remove': []
-}
-
-# see: https://docs.python.org/3/library/enum.html#functional-api
-disable = Enum('disable','none was now both')
-
-def disable_state(conf, check=[3,5,7]):
+def T2665_default_dict_cleanup(dict):
+    """ Cleanup default keys for tag nodes https://phabricator.vyos.net/T2665. """
+    # Cleanup
+    for vif in ['vif', 'vif_s']:
+        if vif in dict.keys():
+            for key in ['ip', 'mtu']:
+                if key in dict[vif].keys():
+                    del dict[vif][key]
+
+            # cleanup VIF-S defaults
+            if 'vif_c' in dict[vif].keys():
+                for key in ['ip', 'mtu']:
+                    if key in dict[vif]['vif_c'].keys():
+                        del dict[vif]['vif_c'][key]
+                # If there is no vif-c defined and we just cleaned the default
+                # keys - we can clean the entire vif-c dict as it's useless
+                if not dict[vif]['vif_c']:
+                    del dict[vif]['vif_c']
+
+            # If there is no real vif/vif-s defined and we just cleaned the default
+            # keys - we can clean the entire vif dict as it's useless
+            if not dict[vif]:
+                del dict[vif]
+
+    return dict
+
+def leaf_node_changed(conf, path):
     """
-    return if and how a particual section of the configuration is has disable'd
-    using "disable" including if it was disabled by one of its parent.
-
-    check: a list of the level we should check, here 7,5 and 3
-          interfaces ethernet eth1 vif-s 1 vif-c 2 disable
-          interfaces ethernet eth1 vif 1 disable
-          interfaces ethernet eth1 disable
-
-    it returns an enum (none, was, now, both)
+    Check if a leaf node was altered. If it has been altered - values has been
+    changed, or it was added/removed, we will return the old value. If nothing
+    has been changed, None is returned
     """
-
-    # save where we are in the config
-    current_level = conf.get_level()
-
-    # logic to figure out if the interface (or one of it parent is disabled)
-    eff_disable = False
-    act_disable = False
-
-    levels = check[:]
-    working_level = current_level[:]
-
-    while levels:
-        position = len(working_level)
-        if not position:
-            break
-        if position not in levels:
-            working_level = working_level[:-1]
-            continue
-
-        levels.remove(position)
-        conf.set_level(working_level)
-        working_level = working_level[:-1]
-
-        eff_disable = eff_disable or conf.exists_effective('disable')
-        act_disable = act_disable or conf.exists('disable')
-
-    conf.set_level(current_level)
-
-    # how the disabling changed
-    if eff_disable and act_disable:
-        return disable.both
-    if eff_disable and not eff_disable:
-        return disable.was
-    if not eff_disable and act_disable:
-        return disable.now
-    return disable.none
-
-
-def intf_to_dict(conf, default):
-    from vyos.ifconfig import Interface
-
+    from vyos.configdiff import get_config_diff
+    D = get_config_diff(conf, key_mangling=('-', '_'))
+    D.set_level(conf.get_level())
+    (new, old) = D.get_value_diff(path)
+    if new != old:
+        if isinstance(old, str):
+            return old
+        elif isinstance(old, list):
+            if isinstance(new, str):
+                new = [new]
+            elif isinstance(new, type(None)):
+                new = []
+            return list_diff(old, new)
+
+    return None
+
+def node_changed(conf, path):
     """
-    Common used function which will extract VLAN related information from config
-    and represent the result as Python dictionary.
-
-    Function call's itself recursively if a vif-s/vif-c pair is detected.
+    Check if a leaf node was altered. If it has been altered - values has been
+    changed, or it was added/removed, we will return the old value. If nothing
+    has been changed, None is returned
     """
+    from vyos.configdiff import get_config_diff, Diff
+    D = get_config_diff(conf, key_mangling=('-', '_'))
+    D.set_level(conf.get_level())
+    # get_child_nodes() will return dict_keys(), mangle this into a list with PEP448
+    keys = D.get_child_nodes_diff(path, expand_nodes=Diff.DELETE)['delete'].keys()
+    return list(keys)
+
+def get_removed_vlans(conf, dict):
+    """
+    Common function to parse a dictionary retrieved via get_config_dict() and
+    determine any added/removed VLAN interfaces - be it 802.1q or Q-in-Q.
+    """
+    from vyos.configdiff import get_config_diff, Diff
 
-    intf = deepcopy(default)
-    intf['intf'] = ifname_from_config(conf)
-
-    current_vif_list = conf.list_nodes(['vif'])
-    previous_vif_list = conf.list_effective_nodes(['vif'])
-
-    # set the vif to be deleted
-    for vif in previous_vif_list:
-        if vif not in current_vif_list:
-            intf['vif_remove'].append(vif)
-
-    # retrieve interface description
-    if conf.exists(['description']):
-        intf['description'] = conf.return_value(['description'])
-
-    # get DHCP client identifier
-    if conf.exists(['dhcp-options', 'client-id']):
-        intf['dhcp_client_id'] = conf.return_value(['dhcp-options', 'client-id'])
-
-    # DHCP client host name (overrides the system host name)
-    if conf.exists(['dhcp-options', 'host-name']):
-        intf['dhcp_hostname'] = conf.return_value(['dhcp-options', 'host-name'])
-
-    # DHCP client vendor identifier
-    if conf.exists(['dhcp-options', 'vendor-class-id']):
-        intf['dhcp_vendor_class_id'] = conf.return_value(
-            ['dhcp-options', 'vendor-class-id'])
-
-    # DHCPv6 only acquire config parameters, no address
-    if conf.exists(['dhcpv6-options', 'parameters-only']):
-        intf['dhcpv6_prm_only'] = True
-
-    # DHCPv6 prefix delegation (RFC3633)
-    current_level = conf.get_level()
-    if conf.exists(['dhcpv6-options', 'prefix-delegation']):
-        dhcpv6_pd_path = current_level + ['dhcpv6-options', 'prefix-delegation']
-        conf.set_level(dhcpv6_pd_path)
-
-        # retriebe DHCPv6-PD prefix helper length as some ISPs only hand out a
-        # /64 by default (https://phabricator.vyos.net/T2506)
-        if conf.exists(['length']):
-            intf['dhcpv6_pd_length'] = conf.return_value(['length'])
+    # Check vif, vif-s/vif-c VLAN interfaces for removal
+    D = get_config_diff(conf, key_mangling=('-', '_'))
+    D.set_level(conf.get_level())
+    # get_child_nodes() will return dict_keys(), mangle this into a list with PEP448
+    keys = D.get_child_nodes_diff(['vif'], expand_nodes=Diff.DELETE)['delete'].keys()
+    if keys:
+        dict.update({'vif_remove': [*keys]})
 
-        for interface in conf.list_nodes(['interface']):
-            conf.set_level(dhcpv6_pd_path + ['interface', interface])
-            pd = {
-                'ifname': interface,
-                'sla_id': '',
-                'sla_len': '',
-                'if_id': ''
-            }
+    # get_child_nodes() will return dict_keys(), mangle this into a list with PEP448
+    keys = D.get_child_nodes_diff(['vif-s'], expand_nodes=Diff.DELETE)['delete'].keys()
+    if keys:
+        dict.update({'vif_s_remove': [*keys]})
 
-            if conf.exists(['sla-id']):
-                pd['sla_id'] = conf.return_value(['sla-id'])
+    for vif in dict.get('vif_s', {}).keys():
+        keys = D.get_child_nodes_diff(['vif-s', vif, 'vif-c'], expand_nodes=Diff.DELETE)['delete'].keys()
+        if keys:
+            dict.update({'vif_s': { vif : {'vif_c_remove': [*keys]}}})
 
-            if conf.exists(['sla-len']):
-                pd['sla_len'] = conf.return_value(['sla-len'])
+    return dict
 
-            if conf.exists(['address']):
-                pd['if_id'] = conf.return_value(['address'])
-
-            intf['dhcpv6_pd_interfaces'].append(pd)
-
-    # re-set config level
-    conf.set_level(current_level)
-
-    # DHCPv6 temporary IPv6 address
-    if conf.exists(['dhcpv6-options', 'temporary']):
-        intf['dhcpv6_temporary'] = True
-
-    # ignore link state changes
-    if conf.exists(['disable-link-detect']):
-        intf['disable_link_detect'] = 2
-
-    # ARP filter configuration
-    if conf.exists(['ip', 'disable-arp-filter']):
-        intf['ip_disable_arp_filter'] = 0
-
-    # ARP enable accept
-    if conf.exists(['ip', 'enable-arp-accept']):
-        intf['ip_enable_arp_accept'] = 1
-
-    # ARP enable announce
-    if conf.exists(['ip', 'enable-arp-announce']):
-        intf['ip_enable_arp_announce'] = 1
-
-    # ARP enable ignore
-    if conf.exists(['ip', 'enable-arp-ignore']):
-        intf['ip_enable_arp_ignore'] = 1
-
-    # Enable Proxy ARP
-    if conf.exists(['ip', 'enable-proxy-arp']):
-        intf['ip_proxy_arp'] = 1
-
-    # Enable acquisition of IPv6 address using stateless autoconfig (SLAAC)
-    if conf.exists(['ipv6', 'address', 'autoconf']):
-        intf['ipv6_autoconf'] = 1
-
-    # Disable IPv6 forwarding on this interface
-    if conf.exists(['ipv6', 'disable-forwarding']):
-        intf['ipv6_forwarding'] = 0
-
-    # check if interface is member of a bridge
-    intf['is_bridge_member'] = is_member(conf, intf['intf'], 'bridge')
-
-    # IPv6 Duplicate Address Detection (DAD) tries
-    if conf.exists(['ipv6', 'dup-addr-detect-transmits']):
-        intf['ipv6_dup_addr_detect'] = int(
-            conf.return_value(['ipv6', 'dup-addr-detect-transmits']))
-
-    # Media Access Control (MAC) address
-    if conf.exists(['mac']):
-        intf['mac'] = conf.return_value(['mac'])
-
-    # Maximum Transmission Unit (MTU)
-    if conf.exists(['mtu']):
-        intf['mtu'] = int(conf.return_value(['mtu']))
-
-    # retrieve VRF instance
-    if conf.exists(['vrf']):
-        intf['vrf'] = conf.return_value(['vrf'])
-
-    #  egress QoS
-    if conf.exists(['egress-qos']):
-        intf['egress_qos'] = conf.return_value(['egress-qos'])
-
-    # egress changes QoS require VLAN interface recreation
-    if conf.return_effective_value(['egress-qos']):
-        if intf['egress_qos'] != conf.return_effective_value(['egress-qos']):
-            intf['egress_qos_changed'] = True
-
-    # ingress QoS
-    if conf.exists(['ingress-qos']):
-        intf['ingress_qos'] = conf.return_value(['ingress-qos'])
-
-    # ingress changes QoS require VLAN interface recreation
-    if conf.return_effective_value(['ingress-qos']):
-        if intf['ingress_qos'] != conf.return_effective_value(['ingress-qos']):
-            intf['ingress_qos_changed'] = True
-
-    # Get the interface addresses
-    intf['address'] = conf.return_values(['address'])
-
-    # addresses to remove - difference between effective and working config
-    intf['address_remove'] = list_diff(
-            conf.return_effective_values(['address']), intf['address'])
-
-    # Get prefixes for IPv6 addressing based on MAC address (EUI-64)
-    intf['ipv6_eui64_prefix'] = conf.return_values(['ipv6', 'address', 'eui64'])
-
-    # EUI64 to remove - difference between effective and working config
-    intf['ipv6_eui64_prefix_remove'] = list_diff(
-            conf.return_effective_values(['ipv6', 'address', 'eui64']),
-            intf['ipv6_eui64_prefix'])
-
-    # Determine if the interface should be disabled
-    disabled = disable_state(conf)
-    if disabled == disable.both:
-        # was and is still disabled
-        intf['disable'] = True
-    elif disabled == disable.now:
-        # it is now disable but was not before
-        intf['disable'] = True
-    elif disabled == disable.was:
-        # it was disable but not anymore
-        intf['disable'] = False
-    else:
-        # normal change
-        intf['disable'] = False
-
-    # Remove the default link-local address if no-default-link-local is set,
-    # if member of a bridge or if disabled (it may not have a MAC if it's down)
-    if ( conf.exists(['ipv6', 'address', 'no-default-link-local'])
-            or intf.get('is_bridge_member') or intf['disable'] ):
-        intf['ipv6_eui64_prefix_remove'].append('fe80::/64')
-    else:
-        # add the link-local by default to make IPv6 work
-        intf['ipv6_eui64_prefix'].append('fe80::/64')
-
-    # If MAC has changed, remove and re-add all IPv6 EUI64 addresses
-    try:
-        interface = Interface(intf['intf'], create=False)
-        if intf['mac'] and intf['mac'] != interface.get_mac():
-            intf['ipv6_eui64_prefix_remove'] += intf['ipv6_eui64_prefix']
-    except Exception:
-        # If the interface does not exist, it could not have changed
-        pass
-
-    # to make IPv6 SLAAC and DHCPv6 work with forwarding=1,
-    # accept_ra must be 2
-    if intf['ipv6_autoconf'] or 'dhcpv6' in intf['address']:
-        intf['ipv6_accept_ra'] = 2
-
-    return intf, disable
-
-
-
-def add_to_dict(conf, disabled, ifdict, section, key):
+def get_interface_dict(config, base, ifname=''):
     """
-    parse a section of vif/vif-s/vif-c and add them to the dict
-    follow the convention to:
-    * use the "key" for what to add
-    * use the "key" what what to remove
-
-    conf:     is the Config() already at the level we need to parse
-    disabled: is a disable enum so we know how to handle to data
-    intf:     if the interface dictionary
-    section:  is the section name to parse (vif/vif-s/vif-c)
-    key:      is the dict key to use (vif/vifs/vifc)
+    Common utility function to retrieve and mandgle the interfaces available
+    in CLI configuration. All interfaces have a common base ground where the
+    value retrival is identical - so it can and should be reused
+
+    Will return a dictionary with the necessary interface configuration
     """
+    from vyos.xml import defaults
+
+    if not ifname:
+        # determine tagNode instance
+        if 'VYOS_TAGNODE_VALUE' not in os.environ:
+            raise ConfigError('Interface (VYOS_TAGNODE_VALUE) not specified')
+        ifname = os.environ['VYOS_TAGNODE_VALUE']
+
+    # retrieve interface default values
+    default_values = defaults(base)
+
+    # setup config level which is extracted in get_removed_vlans()
+    config.set_level(base + [ifname])
+    dict = config.get_config_dict([], key_mangling=('-', '_'), get_first_key=True)
+
+    # Check if interface has been removed
+    if dict == {}:
+        dict.update({'deleted' : ''})
+
+    # Add interface instance name into dictionary
+    dict.update({'ifname': ifname})
+
+    # We have gathered the dict representation of the CLI, but there are
+    # default options which we need to update into the dictionary
+    # retrived.
+    dict = dict_merge(default_values, dict)
+
+    # Check if we are a member of a bridge device
+    bridge = is_member(config, ifname, 'bridge')
+    if bridge:
+        dict.update({'is_bridge_member' : bridge})
+
+    # Check if we are a member of a bond device
+    bond = is_member(config, ifname, 'bonding')
+    if bond:
+        dict.update({'is_bond_member' : bond})
+
+    mac = leaf_node_changed(config, ['mac'])
+    if mac:
+        dict.update({'mac_old' : mac})
+
+    eui64 = leaf_node_changed(config, ['ipv6', 'address', 'eui64'])
+    if eui64:
+        # XXX: T2636 workaround: convert string to a list with one element
+        if isinstance(eui64, str):
+            eui64 = [eui64]
+        tmp = jmespath.search('ipv6.address', dict)
+        if not tmp:
+            dict.update({'ipv6': {'address': {'eui64_old': eui64}}})
+        else:
+            dict['ipv6']['address'].update({'eui64_old': eui64})
+
+    # remove wrongly inserted values
+    dict = T2665_default_dict_cleanup(dict)
+
+    # The values are identical for vif, vif-s and vif-c as the all include the same
+    # XML definitions which hold the defaults
+    default_vif_values = defaults(base + ['vif'])
+    for vif, vif_config in dict.get('vif', {}).items():
+        vif_config = dict_merge(default_vif_values, vif_config)
+    for vif_s, vif_s_config in dict.get('vif_s', {}).items():
+        vif_s_config = dict_merge(default_vif_values, vif_s_config)
+        for vif_c, vif_c_config in vif_s_config.get('vif_c', {}).items():
+            vif_c_config = dict_merge(default_vif_values, vif_c_config)
+
+    # Check vif, vif-s/vif-c VLAN interfaces for removal
+    dict = get_removed_vlans(config, dict)
+
+    return dict
 
-    if not conf.exists(section):
-        return ifdict
-
-    effect = conf.list_effective_nodes(section)
-    active = conf.list_nodes(section)
-
-    # the section to parse for vlan
-    sections = []
-
-    # determine which interfaces to add or remove based on disable state
-    if disabled == disable.both:
-        # was and is still disabled
-        ifdict[f'{key}_remove'] = []
-    elif disabled == disable.now:
-        # it is now disable but was not before
-        ifdict[f'{key}_remove'] = effect
-    elif disabled == disable.was:
-        # it was disable but not anymore
-        ifdict[f'{key}_remove'] = []
-        sections = active
-    else:
-        # normal change
-        # get interfaces (currently effective) - to determine which
-        # interface is no longer present and needs to be removed
-        ifdict[f'{key}_remove'] = list_diff(effect, active)
-        sections = active
-
-    current_level = conf.get_level()
-
-    # add each section, the key must already exists
-    for s in sections:
-        # set config level to vif interface
-        conf.set_level(current_level + [section, s])
-        # add the vlan config as a key (vlan id) - value (config) pair
-        ifdict[key][s] = vlan_to_dict(conf)
-
-    # re-set configuration level to leave things as found
-    conf.set_level(current_level)
-
-    return ifdict
-
-
-def vlan_to_dict(conf, default=vlan_default):
-    vlan, disabled = intf_to_dict(conf, default)
-
-    # if this is a not within vif-s node, we are done
-    if conf.get_level()[-2] != 'vif-s':
-        return vlan
-
-    # ethertype is mandatory on vif-s nodes and only exists here!
-    # ethertype uses a default of 0x88A8
-    tmp = '0x88A8'
-    if conf.exists('ethertype'):
-        tmp = conf.return_value('ethertype')
-    vlan['ethertype'] = get_ethertype(tmp)
-
-    # check if there is a Q-in-Q vlan customer interface
-    # and call this function recursively
-    add_to_dict(conf, disable, vlan, 'vif-c', 'vif_c')
-
-    return vlan