vyos.configdict: T4391: enable get_interface_dict() ti be used with ConfigTreeQuery()

When VyOS is booting and an interface is brought up (PPPoE) which requires a
user callback script that is executed asynchronously when the interface is up
we can not use Config(). The problem is, Config() is not available when
the system starts and the initial commit is still processed.

We need to move to ConfigTreeQuery() which was build for this exact same
purpose. TO reduce side effects and also dependencies on the entire
vyos.configdict library the set_level()/get_level() calls got eliminated
from within the library. All calls to functions like:

* get_removed_vlans()
* is_node_changed()
* leaf_node_changed()
* is_mirror_intf()
* ...

Now require that the full config path to the node is passed.

18 files changed, 61 insertions, 65 deletions

diff --git a/src/conf_mode/containers.py b/src/conf_mode/containers.py
index 516671844..3c1a61f19 100755
--- a/src/conf_mode/containers.py
+++ b/src/conf_mode/containers.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021 VyOS maintainers and contributors
+# Copyright (C) 2021-2022 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -28,7 +28,6 @@ from vyos.configdict import node_changed
 from vyos.util import call
 from vyos.util import cmd
 from vyos.util import run
-from vyos.util import read_file
 from vyos.util import write_file
 from vyos.template import inc_ip
 from vyos.template import is_ipv4
@@ -77,10 +76,10 @@ def get_config(config=None):
             container['name'][name] = dict_merge(default_values, container['name'][name])
 
     # Delete container network, delete containers
-    tmp = node_changed(conf, ['container', 'network'])
+    tmp = node_changed(conf, base + ['container', 'network'])
     if tmp: container.update({'network_remove' : tmp})
 
-    tmp = node_changed(conf, ['container', 'name'])
+    tmp = node_changed(conf, base + ['container', 'name'])
     if tmp: container.update({'container_remove' : tmp})
 
     return container
diff --git a/src/conf_mode/interfaces-bonding.py b/src/conf_mode/interfaces-bonding.py
index ad5a0f499..4167594e3 100755
--- a/src/conf_mode/interfaces-bonding.py
+++ b/src/conf_mode/interfaces-bonding.py
@@ -68,7 +68,7 @@ def get_config(config=None):
     else:
         conf = Config()
     base = ['interfaces', 'bonding']
-    bond = get_interface_dict(conf, base)
+    ifname, bond = get_interface_dict(conf, base)
 
     # To make our own life easier transfor the list of member interfaces
     # into a dictionary - we will use this to add additional information
@@ -81,14 +81,14 @@ def get_config(config=None):
     if 'mode' in bond:
         bond['mode'] = get_bond_mode(bond['mode'])
 
-    tmp = leaf_node_changed(conf, ['mode'])
+    tmp = leaf_node_changed(conf, base + [ifname, 'mode'])
     if tmp: bond.update({'shutdown_required': {}})
 
-    tmp = leaf_node_changed(conf, ['lacp-rate'])
+    tmp = leaf_node_changed(conf, base + [ifname, 'lacp-rate'])
     if tmp: bond.update({'shutdown_required': {}})
 
     # determine which members have been removed
-    interfaces_removed = leaf_node_changed(conf, ['member', 'interface'])
+    interfaces_removed = leaf_node_changed(conf, base + [ifname, 'member', 'interface'])
     if interfaces_removed:
         bond.update({'shutdown_required': {}})
         if 'member' not in bond:
diff --git a/src/conf_mode/interfaces-bridge.py b/src/conf_mode/interfaces-bridge.py
index b1f7e6d7c..38ae727c1 100755
--- a/src/conf_mode/interfaces-bridge.py
+++ b/src/conf_mode/interfaces-bridge.py
@@ -50,15 +50,15 @@ def get_config(config=None):
     else:
         conf = Config()
     base = ['interfaces', 'bridge']
-    bridge = get_interface_dict(conf, base)
+    ifname, bridge = get_interface_dict(conf, base)
 
     # determine which members have been removed
-    tmp = node_changed(conf, ['member', 'interface'], key_mangling=('-', '_'))
+    tmp = node_changed(conf, base + [ifname, 'member', 'interface'], key_mangling=('-', '_'))
     if tmp:
         if 'member' in bridge:
-            bridge['member'].update({'interface_remove': tmp })
+            bridge['member'].update({'interface_remove' : tmp })
         else:
-            bridge.update({'member': {'interface_remove': tmp }})
+            bridge.update({'member' : {'interface_remove' : tmp }})
 
     if dict_search('member.interface', bridge):
         # XXX: T2665: we need a copy of the dict keys for iteration, else we will get:
diff --git a/src/conf_mode/interfaces-dummy.py b/src/conf_mode/interfaces-dummy.py
index 4a1eb7b93..e771581e1 100755
--- a/src/conf_mode/interfaces-dummy.py
+++ b/src/conf_mode/interfaces-dummy.py
@@ -37,7 +37,7 @@ def get_config(config=None):
     else:
         conf = Config()
     base = ['interfaces', 'dummy']
-    dummy = get_interface_dict(conf, base)
+    _, dummy = get_interface_dict(conf, base)
     return dummy
 
 def verify(dummy):
diff --git a/src/conf_mode/interfaces-ethernet.py b/src/conf_mode/interfaces-ethernet.py
index 333d39e0e..fec4456fb 100755
--- a/src/conf_mode/interfaces-ethernet.py
+++ b/src/conf_mode/interfaces-ethernet.py
@@ -65,7 +65,7 @@ def get_config(config=None):
                                get_first_key=True, no_tag_node_value_mangle=True)
 
     base = ['interfaces', 'ethernet']
-    ethernet = get_interface_dict(conf, base)
+    _, ethernet = get_interface_dict(conf, base)
 
     if 'deleted' not in ethernet:
        if pki: ethernet['pki'] = pki
diff --git a/src/conf_mode/interfaces-geneve.py b/src/conf_mode/interfaces-geneve.py
index 26d248579..b9cf2fa3c 100755
--- a/src/conf_mode/interfaces-geneve.py
+++ b/src/conf_mode/interfaces-geneve.py
@@ -22,7 +22,7 @@ from netifaces import interfaces
 from vyos.config import Config
 from vyos.configdict import get_interface_dict
 from vyos.configdict import leaf_node_changed
-from vyos.configdict import node_changed
+from vyos.configdict import is_node_changed
 from vyos.configverify import verify_address
 from vyos.configverify import verify_mtu_ipv6
 from vyos.configverify import verify_bridge_delete
@@ -43,16 +43,16 @@ def get_config(config=None):
     else:
         conf = Config()
     base = ['interfaces', 'geneve']
-    geneve = get_interface_dict(conf, base)
+    ifname, geneve = get_interface_dict(conf, base)
 
     # GENEVE interfaces are picky and require recreation if certain parameters
     # change. But a GENEVE interface should - of course - not be re-created if
     # it's description or IP address is adjusted. Feels somehow logic doesn't it?
     for cli_option in ['remote', 'vni']:
-        if leaf_node_changed(conf, cli_option):
+        if leaf_node_changed(conf, base + [ifname, cli_option]):
             geneve.update({'rebuild_required': {}})
 
-    if node_changed(conf, ['parameters'], recursive=True):
+    if is_node_changed(conf, base + [ifname, 'parameters']):
         geneve.update({'rebuild_required': {}})
 
     return geneve
diff --git a/src/conf_mode/interfaces-l2tpv3.py b/src/conf_mode/interfaces-l2tpv3.py
index 22256bf4f..6a486f969 100755
--- a/src/conf_mode/interfaces-l2tpv3.py
+++ b/src/conf_mode/interfaces-l2tpv3.py
@@ -45,15 +45,15 @@ def get_config(config=None):
     else:
         conf = Config()
     base = ['interfaces', 'l2tpv3']
-    l2tpv3 = get_interface_dict(conf, base)
+    ifname, l2tpv3 = get_interface_dict(conf, base)
 
     # To delete an l2tpv3 interface we need the current tunnel and session-id
     if 'deleted' in l2tpv3:
-        tmp = leaf_node_changed(conf, ['tunnel-id'])
+        tmp = leaf_node_changed(conf, base + [ifname, 'tunnel-id'])
         # leaf_node_changed() returns a list
         l2tpv3.update({'tunnel_id': tmp[0]})
 
-        tmp = leaf_node_changed(conf, ['session-id'])
+        tmp = leaf_node_changed(conf, base + [ifname, 'session-id'])
         l2tpv3.update({'session_id': tmp[0]})
 
     return l2tpv3
diff --git a/src/conf_mode/interfaces-loopback.py b/src/conf_mode/interfaces-loopback.py
index e4bc15bb5..08d34477a 100755
--- a/src/conf_mode/interfaces-loopback.py
+++ b/src/conf_mode/interfaces-loopback.py
@@ -36,7 +36,7 @@ def get_config(config=None):
     else:
         conf = Config()
     base = ['interfaces', 'loopback']
-    loopback = get_interface_dict(conf, base)
+    _, loopback = get_interface_dict(conf, base)
     return loopback
 
 def verify(loopback):
diff --git a/src/conf_mode/interfaces-macsec.py b/src/conf_mode/interfaces-macsec.py
index c71863e61..279dd119b 100755
--- a/src/conf_mode/interfaces-macsec.py
+++ b/src/conf_mode/interfaces-macsec.py
@@ -48,7 +48,7 @@ def get_config(config=None):
     else:
         conf = Config()
     base = ['interfaces', 'macsec']
-    macsec = get_interface_dict(conf, base)
+    ifname, macsec = get_interface_dict(conf, base)
 
     # Check if interface has been removed
     if 'deleted' in macsec:
diff --git a/src/conf_mode/interfaces-openvpn.py b/src/conf_mode/interfaces-openvpn.py
index 6c1a01dab..4750ca3e8 100755
--- a/src/conf_mode/interfaces-openvpn.py
+++ b/src/conf_mode/interfaces-openvpn.py
@@ -85,12 +85,11 @@ def get_config(config=None):
     tmp_pki = conf.get_config_dict(['pki'], key_mangling=('-', '_'),
                                 get_first_key=True, no_tag_node_value_mangle=True)
 
-    openvpn = get_interface_dict(conf, base)
+    ifname, openvpn = get_interface_dict(conf, base)
 
     if 'deleted' not in openvpn:
         openvpn['pki'] = tmp_pki
-
-        if is_node_changed(conf, ['openvpn-option']):
+        if is_node_changed(conf, base + [ifname, 'openvpn-option']):
             openvpn.update({'restart_required': {}})
 
         # We have to get the dict using 'get_config_dict' instead of 'get_interface_dict'
diff --git a/src/conf_mode/interfaces-pppoe.py b/src/conf_mode/interfaces-pppoe.py
index 279369a32..26daa8381 100755
--- a/src/conf_mode/interfaces-pppoe.py
+++ b/src/conf_mode/interfaces-pppoe.py
@@ -49,14 +49,14 @@ def get_config(config=None):
     else:
         conf = Config()
     base = ['interfaces', 'pppoe']
-    pppoe = get_interface_dict(conf, base)
+    ifname, pppoe = get_interface_dict(conf, base)
 
     # We should only terminate the PPPoE session if critical parameters change.
     # All parameters that can be changed on-the-fly (like interface description)
     # should not lead to a reconnect!
     for options in ['access-concentrator', 'connect-on-demand', 'service-name',
                     'source-interface', 'vrf', 'no-default-route', 'authentication']:
-        if is_node_changed(conf, options):
+        if is_node_changed(conf, base + [ifname, options]):
             pppoe.update({'shutdown_required': {}})
             # bail out early - no need to further process other nodes
             break
diff --git a/src/conf_mode/interfaces-pseudo-ethernet.py b/src/conf_mode/interfaces-pseudo-ethernet.py
index f2c85554f..1cd3fe276 100755
--- a/src/conf_mode/interfaces-pseudo-ethernet.py
+++ b/src/conf_mode/interfaces-pseudo-ethernet.py
@@ -18,7 +18,7 @@ from sys import exit
 
 from vyos.config import Config
 from vyos.configdict import get_interface_dict
-from vyos.configdict import leaf_node_changed
+from vyos.configdict import is_node_changed
 from vyos.configverify import verify_vrf
 from vyos.configverify import verify_address
 from vyos.configverify import verify_bridge_delete
@@ -42,14 +42,14 @@ def get_config(config=None):
     else:
         conf = Config()
     base = ['interfaces', 'pseudo-ethernet']
-    peth = get_interface_dict(conf, base)
+    ifname, peth = get_interface_dict(conf, base)
 
-    mode = leaf_node_changed(conf, ['mode'])
-    if mode: peth.update({'mode_old' : mode})
+    mode = is_node_changed(conf, ['mode'])
+    if mode: peth.update({'shutdown_required' : {}})
 
     if 'source_interface' in peth:
-        peth['parent'] = get_interface_dict(conf, ['interfaces', 'ethernet'],
-                                            peth['source_interface'])
+        _, peth['parent'] = get_interface_dict(conf, ['interfaces', 'ethernet'],
+                                               peth['source_interface'])
     return peth
 
 def verify(peth):
diff --git a/src/conf_mode/interfaces-tunnel.py b/src/conf_mode/interfaces-tunnel.py
index f4668d976..eff7f373c 100755
--- a/src/conf_mode/interfaces-tunnel.py
+++ b/src/conf_mode/interfaces-tunnel.py
@@ -48,10 +48,10 @@ def get_config(config=None):
     else:
         conf = Config()
     base = ['interfaces', 'tunnel']
-    tunnel = get_interface_dict(conf, base)
+    ifname, tunnel = get_interface_dict(conf, base)
 
     if 'deleted' not in tunnel:
-        tmp = leaf_node_changed(conf, ['encapsulation'])
+        tmp = leaf_node_changed(conf, base + [ifname, 'encapsulation'])
         if tmp: tunnel.update({'encapsulation_changed': {}})
 
         # We also need to inspect other configured tunnels as there are Kernel
diff --git a/src/conf_mode/interfaces-vti.py b/src/conf_mode/interfaces-vti.py
index f06fdff1b..f4b0436af 100755
--- a/src/conf_mode/interfaces-vti.py
+++ b/src/conf_mode/interfaces-vti.py
@@ -36,7 +36,7 @@ def get_config(config=None):
     else:
         conf = Config()
     base = ['interfaces', 'vti']
-    vti = get_interface_dict(conf, base)
+    _, vti = get_interface_dict(conf, base)
     return vti
 
 def verify(vti):
diff --git a/src/conf_mode/interfaces-vxlan.py b/src/conf_mode/interfaces-vxlan.py
index 53704827e..f44d754ba 100755
--- a/src/conf_mode/interfaces-vxlan.py
+++ b/src/conf_mode/interfaces-vxlan.py
@@ -23,7 +23,7 @@ from vyos.base import Warning
 from vyos.config import Config
 from vyos.configdict import get_interface_dict
 from vyos.configdict import leaf_node_changed
-from vyos.configdict import node_changed
+from vyos.configdict import is_node_changed
 from vyos.configverify import verify_address
 from vyos.configverify import verify_bridge_delete
 from vyos.configverify import verify_mtu_ipv6
@@ -46,17 +46,17 @@ def get_config(config=None):
     else:
         conf = Config()
     base = ['interfaces', 'vxlan']
-    vxlan = get_interface_dict(conf, base)
+    ifname, vxlan = get_interface_dict(conf, base)
 
     # VXLAN interfaces are picky and require recreation if certain parameters
     # change. But a VXLAN interface should - of course - not be re-created if
     # it's description or IP address is adjusted. Feels somehow logic doesn't it?
     for cli_option in ['external', 'gpe', 'group', 'port', 'remote',
                        'source-address', 'source-interface', 'vni']:
-        if leaf_node_changed(conf, cli_option):
+        if leaf_node_changed(conf, base + [ifname, cli_option]):
             vxlan.update({'rebuild_required': {}})
 
-    if node_changed(conf, ['parameters'], recursive=True):
+    if is_node_changed(conf, base + [ifname, 'parameters']):
         vxlan.update({'rebuild_required': {}})
 
     # We need to verify that no other VXLAN tunnel is configured when external
diff --git a/src/conf_mode/interfaces-wireguard.py b/src/conf_mode/interfaces-wireguard.py
index b404375d6..180ffa507 100755
--- a/src/conf_mode/interfaces-wireguard.py
+++ b/src/conf_mode/interfaces-wireguard.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2018-2020 VyOS maintainers and contributors
+# Copyright (C) 2018-2022 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -46,17 +46,17 @@ def get_config(config=None):
     else:
         conf = Config()
     base = ['interfaces', 'wireguard']
-    wireguard = get_interface_dict(conf, base)
+    ifname, wireguard = get_interface_dict(conf, base)
 
     # Check if a port was changed
-    wireguard['port_changed'] = leaf_node_changed(conf, ['port'])
+    wireguard['port_changed'] = leaf_node_changed(conf, base + [ifname, 'port'])
 
     # Determine which Wireguard peer has been removed.
     # Peers can only be removed with their public key!
     dict = {}
-    tmp = node_changed(conf, ['peer'], key_mangling=('-', '_'))
+    tmp = node_changed(conf, base + [ifname, 'peer'], key_mangling=('-', '_'))
     for peer in (tmp or []):
-        public_key = leaf_node_changed(conf, ['peer', peer, 'public_key'])
+        public_key = leaf_node_changed(conf, base + [ifname, 'peer', peer, 'public_key'])
         if public_key:
             dict = dict_merge({'peer_remove' : {peer : {'public_key' : public_key[0]}}}, dict)
             wireguard.update(dict)
diff --git a/src/conf_mode/interfaces-wireless.py b/src/conf_mode/interfaces-wireless.py
index 7fc22cdab..d34297063 100755
--- a/src/conf_mode/interfaces-wireless.py
+++ b/src/conf_mode/interfaces-wireless.py
@@ -76,15 +76,19 @@ def get_config(config=None):
         conf = Config()
     base = ['interfaces', 'wireless']
 
-    wifi = get_interface_dict(conf, base)
+    ifname, wifi = get_interface_dict(conf, base)
 
     # Cleanup "delete" default values when required user selectable values are
     # not defined at all
-    tmp = conf.get_config_dict([], key_mangling=('-', '_'), get_first_key=True)
+    tmp = conf.get_config_dict(base + [ifname], key_mangling=('-', '_'),
+                               get_first_key=True)
     if not (dict_search('security.wpa.passphrase', tmp) or
             dict_search('security.wpa.radius', tmp)):
         if 'deleted' not in wifi:
             del wifi['security']['wpa']
+            # if 'security' key is empty, drop it too
+            if len(wifi['security']) == 0:
+                del wifi['security']
 
     # defaults include RADIUS server specifics per TAG node which need to be
     # added to individual RADIUS servers instead - so we can simply delete them
diff --git a/src/conf_mode/interfaces-wwan.py b/src/conf_mode/interfaces-wwan.py
index 9a33039a3..e275ace84 100755
--- a/src/conf_mode/interfaces-wwan.py
+++ b/src/conf_mode/interfaces-wwan.py
@@ -21,7 +21,7 @@ from time import sleep
 
 from vyos.config import Config
 from vyos.configdict import get_interface_dict
-from vyos.configdict import leaf_node_changed
+from vyos.configdict import is_node_changed
 from vyos.configverify import verify_authentication
 from vyos.configverify import verify_interface_exists
 from vyos.configverify import verify_mirror_redirect
@@ -50,42 +50,36 @@ def get_config(config=None):
     else:
         conf = Config()
     base = ['interfaces', 'wwan']
-    wwan = get_interface_dict(conf, base)
+    ifname, wwan = get_interface_dict(conf, base)
 
     # We should only terminate the WWAN session if critical parameters change.
     # All parameters that can be changed on-the-fly (like interface description)
     # should not lead to a reconnect!
-    tmp = leaf_node_changed(conf, ['address'])
+    tmp = is_node_changed(conf, base + [ifname, 'address'])
     if tmp: wwan.update({'shutdown_required': {}})
 
-    tmp = leaf_node_changed(conf, ['apn'])
+    tmp = is_node_changed(conf, base + [ifname, 'apn'])
     if tmp: wwan.update({'shutdown_required': {}})
 
-    tmp = leaf_node_changed(conf, ['disable'])
+    tmp = is_node_changed(conf, base + [ifname, 'disable'])
     if tmp: wwan.update({'shutdown_required': {}})
 
-    tmp = leaf_node_changed(conf, ['vrf'])
-    # leaf_node_changed() returns a list, as VRF is a non-multi node, there
-    # will be only one list element
-    if tmp: wwan.update({'vrf_old': tmp[0]})
-
-    tmp = leaf_node_changed(conf, ['authentication', 'user'])
+    tmp = is_node_changed(conf, base + [ifname, 'vrf'])
     if tmp: wwan.update({'shutdown_required': {}})
 
-    tmp = leaf_node_changed(conf, ['authentication', 'password'])
+    tmp = is_node_changed(conf, base + [ifname, 'authentication'])
     if tmp: wwan.update({'shutdown_required': {}})
 
-    tmp = leaf_node_changed(conf, ['ipv6', 'address', 'autoconf'])
+    tmp = is_node_changed(conf, base + [ifname, 'ipv6', 'address', 'autoconf'])
     if tmp: wwan.update({'shutdown_required': {}})
 
     # We need to know the amount of other WWAN interfaces as ModemManager needs
     # to be started or stopped.
     conf.set_level(base)
-    wwan['other_interfaces'] = conf.get_config_dict([], key_mangling=('-', '_'),
-                                                    get_first_key=True,
-                                                    no_tag_node_value_mangle=True)
+    _, wwan['other_interfaces'] = conf.get_config_dict([], key_mangling=('-', '_'),
+                                                       get_first_key=True,
+                                                       no_tag_node_value_mangle=True)
 
-    ifname = wwan['ifname']
     # This if-clause is just to be sure - it will always evaluate to true
     if ifname in wwan['other_interfaces']:
         del wwan['other_interfaces'][ifname]