configdict: T5894: add get_config_dict() flag with_pki

VyOS has several services relaying on the PKI CLI tree to retrieve certificates.
Consuming services like ethernet, openvpn or ipsec all re-implemented the same
code to retrieve the certificates from the CLI.

This commit extends the signature of get_config_dict() with a new option with_pki
that defaults to false. If this option is set, the PKI CLI tree will be blended
into the resulting dictionary.

(cherry picked from commit b152b52023ba0cf0d4919eae39e92de28a458917)

11 files changed, 46 insertions, 69 deletions

diff --git a/python/vyos/config.py b/python/vyos/config.py
index 0ca41718f..ca7b035e5 100644
--- a/python/vyos/config.py
+++ b/python/vyos/config.py
@@ -29,7 +29,7 @@ There are multiple types of config tree nodes in VyOS, each requires
 its own set of operations.
 
 *Leaf nodes* (such as "address" in interfaces) can have values, but cannot
-have children. 
+have children.
 Leaf nodes can have one value, multiple values, or no values at all.
 
 For example, "system host-name" is a single-value leaf node,
@@ -258,7 +258,9 @@ class Config(object):
     def get_config_dict(self, path=[], effective=False, key_mangling=None,
                         get_first_key=False, no_multi_convert=False,
                         no_tag_node_value_mangle=False,
-                        with_defaults=False, with_recursive_defaults=False):
+                        with_defaults=False,
+                        with_recursive_defaults=False,
+                        with_pki=False):
         """
         Args:
             path (str list): Configuration tree path, can be empty
@@ -274,6 +276,7 @@ class Config(object):
         del kwargs['no_multi_convert']
         del kwargs['with_defaults']
         del kwargs['with_recursive_defaults']
+        del kwargs['with_pki']
 
         lpath = self._make_path(path)
         root_dict = self.get_cached_root_dict(effective)
@@ -298,6 +301,13 @@ class Config(object):
         else:
             conf_dict = ConfigDict(conf_dict)
 
+        if with_pki and conf_dict:
+            pki_dict = self.get_config_dict(['pki'], key_mangling=('-', '_'),
+                                            no_tag_node_value_mangle=True,
+                                            get_first_key=True)
+            if pki_dict:
+                conf_dict['pki'] = pki_dict
+
         # save optional args for a call to get_config_defaults
         setattr(conf_dict, '_dict_kwargs', kwargs)
 
diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py
index 089d9d3d5..4111d7271 100644
--- a/python/vyos/configdict.py
+++ b/python/vyos/configdict.py
@@ -427,7 +427,7 @@ def get_pppoe_interfaces(conf, vrf=None):
 
     return pppoe_interfaces
 
-def get_interface_dict(config, base, ifname='', recursive_defaults=True):
+def get_interface_dict(config, base, ifname='', recursive_defaults=True, with_pki=False):
     """
     Common utility function to retrieve and mangle the interfaces configuration
     from the CLI input nodes. All interfaces have a common base where value
@@ -459,7 +459,8 @@ def get_interface_dict(config, base, ifname='', recursive_defaults=True):
                                       get_first_key=True,
                                       no_tag_node_value_mangle=True,
                                       with_defaults=True,
-                                      with_recursive_defaults=recursive_defaults)
+                                      with_recursive_defaults=recursive_defaults,
+                                      with_pki=with_pki)
 
         # If interface does not request an IPv4 DHCP address there is no need
         # to keep the dhcp-options key
@@ -623,7 +624,7 @@ def get_vlan_ids(interface):
 
     return vlan_ids
 
-def get_accel_dict(config, base, chap_secrets):
+def get_accel_dict(config, base, chap_secrets, with_pki=False):
     """
     Common utility function to retrieve and mangle the Accel-PPP configuration
     from different CLI input nodes. All Accel-PPP services have a common base
@@ -638,7 +639,8 @@ def get_accel_dict(config, base, chap_secrets):
     dict = config.get_config_dict(base, key_mangling=('-', '_'),
                                   get_first_key=True,
                                   no_tag_node_value_mangle=True,
-                                  with_recursive_defaults=True)
+                                  with_recursive_defaults=True,
+                                  with_pki=with_pki)
 
     # set CPUs cores to process requests
     dict.update({'thread_count' : get_half_cpus()})
diff --git a/smoketest/scripts/cli/test_service_https.py b/smoketest/scripts/cli/test_service_https.py
index 703e3e8c4..280932fd7 100755
--- a/smoketest/scripts/cli/test_service_https.py
+++ b/smoketest/scripts/cli/test_service_https.py
@@ -395,8 +395,7 @@ class TestHTTPSService(VyOSUnitTestSHIM.TestCase):
 
     @ignore_warning(InsecureRequestWarning)
     def test_api_config_file_load_http(self):
-        """Test load config from HTTP URL
-        """
+        # Test load config from HTTP URL
         address = '127.0.0.1'
         key = 'VyOS-key'
         url = f'https://{address}/config-file'
diff --git a/src/conf_mode/interfaces_ethernet.py b/src/conf_mode/interfaces_ethernet.py
index 7374a29f7..2c0f846c3 100755
--- a/src/conf_mode/interfaces_ethernet.py
+++ b/src/conf_mode/interfaces_ethernet.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2019-2021 VyOS maintainers and contributors
+# Copyright (C) 2019-2024 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -150,19 +150,12 @@ def get_config(config=None):
     else:
         conf = Config()
 
-    # This must be called prior to get_interface_dict(), as this function will
-    # alter the config level (config.set_level())
-    pki = conf.get_config_dict(['pki'], key_mangling=('-', '_'),
-                               get_first_key=True, no_tag_node_value_mangle=True)
-
     base = ['interfaces', 'ethernet']
-    ifname, ethernet = get_interface_dict(conf, base)
+    ifname, ethernet = get_interface_dict(conf, base, with_pki=True)
+
     if 'is_bond_member' in ethernet:
         update_bond_options(conf, ethernet)
 
-    if 'deleted' not in ethernet:
-       if pki: ethernet['pki'] = pki
-
     tmp = is_node_changed(conf, base + [ifname, 'speed'])
     if tmp: ethernet.update({'speed_duplex_changed': {}})
 
@@ -171,8 +164,6 @@ def get_config(config=None):
 
     return ethernet
 
-
-
 def verify_speed_duplex(ethernet: dict, ethtool: Ethtool):
     """
      Verify speed and duplex
diff --git a/src/conf_mode/interfaces_openvpn.py b/src/conf_mode/interfaces_openvpn.py
index 85905fd9a..5795ce0bc 100755
--- a/src/conf_mode/interfaces_openvpn.py
+++ b/src/conf_mode/interfaces_openvpn.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2019-2023 VyOS maintainers and contributors
+# Copyright (C) 2019-2024 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -89,16 +89,12 @@ def get_config(config=None):
         conf = Config()
     base = ['interfaces', 'openvpn']
 
-    ifname, openvpn = get_interface_dict(conf, base)
+    ifname, openvpn = get_interface_dict(conf, base, with_pki=True)
     openvpn['auth_user_pass_file'] = '/run/openvpn/{ifname}.pw'.format(**openvpn)
 
     if 'deleted' in openvpn:
         return openvpn
 
-    openvpn['pki'] = conf.get_config_dict(['pki'], key_mangling=('-', '_'),
-                                        get_first_key=True,
-                                        no_tag_node_value_mangle=True)
-
     if is_node_changed(conf, base + [ifname, 'openvpn-option']):
         openvpn.update({'restart_required': {}})
     if is_node_changed(conf, base + [ifname, 'enable-dco']):
@@ -167,9 +163,10 @@ def verify_pki(openvpn):
             raise ConfigError(f'Invalid shared-secret on openvpn interface {interface}')
 
         # If PSK settings are correct, warn about its deprecation
-        DeprecationWarning("OpenVPN shared-secret support will be removed in future VyOS versions.\n\
-        Please migrate your site-to-site tunnels to TLS.\n\
-        You can use self-signed certificates with peer fingerprint verification, consult the documentation for details.")
+        DeprecationWarning('OpenVPN shared-secret support will be removed in future '\
+                           'VyOS versions. Please migrate your site-to-site tunnels to '\
+                           'TLS. You can use self-signed certificates with peer fingerprint '\
+                           'verification, consult the documentation for details.')
 
     if tls:
         if (mode in ['server', 'client']) and ('ca_certificate' not in tls):
@@ -728,4 +725,3 @@ if __name__ == '__main__':
     except ConfigError as e:
         print(e)
         exit(1)
-
diff --git a/src/conf_mode/interfaces_sstpc.py b/src/conf_mode/interfaces_sstpc.py
index b588910dc..b9d7a74fb 100755
--- a/src/conf_mode/interfaces_sstpc.py
+++ b/src/conf_mode/interfaces_sstpc.py
@@ -45,7 +45,7 @@ def get_config(config=None):
     else:
         conf = Config()
     base = ['interfaces', 'sstpc']
-    ifname, sstpc = get_interface_dict(conf, base)
+    ifname, sstpc = get_interface_dict(conf, base, with_pki=True)
 
     # We should only terminate the SSTP client session if critical parameters
     # change. All parameters that can be changed on-the-fly (like interface
@@ -57,10 +57,6 @@ def get_config(config=None):
             # bail out early - no need to further process other nodes
             break
 
-    # Load PKI certificates for later processing
-    sstpc['pki'] = conf.get_config_dict(['pki'], key_mangling=('-', '_'),
-                                        get_first_key=True,
-                                        no_tag_node_value_mangle=True)
     return sstpc
 
 def verify(sstpc):
diff --git a/src/conf_mode/load-balancing_reverse-proxy.py b/src/conf_mode/load-balancing_reverse-proxy.py
index 333ebc66c..7338fe573 100755
--- a/src/conf_mode/load-balancing_reverse-proxy.py
+++ b/src/conf_mode/load-balancing_reverse-proxy.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2023 VyOS maintainers and contributors
+# Copyright (C) 2023-2024 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -43,17 +43,14 @@ def get_config(config=None):
         conf = Config()
 
     base = ['load-balancing', 'reverse-proxy']
+    if not conf.exists(base):
+        return None
     lb = conf.get_config_dict(base,
                               get_first_key=True,
                               key_mangling=('-', '_'),
-                              no_tag_node_value_mangle=True)
-
-    if lb:
-        lb['pki'] = conf.get_config_dict(['pki'], key_mangling=('-', '_'),
-                                    get_first_key=True, no_tag_node_value_mangle=True)
-
-    if lb:
-        lb = conf.merge_defaults(lb, recursive=True)
+                              no_tag_node_value_mangle=True,
+                              with_recursive_defaults=True,
+                              with_pki=True)
 
     return lb
 
diff --git a/src/conf_mode/service_https.py b/src/conf_mode/service_https.py
index 3dc5dfc01..cb40acc9f 100755
--- a/src/conf_mode/service_https.py
+++ b/src/conf_mode/service_https.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2019-2023 VyOS maintainers and contributors
+# Copyright (C) 2019-2024 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -78,12 +78,7 @@ def get_config(config=None):
 
     diff = get_config_diff(conf)
 
-    https = conf.get_config_dict(base, get_first_key=True)
-
-    if https:
-        https['pki'] = conf.get_config_dict(['pki'], key_mangling=('-', '_'),
-                                            no_tag_node_value_mangle=True,
-                                            get_first_key=True)
+    https = conf.get_config_dict(base, get_first_key=True, with_pki=True)
 
     https['children_changed'] = diff.node_changed_children(base)
     https['api_add_or_delete'] = diff.node_changed_presence(base + ['api'])
@@ -119,7 +114,7 @@ def verify(https):
 
         if 'certificate' in certificates:
             if not https['pki']:
-                raise ConfigError("PKI is not configured")
+                raise ConfigError('PKI is not configured')
 
             cert_name = certificates['certificate']
 
diff --git a/src/conf_mode/vpn_ipsec.py b/src/conf_mode/vpn_ipsec.py
index 9e9385ddb..7fd32c230 100755
--- a/src/conf_mode/vpn_ipsec.py
+++ b/src/conf_mode/vpn_ipsec.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2021-2023 VyOS maintainers and contributors
+# Copyright (C) 2021-2024 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -87,15 +87,13 @@ def get_config(config=None):
     ipsec = conf.get_config_dict(base, key_mangling=('-', '_'),
                                  no_tag_node_value_mangle=True,
                                  get_first_key=True,
-                                 with_recursive_defaults=True)
+                                 with_recursive_defaults=True,
+                                 with_pki=True)
 
     ipsec['dhcp_no_address'] = {}
     ipsec['install_routes'] = 'no' if conf.exists(base + ["options", "disable-route-autoinstall"]) else default_install_routes
     ipsec['interface_change'] = leaf_node_changed(conf, base + ['interface'])
     ipsec['nhrp_exists'] = conf.exists(['protocols', 'nhrp', 'tunnel'])
-    ipsec['pki'] = conf.get_config_dict(['pki'], key_mangling=('-', '_'),
-                                        no_tag_node_value_mangle=True,
-                                        get_first_key=True)
 
     tmp = conf.get_config_dict(l2tp_base, key_mangling=('-', '_'),
                                no_tag_node_value_mangle=True,
diff --git a/src/conf_mode/vpn_openconnect.py b/src/conf_mode/vpn_openconnect.py
index a039172c4..421ac6997 100755
--- a/src/conf_mode/vpn_openconnect.py
+++ b/src/conf_mode/vpn_openconnect.py
@@ -56,12 +56,8 @@ def get_config(config=None):
 
     ocserv = conf.get_config_dict(base, key_mangling=('-', '_'),
                                   get_first_key=True,
-                                  with_recursive_defaults=True)
-
-    if ocserv:
-        ocserv['pki'] = conf.get_config_dict(['pki'], key_mangling=('-', '_'),
-                                             no_tag_node_value_mangle=True,
-                                             get_first_key=True)
+                                  with_recursive_defaults=True,
+                                  with_pki=True)
 
     return ocserv
 
diff --git a/src/conf_mode/vpn_sstp.py b/src/conf_mode/vpn_sstp.py
index ac053cc76..6bf9307e1 100755
--- a/src/conf_mode/vpn_sstp.py
+++ b/src/conf_mode/vpn_sstp.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2018-2023 VyOS maintainers and contributors
+# Copyright (C) 2018-2024 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -54,14 +54,11 @@ def get_config(config=None):
         return None
 
     # retrieve common dictionary keys
-    sstp = get_accel_dict(conf, base, sstp_chap_secrets)
+    sstp = get_accel_dict(conf, base, sstp_chap_secrets, with_pki=True)
     if dict_search('client_ip_pool', sstp):
         # Multiple named pools require ordered values T5099
         sstp['ordered_named_pools'] = get_pools_in_order(dict_search('client_ip_pool', sstp))
-    if sstp:
-        sstp['pki'] = conf.get_config_dict(['pki'], key_mangling=('-', '_'),
-                                           get_first_key=True,
-                                           no_tag_node_value_mangle=True)
+
     sstp['server_type'] = 'sstp'
     return sstp