vyos: T9261: defer more heavy imports off the CLI hot paths

Follow-up sweep to f696b2d65. Every conf-mode script not handled in-daemon by
vyos-configd, every op-mode script and every Python validator is a fresh
interpreter, so module-scope imports of heavy dependencies are paid over and
over - which is what makes the CLI feel sluggish on CPUs with little cache.

configsource.py imported vyos.vyconf_session, and with it google.protobuf, for
a class that is only instantiated when the vyconf backend is active. opmode.py
imported ConfigTreeQuery for a single decorator most op-mode scripts never use.
template.py imported jinja2 at module scope although Environment is only touched
in the lru_cache'd _get_environment(), while most callers want just the
is_ipv4/is_ipv6 predicates. ethtool.py pulled pyroute2 via vyos.netlink for two
calls in Ethtool.__init__. ifconfig/interface.py pulled the whole cryptography
stack via vyos.pki for set_eapol() alone. Also deferred: netaddr, jmespath,
tabulate, hurry.filesize, vyos.ifconfig in qos, and cracklib - the last of which
also opens its dictionary database on import.

  import vyos.opmode      34.0 -> 4.6 ms   (-86%), 206 -> 71 modules
  import vyos.ifconfig   132.9 -> 30.8 ms  (-77%), 618 -> 193 modules
  import vyos.config      40.7 -> 20.6 ms  (-49%), 194 -> 130 modules
  import vyos.qos        134.1 -> 12.5 ms  (-91%), 631 -> 124 modules
  import vyos.template    18.8 -> 6.3 ms   (-66%), 153 -> 89 modules
  import vyos.firewall    22.2 -> 12.2 ms  (-45%), 168 -> 114 modules

Deferring textwrap in base.py and logging in configtree.py was measured too and
made no difference; both were left alone.

11 files changed, 100 insertions, 40 deletions

diff --git a/python/vyos/configsource.py b/python/vyos/configsource.py
index b54f6283a..ce9b9a62a 100644
--- a/python/vyos/configsource.py
+++ b/python/vyos/configsource.py
@@ -21,8 +21,6 @@ from typing import Union
 
 from vyos.configtree import ConfigTree
 from vyos.utils.boot import boot_configuration_complete
-from vyos.vyconf_session import VyconfSession
-from vyos.vyconf_session import VyconfSessionError
 from vyos.defaults import directories
 from vyos.xml_ref import is_tag
 from vyos.xml_ref import is_leaf
@@ -332,6 +330,13 @@ class ConfigSourceVyconfSession(ConfigSource):
             pid = os.environ.get('SESSION_PID', '')
             self.pid = int(pid) if pid else os.getppid()
 
+        # Imported here rather than at module scope: vyos.vyconf_session pulls
+        # in vyos.proto.vyconf_client and with it google.protobuf, which every
+        # consumer of vyos.config would otherwise pay for on import even when
+        # the vyconf backend is not in use.
+        from vyos.vyconf_session import VyconfSession
+        from vyos.vyconf_session import VyconfSessionError
+
         self._vyconf_session = VyconfSession(pid=self.pid)
         try:
             out = self._vyconf_session.get_config()
@@ -366,6 +371,8 @@ class ConfigSourceVyconfSession(ConfigSource):
         Returns:
             True if the config session has uncommitted changes, False otherwise.
         """
+        from vyos.vyconf_session import VyconfSessionError
+
         try:
             return self._vyconf_session.session_changed()
         except VyconfSessionError:
diff --git a/python/vyos/ethtool.py b/python/vyos/ethtool.py
index cbc13a0e9..66dc6bb89 100644
--- a/python/vyos/ethtool.py
+++ b/python/vyos/ethtool.py
@@ -19,8 +19,6 @@ import contextlib
 from json import loads
 from vyos.utils.network import interface_exists
 from vyos.utils.process import popen
-from vyos.netlink import coalesce
-from vyos.netlink import timestamp
 
 # These drivers do not support using ethtool to change the speed, duplex, or
 # flow control settings
@@ -127,6 +125,12 @@ class Ethtool:
         if not bool(err):
             self._channels = out.lower()
 
+        # Imported here rather than at module scope: vyos.netlink.* imports
+        # pyroute2, which is by far the largest single cost on the vyos.ifconfig
+        # import path. Only code that actually constructs an Ethtool needs it.
+        from vyos.netlink import coalesce
+        from vyos.netlink import timestamp
+
         # Get information about NIC coalesce settings
         with contextlib.suppress(coalesce.CoalesceError, coalesce.GeneralNetlinkError):
             self._coalesce = coalesce.get_coalesce(ifname)
diff --git a/python/vyos/ifconfig/ethernet.py b/python/vyos/ifconfig/ethernet.py
index 84c002550..bb5119f77 100644
--- a/python/vyos/ifconfig/ethernet.py
+++ b/python/vyos/ifconfig/ethernet.py
@@ -19,7 +19,6 @@ from glob import glob
 
 from vyos.base import Warning
 from vyos.ethtool import Ethtool
-from vyos.netlink import coalesce
 from vyos.ifconfig import Section
 from vyos.ifconfig.interface import Interface
 from vyos.utils.dict import dict_search
@@ -512,6 +511,11 @@ class EthernetIf(Interface):
         if not params:
             return None
 
+        # Imported here rather than at module scope: vyos.netlink.coalesce
+        # imports pyroute2, which every consumer of vyos.ifconfig would
+        # otherwise pay for on import.
+        from vyos.netlink import coalesce
+
         # Override boolean parameters to true if they exist and supported by NIC driver
         for boolean_param in coalesce.get_all_params(boolean=True):
             supported = self.ethtool.check_coalesce(boolean_param)
diff --git a/python/vyos/ifconfig/interface.py b/python/vyos/ifconfig/interface.py
index e62812d31..3a5efec8c 100644
--- a/python/vyos/ifconfig/interface.py
+++ b/python/vyos/ifconfig/interface.py
@@ -16,7 +16,6 @@
 import os
 import re
 import json
-import jmespath
 
 from copy import deepcopy
 from glob import glob
@@ -26,17 +25,11 @@ from ipaddress import IPv6Interface
 from netifaces import ifaddresses # pylint: disable = no-name-in-module
 from socket import AF_INET
 from socket import AF_INET6
-from netaddr import EUI
-from netaddr import mac_unix_expanded
 
 from vyos.configdict import list_diff
 from vyos.configdict import dict_merge
 from vyos.configdict import get_vlan_ids
 from vyos.defaults import directories
-from vyos.pki import find_chain
-from vyos.pki import encode_certificate
-from vyos.pki import load_certificate
-from vyos.pki import wrap_private_key
 from vyos.template import is_ipv4
 from vyos.template import is_ipv6
 from vyos.template import render
@@ -68,6 +61,18 @@ from vyos.ifconfig import Section
 
 link_local_prefix = 'fe80::/64'
 
+def _jmes_search(expression, data):
+    """jmespath.search with a deferred import.
+
+    The lookups below live in lambdas evaluated at call time, so importing
+    jmespath at module scope would put it on the vyos.ifconfig import path for
+    no reason.
+    """
+    import jmespath
+
+    return jmespath.search(expression, data)
+
+
 class Interface(Control):
     # This is the class which will be used to create
     # self.operational, it allows subclasses, such as
@@ -92,39 +97,39 @@ class Interface(Control):
     _command_get = {
         'admin_state': {
             'shellcmd': 'ip -json link show dev {ifname}',
-            'format': lambda j: 'up' if 'UP' in jmespath.search('[*].flags | [0]', json.loads(j)) else 'down',
+            'format': lambda j: 'up' if 'UP' in _jmes_search('[*].flags | [0]', json.loads(j)) else 'down',
         },
         'alias': {
             'shellcmd': 'ip -json -detail link list dev {ifname}',
-            'format': lambda j: jmespath.search('[*].ifalias | [0]', json.loads(j)) or '',
+            'format': lambda j: _jmes_search('[*].ifalias | [0]', json.loads(j)) or '',
         },
         'ifindex': {
             'shellcmd': 'ip -json -detail link list dev {ifname}',
-            'format': lambda j: jmespath.search('[*].ifindex | [0]', json.loads(j)) or '',
+            'format': lambda j: _jmes_search('[*].ifindex | [0]', json.loads(j)) or '',
         },
         'mac': {
             'shellcmd': 'ip -json -detail link list dev {ifname}',
-            'format': lambda j: jmespath.search('[*].address | [0]', json.loads(j)),
+            'format': lambda j: _jmes_search('[*].address | [0]', json.loads(j)),
         },
         'min_mtu': {
             'shellcmd': 'ip -json -detail link list dev {ifname}',
-            'format': lambda j: jmespath.search('[*].min_mtu | [0]', json.loads(j)),
+            'format': lambda j: _jmes_search('[*].min_mtu | [0]', json.loads(j)),
         },
         'max_mtu': {
             'shellcmd': 'ip -json -detail link list dev {ifname}',
-            'format': lambda j: jmespath.search('[*].max_mtu | [0]', json.loads(j)),
+            'format': lambda j: _jmes_search('[*].max_mtu | [0]', json.loads(j)),
         },
         'mtu': {
             'shellcmd': 'ip -json -detail link list dev {ifname}',
-            'format': lambda j: jmespath.search('[*].mtu | [0]', json.loads(j)),
+            'format': lambda j: _jmes_search('[*].mtu | [0]', json.loads(j)),
         },
         'oper_state': {
             'shellcmd': 'ip -json -detail link list dev {ifname}',
-            'format': lambda j: jmespath.search('[*].operstate | [0]', json.loads(j)),
+            'format': lambda j: _jmes_search('[*].operstate | [0]', json.loads(j)),
         },
         'vrf': {
             'shellcmd': 'ip -json -detail link list dev {ifname}',
-            'format': lambda j: jmespath.search('[?linkinfo.info_slave_kind == `vrf`].master | [0]', json.loads(j)),
+            'format': lambda j: _jmes_search('[?linkinfo.info_slave_kind == `vrf`].master | [0]', json.loads(j)),
         },
     }
 
@@ -552,6 +557,11 @@ class Interface(Control):
         sha.update(cpu_id.encode())
         sha.update(first_mac.encode())
         sha.update(self.ifname.encode())
+        # Imported here rather than at module scope: netaddr is only needed
+        # by this one method.
+        from netaddr import EUI
+        from netaddr import mac_unix_expanded
+
         # take the most significant 48 bits from the SHA256 string
         tmp = sha.hexdigest()[:12]
         # Convert pseudo random string into EUI format which now represents a
@@ -1709,13 +1719,22 @@ class Interface(Control):
 
     def set_eapol(self) -> None:
         """ Take care about EAPoL supplicant daemon """
-
         # XXX: wpa_supplicant works on the source interface
         cfg_dir = '/run/wpa_supplicant'
         wpa_supplicant_conf = f'{cfg_dir}/{self.ifname}.conf'
         eapol_action='stop'
 
         if 'eapol' in self.config:
+            # Imported here rather than at module scope: vyos.pki imports the
+            # whole cryptography stack, which every consumer of vyos.ifconfig
+            # would otherwise pay for on import. set_eapol() itself runs on
+            # every ethernet/bond update, so keep it inside this guard too.
+            # vyos/config.py and vyos/configverify.py defer vyos.pki the same way.
+            from vyos.pki import find_chain
+            from vyos.pki import encode_certificate
+            from vyos.pki import load_certificate
+            from vyos.pki import wrap_private_key
+
             # The default is a fallback to hw_id which is not present for any interface
             # other then an ethernet interface. Thus we emulate hw_id by reading back the
             # Kernel assigned MAC address
diff --git a/python/vyos/ifconfig/operational.py b/python/vyos/ifconfig/operational.py
index caca40b3d..f513b9157 100644
--- a/python/vyos/ifconfig/operational.py
+++ b/python/vyos/ifconfig/operational.py
@@ -18,7 +18,6 @@ import os
 from time import time
 from datetime import datetime
 from functools import reduce
-from tabulate import tabulate
 
 from vyos.ifconfig import Control
 
@@ -169,6 +168,8 @@ class Operational(Control):
             tabs.append([f'{rtx.upper()}:', ] + [_ for _ in self._stat_names[rtx]])
             tabs.append(['', ] + [stats[_] for _ in self._stats_dir[rtx]])
 
+        from tabulate import tabulate
+
         s = tabulate(
             tabs,
             stralign="right",
diff --git a/python/vyos/ifconfig/vrrp.py b/python/vyos/ifconfig/vrrp.py
index 4949fe571..8c0c473d0 100644
--- a/python/vyos/ifconfig/vrrp.py
+++ b/python/vyos/ifconfig/vrrp.py
@@ -18,7 +18,6 @@ import json
 import signal
 
 from time import time
-from tabulate import tabulate
 
 from vyos.configquery import ConfigTreeQuery
 from vyos.utils.convert import seconds_to_human
@@ -166,4 +165,6 @@ class VRRP(object):
 
         # add to the active list disabled instances
         groups.extend(cls.disabled())
+        from tabulate import tabulate
+
         return tabulate(groups, headers)
diff --git a/python/vyos/ifconfig/wireguard.py b/python/vyos/ifconfig/wireguard.py
index 4609aff25..8801ff43c 100644
--- a/python/vyos/ifconfig/wireguard.py
+++ b/python/vyos/ifconfig/wireguard.py
@@ -19,11 +19,7 @@ import time
 from datetime import timedelta
 from tempfile import NamedTemporaryFile
 
-from hurry.filesize import size
-from hurry.filesize import alternative
-
 from vyos.base import Warning
-from vyos.configquery import ConfigTreeQuery
 from vyos.ifconfig import Interface
 from vyos.ifconfig import Operational
 from vyos.template import is_ipv6
@@ -84,6 +80,11 @@ class WireGuardOperational(Operational):
         return output
 
     def show_interface(self):
+        # Imported here rather than at module scope: display-only helpers should
+        # not be on the vyos.ifconfig import path.
+        from hurry.filesize import size
+        from hurry.filesize import alternative
+
         from vyos.config import Config
 
         c = Config()
@@ -183,6 +184,10 @@ class WireGuardOperational(Operational):
         return output
 
     def reset_peer(self, peer_name=None, public_key=None):
+        # Imported here rather than at module scope: vyos.configquery pulls in
+        # the whole vyos.config tree.
+        from vyos.configquery import ConfigTreeQuery
+
         c = ConfigTreeQuery()
         tmp = c.get_config_dict(['interfaces', 'wireguard', self.ifname],
                                 effective=True, get_first_key=True,
diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py
index 421054f6e..6c3158ee6 100644
--- a/python/vyos/opmode.py
+++ b/python/vyos/opmode.py
@@ -17,8 +17,6 @@ import re
 import sys
 import typing
 
-from humps import decamelize
-from vyos.configquery import ConfigTreeQuery
 
 class Error(Exception):
     """Any error that makes requested operation impossible to complete
@@ -319,9 +317,11 @@ def run(module):
                     f'Bare literal is not an acceptable raw output, must be a list or an object.\
                     The output was:{res}'
                 )
+            from humps import decamelize
+            from json import dumps
+
             res = decamelize(res)
             res = _normalize_field_names(res)
-            from json import dumps
 
             return dumps(res, indent=4)
     else:
@@ -336,6 +336,11 @@ def verify_cli_exists(cli_path: list, error_msg: str=''):
 
         @wraps(func)
         def _wrapper(*args, **kwargs):
+            # Imported here rather than at module scope: vyos.configquery pulls
+            # in the whole vyos.config tree, and most op-mode scripts import
+            # vyos.opmode without ever using this decorator.
+            from vyos.configquery import ConfigTreeQuery
+
             config = ConfigTreeQuery()
             interface = kwargs.get('intf_name')
 
diff --git a/python/vyos/qos/base.py b/python/vyos/qos/base.py
index 65aeb552e..651d3f1e1 100644
--- a/python/vyos/qos/base.py
+++ b/python/vyos/qos/base.py
@@ -14,10 +14,8 @@
 # License along with this library.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import jmespath
 
 from vyos.base import Warning
-from vyos.ifconfig import Interface
 from vyos.utils.process import cmdl
 from vyos.utils.dict import dict_search
 from vyos.utils.file import read_file
@@ -233,6 +231,13 @@ class QoSBase:
 
     def update(self, config, direction, priority=None):
         """ method must be called from derived class after it has completed qdisc setup """
+        # Imported here rather than at module scope: pulling vyos.ifconfig in
+        # at import time is what made vyos.qos one of the most expensive
+        # modules in the tree.
+        import jmespath
+
+        from vyos.ifconfig import Interface
+
         if self._debug:
             import pprint
             pprint.pprint(config)
diff --git a/python/vyos/template.py b/python/vyos/template.py
index dbbc4df84..83805a73c 100755
--- a/python/vyos/template.py
+++ b/python/vyos/template.py
@@ -16,9 +16,6 @@
 import functools
 import os
 
-from jinja2 import Environment
-from jinja2 import FileSystemLoader
-from jinja2 import ChainableUndefined
 from vyos.defaults import directories
 from vyos.utils.dict import dict_search_args
 from vyos.utils.file import makedir
@@ -41,6 +38,13 @@ _CLEVER_FUNCTIONS = {}
 # reuse Environments with identical settings to improve performance
 @functools.lru_cache(maxsize=2)
 def _get_environment(location=None):
+    # Imported here rather than at module scope: most importers of this module
+    # want only the pure-Python predicates (is_ipv4, is_ipv6, ...) and never
+    # render a template, so they should not pay for jinja2. This function is
+    # lru_cache'd, so the import happens at most once per process.
+    from jinja2 import Environment
+    from jinja2 import FileSystemLoader
+    from jinja2 import ChainableUndefined
     from os import getenv
 
     if location is None:
diff --git a/python/vyos/utils/auth.py b/python/vyos/utils/auth.py
index e85dc9984..3950de649 100644
--- a/python/vyos/utils/auth.py
+++ b/python/vyos/utils/auth.py
@@ -13,7 +13,6 @@
 # You should have received a copy of the GNU Lesser General Public License along with this library;
 # if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 
-import cracklib
 import math
 import re
 import string
@@ -76,15 +75,21 @@ def calculate_entropy(charset: str, passwd: str) -> float:
 
 def evaluate_strength(passwd: str) -> dict[str, str]:
     """ Evaluates password strength and returns a check result dict """
-    charset = (cracklib.ASCII_UPPERCASE + cracklib.ASCII_LOWERCASE +
-        string.punctuation + string.digits)
-
     result = {
         'strength': '',
         'error': '',
     }
 
     try:
+        # Imported here rather than at module scope: importing cracklib opens
+        # its dictionary database, so this module stays importable without it.
+        # Kept inside the try so that a missing or broken package is reported
+        # as EPasswdStrength.ERROR instead of escaping this function.
+        import cracklib
+
+        charset = (cracklib.ASCII_UPPERCASE + cracklib.ASCII_LOWERCASE +
+            string.punctuation + string.digits)
+
         cracklib.FascistCheck(passwd)
     except ValueError as e:
         # The password is vulnerable to dictionary attack no matter the entropy