Merge branch 'current' into dhcpd

23 files changed, 681 insertions, 193 deletions

diff --git a/python/vyos/base.py b/python/vyos/base.py
index c78045548..fd22eaccd 100644
--- a/python/vyos/base.py
+++ b/python/vyos/base.py
@@ -15,6 +15,12 @@
 
 from textwrap import fill
 
+class DeprecationWarning():
+    def __init__(self, message):
+        # Reformat the message and trim it to 72 characters in length
+        message = fill(message, width=72)
+        print(f'\nDEPRECATION WARNING: {message}\n')
+
 class ConfigError(Exception):
     def __init__(self, message):
         # Reformat the message and trim it to 72 characters in length
diff --git a/python/vyos/configdict.py b/python/vyos/configdict.py
index d974a7565..551c27b67 100644
--- a/python/vyos/configdict.py
+++ b/python/vyos/configdict.py
@@ -117,6 +117,12 @@ def leaf_node_changed(conf, path):
     D.set_level(conf.get_level())
     (new, old) = D.get_value_diff(path)
     if new != old:
+        if isinstance(old, dict):
+            # valueLess nodes return {} if node is deleted
+            return True
+        if old is None and isinstance(new, dict):
+            # valueLess nodes return {} if node was added
+            return True
         if old is None:
             return []
         if isinstance(old, str):
@@ -130,7 +136,7 @@ def leaf_node_changed(conf, path):
 
     return None
 
-def node_changed(conf, path, key_mangling=None):
+def node_changed(conf, path, key_mangling=None, recursive=False):
     """
     Check if a leaf node was altered. If it has been altered - values has been
     changed, or it was added/removed, we will return the old value. If nothing
@@ -140,7 +146,7 @@ def node_changed(conf, path, key_mangling=None):
     D = get_config_diff(conf, key_mangling)
     D.set_level(conf.get_level())
     # get_child_nodes() will return dict_keys(), mangle this into a list with PEP448
-    keys = D.get_child_nodes_diff(path, expand_nodes=Diff.DELETE)['delete'].keys()
+    keys = D.get_child_nodes_diff(path, expand_nodes=Diff.DELETE, recursive=recursive)['delete'].keys()
     return list(keys)
 
 def get_removed_vlans(conf, dict):
@@ -196,7 +202,7 @@ def is_member(conf, interface, intftype=None):
     interface name -> Interface is a member of this interface
     False -> interface type cannot have members
     """
-    ret_val = None
+    ret_val = {}
     intftypes = ['bonding', 'bridge']
 
     if intftype not in intftypes + [None]:
@@ -216,8 +222,8 @@ def is_member(conf, interface, intftype=None):
             member = base + [intf, 'member', 'interface', interface]
             if conf.exists(member):
                 tmp = conf.get_config_dict(member, key_mangling=('-', '_'),
-                                           get_first_key=True)
-                ret_val = {intf : tmp}
+                                           get_first_key=True, no_tag_node_value_mangle=True)
+                ret_val.update({intf : tmp})
 
     old_level = conf.set_level(old_level)
     return ret_val
@@ -319,34 +325,42 @@ def is_source_interface(conf, interface, intftype=None):
 def get_dhcp_interfaces(conf, vrf=None):
     """ Common helper functions to retrieve all interfaces from current CLI
     sessions that have DHCP configured. """
-    dhcp_interfaces = []
+    dhcp_interfaces = {}
     dict = conf.get_config_dict(['interfaces'], get_first_key=True)
     if not dict:
         return dhcp_interfaces
 
     def check_dhcp(config, ifname):
-        out = []
+        tmp = {}
         if 'address' in config and 'dhcp' in config['address']:
+            options = {}
+            if 'dhcp_options' in config and 'default_route_distance' in config['dhcp_options']:
+                options.update({'distance' : config['dhcp_options']['default_route_distance']})
             if 'vrf' in config:
-                if vrf is config['vrf']: out.append(ifname)
-            else: out.append(ifname)
-        return out
+                if vrf is config['vrf']: tmp.update({ifname : options})
+            else: tmp.update({ifname : options})
+        return tmp
 
     for section, interface in dict.items():
-        for ifname, ifconfig in interface.items():
+        for ifname in interface:
+            # we already have a dict representation of the config from get_config_dict(),
+            # but with the extended information from get_interface_dict() we also
+            # get the DHCP client default-route-distance default option if not specified.
+            ifconfig = get_interface_dict(conf, ['interfaces', section], ifname)
+
             tmp = check_dhcp(ifconfig, ifname)
-            dhcp_interfaces.extend(tmp)
+            dhcp_interfaces.update(tmp)
             # check per VLAN interfaces
             for vif, vif_config in ifconfig.get('vif', {}).items():
                 tmp = check_dhcp(vif_config, f'{ifname}.{vif}')
-                dhcp_interfaces.extend(tmp)
+                dhcp_interfaces.update(tmp)
             # check QinQ VLAN interfaces
             for vif_s, vif_s_config in ifconfig.get('vif-s', {}).items():
                 tmp = check_dhcp(vif_s_config, f'{ifname}.{vif_s}')
-                dhcp_interfaces.extend(tmp)
+                dhcp_interfaces.update(tmp)
                 for vif_c, vif_c_config in vif_s_config.get('vif-c', {}).items():
                     tmp = check_dhcp(vif_c_config, f'{ifname}.{vif_s}.{vif_c}')
-                    dhcp_interfaces.extend(tmp)
+                    dhcp_interfaces.update(tmp)
 
     return dhcp_interfaces
 
@@ -405,6 +419,12 @@ def get_interface_dict(config, base, ifname=''):
     if 'deleted' not in dict:
         dict = dict_merge(default_values, dict)
 
+        # If interface does not request an IPv4 DHCP address there is no need
+        # to keep the dhcp-options key
+        if 'address' not in dict or 'dhcp' not in dict['address']:
+            if 'dhcp_options' in dict:
+                del dict['dhcp_options']
+
     # XXX: T2665: blend in proper DHCPv6-PD default values
     dict = T2665_set_dhcpv6pd_defaults(dict)
 
@@ -423,6 +443,10 @@ def get_interface_dict(config, base, ifname=''):
     bond = is_member(config, ifname, 'bonding')
     if bond: dict.update({'is_bond_member' : bond})
 
+    # Check if any DHCP options changed which require a client restat
+    dhcp = node_changed(config, ['dhcp-options'], recursive=True)
+    if dhcp: dict.update({'dhcp_options_changed' : ''})
+
     # Some interfaces come with a source_interface which must also not be part
     # of any other bond or bridge interface as it is exclusivly assigned as the
     # Kernels "lower" interface to this new "virtual/upper" interface.
@@ -466,10 +490,20 @@ def get_interface_dict(config, base, ifname=''):
             # XXX: T2665: blend in proper DHCPv6-PD default values
             dict['vif'][vif] = T2665_set_dhcpv6pd_defaults(dict['vif'][vif])
 
+            # If interface does not request an IPv4 DHCP address there is no need
+            # to keep the dhcp-options key
+            if 'address' not in dict['vif'][vif] or 'dhcp' not in dict['vif'][vif]['address']:
+                if 'dhcp_options' in dict['vif'][vif]:
+                    del dict['vif'][vif]['dhcp_options']
+
         # Check if we are a member of a bridge device
         bridge = is_member(config, f'{ifname}.{vif}', 'bridge')
         if bridge: dict['vif'][vif].update({'is_bridge_member' : bridge})
 
+        # Check if any DHCP options changed which require a client restat
+        dhcp = node_changed(config, ['vif', vif, 'dhcp-options'], recursive=True)
+        if dhcp: dict['vif'][vif].update({'dhcp_options_changed' : ''})
+
     for vif_s, vif_s_config in dict.get('vif_s', {}).items():
         default_vif_s_values = defaults(base + ['vif-s'])
         # XXX: T2665: we only wan't the vif-s defaults - do not care about vif-c
@@ -491,10 +525,21 @@ def get_interface_dict(config, base, ifname=''):
             # XXX: T2665: blend in proper DHCPv6-PD default values
             dict['vif_s'][vif_s] = T2665_set_dhcpv6pd_defaults(dict['vif_s'][vif_s])
 
+            # If interface does not request an IPv4 DHCP address there is no need
+            # to keep the dhcp-options key
+            if 'address' not in dict['vif_s'][vif_s] or 'dhcp' not in \
+                dict['vif_s'][vif_s]['address']:
+                if 'dhcp_options' in dict['vif_s'][vif_s]:
+                    del dict['vif_s'][vif_s]['dhcp_options']
+
         # Check if we are a member of a bridge device
         bridge = is_member(config, f'{ifname}.{vif_s}', 'bridge')
         if bridge: dict['vif_s'][vif_s].update({'is_bridge_member' : bridge})
 
+        # Check if any DHCP options changed which require a client restat
+        dhcp = node_changed(config, ['vif-s', vif_s, 'dhcp-options'], recursive=True)
+        if dhcp: dict['vif_s'][vif_s].update({'dhcp_options_changed' : ''})
+
         for vif_c, vif_c_config in vif_s_config.get('vif_c', {}).items():
             default_vif_c_values = defaults(base + ['vif-s', 'vif-c'])
 
@@ -516,11 +561,22 @@ def get_interface_dict(config, base, ifname=''):
                 dict['vif_s'][vif_s]['vif_c'][vif_c] = T2665_set_dhcpv6pd_defaults(
                     dict['vif_s'][vif_s]['vif_c'][vif_c])
 
+                # If interface does not request an IPv4 DHCP address there is no need
+                # to keep the dhcp-options key
+                if 'address' not in dict['vif_s'][vif_s]['vif_c'][vif_c] or 'dhcp' \
+                    not in dict['vif_s'][vif_s]['vif_c'][vif_c]['address']:
+                    if 'dhcp_options' in dict['vif_s'][vif_s]['vif_c'][vif_c]:
+                        del dict['vif_s'][vif_s]['vif_c'][vif_c]['dhcp_options']
+
             # Check if we are a member of a bridge device
             bridge = is_member(config, f'{ifname}.{vif_s}.{vif_c}', 'bridge')
             if bridge: dict['vif_s'][vif_s]['vif_c'][vif_c].update(
                 {'is_bridge_member' : bridge})
 
+            # Check if any DHCP options changed which require a client restat
+            dhcp = node_changed(config, ['vif-s', vif_s, 'vif-c', vif_c, 'dhcp-options'], recursive=True)
+            if dhcp: dict['vif_s'][vif_s]['vif_c'][vif_c].update({'dhcp_options_changed' : ''})
+
     # Check vif, vif-s/vif-c VLAN interfaces for removal
     dict = get_removed_vlans(config, dict)
     return dict
@@ -545,7 +601,6 @@ def get_vlan_ids(interface):
 
     return vlan_ids
 
-
 def get_accel_dict(config, base, chap_secrets):
     """
     Common utility function to retrieve and mangle the Accel-PPP configuration
diff --git a/python/vyos/configdiff.py b/python/vyos/configdiff.py
index 4ad7443d7..9185575df 100644
--- a/python/vyos/configdiff.py
+++ b/python/vyos/configdiff.py
@@ -16,6 +16,7 @@
 from enum import IntFlag, auto
 
 from vyos.config import Config
+from vyos.configtree import DiffTree
 from vyos.configdict import dict_merge
 from vyos.configdict import list_diff
 from vyos.util import get_sub_dict, mangle_dict_keys
@@ -38,6 +39,8 @@ class Diff(IntFlag):
     ADD = auto()
     STABLE = auto()
 
+ALL = Diff.MERGE | Diff.DELETE | Diff.ADD | Diff.STABLE
+
 requires_effective = [enum_to_key(Diff.DELETE)]
 target_defaults = [enum_to_key(Diff.MERGE)]
 
@@ -75,19 +78,24 @@ def get_config_diff(config, key_mangling=None):
             isinstance(key_mangling[1], str)):
         raise ValueError("key_mangling must be a tuple of two strings")
 
-    return ConfigDiff(config, key_mangling)
+    diff_t = DiffTree(config._running_config, config._session_config)
+
+    return ConfigDiff(config, key_mangling, diff_tree=diff_t)
 
 class ConfigDiff(object):
     """
     The class of config changes as represented by comparison between the
     session config dict and the effective config dict.
     """
-    def __init__(self, config, key_mangling=None):
+    def __init__(self, config, key_mangling=None, diff_tree=None):
         self._level = config.get_level()
         self._session_config_dict = config.get_cached_root_dict(effective=False)
         self._effective_config_dict = config.get_cached_root_dict(effective=True)
         self._key_mangling = key_mangling
 
+        self._diff_tree = diff_tree
+        self._diff_dict = diff_tree.dict if diff_tree else {}
+
     # mirrored from Config; allow path arguments relative to level
     def _make_path(self, path):
         if isinstance(path, str):
@@ -136,6 +144,15 @@ class ConfigDiff(object):
                                                     self._key_mangling[1])
         return config_dict
 
+    def is_node_changed(self, path=[]):
+        if self._diff_tree is None:
+            raise NotImplementedError("diff_tree class not available")
+
+        if (self._diff_tree.add.exists(self._make_path(path)) or
+            self._diff_tree.sub.exists(self._make_path(path))):
+            return True
+        return False
+
     def get_child_nodes_diff_str(self, path=[]):
         ret = {'add': {}, 'change': {}, 'delete': {}}
 
@@ -164,7 +181,8 @@ class ConfigDiff(object):
 
         return ret
 
-    def get_child_nodes_diff(self, path=[], expand_nodes=Diff(0), no_defaults=False):
+    def get_child_nodes_diff(self, path=[], expand_nodes=Diff(0), no_defaults=False,
+                             recursive=False):
         """
         Args:
             path (str|list): config path
@@ -174,6 +192,8 @@ class ConfigDiff(object):
                                   value
             no_detaults=False: if expand_nodes & Diff.MERGE, do not merge default
                                values to ret['merge']
+            recursive: if true, use config_tree diff algorithm provided by
+                       diff_tree class
 
         Returns: dict of lists, representing differences between session
                                 and effective config, under path
@@ -184,6 +204,34 @@ class ConfigDiff(object):
         """
         session_dict = get_sub_dict(self._session_config_dict,
                                     self._make_path(path), get_first_key=True)
+
+        if recursive:
+            if self._diff_tree is None:
+                raise NotImplementedError("diff_tree class not available")
+            else:
+                add = get_sub_dict(self._diff_tree.dict, ['add'], get_first_key=True)
+                sub = get_sub_dict(self._diff_tree.dict, ['sub'], get_first_key=True)
+                inter = get_sub_dict(self._diff_tree.dict, ['inter'], get_first_key=True)
+                ret = {}
+                ret[enum_to_key(Diff.MERGE)] = session_dict
+                ret[enum_to_key(Diff.DELETE)] = get_sub_dict(sub, self._make_path(path),
+                                                             get_first_key=True)
+                ret[enum_to_key(Diff.ADD)] = get_sub_dict(add, self._make_path(path),
+                                                          get_first_key=True)
+                ret[enum_to_key(Diff.STABLE)] = get_sub_dict(inter, self._make_path(path),
+                                                             get_first_key=True)
+                for e in Diff:
+                    k = enum_to_key(e)
+                    if not (e & expand_nodes):
+                        ret[k] = list(ret[k])
+                    else:
+                        if self._key_mangling:
+                            ret[k] = self._mangle_dict_keys(ret[k])
+                        if k in target_defaults and not no_defaults:
+                            default_values = defaults(self._make_path(path))
+                            ret[k] = dict_merge(default_values, ret[k])
+                return ret
+
         effective_dict = get_sub_dict(self._effective_config_dict,
                                       self._make_path(path), get_first_key=True)
 
@@ -209,7 +257,8 @@ class ConfigDiff(object):
 
         return ret
 
-    def get_node_diff(self, path=[], expand_nodes=Diff(0), no_defaults=False):
+    def get_node_diff(self, path=[], expand_nodes=Diff(0), no_defaults=False,
+                      recursive=False):
         """
         Args:
             path (str|list): config path
@@ -219,6 +268,8 @@ class ConfigDiff(object):
                                   value
             no_detaults=False: if expand_nodes & Diff.MERGE, do not merge default
                                values to ret['merge']
+            recursive: if true, use config_tree diff algorithm provided by
+                       diff_tree class
 
         Returns: dict of lists, representing differences between session
                                 and effective config, at path
@@ -228,6 +279,31 @@ class ConfigDiff(object):
                  dict['stable'] = config values in both session and effective
         """
         session_dict = get_sub_dict(self._session_config_dict, self._make_path(path))
+
+        if recursive:
+            if self._diff_tree is None:
+                raise NotImplementedError("diff_tree class not available")
+            else:
+                add = get_sub_dict(self._diff_tree.dict, ['add'], get_first_key=True)
+                sub = get_sub_dict(self._diff_tree.dict, ['sub'], get_first_key=True)
+                inter = get_sub_dict(self._diff_tree.dict, ['inter'], get_first_key=True)
+                ret = {}
+                ret[enum_to_key(Diff.MERGE)] = session_dict
+                ret[enum_to_key(Diff.DELETE)] = get_sub_dict(sub, self._make_path(path))
+                ret[enum_to_key(Diff.ADD)] = get_sub_dict(add, self._make_path(path))
+                ret[enum_to_key(Diff.STABLE)] = get_sub_dict(inter, self._make_path(path))
+                for e in Diff:
+                    k = enum_to_key(e)
+                    if not (e & expand_nodes):
+                        ret[k] = list(ret[k])
+                    else:
+                        if self._key_mangling:
+                            ret[k] = self._mangle_dict_keys(ret[k])
+                        if k in target_defaults and not no_defaults:
+                            default_values = defaults(self._make_path(path))
+                            ret[k] = dict_merge(default_values, ret[k])
+                return ret
+
         effective_dict = get_sub_dict(self._effective_config_dict, self._make_path(path))
 
         ret = _key_sets_from_dicts(session_dict, effective_dict)
diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py
index d8ffaca99..e9cdb69e4 100644
--- a/python/vyos/configtree.py
+++ b/python/vyos/configtree.py
@@ -17,6 +17,7 @@ import json
 
 from ctypes import cdll, c_char_p, c_void_p, c_int
 
+LIBPATH = '/usr/lib/libvyosconfig.so.0'
 
 def escape_backslash(string: str) -> str:
     """Escape single backslashes in string that are not in escape sequence"""
@@ -42,7 +43,9 @@ class ConfigTreeError(Exception):
 
 
 class ConfigTree(object):
-    def __init__(self, config_string, libpath='/usr/lib/libvyosconfig.so.0'):
+    def __init__(self, config_string=None, address=None, libpath=LIBPATH):
+        if config_string is None and address is None:
+            raise TypeError("ConfigTree() requires one of 'config_string' or 'address'")
         self.__config = None
         self.__lib = cdll.LoadLibrary(libpath)
 
@@ -60,7 +63,7 @@ class ConfigTree(object):
         self.__to_string.restype = c_char_p
 
         self.__to_commands = self.__lib.to_commands
-        self.__to_commands.argtypes = [c_void_p]
+        self.__to_commands.argtypes = [c_void_p, c_char_p]
         self.__to_commands.restype = c_char_p
 
         self.__to_json = self.__lib.to_json
@@ -123,18 +126,26 @@ class ConfigTree(object):
         self.__set_tag.argtypes = [c_void_p, c_char_p]
         self.__set_tag.restype = c_int
 
+        self.__get_subtree = self.__lib.get_subtree
+        self.__get_subtree.argtypes = [c_void_p, c_char_p]
+        self.__get_subtree.restype = c_void_p
+
         self.__destroy = self.__lib.destroy
         self.__destroy.argtypes = [c_void_p]
 
-        config_section, version_section = extract_version(config_string)
-        config_section = escape_backslash(config_section)
-        config = self.__from_string(config_section.encode())
-        if config is None:
-            msg = self.__get_error().decode()
-            raise ValueError("Failed to parse config: {0}".format(msg))
+        if address is None:
+            config_section, version_section = extract_version(config_string)
+            config_section = escape_backslash(config_section)
+            config = self.__from_string(config_section.encode())
+            if config is None:
+                msg = self.__get_error().decode()
+                raise ValueError("Failed to parse config: {0}".format(msg))
+            else:
+                self.__config = config
+                self.__version = version_section
         else:
-            self.__config = config
-            self.__version = version_section
+            self.__config = address
+            self.__version = ''
 
     def __del__(self):
         if self.__config is not None:
@@ -143,13 +154,16 @@ class ConfigTree(object):
     def __str__(self):
         return self.to_string()
 
+    def _get_config(self):
+        return self.__config
+
     def to_string(self):
         config_string = self.__to_string(self.__config).decode()
         config_string = "{0}\n{1}".format(config_string, self.__version)
         return config_string
 
-    def to_commands(self):
-        return self.__to_commands(self.__config).decode()
+    def to_commands(self, op="set"):
+        return self.__to_commands(self.__config, op.encode()).decode()
 
     def to_json(self):
         return self.__to_json(self.__config).decode()
@@ -281,3 +295,61 @@ class ConfigTree(object):
         else:
             raise ConfigTreeError("Path [{}] doesn't exist".format(path_str))
 
+    def get_subtree(self, path, with_node=False):
+        check_path(path)
+        path_str = " ".join(map(str, path)).encode()
+
+        res = self.__get_subtree(self.__config, path_str, with_node)
+        subt = ConfigTree(address=res)
+        return subt
+
+class DiffTree:
+    def __init__(self, left, right, path=[], libpath=LIBPATH):
+        if left is None:
+            left = ConfigTree(config_string='\n')
+        if right is None:
+            right = ConfigTree(config_string='\n')
+        if not (isinstance(left, ConfigTree) and isinstance(right, ConfigTree)):
+            raise TypeError("Arguments must be instances of ConfigTree")
+        if path:
+            if not left.exists(path):
+                raise ConfigTreeError(f"Path {path} doesn't exist in lhs tree")
+            if not right.exists(path):
+                raise ConfigTreeError(f"Path {path} doesn't exist in rhs tree")
+
+        self.left = left
+        self.right = right
+
+        self.__lib = cdll.LoadLibrary(libpath)
+
+        self.__diff_tree = self.__lib.diff_tree
+        self.__diff_tree.argtypes = [c_char_p, c_void_p, c_void_p]
+        self.__diff_tree.restype = c_void_p
+
+        self.__trim_tree = self.__lib.trim_tree
+        self.__trim_tree.argtypes = [c_void_p, c_void_p]
+        self.__trim_tree.restype = c_void_p
+
+        check_path(path)
+        path_str = " ".join(map(str, path)).encode()
+
+        res = self.__diff_tree(path_str, left._get_config(), right._get_config())
+
+        # full diff config_tree and python dict representation
+        self.full = ConfigTree(address=res)
+        self.dict = json.loads(self.full.to_json())
+
+        # config_tree sub-trees
+        self.add = self.full.get_subtree(['add'])
+        self.sub = self.full.get_subtree(['sub'])
+        self.inter = self.full.get_subtree(['inter'])
+
+        # trim sub(-tract) tree to get delete tree for commands
+        ref = self.right.get_subtree(path, with_node=True) if path else self.right
+        res = self.__trim_tree(self.sub._get_config(), ref._get_config())
+        self.delete = ConfigTree(address=res)
+
+    def to_commands(self):
+        add = self.add.to_commands()
+        delete = self.delete.to_commands(op="delete")
+        return delete + "\n" + add
diff --git a/python/vyos/configverify.py b/python/vyos/configverify.py
index 365a28feb..1062d51ee 100644
--- a/python/vyos/configverify.py
+++ b/python/vyos/configverify.py
@@ -173,24 +173,43 @@ def verify_eapol(config):
             if ca_cert_name not in config['pki']['ca']:
                 raise ConfigError('Invalid CA certificate specified for EAPoL')
 
-            ca_cert = config['pki']['ca'][cert_name]
+            ca_cert = config['pki']['ca'][ca_cert_name]
 
             if 'certificate' not in ca_cert:
                 raise ConfigError('Invalid CA certificate specified for EAPoL')
 
-def verify_mirror(config):
+def verify_mirror_redirect(config):
     """
     Common helper function used by interface implementations to perform
-    recurring validation of mirror interface configuration.
+    recurring validation of mirror and redirect interface configuration via tc(8)
 
     It makes no sense to mirror traffic back at yourself!
     """
+    import os
+    if {'mirror', 'redirect'} <= set(config):
+        raise ConfigError('Mirror and redirect can not be enabled at the same time!')
+
     if 'mirror' in config:
         for direction, mirror_interface in config['mirror'].items():
+            if not os.path.exists(f'/sys/class/net/{mirror_interface}'):
+                raise ConfigError(f'Requested mirror interface "{mirror_interface}" '\
+                                   'does not exist!')
+
             if mirror_interface == config['ifname']:
-                raise ConfigError(f'Can not mirror "{direction}" traffic back ' \
+                raise ConfigError(f'Can not mirror "{direction}" traffic back '\
                                    'the originating interface!')
 
+    if 'redirect' in config:
+        redirect_ifname = config['redirect']
+        if not os.path.exists(f'/sys/class/net/{redirect_ifname}'):
+            raise ConfigError(f'Requested redirect interface "{redirect_ifname}" '\
+                               'does not exist!')
+
+    if dict_search('traffic_policy.in', config) != None:
+        # XXX: support combination of limiting and redirect/mirror - this is an
+        # artificial limitation
+        raise ConfigError('Can not use ingress policy tigether with mirror or redirect!')
+
 def verify_authentication(config):
     """
     Common helper function used by interface implementations to perform
@@ -224,9 +243,10 @@ def verify_bridge_delete(config):
     when interface also is part of a bridge.
     """
     if 'is_bridge_member' in config:
-        raise ConfigError(
-            'Interface "{ifname}" cannot be deleted as it is a '
-            'member of bridge "{is_bridge_member}"!'.format(**config))
+        interface = config['ifname']
+        for bridge in config['is_bridge_member']:
+            raise ConfigError(f'Interface "{interface}" cannot be deleted as it '
+                              f'is a member of bridge "{bridge}"!')
 
 def verify_interface_exists(ifname):
     """
@@ -308,27 +328,37 @@ def verify_vlan_config(config):
         if duplicate:
             raise ConfigError(f'Duplicate VLAN id "{duplicate[0]}" used for vif and vif-s interfaces!')
 
+    parent_ifname = config['ifname']
     # 802.1q VLANs
-    for vlan in config.get('vif', {}):
-        vlan = config['vif'][vlan]
+    for vlan_id in config.get('vif', {}):
+        vlan = config['vif'][vlan_id]
+        vlan['ifname'] = f'{parent_ifname}.{vlan_id}'
+
         verify_dhcpv6(vlan)
         verify_address(vlan)
         verify_vrf(vlan)
+        verify_mirror_redirect(vlan)
         verify_mtu_parent(vlan, config)
 
     # 802.1ad (Q-in-Q) VLANs
-    for s_vlan in config.get('vif_s', {}):
-        s_vlan = config['vif_s'][s_vlan]
+    for s_vlan_id in config.get('vif_s', {}):
+        s_vlan = config['vif_s'][s_vlan_id]
+        s_vlan['ifname'] = f'{parent_ifname}.{s_vlan_id}'
+
         verify_dhcpv6(s_vlan)
         verify_address(s_vlan)
         verify_vrf(s_vlan)
+        verify_mirror_redirect(s_vlan)
         verify_mtu_parent(s_vlan, config)
 
-        for c_vlan in s_vlan.get('vif_c', {}):
-            c_vlan = s_vlan['vif_c'][c_vlan]
+        for c_vlan_id in s_vlan.get('vif_c', {}):
+            c_vlan = s_vlan['vif_c'][c_vlan_id]
+            c_vlan['ifname'] = f'{parent_ifname}.{s_vlan_id}.{c_vlan_id}'
+
             verify_dhcpv6(c_vlan)
             verify_address(c_vlan)
             verify_vrf(c_vlan)
+            verify_mirror_redirect(c_vlan)
             verify_mtu_parent(c_vlan, config)
             verify_mtu_parent(c_vlan, s_vlan)
 
diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py
index c77b695bd..fcb6a7fbc 100644
--- a/python/vyos/defaults.py
+++ b/python/vyos/defaults.py
@@ -48,6 +48,7 @@ api_data = {
     'port' : '8080',
     'socket' : False,
     'strict' : False,
+    'gql' : False,
     'debug' : False,
     'api_keys' : [ {"id": "testapp", "key": "qwerty"} ]
 }
diff --git a/python/vyos/ethtool.py b/python/vyos/ethtool.py
index e45b0f041..672ee2cc9 100644
--- a/python/vyos/ethtool.py
+++ b/python/vyos/ethtool.py
@@ -18,6 +18,9 @@ import re
 
 from vyos.util import popen
 
+# These drivers do not support using ethtool to change the speed, duplex, or flow control settings
+_drivers_without_speed_duplex_flow = ['vmxnet3', 'virtio_net', 'xen_netfront', 'iavf', 'ice', 'i40e']
+
 class Ethtool:
     """
     Class is used to retrive and cache information about an ethernet adapter
@@ -41,7 +44,7 @@ class Ethtool:
     #   '100' : {'full': '', 'half': ''},
     #   '1000': {'full': ''}
     #  }
-    _speed_duplex = { }
+    _speed_duplex = {'auto': {'auto': ''}}
     _ring_buffers = { }
     _ring_buffers_max = { }
     _driver_name = None
@@ -188,7 +191,7 @@ class Ethtool:
         if duplex not in ['auto', 'full', 'half']:
             raise ValueError(f'Value "{duplex}" for duplex is invalid!')
 
-        if self.get_driver_name() in ['vmxnet3', 'virtio_net', 'xen_netfront']:
+        if self.get_driver_name() in _drivers_without_speed_duplex_flow:
             return False
 
         if speed in self._speed_duplex:
@@ -198,7 +201,7 @@ class Ethtool:
 
     def check_flow_control(self):
         """ Check if the NIC supports flow-control """
-        if self.get_driver_name() in ['vmxnet3', 'virtio_net', 'xen_netfront']:
+        if self.get_driver_name() in _drivers_without_speed_duplex_flow:
             return False
         return self._flow_control
 
diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py
index 8b7402b7e..ff8623592 100644
--- a/python/vyos/firewall.py
+++ b/python/vyos/firewall.py
@@ -45,13 +45,19 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name):
 
     if 'state' in rule_conf and rule_conf['state']:
         states = ",".join([s for s, v in rule_conf['state'].items() if v == 'enable'])
-        output.append(f'ct state {{{states}}}')
+
+        if states:
+            output.append(f'ct state {{{states}}}')
 
     if 'protocol' in rule_conf and rule_conf['protocol'] != 'all':
         proto = rule_conf['protocol']
+        operator = ''
+        if proto[0] == '!':
+            operator = '!='
+            proto = proto[1:]
         if proto == 'tcp_udp':
             proto = '{tcp, udp}'
-        output.append('meta l4proto ' + proto)
+        output.append(f'meta l4proto {operator} {proto}')
 
     for side in ['destination', 'source']:
         if side in rule_conf:
@@ -59,7 +65,10 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name):
             side_conf = rule_conf[side]
 
             if 'address' in side_conf:
-                output.append(f'{ip_name} {prefix}addr {side_conf["address"]}')
+                suffix = side_conf['address']
+                if suffix[0] == '!':
+                    suffix = f'!= {suffix[1:]}'
+                output.append(f'{ip_name} {prefix}addr {suffix}')
 
             if 'mac_address' in side_conf:
                 suffix = side_conf["mac_address"]
@@ -69,24 +78,51 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name):
 
             if 'port' in side_conf:
                 proto = rule_conf['protocol']
-                port = side_conf["port"]
+                port = side_conf['port'].split(',')
+
+                ports = []
+                negated_ports = []
 
-                if isinstance(port, list):
-                    port = ",".join(port)
+                for p in port:
+                    if p[0] == '!':
+                        negated_ports.append(p[1:])
+                    else:
+                        ports.append(p)
 
                 if proto == 'tcp_udp':
                     proto = 'th'
 
-                output.append(f'{proto} {prefix}port {{{port}}}')
+                if ports:
+                    ports_str = ','.join(ports)
+                    output.append(f'{proto} {prefix}port {{{ports_str}}}')
+
+                if negated_ports:
+                    negated_ports_str = ','.join(negated_ports)
+                    output.append(f'{proto} {prefix}port != {{{negated_ports_str}}}')
 
             if 'group' in side_conf:
                 group = side_conf['group']
                 if 'address_group' in group:
                     group_name = group['address_group']
-                    output.append(f'{ip_name} {prefix}addr $A{def_suffix}_{group_name}')
+                    operator = ''
+                    if group_name[0] == '!':
+                        operator = '!='
+                        group_name = group_name[1:]
+                    output.append(f'{ip_name} {prefix}addr {operator} $A{def_suffix}_{group_name}')
                 elif 'network_group' in group:
                     group_name = group['network_group']
-                    output.append(f'{ip_name} {prefix}addr $N{def_suffix}_{group_name}')
+                    operator = ''
+                    if group_name[0] == '!':
+                        operator = '!='
+                        group_name = group_name[1:]
+                    output.append(f'{ip_name} {prefix}addr {operator} $N{def_suffix}_{group_name}')
+                if 'mac_group' in group:
+                    group_name = group['mac_group']
+                    operator = ''
+                    if group_name[0] == '!':
+                        operator = '!='
+                        group_name = group_name[1:]
+                    output.append(f'ether {prefix}addr {operator} $M_{group_name}')
                 if 'port_group' in group:
                     proto = rule_conf['protocol']
                     group_name = group['port_group']
@@ -94,10 +130,16 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name):
                     if proto == 'tcp_udp':
                         proto = 'th'
 
-                    output.append(f'{proto} {prefix}port $P_{group_name}')
+                    operator = ''
+                    if group_name[0] == '!':
+                        operator = '!='
+                        group_name = group_name[1:]
+
+                    output.append(f'{proto} {prefix}port {operator} $P_{group_name}')
 
     if 'log' in rule_conf and rule_conf['log'] == 'enable':
-        output.append('log')
+        action = rule_conf['action'] if 'action' in rule_conf else 'accept'
+        output.append(f'log prefix "[{fw_name[:19]}-{rule_id}-{action[:1].upper()}] "')
 
     if 'hop_limit' in rule_conf:
         operators = {'eq': '==', 'gt': '>', 'lt': '<'}
@@ -132,16 +174,14 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name):
 
     if 'limit' in rule_conf:
         if 'rate' in rule_conf['limit']:
-            output.append(f'limit rate {rule_conf["limit"]["rate"]}/second')
+            output.append(f'limit rate {rule_conf["limit"]["rate"]}')
             if 'burst' in rule_conf['limit']:
                 output.append(f'burst {rule_conf["limit"]["burst"]} packets')
 
     if 'recent' in rule_conf:
         count = rule_conf['recent']['count']
         time = rule_conf['recent']['time']
-        # output.append(f'meter {fw_name}_{rule_id} {{ ip saddr and 255.255.255.255 limit rate over {count}/{time} burst {count} packets }}')
-        # Waiting on input from nftables developers due to
-        # bug with above line and atomic chain flushing.
+        output.append(f'add @RECENT{def_suffix}_{fw_name}_{rule_id} {{ {ip_name} saddr limit rate over {count}/{time} burst {count} packets }}')
 
     if 'time' in rule_conf:
         output.append(parse_time(rule_conf['time']))
@@ -150,7 +190,6 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name):
     if tcp_flags:
         output.append(parse_tcp_flags(tcp_flags))
 
-
     output.append('counter')
 
     if 'set' in rule_conf:
@@ -165,15 +204,9 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name):
     return " ".join(output)
 
 def parse_tcp_flags(flags):
-    all_flags = []
-    include = []
-    for flag in flags.split(","):
-        if flag[0] == '!':
-            flag = flag[1:]
-        else:
-            include.append(flag)
-        all_flags.append(flag)
-    return f'tcp flags & ({"|".join(all_flags)}) == {"|".join(include)}'
+    include = [flag for flag in flags if flag != 'not']
+    exclude = list(flags['not']) if 'not' in flags else []
+    return f'tcp flags & ({"|".join(include + exclude)}) == {"|".join(include) if include else "0x0"}'
 
 def parse_time(time):
     out = []
@@ -209,7 +242,7 @@ def parse_policy_set(set_conf, def_suffix):
         table = set_conf['table']
         if table == 'main':
             table = '254'
-        mark = 0x7FFFFFFF - int(set_conf['table'])
+        mark = 0x7FFFFFFF - int(table)
         out.append(f'meta mark set {mark}')
     if 'tcp_mss' in set_conf:
         mss = set_conf['tcp_mss']
diff --git a/python/vyos/frr.py b/python/vyos/frr.py
index a8f115d9a..cbba19ab7 100644
--- a/python/vyos/frr.py
+++ b/python/vyos/frr.py
@@ -73,15 +73,15 @@ from vyos.util import cmd
 import logging
 from logging.handlers import SysLogHandler
 import os
+import sys
+
 LOG = logging.getLogger(__name__)
+DEBUG = False
 
-DEBUG = os.path.exists('/tmp/vyos.frr.debug')
-if DEBUG:
-    LOG.setLevel(logging.DEBUG)
-    ch = SysLogHandler(address='/dev/log')
-    ch2 = logging.StreamHandler()
-    LOG.addHandler(ch)
-    LOG.addHandler(ch2)
+ch = SysLogHandler(address='/dev/log')
+ch2 = logging.StreamHandler(stream=sys.stdout)
+LOG.addHandler(ch)
+LOG.addHandler(ch2)
 
 _frr_daemons = ['zebra', 'bgpd', 'fabricd', 'isisd', 'ospf6d', 'ospfd', 'pbrd',
                 'pimd', 'ripd', 'ripngd', 'sharpd', 'staticd', 'vrrpd', 'ldpd',
@@ -121,6 +121,12 @@ class ConfigSectionNotFound(FrrError):
     """
     pass
 
+def init_debugging():
+    global DEBUG
+
+    DEBUG = os.path.exists('/tmp/vyos.frr.debug')
+    if DEBUG:
+        LOG.setLevel(logging.DEBUG)
 
 def get_configuration(daemon=None, marked=False):
     """ Get current running FRR configuration
@@ -424,6 +430,8 @@ class FRRConfig:
 
         Using this overwrites the current loaded config objects and replaces the original loaded config
         '''
+        init_debugging()
+
         self.imported_config = get_configuration(daemon=daemon)
         if daemon:
             LOG.debug(f'load_configuration: Configuration loaded from FRR daemon {daemon}')
diff --git a/python/vyos/ifconfig/bridge.py b/python/vyos/ifconfig/bridge.py
index 27073b266..ffd9c590f 100644
--- a/python/vyos/ifconfig/bridge.py
+++ b/python/vyos/ifconfig/bridge.py
@@ -298,7 +298,6 @@ class BridgeIf(Interface):
 
         tmp = dict_search('member.interface', config)
         if tmp:
-
             for interface, interface_config in tmp.items():
                 # if interface does yet not exist bail out early and
                 # add it later
@@ -316,10 +315,13 @@ class BridgeIf(Interface):
                 # enslave interface port to bridge
                 self.add_port(interface)
 
-                # always set private-vlan/port isolation
-                tmp = dict_search('isolated', interface_config)
-                value = 'on' if (tmp != None) else 'off'
-                lower.set_port_isolation(value)
+                if not interface.startswith('wlan'):
+                    # always set private-vlan/port isolation - this can not be
+                    # done when lower link is a wifi link, as it will trigger:
+                    # RTNETLINK answers: Operation not supported
+                    tmp = dict_search('isolated', interface_config)
+                    value = 'on' if (tmp != None) else 'off'
+                    lower.set_port_isolation(value)
 
                 # set bridge port path cost
                 if 'cost' in interface_config:
diff --git a/python/vyos/ifconfig/interface.py b/python/vyos/ifconfig/interface.py
index 91c7f0c33..6b0f08fd4 100755
--- a/python/vyos/ifconfig/interface.py
+++ b/python/vyos/ifconfig/interface.py
@@ -1,4 +1,4 @@
-# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io>
+# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -13,7 +13,6 @@
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library.  If not, see <http://www.gnu.org/licenses/>.
 
-from netifaces import interfaces
 import os
 import re
 import json
@@ -1080,16 +1079,17 @@ class Interface(Control):
         if addr in self._addr:
             return False
 
-        addr_is_v4 = is_ipv4(addr)
-
         # add to interface
         if addr == 'dhcp':
             self.set_dhcp(True)
         elif addr == 'dhcpv6':
             self.set_dhcpv6(True)
         elif not is_intf_addr_assigned(self.ifname, addr):
-            self._cmd(f'ip addr add "{addr}" '
-                    f'{"brd + " if addr_is_v4 else ""}dev "{self.ifname}"')
+            tmp = f'ip addr add {addr} dev {self.ifname}'
+            # Add broadcast address for IPv4
+            if is_ipv4(addr): tmp += ' brd +'
+
+            self._cmd(tmp)
         else:
             return False
 
@@ -1228,12 +1228,11 @@ class Interface(Control):
         options_file = f'{config_base}_{ifname}.options'
         pid_file = f'{config_base}_{ifname}.pid'
         lease_file = f'{config_base}_{ifname}.leases'
-
-        # Stop client with old config files to get the right IF_METRIC.
         systemd_service = f'dhclient@{ifname}.service'
-        if is_systemd_service_active(systemd_service):
-            self._cmd(f'systemctl stop {systemd_service}')
 
+        # 'up' check is mandatory b/c even if the interface is A/D, as soon as
+        # the DHCP client is started the interface will be placed in u/u state.
+        # This is not what we intended to do when disabling an interface.
         if enable and 'disable' not in self._config:
             if dict_search('dhcp_options.host_name', self._config) == None:
                 # read configured system hostname.
@@ -1244,16 +1243,19 @@ class Interface(Control):
                     tmp = {'dhcp_options' : { 'host_name' : hostname}}
                     self._config = dict_merge(tmp, self._config)
 
-            render(options_file, 'dhcp-client/daemon-options.tmpl',
-                   self._config)
-            render(config_file, 'dhcp-client/ipv4.tmpl',
-                   self._config)
+            render(options_file, 'dhcp-client/daemon-options.tmpl', self._config)
+            render(config_file, 'dhcp-client/ipv4.tmpl', self._config)
 
-            # 'up' check is mandatory b/c even if the interface is A/D, as soon as
-            # the DHCP client is started the interface will be placed in u/u state.
-            # This is not what we intended to do when disabling an interface.
-            return self._cmd(f'systemctl restart {systemd_service}')
+            # When the DHCP client is restarted a brief outage will occur, as
+            # the old lease is released a new one is acquired (T4203). We will
+            # only restart DHCP client if it's option changed, or if it's not
+            # running, but it should be running (e.g. on system startup)
+            if 'dhcp_options_changed' in self._config or not is_systemd_service_active(systemd_service):
+                return self._cmd(f'systemctl restart {systemd_service}')
+            return None
         else:
+            if is_systemd_service_active(systemd_service):
+                self._cmd(f'systemctl stop {systemd_service}')
             # cleanup old config files
             for file in [config_file, options_file, pid_file, lease_file]:
                 if os.path.isfile(file):
@@ -1284,48 +1286,57 @@ class Interface(Control):
             if os.path.isfile(config_file):
                 os.remove(config_file)
 
-    def set_mirror(self):
+    def set_mirror_redirect(self):
         # Please refer to the document for details
         #   - https://man7.org/linux/man-pages/man8/tc.8.html
         #   - https://man7.org/linux/man-pages/man8/tc-mirred.8.html
         # Depening if we are the source or the target interface of the port
         # mirror we need to setup some variables.
         source_if = self._config['ifname']
-        config = self._config.get('mirror', None)
 
+        mirror_config = None
+        if 'mirror' in self._config:
+            mirror_config = self._config['mirror']
         if 'is_mirror_intf' in self._config:
             source_if = next(iter(self._config['is_mirror_intf']))
-            config = self._config['is_mirror_intf'][source_if].get('mirror', None)
-
-        # Check configuration stored by old perl code before delete T3782/T4056
-        if not 'redirect' in self._config and not 'traffic_policy' in self._config:
-            # Please do not clear the 'set $? = 0 '. It's meant to force a return of 0
-            # Remove existing mirroring rules
-            delete_tc_cmd  = f'tc qdisc del dev {source_if} handle ffff: ingress 2> /dev/null;'
-            delete_tc_cmd += f'tc qdisc del dev {source_if} handle 1: root prio 2> /dev/null;'
-            delete_tc_cmd += 'set $?=0'
-            self._popen(delete_tc_cmd)
-
-        # Bail out early if nothing needs to be configured
-        if not config:
-            return
-
-        for direction, mirror_if in config.items():
-            if mirror_if not in interfaces():
-                continue
-
-            if direction == 'ingress':
-                handle = 'ffff: ingress'
-                parent = 'ffff:'
-            elif direction == 'egress':
-                handle = '1: root prio'
-                parent = '1:'
-
-            # Mirror egress traffic
-            mirror_cmd  = f'tc qdisc add dev {source_if} handle {handle}; '
-            # Export the mirrored traffic to the interface
-            mirror_cmd += f'tc filter add dev {source_if} parent {parent} protocol all prio 10 u32 match u32 0 0 flowid 1:1 action mirred egress mirror dev {mirror_if}'
-            self._popen(mirror_cmd)
+            mirror_config = self._config['is_mirror_intf'][source_if].get('mirror', None)
+
+        redirect_config = None
+
+        # clear existing ingess - ignore errors (e.g. "Error: Cannot find specified
+        # qdisc on specified device") - we simply cleanup all stuff here
+        self._popen(f'tc qdisc del dev {source_if} parent ffff: 2>/dev/null');
+        self._popen(f'tc qdisc del dev {source_if} parent 1: 2>/dev/null');
+
+        # Apply interface mirror policy
+        if mirror_config:
+            for direction, target_if in mirror_config.items():
+                if direction == 'ingress':
+                    handle = 'ffff: ingress'
+                    parent = 'ffff:'
+                elif direction == 'egress':
+                    handle = '1: root prio'
+                    parent = '1:'
+
+                # Mirror egress traffic
+                mirror_cmd  = f'tc qdisc add dev {source_if} handle {handle}; '
+                # Export the mirrored traffic to the interface
+                mirror_cmd += f'tc filter add dev {source_if} parent {parent} protocol '\
+                              f'all prio 10 u32 match u32 0 0 flowid 1:1 action mirred '\
+                              f'egress mirror dev {target_if}'
+                _, err = self._popen(mirror_cmd)
+                if err: print('tc qdisc(filter for mirror port failed')
+
+        # Apply interface traffic redirection policy
+        elif 'redirect' in self._config:
+            _, err = self._popen(f'tc qdisc add dev {source_if} handle ffff: ingress')
+            if err: print(f'tc qdisc add for redirect failed!')
+
+            target_if = self._config['redirect']
+            _, err = self._popen(f'tc filter add dev {source_if} parent ffff: protocol '\
+                                 f'all prio 10 u32 match u32 0 0 flowid 1:1 action mirred '\
+                                 f'egress redirect dev {target_if}')
+            if err: print('tc filter add for redirect failed')
 
     def set_xdp(self, state):
         """
@@ -1424,9 +1435,6 @@ class Interface(Control):
                 else:
                     self.del_addr(addr)
 
-        for addr in new_addr:
-            self.add_addr(addr)
-
         # start DHCPv6 client when only PD was configured
         if dhcpv6pd:
             self.set_dhcpv6(True)
@@ -1441,16 +1449,15 @@ class Interface(Control):
             # checked before
             self.set_vrf(config.get('vrf', ''))
 
+        # Add this section after vrf T4331
+        for addr in new_addr:
+            self.add_addr(addr)
+
         # Configure MSS value for IPv4 TCP connections
         tmp = dict_search('ip.adjust_mss', config)
         value = tmp if (tmp != None) else '0'
         self.set_tcp_ipv4_mss(value)
 
-        # Configure MSS value for IPv6 TCP connections
-        tmp = dict_search('ipv6.adjust_mss', config)
-        value = tmp if (tmp != None) else '0'
-        self.set_tcp_ipv6_mss(value)
-
         # Configure ARP cache timeout in milliseconds - has default value
         tmp = dict_search('ip.arp_cache_timeout', config)
         value = tmp if (tmp != None) else '30'
@@ -1496,6 +1503,18 @@ class Interface(Control):
         value = tmp if (tmp != None) else '0'
         self.set_ipv4_source_validation(value)
 
+        # MTU - Maximum Transfer Unit has a default value. It must ALWAYS be set
+        # before mangling any IPv6 option. If MTU is less then 1280 IPv6 will be
+        # automatically disabled by the kernel. Also MTU must be increased before
+        # configuring any IPv6 address on the interface.
+        if 'mtu' in config:
+            self.set_mtu(config.get('mtu'))
+
+        # Configure MSS value for IPv6 TCP connections
+        tmp = dict_search('ipv6.adjust_mss', config)
+        value = tmp if (tmp != None) else '0'
+        self.set_tcp_ipv6_mss(value)
+
         # IPv6 forwarding
         tmp = dict_search('ipv6.disable_forwarding', config)
         value = '0' if (tmp != None) else '1'
@@ -1518,10 +1537,6 @@ class Interface(Control):
         value = tmp if (tmp != None) else '1'
         self.set_ipv6_dad_messages(value)
 
-        # MTU - Maximum Transfer Unit
-        if 'mtu' in config:
-            self.set_mtu(config.get('mtu'))
-
         # Delete old IPv6 EUI64 addresses before changing MAC
         for addr in (dict_search('ipv6.address.eui64_old', config) or []):
             self.del_ipv6_eui64_address(addr)
@@ -1546,8 +1561,8 @@ class Interface(Control):
         # eXpress Data Path - highly experimental
         self.set_xdp('xdp' in config)
 
-        # configure port mirror
-        self.set_mirror()
+        # configure interface mirror or redirection target
+        self.set_mirror_redirect()
 
         # Enable/Disable of an interface must always be done at the end of the
         # derived class to make use of the ref-counting set_admin_state()
@@ -1706,6 +1721,3 @@ class VLANIf(Interface):
             return None
 
         return super().set_admin_state(state)
-
-    def set_mirror(self):
-        return
diff --git a/python/vyos/ifconfig/loopback.py b/python/vyos/ifconfig/loopback.py
index 192c12f5c..b3babfadc 100644
--- a/python/vyos/ifconfig/loopback.py
+++ b/python/vyos/ifconfig/loopback.py
@@ -1,4 +1,4 @@
-# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io>
+# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -23,7 +23,6 @@ class LoopbackIf(Interface):
     """
     _persistent_addresses = ['127.0.0.1/8', '::1/128']
     iftype = 'loopback'
-
     definition = {
         **Interface.definition,
         **{
@@ -32,6 +31,7 @@ class LoopbackIf(Interface):
             'bridgeable': True,
         }
     }
+
     def remove(self):
         """
         Loopback interface can not be deleted from operating system. We can
@@ -57,12 +57,14 @@ class LoopbackIf(Interface):
         interface setup code and provide a single point of entry when workin
         on any interface. """
 
-        addr = config.get('address', [])
+        address = config.get('address', [])
         # We must ensure that the loopback addresses are never deleted from the system
-        addr += self._persistent_addresses
+        for tmp in self._persistent_addresses:
+            if tmp not in address:
+                address.append(tmp)
 
         # Update IP address entry in our dictionary
-        config.update({'address' : addr})
+        config.update({'address' : address})
 
         # call base class
         super().update(config)
diff --git a/python/vyos/ifconfig/vxlan.py b/python/vyos/ifconfig/vxlan.py
index 0c5282db4..516a19f24 100644
--- a/python/vyos/ifconfig/vxlan.py
+++ b/python/vyos/ifconfig/vxlan.py
@@ -1,4 +1,4 @@
-# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io>
+# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -68,6 +68,16 @@ class VXLANIf(Interface):
             'vni'                        : 'id',
         }
 
+        # IPv6 flowlabels can only be used on IPv6 tunnels, thus we need to
+        # ensure that at least the first remote IP address is passed to the
+        # tunnel creation command. Subsequent tunnel remote addresses can later
+        # be added to the FDB
+        remote_list = None
+        if 'remote' in self.config:
+            # skip first element as this is already configured as remote
+            remote_list = self.config['remote'][1:]
+            self.config['remote'] = self.config['remote'][0]
+
         cmd = 'ip link add {ifname} type {type} dstport {port}'
         for vyos_key, iproute2_key in mapping.items():
             # dict_search will return an empty dict "{}" for valueless nodes like
@@ -82,3 +92,10 @@ class VXLANIf(Interface):
         self._cmd(cmd.format(**self.config))
         # interface is always A/D down. It needs to be enabled explicitly
         self.set_admin_state('down')
+
+        # VXLAN tunnel is always recreated on any change - see interfaces-vxlan.py
+        if remote_list:
+            for remote in remote_list:
+                cmd = f'bridge fdb append to 00:00:00:00:00:00 dst {remote} ' \
+                       'port {port} dev {ifname}'
+                self._cmd(cmd.format(**self.config))
diff --git a/python/vyos/ifconfig/wireless.py b/python/vyos/ifconfig/wireless.py
index 748b6e02d..88eaa772b 100644
--- a/python/vyos/ifconfig/wireless.py
+++ b/python/vyos/ifconfig/wireless.py
@@ -49,10 +49,10 @@ class WiFiIf(Interface):
         on any interface. """
 
         # We can not call add_to_bridge() until wpa_supplicant is running, thus
-        # we will remove the key from the config dict and react to this specal
-        # case in thie derived class.
+        # we will remove the key from the config dict and react to this special
+        # case in this derived class.
         # re-add ourselves to any bridge we might have fallen out of
-        bridge_member = ''
+        bridge_member = None
         if 'is_bridge_member' in config:
             bridge_member = config['is_bridge_member']
             del config['is_bridge_member']
diff --git a/python/vyos/migrator.py b/python/vyos/migrator.py
index 4574bb6d1..a2e0daabd 100644
--- a/python/vyos/migrator.py
+++ b/python/vyos/migrator.py
@@ -195,7 +195,7 @@ class Migrator(object):
             # This will force calling all migration scripts:
             cfg_versions = {}
 
-        sys_versions = systemversions.get_system_versions()
+        sys_versions = systemversions.get_system_component_version()
 
         # save system component versions in json file for easy reference
         self.save_json_record(sys_versions)
diff --git a/python/vyos/pki.py b/python/vyos/pki.py
index 68ad73bf2..0b916eaae 100644
--- a/python/vyos/pki.py
+++ b/python/vyos/pki.py
@@ -331,3 +331,29 @@ def verify_certificate(cert, ca_cert):
         return True
     except InvalidSignature:
         return False
+
+# Certificate chain
+
+def find_parent(cert, ca_certs):
+    for ca_cert in ca_certs:
+        if verify_certificate(cert, ca_cert):
+            return ca_cert
+    return None
+
+def find_chain(cert, ca_certs):
+    remaining = ca_certs.copy()
+    chain = [cert]
+
+    while remaining:
+        parent = find_parent(chain[-1], remaining)
+        if parent is None:
+            # No parent in the list of remaining certificates or there's a circular dependency
+            break
+        elif parent == chain[-1]:
+            # Self-signed: must be root CA (end of chain)
+            break
+        else:
+            remaining.remove(parent)
+            chain.append(parent)
+
+    return chain
diff --git a/python/vyos/remote.py b/python/vyos/remote.py
index aa62ac60d..66044fa52 100644
--- a/python/vyos/remote.py
+++ b/python/vyos/remote.py
@@ -83,8 +83,7 @@ def check_storage(path, size):
     directory = path if os.path.isdir(path) else (os.path.dirname(os.path.expanduser(path)) or os.getcwd())
     # `size` can be None or 0 to indicate unknown size.
     if not size:
-        print_error('Warning: Cannot determine size of remote file.')
-        print_error('Bravely continuing regardless.')
+        print_error('Warning: Cannot determine size of remote file. Bravely continuing regardless.')
         return
 
     if size < 1024 * 1024:
@@ -227,7 +226,7 @@ class HttpC:
                 r.raise_for_status()
                 # If the request got redirected, keep the last URL we ended up with.
                 final_urlstring = r.url
-                if r.history:
+                if r.history and self.progressbar:
                     print_error('Redirecting to ' + final_urlstring)
                 # Check for the prospective file size.
                 try:
@@ -317,11 +316,12 @@ def friendly_download(local_path, urlstring, source_host='', source_port=0):
         sys.exit(1)
     except:
         import traceback
+        print_error(f'Failed to download {urlstring}.')
         # There are a myriad different reasons a download could fail.
         # SSH errors, FTP errors, I/O errors, HTTP errors (403, 404...)
         # We omit the scary stack trace but print the error nevertheless.
-        print_error(f'Failed to download {urlstring}.')
-        traceback.print_exception(*sys.exc_info()[:2], None)
+        exc_type, exc_value, exc_traceback = sys.exc_info()
+        traceback.print_exception(exc_type, exc_value, None, 0, None, False)
         sys.exit(1)
     else:
         print_error('Download complete.')
diff --git a/python/vyos/systemversions.py b/python/vyos/systemversions.py
index 9b3f4f413..f2da76d4f 100644
--- a/python/vyos/systemversions.py
+++ b/python/vyos/systemversions.py
@@ -17,7 +17,10 @@ import os
 import re
 import sys
 import vyos.defaults
+from vyos.xml import component_version
 
+# legacy version, reading from the file names in
+# /opt/vyatta/etc/config-migrate/current
 def get_system_versions():
     """
     Get component versions from running system; critical failure if
@@ -37,3 +40,7 @@ def get_system_versions():
             system_versions[pair[0]] = int(pair[1])
 
     return system_versions
+
+# read from xml cache
+def get_system_component_version():
+    return component_version()
diff --git a/python/vyos/template.py b/python/vyos/template.py
index 6f65c6c98..132f5ddde 100644
--- a/python/vyos/template.py
+++ b/python/vyos/template.py
@@ -18,7 +18,7 @@ import os
 
 from jinja2 import Environment
 from jinja2 import FileSystemLoader
-
+from jinja2 import ChainableUndefined
 from vyos.defaults import directories
 from vyos.util import chmod
 from vyos.util import chown
@@ -27,6 +27,7 @@ from vyos.util import makedir
 
 # Holds template filters registered via register_filter()
 _FILTERS = {}
+_TESTS = {}
 
 # reuse Environments with identical settings to improve performance
 @functools.lru_cache(maxsize=2)
@@ -42,8 +43,10 @@ def _get_environment(location=None):
         cache_size=100,
         loader=loc_loader,
         trim_blocks=True,
+        undefined=ChainableUndefined,
     )
     env.filters.update(_FILTERS)
+    env.tests.update(_TESTS)
     return env
 
 
@@ -67,6 +70,26 @@ def register_filter(name, func=None):
     _FILTERS[name] = func
     return func
 
+def register_test(name, func=None):
+    """Register a function to be available as test in templates under given name.
+
+    It can also be used as a decorator, see below in this module for examples.
+
+    :raise RuntimeError:
+        when trying to register a test after a template has been rendered already
+    :raise ValueError: when trying to register a name which was taken already
+    """
+    if func is None:
+        return functools.partial(register_test, name)
+    if _get_environment.cache_info().currsize:
+        raise RuntimeError(
+            "Tests can only be registered before rendering the first template"
+            )
+    if name in _TESTS:
+        raise ValueError(f"A test with name {name!r} was registered already")
+    _TESTS[name] = func
+    return func
+
 
 def render_to_string(template, content, formater=None, location=None):
     """Render a template from the template directory, raise on any errors.
@@ -127,6 +150,14 @@ def render(
 ##################################
 # Custom template filters follow #
 ##################################
+@register_filter('force_to_list')
+def force_to_list(value):
+    """ Convert scalars to single-item lists and leave lists untouched """
+    if isinstance(value, list):
+        return value
+    else:
+        return [value]
+
 @register_filter('ip_from_cidr')
 def ip_from_cidr(prefix):
     """ Take an IPv4/IPv6 CIDR host and strip cidr mask.
@@ -516,6 +547,19 @@ def nft_rule(rule_conf, fw_name, rule_id, ip_name='ip'):
     from vyos.firewall import parse_rule
     return parse_rule(rule_conf, fw_name, rule_id, ip_name)
 
+@register_filter('nft_default_rule')
+def nft_default_rule(fw_conf, fw_name):
+    output = ['counter']
+    default_action = fw_conf.get('default_action', 'accept')
+
+    if 'enable_default_log' in fw_conf:
+        action_suffix = default_action[:1].upper()
+        output.append(f'log prefix "[{fw_name[:19]}-default-{action_suffix}] "')
+
+    output.append(nft_action(default_action))
+    output.append(f'comment "{fw_name} default-action {default_action}"')
+    return " ".join(output)
+
 @register_filter('nft_state_policy')
 def nft_state_policy(conf, state, ipv6=False):
     out = [f'ct state {state}']
@@ -535,6 +579,7 @@ def nft_intra_zone_action(zone_conf, ipv6=False):
     if 'intra_zone_filtering' in zone_conf:
         intra_zone = zone_conf['intra_zone_filtering']
         fw_name = 'ipv6_name' if ipv6 else 'name'
+        name_prefix = 'NAME6_' if ipv6 else 'NAME_'
 
         if 'action' in intra_zone:
             if intra_zone['action'] == 'accept':
@@ -542,5 +587,57 @@ def nft_intra_zone_action(zone_conf, ipv6=False):
             return intra_zone['action']
         elif dict_search_args(intra_zone, 'firewall', fw_name):
             name = dict_search_args(intra_zone, 'firewall', fw_name)
-            return f'jump {name}'
+            return f'jump {name_prefix}{name}'
     return 'return'
+
+@register_test('vyos_defined')
+def vyos_defined(value, test_value=None, var_type=None):
+    """
+    Jinja2 plugin to test if a variable is defined and not none - vyos_defined
+    will test value if defined and is not none and return true or false.
+
+    If test_value is supplied, the value must also pass == test_value to return true.
+    If var_type is supplied, the value must also be of the specified class/type
+
+    Examples:
+    1. Test if var is defined and not none:
+    {% if foo is vyos_defined %}
+    ...
+    {% endif %}
+
+    2. Test if variable is defined, not none and has value "something"
+    {% if bar is vyos_defined("something") %}
+    ...
+    {% endif %}
+
+    Parameters
+    ----------
+    value : any
+        Value to test from ansible
+    test_value : any, optional
+        Value to test in addition of defined and not none, by default None
+    var_type : ['float', 'int', 'str', 'list', 'dict', 'tuple', 'bool'], optional
+        Type or Class to test for
+
+    Returns
+    -------
+    boolean
+        True if variable matches criteria, False in other cases.
+
+    Implementation inspired and re-used from https://github.com/aristanetworks/ansible-avd/
+    """
+
+    from jinja2 import Undefined
+
+    if isinstance(value, Undefined) or value is None:
+        # Invalid value - return false
+        return False
+    elif test_value is not None and value != test_value:
+        # Valid value but not matching the optional argument
+        return False
+    elif str(var_type).lower() in ['float', 'int', 'str', 'list', 'dict', 'tuple', 'bool'] and str(var_type).lower() != type(value).__name__:
+        # Invalid class - return false
+        return False
+    else:
+        # Valid value and is matching optional argument if provided - return true
+        return True
diff --git a/python/vyos/util.py b/python/vyos/util.py
index 954c6670d..de55e108b 100644
--- a/python/vyos/util.py
+++ b/python/vyos/util.py
@@ -774,6 +774,14 @@ def dict_search_recursive(dict_object, key):
             for x in dict_search_recursive(j, key):
                 yield x
 
+def get_bridge_fdb(interface):
+    """ Returns the forwarding database entries for a given interface """
+    if not os.path.exists(f'/sys/class/net/{interface}'):
+        return None
+    from json import loads
+    tmp = loads(cmd(f'bridge -j fdb show dev {interface}'))
+    return tmp
+
 def get_interface_config(interface):
     """ Returns the used encapsulation protocol for given interface.
         If interface does not exist, None is returned.
@@ -952,14 +960,23 @@ def install_into_config(conf, config_paths, override_prompt=True):
         return None
 
     count = 0
+    failed = []
 
     for path in config_paths:
         if override_prompt and conf.exists(path) and not conf.is_multi(path):
             if not ask_yes_no(f'Config node "{node}" already exists. Do you want to overwrite it?'):
                 continue
 
-        cmd(f'/opt/vyatta/sbin/my_set {path}')
-        count += 1
+        try:
+            cmd(f'/opt/vyatta/sbin/my_set {path}')
+            count += 1
+        except:
+            failed.append(path)
+
+    if failed:
+        print(f'Failed to install {len(failed)} value(s). Commands to manually install:')
+        for path in failed:
+            print(f'set {path}')
 
     if count > 0:
         print(f'{count} value(s) installed. Use "compare" to see the pending changes, and "commit" to apply.')
@@ -972,6 +989,11 @@ def is_wwan_connected(interface):
     if not interface.startswith('wwan'):
         raise ValueError(f'Specified interface "{interface}" is not a WWAN interface')
 
+    # ModemManager is required for connection(s) - if service is not running,
+    # there won't be any connection at all!
+    if not is_systemd_service_active('ModemManager.service'):
+        return False
+
     modem = interface.lstrip('wwan')
 
     tmp = cmd(f'mmcli --modem {modem} --output-json')
@@ -988,3 +1010,17 @@ def boot_configuration_complete() -> bool:
     if os.path.isfile(config_status):
         return True
     return False
+
+def sysctl_read(name):
+    """ Read and return current value of sysctl() option """
+    tmp = cmd(f'sysctl {name}')
+    return tmp.split()[-1]
+
+def sysctl_write(name, value):
+    """ Change value via sysctl() - return True if changed, False otherwise """
+    tmp = cmd(f'sysctl {name}')
+    # last list index contains the actual value - only write if value differs
+    if sysctl_read(name) != str(value):
+        call(f'sysctl -wq {name}={value}')
+        return True
+    return False
diff --git a/python/vyos/validate.py b/python/vyos/validate.py
index 0dad2a6cb..e005da0e4 100644
--- a/python/vyos/validate.py
+++ b/python/vyos/validate.py
@@ -43,19 +43,14 @@ def _are_same_ip(one, two):
     s_two = AF_INET if is_ipv4(two) else AF_INET6
     return inet_pton(f_one, one) == inet_pton(f_one, two)
 
-def is_intf_addr_assigned(intf, addr):
-    if '/' in addr:
-        ip,mask = addr.split('/')
-        return _is_intf_addr_assigned(intf, ip, mask)
-    return _is_intf_addr_assigned(intf, addr)
-
-def _is_intf_addr_assigned(intf, address, netmask=None):
+def is_intf_addr_assigned(intf, address) -> bool:
     """
     Verify if the given IPv4/IPv6 address is assigned to specific interface.
     It can check both a single IP address (e.g. 192.0.2.1 or a assigned CIDR
     address 192.0.2.1/24.
     """
     from vyos.template import is_ipv4
+
     from netifaces import ifaddresses
     from netifaces import AF_INET
     from netifaces import AF_INET6
@@ -76,6 +71,9 @@ def _is_intf_addr_assigned(intf, address, netmask=None):
     addr_type = AF_INET if is_ipv4(address) else AF_INET6
 
     # Check every IP address on this interface for a match
+    netmask = None
+    if '/' in address:
+        address, netmask = address.split('/')
     for ip in ifaces.get(addr_type,[]):
         # ip can have the interface name in the 'addr' field, we need to remove it
         # {'addr': 'fe80::a00:27ff:fec5:f821%eth2', 'netmask': 'ffff:ffff:ffff:ffff::'}
@@ -99,14 +97,20 @@ def _is_intf_addr_assigned(intf, address, netmask=None):
 
     return False
 
-def is_addr_assigned(addr):
-    """
-    Verify if the given IPv4/IPv6 address is assigned to any interface
-    """
+def is_addr_assigned(ip_address, vrf=None) -> bool:
+    """ Verify if the given IPv4/IPv6 address is assigned to any interfac """
     from netifaces import interfaces
-    for intf in interfaces():
-        tmp = is_intf_addr_assigned(intf, addr)
-        if tmp == True:
+    from vyos.util import get_interface_config
+    from vyos.util import dict_search
+    for interface in interfaces():
+        # Check if interface belongs to the requested VRF, if this is not the
+        # case there is no need to proceed with this data set - continue loop
+        # with next element
+        tmp = get_interface_config(interface)
+        if dict_search('master', tmp) != vrf:
+            continue
+
+        if is_intf_addr_assigned(interface, ip_address):
             return True
 
     return False
diff --git a/python/vyos/xml/__init__.py b/python/vyos/xml/__init__.py
index e0eacb2d1..6db446a40 100644
--- a/python/vyos/xml/__init__.py
+++ b/python/vyos/xml/__init__.py
@@ -46,8 +46,8 @@ def is_tag(lpath):
 def is_leaf(lpath, flat=True):
     return load_configuration().is_leaf(lpath, flat)
 
-def component_versions():
-    return load_configuration().component_versions()
+def component_version():
+    return load_configuration().component_version()
 
 def defaults(lpath, flat=False):
     return load_configuration().defaults(lpath, flat)
diff --git a/python/vyos/xml/definition.py b/python/vyos/xml/definition.py
index 5e0d5282c..bc3892b42 100644
--- a/python/vyos/xml/definition.py
+++ b/python/vyos/xml/definition.py
@@ -249,10 +249,11 @@ class XML(dict):
     # @lru_cache(maxsize=100)
     # XXX: need to use cachetool instead - for later
 
-    def component_versions(self) -> dict:
-        sort_component = sorted(self[kw.component_version].items(),
-                                key = lambda kv: kv[0])
-        return dict(sort_component)
+    def component_version(self) -> dict:
+        d = {}
+        for k in sorted(self[kw.component_version]):
+            d[k] = int(self[kw.component_version][k])
+        return d
 
     def defaults(self, lpath, flat):
         d = self[kw.default]