Merge branch 'current' into goodnetnick-shloginotp-T4754

19 files changed, 789 insertions, 348 deletions

diff --git a/python/vyos/accel_ppp.py b/python/vyos/accel_ppp.py
new file mode 100644
index 000000000..bfc8ee5a9
--- /dev/null
+++ b/python/vyos/accel_ppp.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2022 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import sys
+
+import vyos.opmode
+from vyos.util import rc_cmd
+
+
+def get_server_statistics(accel_statistics, pattern, sep=':') -> dict:
+    import re
+
+    stat_dict = {'sessions': {}}
+
+    cpu = re.search(r'cpu(.*)', accel_statistics).group(0)
+    # Find all lines with pattern, for example 'sstp:'
+    data = re.search(rf'{pattern}(.*)', accel_statistics, re.DOTALL).group(0)
+    session_starting = re.search(r'starting(.*)', data).group(0)
+    session_active = re.search(r'active(.*)', data).group(0)
+
+    for entry in {cpu, session_starting, session_active}:
+        if sep in entry:
+            key, value = entry.split(sep)
+            if key in ['starting', 'active', 'finishing']:
+                stat_dict['sessions'][key] = value.strip()
+                continue
+            stat_dict[key] = value.strip()
+    return stat_dict
+
+
+def accel_cmd(port: int, command: str) -> str:
+    _, output = rc_cmd(f'/usr/bin/accel-cmd -p{port} {command}')
+    return output
+
+
+def accel_out_parse(accel_output: list[str]) -> list[dict[str, str]]:
+    """ Parse accel-cmd show sessions output """
+    data_list: list[dict[str, str]] = list()
+    field_names: list[str] = list()
+
+    field_names_unstripped: list[str] = accel_output.pop(0).split('|')
+    for field_name in field_names_unstripped:
+        field_names.append(field_name.strip())
+
+    while accel_output:
+        if '|' not in accel_output[0]:
+            accel_output.pop(0)
+            continue
+
+        current_item: list[str] = accel_output.pop(0).split('|')
+        item_dict: dict[str, str] = {}
+
+        for field_index in range(len(current_item)):
+            field_name: str = field_names[field_index]
+            field_value: str = current_item[field_index].strip()
+            item_dict[field_name] = field_value
+
+        data_list.append(item_dict)
+
+    return data_list
diff --git a/python/vyos/base.py b/python/vyos/base.py
index 78067d5b2..9b93cb2f2 100644
--- a/python/vyos/base.py
+++ b/python/vyos/base.py
@@ -15,17 +15,47 @@
 
 from textwrap import fill
 
+
+class BaseWarning:
+    def __init__(self, header, message, **kwargs):
+        self.message = message
+        self.kwargs = kwargs
+        if 'width' not in kwargs:
+            self.width = 72
+        if 'initial_indent' in kwargs:
+            del self.kwargs['initial_indent']
+        if 'subsequent_indent' in kwargs:
+            del self.kwargs['subsequent_indent']
+        self.textinitindent = header
+        self.standardindent = ''
+
+    def print(self):
+        messages = self.message.split('\n')
+        isfirstmessage = True
+        initial_indent = self.textinitindent
+        print('')
+        for mes in messages:
+            mes = fill(mes, initial_indent=initial_indent,
+                       subsequent_indent=self.standardindent, **self.kwargs)
+            if isfirstmessage:
+                isfirstmessage = False
+                initial_indent = self.standardindent
+            print(f'{mes}')
+        print('')
+
+
 class Warning():
-    def __init__(self, message):
-        # Reformat the message and trim it to 72 characters in length
-        message = fill(message, width=72)
-        print(f'\nWARNING: {message}')
+    def __init__(self, message, **kwargs):
+        self.BaseWarn = BaseWarning('WARNING: ', message, **kwargs)
+        self.BaseWarn.print()
+
 
 class DeprecationWarning():
-    def __init__(self, message):
+    def __init__(self, message, **kwargs):
         # Reformat the message and trim it to 72 characters in length
-        message = fill(message, width=72)
-        print(f'\nDEPRECATION WARNING: {message}\n')
+        self.BaseWarn = BaseWarning('DEPRECATION WARNING: ', message, **kwargs)
+        self.BaseWarn.print()
+
 
 class ConfigError(Exception):
     def __init__(self, message):
diff --git a/python/vyos/component_version.py b/python/vyos/component_version.py
new file mode 100644
index 000000000..a4e318d08
--- /dev/null
+++ b/python/vyos/component_version.py
@@ -0,0 +1,192 @@
+# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Functions for reading/writing component versions.
+
+The config file version string has the following form:
+
+VyOS 1.3/1.4:
+
+// Warning: Do not remove the following line.
+// vyos-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack@3:conntrack-sync@2:dhcp-relay@2:dhcp-server@6:dhcpv6-server@1:dns-forwarding@3:firewall@5:https@2:interfaces@22:ipoe-server@1:ipsec@5:isis@1:l2tp@3:lldp@1:mdns@1:nat@5:ntp@1:pppoe-server@5:pptp@2:qos@1:quagga@8:rpki@1:salt@1:snmp@2:ssh@2:sstp@3:system@21:vrrp@2:vyos-accel-ppp@2:wanloadbalance@3:webproxy@2:zone-policy@1"
+// Release version: 1.3.0
+
+VyOS 1.2:
+
+/* Warning: Do not remove the following line. */
+/* === vyatta-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack-sync@1:conntrack@1:dhcp-relay@2:dhcp-server@5:dns-forwarding@1:firewall@5:ipsec@5:l2tp@1:mdns@1:nat@4:ntp@1:pppoe-server@2:pptp@1:qos@1:quagga@7:snmp@1:ssh@1:system@10:vrrp@2:wanloadbalance@3:webgui@1:webproxy@2:zone-policy@1" === */
+/* Release version: 1.2.8 */
+
+"""
+
+import os
+import re
+import sys
+import fileinput
+
+from vyos.xml import component_version
+from vyos.version import get_version
+from vyos.defaults import directories
+
+DEFAULT_CONFIG_PATH = os.path.join(directories['config'], 'config.boot')
+
+def from_string(string_line, vintage='vyos'):
+    """
+    Get component version dictionary from string.
+    Return empty dictionary if string contains no config information
+    or raise error if component version string malformed.
+    """
+    version_dict = {}
+
+    if vintage == 'vyos':
+        if re.match(r'// vyos-config-version:.+', string_line):
+            if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', string_line):
+                raise ValueError(f"malformed configuration string: {string_line}")
+
+            for pair in re.findall(r'([\w,-]+)@(\d+)', string_line):
+                version_dict[pair[0]] = int(pair[1])
+
+    elif vintage == 'vyatta':
+        if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line):
+            if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line):
+                raise ValueError(f"malformed configuration string: {string_line}")
+
+            for pair in re.findall(r'([\w,-]+)@(\d+)', string_line):
+                version_dict[pair[0]] = int(pair[1])
+    else:
+        raise ValueError("Unknown config string vintage")
+
+    return version_dict
+
+def from_file(config_file_name=DEFAULT_CONFIG_PATH, vintage='vyos'):
+    """
+    Get component version dictionary parsing config file line by line
+    """
+    with open(config_file_name, 'r') as f:
+        for line_in_config in f:
+            version_dict = from_string(line_in_config, vintage=vintage)
+            if version_dict:
+                return version_dict
+
+    # no version information
+    return {}
+
+def from_system():
+    """
+    Get system component version dict.
+    """
+    return component_version()
+
+def legacy_from_system():
+    """
+    Get system component version dict from legacy location.
+    This is for a transitional sanity check; the directory will eventually
+    be removed.
+    """
+    system_versions = {}
+    legacy_dir = directories['current']
+
+    # To be removed:
+    if not os.path.isdir(legacy_dir):
+        return system_versions
+
+    try:
+        version_info = os.listdir(legacy_dir)
+    except OSError as err:
+        sys.exit(repr(err))
+
+    for info in version_info:
+        if re.match(r'[\w,-]+@\d+', info):
+            pair = info.split('@')
+            system_versions[pair[0]] = int(pair[1])
+
+    return system_versions
+
+def format_string(ver: dict) -> str:
+    """
+    Version dict to string.
+    """
+    keys = list(ver)
+    keys.sort()
+    l = []
+    for k in keys:
+        v = ver[k]
+        l.append(f'{k}@{v}')
+    sep = ':'
+    return sep.join(l)
+
+def version_footer(ver: dict, vintage='vyos') -> str:
+    """
+    Version footer as string.
+    """
+    ver_str = format_string(ver)
+    release = get_version()
+    if vintage == 'vyos':
+        ret_str = (f'// Warning: Do not remove the following line.\n'
+                +  f'// vyos-config-version: "{ver_str}"\n'
+                +  f'// Release version: {release}\n')
+    elif vintage == 'vyatta':
+        ret_str = (f'/* Warning: Do not remove the following line. */\n'
+                +  f'/* === vyatta-config-version: "{ver_str}" === */\n'
+                +  f'/* Release version: {release} */\n')
+    else:
+        raise ValueError("Unknown config string vintage")
+
+    return ret_str
+
+def system_footer(vintage='vyos') -> str:
+    """
+    System version footer as string.
+    """
+    ver_d = from_system()
+    return version_footer(ver_d, vintage=vintage)
+
+def write_version_footer(ver: dict, file_name, vintage='vyos'):
+    """
+    Write version footer to file.
+    """
+    footer = version_footer(ver=ver, vintage=vintage)
+    if file_name:
+        with open(file_name, 'a') as f:
+            f.write(footer)
+    else:
+        sys.stdout.write(footer)
+
+def write_system_footer(file_name, vintage='vyos'):
+    """
+    Write system version footer to file.
+    """
+    ver_d = from_system()
+    return write_version_footer(ver_d, file_name=file_name, vintage=vintage)
+
+def remove_footer(file_name):
+    """
+    Remove old version footer.
+    """
+    for line in fileinput.input(file_name, inplace=True):
+        if re.match(r'/\* Warning:.+ \*/$', line):
+            continue
+        if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line):
+            continue
+        if re.match(r'/\* Release version:.+ \*/$', line):
+            continue
+        if re.match('// vyos-config-version:.+', line):
+            continue
+        if re.match('// Warning:.+', line):
+            continue
+        if re.match('// Release version:.+', line):
+            continue
+        sys.stdout.write(line)
diff --git a/python/vyos/component_versions.py b/python/vyos/component_versions.py
deleted file mode 100644
index 90b458aae..000000000
--- a/python/vyos/component_versions.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# Copyright 2017 VyOS maintainers and contributors <maintainers@vyos.io>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library.  If not, see <http://www.gnu.org/licenses/>.
-
-"""
-The version data looks like:
-
-/* Warning: Do not remove the following line. */
-/* === vyatta-config-version:
-"cluster@1:config-management@1:conntrack-sync@1:conntrack@1:dhcp-relay@1:dhcp-server@4:firewall@5:ipsec@4:nat@4:qos@1:quagga@2:system@8:vrrp@1:wanloadbalance@3:webgui@1:webproxy@1:zone-policy@1"
-=== */
-/* Release version: 1.2.0-rolling+201806131737 */
-"""
-
-import re
-
-def get_component_version(string_line):
-    """
-    Get component version dictionary from string
-    return empty dictionary if string contains no config information
-    or raise error if component version string malformed
-    """
-    return_value = {}
-    if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line):
-
-        if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line):
-            raise ValueError("malformed configuration string: " + str(string_line))
-
-        for pair in re.findall(r'([\w,-]+)@(\d+)', string_line):
-            if pair[0] in return_value.keys():
-                raise ValueError("duplicate unit name: \"" + str(pair[0]) + "\" in string: \"" + string_line + "\"")
-            return_value[pair[0]] = int(pair[1])
-
-    return return_value
-
-
-def get_component_versions_from_file(config_file_name='/opt/vyatta/etc/config/config.boot'):
-    """
-    Get component version dictionary parsing config file line by line
-    """
-    f = open(config_file_name, 'r')
-    for line_in_config in f:
-        component_version = get_component_version(line_in_config)
-        if component_version:
-            return component_version
-    raise ValueError("no config string in file:", config_file_name)
diff --git a/python/vyos/configdep.py b/python/vyos/configdep.py
new file mode 100644
index 000000000..d4b2cc78f
--- /dev/null
+++ b/python/vyos/configdep.py
@@ -0,0 +1,95 @@
+# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import json
+import typing
+from inspect import stack
+
+from vyos.util import load_as_module
+from vyos.defaults import directories
+from vyos.configsource import VyOSError
+from vyos import ConfigError
+
+# https://peps.python.org/pep-0484/#forward-references
+# for type 'Config'
+if typing.TYPE_CHECKING:
+    from vyos.config import Config
+
+dependent_func: dict[str, list[typing.Callable]] = {}
+
+def canon_name(name: str) -> str:
+    return os.path.splitext(name)[0].replace('-', '_')
+
+def canon_name_of_path(path: str) -> str:
+    script = os.path.basename(path)
+    return canon_name(script)
+
+def caller_name() -> str:
+    return stack()[-1].filename
+
+def read_dependency_dict() -> dict:
+    path = os.path.join(directories['data'],
+                        'config-mode-dependencies.json')
+    with open(path) as f:
+        d = json.load(f)
+    return d
+
+def get_dependency_dict(config: 'Config') -> dict:
+    if hasattr(config, 'cached_dependency_dict'):
+        d = getattr(config, 'cached_dependency_dict')
+    else:
+        d = read_dependency_dict()
+        setattr(config, 'cached_dependency_dict', d)
+    return d
+
+def run_config_mode_script(script: str, config: 'Config'):
+    path = os.path.join(directories['conf_mode'], script)
+    name = canon_name(script)
+    mod = load_as_module(name, path)
+
+    config.set_level([])
+    try:
+        c = mod.get_config(config)
+        mod.verify(c)
+        mod.generate(c)
+        mod.apply(c)
+    except (VyOSError, ConfigError) as e:
+        raise ConfigError(repr(e))
+
+def def_closure(target: str, config: 'Config',
+                tagnode: typing.Optional[str] = None) -> typing.Callable:
+    script = target + '.py'
+    def func_impl():
+        if tagnode:
+            os.environ['VYOS_TAGNODE_VALUE'] = tagnode
+        run_config_mode_script(script, config)
+    return func_impl
+
+def set_dependents(case: str, config: 'Config',
+                   tagnode: typing.Optional[str] = None):
+    d = get_dependency_dict(config)
+    k = canon_name_of_path(caller_name())
+    l = dependent_func.setdefault(k, [])
+    for target in d[k][case]:
+        func = def_closure(target, config, tagnode)
+        l.append(func)
+
+def call_dependents():
+    k = canon_name_of_path(caller_name())
+    l = dependent_func.get(k, [])
+    while l:
+        f = l.pop(0)
+        f()
diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py
index e9cdb69e4..b88615513 100644
--- a/python/vyos/configtree.py
+++ b/python/vyos/configtree.py
@@ -1,5 +1,5 @@
 # configtree -- a standalone VyOS config file manipulation library (Python bindings)
-# Copyright (C) 2018 VyOS maintainers and contributors
+# Copyright (C) 2018-2022 VyOS maintainers and contributors
 #
 # This library is free software; you can redistribute it and/or modify it under the terms of
 # the GNU Lesser General Public License as published by the Free Software Foundation;
@@ -12,6 +12,7 @@
 # You should have received a copy of the GNU Lesser General Public License along with this library;
 # if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 
+import os
 import re
 import json
 
@@ -147,6 +148,8 @@ class ConfigTree(object):
             self.__config = address
             self.__version = ''
 
+        self.__migration = os.environ.get('VYOS_MIGRATION')
+
     def __del__(self):
         if self.__config is not None:
             self.__destroy(self.__config)
@@ -191,18 +194,27 @@ class ConfigTree(object):
             else:
                 self.__set_add_value(self.__config, path_str, str(value).encode())
 
+        if self.__migration:
+            print(f"- op: set path: {path} value: {value} replace: {replace}")
+
     def delete(self, path):
         check_path(path)
         path_str = " ".join(map(str, path)).encode()
 
         self.__delete(self.__config, path_str)
 
+        if self.__migration:
+            print(f"- op: delete path: {path}")
+
     def delete_value(self, path, value):
         check_path(path)
         path_str = " ".join(map(str, path)).encode()
 
         self.__delete_value(self.__config, path_str, value.encode())
 
+        if self.__migration:
+            print(f"- op: delete_value path: {path} value: {value}")
+
     def rename(self, path, new_name):
         check_path(path)
         path_str = " ".join(map(str, path)).encode()
@@ -216,6 +228,9 @@ class ConfigTree(object):
         if (res != 0):
             raise ConfigTreeError("Path [{}] doesn't exist".format(path))
 
+        if self.__migration:
+            print(f"- op: rename old_path: {path} new_path: {new_path}")
+
     def copy(self, old_path, new_path):
         check_path(old_path)
         check_path(new_path)
@@ -229,6 +244,9 @@ class ConfigTree(object):
         if (res != 0):
             raise ConfigTreeError("Path [{}] doesn't exist".format(old_path))
 
+        if self.__migration:
+            print(f"- op: copy old_path: {old_path} new_path: {new_path}")
+
     def exists(self, path):
         check_path(path)
         path_str = " ".join(map(str, path)).encode()
diff --git a/python/vyos/configverify.py b/python/vyos/configverify.py
index afa0c5b33..8e0ce701e 100644
--- a/python/vyos/configverify.py
+++ b/python/vyos/configverify.py
@@ -388,8 +388,10 @@ def verify_accel_ppp_base_service(config, local_users=True):
     """
     # vertify auth settings
     if local_users and dict_search('authentication.mode', config) == 'local':
-        if dict_search(f'authentication.local_users', config) == None:
-            raise ConfigError('Authentication mode local requires local users to be configured!')
+        if (dict_search(f'authentication.local_users', config) is None or
+                dict_search(f'authentication.local_users', config) == {}):
+            raise ConfigError(
+                'Authentication mode local requires local users to be configured!')
 
         for user in dict_search('authentication.local_users.username', config):
             user_config = config['authentication']['local_users']['username'][user]
diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py
index 4075e55b0..48263eef5 100644
--- a/python/vyos/firewall.py
+++ b/python/vyos/firewall.py
@@ -20,6 +20,9 @@ import os
 import re
 
 from pathlib import Path
+from socket import AF_INET
+from socket import AF_INET6
+from socket import getaddrinfo
 from time import strftime
 
 from vyos.remote import download
@@ -31,65 +34,31 @@ from vyos.util import dict_search_args
 from vyos.util import dict_search_recursive
 from vyos.util import run
 
+# Domain Resolver
 
-# Functions for firewall group domain-groups
-def get_ips_domains_dict(list_domains):
-    """
-    Get list of IPv4 addresses by list of domains
-    Ex: get_ips_domains_dict(['ex1.com', 'ex2.com'])
-        {'ex1.com': ['192.0.2.1'], 'ex2.com': ['192.0.2.2', '192.0.2.3']}
-    """
-    from socket import gethostbyname_ex
-    from socket import gaierror
-
-    ip_dict = {}
-    for domain in list_domains:
-        try:
-            _, _, ips = gethostbyname_ex(domain)
-            ip_dict[domain] = ips
-        except gaierror:
-            pass
-
-    return ip_dict
-
-def nft_init_set(group_name, table="vyos_filter", family="ip"):
-    """
-    table ip vyos_filter {
-        set GROUP_NAME
-            type ipv4_addr
-           flags interval
-        }
-    """
-    return call(f'nft add set ip {table} {group_name} {{ type ipv4_addr\\; flags interval\\; }}')
-
-
-def nft_add_set_elements(group_name, elements, table="vyos_filter", family="ip"):
-    """
-    table ip vyos_filter {
-        set GROUP_NAME {
-            type ipv4_addr
-            flags interval
-            elements = { 192.0.2.1, 192.0.2.2 }
-        }
-    """
-    elements = ", ".join(elements)
-    return call(f'nft add element {family} {table} {group_name} {{ {elements} }} ')
-
-def nft_flush_set(group_name, table="vyos_filter", family="ip"):
-    """
-    Flush elements of nft set
-    """
-    return call(f'nft flush set {family} {table} {group_name}')
-
-def nft_update_set_elements(group_name, elements, table="vyos_filter", family="ip"):
-    """
-    Update elements of nft set
-    """
-    flush_set = nft_flush_set(group_name, table="vyos_filter", family="ip")
-    nft_add_set = nft_add_set_elements(group_name, elements, table="vyos_filter", family="ip")
-    return flush_set, nft_add_set
-
-# END firewall group domain-group (sets)
+def fqdn_config_parse(firewall):
+    firewall['ip_fqdn'] = {}
+    firewall['ip6_fqdn'] = {}
+
+    for domain, path in dict_search_recursive(firewall, 'fqdn'):
+        fw_name = path[1] # name/ipv6-name
+        rule = path[3] # rule id
+        suffix = path[4][0] # source/destination (1 char)
+        set_name = f'{fw_name}_{rule}_{suffix}'
+            
+        if path[0] == 'name':
+            firewall['ip_fqdn'][set_name] = domain
+        elif path[0] == 'ipv6_name':
+            firewall['ip6_fqdn'][set_name] = domain
+
+def fqdn_resolve(fqdn, ipv6=False):
+    try:
+        res = getaddrinfo(fqdn, None, AF_INET6 if ipv6 else AF_INET)
+        return set(item[4][0] for item in res)
+    except:
+        return None
+
+# End Domain Resolver
 
 def find_nftables_rule(table, chain, rule_matches=[]):
     # Find rule in table/chain that matches all criteria and return the handle
@@ -144,12 +113,26 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name):
         if side in rule_conf:
             prefix = side[0]
             side_conf = rule_conf[side]
+            address_mask = side_conf.get('address_mask', None)
 
             if 'address' in side_conf:
                 suffix = side_conf['address']
-                if suffix[0] == '!':
-                    suffix = f'!= {suffix[1:]}'
-                output.append(f'{ip_name} {prefix}addr {suffix}')
+                operator = ''
+                exclude = suffix[0] == '!'
+                if exclude:
+                    operator = '!= '
+                    suffix = suffix[1:]
+                if address_mask:
+                    operator = '!=' if exclude else '=='
+                    operator = f'& {address_mask} {operator} '
+                output.append(f'{ip_name} {prefix}addr {operator}{suffix}')
+
+            if 'fqdn' in side_conf:
+                fqdn = side_conf['fqdn']
+                operator = ''
+                if fqdn[0] == '!':
+                    operator = '!='
+                output.append(f'{ip_name} {prefix}addr {operator} @FQDN_{fw_name}_{rule_id}_{prefix}')
 
             if dict_search_args(side_conf, 'geoip', 'country_code'):
                 operator = ''
@@ -192,9 +175,13 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name):
                 if 'address_group' in group:
                     group_name = group['address_group']
                     operator = ''
-                    if group_name[0] == '!':
+                    exclude = group_name[0] == "!"
+                    if exclude:
                         operator = '!='
                         group_name = group_name[1:]
+                    if address_mask:
+                        operator = '!=' if exclude else '=='
+                        operator = f'& {address_mask} {operator}'
                     output.append(f'{ip_name} {prefix}addr {operator} @A{def_suffix}_{group_name}')
                 # Generate firewall group domain-group
                 elif 'domain_group' in group:
diff --git a/python/vyos/formatversions.py b/python/vyos/formatversions.py
deleted file mode 100644
index 29117a5d3..000000000
--- a/python/vyos/formatversions.py
+++ /dev/null
@@ -1,109 +0,0 @@
-# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this library.  If not, see <http://www.gnu.org/licenses/>.
-
-import sys
-import os
-import re
-import fileinput
-
-def read_vyatta_versions(config_file):
-    config_file_versions = {}
-
-    with open(config_file, 'r') as config_file_handle:
-        for config_line in config_file_handle:
-            if re.match(r'/\* === vyatta-config-version:.+=== \*/$', config_line):
-                if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', config_line):
-                    raise ValueError("malformed configuration string: "
-                            "{}".format(config_line))
-
-                for pair in re.findall(r'([\w,-]+)@(\d+)', config_line):
-                    config_file_versions[pair[0]] = int(pair[1])
-
-
-    return config_file_versions
-
-def read_vyos_versions(config_file):
-    config_file_versions = {}
-
-    with open(config_file, 'r') as config_file_handle:
-        for config_line in config_file_handle:
-            if re.match(r'// vyos-config-version:.+', config_line):
-                if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', config_line):
-                    raise ValueError("malformed configuration string: "
-                            "{}".format(config_line))
-
-                for pair in re.findall(r'([\w,-]+)@(\d+)', config_line):
-                    config_file_versions[pair[0]] = int(pair[1])
-
-    return config_file_versions
-
-def remove_versions(config_file):
-    """
-    Remove old version string.
-    """
-    for line in fileinput.input(config_file, inplace=True):
-        if re.match(r'/\* Warning:.+ \*/$', line):
-            continue
-        if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line):
-            continue
-        if re.match(r'/\* Release version:.+ \*/$', line):
-            continue
-        if re.match('// vyos-config-version:.+', line):
-            continue
-        if re.match('// Warning:.+', line):
-            continue
-        if re.match('// Release version:.+', line):
-            continue
-        sys.stdout.write(line)
-
-def format_versions_string(config_versions):
-    cfg_keys = list(config_versions.keys())
-    cfg_keys.sort()
-
-    component_version_strings = []
-
-    for key in cfg_keys:
-        cfg_vers = config_versions[key]
-        component_version_strings.append('{}@{}'.format(key, cfg_vers))
-
-    separator = ":"
-    component_version_string = separator.join(component_version_strings)
-
-    return component_version_string
-
-def write_vyatta_versions_foot(config_file, component_version_string,
-                                 os_version_string):
-    if config_file:
-        with open(config_file, 'a') as config_file_handle:
-            config_file_handle.write('/* Warning: Do not remove the following line. */\n')
-            config_file_handle.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string))
-            config_file_handle.write('/* Release version: {} */\n'.format(os_version_string))
-    else:
-        sys.stdout.write('/* Warning: Do not remove the following line. */\n')
-        sys.stdout.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string))
-        sys.stdout.write('/* Release version: {} */\n'.format(os_version_string))
-
-def write_vyos_versions_foot(config_file, component_version_string,
-                               os_version_string):
-    if config_file:
-        with open(config_file, 'a') as config_file_handle:
-            config_file_handle.write('// Warning: Do not remove the following line.\n')
-            config_file_handle.write('// vyos-config-version: "{}"\n'.format(component_version_string))
-            config_file_handle.write('// Release version: {}\n'.format(os_version_string))
-    else:
-        sys.stdout.write('// Warning: Do not remove the following line.\n')
-        sys.stdout.write('// vyos-config-version: "{}"\n'.format(component_version_string))
-        sys.stdout.write('// Release version: {}\n'.format(os_version_string))
-
diff --git a/python/vyos/frr.py b/python/vyos/frr.py
index 0ffd5cba9..ccb132dd5 100644
--- a/python/vyos/frr.py
+++ b/python/vyos/frr.py
@@ -477,7 +477,7 @@ class FRRConfig:
                 # for the listed FRR issues above
                 pass
         if count >= count_max:
-            raise ConfigurationNotValid(f'Config commit retry counter ({count_max}) exceeded')
+            raise ConfigurationNotValid(f'Config commit retry counter ({count_max}) exceeded for {daemon} dameon!')
 
         # Save configuration to /run/frr/config/frr.conf
         save_configuration()
diff --git a/python/vyos/ifconfig/__init__.py b/python/vyos/ifconfig/__init__.py
index a37615c8f..206b2bba1 100644
--- a/python/vyos/ifconfig/__init__.py
+++ b/python/vyos/ifconfig/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io>
+# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -36,4 +36,6 @@ from vyos.ifconfig.tunnel import TunnelIf
 from vyos.ifconfig.wireless import WiFiIf
 from vyos.ifconfig.l2tpv3 import L2TPv3If
 from vyos.ifconfig.macsec import MACsecIf
+from vyos.ifconfig.veth import VethIf
 from vyos.ifconfig.wwan import WWANIf
+from vyos.ifconfig.sstpc import SSTPCIf
diff --git a/python/vyos/ifconfig/macvlan.py b/python/vyos/ifconfig/macvlan.py
index 776014bc3..2266879ec 100644
--- a/python/vyos/ifconfig/macvlan.py
+++ b/python/vyos/ifconfig/macvlan.py
@@ -1,4 +1,4 @@
-# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io>
+# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -30,10 +30,17 @@ class MACVLANIf(Interface):
     }
 
     def _create(self):
+        """
+        Create MACvlan interface in OS kernel. Interface is administrative
+        down by default.
+        """
         # please do not change the order when assembling the command
         cmd = 'ip link add {ifname} link {source_interface} type {type} mode {mode}'
         self._cmd(cmd.format(**self.config))
 
+        # interface is always A/D down. It needs to be enabled explicitly
+        self.set_admin_state('down')
+
     def set_mode(self, mode):
         ifname = self.config['ifname']
         cmd = f'ip link set dev {ifname} type macvlan mode {mode}'
diff --git a/python/vyos/ifconfig/sstpc.py b/python/vyos/ifconfig/sstpc.py
new file mode 100644
index 000000000..50fc6ee6b
--- /dev/null
+++ b/python/vyos/ifconfig/sstpc.py
@@ -0,0 +1,40 @@
+# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/>.
+
+from vyos.ifconfig.interface import Interface
+
+@Interface.register
+class SSTPCIf(Interface):
+    iftype = 'sstpc'
+    definition = {
+        **Interface.definition,
+        **{
+            'section': 'sstpc',
+            'prefixes': ['sstpc', ],
+            'eternal': 'sstpc[0-9]+$',
+        },
+    }
+
+    def _create(self):
+        # we can not create this interface as it is managed outside
+        pass
+
+    def _delete(self):
+        # we can not create this interface as it is managed outside
+        pass
+
+    def get_mac(self):
+        """ Get a synthetic MAC address. """
+        return self.get_mac_synthetic()
diff --git a/python/vyos/ifconfig/veth.py b/python/vyos/ifconfig/veth.py
new file mode 100644
index 000000000..aafbf226a
--- /dev/null
+++ b/python/vyos/ifconfig/veth.py
@@ -0,0 +1,54 @@
+# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/>.
+
+from vyos.ifconfig.interface import Interface
+
+
+@Interface.register
+class VethIf(Interface):
+    """
+    Abstraction of a Linux veth interface
+    """
+    iftype = 'veth'
+    definition = {
+        **Interface.definition,
+        **{
+            'section': 'virtual-ethernet',
+            'prefixes': ['veth', ],
+            'bridgeable': True,
+        },
+    }
+
+    def _create(self):
+        """
+        Create veth interface in OS kernel. Interface is administrative
+        down by default.
+        """
+        # check before create, as we have 2 veth interfaces in our CLI
+        # interface virtual-ethernet veth0 peer-name 'veth1'
+        # interface virtual-ethernet veth1 peer-name 'veth0'
+        #
+        # but iproute2 creates the pair with one command:
+        # ip link add vet0 type veth peer name veth1
+        if self.exists(self.config['peer_name']):
+            return
+
+        # create virtual-ethernet interface
+        cmd = 'ip link add {ifname} type {type}'.format(**self.config)
+        cmd += f' peer name {self.config["peer_name"]}'
+        self._cmd(cmd)
+
+        # interface is always A/D down. It needs to be enabled explicitly
+        self.set_admin_state('down')
diff --git a/python/vyos/migrator.py b/python/vyos/migrator.py
index c6e3435ca..87c74e1ea 100644
--- a/python/vyos/migrator.py
+++ b/python/vyos/migrator.py
@@ -1,4 +1,4 @@
-# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
+# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -16,11 +16,13 @@
 import sys
 import os
 import json
-import subprocess
-import vyos.version
+import logging
+
 import vyos.defaults
-import vyos.systemversions as systemversions
-import vyos.formatversions as formatversions
+import vyos.component_version as component_version
+from vyos.util import cmd
+
+log_file = os.path.join(vyos.defaults.directories['config'], 'vyos-migrate.log')
 
 class MigratorError(Exception):
     pass
@@ -31,9 +33,21 @@ class Migrator(object):
         self._force = force
         self._set_vintage = set_vintage
         self._config_file_vintage = None
-        self._log_file = None
         self._changed = False
 
+    def init_logger(self):
+        self.logger = logging.getLogger(__name__)
+        self.logger.setLevel(logging.DEBUG)
+
+        # on adding the file handler, allow write permission for cfg_group;
+        # restore original umask on exit
+        mask = os.umask(0o113)
+        fh = logging.FileHandler(log_file)
+        formatter = logging.Formatter('%(message)s')
+        fh.setFormatter(formatter)
+        self.logger.addHandler(fh)
+        os.umask(mask)
+
     def read_config_file_versions(self):
         """
         Get component versions from config file footer and set vintage;
@@ -42,13 +56,13 @@ class Migrator(object):
         cfg_file = self._config_file
         component_versions = {}
 
-        cfg_versions = formatversions.read_vyatta_versions(cfg_file)
+        cfg_versions = component_version.from_file(cfg_file, vintage='vyatta')
 
         if cfg_versions:
             self._config_file_vintage = 'vyatta'
             component_versions = cfg_versions
 
-        cfg_versions = formatversions.read_vyos_versions(cfg_file)
+        cfg_versions = component_version.from_file(cfg_file, vintage='vyos')
 
         if cfg_versions:
             self._config_file_vintage = 'vyos'
@@ -70,34 +84,15 @@ class Migrator(object):
         else:
             return True
 
-    def open_log_file(self):
-        """
-        Open log file for migration, catching any error.
-        Note that, on boot, migration takes place before the canonical log
-        directory is created, hence write to the config file directory.
-        """
-        self._log_file = os.path.join(vyos.defaults.directories['config'],
-                                      'vyos-migrate.log')
-        # on creation, allow write permission for cfg_group;
-        # restore original umask on exit
-        mask = os.umask(0o113)
-        try:
-            log = open('{0}'.format(self._log_file), 'w')
-            log.write("List of executed migration scripts:\n")
-        except Exception as e:
-            os.umask(mask)
-            print("Logging error: {0}".format(e))
-            return None
-
-        os.umask(mask)
-        return log
-
     def run_migration_scripts(self, config_file_versions, system_versions):
         """
         Run migration scripts iteratively, until config file version equals
         system component version.
         """
-        log = self.open_log_file()
+        os.environ['VYOS_MIGRATION'] = '1'
+        self.init_logger()
+
+        self.logger.info("List of executed migration scripts:")
 
         cfg_versions = config_file_versions
         sys_versions = system_versions
@@ -129,8 +124,9 @@ class Migrator(object):
                         '{}-to-{}'.format(cfg_ver, next_ver))
 
                 try:
-                    subprocess.check_call([migrate_script,
-                        self._config_file])
+                    out = cmd([migrate_script, self._config_file])
+                    self.logger.info(f'{migrate_script}')
+                    if out: self.logger.info(out)
                 except FileNotFoundError:
                     pass
                 except Exception as err:
@@ -138,38 +134,25 @@ class Migrator(object):
                           "".format(migrate_script, err))
                     sys.exit(1)
 
-                if log:
-                    try:
-                        log.write('{0}\n'.format(migrate_script))
-                    except Exception as e:
-                        print("Error writing log: {0}".format(e))
-
                 cfg_ver = next_ver
-
             rev_versions[key] = cfg_ver
 
-        if log:
-            log.close()
-
+        del os.environ['VYOS_MIGRATION']
         return rev_versions
 
     def write_config_file_versions(self, cfg_versions):
         """
         Write new versions string.
         """
-        versions_string = formatversions.format_versions_string(cfg_versions)
-
-        os_version_string = vyos.version.get_version()
-
         if self._config_file_vintage == 'vyatta':
-            formatversions.write_vyatta_versions_foot(self._config_file,
-                                                      versions_string,
-                                                      os_version_string)
+            component_version.write_version_footer(cfg_versions,
+                                                   self._config_file,
+                                                   vintage='vyatta')
 
         if self._config_file_vintage == 'vyos':
-            formatversions.write_vyos_versions_foot(self._config_file,
-                                                    versions_string,
-                                                    os_version_string)
+            component_version.write_version_footer(cfg_versions,
+                                                   self._config_file,
+                                                   vintage='vyos')
 
     def save_json_record(self, component_versions: dict):
         """
@@ -200,7 +183,7 @@ class Migrator(object):
             # This will force calling all migration scripts:
             cfg_versions = {}
 
-        sys_versions = systemversions.get_system_component_version()
+        sys_versions = component_version.from_system()
 
         # save system component versions in json file for easy reference
         self.save_json_record(sys_versions)
@@ -216,7 +199,7 @@ class Migrator(object):
         if not self._changed:
             return
 
-        formatversions.remove_versions(cfg_file)
+        component_version.remove_footer(cfg_file)
 
         self.write_config_file_versions(rev_versions)
 
@@ -237,7 +220,7 @@ class VirtualMigrator(Migrator):
         if not self._changed:
             return
 
-        formatversions.remove_versions(cfg_file)
+        component_version.remove_footer(cfg_file)
 
         self.write_config_file_versions(cfg_versions)
 
diff --git a/python/vyos/nat.py b/python/vyos/nat.py
index 31bbdc386..8a311045a 100644
--- a/python/vyos/nat.py
+++ b/python/vyos/nat.py
@@ -16,6 +16,8 @@
 
 from vyos.template import is_ip_network
 from vyos.util import dict_search_args
+from vyos.template import bracketize_ipv6
+
 
 def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False):
     output = []
@@ -69,6 +71,7 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False):
         else:
             translation_output.append('to')
             if addr:
+                addr = bracketize_ipv6(addr)
                 translation_output.append(addr)
 
         options = []
@@ -85,8 +88,13 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False):
             translation_str += f' {",".join(options)}'
 
     for target in ['source', 'destination']:
+        if target not in rule_conf:
+            continue
+
+        side_conf = rule_conf[target]
         prefix = target[:1]
-        addr = dict_search_args(rule_conf, target, 'address')
+
+        addr = dict_search_args(side_conf, 'address')
         if addr and not (ignore_type_addr and target == nat_type):
             operator = ''
             if addr[:1] == '!':
@@ -94,7 +102,7 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False):
                 addr = addr[1:]
             output.append(f'{ip_prefix} {prefix}addr {operator} {addr}')
 
-        addr_prefix = dict_search_args(rule_conf, target, 'prefix')
+        addr_prefix = dict_search_args(side_conf, 'prefix')
         if addr_prefix and ipv6:
             operator = ''
             if addr_prefix[:1] == '!':
@@ -102,7 +110,7 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False):
                 addr_prefix = addr[1:]
             output.append(f'ip6 {prefix}addr {operator} {addr_prefix}')
 
-        port = dict_search_args(rule_conf, target, 'port')
+        port = dict_search_args(side_conf, 'port')
         if port:
             protocol = rule_conf['protocol']
             if protocol == 'tcp_udp':
@@ -113,6 +121,51 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False):
                 port = port[1:]
             output.append(f'{protocol} {prefix}port {operator} {{ {port} }}')
 
+        if 'group' in side_conf:
+            group = side_conf['group']
+            if 'address_group' in group and not (ignore_type_addr and target == nat_type):
+                group_name = group['address_group']
+                operator = ''
+                if group_name[0] == '!':
+                    operator = '!='
+                    group_name = group_name[1:]
+                output.append(f'{ip_prefix} {prefix}addr {operator} @A_{group_name}')
+            # Generate firewall group domain-group
+            elif 'domain_group' in group and not (ignore_type_addr and target == nat_type):
+                group_name = group['domain_group']
+                operator = ''
+                if group_name[0] == '!':
+                    operator = '!='
+                    group_name = group_name[1:]
+                output.append(f'{ip_prefix} {prefix}addr {operator} @D_{group_name}')
+            elif 'network_group' in group and not (ignore_type_addr and target == nat_type):
+                group_name = group['network_group']
+                operator = ''
+                if group_name[0] == '!':
+                    operator = '!='
+                    group_name = group_name[1:]
+                output.append(f'{ip_prefix} {prefix}addr {operator} @N_{group_name}')
+            if 'mac_group' in group:
+                group_name = group['mac_group']
+                operator = ''
+                if group_name[0] == '!':
+                    operator = '!='
+                    group_name = group_name[1:]
+                output.append(f'ether {prefix}addr {operator} @M_{group_name}')
+            if 'port_group' in group:
+                proto = rule_conf['protocol']
+                group_name = group['port_group']
+
+                if proto == 'tcp_udp':
+                    proto = 'th'
+
+                operator = ''
+                if group_name[0] == '!':
+                    operator = '!='
+                    group_name = group_name[1:]
+
+                output.append(f'{proto} {prefix}port {operator} @P_{group_name}')
+
     output.append('counter')
 
     if 'log' in rule_conf:
diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py
index 7e3545c87..5ff768859 100644
--- a/python/vyos/opmode.py
+++ b/python/vyos/opmode.py
@@ -16,6 +16,7 @@
 import re
 import sys
 import typing
+from humps import decamelize
 
 
 class Error(Exception):
@@ -44,6 +45,25 @@ class PermissionDenied(Error):
     """
     pass
 
+class IncorrectValue(Error):
+    """ Requested operation is valid, but an argument provided has an
+        incorrect value, preventing successful completion.
+    """
+    pass
+
+class CommitInProgress(Error):
+    """ Requested operation is valid, but not possible at the time due
+    to a commit being in progress.
+    """
+    pass
+
+class InternalError(Error):
+    """ Any situation when VyOS detects that it could not perform
+        an operation correctly due to logic errors in its own code
+        or errors in underlying software.
+    """
+    pass
+
 
 def _is_op_mode_function_name(name):
     if re.match(r"^(show|clear|reset|restart)", name):
@@ -93,6 +113,51 @@ def _get_arg_type(t):
     else:
         return t
 
+def _normalize_field_name(name):
+    # Convert the name to string if it is not
+    # (in some cases they may be numbers)
+    name = str(name)
+
+    # Replace all separators with underscores
+    name = re.sub(r'(\s|[\(\)\[\]\{\}\-\.\,:\"\'\`])+', '_', name)
+
+    # Replace specific characters with textual descriptions
+    name = re.sub(r'@', '_at_', name)
+    name = re.sub(r'%', '_percentage_', name)
+    name = re.sub(r'~', '_tilde_', name)
+
+    # Force all letters to lowercase
+    name = name.lower()
+
+    # Remove leading and trailing underscores, if any
+    name = re.sub(r'(^(_+)(?=[^_])|_+$)', '', name)
+
+    # Ensure there are only single underscores
+    name = re.sub(r'_+', '_', name)
+
+    return name
+
+def _normalize_dict_field_names(old_dict):
+    new_dict = {}
+
+    for key in old_dict:
+        new_key = _normalize_field_name(key)
+        new_dict[new_key] = _normalize_field_names(old_dict[key])
+
+    # Sanity check
+    if len(old_dict) != len(new_dict):
+        raise InternalError("Dictionary fields do not allow unique normalization")
+    else:
+        return new_dict
+
+def _normalize_field_names(value):
+    if isinstance(value, dict):
+        return _normalize_dict_field_names(value)
+    elif isinstance(value, list):
+        return list(map(lambda v: _normalize_field_names(v), value))
+    else:
+        return value
+
 def run(module):
     from argparse import ArgumentParser
 
@@ -148,6 +213,8 @@ def run(module):
         if not args["raw"]:
             return res
         else:
+            res = decamelize(res)
+            res = _normalize_field_names(res)
             from json import dumps
             return dumps(res, indent=4)
     else:
diff --git a/python/vyos/systemversions.py b/python/vyos/systemversions.py
deleted file mode 100644
index f2da76d4f..000000000
--- a/python/vyos/systemversions.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this library.  If not, see <http://www.gnu.org/licenses/>.
-
-import os
-import re
-import sys
-import vyos.defaults
-from vyos.xml import component_version
-
-# legacy version, reading from the file names in
-# /opt/vyatta/etc/config-migrate/current
-def get_system_versions():
-    """
-    Get component versions from running system; critical failure if
-    unable to read migration directory.
-    """
-    system_versions = {}
-
-    try:
-        version_info = os.listdir(vyos.defaults.directories['current'])
-    except OSError as err:
-        print("OS error: {}".format(err))
-        sys.exit(1)
-
-    for info in version_info:
-        if re.match(r'[\w,-]+@\d+', info):
-            pair = info.split('@')
-            system_versions[pair[0]] = int(pair[1])
-
-    return system_versions
-
-# read from xml cache
-def get_system_component_version():
-    return component_version()
diff --git a/python/vyos/util.py b/python/vyos/util.py
index 461df9a6e..6a828c0ac 100644
--- a/python/vyos/util.py
+++ b/python/vyos/util.py
@@ -539,13 +539,16 @@ def seconds_to_human(s, separator=""):
 
     return result
 
-def bytes_to_human(bytes, initial_exponent=0):
+def bytes_to_human(bytes, initial_exponent=0, precision=2):
     """ Converts a value in bytes to a human-readable size string like 640 KB
 
     The initial_exponent parameter is the exponent of 2,
     e.g. 10 (1024) for kilobytes, 20 (1024 * 1024) for megabytes.
     """
 
+    if bytes == 0:
+        return "0 B"
+
     from math import log2
 
     bytes = bytes * (2**initial_exponent)
@@ -571,9 +574,40 @@ def bytes_to_human(bytes, initial_exponent=0):
     # Add a new case when the first machine with petabyte RAM
     # hits the market.
 
-    size_string = "{0:.2f} {1}".format(value, suffix)
+    size_string = "{0:.{1}f} {2}".format(value, precision, suffix)
     return size_string
 
+def human_to_bytes(value):
+    """ Converts a data amount with a unit suffix to bytes, like 2K to 2048 """
+
+    from re import match as re_match
+
+    res = re_match(r'^\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]+)\s*$', value)
+
+    if not res:
+        raise ValueError(f"'{value}' is not a valid data amount")
+    else:
+        amount = float(res.group(1))
+        unit = res.group(2).lower()
+
+        if unit == 'b':
+            res = amount
+        elif (unit == 'k') or (unit == 'kb'):
+            res = amount * 1024
+        elif (unit == 'm') or (unit == 'mb'):
+            res = amount * 1024**2
+        elif (unit == 'g') or (unit == 'gb'):
+            res = amount * 1024**3
+        elif (unit == 't') or (unit == 'tb'):
+            res = amount * 1024**4
+        else:
+            raise ValueError(f"Unsupported data unit '{unit}'")
+
+    # There cannot be fractional bytes, so we convert them to integer.
+    # However, truncating causes problems with conversion back to human unit,
+    # so we round instead -- that seems to work well enough.
+    return round(res)
+
 def get_cfg_group_id():
     from grp import getgrnam
     from vyos.defaults import cfg_group
@@ -1105,3 +1139,18 @@ def sysctl_write(name, value):
         call(f'sysctl -wq {name}={value}')
         return True
     return False
+
+# approach follows a discussion in:
+# https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case
+def camel_to_snake_case(name: str) -> str:
+    pattern = r'\d+|[A-Z]?[a-z]+|\W|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)'
+    words = re.findall(pattern, name)
+    return '_'.join(map(str.lower, words))
+
+def load_as_module(name: str, path: str):
+    import importlib.util
+
+    spec = importlib.util.spec_from_file_location(name, path)
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod