T4758: Fix conflicts op-mode-standardized

7 files changed, 296 insertions, 231 deletions

diff --git a/python/vyos/component_version.py b/python/vyos/component_version.py
new file mode 100644
index 000000000..a4e318d08
--- /dev/null
+++ b/python/vyos/component_version.py
@@ -0,0 +1,192 @@
+# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Functions for reading/writing component versions.
+
+The config file version string has the following form:
+
+VyOS 1.3/1.4:
+
+// Warning: Do not remove the following line.
+// vyos-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack@3:conntrack-sync@2:dhcp-relay@2:dhcp-server@6:dhcpv6-server@1:dns-forwarding@3:firewall@5:https@2:interfaces@22:ipoe-server@1:ipsec@5:isis@1:l2tp@3:lldp@1:mdns@1:nat@5:ntp@1:pppoe-server@5:pptp@2:qos@1:quagga@8:rpki@1:salt@1:snmp@2:ssh@2:sstp@3:system@21:vrrp@2:vyos-accel-ppp@2:wanloadbalance@3:webproxy@2:zone-policy@1"
+// Release version: 1.3.0
+
+VyOS 1.2:
+
+/* Warning: Do not remove the following line. */
+/* === vyatta-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack-sync@1:conntrack@1:dhcp-relay@2:dhcp-server@5:dns-forwarding@1:firewall@5:ipsec@5:l2tp@1:mdns@1:nat@4:ntp@1:pppoe-server@2:pptp@1:qos@1:quagga@7:snmp@1:ssh@1:system@10:vrrp@2:wanloadbalance@3:webgui@1:webproxy@2:zone-policy@1" === */
+/* Release version: 1.2.8 */
+
+"""
+
+import os
+import re
+import sys
+import fileinput
+
+from vyos.xml import component_version
+from vyos.version import get_version
+from vyos.defaults import directories
+
+DEFAULT_CONFIG_PATH = os.path.join(directories['config'], 'config.boot')
+
+def from_string(string_line, vintage='vyos'):
+    """
+    Get component version dictionary from string.
+    Return empty dictionary if string contains no config information
+    or raise error if component version string malformed.
+    """
+    version_dict = {}
+
+    if vintage == 'vyos':
+        if re.match(r'// vyos-config-version:.+', string_line):
+            if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', string_line):
+                raise ValueError(f"malformed configuration string: {string_line}")
+
+            for pair in re.findall(r'([\w,-]+)@(\d+)', string_line):
+                version_dict[pair[0]] = int(pair[1])
+
+    elif vintage == 'vyatta':
+        if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line):
+            if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line):
+                raise ValueError(f"malformed configuration string: {string_line}")
+
+            for pair in re.findall(r'([\w,-]+)@(\d+)', string_line):
+                version_dict[pair[0]] = int(pair[1])
+    else:
+        raise ValueError("Unknown config string vintage")
+
+    return version_dict
+
+def from_file(config_file_name=DEFAULT_CONFIG_PATH, vintage='vyos'):
+    """
+    Get component version dictionary parsing config file line by line
+    """
+    with open(config_file_name, 'r') as f:
+        for line_in_config in f:
+            version_dict = from_string(line_in_config, vintage=vintage)
+            if version_dict:
+                return version_dict
+
+    # no version information
+    return {}
+
+def from_system():
+    """
+    Get system component version dict.
+    """
+    return component_version()
+
+def legacy_from_system():
+    """
+    Get system component version dict from legacy location.
+    This is for a transitional sanity check; the directory will eventually
+    be removed.
+    """
+    system_versions = {}
+    legacy_dir = directories['current']
+
+    # To be removed:
+    if not os.path.isdir(legacy_dir):
+        return system_versions
+
+    try:
+        version_info = os.listdir(legacy_dir)
+    except OSError as err:
+        sys.exit(repr(err))
+
+    for info in version_info:
+        if re.match(r'[\w,-]+@\d+', info):
+            pair = info.split('@')
+            system_versions[pair[0]] = int(pair[1])
+
+    return system_versions
+
+def format_string(ver: dict) -> str:
+    """
+    Version dict to string.
+    """
+    keys = list(ver)
+    keys.sort()
+    l = []
+    for k in keys:
+        v = ver[k]
+        l.append(f'{k}@{v}')
+    sep = ':'
+    return sep.join(l)
+
+def version_footer(ver: dict, vintage='vyos') -> str:
+    """
+    Version footer as string.
+    """
+    ver_str = format_string(ver)
+    release = get_version()
+    if vintage == 'vyos':
+        ret_str = (f'// Warning: Do not remove the following line.\n'
+                +  f'// vyos-config-version: "{ver_str}"\n'
+                +  f'// Release version: {release}\n')
+    elif vintage == 'vyatta':
+        ret_str = (f'/* Warning: Do not remove the following line. */\n'
+                +  f'/* === vyatta-config-version: "{ver_str}" === */\n'
+                +  f'/* Release version: {release} */\n')
+    else:
+        raise ValueError("Unknown config string vintage")
+
+    return ret_str
+
+def system_footer(vintage='vyos') -> str:
+    """
+    System version footer as string.
+    """
+    ver_d = from_system()
+    return version_footer(ver_d, vintage=vintage)
+
+def write_version_footer(ver: dict, file_name, vintage='vyos'):
+    """
+    Write version footer to file.
+    """
+    footer = version_footer(ver=ver, vintage=vintage)
+    if file_name:
+        with open(file_name, 'a') as f:
+            f.write(footer)
+    else:
+        sys.stdout.write(footer)
+
+def write_system_footer(file_name, vintage='vyos'):
+    """
+    Write system version footer to file.
+    """
+    ver_d = from_system()
+    return write_version_footer(ver_d, file_name=file_name, vintage=vintage)
+
+def remove_footer(file_name):
+    """
+    Remove old version footer.
+    """
+    for line in fileinput.input(file_name, inplace=True):
+        if re.match(r'/\* Warning:.+ \*/$', line):
+            continue
+        if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line):
+            continue
+        if re.match(r'/\* Release version:.+ \*/$', line):
+            continue
+        if re.match('// vyos-config-version:.+', line):
+            continue
+        if re.match('// Warning:.+', line):
+            continue
+        if re.match('// Release version:.+', line):
+            continue
+        sys.stdout.write(line)
diff --git a/python/vyos/component_versions.py b/python/vyos/component_versions.py
deleted file mode 100644
index 90b458aae..000000000
--- a/python/vyos/component_versions.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# Copyright 2017 VyOS maintainers and contributors <maintainers@vyos.io>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library.  If not, see <http://www.gnu.org/licenses/>.
-
-"""
-The version data looks like:
-
-/* Warning: Do not remove the following line. */
-/* === vyatta-config-version:
-"cluster@1:config-management@1:conntrack-sync@1:conntrack@1:dhcp-relay@1:dhcp-server@4:firewall@5:ipsec@4:nat@4:qos@1:quagga@2:system@8:vrrp@1:wanloadbalance@3:webgui@1:webproxy@1:zone-policy@1"
-=== */
-/* Release version: 1.2.0-rolling+201806131737 */
-"""
-
-import re
-
-def get_component_version(string_line):
-    """
-    Get component version dictionary from string
-    return empty dictionary if string contains no config information
-    or raise error if component version string malformed
-    """
-    return_value = {}
-    if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line):
-
-        if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line):
-            raise ValueError("malformed configuration string: " + str(string_line))
-
-        for pair in re.findall(r'([\w,-]+)@(\d+)', string_line):
-            if pair[0] in return_value.keys():
-                raise ValueError("duplicate unit name: \"" + str(pair[0]) + "\" in string: \"" + string_line + "\"")
-            return_value[pair[0]] = int(pair[1])
-
-    return return_value
-
-
-def get_component_versions_from_file(config_file_name='/opt/vyatta/etc/config/config.boot'):
-    """
-    Get component version dictionary parsing config file line by line
-    """
-    f = open(config_file_name, 'r')
-    for line_in_config in f:
-        component_version = get_component_version(line_in_config)
-        if component_version:
-            return component_version
-    raise ValueError("no config string in file:", config_file_name)
diff --git a/python/vyos/formatversions.py b/python/vyos/formatversions.py
deleted file mode 100644
index 29117a5d3..000000000
--- a/python/vyos/formatversions.py
+++ /dev/null
@@ -1,109 +0,0 @@
-# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this library.  If not, see <http://www.gnu.org/licenses/>.
-
-import sys
-import os
-import re
-import fileinput
-
-def read_vyatta_versions(config_file):
-    config_file_versions = {}
-
-    with open(config_file, 'r') as config_file_handle:
-        for config_line in config_file_handle:
-            if re.match(r'/\* === vyatta-config-version:.+=== \*/$', config_line):
-                if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', config_line):
-                    raise ValueError("malformed configuration string: "
-                            "{}".format(config_line))
-
-                for pair in re.findall(r'([\w,-]+)@(\d+)', config_line):
-                    config_file_versions[pair[0]] = int(pair[1])
-
-
-    return config_file_versions
-
-def read_vyos_versions(config_file):
-    config_file_versions = {}
-
-    with open(config_file, 'r') as config_file_handle:
-        for config_line in config_file_handle:
-            if re.match(r'// vyos-config-version:.+', config_line):
-                if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', config_line):
-                    raise ValueError("malformed configuration string: "
-                            "{}".format(config_line))
-
-                for pair in re.findall(r'([\w,-]+)@(\d+)', config_line):
-                    config_file_versions[pair[0]] = int(pair[1])
-
-    return config_file_versions
-
-def remove_versions(config_file):
-    """
-    Remove old version string.
-    """
-    for line in fileinput.input(config_file, inplace=True):
-        if re.match(r'/\* Warning:.+ \*/$', line):
-            continue
-        if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line):
-            continue
-        if re.match(r'/\* Release version:.+ \*/$', line):
-            continue
-        if re.match('// vyos-config-version:.+', line):
-            continue
-        if re.match('// Warning:.+', line):
-            continue
-        if re.match('// Release version:.+', line):
-            continue
-        sys.stdout.write(line)
-
-def format_versions_string(config_versions):
-    cfg_keys = list(config_versions.keys())
-    cfg_keys.sort()
-
-    component_version_strings = []
-
-    for key in cfg_keys:
-        cfg_vers = config_versions[key]
-        component_version_strings.append('{}@{}'.format(key, cfg_vers))
-
-    separator = ":"
-    component_version_string = separator.join(component_version_strings)
-
-    return component_version_string
-
-def write_vyatta_versions_foot(config_file, component_version_string,
-                                 os_version_string):
-    if config_file:
-        with open(config_file, 'a') as config_file_handle:
-            config_file_handle.write('/* Warning: Do not remove the following line. */\n')
-            config_file_handle.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string))
-            config_file_handle.write('/* Release version: {} */\n'.format(os_version_string))
-    else:
-        sys.stdout.write('/* Warning: Do not remove the following line. */\n')
-        sys.stdout.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string))
-        sys.stdout.write('/* Release version: {} */\n'.format(os_version_string))
-
-def write_vyos_versions_foot(config_file, component_version_string,
-                               os_version_string):
-    if config_file:
-        with open(config_file, 'a') as config_file_handle:
-            config_file_handle.write('// Warning: Do not remove the following line.\n')
-            config_file_handle.write('// vyos-config-version: "{}"\n'.format(component_version_string))
-            config_file_handle.write('// Release version: {}\n'.format(os_version_string))
-    else:
-        sys.stdout.write('// Warning: Do not remove the following line.\n')
-        sys.stdout.write('// vyos-config-version: "{}"\n'.format(component_version_string))
-        sys.stdout.write('// Release version: {}\n'.format(os_version_string))
-
diff --git a/python/vyos/migrator.py b/python/vyos/migrator.py
index c6e3435ca..45ea8b0eb 100644
--- a/python/vyos/migrator.py
+++ b/python/vyos/migrator.py
@@ -1,4 +1,4 @@
-# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
+# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,10 +17,8 @@ import sys
 import os
 import json
 import subprocess
-import vyos.version
 import vyos.defaults
-import vyos.systemversions as systemversions
-import vyos.formatversions as formatversions
+import vyos.component_version as component_version
 
 class MigratorError(Exception):
     pass
@@ -42,13 +40,13 @@ class Migrator(object):
         cfg_file = self._config_file
         component_versions = {}
 
-        cfg_versions = formatversions.read_vyatta_versions(cfg_file)
+        cfg_versions = component_version.from_file(cfg_file, vintage='vyatta')
 
         if cfg_versions:
             self._config_file_vintage = 'vyatta'
             component_versions = cfg_versions
 
-        cfg_versions = formatversions.read_vyos_versions(cfg_file)
+        cfg_versions = component_version.from_file(cfg_file, vintage='vyos')
 
         if cfg_versions:
             self._config_file_vintage = 'vyos'
@@ -157,19 +155,15 @@ class Migrator(object):
         """
         Write new versions string.
         """
-        versions_string = formatversions.format_versions_string(cfg_versions)
-
-        os_version_string = vyos.version.get_version()
-
         if self._config_file_vintage == 'vyatta':
-            formatversions.write_vyatta_versions_foot(self._config_file,
-                                                      versions_string,
-                                                      os_version_string)
+            component_version.write_version_footer(cfg_versions,
+                                                   self._config_file,
+                                                   vintage='vyatta')
 
         if self._config_file_vintage == 'vyos':
-            formatversions.write_vyos_versions_foot(self._config_file,
-                                                    versions_string,
-                                                    os_version_string)
+            component_version.write_version_footer(cfg_versions,
+                                                   self._config_file,
+                                                   vintage='vyos')
 
     def save_json_record(self, component_versions: dict):
         """
@@ -200,7 +194,7 @@ class Migrator(object):
             # This will force calling all migration scripts:
             cfg_versions = {}
 
-        sys_versions = systemversions.get_system_component_version()
+        sys_versions = component_version.from_system()
 
         # save system component versions in json file for easy reference
         self.save_json_record(sys_versions)
@@ -216,7 +210,7 @@ class Migrator(object):
         if not self._changed:
             return
 
-        formatversions.remove_versions(cfg_file)
+        component_version.remove_footer(cfg_file)
 
         self.write_config_file_versions(rev_versions)
 
@@ -237,7 +231,7 @@ class VirtualMigrator(Migrator):
         if not self._changed:
             return
 
-        formatversions.remove_versions(cfg_file)
+        component_version.remove_footer(cfg_file)
 
         self.write_config_file_versions(cfg_versions)
 
diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py
index 7e3545c87..727e118a8 100644
--- a/python/vyos/opmode.py
+++ b/python/vyos/opmode.py
@@ -44,6 +44,13 @@ class PermissionDenied(Error):
     """
     pass
 
+class InternalError(Error):
+    """ Any situation when VyOS detects that it could not perform
+        an operation correctly due to logic errors in its own code
+        or errors in underlying software.
+    """
+    pass
+
 
 def _is_op_mode_function_name(name):
     if re.match(r"^(show|clear|reset|restart)", name):
@@ -93,6 +100,51 @@ def _get_arg_type(t):
     else:
         return t
 
+def _normalize_field_name(name):
+    # Convert the name to string if it is not
+    # (in some cases they may be numbers)
+    name = str(name)
+
+    # Replace all separators with underscores
+    name = re.sub(r'(\s|[\(\)\[\]\{\}\-\.\,:\"\'\`])+', '_', name)
+
+    # Replace specific characters with textual descriptions
+    name = re.sub(r'@', '_at_', name)
+    name = re.sub(r'%', '_percentage_', name)
+    name = re.sub(r'~', '_tilde_', name)
+
+    # Force all letters to lowercase
+    name = name.lower()
+
+    # Remove leading and trailing underscores, if any
+    name = re.sub(r'(^(_+)(?=[^_])|_+$)', '', name)
+
+    # Ensure there are only single underscores
+    name = re.sub(r'_+', '_', name)
+
+    return name
+
+def _normalize_dict_field_names(old_dict):
+    new_dict = {}
+
+    for key in old_dict:
+        new_key = _normalize_field_name(key)
+        new_dict[new_key] = _normalize_field_names(old_dict[key])
+
+    # Sanity check
+    if len(old_dict) != len(new_dict):
+        raise InternalError("Dictionary fields do not allow unique normalization")
+    else:
+        return new_dict
+
+def _normalize_field_names(value):
+    if isinstance(value, dict):
+        return _normalize_dict_field_names(value)
+    elif isinstance(value, list):
+        return list(map(lambda v: _normalize_field_names(v), value))
+    else:
+        return value
+
 def run(module):
     from argparse import ArgumentParser
 
@@ -148,6 +200,7 @@ def run(module):
         if not args["raw"]:
             return res
         else:
+            res = _normalize_field_names(res)
             from json import dumps
             return dumps(res, indent=4)
     else:
diff --git a/python/vyos/systemversions.py b/python/vyos/systemversions.py
deleted file mode 100644
index f2da76d4f..000000000
--- a/python/vyos/systemversions.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this library.  If not, see <http://www.gnu.org/licenses/>.
-
-import os
-import re
-import sys
-import vyos.defaults
-from vyos.xml import component_version
-
-# legacy version, reading from the file names in
-# /opt/vyatta/etc/config-migrate/current
-def get_system_versions():
-    """
-    Get component versions from running system; critical failure if
-    unable to read migration directory.
-    """
-    system_versions = {}
-
-    try:
-        version_info = os.listdir(vyos.defaults.directories['current'])
-    except OSError as err:
-        print("OS error: {}".format(err))
-        sys.exit(1)
-
-    for info in version_info:
-        if re.match(r'[\w,-]+@\d+', info):
-            pair = info.split('@')
-            system_versions[pair[0]] = int(pair[1])
-
-    return system_versions
-
-# read from xml cache
-def get_system_component_version():
-    return component_version()
diff --git a/python/vyos/util.py b/python/vyos/util.py
index 461df9a6e..a80584c5a 100644
--- a/python/vyos/util.py
+++ b/python/vyos/util.py
@@ -574,6 +574,37 @@ def bytes_to_human(bytes, initial_exponent=0):
     size_string = "{0:.2f} {1}".format(value, suffix)
     return size_string
 
+def human_to_bytes(value):
+    """ Converts a data amount with a unit suffix to bytes, like 2K to 2048 """
+
+    from re import match as re_match
+
+    res = re_match(r'^\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]+)\s*$', value)
+
+    if not res:
+        raise ValueError(f"'{value}' is not a valid data amount")
+    else:
+        amount = float(res.group(1))
+        unit = res.group(2).lower()
+
+        if unit == 'b':
+            res = amount
+        elif (unit == 'k') or (unit == 'kb'):
+            res = amount * 1024
+        elif (unit == 'm') or (unit == 'mb'):
+            res = amount * 1024**2
+        elif (unit == 'g') or (unit == 'gb'):
+            res = amount * 1024**3
+        elif (unit == 't') or (unit == 'tb'):
+            res = amount * 1024**4
+        else:
+            raise ValueError(f"Unsupported data unit '{unit}'")
+
+    # There cannot be fractional bytes, so we convert them to integer.
+    # However, truncating causes problems with conversion back to human unit,
+    # so we round instead -- that seems to work well enough.
+    return round(res)
+
 def get_cfg_group_id():
     from grp import getgrnam
     from vyos.defaults import cfg_group
@@ -1105,3 +1136,10 @@ def sysctl_write(name, value):
         call(f'sysctl -wq {name}={value}')
         return True
     return False
+
+# approach follows a discussion in:
+# https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case
+def camel_to_snake_case(name: str) -> str:
+    pattern = r'\d+|[A-Z]?[a-z]+|\W|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)'
+    words = re.findall(pattern, name)
+    return '_'.join(map(str.lower, words))