From 36c475ec3524739f9ae49420e60a57a5266fa575 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Thu, 20 Oct 2022 09:14:53 -0400 Subject: T4765: normalize dict fields in op mode ouputs --- python/vyos/opmode.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) (limited to 'python') diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index 7e3545c87..ac9c0c353 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -44,6 +44,13 @@ class PermissionDenied(Error): """ pass +class InternalError(Error): + """ Any situation when VyOS detects that it could not perform + an operation correctly due to logic errors in its own code + or errors in underlying software. + """ + pass + def _is_op_mode_function_name(name): if re.match(r"^(show|clear|reset|restart)", name): @@ -93,6 +100,39 @@ def _get_arg_type(t): else: return t +def _normalize_field_name(name): + # Replace all separators with underscores + name = re.sub(r'(\s|[\(\)\[\]\{\}\-\.\,:\"\'\`])+', '_', name) + + # Replace specific characters with textual descriptions + name = re.sub(r'@', '_at_', name) + name = re.sub(r'%', '_percentage_', name) + name = re.sub(r'~', '_tilde_', name) + + # Force all letters to lowercase + name = name.lower() + + # Remove leading and trailing underscores, if any + name = re.sub(r'(^(_+)(?=[^_])|_+$)', '', name) + + # Ensure there are only single underscores + name = re.sub(r'_+', '_', name) + + return name + +def _normalize_field_names(old_dict): + new_dict = {} + + for key in old_dict: + new_key = _normalize_field_name(key) + new_dict[new_key] = old_dict[key] + + # Sanity check + if len(old_dict) != len(new_dict): + raise InternalError("Dictionary fields do not allow unique normalization") + else: + return new_dict + def run(module): from argparse import ArgumentParser @@ -145,6 +185,7 @@ def run(module): # they may return human-formatted output # or a raw dict that we need to serialize in JSON for printing res = func(**args) + res = _normalize_field_names(res) if not args["raw"]: return res else: -- cgit v1.2.3 From 40cf5f7c1b8d8ae27b5c404807294f8a2a76842d Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 20 Oct 2022 16:25:51 -0500 Subject: T4765: normalize fields only if 'raw' is true; output must be dict --- python/vyos/opmode.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'python') diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index ac9c0c353..5f9c2c1ce 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -185,10 +185,12 @@ def run(module): # they may return human-formatted output # or a raw dict that we need to serialize in JSON for printing res = func(**args) - res = _normalize_field_names(res) if not args["raw"]: return res else: + if not isinstance(res, dict): + raise InternalError("'raw' output of 'show_*' command must be a dict") + res = _normalize_field_names(res) from json import dumps return dumps(res, indent=4) else: -- cgit v1.2.3 From b6d2e0a4b08c81814cb2d9b5b611cbc3fc31dbeb Mon Sep 17 00:00:00 2001 From: create with ansible Date: Fri, 21 Oct 2022 11:50:17 -0400 Subject: T4765: support list and primitives in op mode output normalization --- python/vyos/opmode.py | 14 ++++++++++---- src/tests/test_op_mode.py | 25 +++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 6 deletions(-) (limited to 'python') diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index 5f9c2c1ce..c9827d634 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -120,12 +120,12 @@ def _normalize_field_name(name): return name -def _normalize_field_names(old_dict): +def _normalize_dict_field_names(old_dict): new_dict = {} for key in old_dict: new_key = _normalize_field_name(key) - new_dict[new_key] = old_dict[key] + new_dict[new_key] = _normalize_field_names(old_dict[key]) # Sanity check if len(old_dict) != len(new_dict): @@ -133,6 +133,14 @@ def _normalize_field_names(old_dict): else: return new_dict +def _normalize_field_names(value): + if isinstance(value, dict): + return _normalize_dict_field_names(value) + elif isinstance(value, list): + return list(map(lambda v: _normalize_field_names(v), value)) + else: + return value + def run(module): from argparse import ArgumentParser @@ -188,8 +196,6 @@ def run(module): if not args["raw"]: return res else: - if not isinstance(res, dict): - raise InternalError("'raw' output of 'show_*' command must be a dict") res = _normalize_field_names(res) from json import dumps return dumps(res, indent=4) diff --git a/src/tests/test_op_mode.py b/src/tests/test_op_mode.py index 4786357c5..90963b3c5 100644 --- a/src/tests/test_op_mode.py +++ b/src/tests/test_op_mode.py @@ -37,8 +37,29 @@ class TestVyOSOpMode(TestCase): with self.assertRaises(vyos.opmode.InternalError): _normalize_field_names(data) - def test_dict_fields_normalization(self): + def test_dict_fields_normalization_simple_dict(self): from vyos.opmode import _normalize_field_names - data = {"foo bar": True, "bar-baz": False} + data = {"foo bar": True, "Bar-Baz": False} self.assertEqual(_normalize_field_names(data), {"foo_bar": True, "bar_baz": False}) + + def test_dict_fields_normalization_nested_dict(self): + from vyos.opmode import _normalize_field_names + + data = {"foo bar": True, "bar-baz": {"baz-quux": {"quux-xyzzy": False}}} + self.assertEqual(_normalize_field_names(data), + {"foo_bar": True, "bar_baz": {"baz_quux": {"quux_xyzzy": False}}}) + + def test_dict_fields_normalization_mixed(self): + from vyos.opmode import _normalize_field_names + + data = [{"foo bar": True, "bar-baz": [{"baz-quux": {"quux-xyzzy": [False]}}]}] + self.assertEqual(_normalize_field_names(data), + [{"foo_bar": True, "bar_baz": [{"baz_quux": {"quux_xyzzy": [False]}}]}]) + + def test_dict_fields_normalization_primitive(self): + from vyos.opmode import _normalize_field_names + + data = [1, False, "foo"] + self.assertEqual(_normalize_field_names(data), [1, False, "foo"]) + -- cgit v1.2.3 From 29b656f14d8bb7622ff861208d26cf8eb018670d Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 25 Oct 2022 13:42:30 -0500 Subject: vyos.util: T4773: add camel_to_snake_case conversion --- python/vyos/util.py | 7 +++++++ src/tests/test_util.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+) (limited to 'python') diff --git a/python/vyos/util.py b/python/vyos/util.py index 461df9a6e..e4e2a44ec 100644 --- a/python/vyos/util.py +++ b/python/vyos/util.py @@ -1105,3 +1105,10 @@ def sysctl_write(name, value): call(f'sysctl -wq {name}={value}') return True return False + +# approach follows a discussion in: +# https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case +def camel_to_snake_case(name: str) -> str: + pattern = r'\d+|[A-Z]?[a-z]+|\W|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)' + words = re.findall(pattern, name) + return '_'.join(map(str.lower, words)) diff --git a/src/tests/test_util.py b/src/tests/test_util.py index 8ac9a500a..d8b2b7940 100644 --- a/src/tests/test_util.py +++ b/src/tests/test_util.py @@ -26,3 +26,17 @@ class TestVyOSUtil(TestCase): def test_sysctl_read(self): self.assertEqual(sysctl_read('net.ipv4.conf.lo.forwarding'), '1') + + def test_camel_to_snake_case(self): + self.assertEqual(camel_to_snake_case('ConnectionTimeout'), + 'connection_timeout') + self.assertEqual(camel_to_snake_case('connectionTimeout'), + 'connection_timeout') + self.assertEqual(camel_to_snake_case('TCPConnectionTimeout'), + 'tcp_connection_timeout') + self.assertEqual(camel_to_snake_case('TCPPort'), + 'tcp_port') + self.assertEqual(camel_to_snake_case('UseHTTPProxy'), + 'use_http_proxy') + self.assertEqual(camel_to_snake_case('CustomerID'), + 'customer_id') -- cgit v1.2.3 From 0e63712195465c9bf0bf55c369b86961d54dfaac Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 27 Oct 2022 12:24:44 -0500 Subject: T4291: consolidate component version string read/write functions --- python/vyos/component_version.py | 192 ++++++++++++++++++++++++ python/vyos/component_versions.py | 57 ------- python/vyos/formatversions.py | 109 -------------- python/vyos/migrator.py | 32 ++-- python/vyos/systemversions.py | 46 ------ smoketest/scripts/cli/test_component_version.py | 6 +- src/helpers/system-versions-foot.py | 21 +-- 7 files changed, 213 insertions(+), 250 deletions(-) create mode 100644 python/vyos/component_version.py delete mode 100644 python/vyos/component_versions.py delete mode 100644 python/vyos/formatversions.py delete mode 100644 python/vyos/systemversions.py (limited to 'python') diff --git a/python/vyos/component_version.py b/python/vyos/component_version.py new file mode 100644 index 000000000..a4e318d08 --- /dev/null +++ b/python/vyos/component_version.py @@ -0,0 +1,192 @@ +# Copyright 2022 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . + +""" +Functions for reading/writing component versions. + +The config file version string has the following form: + +VyOS 1.3/1.4: + +// Warning: Do not remove the following line. +// vyos-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack@3:conntrack-sync@2:dhcp-relay@2:dhcp-server@6:dhcpv6-server@1:dns-forwarding@3:firewall@5:https@2:interfaces@22:ipoe-server@1:ipsec@5:isis@1:l2tp@3:lldp@1:mdns@1:nat@5:ntp@1:pppoe-server@5:pptp@2:qos@1:quagga@8:rpki@1:salt@1:snmp@2:ssh@2:sstp@3:system@21:vrrp@2:vyos-accel-ppp@2:wanloadbalance@3:webproxy@2:zone-policy@1" +// Release version: 1.3.0 + +VyOS 1.2: + +/* Warning: Do not remove the following line. */ +/* === vyatta-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack-sync@1:conntrack@1:dhcp-relay@2:dhcp-server@5:dns-forwarding@1:firewall@5:ipsec@5:l2tp@1:mdns@1:nat@4:ntp@1:pppoe-server@2:pptp@1:qos@1:quagga@7:snmp@1:ssh@1:system@10:vrrp@2:wanloadbalance@3:webgui@1:webproxy@2:zone-policy@1" === */ +/* Release version: 1.2.8 */ + +""" + +import os +import re +import sys +import fileinput + +from vyos.xml import component_version +from vyos.version import get_version +from vyos.defaults import directories + +DEFAULT_CONFIG_PATH = os.path.join(directories['config'], 'config.boot') + +def from_string(string_line, vintage='vyos'): + """ + Get component version dictionary from string. + Return empty dictionary if string contains no config information + or raise error if component version string malformed. + """ + version_dict = {} + + if vintage == 'vyos': + if re.match(r'// vyos-config-version:.+', string_line): + if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', string_line): + raise ValueError(f"malformed configuration string: {string_line}") + + for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): + version_dict[pair[0]] = int(pair[1]) + + elif vintage == 'vyatta': + if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line): + if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line): + raise ValueError(f"malformed configuration string: {string_line}") + + for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): + version_dict[pair[0]] = int(pair[1]) + else: + raise ValueError("Unknown config string vintage") + + return version_dict + +def from_file(config_file_name=DEFAULT_CONFIG_PATH, vintage='vyos'): + """ + Get component version dictionary parsing config file line by line + """ + with open(config_file_name, 'r') as f: + for line_in_config in f: + version_dict = from_string(line_in_config, vintage=vintage) + if version_dict: + return version_dict + + # no version information + return {} + +def from_system(): + """ + Get system component version dict. + """ + return component_version() + +def legacy_from_system(): + """ + Get system component version dict from legacy location. + This is for a transitional sanity check; the directory will eventually + be removed. + """ + system_versions = {} + legacy_dir = directories['current'] + + # To be removed: + if not os.path.isdir(legacy_dir): + return system_versions + + try: + version_info = os.listdir(legacy_dir) + except OSError as err: + sys.exit(repr(err)) + + for info in version_info: + if re.match(r'[\w,-]+@\d+', info): + pair = info.split('@') + system_versions[pair[0]] = int(pair[1]) + + return system_versions + +def format_string(ver: dict) -> str: + """ + Version dict to string. + """ + keys = list(ver) + keys.sort() + l = [] + for k in keys: + v = ver[k] + l.append(f'{k}@{v}') + sep = ':' + return sep.join(l) + +def version_footer(ver: dict, vintage='vyos') -> str: + """ + Version footer as string. + """ + ver_str = format_string(ver) + release = get_version() + if vintage == 'vyos': + ret_str = (f'// Warning: Do not remove the following line.\n' + + f'// vyos-config-version: "{ver_str}"\n' + + f'// Release version: {release}\n') + elif vintage == 'vyatta': + ret_str = (f'/* Warning: Do not remove the following line. */\n' + + f'/* === vyatta-config-version: "{ver_str}" === */\n' + + f'/* Release version: {release} */\n') + else: + raise ValueError("Unknown config string vintage") + + return ret_str + +def system_footer(vintage='vyos') -> str: + """ + System version footer as string. + """ + ver_d = from_system() + return version_footer(ver_d, vintage=vintage) + +def write_version_footer(ver: dict, file_name, vintage='vyos'): + """ + Write version footer to file. + """ + footer = version_footer(ver=ver, vintage=vintage) + if file_name: + with open(file_name, 'a') as f: + f.write(footer) + else: + sys.stdout.write(footer) + +def write_system_footer(file_name, vintage='vyos'): + """ + Write system version footer to file. + """ + ver_d = from_system() + return write_version_footer(ver_d, file_name=file_name, vintage=vintage) + +def remove_footer(file_name): + """ + Remove old version footer. + """ + for line in fileinput.input(file_name, inplace=True): + if re.match(r'/\* Warning:.+ \*/$', line): + continue + if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line): + continue + if re.match(r'/\* Release version:.+ \*/$', line): + continue + if re.match('// vyos-config-version:.+', line): + continue + if re.match('// Warning:.+', line): + continue + if re.match('// Release version:.+', line): + continue + sys.stdout.write(line) diff --git a/python/vyos/component_versions.py b/python/vyos/component_versions.py deleted file mode 100644 index 90b458aae..000000000 --- a/python/vyos/component_versions.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2017 VyOS maintainers and contributors -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see . - -""" -The version data looks like: - -/* Warning: Do not remove the following line. */ -/* === vyatta-config-version: -"cluster@1:config-management@1:conntrack-sync@1:conntrack@1:dhcp-relay@1:dhcp-server@4:firewall@5:ipsec@4:nat@4:qos@1:quagga@2:system@8:vrrp@1:wanloadbalance@3:webgui@1:webproxy@1:zone-policy@1" -=== */ -/* Release version: 1.2.0-rolling+201806131737 */ -""" - -import re - -def get_component_version(string_line): - """ - Get component version dictionary from string - return empty dictionary if string contains no config information - or raise error if component version string malformed - """ - return_value = {} - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line): - - if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line): - raise ValueError("malformed configuration string: " + str(string_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): - if pair[0] in return_value.keys(): - raise ValueError("duplicate unit name: \"" + str(pair[0]) + "\" in string: \"" + string_line + "\"") - return_value[pair[0]] = int(pair[1]) - - return return_value - - -def get_component_versions_from_file(config_file_name='/opt/vyatta/etc/config/config.boot'): - """ - Get component version dictionary parsing config file line by line - """ - f = open(config_file_name, 'r') - for line_in_config in f: - component_version = get_component_version(line_in_config) - if component_version: - return component_version - raise ValueError("no config string in file:", config_file_name) diff --git a/python/vyos/formatversions.py b/python/vyos/formatversions.py deleted file mode 100644 index 29117a5d3..000000000 --- a/python/vyos/formatversions.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2019 VyOS maintainers and contributors -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see . - -import sys -import os -import re -import fileinput - -def read_vyatta_versions(config_file): - config_file_versions = {} - - with open(config_file, 'r') as config_file_handle: - for config_line in config_file_handle: - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', config_line): - if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', config_line): - raise ValueError("malformed configuration string: " - "{}".format(config_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', config_line): - config_file_versions[pair[0]] = int(pair[1]) - - - return config_file_versions - -def read_vyos_versions(config_file): - config_file_versions = {} - - with open(config_file, 'r') as config_file_handle: - for config_line in config_file_handle: - if re.match(r'// vyos-config-version:.+', config_line): - if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', config_line): - raise ValueError("malformed configuration string: " - "{}".format(config_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', config_line): - config_file_versions[pair[0]] = int(pair[1]) - - return config_file_versions - -def remove_versions(config_file): - """ - Remove old version string. - """ - for line in fileinput.input(config_file, inplace=True): - if re.match(r'/\* Warning:.+ \*/$', line): - continue - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line): - continue - if re.match(r'/\* Release version:.+ \*/$', line): - continue - if re.match('// vyos-config-version:.+', line): - continue - if re.match('// Warning:.+', line): - continue - if re.match('// Release version:.+', line): - continue - sys.stdout.write(line) - -def format_versions_string(config_versions): - cfg_keys = list(config_versions.keys()) - cfg_keys.sort() - - component_version_strings = [] - - for key in cfg_keys: - cfg_vers = config_versions[key] - component_version_strings.append('{}@{}'.format(key, cfg_vers)) - - separator = ":" - component_version_string = separator.join(component_version_strings) - - return component_version_string - -def write_vyatta_versions_foot(config_file, component_version_string, - os_version_string): - if config_file: - with open(config_file, 'a') as config_file_handle: - config_file_handle.write('/* Warning: Do not remove the following line. */\n') - config_file_handle.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string)) - config_file_handle.write('/* Release version: {} */\n'.format(os_version_string)) - else: - sys.stdout.write('/* Warning: Do not remove the following line. */\n') - sys.stdout.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string)) - sys.stdout.write('/* Release version: {} */\n'.format(os_version_string)) - -def write_vyos_versions_foot(config_file, component_version_string, - os_version_string): - if config_file: - with open(config_file, 'a') as config_file_handle: - config_file_handle.write('// Warning: Do not remove the following line.\n') - config_file_handle.write('// vyos-config-version: "{}"\n'.format(component_version_string)) - config_file_handle.write('// Release version: {}\n'.format(os_version_string)) - else: - sys.stdout.write('// Warning: Do not remove the following line.\n') - sys.stdout.write('// vyos-config-version: "{}"\n'.format(component_version_string)) - sys.stdout.write('// Release version: {}\n'.format(os_version_string)) - diff --git a/python/vyos/migrator.py b/python/vyos/migrator.py index c6e3435ca..45ea8b0eb 100644 --- a/python/vyos/migrator.py +++ b/python/vyos/migrator.py @@ -1,4 +1,4 @@ -# Copyright 2019 VyOS maintainers and contributors +# Copyright 2019-2022 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -17,10 +17,8 @@ import sys import os import json import subprocess -import vyos.version import vyos.defaults -import vyos.systemversions as systemversions -import vyos.formatversions as formatversions +import vyos.component_version as component_version class MigratorError(Exception): pass @@ -42,13 +40,13 @@ class Migrator(object): cfg_file = self._config_file component_versions = {} - cfg_versions = formatversions.read_vyatta_versions(cfg_file) + cfg_versions = component_version.from_file(cfg_file, vintage='vyatta') if cfg_versions: self._config_file_vintage = 'vyatta' component_versions = cfg_versions - cfg_versions = formatversions.read_vyos_versions(cfg_file) + cfg_versions = component_version.from_file(cfg_file, vintage='vyos') if cfg_versions: self._config_file_vintage = 'vyos' @@ -157,19 +155,15 @@ class Migrator(object): """ Write new versions string. """ - versions_string = formatversions.format_versions_string(cfg_versions) - - os_version_string = vyos.version.get_version() - if self._config_file_vintage == 'vyatta': - formatversions.write_vyatta_versions_foot(self._config_file, - versions_string, - os_version_string) + component_version.write_version_footer(cfg_versions, + self._config_file, + vintage='vyatta') if self._config_file_vintage == 'vyos': - formatversions.write_vyos_versions_foot(self._config_file, - versions_string, - os_version_string) + component_version.write_version_footer(cfg_versions, + self._config_file, + vintage='vyos') def save_json_record(self, component_versions: dict): """ @@ -200,7 +194,7 @@ class Migrator(object): # This will force calling all migration scripts: cfg_versions = {} - sys_versions = systemversions.get_system_component_version() + sys_versions = component_version.from_system() # save system component versions in json file for easy reference self.save_json_record(sys_versions) @@ -216,7 +210,7 @@ class Migrator(object): if not self._changed: return - formatversions.remove_versions(cfg_file) + component_version.remove_footer(cfg_file) self.write_config_file_versions(rev_versions) @@ -237,7 +231,7 @@ class VirtualMigrator(Migrator): if not self._changed: return - formatversions.remove_versions(cfg_file) + component_version.remove_footer(cfg_file) self.write_config_file_versions(cfg_versions) diff --git a/python/vyos/systemversions.py b/python/vyos/systemversions.py deleted file mode 100644 index f2da76d4f..000000000 --- a/python/vyos/systemversions.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2019 VyOS maintainers and contributors -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see . - -import os -import re -import sys -import vyos.defaults -from vyos.xml import component_version - -# legacy version, reading from the file names in -# /opt/vyatta/etc/config-migrate/current -def get_system_versions(): - """ - Get component versions from running system; critical failure if - unable to read migration directory. - """ - system_versions = {} - - try: - version_info = os.listdir(vyos.defaults.directories['current']) - except OSError as err: - print("OS error: {}".format(err)) - sys.exit(1) - - for info in version_info: - if re.match(r'[\w,-]+@\d+', info): - pair = info.split('@') - system_versions[pair[0]] = int(pair[1]) - - return system_versions - -# read from xml cache -def get_system_component_version(): - return component_version() diff --git a/smoketest/scripts/cli/test_component_version.py b/smoketest/scripts/cli/test_component_version.py index 1355c1f94..7b1b12c53 100755 --- a/smoketest/scripts/cli/test_component_version.py +++ b/smoketest/scripts/cli/test_component_version.py @@ -16,7 +16,7 @@ import unittest -from vyos.systemversions import get_system_versions, get_system_component_version +import vyos.component_version as component_version # After T3474, component versions should be updated in the files in # vyos-1x/interface-definitions/include/version/ @@ -24,8 +24,8 @@ from vyos.systemversions import get_system_versions, get_system_component_versio # that in the xml cache. class TestComponentVersion(unittest.TestCase): def setUp(self): - self.legacy_d = get_system_versions() - self.xml_d = get_system_component_version() + self.legacy_d = component_version.legacy_from_system() + self.xml_d = component_version.from_system() self.set_legacy_d = set(self.legacy_d) self.set_xml_d = set(self.xml_d) diff --git a/src/helpers/system-versions-foot.py b/src/helpers/system-versions-foot.py index 2aa687221..9614f0d28 100755 --- a/src/helpers/system-versions-foot.py +++ b/src/helpers/system-versions-foot.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -# Copyright 2019 VyOS maintainers and contributors +# Copyright 2019, 2022 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,24 +16,13 @@ # along with this library. If not, see . import sys -import vyos.formatversions as formatversions -import vyos.systemversions as systemversions import vyos.defaults -import vyos.version - -sys_versions = systemversions.get_system_component_version() - -component_string = formatversions.format_versions_string(sys_versions) - -os_version_string = vyos.version.get_version() +from vyos.component_version import write_system_footer sys.stdout.write("\n\n") if vyos.defaults.cfg_vintage == 'vyos': - formatversions.write_vyos_versions_foot(None, component_string, - os_version_string) + write_system_footer(None, vintage='vyos') elif vyos.defaults.cfg_vintage == 'vyatta': - formatversions.write_vyatta_versions_foot(None, component_string, - os_version_string) + write_system_footer(None, vintage='vyatta') else: - formatversions.write_vyatta_versions_foot(None, component_string, - os_version_string) + write_system_footer(None, vintage='vyos') -- cgit v1.2.3 From 3f75a38abe3e97ff243d01931a4defe8f4eef98a Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Fri, 28 Oct 2022 10:38:26 -0400 Subject: T4779: add vyos.util.human_to_bytes --- python/vyos/util.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'python') diff --git a/python/vyos/util.py b/python/vyos/util.py index e4e2a44ec..a80584c5a 100644 --- a/python/vyos/util.py +++ b/python/vyos/util.py @@ -574,6 +574,37 @@ def bytes_to_human(bytes, initial_exponent=0): size_string = "{0:.2f} {1}".format(value, suffix) return size_string +def human_to_bytes(value): + """ Converts a data amount with a unit suffix to bytes, like 2K to 2048 """ + + from re import match as re_match + + res = re_match(r'^\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]+)\s*$', value) + + if not res: + raise ValueError(f"'{value}' is not a valid data amount") + else: + amount = float(res.group(1)) + unit = res.group(2).lower() + + if unit == 'b': + res = amount + elif (unit == 'k') or (unit == 'kb'): + res = amount * 1024 + elif (unit == 'm') or (unit == 'mb'): + res = amount * 1024**2 + elif (unit == 'g') or (unit == 'gb'): + res = amount * 1024**3 + elif (unit == 't') or (unit == 'tb'): + res = amount * 1024**4 + else: + raise ValueError(f"Unsupported data unit '{unit}'") + + # There cannot be fractional bytes, so we convert them to integer. + # However, truncating causes problems with conversion back to human unit, + # so we round instead -- that seems to work well enough. + return round(res) + def get_cfg_group_id(): from grp import getgrnam from vyos.defaults import cfg_group -- cgit v1.2.3 From fca46598415f0c6f11c272d6b384ac98500fd69d Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Fri, 28 Oct 2022 11:04:51 -0400 Subject: T4765: handle non-string fields in the raw op mode output normalizer --- python/vyos/opmode.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'python') diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index c9827d634..727e118a8 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -101,6 +101,10 @@ def _get_arg_type(t): return t def _normalize_field_name(name): + # Convert the name to string if it is not + # (in some cases they may be numbers) + name = str(name) + # Replace all separators with underscores name = re.sub(r'(\s|[\(\)\[\]\{\}\-\.\,:\"\'\`])+', '_', name) -- cgit v1.2.3