diff options
Diffstat (limited to 'python')
-rw-r--r-- | python/vyos/accel_ppp.py | 74 | ||||
-rw-r--r-- | python/vyos/base.py | 44 | ||||
-rw-r--r-- | python/vyos/component_version.py | 192 | ||||
-rw-r--r-- | python/vyos/component_versions.py | 57 | ||||
-rw-r--r-- | python/vyos/configdep.py | 65 | ||||
-rw-r--r-- | python/vyos/configtree.py | 20 | ||||
-rw-r--r-- | python/vyos/defaults.py | 2 | ||||
-rw-r--r-- | python/vyos/firewall.py | 92 | ||||
-rw-r--r-- | python/vyos/formatversions.py | 109 | ||||
-rw-r--r-- | python/vyos/ifconfig/__init__.py | 1 | ||||
-rw-r--r-- | python/vyos/ifconfig/macvlan.py | 9 | ||||
-rw-r--r-- | python/vyos/ifconfig/veth.py | 54 | ||||
-rw-r--r-- | python/vyos/migrator.py | 95 | ||||
-rw-r--r-- | python/vyos/nat.py | 59 | ||||
-rw-r--r-- | python/vyos/opmode.py | 61 | ||||
-rw-r--r-- | python/vyos/systemversions.py | 46 | ||||
-rw-r--r-- | python/vyos/template.py | 13 | ||||
-rw-r--r-- | python/vyos/util.py | 46 |
18 files changed, 695 insertions, 344 deletions
diff --git a/python/vyos/accel_ppp.py b/python/vyos/accel_ppp.py new file mode 100644 index 000000000..bfc8ee5a9 --- /dev/null +++ b/python/vyos/accel_ppp.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +import sys + +import vyos.opmode +from vyos.util import rc_cmd + + +def get_server_statistics(accel_statistics, pattern, sep=':') -> dict: + import re + + stat_dict = {'sessions': {}} + + cpu = re.search(r'cpu(.*)', accel_statistics).group(0) + # Find all lines with pattern, for example 'sstp:' + data = re.search(rf'{pattern}(.*)', accel_statistics, re.DOTALL).group(0) + session_starting = re.search(r'starting(.*)', data).group(0) + session_active = re.search(r'active(.*)', data).group(0) + + for entry in {cpu, session_starting, session_active}: + if sep in entry: + key, value = entry.split(sep) + if key in ['starting', 'active', 'finishing']: + stat_dict['sessions'][key] = value.strip() + continue + stat_dict[key] = value.strip() + return stat_dict + + +def accel_cmd(port: int, command: str) -> str: + _, output = rc_cmd(f'/usr/bin/accel-cmd -p{port} {command}') + return output + + +def accel_out_parse(accel_output: list[str]) -> list[dict[str, str]]: + """ Parse accel-cmd show sessions output """ + data_list: list[dict[str, str]] = list() + field_names: list[str] = list() + + field_names_unstripped: list[str] = accel_output.pop(0).split('|') + for field_name in field_names_unstripped: + field_names.append(field_name.strip()) + + while accel_output: + if '|' not in accel_output[0]: + accel_output.pop(0) + continue + + current_item: list[str] = accel_output.pop(0).split('|') + item_dict: dict[str, str] = {} + + for field_index in range(len(current_item)): + field_name: str = field_names[field_index] + field_value: str = current_item[field_index].strip() + item_dict[field_name] = field_value + + data_list.append(item_dict) + + return data_list diff --git a/python/vyos/base.py b/python/vyos/base.py index 78067d5b2..9b93cb2f2 100644 --- a/python/vyos/base.py +++ b/python/vyos/base.py @@ -15,17 +15,47 @@ from textwrap import fill + +class BaseWarning: + def __init__(self, header, message, **kwargs): + self.message = message + self.kwargs = kwargs + if 'width' not in kwargs: + self.width = 72 + if 'initial_indent' in kwargs: + del self.kwargs['initial_indent'] + if 'subsequent_indent' in kwargs: + del self.kwargs['subsequent_indent'] + self.textinitindent = header + self.standardindent = '' + + def print(self): + messages = self.message.split('\n') + isfirstmessage = True + initial_indent = self.textinitindent + print('') + for mes in messages: + mes = fill(mes, initial_indent=initial_indent, + subsequent_indent=self.standardindent, **self.kwargs) + if isfirstmessage: + isfirstmessage = False + initial_indent = self.standardindent + print(f'{mes}') + print('') + + class Warning(): - def __init__(self, message): - # Reformat the message and trim it to 72 characters in length - message = fill(message, width=72) - print(f'\nWARNING: {message}') + def __init__(self, message, **kwargs): + self.BaseWarn = BaseWarning('WARNING: ', message, **kwargs) + self.BaseWarn.print() + class DeprecationWarning(): - def __init__(self, message): + def __init__(self, message, **kwargs): # Reformat the message and trim it to 72 characters in length - message = fill(message, width=72) - print(f'\nDEPRECATION WARNING: {message}\n') + self.BaseWarn = BaseWarning('DEPRECATION WARNING: ', message, **kwargs) + self.BaseWarn.print() + class ConfigError(Exception): def __init__(self, message): diff --git a/python/vyos/component_version.py b/python/vyos/component_version.py new file mode 100644 index 000000000..a4e318d08 --- /dev/null +++ b/python/vyos/component_version.py @@ -0,0 +1,192 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +""" +Functions for reading/writing component versions. + +The config file version string has the following form: + +VyOS 1.3/1.4: + +// Warning: Do not remove the following line. +// vyos-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack@3:conntrack-sync@2:dhcp-relay@2:dhcp-server@6:dhcpv6-server@1:dns-forwarding@3:firewall@5:https@2:interfaces@22:ipoe-server@1:ipsec@5:isis@1:l2tp@3:lldp@1:mdns@1:nat@5:ntp@1:pppoe-server@5:pptp@2:qos@1:quagga@8:rpki@1:salt@1:snmp@2:ssh@2:sstp@3:system@21:vrrp@2:vyos-accel-ppp@2:wanloadbalance@3:webproxy@2:zone-policy@1" +// Release version: 1.3.0 + +VyOS 1.2: + +/* Warning: Do not remove the following line. */ +/* === vyatta-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack-sync@1:conntrack@1:dhcp-relay@2:dhcp-server@5:dns-forwarding@1:firewall@5:ipsec@5:l2tp@1:mdns@1:nat@4:ntp@1:pppoe-server@2:pptp@1:qos@1:quagga@7:snmp@1:ssh@1:system@10:vrrp@2:wanloadbalance@3:webgui@1:webproxy@2:zone-policy@1" === */ +/* Release version: 1.2.8 */ + +""" + +import os +import re +import sys +import fileinput + +from vyos.xml import component_version +from vyos.version import get_version +from vyos.defaults import directories + +DEFAULT_CONFIG_PATH = os.path.join(directories['config'], 'config.boot') + +def from_string(string_line, vintage='vyos'): + """ + Get component version dictionary from string. + Return empty dictionary if string contains no config information + or raise error if component version string malformed. + """ + version_dict = {} + + if vintage == 'vyos': + if re.match(r'// vyos-config-version:.+', string_line): + if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', string_line): + raise ValueError(f"malformed configuration string: {string_line}") + + for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): + version_dict[pair[0]] = int(pair[1]) + + elif vintage == 'vyatta': + if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line): + if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line): + raise ValueError(f"malformed configuration string: {string_line}") + + for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): + version_dict[pair[0]] = int(pair[1]) + else: + raise ValueError("Unknown config string vintage") + + return version_dict + +def from_file(config_file_name=DEFAULT_CONFIG_PATH, vintage='vyos'): + """ + Get component version dictionary parsing config file line by line + """ + with open(config_file_name, 'r') as f: + for line_in_config in f: + version_dict = from_string(line_in_config, vintage=vintage) + if version_dict: + return version_dict + + # no version information + return {} + +def from_system(): + """ + Get system component version dict. + """ + return component_version() + +def legacy_from_system(): + """ + Get system component version dict from legacy location. + This is for a transitional sanity check; the directory will eventually + be removed. + """ + system_versions = {} + legacy_dir = directories['current'] + + # To be removed: + if not os.path.isdir(legacy_dir): + return system_versions + + try: + version_info = os.listdir(legacy_dir) + except OSError as err: + sys.exit(repr(err)) + + for info in version_info: + if re.match(r'[\w,-]+@\d+', info): + pair = info.split('@') + system_versions[pair[0]] = int(pair[1]) + + return system_versions + +def format_string(ver: dict) -> str: + """ + Version dict to string. + """ + keys = list(ver) + keys.sort() + l = [] + for k in keys: + v = ver[k] + l.append(f'{k}@{v}') + sep = ':' + return sep.join(l) + +def version_footer(ver: dict, vintage='vyos') -> str: + """ + Version footer as string. + """ + ver_str = format_string(ver) + release = get_version() + if vintage == 'vyos': + ret_str = (f'// Warning: Do not remove the following line.\n' + + f'// vyos-config-version: "{ver_str}"\n' + + f'// Release version: {release}\n') + elif vintage == 'vyatta': + ret_str = (f'/* Warning: Do not remove the following line. */\n' + + f'/* === vyatta-config-version: "{ver_str}" === */\n' + + f'/* Release version: {release} */\n') + else: + raise ValueError("Unknown config string vintage") + + return ret_str + +def system_footer(vintage='vyos') -> str: + """ + System version footer as string. + """ + ver_d = from_system() + return version_footer(ver_d, vintage=vintage) + +def write_version_footer(ver: dict, file_name, vintage='vyos'): + """ + Write version footer to file. + """ + footer = version_footer(ver=ver, vintage=vintage) + if file_name: + with open(file_name, 'a') as f: + f.write(footer) + else: + sys.stdout.write(footer) + +def write_system_footer(file_name, vintage='vyos'): + """ + Write system version footer to file. + """ + ver_d = from_system() + return write_version_footer(ver_d, file_name=file_name, vintage=vintage) + +def remove_footer(file_name): + """ + Remove old version footer. + """ + for line in fileinput.input(file_name, inplace=True): + if re.match(r'/\* Warning:.+ \*/$', line): + continue + if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line): + continue + if re.match(r'/\* Release version:.+ \*/$', line): + continue + if re.match('// vyos-config-version:.+', line): + continue + if re.match('// Warning:.+', line): + continue + if re.match('// Release version:.+', line): + continue + sys.stdout.write(line) diff --git a/python/vyos/component_versions.py b/python/vyos/component_versions.py deleted file mode 100644 index 90b458aae..000000000 --- a/python/vyos/component_versions.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2017 VyOS maintainers and contributors <maintainers@vyos.io> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. - -""" -The version data looks like: - -/* Warning: Do not remove the following line. */ -/* === vyatta-config-version: -"cluster@1:config-management@1:conntrack-sync@1:conntrack@1:dhcp-relay@1:dhcp-server@4:firewall@5:ipsec@4:nat@4:qos@1:quagga@2:system@8:vrrp@1:wanloadbalance@3:webgui@1:webproxy@1:zone-policy@1" -=== */ -/* Release version: 1.2.0-rolling+201806131737 */ -""" - -import re - -def get_component_version(string_line): - """ - Get component version dictionary from string - return empty dictionary if string contains no config information - or raise error if component version string malformed - """ - return_value = {} - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line): - - if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line): - raise ValueError("malformed configuration string: " + str(string_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): - if pair[0] in return_value.keys(): - raise ValueError("duplicate unit name: \"" + str(pair[0]) + "\" in string: \"" + string_line + "\"") - return_value[pair[0]] = int(pair[1]) - - return return_value - - -def get_component_versions_from_file(config_file_name='/opt/vyatta/etc/config/config.boot'): - """ - Get component version dictionary parsing config file line by line - """ - f = open(config_file_name, 'r') - for line_in_config in f: - component_version = get_component_version(line_in_config) - if component_version: - return component_version - raise ValueError("no config string in file:", config_file_name) diff --git a/python/vyos/configdep.py b/python/vyos/configdep.py new file mode 100644 index 000000000..e6b82ca93 --- /dev/null +++ b/python/vyos/configdep.py @@ -0,0 +1,65 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +import os +from inspect import stack + +from vyos.util import load_as_module + +dependents = {} + +def canon_name(name: str) -> str: + return os.path.splitext(name)[0].replace('-', '_') + +def canon_name_of_path(path: str) -> str: + script = os.path.basename(path) + return canon_name(script) + +def caller_name() -> str: + return stack()[-1].filename + +def run_config_mode_script(script: str, config): + from vyos.defaults import directories + + path = os.path.join(directories['conf_mode'], script) + name = canon_name(script) + mod = load_as_module(name, path) + + config.set_level([]) + try: + c = mod.get_config(config) + mod.verify(c) + mod.generate(c) + mod.apply(c) + except (VyOSError, ConfigError) as e: + raise ConfigError(repr(e)) + +def def_closure(script: str, config): + def func_impl(): + run_config_mode_script(script, config) + return func_impl + +def set_dependent(target: str, config): + k = canon_name_of_path(caller_name()) + l = dependents.setdefault(k, []) + func = def_closure(target, config) + l.append(func) + +def call_dependents(): + k = canon_name_of_path(caller_name()) + l = dependents.get(k, []) + while l: + f = l.pop(0) + f() diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index e9cdb69e4..b88615513 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -1,5 +1,5 @@ # configtree -- a standalone VyOS config file manipulation library (Python bindings) -# Copyright (C) 2018 VyOS maintainers and contributors +# Copyright (C) 2018-2022 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or modify it under the terms of # the GNU Lesser General Public License as published by the Free Software Foundation; @@ -12,6 +12,7 @@ # You should have received a copy of the GNU Lesser General Public License along with this library; # if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +import os import re import json @@ -147,6 +148,8 @@ class ConfigTree(object): self.__config = address self.__version = '' + self.__migration = os.environ.get('VYOS_MIGRATION') + def __del__(self): if self.__config is not None: self.__destroy(self.__config) @@ -191,18 +194,27 @@ class ConfigTree(object): else: self.__set_add_value(self.__config, path_str, str(value).encode()) + if self.__migration: + print(f"- op: set path: {path} value: {value} replace: {replace}") + def delete(self, path): check_path(path) path_str = " ".join(map(str, path)).encode() self.__delete(self.__config, path_str) + if self.__migration: + print(f"- op: delete path: {path}") + def delete_value(self, path, value): check_path(path) path_str = " ".join(map(str, path)).encode() self.__delete_value(self.__config, path_str, value.encode()) + if self.__migration: + print(f"- op: delete_value path: {path} value: {value}") + def rename(self, path, new_name): check_path(path) path_str = " ".join(map(str, path)).encode() @@ -216,6 +228,9 @@ class ConfigTree(object): if (res != 0): raise ConfigTreeError("Path [{}] doesn't exist".format(path)) + if self.__migration: + print(f"- op: rename old_path: {path} new_path: {new_path}") + def copy(self, old_path, new_path): check_path(old_path) check_path(new_path) @@ -229,6 +244,9 @@ class ConfigTree(object): if (res != 0): raise ConfigTreeError("Path [{}] doesn't exist".format(old_path)) + if self.__migration: + print(f"- op: copy old_path: {old_path} new_path: {new_path}") + def exists(self, path): check_path(path) path_str = " ".join(map(str, path)).encode() diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index 6894fc4da..7de458960 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -49,8 +49,6 @@ api_data = { 'port' : '8080', 'socket' : False, 'strict' : False, - 'gql' : False, - 'introspection' : False, 'debug' : False, 'api_keys' : [ {"id": "testapp", "key": "qwerty"} ] } diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index 2ebb220fe..48263eef5 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -20,6 +20,9 @@ import os import re from pathlib import Path +from socket import AF_INET +from socket import AF_INET6 +from socket import getaddrinfo from time import strftime from vyos.remote import download @@ -31,65 +34,31 @@ from vyos.util import dict_search_args from vyos.util import dict_search_recursive from vyos.util import run +# Domain Resolver -# Functions for firewall group domain-groups -def get_ips_domains_dict(list_domains): - """ - Get list of IPv4 addresses by list of domains - Ex: get_ips_domains_dict(['ex1.com', 'ex2.com']) - {'ex1.com': ['192.0.2.1'], 'ex2.com': ['192.0.2.2', '192.0.2.3']} - """ - from socket import gethostbyname_ex - from socket import gaierror - - ip_dict = {} - for domain in list_domains: - try: - _, _, ips = gethostbyname_ex(domain) - ip_dict[domain] = ips - except gaierror: - pass - - return ip_dict - -def nft_init_set(group_name, table="vyos_filter", family="ip"): - """ - table ip vyos_filter { - set GROUP_NAME - type ipv4_addr - flags interval - } - """ - return call(f'nft add set ip {table} {group_name} {{ type ipv4_addr\\; flags interval\\; }}') - - -def nft_add_set_elements(group_name, elements, table="vyos_filter", family="ip"): - """ - table ip vyos_filter { - set GROUP_NAME { - type ipv4_addr - flags interval - elements = { 192.0.2.1, 192.0.2.2 } - } - """ - elements = ", ".join(elements) - return call(f'nft add element {family} {table} {group_name} {{ {elements} }} ') - -def nft_flush_set(group_name, table="vyos_filter", family="ip"): - """ - Flush elements of nft set - """ - return call(f'nft flush set {family} {table} {group_name}') - -def nft_update_set_elements(group_name, elements, table="vyos_filter", family="ip"): - """ - Update elements of nft set - """ - flush_set = nft_flush_set(group_name, table="vyos_filter", family="ip") - nft_add_set = nft_add_set_elements(group_name, elements, table="vyos_filter", family="ip") - return flush_set, nft_add_set - -# END firewall group domain-group (sets) +def fqdn_config_parse(firewall): + firewall['ip_fqdn'] = {} + firewall['ip6_fqdn'] = {} + + for domain, path in dict_search_recursive(firewall, 'fqdn'): + fw_name = path[1] # name/ipv6-name + rule = path[3] # rule id + suffix = path[4][0] # source/destination (1 char) + set_name = f'{fw_name}_{rule}_{suffix}' + + if path[0] == 'name': + firewall['ip_fqdn'][set_name] = domain + elif path[0] == 'ipv6_name': + firewall['ip6_fqdn'][set_name] = domain + +def fqdn_resolve(fqdn, ipv6=False): + try: + res = getaddrinfo(fqdn, None, AF_INET6 if ipv6 else AF_INET) + return set(item[4][0] for item in res) + except: + return None + +# End Domain Resolver def find_nftables_rule(table, chain, rule_matches=[]): # Find rule in table/chain that matches all criteria and return the handle @@ -158,6 +127,13 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name): operator = f'& {address_mask} {operator} ' output.append(f'{ip_name} {prefix}addr {operator}{suffix}') + if 'fqdn' in side_conf: + fqdn = side_conf['fqdn'] + operator = '' + if fqdn[0] == '!': + operator = '!=' + output.append(f'{ip_name} {prefix}addr {operator} @FQDN_{fw_name}_{rule_id}_{prefix}') + if dict_search_args(side_conf, 'geoip', 'country_code'): operator = '' if dict_search_args(side_conf, 'geoip', 'inverse_match') != None: diff --git a/python/vyos/formatversions.py b/python/vyos/formatversions.py deleted file mode 100644 index 29117a5d3..000000000 --- a/python/vyos/formatversions.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see <http://www.gnu.org/licenses/>. - -import sys -import os -import re -import fileinput - -def read_vyatta_versions(config_file): - config_file_versions = {} - - with open(config_file, 'r') as config_file_handle: - for config_line in config_file_handle: - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', config_line): - if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', config_line): - raise ValueError("malformed configuration string: " - "{}".format(config_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', config_line): - config_file_versions[pair[0]] = int(pair[1]) - - - return config_file_versions - -def read_vyos_versions(config_file): - config_file_versions = {} - - with open(config_file, 'r') as config_file_handle: - for config_line in config_file_handle: - if re.match(r'// vyos-config-version:.+', config_line): - if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', config_line): - raise ValueError("malformed configuration string: " - "{}".format(config_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', config_line): - config_file_versions[pair[0]] = int(pair[1]) - - return config_file_versions - -def remove_versions(config_file): - """ - Remove old version string. - """ - for line in fileinput.input(config_file, inplace=True): - if re.match(r'/\* Warning:.+ \*/$', line): - continue - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line): - continue - if re.match(r'/\* Release version:.+ \*/$', line): - continue - if re.match('// vyos-config-version:.+', line): - continue - if re.match('// Warning:.+', line): - continue - if re.match('// Release version:.+', line): - continue - sys.stdout.write(line) - -def format_versions_string(config_versions): - cfg_keys = list(config_versions.keys()) - cfg_keys.sort() - - component_version_strings = [] - - for key in cfg_keys: - cfg_vers = config_versions[key] - component_version_strings.append('{}@{}'.format(key, cfg_vers)) - - separator = ":" - component_version_string = separator.join(component_version_strings) - - return component_version_string - -def write_vyatta_versions_foot(config_file, component_version_string, - os_version_string): - if config_file: - with open(config_file, 'a') as config_file_handle: - config_file_handle.write('/* Warning: Do not remove the following line. */\n') - config_file_handle.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string)) - config_file_handle.write('/* Release version: {} */\n'.format(os_version_string)) - else: - sys.stdout.write('/* Warning: Do not remove the following line. */\n') - sys.stdout.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string)) - sys.stdout.write('/* Release version: {} */\n'.format(os_version_string)) - -def write_vyos_versions_foot(config_file, component_version_string, - os_version_string): - if config_file: - with open(config_file, 'a') as config_file_handle: - config_file_handle.write('// Warning: Do not remove the following line.\n') - config_file_handle.write('// vyos-config-version: "{}"\n'.format(component_version_string)) - config_file_handle.write('// Release version: {}\n'.format(os_version_string)) - else: - sys.stdout.write('// Warning: Do not remove the following line.\n') - sys.stdout.write('// vyos-config-version: "{}"\n'.format(component_version_string)) - sys.stdout.write('// Release version: {}\n'.format(os_version_string)) - diff --git a/python/vyos/ifconfig/__init__.py b/python/vyos/ifconfig/__init__.py index a37615c8f..d1ddaa13e 100644 --- a/python/vyos/ifconfig/__init__.py +++ b/python/vyos/ifconfig/__init__.py @@ -36,4 +36,5 @@ from vyos.ifconfig.tunnel import TunnelIf from vyos.ifconfig.wireless import WiFiIf from vyos.ifconfig.l2tpv3 import L2TPv3If from vyos.ifconfig.macsec import MACsecIf +from vyos.ifconfig.veth import VethIf from vyos.ifconfig.wwan import WWANIf diff --git a/python/vyos/ifconfig/macvlan.py b/python/vyos/ifconfig/macvlan.py index 776014bc3..2266879ec 100644 --- a/python/vyos/ifconfig/macvlan.py +++ b/python/vyos/ifconfig/macvlan.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -30,10 +30,17 @@ class MACVLANIf(Interface): } def _create(self): + """ + Create MACvlan interface in OS kernel. Interface is administrative + down by default. + """ # please do not change the order when assembling the command cmd = 'ip link add {ifname} link {source_interface} type {type} mode {mode}' self._cmd(cmd.format(**self.config)) + # interface is always A/D down. It needs to be enabled explicitly + self.set_admin_state('down') + def set_mode(self, mode): ifname = self.config['ifname'] cmd = f'ip link set dev {ifname} type macvlan mode {mode}' diff --git a/python/vyos/ifconfig/veth.py b/python/vyos/ifconfig/veth.py new file mode 100644 index 000000000..aafbf226a --- /dev/null +++ b/python/vyos/ifconfig/veth.py @@ -0,0 +1,54 @@ +# Copyright 2022 VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from vyos.ifconfig.interface import Interface + + +@Interface.register +class VethIf(Interface): + """ + Abstraction of a Linux veth interface + """ + iftype = 'veth' + definition = { + **Interface.definition, + **{ + 'section': 'virtual-ethernet', + 'prefixes': ['veth', ], + 'bridgeable': True, + }, + } + + def _create(self): + """ + Create veth interface in OS kernel. Interface is administrative + down by default. + """ + # check before create, as we have 2 veth interfaces in our CLI + # interface virtual-ethernet veth0 peer-name 'veth1' + # interface virtual-ethernet veth1 peer-name 'veth0' + # + # but iproute2 creates the pair with one command: + # ip link add vet0 type veth peer name veth1 + if self.exists(self.config['peer_name']): + return + + # create virtual-ethernet interface + cmd = 'ip link add {ifname} type {type}'.format(**self.config) + cmd += f' peer name {self.config["peer_name"]}' + self._cmd(cmd) + + # interface is always A/D down. It needs to be enabled explicitly + self.set_admin_state('down') diff --git a/python/vyos/migrator.py b/python/vyos/migrator.py index c6e3435ca..87c74e1ea 100644 --- a/python/vyos/migrator.py +++ b/python/vyos/migrator.py @@ -1,4 +1,4 @@ -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2019-2022 VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,11 +16,13 @@ import sys import os import json -import subprocess -import vyos.version +import logging + import vyos.defaults -import vyos.systemversions as systemversions -import vyos.formatversions as formatversions +import vyos.component_version as component_version +from vyos.util import cmd + +log_file = os.path.join(vyos.defaults.directories['config'], 'vyos-migrate.log') class MigratorError(Exception): pass @@ -31,9 +33,21 @@ class Migrator(object): self._force = force self._set_vintage = set_vintage self._config_file_vintage = None - self._log_file = None self._changed = False + def init_logger(self): + self.logger = logging.getLogger(__name__) + self.logger.setLevel(logging.DEBUG) + + # on adding the file handler, allow write permission for cfg_group; + # restore original umask on exit + mask = os.umask(0o113) + fh = logging.FileHandler(log_file) + formatter = logging.Formatter('%(message)s') + fh.setFormatter(formatter) + self.logger.addHandler(fh) + os.umask(mask) + def read_config_file_versions(self): """ Get component versions from config file footer and set vintage; @@ -42,13 +56,13 @@ class Migrator(object): cfg_file = self._config_file component_versions = {} - cfg_versions = formatversions.read_vyatta_versions(cfg_file) + cfg_versions = component_version.from_file(cfg_file, vintage='vyatta') if cfg_versions: self._config_file_vintage = 'vyatta' component_versions = cfg_versions - cfg_versions = formatversions.read_vyos_versions(cfg_file) + cfg_versions = component_version.from_file(cfg_file, vintage='vyos') if cfg_versions: self._config_file_vintage = 'vyos' @@ -70,34 +84,15 @@ class Migrator(object): else: return True - def open_log_file(self): - """ - Open log file for migration, catching any error. - Note that, on boot, migration takes place before the canonical log - directory is created, hence write to the config file directory. - """ - self._log_file = os.path.join(vyos.defaults.directories['config'], - 'vyos-migrate.log') - # on creation, allow write permission for cfg_group; - # restore original umask on exit - mask = os.umask(0o113) - try: - log = open('{0}'.format(self._log_file), 'w') - log.write("List of executed migration scripts:\n") - except Exception as e: - os.umask(mask) - print("Logging error: {0}".format(e)) - return None - - os.umask(mask) - return log - def run_migration_scripts(self, config_file_versions, system_versions): """ Run migration scripts iteratively, until config file version equals system component version. """ - log = self.open_log_file() + os.environ['VYOS_MIGRATION'] = '1' + self.init_logger() + + self.logger.info("List of executed migration scripts:") cfg_versions = config_file_versions sys_versions = system_versions @@ -129,8 +124,9 @@ class Migrator(object): '{}-to-{}'.format(cfg_ver, next_ver)) try: - subprocess.check_call([migrate_script, - self._config_file]) + out = cmd([migrate_script, self._config_file]) + self.logger.info(f'{migrate_script}') + if out: self.logger.info(out) except FileNotFoundError: pass except Exception as err: @@ -138,38 +134,25 @@ class Migrator(object): "".format(migrate_script, err)) sys.exit(1) - if log: - try: - log.write('{0}\n'.format(migrate_script)) - except Exception as e: - print("Error writing log: {0}".format(e)) - cfg_ver = next_ver - rev_versions[key] = cfg_ver - if log: - log.close() - + del os.environ['VYOS_MIGRATION'] return rev_versions def write_config_file_versions(self, cfg_versions): """ Write new versions string. """ - versions_string = formatversions.format_versions_string(cfg_versions) - - os_version_string = vyos.version.get_version() - if self._config_file_vintage == 'vyatta': - formatversions.write_vyatta_versions_foot(self._config_file, - versions_string, - os_version_string) + component_version.write_version_footer(cfg_versions, + self._config_file, + vintage='vyatta') if self._config_file_vintage == 'vyos': - formatversions.write_vyos_versions_foot(self._config_file, - versions_string, - os_version_string) + component_version.write_version_footer(cfg_versions, + self._config_file, + vintage='vyos') def save_json_record(self, component_versions: dict): """ @@ -200,7 +183,7 @@ class Migrator(object): # This will force calling all migration scripts: cfg_versions = {} - sys_versions = systemversions.get_system_component_version() + sys_versions = component_version.from_system() # save system component versions in json file for easy reference self.save_json_record(sys_versions) @@ -216,7 +199,7 @@ class Migrator(object): if not self._changed: return - formatversions.remove_versions(cfg_file) + component_version.remove_footer(cfg_file) self.write_config_file_versions(rev_versions) @@ -237,7 +220,7 @@ class VirtualMigrator(Migrator): if not self._changed: return - formatversions.remove_versions(cfg_file) + component_version.remove_footer(cfg_file) self.write_config_file_versions(cfg_versions) diff --git a/python/vyos/nat.py b/python/vyos/nat.py index 31bbdc386..8a311045a 100644 --- a/python/vyos/nat.py +++ b/python/vyos/nat.py @@ -16,6 +16,8 @@ from vyos.template import is_ip_network from vyos.util import dict_search_args +from vyos.template import bracketize_ipv6 + def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): output = [] @@ -69,6 +71,7 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): else: translation_output.append('to') if addr: + addr = bracketize_ipv6(addr) translation_output.append(addr) options = [] @@ -85,8 +88,13 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): translation_str += f' {",".join(options)}' for target in ['source', 'destination']: + if target not in rule_conf: + continue + + side_conf = rule_conf[target] prefix = target[:1] - addr = dict_search_args(rule_conf, target, 'address') + + addr = dict_search_args(side_conf, 'address') if addr and not (ignore_type_addr and target == nat_type): operator = '' if addr[:1] == '!': @@ -94,7 +102,7 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): addr = addr[1:] output.append(f'{ip_prefix} {prefix}addr {operator} {addr}') - addr_prefix = dict_search_args(rule_conf, target, 'prefix') + addr_prefix = dict_search_args(side_conf, 'prefix') if addr_prefix and ipv6: operator = '' if addr_prefix[:1] == '!': @@ -102,7 +110,7 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): addr_prefix = addr[1:] output.append(f'ip6 {prefix}addr {operator} {addr_prefix}') - port = dict_search_args(rule_conf, target, 'port') + port = dict_search_args(side_conf, 'port') if port: protocol = rule_conf['protocol'] if protocol == 'tcp_udp': @@ -113,6 +121,51 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): port = port[1:] output.append(f'{protocol} {prefix}port {operator} {{ {port} }}') + if 'group' in side_conf: + group = side_conf['group'] + if 'address_group' in group and not (ignore_type_addr and target == nat_type): + group_name = group['address_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_prefix} {prefix}addr {operator} @A_{group_name}') + # Generate firewall group domain-group + elif 'domain_group' in group and not (ignore_type_addr and target == nat_type): + group_name = group['domain_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_prefix} {prefix}addr {operator} @D_{group_name}') + elif 'network_group' in group and not (ignore_type_addr and target == nat_type): + group_name = group['network_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_prefix} {prefix}addr {operator} @N_{group_name}') + if 'mac_group' in group: + group_name = group['mac_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'ether {prefix}addr {operator} @M_{group_name}') + if 'port_group' in group: + proto = rule_conf['protocol'] + group_name = group['port_group'] + + if proto == 'tcp_udp': + proto = 'th' + + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + + output.append(f'{proto} {prefix}port {operator} @P_{group_name}') + output.append('counter') if 'log' in rule_conf: diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index 7e3545c87..9dba8d30f 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -16,6 +16,7 @@ import re import sys import typing +from humps import decamelize class Error(Exception): @@ -44,6 +45,19 @@ class PermissionDenied(Error): """ pass +class IncorrectValue(Error): + """ Requested operation is valid, but an argument provided has an + incorrect value, preventing successful completion. + """ + pass + +class InternalError(Error): + """ Any situation when VyOS detects that it could not perform + an operation correctly due to logic errors in its own code + or errors in underlying software. + """ + pass + def _is_op_mode_function_name(name): if re.match(r"^(show|clear|reset|restart)", name): @@ -93,6 +107,51 @@ def _get_arg_type(t): else: return t +def _normalize_field_name(name): + # Convert the name to string if it is not + # (in some cases they may be numbers) + name = str(name) + + # Replace all separators with underscores + name = re.sub(r'(\s|[\(\)\[\]\{\}\-\.\,:\"\'\`])+', '_', name) + + # Replace specific characters with textual descriptions + name = re.sub(r'@', '_at_', name) + name = re.sub(r'%', '_percentage_', name) + name = re.sub(r'~', '_tilde_', name) + + # Force all letters to lowercase + name = name.lower() + + # Remove leading and trailing underscores, if any + name = re.sub(r'(^(_+)(?=[^_])|_+$)', '', name) + + # Ensure there are only single underscores + name = re.sub(r'_+', '_', name) + + return name + +def _normalize_dict_field_names(old_dict): + new_dict = {} + + for key in old_dict: + new_key = _normalize_field_name(key) + new_dict[new_key] = _normalize_field_names(old_dict[key]) + + # Sanity check + if len(old_dict) != len(new_dict): + raise InternalError("Dictionary fields do not allow unique normalization") + else: + return new_dict + +def _normalize_field_names(value): + if isinstance(value, dict): + return _normalize_dict_field_names(value) + elif isinstance(value, list): + return list(map(lambda v: _normalize_field_names(v), value)) + else: + return value + def run(module): from argparse import ArgumentParser @@ -148,6 +207,8 @@ def run(module): if not args["raw"]: return res else: + res = decamelize(res) + res = _normalize_field_names(res) from json import dumps return dumps(res, indent=4) else: diff --git a/python/vyos/systemversions.py b/python/vyos/systemversions.py deleted file mode 100644 index f2da76d4f..000000000 --- a/python/vyos/systemversions.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2019 VyOS maintainers and contributors <maintainers@vyos.io> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see <http://www.gnu.org/licenses/>. - -import os -import re -import sys -import vyos.defaults -from vyos.xml import component_version - -# legacy version, reading from the file names in -# /opt/vyatta/etc/config-migrate/current -def get_system_versions(): - """ - Get component versions from running system; critical failure if - unable to read migration directory. - """ - system_versions = {} - - try: - version_info = os.listdir(vyos.defaults.directories['current']) - except OSError as err: - print("OS error: {}".format(err)) - sys.exit(1) - - for info in version_info: - if re.match(r'[\w,-]+@\d+', info): - pair = info.split('@') - system_versions[pair[0]] = int(pair[1]) - - return system_versions - -# read from xml cache -def get_system_component_version(): - return component_version() diff --git a/python/vyos/template.py b/python/vyos/template.py index 0870a0523..2a4135f9e 100644 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -566,12 +566,17 @@ def nft_default_rule(fw_conf, fw_name, ipv6=False): return " ".join(output) @register_filter('nft_state_policy') -def nft_state_policy(conf, state, ipv6=False): +def nft_state_policy(conf, state): out = [f'ct state {state}'] - if 'log' in conf: - log_level = conf['log'] - out.append(f'log level {log_level}') + if 'log' in conf and 'enable' in conf['log']: + log_state = state[:3].upper() + log_action = (conf['action'] if 'action' in conf else 'accept')[:1].upper() + out.append(f'log prefix "[STATE-POLICY-{log_state}-{log_action}]"') + + if 'log_level' in conf: + log_level = conf['log_level'] + out.append(f'level {log_level}') out.append('counter') diff --git a/python/vyos/util.py b/python/vyos/util.py index 461df9a6e..9ebe69b6c 100644 --- a/python/vyos/util.py +++ b/python/vyos/util.py @@ -574,6 +574,37 @@ def bytes_to_human(bytes, initial_exponent=0): size_string = "{0:.2f} {1}".format(value, suffix) return size_string +def human_to_bytes(value): + """ Converts a data amount with a unit suffix to bytes, like 2K to 2048 """ + + from re import match as re_match + + res = re_match(r'^\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]+)\s*$', value) + + if not res: + raise ValueError(f"'{value}' is not a valid data amount") + else: + amount = float(res.group(1)) + unit = res.group(2).lower() + + if unit == 'b': + res = amount + elif (unit == 'k') or (unit == 'kb'): + res = amount * 1024 + elif (unit == 'm') or (unit == 'mb'): + res = amount * 1024**2 + elif (unit == 'g') or (unit == 'gb'): + res = amount * 1024**3 + elif (unit == 't') or (unit == 'tb'): + res = amount * 1024**4 + else: + raise ValueError(f"Unsupported data unit '{unit}'") + + # There cannot be fractional bytes, so we convert them to integer. + # However, truncating causes problems with conversion back to human unit, + # so we round instead -- that seems to work well enough. + return round(res) + def get_cfg_group_id(): from grp import getgrnam from vyos.defaults import cfg_group @@ -1105,3 +1136,18 @@ def sysctl_write(name, value): call(f'sysctl -wq {name}={value}') return True return False + +# approach follows a discussion in: +# https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case +def camel_to_snake_case(name: str) -> str: + pattern = r'\d+|[A-Z]?[a-z]+|\W|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)' + words = re.findall(pattern, name) + return '_'.join(map(str.lower, words)) + +def load_as_module(name: str, path: str): + import importlib.util + + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod |