From 85f04237160a6ea98eea4ec58f1ccab9f6bfc31a Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 17 Oct 2022 12:15:22 +0000 Subject: ssh: T4720: Ability to configure SSH-server HostKeyAlgorithms Ability to configure SSH-server HostKeyAlgorithms. Specifies the host key signature algorithms that the server offers. Can accept multiple values. --- data/templates/ssh/sshd_config.j2 | 5 +++++ interface-definitions/ssh.xml.in | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/data/templates/ssh/sshd_config.j2 b/data/templates/ssh/sshd_config.j2 index 5bbfdeb88..93735020c 100644 --- a/data/templates/ssh/sshd_config.j2 +++ b/data/templates/ssh/sshd_config.j2 @@ -62,6 +62,11 @@ ListenAddress {{ address }} Ciphers {{ ciphers | join(',') }} {% endif %} +{% if hostkey_algorithm is vyos_defined %} +# Specifies the available Host Key signature algorithms +HostKeyAlgorithms {{ hostkey_algorithm | join(',') }} +{% endif %} + {% if mac is vyos_defined %} # Specifies the available MAC (message authentication code) algorithms MACs {{ mac | join(',') }} diff --git a/interface-definitions/ssh.xml.in b/interface-definitions/ssh.xml.in index f3c731fe5..2bcce2cf0 100644 --- a/interface-definitions/ssh.xml.in +++ b/interface-definitions/ssh.xml.in @@ -133,6 +133,19 @@ + + + Allowed host key signature algorithms + + + ssh-ed25519 ssh-ed25519-cert-v01@openssh.com sk-ssh-ed25519@openssh.com sk-ssh-ed25519-cert-v01@openssh.com ssh-rsa rsa-sha2-256 rsa-sha2-512 ssh-dss ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 sk-ecdsa-sha2-nistp256@openssh.com webauthn-sk-ecdsa-sha2-nistp256@openssh.com ssh-rsa-cert-v01@openssh.com rsa-sha2-256-cert-v01@openssh.com rsa-sha2-512-cert-v01@openssh.com ssh-dss-cert-v01@openssh.com ecdsa-sha2-nistp256-cert-v01@openssh.com ecdsa-sha2-nistp384-cert-v01@openssh.com ecdsa-sha2-nistp521-cert-v01@openssh.com sk-ecdsa-sha2-nistp256-cert-v01@openssh.com + + + + (ssh-ed25519|ssh-ed25519-cert-v01@openssh.com|sk-ssh-ed25519@openssh.com|sk-ssh-ed25519-cert-v01@openssh.com|ssh-rsa|rsa-sha2-256|rsa-sha2-512|ssh-dss|ecdsa-sha2-nistp256|ecdsa-sha2-nistp384|ecdsa-sha2-nistp521|sk-ecdsa-sha2-nistp256@openssh.com|webauthn-sk-ecdsa-sha2-nistp256@openssh.com|ssh-rsa-cert-v01@openssh.com|rsa-sha2-256-cert-v01@openssh.com|rsa-sha2-512-cert-v01@openssh.com|ssh-dss-cert-v01@openssh.com|ecdsa-sha2-nistp256-cert-v01@openssh.com|ecdsa-sha2-nistp384-cert-v01@openssh.com|ecdsa-sha2-nistp521-cert-v01@openssh.com|sk-ecdsa-sha2-nistp256-cert-v01@openssh.com) + + + Allowed key exchange (KEX) algorithms -- cgit v1.2.3 From 3ff47d3388fbbcd538d262170c4950aaa61d0efe Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 17 Oct 2022 13:24:48 +0000 Subject: T4720: Add smoketest for SSH NDcPP --- smoketest/scripts/cli/test_service_ssh.py | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/smoketest/scripts/cli/test_service_ssh.py b/smoketest/scripts/cli/test_service_ssh.py index 0b029dd00..8de98f34f 100755 --- a/smoketest/scripts/cli/test_service_ssh.py +++ b/smoketest/scripts/cli/test_service_ssh.py @@ -262,5 +262,42 @@ class TestServiceSSH(VyOSUnitTestSHIM.TestCase): self.assertFalse(process_named_running(SSHGUARD_PROCESS)) + + # Network Device Collaborative Protection Profile + def test_ssh_ndcpp(self): + ciphers = ['aes128-cbc', 'aes128-ctr', 'aes256-cbc', 'aes256-ctr'] + host_key_algs = ['sk-ssh-ed25519@openssh.com', 'ssh-rsa', 'ssh-ed25519'] + kexes = ['diffie-hellman-group14-sha1', 'ecdh-sha2-nistp256', 'ecdh-sha2-nistp384', 'ecdh-sha2-nistp521'] + macs = ['hmac-sha1', 'hmac-sha2-256', 'hmac-sha2-512'] + rekey_time = '60' + rekey_data = '1024' + + for cipher in ciphers: + self.cli_set(base_path + ['ciphers', cipher]) + for host_key in host_key_algs: + self.cli_set(base_path + ['hostkey-algorithm', host_key]) + for kex in kexes: + self.cli_set(base_path + ['key-exchange', kex]) + for mac in macs: + self.cli_set(base_path + ['mac', mac]) + # Optional rekey parameters + self.cli_set(base_path + ['rekey', 'data', rekey_data]) + self.cli_set(base_path + ['rekey', 'time', rekey_time]) + + # commit changes + self.cli_commit() + + ssh_lines = ['Ciphers aes128-cbc,aes128-ctr,aes256-cbc,aes256-ctr', + 'HostKeyAlgorithms sk-ssh-ed25519@openssh.com,ssh-rsa,ssh-ed25519', + 'MACs hmac-sha1,hmac-sha2-256,hmac-sha2-512', + 'KexAlgorithms diffie-hellman-group14-sha1,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521', + 'RekeyLimit 1024M 60M' + ] + tmp_sshd_conf = read_file(SSHD_CONF) + + for line in ssh_lines: + self.assertIn(line, tmp_sshd_conf) + + if __name__ == '__main__': unittest.main(verbosity=2) -- cgit v1.2.3 From 6acf41ea7d11d549cb4453f9aa6f66aaa121aa5e Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Thu, 20 Oct 2022 11:42:38 +0000 Subject: T4763: Use nat.py for show nat destination statistics Use nat.py instead of old op-mode script --- op-mode-definitions/nat.xml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/op-mode-definitions/nat.xml.in b/op-mode-definitions/nat.xml.in index ce0544390..50abb1555 100644 --- a/op-mode-definitions/nat.xml.in +++ b/op-mode-definitions/nat.xml.in @@ -64,7 +64,7 @@ Show statistics for configured destination NAT rules - ${vyos_op_scripts_dir}/show_nat_statistics.py --destination + ${vyos_op_scripts_dir}/nat.py show_statistics --direction destination --family inet -- cgit v1.2.3 From 36c475ec3524739f9ae49420e60a57a5266fa575 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Thu, 20 Oct 2022 09:14:53 -0400 Subject: T4765: normalize dict fields in op mode ouputs --- python/vyos/opmode.py | 41 +++++++++++++++++++++++++++++++++++++++++ src/tests/test_op_mode.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 src/tests/test_op_mode.py diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index 7e3545c87..ac9c0c353 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -44,6 +44,13 @@ class PermissionDenied(Error): """ pass +class InternalError(Error): + """ Any situation when VyOS detects that it could not perform + an operation correctly due to logic errors in its own code + or errors in underlying software. + """ + pass + def _is_op_mode_function_name(name): if re.match(r"^(show|clear|reset|restart)", name): @@ -93,6 +100,39 @@ def _get_arg_type(t): else: return t +def _normalize_field_name(name): + # Replace all separators with underscores + name = re.sub(r'(\s|[\(\)\[\]\{\}\-\.\,:\"\'\`])+', '_', name) + + # Replace specific characters with textual descriptions + name = re.sub(r'@', '_at_', name) + name = re.sub(r'%', '_percentage_', name) + name = re.sub(r'~', '_tilde_', name) + + # Force all letters to lowercase + name = name.lower() + + # Remove leading and trailing underscores, if any + name = re.sub(r'(^(_+)(?=[^_])|_+$)', '', name) + + # Ensure there are only single underscores + name = re.sub(r'_+', '_', name) + + return name + +def _normalize_field_names(old_dict): + new_dict = {} + + for key in old_dict: + new_key = _normalize_field_name(key) + new_dict[new_key] = old_dict[key] + + # Sanity check + if len(old_dict) != len(new_dict): + raise InternalError("Dictionary fields do not allow unique normalization") + else: + return new_dict + def run(module): from argparse import ArgumentParser @@ -145,6 +185,7 @@ def run(module): # they may return human-formatted output # or a raw dict that we need to serialize in JSON for printing res = func(**args) + res = _normalize_field_names(res) if not args["raw"]: return res else: diff --git a/src/tests/test_op_mode.py b/src/tests/test_op_mode.py new file mode 100644 index 000000000..4786357c5 --- /dev/null +++ b/src/tests/test_op_mode.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from unittest import TestCase + +import vyos.opmode + +class TestVyOSOpMode(TestCase): + def test_field_name_normalization(self): + from vyos.opmode import _normalize_field_name + + self.assertEqual(_normalize_field_name(" foo bar "), "foo_bar") + self.assertEqual(_normalize_field_name("foo-bar"), "foo_bar") + self.assertEqual(_normalize_field_name("foo (bar) baz"), "foo_bar_baz") + self.assertEqual(_normalize_field_name("load%"), "load_percentage") + + def test_dict_fields_normalization_non_unique(self): + from vyos.opmode import _normalize_field_names + + # Space and dot are both replaced by an underscore, + # so dicts like this cannor be normalized uniquely + data = {"foo bar": True, "foo.bar": False} + + with self.assertRaises(vyos.opmode.InternalError): + _normalize_field_names(data) + + def test_dict_fields_normalization(self): + from vyos.opmode import _normalize_field_names + + data = {"foo bar": True, "bar-baz": False} + self.assertEqual(_normalize_field_names(data), {"foo_bar": True, "bar_baz": False}) -- cgit v1.2.3 From 40cf5f7c1b8d8ae27b5c404807294f8a2a76842d Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 20 Oct 2022 16:25:51 -0500 Subject: T4765: normalize fields only if 'raw' is true; output must be dict --- python/vyos/opmode.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index ac9c0c353..5f9c2c1ce 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -185,10 +185,12 @@ def run(module): # they may return human-formatted output # or a raw dict that we need to serialize in JSON for printing res = func(**args) - res = _normalize_field_names(res) if not args["raw"]: return res else: + if not isinstance(res, dict): + raise InternalError("'raw' output of 'show_*' command must be a dict") + res = _normalize_field_names(res) from json import dumps return dumps(res, indent=4) else: -- cgit v1.2.3 From 89fbe73b9fb9ad178a2a35bdf9c7c477dc72f054 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 21 Oct 2022 08:41:26 -0500 Subject: graphql: T4768: change name of api child node from 'gql' to 'graphql' --- interface-definitions/https.xml.in | 2 +- .../include/version/https-version.xml.i | 2 +- smoketest/scripts/cli/test_service_https.py | 10 ++-- src/conf_mode/http-api.py | 2 +- src/migration-scripts/https/3-to-4 | 53 ++++++++++++++++++++++ src/services/vyos-http-api-server | 10 ++-- 6 files changed, 66 insertions(+), 13 deletions(-) create mode 100755 src/migration-scripts/https/3-to-4 diff --git a/interface-definitions/https.xml.in b/interface-definitions/https.xml.in index d096c4ff1..28656b594 100644 --- a/interface-definitions/https.xml.in +++ b/interface-definitions/https.xml.in @@ -107,7 +107,7 @@ - + GraphQL support diff --git a/interface-definitions/include/version/https-version.xml.i b/interface-definitions/include/version/https-version.xml.i index 586083649..111076974 100644 --- a/interface-definitions/include/version/https-version.xml.i +++ b/interface-definitions/include/version/https-version.xml.i @@ -1,3 +1,3 @@ - + diff --git a/smoketest/scripts/cli/test_service_https.py b/smoketest/scripts/cli/test_service_https.py index 72c1d4e43..719125f0f 100755 --- a/smoketest/scripts/cli/test_service_https.py +++ b/smoketest/scripts/cli/test_service_https.py @@ -143,10 +143,10 @@ class TestHTTPSService(VyOSUnitTestSHIM.TestCase): # caught by the resolver, and returns success 'False', so one must # check the return value. - self.cli_set(base_path + ['api', 'gql']) + self.cli_set(base_path + ['api', 'graphql']) self.cli_commit() - gql_url = f'https://{address}/graphql' + graphql_url = f'https://{address}/graphql' query_valid_key = f""" {{ @@ -160,7 +160,7 @@ class TestHTTPSService(VyOSUnitTestSHIM.TestCase): }} """ - r = request('POST', gql_url, verify=False, headers=headers, json={'query': query_valid_key}) + r = request('POST', graphql_url, verify=False, headers=headers, json={'query': query_valid_key}) success = r.json()['data']['SystemStatus']['success'] self.assertTrue(success) @@ -176,7 +176,7 @@ class TestHTTPSService(VyOSUnitTestSHIM.TestCase): } """ - r = request('POST', gql_url, verify=False, headers=headers, json={'query': query_invalid_key}) + r = request('POST', graphql_url, verify=False, headers=headers, json={'query': query_invalid_key}) success = r.json()['data']['SystemStatus']['success'] self.assertFalse(success) @@ -192,7 +192,7 @@ class TestHTTPSService(VyOSUnitTestSHIM.TestCase): } """ - r = request('POST', gql_url, verify=False, headers=headers, json={'query': query_no_key}) + r = request('POST', graphql_url, verify=False, headers=headers, json={'query': query_no_key}) self.assertEqual(r.status_code, 400) if __name__ == '__main__': diff --git a/src/conf_mode/http-api.py b/src/conf_mode/http-api.py index c196e272b..be80613c6 100755 --- a/src/conf_mode/http-api.py +++ b/src/conf_mode/http-api.py @@ -86,7 +86,7 @@ def get_config(config=None): if 'api_keys' in api_dict: keys_added = True - if 'gql' in api_dict: + if 'graphql' in api_dict: api_dict = dict_merge(defaults(base), api_dict) http_api.update(api_dict) diff --git a/src/migration-scripts/https/3-to-4 b/src/migration-scripts/https/3-to-4 new file mode 100755 index 000000000..5ee528b31 --- /dev/null +++ b/src/migration-scripts/https/3-to-4 @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# T4768 rename node 'gql' to 'graphql'. + +import sys + +from vyos.configtree import ConfigTree + +if (len(sys.argv) < 2): + print("Must specify file name!") + sys.exit(1) + +file_name = sys.argv[1] + +with open(file_name, 'r') as f: + config_file = f.read() + +config = ConfigTree(config_file) + +old_base = ['service', 'https', 'api', 'gql'] +if not config.exists(old_base): + # Nothing to do + sys.exit(0) + +new_base = ['service', 'https', 'api', 'graphql'] +config.set(new_base) + +nodes = config.list_nodes(old_base) +for node in nodes: + config.copy(old_base + [node], new_base + [node]) + +config.delete(old_base) + +try: + with open(file_name, 'w') as f: + f.write(config.to_string()) +except OSError as e: + print("Failed to save the modified config: {}".format(e)) + sys.exit(1) diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server index 4ace981ca..632c1e87d 100755 --- a/src/services/vyos-http-api-server +++ b/src/services/vyos-http-api-server @@ -688,16 +688,16 @@ if __name__ == '__main__': app.state.vyos_debug = server_config['debug'] app.state.vyos_strict = server_config['strict'] app.state.vyos_origins = server_config.get('cors', {}).get('allow_origin', []) - if 'gql' in server_config: - app.state.vyos_gql = True - if isinstance(server_config['gql'], dict) and 'introspection' in server_config['gql']: + if 'graphql' in server_config: + app.state.vyos_graphql = True + if isinstance(server_config['graphql'], dict) and 'introspection' in server_config['graphql']: app.state.vyos_introspection = True else: app.state.vyos_introspection = False else: - app.state.vyos_gql = False + app.state.vyos_graphql = False - if app.state.vyos_gql: + if app.state.vyos_graphql: graphql_init(app) try: -- cgit v1.2.3 From b6d2e0a4b08c81814cb2d9b5b611cbc3fc31dbeb Mon Sep 17 00:00:00 2001 From: create with ansible Date: Fri, 21 Oct 2022 11:50:17 -0400 Subject: T4765: support list and primitives in op mode output normalization --- python/vyos/opmode.py | 14 ++++++++++---- src/tests/test_op_mode.py | 25 +++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index 5f9c2c1ce..c9827d634 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -120,12 +120,12 @@ def _normalize_field_name(name): return name -def _normalize_field_names(old_dict): +def _normalize_dict_field_names(old_dict): new_dict = {} for key in old_dict: new_key = _normalize_field_name(key) - new_dict[new_key] = old_dict[key] + new_dict[new_key] = _normalize_field_names(old_dict[key]) # Sanity check if len(old_dict) != len(new_dict): @@ -133,6 +133,14 @@ def _normalize_field_names(old_dict): else: return new_dict +def _normalize_field_names(value): + if isinstance(value, dict): + return _normalize_dict_field_names(value) + elif isinstance(value, list): + return list(map(lambda v: _normalize_field_names(v), value)) + else: + return value + def run(module): from argparse import ArgumentParser @@ -188,8 +196,6 @@ def run(module): if not args["raw"]: return res else: - if not isinstance(res, dict): - raise InternalError("'raw' output of 'show_*' command must be a dict") res = _normalize_field_names(res) from json import dumps return dumps(res, indent=4) diff --git a/src/tests/test_op_mode.py b/src/tests/test_op_mode.py index 4786357c5..90963b3c5 100644 --- a/src/tests/test_op_mode.py +++ b/src/tests/test_op_mode.py @@ -37,8 +37,29 @@ class TestVyOSOpMode(TestCase): with self.assertRaises(vyos.opmode.InternalError): _normalize_field_names(data) - def test_dict_fields_normalization(self): + def test_dict_fields_normalization_simple_dict(self): from vyos.opmode import _normalize_field_names - data = {"foo bar": True, "bar-baz": False} + data = {"foo bar": True, "Bar-Baz": False} self.assertEqual(_normalize_field_names(data), {"foo_bar": True, "bar_baz": False}) + + def test_dict_fields_normalization_nested_dict(self): + from vyos.opmode import _normalize_field_names + + data = {"foo bar": True, "bar-baz": {"baz-quux": {"quux-xyzzy": False}}} + self.assertEqual(_normalize_field_names(data), + {"foo_bar": True, "bar_baz": {"baz_quux": {"quux_xyzzy": False}}}) + + def test_dict_fields_normalization_mixed(self): + from vyos.opmode import _normalize_field_names + + data = [{"foo bar": True, "bar-baz": [{"baz-quux": {"quux-xyzzy": [False]}}]}] + self.assertEqual(_normalize_field_names(data), + [{"foo_bar": True, "bar_baz": [{"baz_quux": {"quux_xyzzy": [False]}}]}]) + + def test_dict_fields_normalization_primitive(self): + from vyos.opmode import _normalize_field_names + + data = [1, False, "foo"] + self.assertEqual(_normalize_field_names(data), [1, False, "foo"]) + -- cgit v1.2.3 From 28b312d687291cef1e3935b7f39dc28b9e7976ef Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Thu, 20 Oct 2022 11:51:32 +0000 Subject: T4762: Add check for show nat if nat config does not exist Add check for 'show nat xxx' if nat configuration does not exist --- src/op_mode/nat.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/op_mode/nat.py b/src/op_mode/nat.py index 845dbbb2c..f899eb3dc 100755 --- a/src/op_mode/nat.py +++ b/src/op_mode/nat.py @@ -22,12 +22,18 @@ import xmltodict from sys import exit from tabulate import tabulate +from vyos.configquery import ConfigTreeQuery + from vyos.util import cmd from vyos.util import dict_search import vyos.opmode +base = 'nat' +unconf_message = 'NAT is not configured' + + def _get_xml_translation(direction, family): """ Get conntrack XML output --src-nat|--dst-nat @@ -277,6 +283,20 @@ def _get_formatted_translation(dict_data, nat_direction, family): return output +def _verify(func): + """Decorator checks if NAT config exists""" + from functools import wraps + + @wraps(func) + def _wrapper(*args, **kwargs): + config = ConfigTreeQuery() + if not config.exists(base): + raise vyos.opmode.UnconfiguredSubsystem(unconf_message) + return func(*args, **kwargs) + return _wrapper + + +@_verify def show_rules(raw: bool, direction: str, family: str): nat_rules = _get_raw_data_rules(direction, family) if raw: @@ -285,6 +305,7 @@ def show_rules(raw: bool, direction: str, family: str): return _get_formatted_output_rules(nat_rules, direction, family) +@_verify def show_statistics(raw: bool, direction: str, family: str): nat_statistics = _get_raw_data_rules(direction, family) if raw: @@ -293,6 +314,7 @@ def show_statistics(raw: bool, direction: str, family: str): return _get_formatted_output_statistics(nat_statistics, direction) +@_verify def show_translations(raw: bool, direction: str, family: str): family = 'ipv6' if family == 'inet6' else 'ipv4' nat_translation = _get_raw_translation(direction, family) -- cgit v1.2.3 From 1c05f8b09bf5727a6e0c0b124f77684635dcf9a8 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Mon, 24 Oct 2022 09:33:46 -0500 Subject: route: T4772: return list of dicts in 'raw' output --- src/op_mode/route.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/op_mode/route.py b/src/op_mode/route.py index e1eee5bbf..d11b00ba0 100755 --- a/src/op_mode/route.py +++ b/src/op_mode/route.py @@ -83,7 +83,12 @@ def show(raw: bool, if raw: from json import loads - return loads(output) + d = loads(output) + collect = [] + for k,_ in d.items(): + for l in d[k]: + collect.append(l) + return collect else: return output -- cgit v1.2.3 From 093ac258c11894b07afd9e85a61778d23e356718 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 23 Oct 2022 11:05:33 -0500 Subject: graphql: T4574: call all schema definition generation on init --- src/services/api/graphql/bindings.py | 4 + .../api/graphql/graphql/schema/composite.graphql | 18 ---- .../graphql/graphql/schema/configsession.graphql | 115 --------------------- .../api/graphql/utils/schema_from_composite.py | 8 +- .../graphql/utils/schema_from_config_session.py | 8 +- 5 files changed, 12 insertions(+), 141 deletions(-) delete mode 100644 src/services/api/graphql/graphql/schema/composite.graphql delete mode 100644 src/services/api/graphql/graphql/schema/configsession.graphql diff --git a/src/services/api/graphql/bindings.py b/src/services/api/graphql/bindings.py index 0b1260912..c5c4560dd 100644 --- a/src/services/api/graphql/bindings.py +++ b/src/services/api/graphql/bindings.py @@ -19,12 +19,16 @@ from . graphql.mutations import mutation from . graphql.directives import directives_dict from . graphql.errors import op_mode_error from . utils.schema_from_op_mode import generate_op_mode_definitions +from . utils.schema_from_config_session import generate_config_session_definitions +from . utils.schema_from_composite import generate_composite_definitions from ariadne import make_executable_schema, load_schema_from_path, snake_case_fallback_resolvers def generate_schema(): api_schema_dir = vyos.defaults.directories['api_schema'] generate_op_mode_definitions() + generate_config_session_definitions() + generate_composite_definitions() type_defs = load_schema_from_path(api_schema_dir) diff --git a/src/services/api/graphql/graphql/schema/composite.graphql b/src/services/api/graphql/graphql/schema/composite.graphql deleted file mode 100644 index 717fbd89d..000000000 --- a/src/services/api/graphql/graphql/schema/composite.graphql +++ /dev/null @@ -1,18 +0,0 @@ - -input SystemStatusInput { - key: String! -} - -type SystemStatus { - result: Generic -} - -type SystemStatusResult { - data: SystemStatus - success: Boolean! - errors: [String] -} - -extend type Query { - SystemStatus(data: SystemStatusInput) : SystemStatusResult @compositequery -} \ No newline at end of file diff --git a/src/services/api/graphql/graphql/schema/configsession.graphql b/src/services/api/graphql/graphql/schema/configsession.graphql deleted file mode 100644 index b1deac4b3..000000000 --- a/src/services/api/graphql/graphql/schema/configsession.graphql +++ /dev/null @@ -1,115 +0,0 @@ - -input ShowConfigInput { - key: String! - path: [String!]! - configFormat: String = null -} - -type ShowConfig { - result: Generic -} - -type ShowConfigResult { - data: ShowConfig - success: Boolean! - errors: [String] -} - -extend type Query { - ShowConfig(data: ShowConfigInput) : ShowConfigResult @configsessionquery -} - -input ShowInput { - key: String! - path: [String!]! -} - -type Show { - result: Generic -} - -type ShowResult { - data: Show - success: Boolean! - errors: [String] -} - -extend type Query { - Show(data: ShowInput) : ShowResult @configsessionquery -} - -input SaveConfigFileInput { - key: String! - fileName: String = null -} - -type SaveConfigFile { - result: Generic -} - -type SaveConfigFileResult { - data: SaveConfigFile - success: Boolean! - errors: [String] -} - -extend type Mutation { - SaveConfigFile(data: SaveConfigFileInput) : SaveConfigFileResult @configsessionmutation -} - -input LoadConfigFileInput { - key: String! - fileName: String! -} - -type LoadConfigFile { - result: Generic -} - -type LoadConfigFileResult { - data: LoadConfigFile - success: Boolean! - errors: [String] -} - -extend type Mutation { - LoadConfigFile(data: LoadConfigFileInput) : LoadConfigFileResult @configsessionmutation -} - -input AddSystemImageInput { - key: String! - location: String! -} - -type AddSystemImage { - result: Generic -} - -type AddSystemImageResult { - data: AddSystemImage - success: Boolean! - errors: [String] -} - -extend type Mutation { - AddSystemImage(data: AddSystemImageInput) : AddSystemImageResult @configsessionmutation -} - -input DeleteSystemImageInput { - key: String! - name: String! -} - -type DeleteSystemImage { - result: Generic -} - -type DeleteSystemImageResult { - data: DeleteSystemImage - success: Boolean! - errors: [String] -} - -extend type Mutation { - DeleteSystemImage(data: DeleteSystemImageInput) : DeleteSystemImageResult @configsessionmutation -} \ No newline at end of file diff --git a/src/services/api/graphql/utils/schema_from_composite.py b/src/services/api/graphql/utils/schema_from_composite.py index f9983cd98..d5e0ecdf6 100755 --- a/src/services/api/graphql/utils/schema_from_composite.py +++ b/src/services/api/graphql/utils/schema_from_composite.py @@ -23,13 +23,15 @@ import json from inspect import signature, getmembers, isfunction, isclass, getmro from jinja2 import Template +from vyos.defaults import directories if __package__ is None or __package__ == '': from util import snake_to_pascal_case, map_type_name + from composite_function import queries, mutations else: from . util import snake_to_pascal_case, map_type_name + from . composite_function import queries, mutations -# this will be run locally before the build -SCHEMA_PATH = '../graphql/schema' +SCHEMA_PATH = directories['api_schema'] schema_data: dict = {'schema_name': '', 'schema_fields': []} @@ -100,8 +102,6 @@ def create_schema(func_name: str, func: callable, template: str) -> str: return res def generate_composite_definitions(): - from composite_function import queries, mutations - results = [] for name,func in queries.items(): res = create_schema(name, func, query_template) diff --git a/src/services/api/graphql/utils/schema_from_config_session.py b/src/services/api/graphql/utils/schema_from_config_session.py index ea78aaf88..b6609357e 100755 --- a/src/services/api/graphql/utils/schema_from_config_session.py +++ b/src/services/api/graphql/utils/schema_from_config_session.py @@ -23,13 +23,15 @@ import json from inspect import signature, getmembers, isfunction, isclass, getmro from jinja2 import Template +from vyos.defaults import directories if __package__ is None or __package__ == '': from util import snake_to_pascal_case, map_type_name + from config_session_function import queries, mutations else: from . util import snake_to_pascal_case, map_type_name + from . config_session_function import queries, mutations -# this will be run locally before the build -SCHEMA_PATH = '../graphql/schema' +SCHEMA_PATH = directories['api_schema'] schema_data: dict = {'schema_name': '', 'schema_fields': []} @@ -100,8 +102,6 @@ def create_schema(func_name: str, func: callable, template: str) -> str: return res def generate_config_session_definitions(): - from config_session_function import queries, mutations - results = [] for name,func in queries.items(): res = create_schema(name, func, query_template) -- cgit v1.2.3 From 7038b761302be2ec90338981830b8cd7cf887381 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 23 Oct 2022 11:06:03 -0500 Subject: graphql: T4574: reorganize directory structure for clarity --- src/services/api/graphql/__init__.py | 0 src/services/api/graphql/bindings.py | 6 +- .../api/graphql/generate/composite_function.py | 11 ++ .../graphql/generate/config_session_function.py | 28 ++++ .../api/graphql/generate/schema_from_composite.py | 121 ++++++++++++++ .../graphql/generate/schema_from_config_session.py | 121 ++++++++++++++ .../api/graphql/generate/schema_from_op_mode.py | 185 +++++++++++++++++++++ src/services/api/graphql/graphql/mutations.py | 2 +- src/services/api/graphql/graphql/queries.py | 2 +- src/services/api/graphql/key_auth.py | 18 -- src/services/api/graphql/libs/key_auth.py | 18 ++ src/services/api/graphql/libs/op_mode.py | 100 +++++++++++ .../api/graphql/session/composite/system_status.py | 2 +- src/services/api/graphql/session/session.py | 2 +- .../api/graphql/utils/composite_function.py | 11 -- .../api/graphql/utils/config_session_function.py | 28 ---- .../api/graphql/utils/schema_from_composite.py | 119 ------------- .../graphql/utils/schema_from_config_session.py | 119 ------------- .../api/graphql/utils/schema_from_op_mode.py | 183 -------------------- src/services/api/graphql/utils/util.py | 100 ----------- 20 files changed, 591 insertions(+), 585 deletions(-) create mode 100644 src/services/api/graphql/__init__.py create mode 100644 src/services/api/graphql/generate/composite_function.py create mode 100644 src/services/api/graphql/generate/config_session_function.py create mode 100755 src/services/api/graphql/generate/schema_from_composite.py create mode 100755 src/services/api/graphql/generate/schema_from_config_session.py create mode 100755 src/services/api/graphql/generate/schema_from_op_mode.py delete mode 100644 src/services/api/graphql/key_auth.py create mode 100644 src/services/api/graphql/libs/key_auth.py create mode 100644 src/services/api/graphql/libs/op_mode.py delete mode 100644 src/services/api/graphql/utils/composite_function.py delete mode 100644 src/services/api/graphql/utils/config_session_function.py delete mode 100755 src/services/api/graphql/utils/schema_from_composite.py delete mode 100755 src/services/api/graphql/utils/schema_from_config_session.py delete mode 100755 src/services/api/graphql/utils/schema_from_op_mode.py delete mode 100644 src/services/api/graphql/utils/util.py diff --git a/src/services/api/graphql/__init__.py b/src/services/api/graphql/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/services/api/graphql/bindings.py b/src/services/api/graphql/bindings.py index c5c4560dd..d3cff21c7 100644 --- a/src/services/api/graphql/bindings.py +++ b/src/services/api/graphql/bindings.py @@ -18,9 +18,9 @@ from . graphql.queries import query from . graphql.mutations import mutation from . graphql.directives import directives_dict from . graphql.errors import op_mode_error -from . utils.schema_from_op_mode import generate_op_mode_definitions -from . utils.schema_from_config_session import generate_config_session_definitions -from . utils.schema_from_composite import generate_composite_definitions +from . generate.schema_from_op_mode import generate_op_mode_definitions +from . generate.schema_from_config_session import generate_config_session_definitions +from . generate.schema_from_composite import generate_composite_definitions from ariadne import make_executable_schema, load_schema_from_path, snake_case_fallback_resolvers def generate_schema(): diff --git a/src/services/api/graphql/generate/composite_function.py b/src/services/api/graphql/generate/composite_function.py new file mode 100644 index 000000000..bc9d80fbb --- /dev/null +++ b/src/services/api/graphql/generate/composite_function.py @@ -0,0 +1,11 @@ +# typing information for composite functions: those that invoke several +# elementary requests, and return the result as a single dict +import typing + +def system_status(): + pass + +queries = {'system_status': system_status} + +mutations = {} + diff --git a/src/services/api/graphql/generate/config_session_function.py b/src/services/api/graphql/generate/config_session_function.py new file mode 100644 index 000000000..fc0dd7a87 --- /dev/null +++ b/src/services/api/graphql/generate/config_session_function.py @@ -0,0 +1,28 @@ +# typing information for native configsession functions; used to generate +# schema definition files +import typing + +def show_config(path: list[str], configFormat: typing.Optional[str]): + pass + +def show(path: list[str]): + pass + +queries = {'show_config': show_config, + 'show': show} + +def save_config_file(fileName: typing.Optional[str]): + pass +def load_config_file(fileName: str): + pass +def add_system_image(location: str): + pass +def delete_system_image(name: str): + pass + +mutations = {'save_config_file': save_config_file, + 'load_config_file': load_config_file, + 'add_system_image': add_system_image, + 'delete_system_image': delete_system_image} + + diff --git a/src/services/api/graphql/generate/schema_from_composite.py b/src/services/api/graphql/generate/schema_from_composite.py new file mode 100755 index 000000000..7187047a0 --- /dev/null +++ b/src/services/api/graphql/generate/schema_from_composite.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# A utility to generate GraphQL schema defintions from typing information of +# composite functions comprising several requests. + +import os +import sys +import json +from inspect import signature, getmembers, isfunction, isclass, getmro +from jinja2 import Template + +from vyos.defaults import directories +if __package__ is None or __package__ == '': + sys.path.append("/usr/libexec/vyos/services/api") + from graphql.libs.op_mode import snake_to_pascal_case, map_type_name + from composite_function import queries, mutations +else: + from .. libs.op_mode import snake_to_pascal_case, map_type_name + from . composite_function import queries, mutations + +SCHEMA_PATH = directories['api_schema'] + +schema_data: dict = {'schema_name': '', + 'schema_fields': []} + +query_template = """ +input {{ schema_name }}Input { + key: String! + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} + +type {{ schema_name }} { + result: Generic +} + +type {{ schema_name }}Result { + data: {{ schema_name }} + success: Boolean! + errors: [String] +} + +extend type Query { + {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @compositequery +} +""" + +mutation_template = """ +input {{ schema_name }}Input { + key: String! + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} + +type {{ schema_name }} { + result: Generic +} + +type {{ schema_name }}Result { + data: {{ schema_name }} + success: Boolean! + errors: [String] +} + +extend type Mutation { + {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @compositemutation +} +""" + +def create_schema(func_name: str, func: callable, template: str) -> str: + sig = signature(func) + + field_dict = {} + for k in sig.parameters: + field_dict[sig.parameters[k].name] = map_type_name(sig.parameters[k].annotation) + + schema_fields = [] + for k,v in field_dict.items(): + schema_fields.append(k+': '+v) + + schema_data['schema_name'] = snake_to_pascal_case(func_name) + schema_data['schema_fields'] = schema_fields + + j2_template = Template(template) + res = j2_template.render(schema_data) + + return res + +def generate_composite_definitions(): + results = [] + for name,func in queries.items(): + res = create_schema(name, func, query_template) + results.append(res) + + for name,func in mutations.items(): + res = create_schema(name, func, mutation_template) + results.append(res) + + out = '\n'.join(results) + with open(f'{SCHEMA_PATH}/composite.graphql', 'w') as f: + f.write(out) + +if __name__ == '__main__': + generate_composite_definitions() diff --git a/src/services/api/graphql/generate/schema_from_config_session.py b/src/services/api/graphql/generate/schema_from_config_session.py new file mode 100755 index 000000000..cf69cbafd --- /dev/null +++ b/src/services/api/graphql/generate/schema_from_config_session.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# A utility to generate GraphQL schema defintions from typing information of +# (wrappers of) native configsession functions. + +import os +import sys +import json +from inspect import signature, getmembers, isfunction, isclass, getmro +from jinja2 import Template + +from vyos.defaults import directories +if __package__ is None or __package__ == '': + sys.path.append("/usr/libexec/vyos/services/api") + from graphql.libs.op_mode import snake_to_pascal_case, map_type_name + from config_session_function import queries, mutations +else: + from .. libs.op_mode import snake_to_pascal_case, map_type_name + from . config_session_function import queries, mutations + +SCHEMA_PATH = directories['api_schema'] + +schema_data: dict = {'schema_name': '', + 'schema_fields': []} + +query_template = """ +input {{ schema_name }}Input { + key: String! + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} + +type {{ schema_name }} { + result: Generic +} + +type {{ schema_name }}Result { + data: {{ schema_name }} + success: Boolean! + errors: [String] +} + +extend type Query { + {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @configsessionquery +} +""" + +mutation_template = """ +input {{ schema_name }}Input { + key: String! + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} + +type {{ schema_name }} { + result: Generic +} + +type {{ schema_name }}Result { + data: {{ schema_name }} + success: Boolean! + errors: [String] +} + +extend type Mutation { + {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @configsessionmutation +} +""" + +def create_schema(func_name: str, func: callable, template: str) -> str: + sig = signature(func) + + field_dict = {} + for k in sig.parameters: + field_dict[sig.parameters[k].name] = map_type_name(sig.parameters[k].annotation) + + schema_fields = [] + for k,v in field_dict.items(): + schema_fields.append(k+': '+v) + + schema_data['schema_name'] = snake_to_pascal_case(func_name) + schema_data['schema_fields'] = schema_fields + + j2_template = Template(template) + res = j2_template.render(schema_data) + + return res + +def generate_config_session_definitions(): + results = [] + for name,func in queries.items(): + res = create_schema(name, func, query_template) + results.append(res) + + for name,func in mutations.items(): + res = create_schema(name, func, mutation_template) + results.append(res) + + out = '\n'.join(results) + with open(f'{SCHEMA_PATH}/configsession.graphql', 'w') as f: + f.write(out) + +if __name__ == '__main__': + generate_config_session_definitions() diff --git a/src/services/api/graphql/generate/schema_from_op_mode.py b/src/services/api/graphql/generate/schema_from_op_mode.py new file mode 100755 index 000000000..a88816b34 --- /dev/null +++ b/src/services/api/graphql/generate/schema_from_op_mode.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# A utility to generate GraphQL schema defintions from standardized op-mode +# scripts. + +import os +import sys +import json +from inspect import signature, getmembers, isfunction, isclass, getmro +from jinja2 import Template + +from vyos.defaults import directories +if __package__ is None or __package__ == '': + sys.path.append("/usr/libexec/vyos/services/api") + from graphql.libs.op_mode import load_as_module, is_op_mode_function_name, is_show_function_name + from graphql.libs.op_mode import snake_to_pascal_case, map_type_name +else: + from .. libs.op_mode import load_as_module, is_op_mode_function_name, is_show_function_name + from .. libs.op_mode import snake_to_pascal_case, map_type_name + +OP_MODE_PATH = directories['op_mode'] +SCHEMA_PATH = directories['api_schema'] +DATA_DIR = directories['data'] + +op_mode_include_file = os.path.join(DATA_DIR, 'op-mode-standardized.json') +op_mode_error_schema = 'op_mode_error.graphql' + +schema_data: dict = {'schema_name': '', + 'schema_fields': []} + +query_template = """ +input {{ schema_name }}Input { + key: String! + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} + +type {{ schema_name }} { + result: Generic +} + +type {{ schema_name }}Result { + data: {{ schema_name }} + op_mode_error: OpModeError + success: Boolean! + errors: [String] +} + +extend type Query { + {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @genopquery +} +""" + +mutation_template = """ +input {{ schema_name }}Input { + key: String! + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} + +type {{ schema_name }} { + result: Generic +} + +type {{ schema_name }}Result { + data: {{ schema_name }} + op_mode_error: OpModeError + success: Boolean! + errors: [String] +} + +extend type Mutation { + {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @genopmutation +} +""" + +error_template = """ +interface OpModeError { + name: String! + message: String! + vyos_code: Int! +} +{% for name in error_names %} +type {{ name }} implements OpModeError { + name: String! + message: String! + vyos_code: Int! +} +{%- endfor %} +""" + +def create_schema(func_name: str, base_name: str, func: callable) -> str: + sig = signature(func) + + field_dict = {} + for k in sig.parameters: + field_dict[sig.parameters[k].name] = map_type_name(sig.parameters[k].annotation) + + # It is assumed that if one is generating a schema for a 'show_*' + # function, that 'get_raw_data' is present and 'raw' is desired. + if 'raw' in list(field_dict): + del field_dict['raw'] + + schema_fields = [] + for k,v in field_dict.items(): + schema_fields.append(k+': '+v) + + schema_data['schema_name'] = snake_to_pascal_case(func_name + '_' + base_name) + schema_data['schema_fields'] = schema_fields + + if is_show_function_name(func_name): + j2_template = Template(query_template) + else: + j2_template = Template(mutation_template) + + res = j2_template.render(schema_data) + + return res + +def create_error_schema(): + from vyos import opmode + + e = Exception + err_types = getmembers(opmode, isclass) + err_types = [k for k in err_types if issubclass(k[1], e)] + # drop base class, to be replaced by interface type. Find the class + # programmatically, in case the base class name changes. + for i in range(len(err_types)): + if err_types[i][1] in getmro(err_types[i-1][1]): + del err_types[i] + break + err_names = [k[0] for k in err_types] + error_data = {'error_names': err_names} + j2_template = Template(error_template) + res = j2_template.render(error_data) + + return res + +def generate_op_mode_definitions(): + out = create_error_schema() + with open(f'{SCHEMA_PATH}/{op_mode_error_schema}', 'w') as f: + f.write(out) + + with open(op_mode_include_file) as f: + op_mode_files = json.load(f) + + for file in op_mode_files: + basename = os.path.splitext(file)[0].replace('-', '_') + module = load_as_module(basename, os.path.join(OP_MODE_PATH, file)) + + funcs = getmembers(module, isfunction) + funcs = list(filter(lambda ft: is_op_mode_function_name(ft[0]), funcs)) + + funcs_dict = {} + for (name, thunk) in funcs: + funcs_dict[name] = thunk + + results = [] + for name,func in funcs_dict.items(): + res = create_schema(name, basename, func) + results.append(res) + + out = '\n'.join(results) + with open(f'{SCHEMA_PATH}/{basename}.graphql', 'w') as f: + f.write(out) + +if __name__ == '__main__': + generate_op_mode_definitions() diff --git a/src/services/api/graphql/graphql/mutations.py b/src/services/api/graphql/graphql/mutations.py index 32da0eeb7..f0c8b438f 100644 --- a/src/services/api/graphql/graphql/mutations.py +++ b/src/services/api/graphql/graphql/mutations.py @@ -20,7 +20,7 @@ from graphql import GraphQLResolveInfo from makefun import with_signature from .. import state -from .. import key_auth +from .. libs import key_auth from api.graphql.session.session import Session from api.graphql.session.errors.op_mode_errors import op_mode_err_msg, op_mode_err_code from vyos.opmode import Error as OpModeError diff --git a/src/services/api/graphql/graphql/queries.py b/src/services/api/graphql/graphql/queries.py index 791b0d3e0..13eb59ae4 100644 --- a/src/services/api/graphql/graphql/queries.py +++ b/src/services/api/graphql/graphql/queries.py @@ -20,7 +20,7 @@ from graphql import GraphQLResolveInfo from makefun import with_signature from .. import state -from .. import key_auth +from .. libs import key_auth from api.graphql.session.session import Session from api.graphql.session.errors.op_mode_errors import op_mode_err_msg, op_mode_err_code from vyos.opmode import Error as OpModeError diff --git a/src/services/api/graphql/key_auth.py b/src/services/api/graphql/key_auth.py deleted file mode 100644 index f756ed6d8..000000000 --- a/src/services/api/graphql/key_auth.py +++ /dev/null @@ -1,18 +0,0 @@ - -from . import state - -def check_auth(key_list, key): - if not key_list: - return None - key_id = None - for k in key_list: - if k['key'] == key: - key_id = k['id'] - return key_id - -def auth_required(key): - api_keys = None - api_keys = state.settings['app'].state.vyos_keys - key_id = check_auth(api_keys, key) - state.settings['app'].state.vyos_id = key_id - return key_id diff --git a/src/services/api/graphql/libs/key_auth.py b/src/services/api/graphql/libs/key_auth.py new file mode 100644 index 000000000..2db0f7d48 --- /dev/null +++ b/src/services/api/graphql/libs/key_auth.py @@ -0,0 +1,18 @@ + +from .. import state + +def check_auth(key_list, key): + if not key_list: + return None + key_id = None + for k in key_list: + if k['key'] == key: + key_id = k['id'] + return key_id + +def auth_required(key): + api_keys = None + api_keys = state.settings['app'].state.vyos_keys + key_id = check_auth(api_keys, key) + state.settings['app'].state.vyos_id = key_id + return key_id diff --git a/src/services/api/graphql/libs/op_mode.py b/src/services/api/graphql/libs/op_mode.py new file mode 100644 index 000000000..da2bcdb5b --- /dev/null +++ b/src/services/api/graphql/libs/op_mode.py @@ -0,0 +1,100 @@ +# Copyright 2022 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see . + +import os +import re +import typing +import importlib.util + +from vyos.defaults import directories + +def load_as_module(name: str, path: str): + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +def load_op_mode_as_module(name: str): + path = os.path.join(directories['op_mode'], name) + name = os.path.splitext(name)[0].replace('-', '_') + return load_as_module(name, path) + +def is_op_mode_function_name(name): + if re.match(r"^(show|clear|reset|restart)", name): + return True + return False + +def is_show_function_name(name): + if re.match(r"^show", name): + return True + return False + +def _nth_split(delim: str, n: int, s: str): + groups = s.split(delim) + l = len(groups) + if n > l-1 or n < 1: + return (s, '') + return (delim.join(groups[:n]), delim.join(groups[n:])) + +def _nth_rsplit(delim: str, n: int, s: str): + groups = s.split(delim) + l = len(groups) + if n > l-1 or n < 1: + return (s, '') + return (delim.join(groups[:l-n]), delim.join(groups[l-n:])) + +# Since we have mangled possible hyphens in the file name while constructing +# the snake case of the query/mutation name, we will need to recover the +# file name by searching with mangling: +def _filter_on_mangled(test): + def func(elem): + mangle = os.path.splitext(elem)[0].replace('-', '_') + return test == mangle + return func + +# Find longest name in concatenated string that matches the basename of an +# op-mode script. Should one prefer to concatenate in the reverse order +# (script_name + '_' + function_name), use _nth_rsplit. +def split_compound_op_mode_name(name: str, files: list): + for i in range(1, name.count('_') + 1): + pair = _nth_split('_', i, name) + f = list(filter(_filter_on_mangled(pair[1]), files)) + if f: + pair = (pair[0], f[0]) + return pair + return (name, '') + +def snake_to_pascal_case(name: str) -> str: + res = ''.join(map(str.title, name.split('_'))) + return res + +def map_type_name(type_name: type, optional: bool = False) -> str: + if type_name == str: + return 'String!' if not optional else 'String = null' + if type_name == int: + return 'Int!' if not optional else 'Int = null' + if type_name == bool: + return 'Boolean!' if not optional else 'Boolean = false' + if typing.get_origin(type_name) == list: + if not optional: + return f'[{map_type_name(typing.get_args(type_name)[0])}]!' + return f'[{map_type_name(typing.get_args(type_name)[0])}]' + # typing.Optional is typing.Union[_, NoneType] + if (typing.get_origin(type_name) is typing.Union and + typing.get_args(type_name)[1] == type(None)): + return f'{map_type_name(typing.get_args(type_name)[0], optional=True)}' + + # scalar 'Generic' is defined in schema.graphql + return 'Generic' diff --git a/src/services/api/graphql/session/composite/system_status.py b/src/services/api/graphql/session/composite/system_status.py index 3c1a3d45b..d809f32e3 100755 --- a/src/services/api/graphql/session/composite/system_status.py +++ b/src/services/api/graphql/session/composite/system_status.py @@ -23,7 +23,7 @@ import importlib.util from vyos.defaults import directories -from api.graphql.utils.util import load_op_mode_as_module +from api.graphql.libs.op_mode import load_op_mode_as_module def get_system_version() -> dict: show_version = load_op_mode_as_module('version.py') diff --git a/src/services/api/graphql/session/session.py b/src/services/api/graphql/session/session.py index f990e63d0..c2c1db1df 100644 --- a/src/services/api/graphql/session/session.py +++ b/src/services/api/graphql/session/session.py @@ -24,7 +24,7 @@ from vyos.defaults import directories from vyos.template import render from vyos.opmode import Error as OpModeError -from api.graphql.utils.util import load_op_mode_as_module, split_compound_op_mode_name +from api.graphql.libs.op_mode import load_op_mode_as_module, split_compound_op_mode_name op_mode_include_file = os.path.join(directories['data'], 'op-mode-standardized.json') diff --git a/src/services/api/graphql/utils/composite_function.py b/src/services/api/graphql/utils/composite_function.py deleted file mode 100644 index bc9d80fbb..000000000 --- a/src/services/api/graphql/utils/composite_function.py +++ /dev/null @@ -1,11 +0,0 @@ -# typing information for composite functions: those that invoke several -# elementary requests, and return the result as a single dict -import typing - -def system_status(): - pass - -queries = {'system_status': system_status} - -mutations = {} - diff --git a/src/services/api/graphql/utils/config_session_function.py b/src/services/api/graphql/utils/config_session_function.py deleted file mode 100644 index fc0dd7a87..000000000 --- a/src/services/api/graphql/utils/config_session_function.py +++ /dev/null @@ -1,28 +0,0 @@ -# typing information for native configsession functions; used to generate -# schema definition files -import typing - -def show_config(path: list[str], configFormat: typing.Optional[str]): - pass - -def show(path: list[str]): - pass - -queries = {'show_config': show_config, - 'show': show} - -def save_config_file(fileName: typing.Optional[str]): - pass -def load_config_file(fileName: str): - pass -def add_system_image(location: str): - pass -def delete_system_image(name: str): - pass - -mutations = {'save_config_file': save_config_file, - 'load_config_file': load_config_file, - 'add_system_image': add_system_image, - 'delete_system_image': delete_system_image} - - diff --git a/src/services/api/graphql/utils/schema_from_composite.py b/src/services/api/graphql/utils/schema_from_composite.py deleted file mode 100755 index d5e0ecdf6..000000000 --- a/src/services/api/graphql/utils/schema_from_composite.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2022 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# -# A utility to generate GraphQL schema defintions from typing information of -# composite functions comprising several requests. - -import os -import json -from inspect import signature, getmembers, isfunction, isclass, getmro -from jinja2 import Template - -from vyos.defaults import directories -if __package__ is None or __package__ == '': - from util import snake_to_pascal_case, map_type_name - from composite_function import queries, mutations -else: - from . util import snake_to_pascal_case, map_type_name - from . composite_function import queries, mutations - -SCHEMA_PATH = directories['api_schema'] - -schema_data: dict = {'schema_name': '', - 'schema_fields': []} - -query_template = """ -input {{ schema_name }}Input { - key: String! - {%- for field_entry in schema_fields %} - {{ field_entry }} - {%- endfor %} -} - -type {{ schema_name }} { - result: Generic -} - -type {{ schema_name }}Result { - data: {{ schema_name }} - success: Boolean! - errors: [String] -} - -extend type Query { - {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @compositequery -} -""" - -mutation_template = """ -input {{ schema_name }}Input { - key: String! - {%- for field_entry in schema_fields %} - {{ field_entry }} - {%- endfor %} -} - -type {{ schema_name }} { - result: Generic -} - -type {{ schema_name }}Result { - data: {{ schema_name }} - success: Boolean! - errors: [String] -} - -extend type Mutation { - {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @compositemutation -} -""" - -def create_schema(func_name: str, func: callable, template: str) -> str: - sig = signature(func) - - field_dict = {} - for k in sig.parameters: - field_dict[sig.parameters[k].name] = map_type_name(sig.parameters[k].annotation) - - schema_fields = [] - for k,v in field_dict.items(): - schema_fields.append(k+': '+v) - - schema_data['schema_name'] = snake_to_pascal_case(func_name) - schema_data['schema_fields'] = schema_fields - - j2_template = Template(template) - res = j2_template.render(schema_data) - - return res - -def generate_composite_definitions(): - results = [] - for name,func in queries.items(): - res = create_schema(name, func, query_template) - results.append(res) - - for name,func in mutations.items(): - res = create_schema(name, func, mutation_template) - results.append(res) - - out = '\n'.join(results) - with open(f'{SCHEMA_PATH}/composite.graphql', 'w') as f: - f.write(out) - -if __name__ == '__main__': - generate_composite_definitions() diff --git a/src/services/api/graphql/utils/schema_from_config_session.py b/src/services/api/graphql/utils/schema_from_config_session.py deleted file mode 100755 index b6609357e..000000000 --- a/src/services/api/graphql/utils/schema_from_config_session.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2022 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# -# A utility to generate GraphQL schema defintions from typing information of -# (wrappers of) native configsession functions. - -import os -import json -from inspect import signature, getmembers, isfunction, isclass, getmro -from jinja2 import Template - -from vyos.defaults import directories -if __package__ is None or __package__ == '': - from util import snake_to_pascal_case, map_type_name - from config_session_function import queries, mutations -else: - from . util import snake_to_pascal_case, map_type_name - from . config_session_function import queries, mutations - -SCHEMA_PATH = directories['api_schema'] - -schema_data: dict = {'schema_name': '', - 'schema_fields': []} - -query_template = """ -input {{ schema_name }}Input { - key: String! - {%- for field_entry in schema_fields %} - {{ field_entry }} - {%- endfor %} -} - -type {{ schema_name }} { - result: Generic -} - -type {{ schema_name }}Result { - data: {{ schema_name }} - success: Boolean! - errors: [String] -} - -extend type Query { - {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @configsessionquery -} -""" - -mutation_template = """ -input {{ schema_name }}Input { - key: String! - {%- for field_entry in schema_fields %} - {{ field_entry }} - {%- endfor %} -} - -type {{ schema_name }} { - result: Generic -} - -type {{ schema_name }}Result { - data: {{ schema_name }} - success: Boolean! - errors: [String] -} - -extend type Mutation { - {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @configsessionmutation -} -""" - -def create_schema(func_name: str, func: callable, template: str) -> str: - sig = signature(func) - - field_dict = {} - for k in sig.parameters: - field_dict[sig.parameters[k].name] = map_type_name(sig.parameters[k].annotation) - - schema_fields = [] - for k,v in field_dict.items(): - schema_fields.append(k+': '+v) - - schema_data['schema_name'] = snake_to_pascal_case(func_name) - schema_data['schema_fields'] = schema_fields - - j2_template = Template(template) - res = j2_template.render(schema_data) - - return res - -def generate_config_session_definitions(): - results = [] - for name,func in queries.items(): - res = create_schema(name, func, query_template) - results.append(res) - - for name,func in mutations.items(): - res = create_schema(name, func, mutation_template) - results.append(res) - - out = '\n'.join(results) - with open(f'{SCHEMA_PATH}/configsession.graphql', 'w') as f: - f.write(out) - -if __name__ == '__main__': - generate_config_session_definitions() diff --git a/src/services/api/graphql/utils/schema_from_op_mode.py b/src/services/api/graphql/utils/schema_from_op_mode.py deleted file mode 100755 index 57d63628b..000000000 --- a/src/services/api/graphql/utils/schema_from_op_mode.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2022 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# -# A utility to generate GraphQL schema defintions from standardized op-mode -# scripts. - -import os -import json -from inspect import signature, getmembers, isfunction, isclass, getmro -from jinja2 import Template - -from vyos.defaults import directories -if __package__ is None or __package__ == '': - from util import load_as_module, is_op_mode_function_name, is_show_function_name - from util import snake_to_pascal_case, map_type_name -else: - from . util import load_as_module, is_op_mode_function_name, is_show_function_name - from . util import snake_to_pascal_case, map_type_name - -OP_MODE_PATH = directories['op_mode'] -SCHEMA_PATH = directories['api_schema'] -DATA_DIR = directories['data'] - -op_mode_include_file = os.path.join(DATA_DIR, 'op-mode-standardized.json') -op_mode_error_schema = 'op_mode_error.graphql' - -schema_data: dict = {'schema_name': '', - 'schema_fields': []} - -query_template = """ -input {{ schema_name }}Input { - key: String! - {%- for field_entry in schema_fields %} - {{ field_entry }} - {%- endfor %} -} - -type {{ schema_name }} { - result: Generic -} - -type {{ schema_name }}Result { - data: {{ schema_name }} - op_mode_error: OpModeError - success: Boolean! - errors: [String] -} - -extend type Query { - {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @genopquery -} -""" - -mutation_template = """ -input {{ schema_name }}Input { - key: String! - {%- for field_entry in schema_fields %} - {{ field_entry }} - {%- endfor %} -} - -type {{ schema_name }} { - result: Generic -} - -type {{ schema_name }}Result { - data: {{ schema_name }} - op_mode_error: OpModeError - success: Boolean! - errors: [String] -} - -extend type Mutation { - {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @genopmutation -} -""" - -error_template = """ -interface OpModeError { - name: String! - message: String! - vyos_code: Int! -} -{% for name in error_names %} -type {{ name }} implements OpModeError { - name: String! - message: String! - vyos_code: Int! -} -{%- endfor %} -""" - -def create_schema(func_name: str, base_name: str, func: callable) -> str: - sig = signature(func) - - field_dict = {} - for k in sig.parameters: - field_dict[sig.parameters[k].name] = map_type_name(sig.parameters[k].annotation) - - # It is assumed that if one is generating a schema for a 'show_*' - # function, that 'get_raw_data' is present and 'raw' is desired. - if 'raw' in list(field_dict): - del field_dict['raw'] - - schema_fields = [] - for k,v in field_dict.items(): - schema_fields.append(k+': '+v) - - schema_data['schema_name'] = snake_to_pascal_case(func_name + '_' + base_name) - schema_data['schema_fields'] = schema_fields - - if is_show_function_name(func_name): - j2_template = Template(query_template) - else: - j2_template = Template(mutation_template) - - res = j2_template.render(schema_data) - - return res - -def create_error_schema(): - from vyos import opmode - - e = Exception - err_types = getmembers(opmode, isclass) - err_types = [k for k in err_types if issubclass(k[1], e)] - # drop base class, to be replaced by interface type. Find the class - # programmatically, in case the base class name changes. - for i in range(len(err_types)): - if err_types[i][1] in getmro(err_types[i-1][1]): - del err_types[i] - break - err_names = [k[0] for k in err_types] - error_data = {'error_names': err_names} - j2_template = Template(error_template) - res = j2_template.render(error_data) - - return res - -def generate_op_mode_definitions(): - out = create_error_schema() - with open(f'{SCHEMA_PATH}/{op_mode_error_schema}', 'w') as f: - f.write(out) - - with open(op_mode_include_file) as f: - op_mode_files = json.load(f) - - for file in op_mode_files: - basename = os.path.splitext(file)[0].replace('-', '_') - module = load_as_module(basename, os.path.join(OP_MODE_PATH, file)) - - funcs = getmembers(module, isfunction) - funcs = list(filter(lambda ft: is_op_mode_function_name(ft[0]), funcs)) - - funcs_dict = {} - for (name, thunk) in funcs: - funcs_dict[name] = thunk - - results = [] - for name,func in funcs_dict.items(): - res = create_schema(name, basename, func) - results.append(res) - - out = '\n'.join(results) - with open(f'{SCHEMA_PATH}/{basename}.graphql', 'w') as f: - f.write(out) - -if __name__ == '__main__': - generate_op_mode_definitions() diff --git a/src/services/api/graphql/utils/util.py b/src/services/api/graphql/utils/util.py deleted file mode 100644 index da2bcdb5b..000000000 --- a/src/services/api/graphql/utils/util.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2022 VyOS maintainers and contributors -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see . - -import os -import re -import typing -import importlib.util - -from vyos.defaults import directories - -def load_as_module(name: str, path: str): - spec = importlib.util.spec_from_file_location(name, path) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - -def load_op_mode_as_module(name: str): - path = os.path.join(directories['op_mode'], name) - name = os.path.splitext(name)[0].replace('-', '_') - return load_as_module(name, path) - -def is_op_mode_function_name(name): - if re.match(r"^(show|clear|reset|restart)", name): - return True - return False - -def is_show_function_name(name): - if re.match(r"^show", name): - return True - return False - -def _nth_split(delim: str, n: int, s: str): - groups = s.split(delim) - l = len(groups) - if n > l-1 or n < 1: - return (s, '') - return (delim.join(groups[:n]), delim.join(groups[n:])) - -def _nth_rsplit(delim: str, n: int, s: str): - groups = s.split(delim) - l = len(groups) - if n > l-1 or n < 1: - return (s, '') - return (delim.join(groups[:l-n]), delim.join(groups[l-n:])) - -# Since we have mangled possible hyphens in the file name while constructing -# the snake case of the query/mutation name, we will need to recover the -# file name by searching with mangling: -def _filter_on_mangled(test): - def func(elem): - mangle = os.path.splitext(elem)[0].replace('-', '_') - return test == mangle - return func - -# Find longest name in concatenated string that matches the basename of an -# op-mode script. Should one prefer to concatenate in the reverse order -# (script_name + '_' + function_name), use _nth_rsplit. -def split_compound_op_mode_name(name: str, files: list): - for i in range(1, name.count('_') + 1): - pair = _nth_split('_', i, name) - f = list(filter(_filter_on_mangled(pair[1]), files)) - if f: - pair = (pair[0], f[0]) - return pair - return (name, '') - -def snake_to_pascal_case(name: str) -> str: - res = ''.join(map(str.title, name.split('_'))) - return res - -def map_type_name(type_name: type, optional: bool = False) -> str: - if type_name == str: - return 'String!' if not optional else 'String = null' - if type_name == int: - return 'Int!' if not optional else 'Int = null' - if type_name == bool: - return 'Boolean!' if not optional else 'Boolean = false' - if typing.get_origin(type_name) == list: - if not optional: - return f'[{map_type_name(typing.get_args(type_name)[0])}]!' - return f'[{map_type_name(typing.get_args(type_name)[0])}]' - # typing.Optional is typing.Union[_, NoneType] - if (typing.get_origin(type_name) is typing.Union and - typing.get_args(type_name)[1] == type(None)): - return f'{map_type_name(typing.get_args(type_name)[0], optional=True)}' - - # scalar 'Generic' is defined in schema.graphql - return 'Generic' -- cgit v1.2.3 From cbb72ad6d3f5f08ad23c40e29b9463087ca5cade Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 23 Oct 2022 11:06:52 -0500 Subject: graphql: T4574: add interface definitions for authentication settings --- interface-definitions/https.xml.in | 53 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/interface-definitions/https.xml.in b/interface-definitions/https.xml.in index 28656b594..6adb07598 100644 --- a/interface-definitions/https.xml.in +++ b/interface-definitions/https.xml.in @@ -118,6 +118,59 @@ + + + GraphQL authentication + + + + + Authentication type + + key token + + + key + Use API keys + + + token + Use JWT token + + + (key|token) + + + key + + + + Token time to expire in seconds + + u32:60-31536000 + Token lifetime in seconds + + + + + + 3600 + + + + Length of shared secret in bytes + + u32:16-65535 + Byte length of generated shared secret + + + + + + 32 + + + -- cgit v1.2.3 From f76a6f68b08fce1feee2dbbb84658b8eede09655 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 23 Oct 2022 11:07:16 -0500 Subject: graphql: T4574: add mutation for requesting JWT token --- src/services/api/graphql/bindings.py | 8 +++- .../api/graphql/graphql/auth_token_mutation.py | 44 ++++++++++++++++++++++ .../api/graphql/graphql/schema/auth_token.graphql | 19 ++++++++++ src/services/api/graphql/libs/token_auth.py | 38 +++++++++++++++++++ 4 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 src/services/api/graphql/graphql/auth_token_mutation.py create mode 100644 src/services/api/graphql/graphql/schema/auth_token.graphql create mode 100644 src/services/api/graphql/libs/token_auth.py diff --git a/src/services/api/graphql/bindings.py b/src/services/api/graphql/bindings.py index d3cff21c7..aa1ba0eb0 100644 --- a/src/services/api/graphql/bindings.py +++ b/src/services/api/graphql/bindings.py @@ -18,9 +18,12 @@ from . graphql.queries import query from . graphql.mutations import mutation from . graphql.directives import directives_dict from . graphql.errors import op_mode_error +from . graphql.auth_token_mutation import auth_token_mutation from . generate.schema_from_op_mode import generate_op_mode_definitions from . generate.schema_from_config_session import generate_config_session_definitions from . generate.schema_from_composite import generate_composite_definitions +from . libs.token_auth import init_secret +from . import state from ariadne import make_executable_schema, load_schema_from_path, snake_case_fallback_resolvers def generate_schema(): @@ -30,8 +33,11 @@ def generate_schema(): generate_config_session_definitions() generate_composite_definitions() + if state.settings['app'].state.vyos_auth_type == 'token': + init_secret() + type_defs = load_schema_from_path(api_schema_dir) - schema = make_executable_schema(type_defs, query, op_mode_error, mutation, snake_case_fallback_resolvers, directives=directives_dict) + schema = make_executable_schema(type_defs, query, op_mode_error, mutation, auth_token_mutation, snake_case_fallback_resolvers, directives=directives_dict) return schema diff --git a/src/services/api/graphql/graphql/auth_token_mutation.py b/src/services/api/graphql/graphql/auth_token_mutation.py new file mode 100644 index 000000000..33779d4f0 --- /dev/null +++ b/src/services/api/graphql/graphql/auth_token_mutation.py @@ -0,0 +1,44 @@ +# Copyright 2022 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see . + +import jwt +from typing import Any, Dict +from ariadne import ObjectType, UnionType +from graphql import GraphQLResolveInfo + +from .. libs.token_auth import generate_token +from .. import state + +auth_token_mutation = ObjectType("Mutation") + +@auth_token_mutation.field('AuthToken') +def auth_token_resolver(obj: Any, info: GraphQLResolveInfo, data: Dict): + # non-nullable fields + user = data['username'] + passwd = data['password'] + + secret = state.settings['secret'] + res = generate_token(user, passwd, secret) + if res: + data['result'] = res + return { + "success": True, + "data": data + } + + return { + "success": False, + "errors": ['token generation failed'] + } diff --git a/src/services/api/graphql/graphql/schema/auth_token.graphql b/src/services/api/graphql/graphql/schema/auth_token.graphql new file mode 100644 index 000000000..af53a293a --- /dev/null +++ b/src/services/api/graphql/graphql/schema/auth_token.graphql @@ -0,0 +1,19 @@ + +input AuthTokenInput { + username: String! + password: String! +} + +type AuthToken { + result: Generic +} + +type AuthTokenResult { + data: AuthToken + success: Boolean! + errors: [String] +} + +extend type Mutation { + AuthToken(data: AuthTokenInput) : AuthTokenResult +} diff --git a/src/services/api/graphql/libs/token_auth.py b/src/services/api/graphql/libs/token_auth.py new file mode 100644 index 000000000..c53e354b1 --- /dev/null +++ b/src/services/api/graphql/libs/token_auth.py @@ -0,0 +1,38 @@ +import jwt +import uuid +import pam +from secrets import token_hex + +from .. import state + +def _check_passwd_pam(username: str, passwd: str) -> bool: + if pam.authenticate(username, passwd): + return True + return False + +def init_secret(): + secret = token_hex(16) + state.settings['secret'] = secret + +def generate_token(user: str, passwd: str, secret: str) -> dict: + if user is None or passwd is None: + return {} + if _check_passwd_pam(user, passwd): + app = state.settings['app'] + try: + users = app.state.vyos_token_users + except AttributeError: + app.state.vyos_token_users = {} + users = app.state.vyos_token_users + user_id = uuid.uuid1().hex + payload_data = {'iss': user, 'sub': user_id} + secret = state.settings.get('secret') + if secret is None: + return { + "success": False, + "errors": ['failed secret generation'] + } + token = jwt.encode(payload=payload_data, key=secret, algorithm="HS256") + + users |= {user_id: user} + return {'token': token} -- cgit v1.2.3 From af56ddf4615974c6b5f5886520d6abb0781cea80 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 23 Oct 2022 11:07:46 -0500 Subject: graphql: T4574: read config and generate schema with/without key auth --- .../api/graphql/generate/schema_from_composite.py | 46 +++++++++++++++++++++- .../graphql/generate/schema_from_config_session.py | 46 +++++++++++++++++++++- .../api/graphql/generate/schema_from_op_mode.py | 46 +++++++++++++++++++++- src/services/vyos-http-api-server | 15 ++++--- 4 files changed, 144 insertions(+), 9 deletions(-) diff --git a/src/services/api/graphql/generate/schema_from_composite.py b/src/services/api/graphql/generate/schema_from_composite.py index 7187047a0..61a08cb2f 100755 --- a/src/services/api/graphql/generate/schema_from_composite.py +++ b/src/services/api/graphql/generate/schema_from_composite.py @@ -29,22 +29,50 @@ if __package__ is None or __package__ == '': sys.path.append("/usr/libexec/vyos/services/api") from graphql.libs.op_mode import snake_to_pascal_case, map_type_name from composite_function import queries, mutations + from vyos.config import Config + from vyos.configdict import dict_merge + from vyos.xml import defaults else: from .. libs.op_mode import snake_to_pascal_case, map_type_name from . composite_function import queries, mutations + from .. import state SCHEMA_PATH = directories['api_schema'] -schema_data: dict = {'schema_name': '', +if __package__ is None or __package__ == '': + # allow running stand-alone + conf = Config() + base = ['service', 'https', 'api'] + graphql_dict = conf.get_config_dict(base, key_mangling=('-', '_'), + no_tag_node_value_mangle=True, + get_first_key=True) + if 'graphql' not in graphql_dict: + exit("graphql is not configured") + + graphql_dict = dict_merge(defaults(base), graphql_dict) + auth_type = graphql_dict['graphql']['authentication']['type'] +else: + auth_type = state.settings['app'].state.vyos_auth_type + +schema_data: dict = {'auth_type': auth_type, + 'schema_name': '', 'schema_fields': []} query_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -57,17 +85,29 @@ type {{ schema_name }}Result { } extend type Query { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @compositequery +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @compositequery +{%- endif %} } """ mutation_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -80,7 +120,11 @@ type {{ schema_name }}Result { } extend type Mutation { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @compositemutation +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @compositemutation +{%- endif %} } """ diff --git a/src/services/api/graphql/generate/schema_from_config_session.py b/src/services/api/graphql/generate/schema_from_config_session.py index cf69cbafd..49bf2440e 100755 --- a/src/services/api/graphql/generate/schema_from_config_session.py +++ b/src/services/api/graphql/generate/schema_from_config_session.py @@ -29,22 +29,50 @@ if __package__ is None or __package__ == '': sys.path.append("/usr/libexec/vyos/services/api") from graphql.libs.op_mode import snake_to_pascal_case, map_type_name from config_session_function import queries, mutations + from vyos.config import Config + from vyos.configdict import dict_merge + from vyos.xml import defaults else: from .. libs.op_mode import snake_to_pascal_case, map_type_name from . config_session_function import queries, mutations + from .. import state SCHEMA_PATH = directories['api_schema'] -schema_data: dict = {'schema_name': '', +if __package__ is None or __package__ == '': + # allow running stand-alone + conf = Config() + base = ['service', 'https', 'api'] + graphql_dict = conf.get_config_dict(base, key_mangling=('-', '_'), + no_tag_node_value_mangle=True, + get_first_key=True) + if 'graphql' not in graphql_dict: + exit("graphql is not configured") + + graphql_dict = dict_merge(defaults(base), graphql_dict) + auth_type = graphql_dict['graphql']['authentication']['type'] +else: + auth_type = state.settings['app'].state.vyos_auth_type + +schema_data: dict = {'auth_type': auth_type, + 'schema_name': '', 'schema_fields': []} query_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -57,17 +85,29 @@ type {{ schema_name }}Result { } extend type Query { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @configsessionquery +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @configsessionquery +{%- endif %} } """ mutation_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -80,7 +120,11 @@ type {{ schema_name }}Result { } extend type Mutation { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @configsessionmutation +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @configsessionmutation +{%- endif %} } """ diff --git a/src/services/api/graphql/generate/schema_from_op_mode.py b/src/services/api/graphql/generate/schema_from_op_mode.py index a88816b34..1fd198a37 100755 --- a/src/services/api/graphql/generate/schema_from_op_mode.py +++ b/src/services/api/graphql/generate/schema_from_op_mode.py @@ -29,9 +29,13 @@ if __package__ is None or __package__ == '': sys.path.append("/usr/libexec/vyos/services/api") from graphql.libs.op_mode import load_as_module, is_op_mode_function_name, is_show_function_name from graphql.libs.op_mode import snake_to_pascal_case, map_type_name + from vyos.config import Config + from vyos.configdict import dict_merge + from vyos.xml import defaults else: from .. libs.op_mode import load_as_module, is_op_mode_function_name, is_show_function_name from .. libs.op_mode import snake_to_pascal_case, map_type_name + from .. import state OP_MODE_PATH = directories['op_mode'] SCHEMA_PATH = directories['api_schema'] @@ -40,16 +44,40 @@ DATA_DIR = directories['data'] op_mode_include_file = os.path.join(DATA_DIR, 'op-mode-standardized.json') op_mode_error_schema = 'op_mode_error.graphql' -schema_data: dict = {'schema_name': '', +if __package__ is None or __package__ == '': + # allow running stand-alone + conf = Config() + base = ['service', 'https', 'api'] + graphql_dict = conf.get_config_dict(base, key_mangling=('-', '_'), + no_tag_node_value_mangle=True, + get_first_key=True) + if 'graphql' not in graphql_dict: + exit("graphql is not configured") + + graphql_dict = dict_merge(defaults(base), graphql_dict) + auth_type = graphql_dict['graphql']['authentication']['type'] +else: + auth_type = state.settings['app'].state.vyos_auth_type + +schema_data: dict = {'auth_type': auth_type, + 'schema_name': '', 'schema_fields': []} query_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -63,17 +91,29 @@ type {{ schema_name }}Result { } extend type Query { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @genopquery +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @genopquery +{%- endif %} } """ mutation_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -87,7 +127,11 @@ type {{ schema_name }}Result { } extend type Mutation { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @genopmutation +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @genopquery +{%- endif %} } """ diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server index 632c1e87d..7a35546e5 100755 --- a/src/services/vyos-http-api-server +++ b/src/services/vyos-http-api-server @@ -647,11 +647,11 @@ def reset_op(data: ResetModel): ### def graphql_init(fast_api_app): - from api.graphql.bindings import generate_schema - api.graphql.state.init() api.graphql.state.settings['app'] = app + # import after initializaion of state + from api.graphql.bindings import generate_schema schema = generate_schema() in_spec = app.state.vyos_introspection @@ -690,10 +690,13 @@ if __name__ == '__main__': app.state.vyos_origins = server_config.get('cors', {}).get('allow_origin', []) if 'graphql' in server_config: app.state.vyos_graphql = True - if isinstance(server_config['graphql'], dict) and 'introspection' in server_config['graphql']: - app.state.vyos_introspection = True - else: - app.state.vyos_introspection = False + if isinstance(server_config['graphql'], dict): + if 'introspection' in server_config['graphql']: + app.state.vyos_introspection = True + else: + app.state.vyos_introspection = False + # default value is merged in conf_mode http-api.py, if not set + app.state.vyos_auth_type = server_config['graphql']['authentication']['type'] else: app.state.vyos_graphql = False -- cgit v1.2.3 From 28676844e3f4317786e457fcd8651939a05c88ff Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 23 Oct 2022 11:08:06 -0500 Subject: graphql: T4574: add context to read token in queries/mutations --- src/services/api/graphql/graphql/mutations.py | 62 ++++++++++++++++++--------- src/services/api/graphql/graphql/queries.py | 62 ++++++++++++++++++--------- src/services/api/graphql/libs/token_auth.py | 29 +++++++++++++ src/services/vyos-http-api-server | 5 ++- 4 files changed, 116 insertions(+), 42 deletions(-) diff --git a/src/services/api/graphql/graphql/mutations.py b/src/services/api/graphql/graphql/mutations.py index f0c8b438f..2778feb69 100644 --- a/src/services/api/graphql/graphql/mutations.py +++ b/src/services/api/graphql/graphql/mutations.py @@ -42,32 +42,54 @@ def make_mutation_resolver(mutation_name, class_name, session_func): func_base_name = convert_camel_case_to_snake(class_name) resolver_name = f'resolve_{func_base_name}' - func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Dict)' + func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Dict = {})' @mutation.field(mutation_name) @convert_kwargs_to_snake_case @with_signature(func_sig, func_name=resolver_name) async def func_impl(*args, **kwargs): try: - if 'data' not in kwargs: - return { - "success": False, - "errors": ['missing data'] - } - - data = kwargs['data'] - key = data['key'] - - auth = key_auth.auth_required(key) - if auth is None: - return { - "success": False, - "errors": ['invalid API key'] - } - - # We are finished with the 'key' entry, and may remove so as to - # pass the rest of data (if any) to function. - del data['key'] + auth_type = state.settings['app'].state.vyos_auth_type + + if auth_type == 'key': + data = kwargs['data'] + key = data['key'] + + auth = key_auth.auth_required(key) + if auth is None: + return { + "success": False, + "errors": ['invalid API key'] + } + + # We are finished with the 'key' entry, and may remove so as to + # pass the rest of data (if any) to function. + del data['key'] + + elif auth_type == 'token': + # there is a subtlety here: with the removal of the key entry, + # some requests will now have empty input, hence no data arg, so + # make it optional in the func_sig. However, it can not be None, + # as the makefun package provides accurate TypeError exceptions; + # hence set it to {}, but now it is a mutable default argument, + # so clear the key 'result', which is added at the end of + # this function. + data = kwargs['data'] + if 'result' in data: + del data['result'] + + info = kwargs['info'] + user = info.context.get('user') + if user is None: + return { + "success": False, + "errors": ['not authenticated'] + } + else: + # AtrributeError will have already been raised if no + # vyos_auth_type; validation and defaultValue ensure it is + # one of the previous cases, so this is never reached. + pass session = state.settings['app'].state.vyos_session diff --git a/src/services/api/graphql/graphql/queries.py b/src/services/api/graphql/graphql/queries.py index 13eb59ae4..9c8a4f064 100644 --- a/src/services/api/graphql/graphql/queries.py +++ b/src/services/api/graphql/graphql/queries.py @@ -42,32 +42,54 @@ def make_query_resolver(query_name, class_name, session_func): func_base_name = convert_camel_case_to_snake(class_name) resolver_name = f'resolve_{func_base_name}' - func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Dict)' + func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Dict = {})' @query.field(query_name) @convert_kwargs_to_snake_case @with_signature(func_sig, func_name=resolver_name) async def func_impl(*args, **kwargs): try: - if 'data' not in kwargs: - return { - "success": False, - "errors": ['missing data'] - } - - data = kwargs['data'] - key = data['key'] - - auth = key_auth.auth_required(key) - if auth is None: - return { - "success": False, - "errors": ['invalid API key'] - } - - # We are finished with the 'key' entry, and may remove so as to - # pass the rest of data (if any) to function. - del data['key'] + auth_type = state.settings['app'].state.vyos_auth_type + + if auth_type == 'key': + data = kwargs['data'] + key = data['key'] + + auth = key_auth.auth_required(key) + if auth is None: + return { + "success": False, + "errors": ['invalid API key'] + } + + # We are finished with the 'key' entry, and may remove so as to + # pass the rest of data (if any) to function. + del data['key'] + + elif auth_type == 'token': + # there is a subtlety here: with the removal of the key entry, + # some requests will now have empty input, hence no data arg, so + # make it optional in the func_sig. However, it can not be None, + # as the makefun package provides accurate TypeError exceptions; + # hence set it to {}, but now it is a mutable default argument, + # so clear the key 'result', which is added at the end of + # this function. + data = kwargs['data'] + if 'result' in data: + del data['result'] + + info = kwargs['info'] + user = info.context.get('user') + if user is None: + return { + "success": False, + "errors": ['not authenticated'] + } + else: + # AtrributeError will have already been raised if no + # vyos_auth_type; validation and defaultValue ensure it is + # one of the previous cases, so this is never reached. + pass session = state.settings['app'].state.vyos_session diff --git a/src/services/api/graphql/libs/token_auth.py b/src/services/api/graphql/libs/token_auth.py index c53e354b1..2d63a1cc7 100644 --- a/src/services/api/graphql/libs/token_auth.py +++ b/src/services/api/graphql/libs/token_auth.py @@ -36,3 +36,32 @@ def generate_token(user: str, passwd: str, secret: str) -> dict: users |= {user_id: user} return {'token': token} + +def get_user_context(request): + context = {} + context['request'] = request + context['user'] = None + if 'Authorization' in request.headers: + auth = request.headers['Authorization'] + scheme, token = auth.split() + if scheme.lower() != 'bearer': + return context + + try: + secret = state.settings.get('secret') + payload = jwt.decode(token, secret, algorithms=["HS256"]) + user_id: str = payload.get('sub') + if user_id is None: + return context + except jwt.PyJWTError: + return context + try: + users = state.settings['app'].state.vyos_token_users + except AttributeError: + return context + + user = users.get(user_id) + if user is not None: + context['user'] = user + + return context diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server index 7a35546e5..840041b73 100755 --- a/src/services/vyos-http-api-server +++ b/src/services/vyos-http-api-server @@ -647,6 +647,7 @@ def reset_op(data: ResetModel): ### def graphql_init(fast_api_app): + from api.graphql.libs.token_auth import get_user_context api.graphql.state.init() api.graphql.state.settings['app'] = app @@ -658,9 +659,9 @@ def graphql_init(fast_api_app): if app.state.vyos_origins: origins = app.state.vyos_origins - app.add_route('/graphql', CORSMiddleware(GraphQL(schema, debug=True, introspection=in_spec), allow_origins=origins, allow_methods=("GET", "POST", "OPTIONS"))) + app.add_route('/graphql', CORSMiddleware(GraphQL(schema, context_value=get_user_context, debug=True, introspection=in_spec), allow_origins=origins, allow_methods=("GET", "POST", "OPTIONS"))) else: - app.add_route('/graphql', GraphQL(schema, debug=True, introspection=in_spec)) + app.add_route('/graphql', GraphQL(schema, context_value=get_user_context, debug=True, introspection=in_spec)) ### -- cgit v1.2.3 From dc37f30a1273c1d3b7949b1d64e60d37da3b9fd4 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 23 Oct 2022 11:08:19 -0500 Subject: graphql: T4574: set token expiration time in claims --- src/services/api/graphql/graphql/auth_token_mutation.py | 7 ++++++- src/services/api/graphql/libs/token_auth.py | 4 ++-- src/services/vyos-http-api-server | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/services/api/graphql/graphql/auth_token_mutation.py b/src/services/api/graphql/graphql/auth_token_mutation.py index 33779d4f0..21ac40094 100644 --- a/src/services/api/graphql/graphql/auth_token_mutation.py +++ b/src/services/api/graphql/graphql/auth_token_mutation.py @@ -14,6 +14,7 @@ # along with this library. If not, see . import jwt +import datetime from typing import Any, Dict from ariadne import ObjectType, UnionType from graphql import GraphQLResolveInfo @@ -30,7 +31,11 @@ def auth_token_resolver(obj: Any, info: GraphQLResolveInfo, data: Dict): passwd = data['password'] secret = state.settings['secret'] - res = generate_token(user, passwd, secret) + exp_interval = int(state.settings['app'].state.vyos_token_exp) + expiration = (datetime.datetime.now(tz=datetime.timezone.utc) + + datetime.timedelta(seconds=exp_interval)) + + res = generate_token(user, passwd, secret, expiration) if res: data['result'] = res return { diff --git a/src/services/api/graphql/libs/token_auth.py b/src/services/api/graphql/libs/token_auth.py index 2d63a1cc7..fafb0f5af 100644 --- a/src/services/api/graphql/libs/token_auth.py +++ b/src/services/api/graphql/libs/token_auth.py @@ -14,7 +14,7 @@ def init_secret(): secret = token_hex(16) state.settings['secret'] = secret -def generate_token(user: str, passwd: str, secret: str) -> dict: +def generate_token(user: str, passwd: str, secret: str, exp: int) -> dict: if user is None or passwd is None: return {} if _check_passwd_pam(user, passwd): @@ -25,7 +25,7 @@ def generate_token(user: str, passwd: str, secret: str) -> dict: app.state.vyos_token_users = {} users = app.state.vyos_token_users user_id = uuid.uuid1().hex - payload_data = {'iss': user, 'sub': user_id} + payload_data = {'iss': user, 'sub': user_id, 'exp': exp} secret = state.settings.get('secret') if secret is None: return { diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server index 840041b73..4af27b949 100755 --- a/src/services/vyos-http-api-server +++ b/src/services/vyos-http-api-server @@ -698,6 +698,7 @@ if __name__ == '__main__': app.state.vyos_introspection = False # default value is merged in conf_mode http-api.py, if not set app.state.vyos_auth_type = server_config['graphql']['authentication']['type'] + app.state.vyos_token_exp = server_config['graphql']['authentication']['expiration'] else: app.state.vyos_graphql = False -- cgit v1.2.3 From 8ed99cf8662910f8fd28866391591a4fcbfbea47 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 23 Oct 2022 11:49:08 -0500 Subject: graphql: T4574: extend smoketest for token authentication --- smoketest/scripts/cli/test_service_https.py | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/smoketest/scripts/cli/test_service_https.py b/smoketest/scripts/cli/test_service_https.py index 719125f0f..0f4b1393c 100755 --- a/smoketest/scripts/cli/test_service_https.py +++ b/smoketest/scripts/cli/test_service_https.py @@ -195,5 +195,49 @@ class TestHTTPSService(VyOSUnitTestSHIM.TestCase): r = request('POST', graphql_url, verify=False, headers=headers, json={'query': query_no_key}) self.assertEqual(r.status_code, 400) + # GraphQL token authentication test: request token; pass in header + # of query. + + self.cli_set(base_path + ['api', 'graphql', 'authentication', 'type', 'token']) + self.cli_commit() + + mutation = """ + mutation { + AuthToken (data: {username: "vyos", password: "vyos"}) { + success + errors + data { + result + } + } + } + """ + r = request('POST', graphql_url, verify=False, headers=headers, json={'query': mutation}) + + token = r.json()['data']['AuthToken']['data']['result']['token'] + + headers = {'Authorization': f'Bearer {token}'} + + query = """ + { + ShowVersion (data: {}) { + success + errors + op_mode_error { + name + message + vyos_code + } + data { + result + } + } + } + """ + + r = request('POST', graphql_url, verify=False, headers=headers, json={'query': query}) + success = r.json()['data']['ShowVersion']['success'] + self.assertTrue(success) + if __name__ == '__main__': unittest.main(verbosity=2) -- cgit v1.2.3 From 3db5ba8ef354d80f080cc1baacf33d77ccbb6222 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 25 Oct 2022 09:22:50 -0500 Subject: graphql: T4574: set byte length of shared secret from CLI --- src/services/api/graphql/libs/token_auth.py | 3 ++- src/services/vyos-http-api-server | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/services/api/graphql/libs/token_auth.py b/src/services/api/graphql/libs/token_auth.py index fafb0f5af..3ecd8b855 100644 --- a/src/services/api/graphql/libs/token_auth.py +++ b/src/services/api/graphql/libs/token_auth.py @@ -11,7 +11,8 @@ def _check_passwd_pam(username: str, passwd: str) -> bool: return False def init_secret(): - secret = token_hex(16) + length = int(state.settings['app'].state.vyos_secret_len) + secret = token_hex(length) state.settings['secret'] = secret def generate_token(user: str, passwd: str, secret: str, exp: int) -> dict: diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server index 4af27b949..3c390d9dc 100755 --- a/src/services/vyos-http-api-server +++ b/src/services/vyos-http-api-server @@ -699,6 +699,7 @@ if __name__ == '__main__': # default value is merged in conf_mode http-api.py, if not set app.state.vyos_auth_type = server_config['graphql']['authentication']['type'] app.state.vyos_token_exp = server_config['graphql']['authentication']['expiration'] + app.state.vyos_secret_len = server_config['graphql']['authentication']['secret_length'] else: app.state.vyos_graphql = False -- cgit v1.2.3 From c0594071c6c1062cc0069377b6666de89a172d0a Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 25 Oct 2022 14:55:49 -0500 Subject: ci: T4748: add dot to regex char class to allow 'vyos.util: Txxx: ...' --- scripts/check-pr-title-and-commit-messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check-pr-title-and-commit-messages.py b/scripts/check-pr-title-and-commit-messages.py index c30c3ef1f..3317745d6 100755 --- a/scripts/check-pr-title-and-commit-messages.py +++ b/scripts/check-pr-title-and-commit-messages.py @@ -7,7 +7,7 @@ import requests from pprint import pprint # Use the same regex for PR title and commit messages for now -title_regex = r'^(([a-zA-Z]+:\s)?)T\d+:\s+[^\s]+.*' +title_regex = r'^(([a-zA-Z.]+:\s)?)T\d+:\s+[^\s]+.*' commit_regex = title_regex def check_pr_title(title): -- cgit v1.2.3 From 29b656f14d8bb7622ff861208d26cf8eb018670d Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 25 Oct 2022 13:42:30 -0500 Subject: vyos.util: T4773: add camel_to_snake_case conversion --- python/vyos/util.py | 7 +++++++ src/tests/test_util.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/python/vyos/util.py b/python/vyos/util.py index 461df9a6e..e4e2a44ec 100644 --- a/python/vyos/util.py +++ b/python/vyos/util.py @@ -1105,3 +1105,10 @@ def sysctl_write(name, value): call(f'sysctl -wq {name}={value}') return True return False + +# approach follows a discussion in: +# https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case +def camel_to_snake_case(name: str) -> str: + pattern = r'\d+|[A-Z]?[a-z]+|\W|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)' + words = re.findall(pattern, name) + return '_'.join(map(str.lower, words)) diff --git a/src/tests/test_util.py b/src/tests/test_util.py index 8ac9a500a..d8b2b7940 100644 --- a/src/tests/test_util.py +++ b/src/tests/test_util.py @@ -26,3 +26,17 @@ class TestVyOSUtil(TestCase): def test_sysctl_read(self): self.assertEqual(sysctl_read('net.ipv4.conf.lo.forwarding'), '1') + + def test_camel_to_snake_case(self): + self.assertEqual(camel_to_snake_case('ConnectionTimeout'), + 'connection_timeout') + self.assertEqual(camel_to_snake_case('connectionTimeout'), + 'connection_timeout') + self.assertEqual(camel_to_snake_case('TCPConnectionTimeout'), + 'tcp_connection_timeout') + self.assertEqual(camel_to_snake_case('TCPPort'), + 'tcp_port') + self.assertEqual(camel_to_snake_case('UseHTTPProxy'), + 'use_http_proxy') + self.assertEqual(camel_to_snake_case('CustomerID'), + 'customer_id') -- cgit v1.2.3 From 2a5273e650ce1242bc22e992e5a3104961ec1295 Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Tue, 25 Oct 2022 12:29:03 +0200 Subject: nat: T4764: Remove tables on NAT deletion --- data/templates/firewall/nftables-nat.j2 | 18 ++++++++++-------- data/templates/firewall/nftables-static-nat.j2 | 18 ++++++++++-------- smoketest/scripts/cli/test_nat.py | 6 ++++++ src/conf_mode/nat.py | 4 ++++ 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/data/templates/firewall/nftables-nat.j2 b/data/templates/firewall/nftables-nat.j2 index 55fe6024b..c5c0a2c86 100644 --- a/data/templates/firewall/nftables-nat.j2 +++ b/data/templates/firewall/nftables-nat.j2 @@ -24,6 +24,7 @@ add rule ip raw NAT_CONNTRACK counter accept {% if first_install is not vyos_defined %} delete table ip vyos_nat {% endif %} +{% if deleted is not vyos_defined %} table ip vyos_nat { # # Destination NAT rules build up here @@ -31,11 +32,11 @@ table ip vyos_nat { chain PREROUTING { type nat hook prerouting priority -100; policy accept; counter jump VYOS_PRE_DNAT_HOOK -{% if destination.rule is vyos_defined %} -{% for rule, config in destination.rule.items() if config.disable is not vyos_defined %} +{% if destination.rule is vyos_defined %} +{% for rule, config in destination.rule.items() if config.disable is not vyos_defined %} {{ config | nat_rule(rule, 'destination') }} -{% endfor %} -{% endif %} +{% endfor %} +{% endif %} } # @@ -44,11 +45,11 @@ table ip vyos_nat { chain POSTROUTING { type nat hook postrouting priority 100; policy accept; counter jump VYOS_PRE_SNAT_HOOK -{% if source.rule is vyos_defined %} -{% for rule, config in source.rule.items() if config.disable is not vyos_defined %} +{% if source.rule is vyos_defined %} +{% for rule, config in source.rule.items() if config.disable is not vyos_defined %} {{ config | nat_rule(rule, 'source') }} -{% endfor %} -{% endif %} +{% endfor %} +{% endif %} } chain VYOS_PRE_DNAT_HOOK { @@ -59,3 +60,4 @@ table ip vyos_nat { return } } +{% endif %} diff --git a/data/templates/firewall/nftables-static-nat.j2 b/data/templates/firewall/nftables-static-nat.j2 index 790c33ce9..e5e3da867 100644 --- a/data/templates/firewall/nftables-static-nat.j2 +++ b/data/templates/firewall/nftables-static-nat.j2 @@ -3,6 +3,7 @@ {% if first_install is not vyos_defined %} delete table ip vyos_static_nat {% endif %} +{% if deleted is not vyos_defined %} table ip vyos_static_nat { # # Destination NAT rules build up here @@ -10,11 +11,11 @@ table ip vyos_static_nat { chain PREROUTING { type nat hook prerouting priority -100; policy accept; -{% if static.rule is vyos_defined %} -{% for rule, config in static.rule.items() if config.disable is not vyos_defined %} +{% if static.rule is vyos_defined %} +{% for rule, config in static.rule.items() if config.disable is not vyos_defined %} {{ config | nat_static_rule(rule, 'destination') }} -{% endfor %} -{% endif %} +{% endfor %} +{% endif %} } # @@ -22,10 +23,11 @@ table ip vyos_static_nat { # chain POSTROUTING { type nat hook postrouting priority 100; policy accept; -{% if static.rule is vyos_defined %} -{% for rule, config in static.rule.items() if config.disable is not vyos_defined %} +{% if static.rule is vyos_defined %} +{% for rule, config in static.rule.items() if config.disable is not vyos_defined %} {{ config | nat_static_rule(rule, 'source') }} -{% endfor %} -{% endif %} +{% endfor %} +{% endif %} } } +{% endif %} diff --git a/smoketest/scripts/cli/test_nat.py b/smoketest/scripts/cli/test_nat.py index f824838c0..2ae90fcaf 100755 --- a/smoketest/scripts/cli/test_nat.py +++ b/smoketest/scripts/cli/test_nat.py @@ -16,6 +16,7 @@ import jmespath import json +import os import unittest from base_vyostest_shim import VyOSUnitTestSHIM @@ -28,6 +29,9 @@ src_path = base_path + ['source'] dst_path = base_path + ['destination'] static_path = base_path + ['static'] +nftables_nat_config = '/run/nftables_nat.conf' +nftables_static_nat_conf = '/run/nftables_static-nat-rules.nft' + class TestNAT(VyOSUnitTestSHIM.TestCase): @classmethod def setUpClass(cls): @@ -40,6 +44,8 @@ class TestNAT(VyOSUnitTestSHIM.TestCase): def tearDown(self): self.cli_delete(base_path) self.cli_commit() + self.assertFalse(os.path.exists(nftables_nat_config)) + self.assertFalse(os.path.exists(nftables_static_nat_conf)) def verify_nftables(self, nftables_search, table, inverse=False, args=''): nftables_output = cmd(f'sudo nft {args} list table {table}') diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py index 8b1a5a720..1e807753d 100755 --- a/src/conf_mode/nat.py +++ b/src/conf_mode/nat.py @@ -204,6 +204,10 @@ def apply(nat): cmd(f'nft -f {nftables_nat_config}') cmd(f'nft -f {nftables_static_nat_conf}') + if not nat or 'deleted' in nat: + os.unlink(nftables_nat_config) + os.unlink(nftables_static_nat_conf) + return None if __name__ == '__main__': -- cgit v1.2.3 From 16207f7a8ffdbc93fcfcc4b6ba783940a1e40e33 Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Tue, 25 Oct 2022 22:41:55 +0200 Subject: nat: T4706: Verify translation address or port exists --- src/conf_mode/nat.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py index 1e807753d..978c043e9 100755 --- a/src/conf_mode/nat.py +++ b/src/conf_mode/nat.py @@ -146,6 +146,10 @@ def verify(nat): if config['outbound_interface'] not in 'any' and config['outbound_interface'] not in interfaces(): Warning(f'rule "{rule}" interface "{config["outbound_interface"]}" does not exist on this system') + if not dict_search('translation.address', config) and not dict_search('translation.port', config): + if 'exclude' not in config: + raise ConfigError(f'{err_msg} translation requires address and/or port') + addr = dict_search('translation.address', config) if addr != None and addr != 'masquerade' and not is_ip_network(addr): for ip in addr.split('-'): @@ -166,6 +170,10 @@ def verify(nat): elif config['inbound_interface'] not in 'any' and config['inbound_interface'] not in interfaces(): Warning(f'rule "{rule}" interface "{config["inbound_interface"]}" does not exist on this system') + if not dict_search('translation.address', config) and not dict_search('translation.port', config): + if 'exclude' not in config: + raise ConfigError(f'{err_msg} translation requires address and/or port') + # common rule verification verify_rule(config, err_msg) -- cgit v1.2.3 From 413e24400c54f398ef73347df2e877aef422400e Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 27 Oct 2022 13:57:25 -0500 Subject: ipsec: T4778: raise UnconfiguredSubsystem if IPsec not initialized --- src/op_mode/ipsec.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/op_mode/ipsec.py b/src/op_mode/ipsec.py index 7ec35d7bd..aaa0cec5a 100755 --- a/src/op_mode/ipsec.py +++ b/src/op_mode/ipsec.py @@ -43,7 +43,10 @@ def _alphanum_key(key): def _get_vici_sas(): from vici import Session as vici_session - session = vici_session() + try: + session = vici_session() + except Exception: + raise vyos.opmode.UnconfiguredSubsystem("IPsec not initialized") sas = list(session.list_sas()) return sas -- cgit v1.2.3 From c2ff9aa158b81fa66ce9c810e891ad25d4a7f14b Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Thu, 27 Oct 2022 22:37:42 +0200 Subject: wireguard: T4774: Prevent duplicate peer public keys --- smoketest/scripts/cli/test_interfaces_wireguard.py | 10 ++++++++-- src/conf_mode/interfaces-wireguard.py | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/smoketest/scripts/cli/test_interfaces_wireguard.py b/smoketest/scripts/cli/test_interfaces_wireguard.py index f3e9670f7..14fc8d109 100755 --- a/smoketest/scripts/cli/test_interfaces_wireguard.py +++ b/smoketest/scripts/cli/test_interfaces_wireguard.py @@ -62,10 +62,10 @@ class WireGuardInterfaceTest(VyOSUnitTestSHIM.TestCase): self.assertTrue(os.path.isdir(f'/sys/class/net/{intf}')) - def test_wireguard_add_remove_peer(self): # T2939: Create WireGuard interfaces with associated peers. # Remove one of the configured peers. + # T4774: Test prevention of duplicate peer public keys interface = 'wg0' port = '12345' privkey = '6ISOkASm6VhHOOSz/5iIxw+Q9adq9zA17iMM4X40dlc=' @@ -80,11 +80,17 @@ class WireGuardInterfaceTest(VyOSUnitTestSHIM.TestCase): self.cli_set(base_path + [interface, 'peer', 'PEER01', 'allowed-ips', '10.205.212.10/32']) self.cli_set(base_path + [interface, 'peer', 'PEER01', 'address', '192.0.2.1']) - self.cli_set(base_path + [interface, 'peer', 'PEER02', 'public-key', pubkey_2]) + self.cli_set(base_path + [interface, 'peer', 'PEER02', 'public-key', pubkey_1]) self.cli_set(base_path + [interface, 'peer', 'PEER02', 'port', port]) self.cli_set(base_path + [interface, 'peer', 'PEER02', 'allowed-ips', '10.205.212.11/32']) self.cli_set(base_path + [interface, 'peer', 'PEER02', 'address', '192.0.2.2']) + # Duplicate pubkey_1 + with self.assertRaises(ConfigSessionError): + self.cli_commit() + + self.cli_set(base_path + [interface, 'peer', 'PEER02', 'public-key', pubkey_2]) + # Commit peers self.cli_commit() diff --git a/src/conf_mode/interfaces-wireguard.py b/src/conf_mode/interfaces-wireguard.py index 8d738f55e..762bad94f 100755 --- a/src/conf_mode/interfaces-wireguard.py +++ b/src/conf_mode/interfaces-wireguard.py @@ -87,6 +87,8 @@ def verify(wireguard): 'cannot be used for the interface!') # run checks on individual configured WireGuard peer + public_keys = [] + for tmp in wireguard['peer']: peer = wireguard['peer'][tmp] @@ -100,6 +102,11 @@ def verify(wireguard): raise ConfigError('Both Wireguard port and address must be defined ' f'for peer "{tmp}" if either one of them is set!') + if peer['public_key'] in public_keys: + raise ConfigError(f'Duplicate public-key defined on peer "{tmp}"') + + public_keys.append(peer['public_key']) + def apply(wireguard): tmp = WireGuardIf(wireguard['ifname']) if 'deleted' in wireguard: -- cgit v1.2.3 From f35195945daba0a81a93b74b280591dd955c193a Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Fri, 28 Oct 2022 09:00:23 -0400 Subject: T4779: use bytes in the raw output of "show system memory" --- src/op_mode/memory.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/op_mode/memory.py b/src/op_mode/memory.py index 178544be4..7666de646 100755 --- a/src/op_mode/memory.py +++ b/src/op_mode/memory.py @@ -20,7 +20,7 @@ import sys import vyos.opmode -def _get_system_memory(): +def _get_raw_data(): from re import search as re_search def find_value(keyword, mem_data): @@ -38,7 +38,7 @@ def _get_system_memory(): used = total - available - res = { + mem_data = { "total": total, "free": available, "used": used, @@ -46,24 +46,21 @@ def _get_system_memory(): "cached": cached } - return res - -def _get_system_memory_human(): - from vyos.util import bytes_to_human - - mem = _get_system_memory() - - for key in mem: + for key in mem_data: # The Linux kernel exposes memory values in kilobytes, # so we need to normalize them - mem[key] = bytes_to_human(mem[key], initial_exponent=10) + mem_data[key] = mem_data[key] * 1024 - return mem - -def _get_raw_data(): - return _get_system_memory_human() + return mem_data def _get_formatted_output(mem): + from vyos.util import bytes_to_human + + # For human-readable outputs, we convert bytes to more convenient units + # (100M, 1.3G...) + for key in mem: + mem[key] = bytes_to_human(mem[key]) + out = "Total: {}\n".format(mem["total"]) out += "Free: {}\n".format(mem["free"]) out += "Used: {}".format(mem["used"]) -- cgit v1.2.3 From 0e63712195465c9bf0bf55c369b86961d54dfaac Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 27 Oct 2022 12:24:44 -0500 Subject: T4291: consolidate component version string read/write functions --- python/vyos/component_version.py | 192 ++++++++++++++++++++++++ python/vyos/component_versions.py | 57 ------- python/vyos/formatversions.py | 109 -------------- python/vyos/migrator.py | 32 ++-- python/vyos/systemversions.py | 46 ------ smoketest/scripts/cli/test_component_version.py | 6 +- src/helpers/system-versions-foot.py | 21 +-- 7 files changed, 213 insertions(+), 250 deletions(-) create mode 100644 python/vyos/component_version.py delete mode 100644 python/vyos/component_versions.py delete mode 100644 python/vyos/formatversions.py delete mode 100644 python/vyos/systemversions.py diff --git a/python/vyos/component_version.py b/python/vyos/component_version.py new file mode 100644 index 000000000..a4e318d08 --- /dev/null +++ b/python/vyos/component_version.py @@ -0,0 +1,192 @@ +# Copyright 2022 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . + +""" +Functions for reading/writing component versions. + +The config file version string has the following form: + +VyOS 1.3/1.4: + +// Warning: Do not remove the following line. +// vyos-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack@3:conntrack-sync@2:dhcp-relay@2:dhcp-server@6:dhcpv6-server@1:dns-forwarding@3:firewall@5:https@2:interfaces@22:ipoe-server@1:ipsec@5:isis@1:l2tp@3:lldp@1:mdns@1:nat@5:ntp@1:pppoe-server@5:pptp@2:qos@1:quagga@8:rpki@1:salt@1:snmp@2:ssh@2:sstp@3:system@21:vrrp@2:vyos-accel-ppp@2:wanloadbalance@3:webproxy@2:zone-policy@1" +// Release version: 1.3.0 + +VyOS 1.2: + +/* Warning: Do not remove the following line. */ +/* === vyatta-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack-sync@1:conntrack@1:dhcp-relay@2:dhcp-server@5:dns-forwarding@1:firewall@5:ipsec@5:l2tp@1:mdns@1:nat@4:ntp@1:pppoe-server@2:pptp@1:qos@1:quagga@7:snmp@1:ssh@1:system@10:vrrp@2:wanloadbalance@3:webgui@1:webproxy@2:zone-policy@1" === */ +/* Release version: 1.2.8 */ + +""" + +import os +import re +import sys +import fileinput + +from vyos.xml import component_version +from vyos.version import get_version +from vyos.defaults import directories + +DEFAULT_CONFIG_PATH = os.path.join(directories['config'], 'config.boot') + +def from_string(string_line, vintage='vyos'): + """ + Get component version dictionary from string. + Return empty dictionary if string contains no config information + or raise error if component version string malformed. + """ + version_dict = {} + + if vintage == 'vyos': + if re.match(r'// vyos-config-version:.+', string_line): + if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', string_line): + raise ValueError(f"malformed configuration string: {string_line}") + + for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): + version_dict[pair[0]] = int(pair[1]) + + elif vintage == 'vyatta': + if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line): + if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line): + raise ValueError(f"malformed configuration string: {string_line}") + + for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): + version_dict[pair[0]] = int(pair[1]) + else: + raise ValueError("Unknown config string vintage") + + return version_dict + +def from_file(config_file_name=DEFAULT_CONFIG_PATH, vintage='vyos'): + """ + Get component version dictionary parsing config file line by line + """ + with open(config_file_name, 'r') as f: + for line_in_config in f: + version_dict = from_string(line_in_config, vintage=vintage) + if version_dict: + return version_dict + + # no version information + return {} + +def from_system(): + """ + Get system component version dict. + """ + return component_version() + +def legacy_from_system(): + """ + Get system component version dict from legacy location. + This is for a transitional sanity check; the directory will eventually + be removed. + """ + system_versions = {} + legacy_dir = directories['current'] + + # To be removed: + if not os.path.isdir(legacy_dir): + return system_versions + + try: + version_info = os.listdir(legacy_dir) + except OSError as err: + sys.exit(repr(err)) + + for info in version_info: + if re.match(r'[\w,-]+@\d+', info): + pair = info.split('@') + system_versions[pair[0]] = int(pair[1]) + + return system_versions + +def format_string(ver: dict) -> str: + """ + Version dict to string. + """ + keys = list(ver) + keys.sort() + l = [] + for k in keys: + v = ver[k] + l.append(f'{k}@{v}') + sep = ':' + return sep.join(l) + +def version_footer(ver: dict, vintage='vyos') -> str: + """ + Version footer as string. + """ + ver_str = format_string(ver) + release = get_version() + if vintage == 'vyos': + ret_str = (f'// Warning: Do not remove the following line.\n' + + f'// vyos-config-version: "{ver_str}"\n' + + f'// Release version: {release}\n') + elif vintage == 'vyatta': + ret_str = (f'/* Warning: Do not remove the following line. */\n' + + f'/* === vyatta-config-version: "{ver_str}" === */\n' + + f'/* Release version: {release} */\n') + else: + raise ValueError("Unknown config string vintage") + + return ret_str + +def system_footer(vintage='vyos') -> str: + """ + System version footer as string. + """ + ver_d = from_system() + return version_footer(ver_d, vintage=vintage) + +def write_version_footer(ver: dict, file_name, vintage='vyos'): + """ + Write version footer to file. + """ + footer = version_footer(ver=ver, vintage=vintage) + if file_name: + with open(file_name, 'a') as f: + f.write(footer) + else: + sys.stdout.write(footer) + +def write_system_footer(file_name, vintage='vyos'): + """ + Write system version footer to file. + """ + ver_d = from_system() + return write_version_footer(ver_d, file_name=file_name, vintage=vintage) + +def remove_footer(file_name): + """ + Remove old version footer. + """ + for line in fileinput.input(file_name, inplace=True): + if re.match(r'/\* Warning:.+ \*/$', line): + continue + if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line): + continue + if re.match(r'/\* Release version:.+ \*/$', line): + continue + if re.match('// vyos-config-version:.+', line): + continue + if re.match('// Warning:.+', line): + continue + if re.match('// Release version:.+', line): + continue + sys.stdout.write(line) diff --git a/python/vyos/component_versions.py b/python/vyos/component_versions.py deleted file mode 100644 index 90b458aae..000000000 --- a/python/vyos/component_versions.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2017 VyOS maintainers and contributors -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see . - -""" -The version data looks like: - -/* Warning: Do not remove the following line. */ -/* === vyatta-config-version: -"cluster@1:config-management@1:conntrack-sync@1:conntrack@1:dhcp-relay@1:dhcp-server@4:firewall@5:ipsec@4:nat@4:qos@1:quagga@2:system@8:vrrp@1:wanloadbalance@3:webgui@1:webproxy@1:zone-policy@1" -=== */ -/* Release version: 1.2.0-rolling+201806131737 */ -""" - -import re - -def get_component_version(string_line): - """ - Get component version dictionary from string - return empty dictionary if string contains no config information - or raise error if component version string malformed - """ - return_value = {} - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line): - - if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line): - raise ValueError("malformed configuration string: " + str(string_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): - if pair[0] in return_value.keys(): - raise ValueError("duplicate unit name: \"" + str(pair[0]) + "\" in string: \"" + string_line + "\"") - return_value[pair[0]] = int(pair[1]) - - return return_value - - -def get_component_versions_from_file(config_file_name='/opt/vyatta/etc/config/config.boot'): - """ - Get component version dictionary parsing config file line by line - """ - f = open(config_file_name, 'r') - for line_in_config in f: - component_version = get_component_version(line_in_config) - if component_version: - return component_version - raise ValueError("no config string in file:", config_file_name) diff --git a/python/vyos/formatversions.py b/python/vyos/formatversions.py deleted file mode 100644 index 29117a5d3..000000000 --- a/python/vyos/formatversions.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2019 VyOS maintainers and contributors -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see . - -import sys -import os -import re -import fileinput - -def read_vyatta_versions(config_file): - config_file_versions = {} - - with open(config_file, 'r') as config_file_handle: - for config_line in config_file_handle: - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', config_line): - if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', config_line): - raise ValueError("malformed configuration string: " - "{}".format(config_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', config_line): - config_file_versions[pair[0]] = int(pair[1]) - - - return config_file_versions - -def read_vyos_versions(config_file): - config_file_versions = {} - - with open(config_file, 'r') as config_file_handle: - for config_line in config_file_handle: - if re.match(r'// vyos-config-version:.+', config_line): - if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', config_line): - raise ValueError("malformed configuration string: " - "{}".format(config_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', config_line): - config_file_versions[pair[0]] = int(pair[1]) - - return config_file_versions - -def remove_versions(config_file): - """ - Remove old version string. - """ - for line in fileinput.input(config_file, inplace=True): - if re.match(r'/\* Warning:.+ \*/$', line): - continue - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line): - continue - if re.match(r'/\* Release version:.+ \*/$', line): - continue - if re.match('// vyos-config-version:.+', line): - continue - if re.match('// Warning:.+', line): - continue - if re.match('// Release version:.+', line): - continue - sys.stdout.write(line) - -def format_versions_string(config_versions): - cfg_keys = list(config_versions.keys()) - cfg_keys.sort() - - component_version_strings = [] - - for key in cfg_keys: - cfg_vers = config_versions[key] - component_version_strings.append('{}@{}'.format(key, cfg_vers)) - - separator = ":" - component_version_string = separator.join(component_version_strings) - - return component_version_string - -def write_vyatta_versions_foot(config_file, component_version_string, - os_version_string): - if config_file: - with open(config_file, 'a') as config_file_handle: - config_file_handle.write('/* Warning: Do not remove the following line. */\n') - config_file_handle.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string)) - config_file_handle.write('/* Release version: {} */\n'.format(os_version_string)) - else: - sys.stdout.write('/* Warning: Do not remove the following line. */\n') - sys.stdout.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string)) - sys.stdout.write('/* Release version: {} */\n'.format(os_version_string)) - -def write_vyos_versions_foot(config_file, component_version_string, - os_version_string): - if config_file: - with open(config_file, 'a') as config_file_handle: - config_file_handle.write('// Warning: Do not remove the following line.\n') - config_file_handle.write('// vyos-config-version: "{}"\n'.format(component_version_string)) - config_file_handle.write('// Release version: {}\n'.format(os_version_string)) - else: - sys.stdout.write('// Warning: Do not remove the following line.\n') - sys.stdout.write('// vyos-config-version: "{}"\n'.format(component_version_string)) - sys.stdout.write('// Release version: {}\n'.format(os_version_string)) - diff --git a/python/vyos/migrator.py b/python/vyos/migrator.py index c6e3435ca..45ea8b0eb 100644 --- a/python/vyos/migrator.py +++ b/python/vyos/migrator.py @@ -1,4 +1,4 @@ -# Copyright 2019 VyOS maintainers and contributors +# Copyright 2019-2022 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -17,10 +17,8 @@ import sys import os import json import subprocess -import vyos.version import vyos.defaults -import vyos.systemversions as systemversions -import vyos.formatversions as formatversions +import vyos.component_version as component_version class MigratorError(Exception): pass @@ -42,13 +40,13 @@ class Migrator(object): cfg_file = self._config_file component_versions = {} - cfg_versions = formatversions.read_vyatta_versions(cfg_file) + cfg_versions = component_version.from_file(cfg_file, vintage='vyatta') if cfg_versions: self._config_file_vintage = 'vyatta' component_versions = cfg_versions - cfg_versions = formatversions.read_vyos_versions(cfg_file) + cfg_versions = component_version.from_file(cfg_file, vintage='vyos') if cfg_versions: self._config_file_vintage = 'vyos' @@ -157,19 +155,15 @@ class Migrator(object): """ Write new versions string. """ - versions_string = formatversions.format_versions_string(cfg_versions) - - os_version_string = vyos.version.get_version() - if self._config_file_vintage == 'vyatta': - formatversions.write_vyatta_versions_foot(self._config_file, - versions_string, - os_version_string) + component_version.write_version_footer(cfg_versions, + self._config_file, + vintage='vyatta') if self._config_file_vintage == 'vyos': - formatversions.write_vyos_versions_foot(self._config_file, - versions_string, - os_version_string) + component_version.write_version_footer(cfg_versions, + self._config_file, + vintage='vyos') def save_json_record(self, component_versions: dict): """ @@ -200,7 +194,7 @@ class Migrator(object): # This will force calling all migration scripts: cfg_versions = {} - sys_versions = systemversions.get_system_component_version() + sys_versions = component_version.from_system() # save system component versions in json file for easy reference self.save_json_record(sys_versions) @@ -216,7 +210,7 @@ class Migrator(object): if not self._changed: return - formatversions.remove_versions(cfg_file) + component_version.remove_footer(cfg_file) self.write_config_file_versions(rev_versions) @@ -237,7 +231,7 @@ class VirtualMigrator(Migrator): if not self._changed: return - formatversions.remove_versions(cfg_file) + component_version.remove_footer(cfg_file) self.write_config_file_versions(cfg_versions) diff --git a/python/vyos/systemversions.py b/python/vyos/systemversions.py deleted file mode 100644 index f2da76d4f..000000000 --- a/python/vyos/systemversions.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2019 VyOS maintainers and contributors -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see . - -import os -import re -import sys -import vyos.defaults -from vyos.xml import component_version - -# legacy version, reading from the file names in -# /opt/vyatta/etc/config-migrate/current -def get_system_versions(): - """ - Get component versions from running system; critical failure if - unable to read migration directory. - """ - system_versions = {} - - try: - version_info = os.listdir(vyos.defaults.directories['current']) - except OSError as err: - print("OS error: {}".format(err)) - sys.exit(1) - - for info in version_info: - if re.match(r'[\w,-]+@\d+', info): - pair = info.split('@') - system_versions[pair[0]] = int(pair[1]) - - return system_versions - -# read from xml cache -def get_system_component_version(): - return component_version() diff --git a/smoketest/scripts/cli/test_component_version.py b/smoketest/scripts/cli/test_component_version.py index 1355c1f94..7b1b12c53 100755 --- a/smoketest/scripts/cli/test_component_version.py +++ b/smoketest/scripts/cli/test_component_version.py @@ -16,7 +16,7 @@ import unittest -from vyos.systemversions import get_system_versions, get_system_component_version +import vyos.component_version as component_version # After T3474, component versions should be updated in the files in # vyos-1x/interface-definitions/include/version/ @@ -24,8 +24,8 @@ from vyos.systemversions import get_system_versions, get_system_component_versio # that in the xml cache. class TestComponentVersion(unittest.TestCase): def setUp(self): - self.legacy_d = get_system_versions() - self.xml_d = get_system_component_version() + self.legacy_d = component_version.legacy_from_system() + self.xml_d = component_version.from_system() self.set_legacy_d = set(self.legacy_d) self.set_xml_d = set(self.xml_d) diff --git a/src/helpers/system-versions-foot.py b/src/helpers/system-versions-foot.py index 2aa687221..9614f0d28 100755 --- a/src/helpers/system-versions-foot.py +++ b/src/helpers/system-versions-foot.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -# Copyright 2019 VyOS maintainers and contributors +# Copyright 2019, 2022 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,24 +16,13 @@ # along with this library. If not, see . import sys -import vyos.formatversions as formatversions -import vyos.systemversions as systemversions import vyos.defaults -import vyos.version - -sys_versions = systemversions.get_system_component_version() - -component_string = formatversions.format_versions_string(sys_versions) - -os_version_string = vyos.version.get_version() +from vyos.component_version import write_system_footer sys.stdout.write("\n\n") if vyos.defaults.cfg_vintage == 'vyos': - formatversions.write_vyos_versions_foot(None, component_string, - os_version_string) + write_system_footer(None, vintage='vyos') elif vyos.defaults.cfg_vintage == 'vyatta': - formatversions.write_vyatta_versions_foot(None, component_string, - os_version_string) + write_system_footer(None, vintage='vyatta') else: - formatversions.write_vyatta_versions_foot(None, component_string, - os_version_string) + write_system_footer(None, vintage='vyos') -- cgit v1.2.3 From 3f75a38abe3e97ff243d01931a4defe8f4eef98a Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Fri, 28 Oct 2022 10:38:26 -0400 Subject: T4779: add vyos.util.human_to_bytes --- python/vyos/util.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/python/vyos/util.py b/python/vyos/util.py index e4e2a44ec..a80584c5a 100644 --- a/python/vyos/util.py +++ b/python/vyos/util.py @@ -574,6 +574,37 @@ def bytes_to_human(bytes, initial_exponent=0): size_string = "{0:.2f} {1}".format(value, suffix) return size_string +def human_to_bytes(value): + """ Converts a data amount with a unit suffix to bytes, like 2K to 2048 """ + + from re import match as re_match + + res = re_match(r'^\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]+)\s*$', value) + + if not res: + raise ValueError(f"'{value}' is not a valid data amount") + else: + amount = float(res.group(1)) + unit = res.group(2).lower() + + if unit == 'b': + res = amount + elif (unit == 'k') or (unit == 'kb'): + res = amount * 1024 + elif (unit == 'm') or (unit == 'mb'): + res = amount * 1024**2 + elif (unit == 'g') or (unit == 'gb'): + res = amount * 1024**3 + elif (unit == 't') or (unit == 'tb'): + res = amount * 1024**4 + else: + raise ValueError(f"Unsupported data unit '{unit}'") + + # There cannot be fractional bytes, so we convert them to integer. + # However, truncating causes problems with conversion back to human unit, + # so we round instead -- that seems to work well enough. + return round(res) + def get_cfg_group_id(): from grp import getgrnam from vyos.defaults import cfg_group -- cgit v1.2.3 From b8b752d5b3503f2874a490582e212edd38c902fc Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Fri, 28 Oct 2022 10:39:17 -0400 Subject: T4779: switch raw output of "show system storage" to bytes --- src/op_mode/storage.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/op_mode/storage.py b/src/op_mode/storage.py index 75964c493..d16e271bd 100755 --- a/src/op_mode/storage.py +++ b/src/op_mode/storage.py @@ -20,6 +20,16 @@ import sys import vyos.opmode from vyos.util import cmd +# FIY: As of coreutils from Debian Buster and Bullseye, +# the outpt looks like this: +# +# $ df -h -t ext4 --output=source,size,used,avail,pcent +# Filesystem Size Used Avail Use% +# /dev/sda1 16G 7.6G 7.3G 51% +# +# Those field names are automatically normalized by vyos.opmode.run, +# so we don't touch them here, +# and only normalize values. def _get_system_storage(only_persistent=False): if not only_persistent: @@ -32,11 +42,19 @@ def _get_system_storage(only_persistent=False): return res def _get_raw_data(): + from re import sub as re_sub + from vyos.util import human_to_bytes + out = _get_system_storage(only_persistent=True) lines = out.splitlines() lists = [l.split() for l in lines] res = {lists[0][i]: lists[1][i] for i in range(len(lists[0]))} + res["Size"] = human_to_bytes(res["Size"]) + res["Used"] = human_to_bytes(res["Used"]) + res["Avail"] = human_to_bytes(res["Avail"]) + res["Use%"] = re_sub(r'%', '', res["Use%"]) + return res def _get_formatted_output(): -- cgit v1.2.3 From fca46598415f0c6f11c272d6b384ac98500fd69d Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Fri, 28 Oct 2022 11:04:51 -0400 Subject: T4765: handle non-string fields in the raw op mode output normalizer --- python/vyos/opmode.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index c9827d634..727e118a8 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -101,6 +101,10 @@ def _get_arg_type(t): return t def _normalize_field_name(name): + # Convert the name to string if it is not + # (in some cases they may be numbers) + name = str(name) + # Replace all separators with underscores name = re.sub(r'(\s|[\(\)\[\]\{\}\-\.\,:\"\'\`])+', '_', name) -- cgit v1.2.3 From 2b90e401455ec6a3de54e3825068632cc914143c Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Sat, 29 Oct 2022 06:15:21 -0400 Subject: T4783: add stunnel to the image --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index 0ed8f85c4..16b7ee814 100644 --- a/debian/control +++ b/debian/control @@ -154,6 +154,7 @@ Depends: ssl-cert, strongswan (>= 5.9), strongswan-swanctl (>= 5.9), + stunnel4, sudo, systemd, telegraf (>= 1.20), -- cgit v1.2.3 From f9c1277f5cf56fba2fc773d133de0221b06fa511 Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Fri, 28 Oct 2022 21:27:37 +0200 Subject: containers: T3903: Use systemd units for containers * ExecStop action with defined timeout allows for quicker reboot/shutdown with containers --- data/templates/container/systemd-unit.j2 | 17 +++ src/conf_mode/container.py | 172 ++++++++++++++++--------------- 2 files changed, 108 insertions(+), 81 deletions(-) create mode 100644 data/templates/container/systemd-unit.j2 diff --git a/data/templates/container/systemd-unit.j2 b/data/templates/container/systemd-unit.j2 new file mode 100644 index 000000000..fa48384ab --- /dev/null +++ b/data/templates/container/systemd-unit.j2 @@ -0,0 +1,17 @@ +### Autogenerated by container.py ### +[Unit] +Description=VyOS Container {{ name }} + +[Service] +Environment=PODMAN_SYSTEMD_UNIT=%n +Restart=on-failure +ExecStartPre=/bin/rm -f %t/%n.pid %t/%n.cid +ExecStart=/usr/bin/podman run \ + --conmon-pidfile %t/%n.pid --cidfile %t/%n.cid --cgroups=no-conmon \ + {{ run_args }} +ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%n.cid -t 5 +ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%n.cid +ExecStopPost=/bin/rm -f %t/%n.cid +PIDFile=%t/%n.pid +KillMode=none +Type=forking diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py index ac3dc536b..70d149f0d 100755 --- a/src/conf_mode/container.py +++ b/src/conf_mode/container.py @@ -40,20 +40,7 @@ airbag.enable() config_containers_registry = '/etc/containers/registries.conf' config_containers_storage = '/etc/containers/storage.conf' - -def _run_rerun(container_cmd): - counter = 0 - while True: - if counter >= 10: - break - try: - _cmd(container_cmd) - break - except: - counter = counter +1 - sleep(0.5) - - return None +systemd_unit_path = '/run/systemd/system' def _cmd(command): if os.path.exists('/tmp/vyos.container.debug'): @@ -122,7 +109,7 @@ def verify(container): # of image upgrade and deletion. image = container_config['image'] if run(f'podman image exists {image}') != 0: - Warning(f'Image "{image}" used in contianer "{name}" does not exist '\ + Warning(f'Image "{image}" used in container "{name}" does not exist '\ f'locally. Please use "add container image {image}" to add it '\ f'to the system! Container "{name}" will not be started!') @@ -136,9 +123,6 @@ def verify(container): raise ConfigError(f'Container network "{network_name}" does not exist!') if 'address' in container_config['network'][network_name]: - if 'network' not in container_config: - raise ConfigError(f'Can not use "address" without "network" for container "{name}"!') - address = container_config['network'][network_name]['address'] network = None if is_ipv4(address): @@ -220,6 +204,71 @@ def verify(container): return None +def generate_run_arguments(name, container_config): + image = container_config['image'] + memory = container_config['memory'] + restart = container_config['restart'] + + # Add capability options. Should be in uppercase + cap_add = '' + if 'cap_add' in container_config: + for c in container_config['cap_add']: + c = c.upper() + c = c.replace('-', '_') + cap_add += f' --cap-add={c}' + + # Add a host device to the container /dev/x:/dev/x + device = '' + if 'device' in container_config: + for dev, dev_config in container_config['device'].items(): + source_dev = dev_config['source'] + dest_dev = dev_config['destination'] + device += f' --device={source_dev}:{dest_dev}' + + # Check/set environment options "-e foo=bar" + env_opt = '' + if 'environment' in container_config: + for k, v in container_config['environment'].items(): + env_opt += f" -e \"{k}={v['value']}\"" + + # Publish ports + port = '' + if 'port' in container_config: + protocol = '' + for portmap in container_config['port']: + if 'protocol' in container_config['port'][portmap]: + protocol = container_config['port'][portmap]['protocol'] + protocol = f'/{protocol}' + else: + protocol = '/tcp' + sport = container_config['port'][portmap]['source'] + dport = container_config['port'][portmap]['destination'] + port += f' -p {sport}:{dport}{protocol}' + + # Bind volume + volume = '' + if 'volume' in container_config: + for vol, vol_config in container_config['volume'].items(): + svol = vol_config['source'] + dvol = vol_config['destination'] + volume += f' -v {svol}:{dvol}' + + container_base_cmd = f'--detach --interactive --tty --replace {cap_add} ' \ + f'--memory {memory}m --memory-swap 0 --restart {restart} ' \ + f'--name {name} {device} {port} {volume} {env_opt}' + + if 'allow_host_networks' in container_config: + return f'{container_base_cmd} --net host {image}' + + ip_param = '' + networks = ",".join(container_config['network']) + for network in container_config['network']: + if 'address' in container_config['network'][network]: + address = container_config['network'][network]['address'] + ip_param = f'--ip {address}' + + return f'{container_base_cmd} --net {networks} {ip_param} {image}' + def generate(container): # bail out early - looks like removal from running config if not container: @@ -263,6 +312,15 @@ def generate(container): render(config_containers_registry, 'container/registries.conf.j2', container) render(config_containers_storage, 'container/storage.conf.j2', container) + if 'name' in container: + for name, container_config in container['name'].items(): + if 'disable' in container_config: + continue + + file_path = os.path.join(systemd_unit_path, f'vyos-container-{name}.service') + run_args = generate_run_arguments(name, container_config) + render(file_path, 'container/systemd-unit.j2', {'name': name, 'run_args': run_args}) + return None def apply(container): @@ -270,8 +328,12 @@ def apply(container): # Option "--force" allows to delete containers with any status if 'container_remove' in container: for name in container['container_remove']: - call(f'podman stop --time 3 {name}') - call(f'podman rm --force {name}') + file_path = os.path.join(systemd_unit_path, f'vyos-container-{name}.service') + call(f'systemctl stop vyos-container-{name}.service') + if os.path.exists(file_path): + os.unlink(file_path) + + call('systemctl daemon-reload') # Delete old networks if needed if 'network_remove' in container: @@ -282,6 +344,7 @@ def apply(container): os.unlink(tmp) # Add container + disabled_new = False if 'name' in container: for name, container_config in container['name'].items(): image = container_config['image'] @@ -295,70 +358,17 @@ def apply(container): # check if there is a container by that name running tmp = _cmd('podman ps -a --format "{{.Names}}"') if name in tmp: - _cmd(f'podman stop --time 3 {name}') - _cmd(f'podman rm --force {name}') + file_path = os.path.join(systemd_unit_path, f'vyos-container-{name}.service') + call(f'systemctl stop vyos-container-{name}.service') + if os.path.exists(file_path): + disabled_new = True + os.unlink(file_path) continue - memory = container_config['memory'] - restart = container_config['restart'] - - # Add capability options. Should be in uppercase - cap_add = '' - if 'cap_add' in container_config: - for c in container_config['cap_add']: - c = c.upper() - c = c.replace('-', '_') - cap_add += f' --cap-add={c}' - - # Add a host device to the container /dev/x:/dev/x - device = '' - if 'device' in container_config: - for dev, dev_config in container_config['device'].items(): - source_dev = dev_config['source'] - dest_dev = dev_config['destination'] - device += f' --device={source_dev}:{dest_dev}' - - # Check/set environment options "-e foo=bar" - env_opt = '' - if 'environment' in container_config: - for k, v in container_config['environment'].items(): - env_opt += f" -e \"{k}={v['value']}\"" - - # Publish ports - port = '' - if 'port' in container_config: - protocol = '' - for portmap in container_config['port']: - if 'protocol' in container_config['port'][portmap]: - protocol = container_config['port'][portmap]['protocol'] - protocol = f'/{protocol}' - else: - protocol = '/tcp' - sport = container_config['port'][portmap]['source'] - dport = container_config['port'][portmap]['destination'] - port += f' -p {sport}:{dport}{protocol}' - - # Bind volume - volume = '' - if 'volume' in container_config: - for vol, vol_config in container_config['volume'].items(): - svol = vol_config['source'] - dvol = vol_config['destination'] - volume += f' -v {svol}:{dvol}' - - container_base_cmd = f'podman run --detach --interactive --tty --replace {cap_add} ' \ - f'--memory {memory}m --memory-swap 0 --restart {restart} ' \ - f'--name {name} {device} {port} {volume} {env_opt}' - if 'allow_host_networks' in container_config: - _run_rerun(f'{container_base_cmd} --net host {image}') - else: - for network in container_config['network']: - ipparam = '' - if 'address' in container_config['network'][network]: - address = container_config['network'][network]['address'] - ipparam = f'--ip {address}' + cmd(f'systemctl restart vyos-container-{name}.service') - _run_rerun(f'{container_base_cmd} --net {network} {ipparam} {image}') + if disabled_new: + call('systemctl daemon-reload') return None -- cgit v1.2.3 From ac73bc2db85bd1c7c28bd41a3f7b7e31ee57ce3f Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Sat, 29 Oct 2022 01:55:47 +0200 Subject: containers: T2216: Re-enable container smoketest using busybox image --- smoketest/scripts/cli/test_container.py | 44 ++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 6 deletions(-) mode change 100644 => 100755 smoketest/scripts/cli/test_container.py diff --git a/smoketest/scripts/cli/test_container.py b/smoketest/scripts/cli/test_container.py old mode 100644 new mode 100755 index cc0cdaec0..b9d308ae1 --- a/smoketest/scripts/cli/test_container.py +++ b/smoketest/scripts/cli/test_container.py @@ -15,6 +15,7 @@ # along with this program. If not, see . import unittest +import glob import json from base_vyostest_shim import VyOSUnitTestSHIM @@ -25,10 +26,13 @@ from vyos.util import process_named_running from vyos.util import read_file base_path = ['container'] -cont_image = 'busybox' +cont_image = 'busybox:stable' # busybox is included in vyos-build prefix = '192.168.205.0/24' net_name = 'NET01' -PROCESS_NAME = 'podman' +PROCESS_NAME = 'conmon' +PROCESS_PIDFILE = '/run/vyos-container-{0}.service.pid' + +busybox_image_path = '/usr/share/vyos/busybox-stable.tar' def cmd_to_json(command): c = cmd(command + ' --format=json') @@ -37,7 +41,31 @@ def cmd_to_json(command): return data -class TesContainer(VyOSUnitTestSHIM.TestCase): +class TestContainer(VyOSUnitTestSHIM.TestCase): + @classmethod + def setUpClass(cls): + super(TestContainer, cls).setUpClass() + + # Load image for smoketest provided in vyos-build + cmd(f'cat {busybox_image_path} | sudo podman load') + + @classmethod + def tearDownClass(cls): + super(TestContainer, cls).tearDownClass() + + # Cleanup podman image + cmd(f'sudo podman image rm -f {cont_image}') + + def tearDown(self): + self.cli_delete(base_path) + self.cli_commit() + + # Ensure no container process remains + self.assertIsNone(process_named_running(PROCESS_NAME)) + + # Ensure systemd units are removed + units = glob.glob('/run/systemd/system/vyos-container-*') + self.assertEqual(units, []) def test_01_basic_container(self): cont_name = 'c1' @@ -53,13 +81,17 @@ class TesContainer(VyOSUnitTestSHIM.TestCase): # commit changes self.cli_commit() + pid = 0 + with open(PROCESS_PIDFILE.format(cont_name), 'r') as f: + pid = int(f.read()) + # Check for running process - self.assertTrue(process_named_running(PROCESS_NAME)) + self.assertEqual(process_named_running(PROCESS_NAME), pid) def test_02_container_network(self): cont_name = 'c2' cont_ip = '192.168.205.25' - self.cli_set(base_path + ['network', net_name, 'ipv4-prefix', prefix]) + self.cli_set(base_path + ['network', net_name, 'prefix', prefix]) self.cli_set(base_path + ['name', cont_name, 'image', cont_image]) self.cli_set(base_path + ['name', cont_name, 'network', net_name, 'address', cont_ip]) @@ -67,7 +99,7 @@ class TesContainer(VyOSUnitTestSHIM.TestCase): self.cli_commit() n = cmd_to_json(f'sudo podman network inspect {net_name}') - json_subnet = n['plugins'][0]['ipam']['ranges'][0][0]['subnet'] + json_subnet = n['subnets'][0]['subnet'] c = cmd_to_json(f'sudo podman container inspect {cont_name}') json_ip = c['NetworkSettings']['Networks'][net_name]['IPAddress'] -- cgit v1.2.3 From 07afb79785ac5005a02df60df1ea427bdabe7de7 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sat, 29 Oct 2022 20:58:04 +0200 Subject: static: T4784: add description node for static route/route6 tagNodes --- interface-definitions/include/static/static-route.xml.i | 1 + interface-definitions/include/static/static-route6.xml.i | 1 + 2 files changed, 2 insertions(+) diff --git a/interface-definitions/include/static/static-route.xml.i b/interface-definitions/include/static/static-route.xml.i index 2de5dc58f..04ee999c7 100644 --- a/interface-definitions/include/static/static-route.xml.i +++ b/interface-definitions/include/static/static-route.xml.i @@ -14,6 +14,7 @@ #include #include #include + #include Next-hop IPv4 router interface diff --git a/interface-definitions/include/static/static-route6.xml.i b/interface-definitions/include/static/static-route6.xml.i index 35feef41c..6131ac7fe 100644 --- a/interface-definitions/include/static/static-route6.xml.i +++ b/interface-definitions/include/static/static-route6.xml.i @@ -13,6 +13,7 @@ #include #include + #include IPv6 gateway interface name -- cgit v1.2.3 From dda62226353ebc198b4dbbd319412bb5d1d1ece2 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sat, 29 Oct 2022 20:58:37 +0200 Subject: snmp: T4785: allow ! in community name --- interface-definitions/snmp.xml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface-definitions/snmp.xml.in b/interface-definitions/snmp.xml.in index b4f72589e..91c2715a0 100644 --- a/interface-definitions/snmp.xml.in +++ b/interface-definitions/snmp.xml.in @@ -13,7 +13,7 @@ Community name - [a-zA-Z0-9\-_]{1,100} + [a-zA-Z0-9\-_!]{1,100} Community string is limited to alphanumerical characters only with a total lenght of 100 -- cgit v1.2.3 From 3f91033927d80748b70e1ef58b2941643d1aca33 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sun, 30 Oct 2022 12:50:36 +0100 Subject: snmp: T4785: allow @, * and # in SNMP community name --- interface-definitions/snmp.xml.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface-definitions/snmp.xml.in b/interface-definitions/snmp.xml.in index 91c2715a0..7ec60b2e7 100644 --- a/interface-definitions/snmp.xml.in +++ b/interface-definitions/snmp.xml.in @@ -13,9 +13,9 @@ Community name - [a-zA-Z0-9\-_!]{1,100} + [a-zA-Z0-9\-_!@*#]{1,100} - Community string is limited to alphanumerical characters only with a total lenght of 100 + Community string is limited to alphanumerical characters, !, @, * and # with a total lenght of 100 -- cgit v1.2.3 From a3ae748608097170063888ce121579ed5a315744 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 31 Oct 2022 10:58:34 +0000 Subject: T4786: Add package python3-pyhumps humps Convert strings (and dictionary keys) between snake case, camel case and pascal case in Python % decamelize('superTCPOption') 'super_tcp_option' % % decamelize({'ParamOption': 'one', 'fooBarBaz': True}) {'param_option': 'one', 'foo_bar_baz': True} % --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index 16b7ee814..cf766a825 100644 --- a/debian/control +++ b/debian/control @@ -131,6 +131,7 @@ Depends: python3-netifaces, python3-paramiko, python3-psutil, + python3-pyhumps, python3-pystache, python3-pyudev, python3-six, -- cgit v1.2.3 From 1afb3f8bd5de3748c5b37462eb42235d721d4963 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Fri, 28 Oct 2022 13:07:30 +0000 Subject: T4771: Ability to get raw format for op-mode BGP commands --- data/op-mode-standardized.json | 1 + src/op_mode/bgp.py | 120 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100755 src/op_mode/bgp.py diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index 9500d3aa7..c5e9f9243 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -1,4 +1,5 @@ [ +"bgp.py", "bridge.py", "conntrack.py", "container.py", diff --git a/src/op_mode/bgp.py b/src/op_mode/bgp.py new file mode 100755 index 000000000..23001a9d7 --- /dev/null +++ b/src/op_mode/bgp.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Purpose: +# Displays bgp neighbors information. +# Used by the "show bgp (vrf ) ipv4|ipv6 neighbors" commands. + +import re +import sys +import typing + +import jmespath +from jinja2 import Template +from humps import decamelize + +from vyos.configquery import ConfigTreeQuery + +import vyos.opmode + + +frr_command_template = Template(""" +{% if family %} + show bgp + {{ 'vrf ' ~ vrf if vrf else '' }} + {{ 'ipv6' if family == 'inet6' else 'ipv4'}} + {{ 'neighbor ' ~ peer if peer else 'summary' }} +{% endif %} + +{% if raw %} + json +{% endif %} +""") + + +def _verify(func): + """Decorator checks if BGP config exists + BGP configuration can be present under vrf + If we do npt get arg 'peer' then it can be 'bgp summary' + """ + from functools import wraps + + @wraps(func) + def _wrapper(*args, **kwargs): + config = ConfigTreeQuery() + afi = 'ipv6' if kwargs.get('family') == 'inet6' else 'ipv4' + global_vrfs = ['all', 'default'] + peer = kwargs.get('peer') + vrf = kwargs.get('vrf') + unconf_message = f'BGP or neighbor is not configured' + # Add option to check the specific neighbor if we have arg 'peer' + peer_opt = f'neighbor {peer} address-family {afi}-unicast' if peer else '' + vrf_opt = '' + if vrf and vrf not in global_vrfs: + vrf_opt = f'vrf name {vrf}' + # Check if config does not exist + if not config.exists(f'{vrf_opt} protocols bgp {peer_opt}'): + raise vyos.opmode.UnconfiguredSubsystem(unconf_message) + return func(*args, **kwargs) + + return _wrapper + + +@_verify +def show_neighbors(raw: bool, + family: str, + peer: typing.Optional[str], + vrf: typing.Optional[str]): + kwargs = dict(locals()) + frr_command = frr_command_template.render(kwargs) + frr_command = re.sub(r'\s+', ' ', frr_command) + + from vyos.util import cmd + output = cmd(f"vtysh -c '{frr_command}'") + + if raw: + from json import loads + data = loads(output) + # Get list of the peers + peers = jmespath.search('*.peers | [0]', data) + if peers: + # Create new dict, delete old key 'peers' + # add key 'peers' neighbors to the list + list_peers = [] + new_dict = jmespath.search('* | [0]', data) + if 'peers' in new_dict: + new_dict.pop('peers') + + for neighbor, neighbor_options in peers.items(): + neighbor_options['neighbor'] = neighbor + list_peers.append(neighbor_options) + new_dict['peers'] = list_peers + return decamelize(new_dict) + data = jmespath.search('* | [0]', data) + return decamelize(data) + + else: + return output + + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) -- cgit v1.2.3 From 61bc664137d73c6b73e9db5157e1a7a79ab5cebd Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Mon, 31 Oct 2022 08:14:05 -0400 Subject: T4526: use informative error messages for keepalived-fifo with commit in progress --- src/system/keepalived-fifo.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/system/keepalived-fifo.py b/src/system/keepalived-fifo.py index a0fccd1d0..864ee8419 100755 --- a/src/system/keepalived-fifo.py +++ b/src/system/keepalived-fifo.py @@ -67,13 +67,13 @@ class KeepalivedFifo: # For VRRP configuration to be read, the commit must be finished count = 1 while commit_in_progress(): - if ( count <= 40 ): - logger.debug(f'commit in progress try: {count}') + if ( count <= 20 ): + logger.debug(f'Attempt to load keepalived configuration aborted due to a commit in progress (attempt {count}/20)') else: - logger.error(f'commit still in progress after {count} continuing anyway') + logger.error(f'Forced keepalived configuration loading despite a commit in progress ({count} wait time expired, not waiting further)') break count += 1 - time.sleep(0.5) + time.sleep(1) try: base = ['high-availability', 'vrrp'] -- cgit v1.2.3 From 22c3dcbb01d731f0dab0ffefa2e5a0be7009baf1 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Mon, 31 Oct 2022 15:09:58 +0100 Subject: ipsec: T4787: add support for road-warrior/remote-access RADIUS timeout This enabled users to also use 2FA/MFA authentication with a radius backend as there is enough time to enter the second factor. --- data/templates/ipsec/charon/eap-radius.conf.j2 | 4 +++- interface-definitions/include/radius-timeout.xml.i | 16 ++++++++++++++++ interface-definitions/vpn-ipsec.xml.in | 1 + interface-definitions/vpn-openconnect.xml.in | 15 +-------------- src/conf_mode/vpn_ipsec.py | 17 +++++++++++++++-- 5 files changed, 36 insertions(+), 17 deletions(-) create mode 100644 interface-definitions/include/radius-timeout.xml.i diff --git a/data/templates/ipsec/charon/eap-radius.conf.j2 b/data/templates/ipsec/charon/eap-radius.conf.j2 index 8495011fe..364377473 100644 --- a/data/templates/ipsec/charon/eap-radius.conf.j2 +++ b/data/templates/ipsec/charon/eap-radius.conf.j2 @@ -49,8 +49,10 @@ eap-radius { # Base to use for calculating exponential back off. # retransmit_base = 1.4 +{% if remote_access.radius.timeout is vyos_defined %} # Timeout in seconds before sending first retransmit. - # retransmit_timeout = 2.0 + retransmit_timeout = {{ remote_access.radius.timeout | float }} +{% endif %} # Number of times to retransmit a packet before giving up. # retransmit_tries = 4 diff --git a/interface-definitions/include/radius-timeout.xml.i b/interface-definitions/include/radius-timeout.xml.i new file mode 100644 index 000000000..22bb6d312 --- /dev/null +++ b/interface-definitions/include/radius-timeout.xml.i @@ -0,0 +1,16 @@ + + + + Session timeout + + u32:1-240 + Session timeout in seconds (default: 2) + + + + + Timeout must be between 1 and 240 seconds + + 2 + + diff --git a/interface-definitions/vpn-ipsec.xml.in b/interface-definitions/vpn-ipsec.xml.in index 4776c53dc..64966b540 100644 --- a/interface-definitions/vpn-ipsec.xml.in +++ b/interface-definitions/vpn-ipsec.xml.in @@ -888,6 +888,7 @@ #include + #include #include diff --git a/interface-definitions/vpn-openconnect.xml.in b/interface-definitions/vpn-openconnect.xml.in index 3b3a83bd4..8b60f2e6e 100644 --- a/interface-definitions/vpn-openconnect.xml.in +++ b/interface-definitions/vpn-openconnect.xml.in @@ -140,20 +140,7 @@ #include - - - Session timeout - - u32:1-240 - Session timeout in seconds (default: 2) - - - - - Timeout must be between 1 and 240 seconds - - 2 - + #include If the groupconfig option is set, then config-per-user will be overriden, and all configuration will be read from RADIUS. diff --git a/src/conf_mode/vpn_ipsec.py b/src/conf_mode/vpn_ipsec.py index 77a425f8b..cfefcfbe8 100755 --- a/src/conf_mode/vpn_ipsec.py +++ b/src/conf_mode/vpn_ipsec.py @@ -117,13 +117,26 @@ def get_config(config=None): ipsec['ike_group'][group]['proposal'][proposal] = dict_merge(default_values, ipsec['ike_group'][group]['proposal'][proposal]) - if 'remote_access' in ipsec and 'connection' in ipsec['remote_access']: + # XXX: T2665: we can not safely rely on the defaults() when there are + # tagNodes in place, it is better to blend in the defaults manually. + if dict_search('remote_access.connection', ipsec): default_values = defaults(base + ['remote-access', 'connection']) for rw in ipsec['remote_access']['connection']: ipsec['remote_access']['connection'][rw] = dict_merge(default_values, ipsec['remote_access']['connection'][rw]) - if 'remote_access' in ipsec and 'radius' in ipsec['remote_access'] and 'server' in ipsec['remote_access']['radius']: + # XXX: T2665: we can not safely rely on the defaults() when there are + # tagNodes in place, it is better to blend in the defaults manually. + if dict_search('remote_access.radius.server', ipsec): + # Fist handle the "base" stuff like RADIUS timeout + default_values = defaults(base + ['remote-access', 'radius']) + if 'server' in default_values: + del default_values['server'] + ipsec['remote_access']['radius'] = dict_merge(default_values, + ipsec['remote_access']['radius']) + + # Take care about individual RADIUS servers implemented as tagNodes - this + # requires special treatment default_values = defaults(base + ['remote-access', 'radius', 'server']) for server in ipsec['remote_access']['radius']['server']: ipsec['remote_access']['radius']['server'][server] = dict_merge(default_values, -- cgit v1.2.3 From f50f7b043a8636a57fc61330d94550734d2826b5 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Tue, 1 Nov 2022 09:03:40 +0100 Subject: login: T4750: add ecdsa-sk and ed25519-sk as supported public key type --- interface-definitions/system-login.xml.in | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/interface-definitions/system-login.xml.in b/interface-definitions/system-login.xml.in index def42544a..027d3f587 100644 --- a/interface-definitions/system-login.xml.in +++ b/interface-definitions/system-login.xml.in @@ -127,32 +127,44 @@ - Public key type + SSH public key type - ssh-dss ssh-rsa ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 ssh-ed25519 + ssh-dss ssh-rsa ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521 ssh-ed25519 ecdsa-sk ed25519-sk ssh-dss - + Digital Signature Algorithm (DSA) key support ssh-rsa - + Key pair based on RSA algorithm ecdsa-sha2-nistp256 - + Elliptic Curve DSA with NIST P-256 curve ecdsa-sha2-nistp384 - + Elliptic Curve DSA with NIST P-384 curve + + + ecdsa-sha2-nistp521 + Elliptic Curve DSA with NIST P-521 curve ssh-ed25519 - + Edwards-curve DSA with elliptic curve 25519 + + + ecdsa-sk + Elliptic Curve DSA security key + + + ed25519-sk + Elliptic curve 25519 security key - (ssh-dss|ssh-rsa|ecdsa-sha2-nistp256|ecdsa-sha2-nistp384|ecdsa-sha2-nistp521|ssh-ed25519) + (ssh-dss|ssh-rsa|ecdsa-sha2-nistp256|ecdsa-sha2-nistp384|ecdsa-sha2-nistp521|ssh-ed25519|ecdsa-sk|ed25519-sk) -- cgit v1.2.3 From f489c5ecdab5bdd8a5faa130f4c79a6f4559353b Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Tue, 1 Nov 2022 17:07:24 +0000 Subject: T4777: Ability to get logs in machine-readable format Ability to get logs in JSON format Possible filter by unit. Options for count lines, UTC time, facility or logs since boot --- data/op-mode-standardized.json | 1 + src/op_mode/log.py | 94 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100755 src/op_mode/log.py diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index 9500d3aa7..a34c3f481 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -3,6 +3,7 @@ "conntrack.py", "container.py", "cpu.py", +"log.py", "memory.py", "nat.py", "neighbor.py", diff --git a/src/op_mode/log.py b/src/op_mode/log.py new file mode 100755 index 000000000..b0abd6191 --- /dev/null +++ b/src/op_mode/log.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import json +import re +import sys +import typing + +from jinja2 import Template + +from vyos.util import rc_cmd + +import vyos.opmode + +journalctl_command_template = Template(""" +--no-hostname +--quiet + +{% if boot %} + --boot +{% endif %} + +{% if count %} + --lines={{ count }} +{% endif %} + +{% if reverse %} + --reverse +{% endif %} + +{% if since %} + --since={{ since }} +{% endif %} + +{% if unit %} + --unit={{ unit }} +{% endif %} + +{% if utc %} + --utc +{% endif %} + +{% if raw %} +{# By default show 100 only lines for raw option if count does not set #} +{# Protection from parsing the full log by default #} +{% if not boot %} + --lines={{ '' ~ count if count else '100' }} +{% endif %} + --no-pager + --output=json +{% endif %} +""") + + +def show(raw: bool, + boot: typing.Optional[bool], + count: typing.Optional[int], + facility: typing.Optional[str], + reverse: typing.Optional[bool], + utc: typing.Optional[bool], + unit: typing.Optional[str]): + kwargs = dict(locals()) + + journalctl_options = journalctl_command_template.render(kwargs) + journalctl_options = re.sub(r'\s+', ' ', journalctl_options) + rc, output = rc_cmd(f'journalctl {journalctl_options}') + if raw: + # Each 'journalctl --output json' line is a separate JSON object + # So we should return list of dict + return [json.loads(line) for line in output.split('\n')] + return output + + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) -- cgit v1.2.3