From b147c020bae07cc58bd9ec96b781e79b732c102b Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sun, 16 Oct 2022 22:02:53 +0200 Subject: xdp: T4284: migrate to Debian libbpf In order to properly retrieve JSON information in the Smoketests for the new QoS implementation we need a recent (>6.0) version of iproute2. This requires the libbpf-dev package and this small source-code change. --- src/xdp/common/common.mk | 2 +- src/xdp/common/common_user_bpf_xdp.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/xdp/common/common.mk b/src/xdp/common/common.mk index ebe23a9ed..ffb86a65c 100644 --- a/src/xdp/common/common.mk +++ b/src/xdp/common/common.mk @@ -39,7 +39,7 @@ KERN_USER_H ?= $(wildcard common_kern_user.h) CFLAGS ?= -g -I../include/ BPF_CFLAGS ?= -I../include/ -LIBS = -l:libbpf.a -lelf $(USER_LIBS) +LIBS = -lbpf -lelf $(USER_LIBS) all: llvm-check $(USER_TARGETS) $(XDP_OBJ) $(COPY_LOADER) $(COPY_STATS) diff --git a/src/xdp/common/common_user_bpf_xdp.c b/src/xdp/common/common_user_bpf_xdp.c index e7ef77174..faf7f4f91 100644 --- a/src/xdp/common/common_user_bpf_xdp.c +++ b/src/xdp/common/common_user_bpf_xdp.c @@ -274,7 +274,7 @@ struct bpf_object *load_bpf_and_xdp_attach(struct config *cfg) exit(EXIT_FAIL_BPF); } - strncpy(cfg->progsec, bpf_program__title(bpf_prog, false), sizeof(cfg->progsec)); + strncpy(cfg->progsec, bpf_program__section_name(bpf_prog), sizeof(cfg->progsec)); prog_fd = bpf_program__fd(bpf_prog); if (prog_fd <= 0) { -- cgit v1.2.3 From 8cafffff3169018abb09bb1652a63dc4be01b983 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 16 Oct 2022 15:56:49 -0500 Subject: graphql: T4753: generalize system_status to composite_{query,mutation} --- src/services/api/graphql/graphql/directives.py | 15 ++- src/services/api/graphql/graphql/mutations.py | 4 + src/services/api/graphql/graphql/queries.py | 5 +- .../api/graphql/graphql/schema/composite.graphql | 18 ++++ .../api/graphql/graphql/schema/schema.graphql | 8 +- .../graphql/graphql/schema/system_status.graphql | 18 ---- .../api/graphql/utils/composite_function.py | 11 ++ .../api/graphql/utils/schema_from_composite.py | 119 +++++++++++++++++++++ 8 files changed, 170 insertions(+), 28 deletions(-) create mode 100644 src/services/api/graphql/graphql/schema/composite.graphql delete mode 100644 src/services/api/graphql/graphql/schema/system_status.graphql create mode 100644 src/services/api/graphql/utils/composite_function.py create mode 100755 src/services/api/graphql/utils/schema_from_composite.py (limited to 'src') diff --git a/src/services/api/graphql/graphql/directives.py b/src/services/api/graphql/graphql/directives.py index d75d72582..a7919854a 100644 --- a/src/services/api/graphql/graphql/directives.py +++ b/src/services/api/graphql/graphql/directives.py @@ -63,16 +63,25 @@ class GenOpMutationDirective(VyosDirective): super().visit_field_definition(field, object_type, make_resolver=make_gen_op_mutation_resolver) -class SystemStatusDirective(VyosDirective): +class CompositeQueryDirective(VyosDirective): """ Class providing implementation of 'system_status' directive in schema. """ def visit_field_definition(self, field, object_type): super().visit_field_definition(field, object_type, - make_resolver=make_system_status_resolver) + make_resolver=make_composite_query_resolver) + +class CompositeMutationDirective(VyosDirective): + """ + Class providing implementation of 'system_status' directive in schema. + """ + def visit_field_definition(self, field, object_type): + super().visit_field_definition(field, object_type, + make_resolver=make_composite_mutation_resolver) directives_dict = {"configsessionquery": ConfigSessionQueryDirective, "configsessionmutation": ConfigSessionMutationDirective, "genopquery": GenOpQueryDirective, "genopmutation": GenOpMutationDirective, - "systemstatus": SystemStatusDirective} + "compositequery": CompositeQueryDirective, + "compositemutation": CompositeMutationDirective} diff --git a/src/services/api/graphql/graphql/mutations.py b/src/services/api/graphql/graphql/mutations.py index f7d285a77..32da0eeb7 100644 --- a/src/services/api/graphql/graphql/mutations.py +++ b/src/services/api/graphql/graphql/mutations.py @@ -112,3 +112,7 @@ def make_config_session_mutation_resolver(mutation_name): def make_gen_op_mutation_resolver(mutation_name): return make_mutation_resolver(mutation_name, mutation_name, 'gen_op_mutation') + +def make_composite_mutation_resolver(mutation_name): + return make_mutation_resolver(mutation_name, mutation_name, + convert_camel_case_to_snake(mutation_name)) diff --git a/src/services/api/graphql/graphql/queries.py b/src/services/api/graphql/graphql/queries.py index 5f3a7d005..791b0d3e0 100644 --- a/src/services/api/graphql/graphql/queries.py +++ b/src/services/api/graphql/graphql/queries.py @@ -113,5 +113,6 @@ def make_config_session_query_resolver(query_name): def make_gen_op_query_resolver(query_name): return make_query_resolver(query_name, query_name, 'gen_op_query') -def make_system_status_resolver(query_name): - return make_query_resolver(query_name, query_name, 'system_status') +def make_composite_query_resolver(query_name): + return make_query_resolver(query_name, query_name, + convert_camel_case_to_snake(query_name)) diff --git a/src/services/api/graphql/graphql/schema/composite.graphql b/src/services/api/graphql/graphql/schema/composite.graphql new file mode 100644 index 000000000..717fbd89d --- /dev/null +++ b/src/services/api/graphql/graphql/schema/composite.graphql @@ -0,0 +1,18 @@ + +input SystemStatusInput { + key: String! +} + +type SystemStatus { + result: Generic +} + +type SystemStatusResult { + data: SystemStatus + success: Boolean! + errors: [String] +} + +extend type Query { + SystemStatus(data: SystemStatusInput) : SystemStatusResult @compositequery +} \ No newline at end of file diff --git a/src/services/api/graphql/graphql/schema/schema.graphql b/src/services/api/graphql/graphql/schema/schema.graphql index 2acecade4..62b0d30bb 100644 --- a/src/services/api/graphql/graphql/schema/schema.graphql +++ b/src/services/api/graphql/graphql/schema/schema.graphql @@ -3,7 +3,8 @@ schema { mutation: Mutation } -directive @systemstatus on FIELD_DEFINITION +directive @compositequery on FIELD_DEFINITION +directive @compositemutation on FIELD_DEFINITION directive @configsessionquery on FIELD_DEFINITION directive @configsessionmutation on FIELD_DEFINITION directive @genopquery on FIELD_DEFINITION @@ -11,8 +12,5 @@ directive @genopmutation on FIELD_DEFINITION scalar Generic -type Query { - SystemStatus(data: SystemStatusInput) : SystemStatusResult @systemstatus -} - +type Query type Mutation diff --git a/src/services/api/graphql/graphql/schema/system_status.graphql b/src/services/api/graphql/graphql/schema/system_status.graphql deleted file mode 100644 index be8d87535..000000000 --- a/src/services/api/graphql/graphql/schema/system_status.graphql +++ /dev/null @@ -1,18 +0,0 @@ -""" -Use 'scalar Generic' for system status output, to avoid attempts to -JSON-serialize in case of JSON output. -""" - -input SystemStatusInput { - key: String! -} - -type SystemStatus { - result: Generic -} - -type SystemStatusResult { - data: SystemStatus - success: Boolean! - errors: [String] -} diff --git a/src/services/api/graphql/utils/composite_function.py b/src/services/api/graphql/utils/composite_function.py new file mode 100644 index 000000000..bc9d80fbb --- /dev/null +++ b/src/services/api/graphql/utils/composite_function.py @@ -0,0 +1,11 @@ +# typing information for composite functions: those that invoke several +# elementary requests, and return the result as a single dict +import typing + +def system_status(): + pass + +queries = {'system_status': system_status} + +mutations = {} + diff --git a/src/services/api/graphql/utils/schema_from_composite.py b/src/services/api/graphql/utils/schema_from_composite.py new file mode 100755 index 000000000..f9983cd98 --- /dev/null +++ b/src/services/api/graphql/utils/schema_from_composite.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# A utility to generate GraphQL schema defintions from typing information of +# composite functions comprising several requests. + +import os +import json +from inspect import signature, getmembers, isfunction, isclass, getmro +from jinja2 import Template + +if __package__ is None or __package__ == '': + from util import snake_to_pascal_case, map_type_name +else: + from . util import snake_to_pascal_case, map_type_name + +# this will be run locally before the build +SCHEMA_PATH = '../graphql/schema' + +schema_data: dict = {'schema_name': '', + 'schema_fields': []} + +query_template = """ +input {{ schema_name }}Input { + key: String! + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} + +type {{ schema_name }} { + result: Generic +} + +type {{ schema_name }}Result { + data: {{ schema_name }} + success: Boolean! + errors: [String] +} + +extend type Query { + {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @compositequery +} +""" + +mutation_template = """ +input {{ schema_name }}Input { + key: String! + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} + +type {{ schema_name }} { + result: Generic +} + +type {{ schema_name }}Result { + data: {{ schema_name }} + success: Boolean! + errors: [String] +} + +extend type Mutation { + {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @compositemutation +} +""" + +def create_schema(func_name: str, func: callable, template: str) -> str: + sig = signature(func) + + field_dict = {} + for k in sig.parameters: + field_dict[sig.parameters[k].name] = map_type_name(sig.parameters[k].annotation) + + schema_fields = [] + for k,v in field_dict.items(): + schema_fields.append(k+': '+v) + + schema_data['schema_name'] = snake_to_pascal_case(func_name) + schema_data['schema_fields'] = schema_fields + + j2_template = Template(template) + res = j2_template.render(schema_data) + + return res + +def generate_composite_definitions(): + from composite_function import queries, mutations + + results = [] + for name,func in queries.items(): + res = create_schema(name, func, query_template) + results.append(res) + + for name,func in mutations.items(): + res = create_schema(name, func, mutation_template) + results.append(res) + + out = '\n'.join(results) + with open(f'{SCHEMA_PATH}/composite.graphql', 'w') as f: + f.write(out) + +if __name__ == '__main__': + generate_composite_definitions() -- cgit v1.2.3 From 36c475ec3524739f9ae49420e60a57a5266fa575 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Thu, 20 Oct 2022 09:14:53 -0400 Subject: T4765: normalize dict fields in op mode ouputs --- python/vyos/opmode.py | 41 +++++++++++++++++++++++++++++++++++++++++ src/tests/test_op_mode.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 src/tests/test_op_mode.py (limited to 'src') diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index 7e3545c87..ac9c0c353 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -44,6 +44,13 @@ class PermissionDenied(Error): """ pass +class InternalError(Error): + """ Any situation when VyOS detects that it could not perform + an operation correctly due to logic errors in its own code + or errors in underlying software. + """ + pass + def _is_op_mode_function_name(name): if re.match(r"^(show|clear|reset|restart)", name): @@ -93,6 +100,39 @@ def _get_arg_type(t): else: return t +def _normalize_field_name(name): + # Replace all separators with underscores + name = re.sub(r'(\s|[\(\)\[\]\{\}\-\.\,:\"\'\`])+', '_', name) + + # Replace specific characters with textual descriptions + name = re.sub(r'@', '_at_', name) + name = re.sub(r'%', '_percentage_', name) + name = re.sub(r'~', '_tilde_', name) + + # Force all letters to lowercase + name = name.lower() + + # Remove leading and trailing underscores, if any + name = re.sub(r'(^(_+)(?=[^_])|_+$)', '', name) + + # Ensure there are only single underscores + name = re.sub(r'_+', '_', name) + + return name + +def _normalize_field_names(old_dict): + new_dict = {} + + for key in old_dict: + new_key = _normalize_field_name(key) + new_dict[new_key] = old_dict[key] + + # Sanity check + if len(old_dict) != len(new_dict): + raise InternalError("Dictionary fields do not allow unique normalization") + else: + return new_dict + def run(module): from argparse import ArgumentParser @@ -145,6 +185,7 @@ def run(module): # they may return human-formatted output # or a raw dict that we need to serialize in JSON for printing res = func(**args) + res = _normalize_field_names(res) if not args["raw"]: return res else: diff --git a/src/tests/test_op_mode.py b/src/tests/test_op_mode.py new file mode 100644 index 000000000..4786357c5 --- /dev/null +++ b/src/tests/test_op_mode.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from unittest import TestCase + +import vyos.opmode + +class TestVyOSOpMode(TestCase): + def test_field_name_normalization(self): + from vyos.opmode import _normalize_field_name + + self.assertEqual(_normalize_field_name(" foo bar "), "foo_bar") + self.assertEqual(_normalize_field_name("foo-bar"), "foo_bar") + self.assertEqual(_normalize_field_name("foo (bar) baz"), "foo_bar_baz") + self.assertEqual(_normalize_field_name("load%"), "load_percentage") + + def test_dict_fields_normalization_non_unique(self): + from vyos.opmode import _normalize_field_names + + # Space and dot are both replaced by an underscore, + # so dicts like this cannor be normalized uniquely + data = {"foo bar": True, "foo.bar": False} + + with self.assertRaises(vyos.opmode.InternalError): + _normalize_field_names(data) + + def test_dict_fields_normalization(self): + from vyos.opmode import _normalize_field_names + + data = {"foo bar": True, "bar-baz": False} + self.assertEqual(_normalize_field_names(data), {"foo_bar": True, "bar_baz": False}) -- cgit v1.2.3 From 89fbe73b9fb9ad178a2a35bdf9c7c477dc72f054 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 21 Oct 2022 08:41:26 -0500 Subject: graphql: T4768: change name of api child node from 'gql' to 'graphql' --- interface-definitions/https.xml.in | 2 +- .../include/version/https-version.xml.i | 2 +- smoketest/scripts/cli/test_service_https.py | 10 ++-- src/conf_mode/http-api.py | 2 +- src/migration-scripts/https/3-to-4 | 53 ++++++++++++++++++++++ src/services/vyos-http-api-server | 10 ++-- 6 files changed, 66 insertions(+), 13 deletions(-) create mode 100755 src/migration-scripts/https/3-to-4 (limited to 'src') diff --git a/interface-definitions/https.xml.in b/interface-definitions/https.xml.in index d096c4ff1..28656b594 100644 --- a/interface-definitions/https.xml.in +++ b/interface-definitions/https.xml.in @@ -107,7 +107,7 @@ - + GraphQL support diff --git a/interface-definitions/include/version/https-version.xml.i b/interface-definitions/include/version/https-version.xml.i index 586083649..111076974 100644 --- a/interface-definitions/include/version/https-version.xml.i +++ b/interface-definitions/include/version/https-version.xml.i @@ -1,3 +1,3 @@ - + diff --git a/smoketest/scripts/cli/test_service_https.py b/smoketest/scripts/cli/test_service_https.py index 72c1d4e43..719125f0f 100755 --- a/smoketest/scripts/cli/test_service_https.py +++ b/smoketest/scripts/cli/test_service_https.py @@ -143,10 +143,10 @@ class TestHTTPSService(VyOSUnitTestSHIM.TestCase): # caught by the resolver, and returns success 'False', so one must # check the return value. - self.cli_set(base_path + ['api', 'gql']) + self.cli_set(base_path + ['api', 'graphql']) self.cli_commit() - gql_url = f'https://{address}/graphql' + graphql_url = f'https://{address}/graphql' query_valid_key = f""" {{ @@ -160,7 +160,7 @@ class TestHTTPSService(VyOSUnitTestSHIM.TestCase): }} """ - r = request('POST', gql_url, verify=False, headers=headers, json={'query': query_valid_key}) + r = request('POST', graphql_url, verify=False, headers=headers, json={'query': query_valid_key}) success = r.json()['data']['SystemStatus']['success'] self.assertTrue(success) @@ -176,7 +176,7 @@ class TestHTTPSService(VyOSUnitTestSHIM.TestCase): } """ - r = request('POST', gql_url, verify=False, headers=headers, json={'query': query_invalid_key}) + r = request('POST', graphql_url, verify=False, headers=headers, json={'query': query_invalid_key}) success = r.json()['data']['SystemStatus']['success'] self.assertFalse(success) @@ -192,7 +192,7 @@ class TestHTTPSService(VyOSUnitTestSHIM.TestCase): } """ - r = request('POST', gql_url, verify=False, headers=headers, json={'query': query_no_key}) + r = request('POST', graphql_url, verify=False, headers=headers, json={'query': query_no_key}) self.assertEqual(r.status_code, 400) if __name__ == '__main__': diff --git a/src/conf_mode/http-api.py b/src/conf_mode/http-api.py index c196e272b..be80613c6 100755 --- a/src/conf_mode/http-api.py +++ b/src/conf_mode/http-api.py @@ -86,7 +86,7 @@ def get_config(config=None): if 'api_keys' in api_dict: keys_added = True - if 'gql' in api_dict: + if 'graphql' in api_dict: api_dict = dict_merge(defaults(base), api_dict) http_api.update(api_dict) diff --git a/src/migration-scripts/https/3-to-4 b/src/migration-scripts/https/3-to-4 new file mode 100755 index 000000000..5ee528b31 --- /dev/null +++ b/src/migration-scripts/https/3-to-4 @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# T4768 rename node 'gql' to 'graphql'. + +import sys + +from vyos.configtree import ConfigTree + +if (len(sys.argv) < 2): + print("Must specify file name!") + sys.exit(1) + +file_name = sys.argv[1] + +with open(file_name, 'r') as f: + config_file = f.read() + +config = ConfigTree(config_file) + +old_base = ['service', 'https', 'api', 'gql'] +if not config.exists(old_base): + # Nothing to do + sys.exit(0) + +new_base = ['service', 'https', 'api', 'graphql'] +config.set(new_base) + +nodes = config.list_nodes(old_base) +for node in nodes: + config.copy(old_base + [node], new_base + [node]) + +config.delete(old_base) + +try: + with open(file_name, 'w') as f: + f.write(config.to_string()) +except OSError as e: + print("Failed to save the modified config: {}".format(e)) + sys.exit(1) diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server index 4ace981ca..632c1e87d 100755 --- a/src/services/vyos-http-api-server +++ b/src/services/vyos-http-api-server @@ -688,16 +688,16 @@ if __name__ == '__main__': app.state.vyos_debug = server_config['debug'] app.state.vyos_strict = server_config['strict'] app.state.vyos_origins = server_config.get('cors', {}).get('allow_origin', []) - if 'gql' in server_config: - app.state.vyos_gql = True - if isinstance(server_config['gql'], dict) and 'introspection' in server_config['gql']: + if 'graphql' in server_config: + app.state.vyos_graphql = True + if isinstance(server_config['graphql'], dict) and 'introspection' in server_config['graphql']: app.state.vyos_introspection = True else: app.state.vyos_introspection = False else: - app.state.vyos_gql = False + app.state.vyos_graphql = False - if app.state.vyos_gql: + if app.state.vyos_graphql: graphql_init(app) try: -- cgit v1.2.3 From b6d2e0a4b08c81814cb2d9b5b611cbc3fc31dbeb Mon Sep 17 00:00:00 2001 From: create with ansible Date: Fri, 21 Oct 2022 11:50:17 -0400 Subject: T4765: support list and primitives in op mode output normalization --- python/vyos/opmode.py | 14 ++++++++++---- src/tests/test_op_mode.py | 25 +++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index 5f9c2c1ce..c9827d634 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -120,12 +120,12 @@ def _normalize_field_name(name): return name -def _normalize_field_names(old_dict): +def _normalize_dict_field_names(old_dict): new_dict = {} for key in old_dict: new_key = _normalize_field_name(key) - new_dict[new_key] = old_dict[key] + new_dict[new_key] = _normalize_field_names(old_dict[key]) # Sanity check if len(old_dict) != len(new_dict): @@ -133,6 +133,14 @@ def _normalize_field_names(old_dict): else: return new_dict +def _normalize_field_names(value): + if isinstance(value, dict): + return _normalize_dict_field_names(value) + elif isinstance(value, list): + return list(map(lambda v: _normalize_field_names(v), value)) + else: + return value + def run(module): from argparse import ArgumentParser @@ -188,8 +196,6 @@ def run(module): if not args["raw"]: return res else: - if not isinstance(res, dict): - raise InternalError("'raw' output of 'show_*' command must be a dict") res = _normalize_field_names(res) from json import dumps return dumps(res, indent=4) diff --git a/src/tests/test_op_mode.py b/src/tests/test_op_mode.py index 4786357c5..90963b3c5 100644 --- a/src/tests/test_op_mode.py +++ b/src/tests/test_op_mode.py @@ -37,8 +37,29 @@ class TestVyOSOpMode(TestCase): with self.assertRaises(vyos.opmode.InternalError): _normalize_field_names(data) - def test_dict_fields_normalization(self): + def test_dict_fields_normalization_simple_dict(self): from vyos.opmode import _normalize_field_names - data = {"foo bar": True, "bar-baz": False} + data = {"foo bar": True, "Bar-Baz": False} self.assertEqual(_normalize_field_names(data), {"foo_bar": True, "bar_baz": False}) + + def test_dict_fields_normalization_nested_dict(self): + from vyos.opmode import _normalize_field_names + + data = {"foo bar": True, "bar-baz": {"baz-quux": {"quux-xyzzy": False}}} + self.assertEqual(_normalize_field_names(data), + {"foo_bar": True, "bar_baz": {"baz_quux": {"quux_xyzzy": False}}}) + + def test_dict_fields_normalization_mixed(self): + from vyos.opmode import _normalize_field_names + + data = [{"foo bar": True, "bar-baz": [{"baz-quux": {"quux-xyzzy": [False]}}]}] + self.assertEqual(_normalize_field_names(data), + [{"foo_bar": True, "bar_baz": [{"baz_quux": {"quux_xyzzy": [False]}}]}]) + + def test_dict_fields_normalization_primitive(self): + from vyos.opmode import _normalize_field_names + + data = [1, False, "foo"] + self.assertEqual(_normalize_field_names(data), [1, False, "foo"]) + -- cgit v1.2.3 From 28b312d687291cef1e3935b7f39dc28b9e7976ef Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Thu, 20 Oct 2022 11:51:32 +0000 Subject: T4762: Add check for show nat if nat config does not exist Add check for 'show nat xxx' if nat configuration does not exist --- src/op_mode/nat.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'src') diff --git a/src/op_mode/nat.py b/src/op_mode/nat.py index 845dbbb2c..f899eb3dc 100755 --- a/src/op_mode/nat.py +++ b/src/op_mode/nat.py @@ -22,12 +22,18 @@ import xmltodict from sys import exit from tabulate import tabulate +from vyos.configquery import ConfigTreeQuery + from vyos.util import cmd from vyos.util import dict_search import vyos.opmode +base = 'nat' +unconf_message = 'NAT is not configured' + + def _get_xml_translation(direction, family): """ Get conntrack XML output --src-nat|--dst-nat @@ -277,6 +283,20 @@ def _get_formatted_translation(dict_data, nat_direction, family): return output +def _verify(func): + """Decorator checks if NAT config exists""" + from functools import wraps + + @wraps(func) + def _wrapper(*args, **kwargs): + config = ConfigTreeQuery() + if not config.exists(base): + raise vyos.opmode.UnconfiguredSubsystem(unconf_message) + return func(*args, **kwargs) + return _wrapper + + +@_verify def show_rules(raw: bool, direction: str, family: str): nat_rules = _get_raw_data_rules(direction, family) if raw: @@ -285,6 +305,7 @@ def show_rules(raw: bool, direction: str, family: str): return _get_formatted_output_rules(nat_rules, direction, family) +@_verify def show_statistics(raw: bool, direction: str, family: str): nat_statistics = _get_raw_data_rules(direction, family) if raw: @@ -293,6 +314,7 @@ def show_statistics(raw: bool, direction: str, family: str): return _get_formatted_output_statistics(nat_statistics, direction) +@_verify def show_translations(raw: bool, direction: str, family: str): family = 'ipv6' if family == 'inet6' else 'ipv4' nat_translation = _get_raw_translation(direction, family) -- cgit v1.2.3 From 1c05f8b09bf5727a6e0c0b124f77684635dcf9a8 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Mon, 24 Oct 2022 09:33:46 -0500 Subject: route: T4772: return list of dicts in 'raw' output --- src/op_mode/route.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/op_mode/route.py b/src/op_mode/route.py index e1eee5bbf..d11b00ba0 100755 --- a/src/op_mode/route.py +++ b/src/op_mode/route.py @@ -83,7 +83,12 @@ def show(raw: bool, if raw: from json import loads - return loads(output) + d = loads(output) + collect = [] + for k,_ in d.items(): + for l in d[k]: + collect.append(l) + return collect else: return output -- cgit v1.2.3 From 093ac258c11894b07afd9e85a61778d23e356718 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 23 Oct 2022 11:05:33 -0500 Subject: graphql: T4574: call all schema definition generation on init --- src/services/api/graphql/bindings.py | 4 + .../api/graphql/graphql/schema/composite.graphql | 18 ---- .../graphql/graphql/schema/configsession.graphql | 115 --------------------- .../api/graphql/utils/schema_from_composite.py | 8 +- .../graphql/utils/schema_from_config_session.py | 8 +- 5 files changed, 12 insertions(+), 141 deletions(-) delete mode 100644 src/services/api/graphql/graphql/schema/composite.graphql delete mode 100644 src/services/api/graphql/graphql/schema/configsession.graphql (limited to 'src') diff --git a/src/services/api/graphql/bindings.py b/src/services/api/graphql/bindings.py index 0b1260912..c5c4560dd 100644 --- a/src/services/api/graphql/bindings.py +++ b/src/services/api/graphql/bindings.py @@ -19,12 +19,16 @@ from . graphql.mutations import mutation from . graphql.directives import directives_dict from . graphql.errors import op_mode_error from . utils.schema_from_op_mode import generate_op_mode_definitions +from . utils.schema_from_config_session import generate_config_session_definitions +from . utils.schema_from_composite import generate_composite_definitions from ariadne import make_executable_schema, load_schema_from_path, snake_case_fallback_resolvers def generate_schema(): api_schema_dir = vyos.defaults.directories['api_schema'] generate_op_mode_definitions() + generate_config_session_definitions() + generate_composite_definitions() type_defs = load_schema_from_path(api_schema_dir) diff --git a/src/services/api/graphql/graphql/schema/composite.graphql b/src/services/api/graphql/graphql/schema/composite.graphql deleted file mode 100644 index 717fbd89d..000000000 --- a/src/services/api/graphql/graphql/schema/composite.graphql +++ /dev/null @@ -1,18 +0,0 @@ - -input SystemStatusInput { - key: String! -} - -type SystemStatus { - result: Generic -} - -type SystemStatusResult { - data: SystemStatus - success: Boolean! - errors: [String] -} - -extend type Query { - SystemStatus(data: SystemStatusInput) : SystemStatusResult @compositequery -} \ No newline at end of file diff --git a/src/services/api/graphql/graphql/schema/configsession.graphql b/src/services/api/graphql/graphql/schema/configsession.graphql deleted file mode 100644 index b1deac4b3..000000000 --- a/src/services/api/graphql/graphql/schema/configsession.graphql +++ /dev/null @@ -1,115 +0,0 @@ - -input ShowConfigInput { - key: String! - path: [String!]! - configFormat: String = null -} - -type ShowConfig { - result: Generic -} - -type ShowConfigResult { - data: ShowConfig - success: Boolean! - errors: [String] -} - -extend type Query { - ShowConfig(data: ShowConfigInput) : ShowConfigResult @configsessionquery -} - -input ShowInput { - key: String! - path: [String!]! -} - -type Show { - result: Generic -} - -type ShowResult { - data: Show - success: Boolean! - errors: [String] -} - -extend type Query { - Show(data: ShowInput) : ShowResult @configsessionquery -} - -input SaveConfigFileInput { - key: String! - fileName: String = null -} - -type SaveConfigFile { - result: Generic -} - -type SaveConfigFileResult { - data: SaveConfigFile - success: Boolean! - errors: [String] -} - -extend type Mutation { - SaveConfigFile(data: SaveConfigFileInput) : SaveConfigFileResult @configsessionmutation -} - -input LoadConfigFileInput { - key: String! - fileName: String! -} - -type LoadConfigFile { - result: Generic -} - -type LoadConfigFileResult { - data: LoadConfigFile - success: Boolean! - errors: [String] -} - -extend type Mutation { - LoadConfigFile(data: LoadConfigFileInput) : LoadConfigFileResult @configsessionmutation -} - -input AddSystemImageInput { - key: String! - location: String! -} - -type AddSystemImage { - result: Generic -} - -type AddSystemImageResult { - data: AddSystemImage - success: Boolean! - errors: [String] -} - -extend type Mutation { - AddSystemImage(data: AddSystemImageInput) : AddSystemImageResult @configsessionmutation -} - -input DeleteSystemImageInput { - key: String! - name: String! -} - -type DeleteSystemImage { - result: Generic -} - -type DeleteSystemImageResult { - data: DeleteSystemImage - success: Boolean! - errors: [String] -} - -extend type Mutation { - DeleteSystemImage(data: DeleteSystemImageInput) : DeleteSystemImageResult @configsessionmutation -} \ No newline at end of file diff --git a/src/services/api/graphql/utils/schema_from_composite.py b/src/services/api/graphql/utils/schema_from_composite.py index f9983cd98..d5e0ecdf6 100755 --- a/src/services/api/graphql/utils/schema_from_composite.py +++ b/src/services/api/graphql/utils/schema_from_composite.py @@ -23,13 +23,15 @@ import json from inspect import signature, getmembers, isfunction, isclass, getmro from jinja2 import Template +from vyos.defaults import directories if __package__ is None or __package__ == '': from util import snake_to_pascal_case, map_type_name + from composite_function import queries, mutations else: from . util import snake_to_pascal_case, map_type_name + from . composite_function import queries, mutations -# this will be run locally before the build -SCHEMA_PATH = '../graphql/schema' +SCHEMA_PATH = directories['api_schema'] schema_data: dict = {'schema_name': '', 'schema_fields': []} @@ -100,8 +102,6 @@ def create_schema(func_name: str, func: callable, template: str) -> str: return res def generate_composite_definitions(): - from composite_function import queries, mutations - results = [] for name,func in queries.items(): res = create_schema(name, func, query_template) diff --git a/src/services/api/graphql/utils/schema_from_config_session.py b/src/services/api/graphql/utils/schema_from_config_session.py index ea78aaf88..b6609357e 100755 --- a/src/services/api/graphql/utils/schema_from_config_session.py +++ b/src/services/api/graphql/utils/schema_from_config_session.py @@ -23,13 +23,15 @@ import json from inspect import signature, getmembers, isfunction, isclass, getmro from jinja2 import Template +from vyos.defaults import directories if __package__ is None or __package__ == '': from util import snake_to_pascal_case, map_type_name + from config_session_function import queries, mutations else: from . util import snake_to_pascal_case, map_type_name + from . config_session_function import queries, mutations -# this will be run locally before the build -SCHEMA_PATH = '../graphql/schema' +SCHEMA_PATH = directories['api_schema'] schema_data: dict = {'schema_name': '', 'schema_fields': []} @@ -100,8 +102,6 @@ def create_schema(func_name: str, func: callable, template: str) -> str: return res def generate_config_session_definitions(): - from config_session_function import queries, mutations - results = [] for name,func in queries.items(): res = create_schema(name, func, query_template) -- cgit v1.2.3 From 7038b761302be2ec90338981830b8cd7cf887381 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 23 Oct 2022 11:06:03 -0500 Subject: graphql: T4574: reorganize directory structure for clarity --- src/services/api/graphql/__init__.py | 0 src/services/api/graphql/bindings.py | 6 +- .../api/graphql/generate/composite_function.py | 11 ++ .../graphql/generate/config_session_function.py | 28 ++++ .../api/graphql/generate/schema_from_composite.py | 121 ++++++++++++++ .../graphql/generate/schema_from_config_session.py | 121 ++++++++++++++ .../api/graphql/generate/schema_from_op_mode.py | 185 +++++++++++++++++++++ src/services/api/graphql/graphql/mutations.py | 2 +- src/services/api/graphql/graphql/queries.py | 2 +- src/services/api/graphql/key_auth.py | 18 -- src/services/api/graphql/libs/key_auth.py | 18 ++ src/services/api/graphql/libs/op_mode.py | 100 +++++++++++ .../api/graphql/session/composite/system_status.py | 2 +- src/services/api/graphql/session/session.py | 2 +- .../api/graphql/utils/composite_function.py | 11 -- .../api/graphql/utils/config_session_function.py | 28 ---- .../api/graphql/utils/schema_from_composite.py | 119 ------------- .../graphql/utils/schema_from_config_session.py | 119 ------------- .../api/graphql/utils/schema_from_op_mode.py | 183 -------------------- src/services/api/graphql/utils/util.py | 100 ----------- 20 files changed, 591 insertions(+), 585 deletions(-) create mode 100644 src/services/api/graphql/__init__.py create mode 100644 src/services/api/graphql/generate/composite_function.py create mode 100644 src/services/api/graphql/generate/config_session_function.py create mode 100755 src/services/api/graphql/generate/schema_from_composite.py create mode 100755 src/services/api/graphql/generate/schema_from_config_session.py create mode 100755 src/services/api/graphql/generate/schema_from_op_mode.py delete mode 100644 src/services/api/graphql/key_auth.py create mode 100644 src/services/api/graphql/libs/key_auth.py create mode 100644 src/services/api/graphql/libs/op_mode.py delete mode 100644 src/services/api/graphql/utils/composite_function.py delete mode 100644 src/services/api/graphql/utils/config_session_function.py delete mode 100755 src/services/api/graphql/utils/schema_from_composite.py delete mode 100755 src/services/api/graphql/utils/schema_from_config_session.py delete mode 100755 src/services/api/graphql/utils/schema_from_op_mode.py delete mode 100644 src/services/api/graphql/utils/util.py (limited to 'src') diff --git a/src/services/api/graphql/__init__.py b/src/services/api/graphql/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/services/api/graphql/bindings.py b/src/services/api/graphql/bindings.py index c5c4560dd..d3cff21c7 100644 --- a/src/services/api/graphql/bindings.py +++ b/src/services/api/graphql/bindings.py @@ -18,9 +18,9 @@ from . graphql.queries import query from . graphql.mutations import mutation from . graphql.directives import directives_dict from . graphql.errors import op_mode_error -from . utils.schema_from_op_mode import generate_op_mode_definitions -from . utils.schema_from_config_session import generate_config_session_definitions -from . utils.schema_from_composite import generate_composite_definitions +from . generate.schema_from_op_mode import generate_op_mode_definitions +from . generate.schema_from_config_session import generate_config_session_definitions +from . generate.schema_from_composite import generate_composite_definitions from ariadne import make_executable_schema, load_schema_from_path, snake_case_fallback_resolvers def generate_schema(): diff --git a/src/services/api/graphql/generate/composite_function.py b/src/services/api/graphql/generate/composite_function.py new file mode 100644 index 000000000..bc9d80fbb --- /dev/null +++ b/src/services/api/graphql/generate/composite_function.py @@ -0,0 +1,11 @@ +# typing information for composite functions: those that invoke several +# elementary requests, and return the result as a single dict +import typing + +def system_status(): + pass + +queries = {'system_status': system_status} + +mutations = {} + diff --git a/src/services/api/graphql/generate/config_session_function.py b/src/services/api/graphql/generate/config_session_function.py new file mode 100644 index 000000000..fc0dd7a87 --- /dev/null +++ b/src/services/api/graphql/generate/config_session_function.py @@ -0,0 +1,28 @@ +# typing information for native configsession functions; used to generate +# schema definition files +import typing + +def show_config(path: list[str], configFormat: typing.Optional[str]): + pass + +def show(path: list[str]): + pass + +queries = {'show_config': show_config, + 'show': show} + +def save_config_file(fileName: typing.Optional[str]): + pass +def load_config_file(fileName: str): + pass +def add_system_image(location: str): + pass +def delete_system_image(name: str): + pass + +mutations = {'save_config_file': save_config_file, + 'load_config_file': load_config_file, + 'add_system_image': add_system_image, + 'delete_system_image': delete_system_image} + + diff --git a/src/services/api/graphql/generate/schema_from_composite.py b/src/services/api/graphql/generate/schema_from_composite.py new file mode 100755 index 000000000..7187047a0 --- /dev/null +++ b/src/services/api/graphql/generate/schema_from_composite.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# A utility to generate GraphQL schema defintions from typing information of +# composite functions comprising several requests. + +import os +import sys +import json +from inspect import signature, getmembers, isfunction, isclass, getmro +from jinja2 import Template + +from vyos.defaults import directories +if __package__ is None or __package__ == '': + sys.path.append("/usr/libexec/vyos/services/api") + from graphql.libs.op_mode import snake_to_pascal_case, map_type_name + from composite_function import queries, mutations +else: + from .. libs.op_mode import snake_to_pascal_case, map_type_name + from . composite_function import queries, mutations + +SCHEMA_PATH = directories['api_schema'] + +schema_data: dict = {'schema_name': '', + 'schema_fields': []} + +query_template = """ +input {{ schema_name }}Input { + key: String! + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} + +type {{ schema_name }} { + result: Generic +} + +type {{ schema_name }}Result { + data: {{ schema_name }} + success: Boolean! + errors: [String] +} + +extend type Query { + {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @compositequery +} +""" + +mutation_template = """ +input {{ schema_name }}Input { + key: String! + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} + +type {{ schema_name }} { + result: Generic +} + +type {{ schema_name }}Result { + data: {{ schema_name }} + success: Boolean! + errors: [String] +} + +extend type Mutation { + {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @compositemutation +} +""" + +def create_schema(func_name: str, func: callable, template: str) -> str: + sig = signature(func) + + field_dict = {} + for k in sig.parameters: + field_dict[sig.parameters[k].name] = map_type_name(sig.parameters[k].annotation) + + schema_fields = [] + for k,v in field_dict.items(): + schema_fields.append(k+': '+v) + + schema_data['schema_name'] = snake_to_pascal_case(func_name) + schema_data['schema_fields'] = schema_fields + + j2_template = Template(template) + res = j2_template.render(schema_data) + + return res + +def generate_composite_definitions(): + results = [] + for name,func in queries.items(): + res = create_schema(name, func, query_template) + results.append(res) + + for name,func in mutations.items(): + res = create_schema(name, func, mutation_template) + results.append(res) + + out = '\n'.join(results) + with open(f'{SCHEMA_PATH}/composite.graphql', 'w') as f: + f.write(out) + +if __name__ == '__main__': + generate_composite_definitions() diff --git a/src/services/api/graphql/generate/schema_from_config_session.py b/src/services/api/graphql/generate/schema_from_config_session.py new file mode 100755 index 000000000..cf69cbafd --- /dev/null +++ b/src/services/api/graphql/generate/schema_from_config_session.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# A utility to generate GraphQL schema defintions from typing information of +# (wrappers of) native configsession functions. + +import os +import sys +import json +from inspect import signature, getmembers, isfunction, isclass, getmro +from jinja2 import Template + +from vyos.defaults import directories +if __package__ is None or __package__ == '': + sys.path.append("/usr/libexec/vyos/services/api") + from graphql.libs.op_mode import snake_to_pascal_case, map_type_name + from config_session_function import queries, mutations +else: + from .. libs.op_mode import snake_to_pascal_case, map_type_name + from . config_session_function import queries, mutations + +SCHEMA_PATH = directories['api_schema'] + +schema_data: dict = {'schema_name': '', + 'schema_fields': []} + +query_template = """ +input {{ schema_name }}Input { + key: String! + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} + +type {{ schema_name }} { + result: Generic +} + +type {{ schema_name }}Result { + data: {{ schema_name }} + success: Boolean! + errors: [String] +} + +extend type Query { + {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @configsessionquery +} +""" + +mutation_template = """ +input {{ schema_name }}Input { + key: String! + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} + +type {{ schema_name }} { + result: Generic +} + +type {{ schema_name }}Result { + data: {{ schema_name }} + success: Boolean! + errors: [String] +} + +extend type Mutation { + {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @configsessionmutation +} +""" + +def create_schema(func_name: str, func: callable, template: str) -> str: + sig = signature(func) + + field_dict = {} + for k in sig.parameters: + field_dict[sig.parameters[k].name] = map_type_name(sig.parameters[k].annotation) + + schema_fields = [] + for k,v in field_dict.items(): + schema_fields.append(k+': '+v) + + schema_data['schema_name'] = snake_to_pascal_case(func_name) + schema_data['schema_fields'] = schema_fields + + j2_template = Template(template) + res = j2_template.render(schema_data) + + return res + +def generate_config_session_definitions(): + results = [] + for name,func in queries.items(): + res = create_schema(name, func, query_template) + results.append(res) + + for name,func in mutations.items(): + res = create_schema(name, func, mutation_template) + results.append(res) + + out = '\n'.join(results) + with open(f'{SCHEMA_PATH}/configsession.graphql', 'w') as f: + f.write(out) + +if __name__ == '__main__': + generate_config_session_definitions() diff --git a/src/services/api/graphql/generate/schema_from_op_mode.py b/src/services/api/graphql/generate/schema_from_op_mode.py new file mode 100755 index 000000000..a88816b34 --- /dev/null +++ b/src/services/api/graphql/generate/schema_from_op_mode.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# A utility to generate GraphQL schema defintions from standardized op-mode +# scripts. + +import os +import sys +import json +from inspect import signature, getmembers, isfunction, isclass, getmro +from jinja2 import Template + +from vyos.defaults import directories +if __package__ is None or __package__ == '': + sys.path.append("/usr/libexec/vyos/services/api") + from graphql.libs.op_mode import load_as_module, is_op_mode_function_name, is_show_function_name + from graphql.libs.op_mode import snake_to_pascal_case, map_type_name +else: + from .. libs.op_mode import load_as_module, is_op_mode_function_name, is_show_function_name + from .. libs.op_mode import snake_to_pascal_case, map_type_name + +OP_MODE_PATH = directories['op_mode'] +SCHEMA_PATH = directories['api_schema'] +DATA_DIR = directories['data'] + +op_mode_include_file = os.path.join(DATA_DIR, 'op-mode-standardized.json') +op_mode_error_schema = 'op_mode_error.graphql' + +schema_data: dict = {'schema_name': '', + 'schema_fields': []} + +query_template = """ +input {{ schema_name }}Input { + key: String! + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} + +type {{ schema_name }} { + result: Generic +} + +type {{ schema_name }}Result { + data: {{ schema_name }} + op_mode_error: OpModeError + success: Boolean! + errors: [String] +} + +extend type Query { + {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @genopquery +} +""" + +mutation_template = """ +input {{ schema_name }}Input { + key: String! + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} + +type {{ schema_name }} { + result: Generic +} + +type {{ schema_name }}Result { + data: {{ schema_name }} + op_mode_error: OpModeError + success: Boolean! + errors: [String] +} + +extend type Mutation { + {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @genopmutation +} +""" + +error_template = """ +interface OpModeError { + name: String! + message: String! + vyos_code: Int! +} +{% for name in error_names %} +type {{ name }} implements OpModeError { + name: String! + message: String! + vyos_code: Int! +} +{%- endfor %} +""" + +def create_schema(func_name: str, base_name: str, func: callable) -> str: + sig = signature(func) + + field_dict = {} + for k in sig.parameters: + field_dict[sig.parameters[k].name] = map_type_name(sig.parameters[k].annotation) + + # It is assumed that if one is generating a schema for a 'show_*' + # function, that 'get_raw_data' is present and 'raw' is desired. + if 'raw' in list(field_dict): + del field_dict['raw'] + + schema_fields = [] + for k,v in field_dict.items(): + schema_fields.append(k+': '+v) + + schema_data['schema_name'] = snake_to_pascal_case(func_name + '_' + base_name) + schema_data['schema_fields'] = schema_fields + + if is_show_function_name(func_name): + j2_template = Template(query_template) + else: + j2_template = Template(mutation_template) + + res = j2_template.render(schema_data) + + return res + +def create_error_schema(): + from vyos import opmode + + e = Exception + err_types = getmembers(opmode, isclass) + err_types = [k for k in err_types if issubclass(k[1], e)] + # drop base class, to be replaced by interface type. Find the class + # programmatically, in case the base class name changes. + for i in range(len(err_types)): + if err_types[i][1] in getmro(err_types[i-1][1]): + del err_types[i] + break + err_names = [k[0] for k in err_types] + error_data = {'error_names': err_names} + j2_template = Template(error_template) + res = j2_template.render(error_data) + + return res + +def generate_op_mode_definitions(): + out = create_error_schema() + with open(f'{SCHEMA_PATH}/{op_mode_error_schema}', 'w') as f: + f.write(out) + + with open(op_mode_include_file) as f: + op_mode_files = json.load(f) + + for file in op_mode_files: + basename = os.path.splitext(file)[0].replace('-', '_') + module = load_as_module(basename, os.path.join(OP_MODE_PATH, file)) + + funcs = getmembers(module, isfunction) + funcs = list(filter(lambda ft: is_op_mode_function_name(ft[0]), funcs)) + + funcs_dict = {} + for (name, thunk) in funcs: + funcs_dict[name] = thunk + + results = [] + for name,func in funcs_dict.items(): + res = create_schema(name, basename, func) + results.append(res) + + out = '\n'.join(results) + with open(f'{SCHEMA_PATH}/{basename}.graphql', 'w') as f: + f.write(out) + +if __name__ == '__main__': + generate_op_mode_definitions() diff --git a/src/services/api/graphql/graphql/mutations.py b/src/services/api/graphql/graphql/mutations.py index 32da0eeb7..f0c8b438f 100644 --- a/src/services/api/graphql/graphql/mutations.py +++ b/src/services/api/graphql/graphql/mutations.py @@ -20,7 +20,7 @@ from graphql import GraphQLResolveInfo from makefun import with_signature from .. import state -from .. import key_auth +from .. libs import key_auth from api.graphql.session.session import Session from api.graphql.session.errors.op_mode_errors import op_mode_err_msg, op_mode_err_code from vyos.opmode import Error as OpModeError diff --git a/src/services/api/graphql/graphql/queries.py b/src/services/api/graphql/graphql/queries.py index 791b0d3e0..13eb59ae4 100644 --- a/src/services/api/graphql/graphql/queries.py +++ b/src/services/api/graphql/graphql/queries.py @@ -20,7 +20,7 @@ from graphql import GraphQLResolveInfo from makefun import with_signature from .. import state -from .. import key_auth +from .. libs import key_auth from api.graphql.session.session import Session from api.graphql.session.errors.op_mode_errors import op_mode_err_msg, op_mode_err_code from vyos.opmode import Error as OpModeError diff --git a/src/services/api/graphql/key_auth.py b/src/services/api/graphql/key_auth.py deleted file mode 100644 index f756ed6d8..000000000 --- a/src/services/api/graphql/key_auth.py +++ /dev/null @@ -1,18 +0,0 @@ - -from . import state - -def check_auth(key_list, key): - if not key_list: - return None - key_id = None - for k in key_list: - if k['key'] == key: - key_id = k['id'] - return key_id - -def auth_required(key): - api_keys = None - api_keys = state.settings['app'].state.vyos_keys - key_id = check_auth(api_keys, key) - state.settings['app'].state.vyos_id = key_id - return key_id diff --git a/src/services/api/graphql/libs/key_auth.py b/src/services/api/graphql/libs/key_auth.py new file mode 100644 index 000000000..2db0f7d48 --- /dev/null +++ b/src/services/api/graphql/libs/key_auth.py @@ -0,0 +1,18 @@ + +from .. import state + +def check_auth(key_list, key): + if not key_list: + return None + key_id = None + for k in key_list: + if k['key'] == key: + key_id = k['id'] + return key_id + +def auth_required(key): + api_keys = None + api_keys = state.settings['app'].state.vyos_keys + key_id = check_auth(api_keys, key) + state.settings['app'].state.vyos_id = key_id + return key_id diff --git a/src/services/api/graphql/libs/op_mode.py b/src/services/api/graphql/libs/op_mode.py new file mode 100644 index 000000000..da2bcdb5b --- /dev/null +++ b/src/services/api/graphql/libs/op_mode.py @@ -0,0 +1,100 @@ +# Copyright 2022 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see . + +import os +import re +import typing +import importlib.util + +from vyos.defaults import directories + +def load_as_module(name: str, path: str): + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +def load_op_mode_as_module(name: str): + path = os.path.join(directories['op_mode'], name) + name = os.path.splitext(name)[0].replace('-', '_') + return load_as_module(name, path) + +def is_op_mode_function_name(name): + if re.match(r"^(show|clear|reset|restart)", name): + return True + return False + +def is_show_function_name(name): + if re.match(r"^show", name): + return True + return False + +def _nth_split(delim: str, n: int, s: str): + groups = s.split(delim) + l = len(groups) + if n > l-1 or n < 1: + return (s, '') + return (delim.join(groups[:n]), delim.join(groups[n:])) + +def _nth_rsplit(delim: str, n: int, s: str): + groups = s.split(delim) + l = len(groups) + if n > l-1 or n < 1: + return (s, '') + return (delim.join(groups[:l-n]), delim.join(groups[l-n:])) + +# Since we have mangled possible hyphens in the file name while constructing +# the snake case of the query/mutation name, we will need to recover the +# file name by searching with mangling: +def _filter_on_mangled(test): + def func(elem): + mangle = os.path.splitext(elem)[0].replace('-', '_') + return test == mangle + return func + +# Find longest name in concatenated string that matches the basename of an +# op-mode script. Should one prefer to concatenate in the reverse order +# (script_name + '_' + function_name), use _nth_rsplit. +def split_compound_op_mode_name(name: str, files: list): + for i in range(1, name.count('_') + 1): + pair = _nth_split('_', i, name) + f = list(filter(_filter_on_mangled(pair[1]), files)) + if f: + pair = (pair[0], f[0]) + return pair + return (name, '') + +def snake_to_pascal_case(name: str) -> str: + res = ''.join(map(str.title, name.split('_'))) + return res + +def map_type_name(type_name: type, optional: bool = False) -> str: + if type_name == str: + return 'String!' if not optional else 'String = null' + if type_name == int: + return 'Int!' if not optional else 'Int = null' + if type_name == bool: + return 'Boolean!' if not optional else 'Boolean = false' + if typing.get_origin(type_name) == list: + if not optional: + return f'[{map_type_name(typing.get_args(type_name)[0])}]!' + return f'[{map_type_name(typing.get_args(type_name)[0])}]' + # typing.Optional is typing.Union[_, NoneType] + if (typing.get_origin(type_name) is typing.Union and + typing.get_args(type_name)[1] == type(None)): + return f'{map_type_name(typing.get_args(type_name)[0], optional=True)}' + + # scalar 'Generic' is defined in schema.graphql + return 'Generic' diff --git a/src/services/api/graphql/session/composite/system_status.py b/src/services/api/graphql/session/composite/system_status.py index 3c1a3d45b..d809f32e3 100755 --- a/src/services/api/graphql/session/composite/system_status.py +++ b/src/services/api/graphql/session/composite/system_status.py @@ -23,7 +23,7 @@ import importlib.util from vyos.defaults import directories -from api.graphql.utils.util import load_op_mode_as_module +from api.graphql.libs.op_mode import load_op_mode_as_module def get_system_version() -> dict: show_version = load_op_mode_as_module('version.py') diff --git a/src/services/api/graphql/session/session.py b/src/services/api/graphql/session/session.py index f990e63d0..c2c1db1df 100644 --- a/src/services/api/graphql/session/session.py +++ b/src/services/api/graphql/session/session.py @@ -24,7 +24,7 @@ from vyos.defaults import directories from vyos.template import render from vyos.opmode import Error as OpModeError -from api.graphql.utils.util import load_op_mode_as_module, split_compound_op_mode_name +from api.graphql.libs.op_mode import load_op_mode_as_module, split_compound_op_mode_name op_mode_include_file = os.path.join(directories['data'], 'op-mode-standardized.json') diff --git a/src/services/api/graphql/utils/composite_function.py b/src/services/api/graphql/utils/composite_function.py deleted file mode 100644 index bc9d80fbb..000000000 --- a/src/services/api/graphql/utils/composite_function.py +++ /dev/null @@ -1,11 +0,0 @@ -# typing information for composite functions: those that invoke several -# elementary requests, and return the result as a single dict -import typing - -def system_status(): - pass - -queries = {'system_status': system_status} - -mutations = {} - diff --git a/src/services/api/graphql/utils/config_session_function.py b/src/services/api/graphql/utils/config_session_function.py deleted file mode 100644 index fc0dd7a87..000000000 --- a/src/services/api/graphql/utils/config_session_function.py +++ /dev/null @@ -1,28 +0,0 @@ -# typing information for native configsession functions; used to generate -# schema definition files -import typing - -def show_config(path: list[str], configFormat: typing.Optional[str]): - pass - -def show(path: list[str]): - pass - -queries = {'show_config': show_config, - 'show': show} - -def save_config_file(fileName: typing.Optional[str]): - pass -def load_config_file(fileName: str): - pass -def add_system_image(location: str): - pass -def delete_system_image(name: str): - pass - -mutations = {'save_config_file': save_config_file, - 'load_config_file': load_config_file, - 'add_system_image': add_system_image, - 'delete_system_image': delete_system_image} - - diff --git a/src/services/api/graphql/utils/schema_from_composite.py b/src/services/api/graphql/utils/schema_from_composite.py deleted file mode 100755 index d5e0ecdf6..000000000 --- a/src/services/api/graphql/utils/schema_from_composite.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2022 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# -# A utility to generate GraphQL schema defintions from typing information of -# composite functions comprising several requests. - -import os -import json -from inspect import signature, getmembers, isfunction, isclass, getmro -from jinja2 import Template - -from vyos.defaults import directories -if __package__ is None or __package__ == '': - from util import snake_to_pascal_case, map_type_name - from composite_function import queries, mutations -else: - from . util import snake_to_pascal_case, map_type_name - from . composite_function import queries, mutations - -SCHEMA_PATH = directories['api_schema'] - -schema_data: dict = {'schema_name': '', - 'schema_fields': []} - -query_template = """ -input {{ schema_name }}Input { - key: String! - {%- for field_entry in schema_fields %} - {{ field_entry }} - {%- endfor %} -} - -type {{ schema_name }} { - result: Generic -} - -type {{ schema_name }}Result { - data: {{ schema_name }} - success: Boolean! - errors: [String] -} - -extend type Query { - {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @compositequery -} -""" - -mutation_template = """ -input {{ schema_name }}Input { - key: String! - {%- for field_entry in schema_fields %} - {{ field_entry }} - {%- endfor %} -} - -type {{ schema_name }} { - result: Generic -} - -type {{ schema_name }}Result { - data: {{ schema_name }} - success: Boolean! - errors: [String] -} - -extend type Mutation { - {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @compositemutation -} -""" - -def create_schema(func_name: str, func: callable, template: str) -> str: - sig = signature(func) - - field_dict = {} - for k in sig.parameters: - field_dict[sig.parameters[k].name] = map_type_name(sig.parameters[k].annotation) - - schema_fields = [] - for k,v in field_dict.items(): - schema_fields.append(k+': '+v) - - schema_data['schema_name'] = snake_to_pascal_case(func_name) - schema_data['schema_fields'] = schema_fields - - j2_template = Template(template) - res = j2_template.render(schema_data) - - return res - -def generate_composite_definitions(): - results = [] - for name,func in queries.items(): - res = create_schema(name, func, query_template) - results.append(res) - - for name,func in mutations.items(): - res = create_schema(name, func, mutation_template) - results.append(res) - - out = '\n'.join(results) - with open(f'{SCHEMA_PATH}/composite.graphql', 'w') as f: - f.write(out) - -if __name__ == '__main__': - generate_composite_definitions() diff --git a/src/services/api/graphql/utils/schema_from_config_session.py b/src/services/api/graphql/utils/schema_from_config_session.py deleted file mode 100755 index b6609357e..000000000 --- a/src/services/api/graphql/utils/schema_from_config_session.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2022 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# -# A utility to generate GraphQL schema defintions from typing information of -# (wrappers of) native configsession functions. - -import os -import json -from inspect import signature, getmembers, isfunction, isclass, getmro -from jinja2 import Template - -from vyos.defaults import directories -if __package__ is None or __package__ == '': - from util import snake_to_pascal_case, map_type_name - from config_session_function import queries, mutations -else: - from . util import snake_to_pascal_case, map_type_name - from . config_session_function import queries, mutations - -SCHEMA_PATH = directories['api_schema'] - -schema_data: dict = {'schema_name': '', - 'schema_fields': []} - -query_template = """ -input {{ schema_name }}Input { - key: String! - {%- for field_entry in schema_fields %} - {{ field_entry }} - {%- endfor %} -} - -type {{ schema_name }} { - result: Generic -} - -type {{ schema_name }}Result { - data: {{ schema_name }} - success: Boolean! - errors: [String] -} - -extend type Query { - {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @configsessionquery -} -""" - -mutation_template = """ -input {{ schema_name }}Input { - key: String! - {%- for field_entry in schema_fields %} - {{ field_entry }} - {%- endfor %} -} - -type {{ schema_name }} { - result: Generic -} - -type {{ schema_name }}Result { - data: {{ schema_name }} - success: Boolean! - errors: [String] -} - -extend type Mutation { - {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @configsessionmutation -} -""" - -def create_schema(func_name: str, func: callable, template: str) -> str: - sig = signature(func) - - field_dict = {} - for k in sig.parameters: - field_dict[sig.parameters[k].name] = map_type_name(sig.parameters[k].annotation) - - schema_fields = [] - for k,v in field_dict.items(): - schema_fields.append(k+': '+v) - - schema_data['schema_name'] = snake_to_pascal_case(func_name) - schema_data['schema_fields'] = schema_fields - - j2_template = Template(template) - res = j2_template.render(schema_data) - - return res - -def generate_config_session_definitions(): - results = [] - for name,func in queries.items(): - res = create_schema(name, func, query_template) - results.append(res) - - for name,func in mutations.items(): - res = create_schema(name, func, mutation_template) - results.append(res) - - out = '\n'.join(results) - with open(f'{SCHEMA_PATH}/configsession.graphql', 'w') as f: - f.write(out) - -if __name__ == '__main__': - generate_config_session_definitions() diff --git a/src/services/api/graphql/utils/schema_from_op_mode.py b/src/services/api/graphql/utils/schema_from_op_mode.py deleted file mode 100755 index 57d63628b..000000000 --- a/src/services/api/graphql/utils/schema_from_op_mode.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2022 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# -# A utility to generate GraphQL schema defintions from standardized op-mode -# scripts. - -import os -import json -from inspect import signature, getmembers, isfunction, isclass, getmro -from jinja2 import Template - -from vyos.defaults import directories -if __package__ is None or __package__ == '': - from util import load_as_module, is_op_mode_function_name, is_show_function_name - from util import snake_to_pascal_case, map_type_name -else: - from . util import load_as_module, is_op_mode_function_name, is_show_function_name - from . util import snake_to_pascal_case, map_type_name - -OP_MODE_PATH = directories['op_mode'] -SCHEMA_PATH = directories['api_schema'] -DATA_DIR = directories['data'] - -op_mode_include_file = os.path.join(DATA_DIR, 'op-mode-standardized.json') -op_mode_error_schema = 'op_mode_error.graphql' - -schema_data: dict = {'schema_name': '', - 'schema_fields': []} - -query_template = """ -input {{ schema_name }}Input { - key: String! - {%- for field_entry in schema_fields %} - {{ field_entry }} - {%- endfor %} -} - -type {{ schema_name }} { - result: Generic -} - -type {{ schema_name }}Result { - data: {{ schema_name }} - op_mode_error: OpModeError - success: Boolean! - errors: [String] -} - -extend type Query { - {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @genopquery -} -""" - -mutation_template = """ -input {{ schema_name }}Input { - key: String! - {%- for field_entry in schema_fields %} - {{ field_entry }} - {%- endfor %} -} - -type {{ schema_name }} { - result: Generic -} - -type {{ schema_name }}Result { - data: {{ schema_name }} - op_mode_error: OpModeError - success: Boolean! - errors: [String] -} - -extend type Mutation { - {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @genopmutation -} -""" - -error_template = """ -interface OpModeError { - name: String! - message: String! - vyos_code: Int! -} -{% for name in error_names %} -type {{ name }} implements OpModeError { - name: String! - message: String! - vyos_code: Int! -} -{%- endfor %} -""" - -def create_schema(func_name: str, base_name: str, func: callable) -> str: - sig = signature(func) - - field_dict = {} - for k in sig.parameters: - field_dict[sig.parameters[k].name] = map_type_name(sig.parameters[k].annotation) - - # It is assumed that if one is generating a schema for a 'show_*' - # function, that 'get_raw_data' is present and 'raw' is desired. - if 'raw' in list(field_dict): - del field_dict['raw'] - - schema_fields = [] - for k,v in field_dict.items(): - schema_fields.append(k+': '+v) - - schema_data['schema_name'] = snake_to_pascal_case(func_name + '_' + base_name) - schema_data['schema_fields'] = schema_fields - - if is_show_function_name(func_name): - j2_template = Template(query_template) - else: - j2_template = Template(mutation_template) - - res = j2_template.render(schema_data) - - return res - -def create_error_schema(): - from vyos import opmode - - e = Exception - err_types = getmembers(opmode, isclass) - err_types = [k for k in err_types if issubclass(k[1], e)] - # drop base class, to be replaced by interface type. Find the class - # programmatically, in case the base class name changes. - for i in range(len(err_types)): - if err_types[i][1] in getmro(err_types[i-1][1]): - del err_types[i] - break - err_names = [k[0] for k in err_types] - error_data = {'error_names': err_names} - j2_template = Template(error_template) - res = j2_template.render(error_data) - - return res - -def generate_op_mode_definitions(): - out = create_error_schema() - with open(f'{SCHEMA_PATH}/{op_mode_error_schema}', 'w') as f: - f.write(out) - - with open(op_mode_include_file) as f: - op_mode_files = json.load(f) - - for file in op_mode_files: - basename = os.path.splitext(file)[0].replace('-', '_') - module = load_as_module(basename, os.path.join(OP_MODE_PATH, file)) - - funcs = getmembers(module, isfunction) - funcs = list(filter(lambda ft: is_op_mode_function_name(ft[0]), funcs)) - - funcs_dict = {} - for (name, thunk) in funcs: - funcs_dict[name] = thunk - - results = [] - for name,func in funcs_dict.items(): - res = create_schema(name, basename, func) - results.append(res) - - out = '\n'.join(results) - with open(f'{SCHEMA_PATH}/{basename}.graphql', 'w') as f: - f.write(out) - -if __name__ == '__main__': - generate_op_mode_definitions() diff --git a/src/services/api/graphql/utils/util.py b/src/services/api/graphql/utils/util.py deleted file mode 100644 index da2bcdb5b..000000000 --- a/src/services/api/graphql/utils/util.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2022 VyOS maintainers and contributors -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see . - -import os -import re -import typing -import importlib.util - -from vyos.defaults import directories - -def load_as_module(name: str, path: str): - spec = importlib.util.spec_from_file_location(name, path) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - -def load_op_mode_as_module(name: str): - path = os.path.join(directories['op_mode'], name) - name = os.path.splitext(name)[0].replace('-', '_') - return load_as_module(name, path) - -def is_op_mode_function_name(name): - if re.match(r"^(show|clear|reset|restart)", name): - return True - return False - -def is_show_function_name(name): - if re.match(r"^show", name): - return True - return False - -def _nth_split(delim: str, n: int, s: str): - groups = s.split(delim) - l = len(groups) - if n > l-1 or n < 1: - return (s, '') - return (delim.join(groups[:n]), delim.join(groups[n:])) - -def _nth_rsplit(delim: str, n: int, s: str): - groups = s.split(delim) - l = len(groups) - if n > l-1 or n < 1: - return (s, '') - return (delim.join(groups[:l-n]), delim.join(groups[l-n:])) - -# Since we have mangled possible hyphens in the file name while constructing -# the snake case of the query/mutation name, we will need to recover the -# file name by searching with mangling: -def _filter_on_mangled(test): - def func(elem): - mangle = os.path.splitext(elem)[0].replace('-', '_') - return test == mangle - return func - -# Find longest name in concatenated string that matches the basename of an -# op-mode script. Should one prefer to concatenate in the reverse order -# (script_name + '_' + function_name), use _nth_rsplit. -def split_compound_op_mode_name(name: str, files: list): - for i in range(1, name.count('_') + 1): - pair = _nth_split('_', i, name) - f = list(filter(_filter_on_mangled(pair[1]), files)) - if f: - pair = (pair[0], f[0]) - return pair - return (name, '') - -def snake_to_pascal_case(name: str) -> str: - res = ''.join(map(str.title, name.split('_'))) - return res - -def map_type_name(type_name: type, optional: bool = False) -> str: - if type_name == str: - return 'String!' if not optional else 'String = null' - if type_name == int: - return 'Int!' if not optional else 'Int = null' - if type_name == bool: - return 'Boolean!' if not optional else 'Boolean = false' - if typing.get_origin(type_name) == list: - if not optional: - return f'[{map_type_name(typing.get_args(type_name)[0])}]!' - return f'[{map_type_name(typing.get_args(type_name)[0])}]' - # typing.Optional is typing.Union[_, NoneType] - if (typing.get_origin(type_name) is typing.Union and - typing.get_args(type_name)[1] == type(None)): - return f'{map_type_name(typing.get_args(type_name)[0], optional=True)}' - - # scalar 'Generic' is defined in schema.graphql - return 'Generic' -- cgit v1.2.3 From f76a6f68b08fce1feee2dbbb84658b8eede09655 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 23 Oct 2022 11:07:16 -0500 Subject: graphql: T4574: add mutation for requesting JWT token --- src/services/api/graphql/bindings.py | 8 +++- .../api/graphql/graphql/auth_token_mutation.py | 44 ++++++++++++++++++++++ .../api/graphql/graphql/schema/auth_token.graphql | 19 ++++++++++ src/services/api/graphql/libs/token_auth.py | 38 +++++++++++++++++++ 4 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 src/services/api/graphql/graphql/auth_token_mutation.py create mode 100644 src/services/api/graphql/graphql/schema/auth_token.graphql create mode 100644 src/services/api/graphql/libs/token_auth.py (limited to 'src') diff --git a/src/services/api/graphql/bindings.py b/src/services/api/graphql/bindings.py index d3cff21c7..aa1ba0eb0 100644 --- a/src/services/api/graphql/bindings.py +++ b/src/services/api/graphql/bindings.py @@ -18,9 +18,12 @@ from . graphql.queries import query from . graphql.mutations import mutation from . graphql.directives import directives_dict from . graphql.errors import op_mode_error +from . graphql.auth_token_mutation import auth_token_mutation from . generate.schema_from_op_mode import generate_op_mode_definitions from . generate.schema_from_config_session import generate_config_session_definitions from . generate.schema_from_composite import generate_composite_definitions +from . libs.token_auth import init_secret +from . import state from ariadne import make_executable_schema, load_schema_from_path, snake_case_fallback_resolvers def generate_schema(): @@ -30,8 +33,11 @@ def generate_schema(): generate_config_session_definitions() generate_composite_definitions() + if state.settings['app'].state.vyos_auth_type == 'token': + init_secret() + type_defs = load_schema_from_path(api_schema_dir) - schema = make_executable_schema(type_defs, query, op_mode_error, mutation, snake_case_fallback_resolvers, directives=directives_dict) + schema = make_executable_schema(type_defs, query, op_mode_error, mutation, auth_token_mutation, snake_case_fallback_resolvers, directives=directives_dict) return schema diff --git a/src/services/api/graphql/graphql/auth_token_mutation.py b/src/services/api/graphql/graphql/auth_token_mutation.py new file mode 100644 index 000000000..33779d4f0 --- /dev/null +++ b/src/services/api/graphql/graphql/auth_token_mutation.py @@ -0,0 +1,44 @@ +# Copyright 2022 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see . + +import jwt +from typing import Any, Dict +from ariadne import ObjectType, UnionType +from graphql import GraphQLResolveInfo + +from .. libs.token_auth import generate_token +from .. import state + +auth_token_mutation = ObjectType("Mutation") + +@auth_token_mutation.field('AuthToken') +def auth_token_resolver(obj: Any, info: GraphQLResolveInfo, data: Dict): + # non-nullable fields + user = data['username'] + passwd = data['password'] + + secret = state.settings['secret'] + res = generate_token(user, passwd, secret) + if res: + data['result'] = res + return { + "success": True, + "data": data + } + + return { + "success": False, + "errors": ['token generation failed'] + } diff --git a/src/services/api/graphql/graphql/schema/auth_token.graphql b/src/services/api/graphql/graphql/schema/auth_token.graphql new file mode 100644 index 000000000..af53a293a --- /dev/null +++ b/src/services/api/graphql/graphql/schema/auth_token.graphql @@ -0,0 +1,19 @@ + +input AuthTokenInput { + username: String! + password: String! +} + +type AuthToken { + result: Generic +} + +type AuthTokenResult { + data: AuthToken + success: Boolean! + errors: [String] +} + +extend type Mutation { + AuthToken(data: AuthTokenInput) : AuthTokenResult +} diff --git a/src/services/api/graphql/libs/token_auth.py b/src/services/api/graphql/libs/token_auth.py new file mode 100644 index 000000000..c53e354b1 --- /dev/null +++ b/src/services/api/graphql/libs/token_auth.py @@ -0,0 +1,38 @@ +import jwt +import uuid +import pam +from secrets import token_hex + +from .. import state + +def _check_passwd_pam(username: str, passwd: str) -> bool: + if pam.authenticate(username, passwd): + return True + return False + +def init_secret(): + secret = token_hex(16) + state.settings['secret'] = secret + +def generate_token(user: str, passwd: str, secret: str) -> dict: + if user is None or passwd is None: + return {} + if _check_passwd_pam(user, passwd): + app = state.settings['app'] + try: + users = app.state.vyos_token_users + except AttributeError: + app.state.vyos_token_users = {} + users = app.state.vyos_token_users + user_id = uuid.uuid1().hex + payload_data = {'iss': user, 'sub': user_id} + secret = state.settings.get('secret') + if secret is None: + return { + "success": False, + "errors": ['failed secret generation'] + } + token = jwt.encode(payload=payload_data, key=secret, algorithm="HS256") + + users |= {user_id: user} + return {'token': token} -- cgit v1.2.3 From af56ddf4615974c6b5f5886520d6abb0781cea80 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 23 Oct 2022 11:07:46 -0500 Subject: graphql: T4574: read config and generate schema with/without key auth --- .../api/graphql/generate/schema_from_composite.py | 46 +++++++++++++++++++++- .../graphql/generate/schema_from_config_session.py | 46 +++++++++++++++++++++- .../api/graphql/generate/schema_from_op_mode.py | 46 +++++++++++++++++++++- src/services/vyos-http-api-server | 15 ++++--- 4 files changed, 144 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/services/api/graphql/generate/schema_from_composite.py b/src/services/api/graphql/generate/schema_from_composite.py index 7187047a0..61a08cb2f 100755 --- a/src/services/api/graphql/generate/schema_from_composite.py +++ b/src/services/api/graphql/generate/schema_from_composite.py @@ -29,22 +29,50 @@ if __package__ is None or __package__ == '': sys.path.append("/usr/libexec/vyos/services/api") from graphql.libs.op_mode import snake_to_pascal_case, map_type_name from composite_function import queries, mutations + from vyos.config import Config + from vyos.configdict import dict_merge + from vyos.xml import defaults else: from .. libs.op_mode import snake_to_pascal_case, map_type_name from . composite_function import queries, mutations + from .. import state SCHEMA_PATH = directories['api_schema'] -schema_data: dict = {'schema_name': '', +if __package__ is None or __package__ == '': + # allow running stand-alone + conf = Config() + base = ['service', 'https', 'api'] + graphql_dict = conf.get_config_dict(base, key_mangling=('-', '_'), + no_tag_node_value_mangle=True, + get_first_key=True) + if 'graphql' not in graphql_dict: + exit("graphql is not configured") + + graphql_dict = dict_merge(defaults(base), graphql_dict) + auth_type = graphql_dict['graphql']['authentication']['type'] +else: + auth_type = state.settings['app'].state.vyos_auth_type + +schema_data: dict = {'auth_type': auth_type, + 'schema_name': '', 'schema_fields': []} query_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -57,17 +85,29 @@ type {{ schema_name }}Result { } extend type Query { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @compositequery +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @compositequery +{%- endif %} } """ mutation_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -80,7 +120,11 @@ type {{ schema_name }}Result { } extend type Mutation { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @compositemutation +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @compositemutation +{%- endif %} } """ diff --git a/src/services/api/graphql/generate/schema_from_config_session.py b/src/services/api/graphql/generate/schema_from_config_session.py index cf69cbafd..49bf2440e 100755 --- a/src/services/api/graphql/generate/schema_from_config_session.py +++ b/src/services/api/graphql/generate/schema_from_config_session.py @@ -29,22 +29,50 @@ if __package__ is None or __package__ == '': sys.path.append("/usr/libexec/vyos/services/api") from graphql.libs.op_mode import snake_to_pascal_case, map_type_name from config_session_function import queries, mutations + from vyos.config import Config + from vyos.configdict import dict_merge + from vyos.xml import defaults else: from .. libs.op_mode import snake_to_pascal_case, map_type_name from . config_session_function import queries, mutations + from .. import state SCHEMA_PATH = directories['api_schema'] -schema_data: dict = {'schema_name': '', +if __package__ is None or __package__ == '': + # allow running stand-alone + conf = Config() + base = ['service', 'https', 'api'] + graphql_dict = conf.get_config_dict(base, key_mangling=('-', '_'), + no_tag_node_value_mangle=True, + get_first_key=True) + if 'graphql' not in graphql_dict: + exit("graphql is not configured") + + graphql_dict = dict_merge(defaults(base), graphql_dict) + auth_type = graphql_dict['graphql']['authentication']['type'] +else: + auth_type = state.settings['app'].state.vyos_auth_type + +schema_data: dict = {'auth_type': auth_type, + 'schema_name': '', 'schema_fields': []} query_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -57,17 +85,29 @@ type {{ schema_name }}Result { } extend type Query { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @configsessionquery +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @configsessionquery +{%- endif %} } """ mutation_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -80,7 +120,11 @@ type {{ schema_name }}Result { } extend type Mutation { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @configsessionmutation +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @configsessionmutation +{%- endif %} } """ diff --git a/src/services/api/graphql/generate/schema_from_op_mode.py b/src/services/api/graphql/generate/schema_from_op_mode.py index a88816b34..1fd198a37 100755 --- a/src/services/api/graphql/generate/schema_from_op_mode.py +++ b/src/services/api/graphql/generate/schema_from_op_mode.py @@ -29,9 +29,13 @@ if __package__ is None or __package__ == '': sys.path.append("/usr/libexec/vyos/services/api") from graphql.libs.op_mode import load_as_module, is_op_mode_function_name, is_show_function_name from graphql.libs.op_mode import snake_to_pascal_case, map_type_name + from vyos.config import Config + from vyos.configdict import dict_merge + from vyos.xml import defaults else: from .. libs.op_mode import load_as_module, is_op_mode_function_name, is_show_function_name from .. libs.op_mode import snake_to_pascal_case, map_type_name + from .. import state OP_MODE_PATH = directories['op_mode'] SCHEMA_PATH = directories['api_schema'] @@ -40,16 +44,40 @@ DATA_DIR = directories['data'] op_mode_include_file = os.path.join(DATA_DIR, 'op-mode-standardized.json') op_mode_error_schema = 'op_mode_error.graphql' -schema_data: dict = {'schema_name': '', +if __package__ is None or __package__ == '': + # allow running stand-alone + conf = Config() + base = ['service', 'https', 'api'] + graphql_dict = conf.get_config_dict(base, key_mangling=('-', '_'), + no_tag_node_value_mangle=True, + get_first_key=True) + if 'graphql' not in graphql_dict: + exit("graphql is not configured") + + graphql_dict = dict_merge(defaults(base), graphql_dict) + auth_type = graphql_dict['graphql']['authentication']['type'] +else: + auth_type = state.settings['app'].state.vyos_auth_type + +schema_data: dict = {'auth_type': auth_type, + 'schema_name': '', 'schema_fields': []} query_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -63,17 +91,29 @@ type {{ schema_name }}Result { } extend type Query { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @genopquery +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @genopquery +{%- endif %} } """ mutation_template = """ +{%- if auth_type == 'key' %} input {{ schema_name }}Input { key: String! {%- for field_entry in schema_fields %} {{ field_entry }} {%- endfor %} } +{%- elif schema_fields %} +input {{ schema_name }}Input { + {%- for field_entry in schema_fields %} + {{ field_entry }} + {%- endfor %} +} +{%- endif %} type {{ schema_name }} { result: Generic @@ -87,7 +127,11 @@ type {{ schema_name }}Result { } extend type Mutation { +{%- if auth_type == 'key' or schema_fields %} {{ schema_name }}(data: {{ schema_name }}Input) : {{ schema_name }}Result @genopmutation +{%- else %} + {{ schema_name }} : {{ schema_name }}Result @genopquery +{%- endif %} } """ diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server index 632c1e87d..7a35546e5 100755 --- a/src/services/vyos-http-api-server +++ b/src/services/vyos-http-api-server @@ -647,11 +647,11 @@ def reset_op(data: ResetModel): ### def graphql_init(fast_api_app): - from api.graphql.bindings import generate_schema - api.graphql.state.init() api.graphql.state.settings['app'] = app + # import after initializaion of state + from api.graphql.bindings import generate_schema schema = generate_schema() in_spec = app.state.vyos_introspection @@ -690,10 +690,13 @@ if __name__ == '__main__': app.state.vyos_origins = server_config.get('cors', {}).get('allow_origin', []) if 'graphql' in server_config: app.state.vyos_graphql = True - if isinstance(server_config['graphql'], dict) and 'introspection' in server_config['graphql']: - app.state.vyos_introspection = True - else: - app.state.vyos_introspection = False + if isinstance(server_config['graphql'], dict): + if 'introspection' in server_config['graphql']: + app.state.vyos_introspection = True + else: + app.state.vyos_introspection = False + # default value is merged in conf_mode http-api.py, if not set + app.state.vyos_auth_type = server_config['graphql']['authentication']['type'] else: app.state.vyos_graphql = False -- cgit v1.2.3 From 28676844e3f4317786e457fcd8651939a05c88ff Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 23 Oct 2022 11:08:06 -0500 Subject: graphql: T4574: add context to read token in queries/mutations --- src/services/api/graphql/graphql/mutations.py | 62 ++++++++++++++++++--------- src/services/api/graphql/graphql/queries.py | 62 ++++++++++++++++++--------- src/services/api/graphql/libs/token_auth.py | 29 +++++++++++++ src/services/vyos-http-api-server | 5 ++- 4 files changed, 116 insertions(+), 42 deletions(-) (limited to 'src') diff --git a/src/services/api/graphql/graphql/mutations.py b/src/services/api/graphql/graphql/mutations.py index f0c8b438f..2778feb69 100644 --- a/src/services/api/graphql/graphql/mutations.py +++ b/src/services/api/graphql/graphql/mutations.py @@ -42,32 +42,54 @@ def make_mutation_resolver(mutation_name, class_name, session_func): func_base_name = convert_camel_case_to_snake(class_name) resolver_name = f'resolve_{func_base_name}' - func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Dict)' + func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Dict = {})' @mutation.field(mutation_name) @convert_kwargs_to_snake_case @with_signature(func_sig, func_name=resolver_name) async def func_impl(*args, **kwargs): try: - if 'data' not in kwargs: - return { - "success": False, - "errors": ['missing data'] - } - - data = kwargs['data'] - key = data['key'] - - auth = key_auth.auth_required(key) - if auth is None: - return { - "success": False, - "errors": ['invalid API key'] - } - - # We are finished with the 'key' entry, and may remove so as to - # pass the rest of data (if any) to function. - del data['key'] + auth_type = state.settings['app'].state.vyos_auth_type + + if auth_type == 'key': + data = kwargs['data'] + key = data['key'] + + auth = key_auth.auth_required(key) + if auth is None: + return { + "success": False, + "errors": ['invalid API key'] + } + + # We are finished with the 'key' entry, and may remove so as to + # pass the rest of data (if any) to function. + del data['key'] + + elif auth_type == 'token': + # there is a subtlety here: with the removal of the key entry, + # some requests will now have empty input, hence no data arg, so + # make it optional in the func_sig. However, it can not be None, + # as the makefun package provides accurate TypeError exceptions; + # hence set it to {}, but now it is a mutable default argument, + # so clear the key 'result', which is added at the end of + # this function. + data = kwargs['data'] + if 'result' in data: + del data['result'] + + info = kwargs['info'] + user = info.context.get('user') + if user is None: + return { + "success": False, + "errors": ['not authenticated'] + } + else: + # AtrributeError will have already been raised if no + # vyos_auth_type; validation and defaultValue ensure it is + # one of the previous cases, so this is never reached. + pass session = state.settings['app'].state.vyos_session diff --git a/src/services/api/graphql/graphql/queries.py b/src/services/api/graphql/graphql/queries.py index 13eb59ae4..9c8a4f064 100644 --- a/src/services/api/graphql/graphql/queries.py +++ b/src/services/api/graphql/graphql/queries.py @@ -42,32 +42,54 @@ def make_query_resolver(query_name, class_name, session_func): func_base_name = convert_camel_case_to_snake(class_name) resolver_name = f'resolve_{func_base_name}' - func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Dict)' + func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Dict = {})' @query.field(query_name) @convert_kwargs_to_snake_case @with_signature(func_sig, func_name=resolver_name) async def func_impl(*args, **kwargs): try: - if 'data' not in kwargs: - return { - "success": False, - "errors": ['missing data'] - } - - data = kwargs['data'] - key = data['key'] - - auth = key_auth.auth_required(key) - if auth is None: - return { - "success": False, - "errors": ['invalid API key'] - } - - # We are finished with the 'key' entry, and may remove so as to - # pass the rest of data (if any) to function. - del data['key'] + auth_type = state.settings['app'].state.vyos_auth_type + + if auth_type == 'key': + data = kwargs['data'] + key = data['key'] + + auth = key_auth.auth_required(key) + if auth is None: + return { + "success": False, + "errors": ['invalid API key'] + } + + # We are finished with the 'key' entry, and may remove so as to + # pass the rest of data (if any) to function. + del data['key'] + + elif auth_type == 'token': + # there is a subtlety here: with the removal of the key entry, + # some requests will now have empty input, hence no data arg, so + # make it optional in the func_sig. However, it can not be None, + # as the makefun package provides accurate TypeError exceptions; + # hence set it to {}, but now it is a mutable default argument, + # so clear the key 'result', which is added at the end of + # this function. + data = kwargs['data'] + if 'result' in data: + del data['result'] + + info = kwargs['info'] + user = info.context.get('user') + if user is None: + return { + "success": False, + "errors": ['not authenticated'] + } + else: + # AtrributeError will have already been raised if no + # vyos_auth_type; validation and defaultValue ensure it is + # one of the previous cases, so this is never reached. + pass session = state.settings['app'].state.vyos_session diff --git a/src/services/api/graphql/libs/token_auth.py b/src/services/api/graphql/libs/token_auth.py index c53e354b1..2d63a1cc7 100644 --- a/src/services/api/graphql/libs/token_auth.py +++ b/src/services/api/graphql/libs/token_auth.py @@ -36,3 +36,32 @@ def generate_token(user: str, passwd: str, secret: str) -> dict: users |= {user_id: user} return {'token': token} + +def get_user_context(request): + context = {} + context['request'] = request + context['user'] = None + if 'Authorization' in request.headers: + auth = request.headers['Authorization'] + scheme, token = auth.split() + if scheme.lower() != 'bearer': + return context + + try: + secret = state.settings.get('secret') + payload = jwt.decode(token, secret, algorithms=["HS256"]) + user_id: str = payload.get('sub') + if user_id is None: + return context + except jwt.PyJWTError: + return context + try: + users = state.settings['app'].state.vyos_token_users + except AttributeError: + return context + + user = users.get(user_id) + if user is not None: + context['user'] = user + + return context diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server index 7a35546e5..840041b73 100755 --- a/src/services/vyos-http-api-server +++ b/src/services/vyos-http-api-server @@ -647,6 +647,7 @@ def reset_op(data: ResetModel): ### def graphql_init(fast_api_app): + from api.graphql.libs.token_auth import get_user_context api.graphql.state.init() api.graphql.state.settings['app'] = app @@ -658,9 +659,9 @@ def graphql_init(fast_api_app): if app.state.vyos_origins: origins = app.state.vyos_origins - app.add_route('/graphql', CORSMiddleware(GraphQL(schema, debug=True, introspection=in_spec), allow_origins=origins, allow_methods=("GET", "POST", "OPTIONS"))) + app.add_route('/graphql', CORSMiddleware(GraphQL(schema, context_value=get_user_context, debug=True, introspection=in_spec), allow_origins=origins, allow_methods=("GET", "POST", "OPTIONS"))) else: - app.add_route('/graphql', GraphQL(schema, debug=True, introspection=in_spec)) + app.add_route('/graphql', GraphQL(schema, context_value=get_user_context, debug=True, introspection=in_spec)) ### -- cgit v1.2.3 From dc37f30a1273c1d3b7949b1d64e60d37da3b9fd4 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 23 Oct 2022 11:08:19 -0500 Subject: graphql: T4574: set token expiration time in claims --- src/services/api/graphql/graphql/auth_token_mutation.py | 7 ++++++- src/services/api/graphql/libs/token_auth.py | 4 ++-- src/services/vyos-http-api-server | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/services/api/graphql/graphql/auth_token_mutation.py b/src/services/api/graphql/graphql/auth_token_mutation.py index 33779d4f0..21ac40094 100644 --- a/src/services/api/graphql/graphql/auth_token_mutation.py +++ b/src/services/api/graphql/graphql/auth_token_mutation.py @@ -14,6 +14,7 @@ # along with this library. If not, see . import jwt +import datetime from typing import Any, Dict from ariadne import ObjectType, UnionType from graphql import GraphQLResolveInfo @@ -30,7 +31,11 @@ def auth_token_resolver(obj: Any, info: GraphQLResolveInfo, data: Dict): passwd = data['password'] secret = state.settings['secret'] - res = generate_token(user, passwd, secret) + exp_interval = int(state.settings['app'].state.vyos_token_exp) + expiration = (datetime.datetime.now(tz=datetime.timezone.utc) + + datetime.timedelta(seconds=exp_interval)) + + res = generate_token(user, passwd, secret, expiration) if res: data['result'] = res return { diff --git a/src/services/api/graphql/libs/token_auth.py b/src/services/api/graphql/libs/token_auth.py index 2d63a1cc7..fafb0f5af 100644 --- a/src/services/api/graphql/libs/token_auth.py +++ b/src/services/api/graphql/libs/token_auth.py @@ -14,7 +14,7 @@ def init_secret(): secret = token_hex(16) state.settings['secret'] = secret -def generate_token(user: str, passwd: str, secret: str) -> dict: +def generate_token(user: str, passwd: str, secret: str, exp: int) -> dict: if user is None or passwd is None: return {} if _check_passwd_pam(user, passwd): @@ -25,7 +25,7 @@ def generate_token(user: str, passwd: str, secret: str) -> dict: app.state.vyos_token_users = {} users = app.state.vyos_token_users user_id = uuid.uuid1().hex - payload_data = {'iss': user, 'sub': user_id} + payload_data = {'iss': user, 'sub': user_id, 'exp': exp} secret = state.settings.get('secret') if secret is None: return { diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server index 840041b73..4af27b949 100755 --- a/src/services/vyos-http-api-server +++ b/src/services/vyos-http-api-server @@ -698,6 +698,7 @@ if __name__ == '__main__': app.state.vyos_introspection = False # default value is merged in conf_mode http-api.py, if not set app.state.vyos_auth_type = server_config['graphql']['authentication']['type'] + app.state.vyos_token_exp = server_config['graphql']['authentication']['expiration'] else: app.state.vyos_graphql = False -- cgit v1.2.3 From 3db5ba8ef354d80f080cc1baacf33d77ccbb6222 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 25 Oct 2022 09:22:50 -0500 Subject: graphql: T4574: set byte length of shared secret from CLI --- src/services/api/graphql/libs/token_auth.py | 3 ++- src/services/vyos-http-api-server | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/services/api/graphql/libs/token_auth.py b/src/services/api/graphql/libs/token_auth.py index fafb0f5af..3ecd8b855 100644 --- a/src/services/api/graphql/libs/token_auth.py +++ b/src/services/api/graphql/libs/token_auth.py @@ -11,7 +11,8 @@ def _check_passwd_pam(username: str, passwd: str) -> bool: return False def init_secret(): - secret = token_hex(16) + length = int(state.settings['app'].state.vyos_secret_len) + secret = token_hex(length) state.settings['secret'] = secret def generate_token(user: str, passwd: str, secret: str, exp: int) -> dict: diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server index 4af27b949..3c390d9dc 100755 --- a/src/services/vyos-http-api-server +++ b/src/services/vyos-http-api-server @@ -699,6 +699,7 @@ if __name__ == '__main__': # default value is merged in conf_mode http-api.py, if not set app.state.vyos_auth_type = server_config['graphql']['authentication']['type'] app.state.vyos_token_exp = server_config['graphql']['authentication']['expiration'] + app.state.vyos_secret_len = server_config['graphql']['authentication']['secret_length'] else: app.state.vyos_graphql = False -- cgit v1.2.3 From 29b656f14d8bb7622ff861208d26cf8eb018670d Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 25 Oct 2022 13:42:30 -0500 Subject: vyos.util: T4773: add camel_to_snake_case conversion --- python/vyos/util.py | 7 +++++++ src/tests/test_util.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+) (limited to 'src') diff --git a/python/vyos/util.py b/python/vyos/util.py index 461df9a6e..e4e2a44ec 100644 --- a/python/vyos/util.py +++ b/python/vyos/util.py @@ -1105,3 +1105,10 @@ def sysctl_write(name, value): call(f'sysctl -wq {name}={value}') return True return False + +# approach follows a discussion in: +# https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case +def camel_to_snake_case(name: str) -> str: + pattern = r'\d+|[A-Z]?[a-z]+|\W|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)' + words = re.findall(pattern, name) + return '_'.join(map(str.lower, words)) diff --git a/src/tests/test_util.py b/src/tests/test_util.py index 8ac9a500a..d8b2b7940 100644 --- a/src/tests/test_util.py +++ b/src/tests/test_util.py @@ -26,3 +26,17 @@ class TestVyOSUtil(TestCase): def test_sysctl_read(self): self.assertEqual(sysctl_read('net.ipv4.conf.lo.forwarding'), '1') + + def test_camel_to_snake_case(self): + self.assertEqual(camel_to_snake_case('ConnectionTimeout'), + 'connection_timeout') + self.assertEqual(camel_to_snake_case('connectionTimeout'), + 'connection_timeout') + self.assertEqual(camel_to_snake_case('TCPConnectionTimeout'), + 'tcp_connection_timeout') + self.assertEqual(camel_to_snake_case('TCPPort'), + 'tcp_port') + self.assertEqual(camel_to_snake_case('UseHTTPProxy'), + 'use_http_proxy') + self.assertEqual(camel_to_snake_case('CustomerID'), + 'customer_id') -- cgit v1.2.3 From 2a5273e650ce1242bc22e992e5a3104961ec1295 Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Tue, 25 Oct 2022 12:29:03 +0200 Subject: nat: T4764: Remove tables on NAT deletion --- data/templates/firewall/nftables-nat.j2 | 18 ++++++++++-------- data/templates/firewall/nftables-static-nat.j2 | 18 ++++++++++-------- smoketest/scripts/cli/test_nat.py | 6 ++++++ src/conf_mode/nat.py | 4 ++++ 4 files changed, 30 insertions(+), 16 deletions(-) (limited to 'src') diff --git a/data/templates/firewall/nftables-nat.j2 b/data/templates/firewall/nftables-nat.j2 index 55fe6024b..c5c0a2c86 100644 --- a/data/templates/firewall/nftables-nat.j2 +++ b/data/templates/firewall/nftables-nat.j2 @@ -24,6 +24,7 @@ add rule ip raw NAT_CONNTRACK counter accept {% if first_install is not vyos_defined %} delete table ip vyos_nat {% endif %} +{% if deleted is not vyos_defined %} table ip vyos_nat { # # Destination NAT rules build up here @@ -31,11 +32,11 @@ table ip vyos_nat { chain PREROUTING { type nat hook prerouting priority -100; policy accept; counter jump VYOS_PRE_DNAT_HOOK -{% if destination.rule is vyos_defined %} -{% for rule, config in destination.rule.items() if config.disable is not vyos_defined %} +{% if destination.rule is vyos_defined %} +{% for rule, config in destination.rule.items() if config.disable is not vyos_defined %} {{ config | nat_rule(rule, 'destination') }} -{% endfor %} -{% endif %} +{% endfor %} +{% endif %} } # @@ -44,11 +45,11 @@ table ip vyos_nat { chain POSTROUTING { type nat hook postrouting priority 100; policy accept; counter jump VYOS_PRE_SNAT_HOOK -{% if source.rule is vyos_defined %} -{% for rule, config in source.rule.items() if config.disable is not vyos_defined %} +{% if source.rule is vyos_defined %} +{% for rule, config in source.rule.items() if config.disable is not vyos_defined %} {{ config | nat_rule(rule, 'source') }} -{% endfor %} -{% endif %} +{% endfor %} +{% endif %} } chain VYOS_PRE_DNAT_HOOK { @@ -59,3 +60,4 @@ table ip vyos_nat { return } } +{% endif %} diff --git a/data/templates/firewall/nftables-static-nat.j2 b/data/templates/firewall/nftables-static-nat.j2 index 790c33ce9..e5e3da867 100644 --- a/data/templates/firewall/nftables-static-nat.j2 +++ b/data/templates/firewall/nftables-static-nat.j2 @@ -3,6 +3,7 @@ {% if first_install is not vyos_defined %} delete table ip vyos_static_nat {% endif %} +{% if deleted is not vyos_defined %} table ip vyos_static_nat { # # Destination NAT rules build up here @@ -10,11 +11,11 @@ table ip vyos_static_nat { chain PREROUTING { type nat hook prerouting priority -100; policy accept; -{% if static.rule is vyos_defined %} -{% for rule, config in static.rule.items() if config.disable is not vyos_defined %} +{% if static.rule is vyos_defined %} +{% for rule, config in static.rule.items() if config.disable is not vyos_defined %} {{ config | nat_static_rule(rule, 'destination') }} -{% endfor %} -{% endif %} +{% endfor %} +{% endif %} } # @@ -22,10 +23,11 @@ table ip vyos_static_nat { # chain POSTROUTING { type nat hook postrouting priority 100; policy accept; -{% if static.rule is vyos_defined %} -{% for rule, config in static.rule.items() if config.disable is not vyos_defined %} +{% if static.rule is vyos_defined %} +{% for rule, config in static.rule.items() if config.disable is not vyos_defined %} {{ config | nat_static_rule(rule, 'source') }} -{% endfor %} -{% endif %} +{% endfor %} +{% endif %} } } +{% endif %} diff --git a/smoketest/scripts/cli/test_nat.py b/smoketest/scripts/cli/test_nat.py index f824838c0..2ae90fcaf 100755 --- a/smoketest/scripts/cli/test_nat.py +++ b/smoketest/scripts/cli/test_nat.py @@ -16,6 +16,7 @@ import jmespath import json +import os import unittest from base_vyostest_shim import VyOSUnitTestSHIM @@ -28,6 +29,9 @@ src_path = base_path + ['source'] dst_path = base_path + ['destination'] static_path = base_path + ['static'] +nftables_nat_config = '/run/nftables_nat.conf' +nftables_static_nat_conf = '/run/nftables_static-nat-rules.nft' + class TestNAT(VyOSUnitTestSHIM.TestCase): @classmethod def setUpClass(cls): @@ -40,6 +44,8 @@ class TestNAT(VyOSUnitTestSHIM.TestCase): def tearDown(self): self.cli_delete(base_path) self.cli_commit() + self.assertFalse(os.path.exists(nftables_nat_config)) + self.assertFalse(os.path.exists(nftables_static_nat_conf)) def verify_nftables(self, nftables_search, table, inverse=False, args=''): nftables_output = cmd(f'sudo nft {args} list table {table}') diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py index 8b1a5a720..1e807753d 100755 --- a/src/conf_mode/nat.py +++ b/src/conf_mode/nat.py @@ -204,6 +204,10 @@ def apply(nat): cmd(f'nft -f {nftables_nat_config}') cmd(f'nft -f {nftables_static_nat_conf}') + if not nat or 'deleted' in nat: + os.unlink(nftables_nat_config) + os.unlink(nftables_static_nat_conf) + return None if __name__ == '__main__': -- cgit v1.2.3 From 16207f7a8ffdbc93fcfcc4b6ba783940a1e40e33 Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Tue, 25 Oct 2022 22:41:55 +0200 Subject: nat: T4706: Verify translation address or port exists --- src/conf_mode/nat.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'src') diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py index 1e807753d..978c043e9 100755 --- a/src/conf_mode/nat.py +++ b/src/conf_mode/nat.py @@ -146,6 +146,10 @@ def verify(nat): if config['outbound_interface'] not in 'any' and config['outbound_interface'] not in interfaces(): Warning(f'rule "{rule}" interface "{config["outbound_interface"]}" does not exist on this system') + if not dict_search('translation.address', config) and not dict_search('translation.port', config): + if 'exclude' not in config: + raise ConfigError(f'{err_msg} translation requires address and/or port') + addr = dict_search('translation.address', config) if addr != None and addr != 'masquerade' and not is_ip_network(addr): for ip in addr.split('-'): @@ -166,6 +170,10 @@ def verify(nat): elif config['inbound_interface'] not in 'any' and config['inbound_interface'] not in interfaces(): Warning(f'rule "{rule}" interface "{config["inbound_interface"]}" does not exist on this system') + if not dict_search('translation.address', config) and not dict_search('translation.port', config): + if 'exclude' not in config: + raise ConfigError(f'{err_msg} translation requires address and/or port') + # common rule verification verify_rule(config, err_msg) -- cgit v1.2.3 From 413e24400c54f398ef73347df2e877aef422400e Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 27 Oct 2022 13:57:25 -0500 Subject: ipsec: T4778: raise UnconfiguredSubsystem if IPsec not initialized --- src/op_mode/ipsec.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/op_mode/ipsec.py b/src/op_mode/ipsec.py index 7ec35d7bd..aaa0cec5a 100755 --- a/src/op_mode/ipsec.py +++ b/src/op_mode/ipsec.py @@ -43,7 +43,10 @@ def _alphanum_key(key): def _get_vici_sas(): from vici import Session as vici_session - session = vici_session() + try: + session = vici_session() + except Exception: + raise vyos.opmode.UnconfiguredSubsystem("IPsec not initialized") sas = list(session.list_sas()) return sas -- cgit v1.2.3 From c2ff9aa158b81fa66ce9c810e891ad25d4a7f14b Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Thu, 27 Oct 2022 22:37:42 +0200 Subject: wireguard: T4774: Prevent duplicate peer public keys --- smoketest/scripts/cli/test_interfaces_wireguard.py | 10 ++++++++-- src/conf_mode/interfaces-wireguard.py | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/smoketest/scripts/cli/test_interfaces_wireguard.py b/smoketest/scripts/cli/test_interfaces_wireguard.py index f3e9670f7..14fc8d109 100755 --- a/smoketest/scripts/cli/test_interfaces_wireguard.py +++ b/smoketest/scripts/cli/test_interfaces_wireguard.py @@ -62,10 +62,10 @@ class WireGuardInterfaceTest(VyOSUnitTestSHIM.TestCase): self.assertTrue(os.path.isdir(f'/sys/class/net/{intf}')) - def test_wireguard_add_remove_peer(self): # T2939: Create WireGuard interfaces with associated peers. # Remove one of the configured peers. + # T4774: Test prevention of duplicate peer public keys interface = 'wg0' port = '12345' privkey = '6ISOkASm6VhHOOSz/5iIxw+Q9adq9zA17iMM4X40dlc=' @@ -80,11 +80,17 @@ class WireGuardInterfaceTest(VyOSUnitTestSHIM.TestCase): self.cli_set(base_path + [interface, 'peer', 'PEER01', 'allowed-ips', '10.205.212.10/32']) self.cli_set(base_path + [interface, 'peer', 'PEER01', 'address', '192.0.2.1']) - self.cli_set(base_path + [interface, 'peer', 'PEER02', 'public-key', pubkey_2]) + self.cli_set(base_path + [interface, 'peer', 'PEER02', 'public-key', pubkey_1]) self.cli_set(base_path + [interface, 'peer', 'PEER02', 'port', port]) self.cli_set(base_path + [interface, 'peer', 'PEER02', 'allowed-ips', '10.205.212.11/32']) self.cli_set(base_path + [interface, 'peer', 'PEER02', 'address', '192.0.2.2']) + # Duplicate pubkey_1 + with self.assertRaises(ConfigSessionError): + self.cli_commit() + + self.cli_set(base_path + [interface, 'peer', 'PEER02', 'public-key', pubkey_2]) + # Commit peers self.cli_commit() diff --git a/src/conf_mode/interfaces-wireguard.py b/src/conf_mode/interfaces-wireguard.py index 8d738f55e..762bad94f 100755 --- a/src/conf_mode/interfaces-wireguard.py +++ b/src/conf_mode/interfaces-wireguard.py @@ -87,6 +87,8 @@ def verify(wireguard): 'cannot be used for the interface!') # run checks on individual configured WireGuard peer + public_keys = [] + for tmp in wireguard['peer']: peer = wireguard['peer'][tmp] @@ -100,6 +102,11 @@ def verify(wireguard): raise ConfigError('Both Wireguard port and address must be defined ' f'for peer "{tmp}" if either one of them is set!') + if peer['public_key'] in public_keys: + raise ConfigError(f'Duplicate public-key defined on peer "{tmp}"') + + public_keys.append(peer['public_key']) + def apply(wireguard): tmp = WireGuardIf(wireguard['ifname']) if 'deleted' in wireguard: -- cgit v1.2.3 From f35195945daba0a81a93b74b280591dd955c193a Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Fri, 28 Oct 2022 09:00:23 -0400 Subject: T4779: use bytes in the raw output of "show system memory" --- src/op_mode/memory.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/op_mode/memory.py b/src/op_mode/memory.py index 178544be4..7666de646 100755 --- a/src/op_mode/memory.py +++ b/src/op_mode/memory.py @@ -20,7 +20,7 @@ import sys import vyos.opmode -def _get_system_memory(): +def _get_raw_data(): from re import search as re_search def find_value(keyword, mem_data): @@ -38,7 +38,7 @@ def _get_system_memory(): used = total - available - res = { + mem_data = { "total": total, "free": available, "used": used, @@ -46,24 +46,21 @@ def _get_system_memory(): "cached": cached } - return res - -def _get_system_memory_human(): - from vyos.util import bytes_to_human - - mem = _get_system_memory() - - for key in mem: + for key in mem_data: # The Linux kernel exposes memory values in kilobytes, # so we need to normalize them - mem[key] = bytes_to_human(mem[key], initial_exponent=10) + mem_data[key] = mem_data[key] * 1024 - return mem - -def _get_raw_data(): - return _get_system_memory_human() + return mem_data def _get_formatted_output(mem): + from vyos.util import bytes_to_human + + # For human-readable outputs, we convert bytes to more convenient units + # (100M, 1.3G...) + for key in mem: + mem[key] = bytes_to_human(mem[key]) + out = "Total: {}\n".format(mem["total"]) out += "Free: {}\n".format(mem["free"]) out += "Used: {}".format(mem["used"]) -- cgit v1.2.3 From 0e63712195465c9bf0bf55c369b86961d54dfaac Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 27 Oct 2022 12:24:44 -0500 Subject: T4291: consolidate component version string read/write functions --- python/vyos/component_version.py | 192 ++++++++++++++++++++++++ python/vyos/component_versions.py | 57 ------- python/vyos/formatversions.py | 109 -------------- python/vyos/migrator.py | 32 ++-- python/vyos/systemversions.py | 46 ------ smoketest/scripts/cli/test_component_version.py | 6 +- src/helpers/system-versions-foot.py | 21 +-- 7 files changed, 213 insertions(+), 250 deletions(-) create mode 100644 python/vyos/component_version.py delete mode 100644 python/vyos/component_versions.py delete mode 100644 python/vyos/formatversions.py delete mode 100644 python/vyos/systemversions.py (limited to 'src') diff --git a/python/vyos/component_version.py b/python/vyos/component_version.py new file mode 100644 index 000000000..a4e318d08 --- /dev/null +++ b/python/vyos/component_version.py @@ -0,0 +1,192 @@ +# Copyright 2022 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . + +""" +Functions for reading/writing component versions. + +The config file version string has the following form: + +VyOS 1.3/1.4: + +// Warning: Do not remove the following line. +// vyos-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack@3:conntrack-sync@2:dhcp-relay@2:dhcp-server@6:dhcpv6-server@1:dns-forwarding@3:firewall@5:https@2:interfaces@22:ipoe-server@1:ipsec@5:isis@1:l2tp@3:lldp@1:mdns@1:nat@5:ntp@1:pppoe-server@5:pptp@2:qos@1:quagga@8:rpki@1:salt@1:snmp@2:ssh@2:sstp@3:system@21:vrrp@2:vyos-accel-ppp@2:wanloadbalance@3:webproxy@2:zone-policy@1" +// Release version: 1.3.0 + +VyOS 1.2: + +/* Warning: Do not remove the following line. */ +/* === vyatta-config-version: "broadcast-relay@1:cluster@1:config-management@1:conntrack-sync@1:conntrack@1:dhcp-relay@2:dhcp-server@5:dns-forwarding@1:firewall@5:ipsec@5:l2tp@1:mdns@1:nat@4:ntp@1:pppoe-server@2:pptp@1:qos@1:quagga@7:snmp@1:ssh@1:system@10:vrrp@2:wanloadbalance@3:webgui@1:webproxy@2:zone-policy@1" === */ +/* Release version: 1.2.8 */ + +""" + +import os +import re +import sys +import fileinput + +from vyos.xml import component_version +from vyos.version import get_version +from vyos.defaults import directories + +DEFAULT_CONFIG_PATH = os.path.join(directories['config'], 'config.boot') + +def from_string(string_line, vintage='vyos'): + """ + Get component version dictionary from string. + Return empty dictionary if string contains no config information + or raise error if component version string malformed. + """ + version_dict = {} + + if vintage == 'vyos': + if re.match(r'// vyos-config-version:.+', string_line): + if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', string_line): + raise ValueError(f"malformed configuration string: {string_line}") + + for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): + version_dict[pair[0]] = int(pair[1]) + + elif vintage == 'vyatta': + if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line): + if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line): + raise ValueError(f"malformed configuration string: {string_line}") + + for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): + version_dict[pair[0]] = int(pair[1]) + else: + raise ValueError("Unknown config string vintage") + + return version_dict + +def from_file(config_file_name=DEFAULT_CONFIG_PATH, vintage='vyos'): + """ + Get component version dictionary parsing config file line by line + """ + with open(config_file_name, 'r') as f: + for line_in_config in f: + version_dict = from_string(line_in_config, vintage=vintage) + if version_dict: + return version_dict + + # no version information + return {} + +def from_system(): + """ + Get system component version dict. + """ + return component_version() + +def legacy_from_system(): + """ + Get system component version dict from legacy location. + This is for a transitional sanity check; the directory will eventually + be removed. + """ + system_versions = {} + legacy_dir = directories['current'] + + # To be removed: + if not os.path.isdir(legacy_dir): + return system_versions + + try: + version_info = os.listdir(legacy_dir) + except OSError as err: + sys.exit(repr(err)) + + for info in version_info: + if re.match(r'[\w,-]+@\d+', info): + pair = info.split('@') + system_versions[pair[0]] = int(pair[1]) + + return system_versions + +def format_string(ver: dict) -> str: + """ + Version dict to string. + """ + keys = list(ver) + keys.sort() + l = [] + for k in keys: + v = ver[k] + l.append(f'{k}@{v}') + sep = ':' + return sep.join(l) + +def version_footer(ver: dict, vintage='vyos') -> str: + """ + Version footer as string. + """ + ver_str = format_string(ver) + release = get_version() + if vintage == 'vyos': + ret_str = (f'// Warning: Do not remove the following line.\n' + + f'// vyos-config-version: "{ver_str}"\n' + + f'// Release version: {release}\n') + elif vintage == 'vyatta': + ret_str = (f'/* Warning: Do not remove the following line. */\n' + + f'/* === vyatta-config-version: "{ver_str}" === */\n' + + f'/* Release version: {release} */\n') + else: + raise ValueError("Unknown config string vintage") + + return ret_str + +def system_footer(vintage='vyos') -> str: + """ + System version footer as string. + """ + ver_d = from_system() + return version_footer(ver_d, vintage=vintage) + +def write_version_footer(ver: dict, file_name, vintage='vyos'): + """ + Write version footer to file. + """ + footer = version_footer(ver=ver, vintage=vintage) + if file_name: + with open(file_name, 'a') as f: + f.write(footer) + else: + sys.stdout.write(footer) + +def write_system_footer(file_name, vintage='vyos'): + """ + Write system version footer to file. + """ + ver_d = from_system() + return write_version_footer(ver_d, file_name=file_name, vintage=vintage) + +def remove_footer(file_name): + """ + Remove old version footer. + """ + for line in fileinput.input(file_name, inplace=True): + if re.match(r'/\* Warning:.+ \*/$', line): + continue + if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line): + continue + if re.match(r'/\* Release version:.+ \*/$', line): + continue + if re.match('// vyos-config-version:.+', line): + continue + if re.match('// Warning:.+', line): + continue + if re.match('// Release version:.+', line): + continue + sys.stdout.write(line) diff --git a/python/vyos/component_versions.py b/python/vyos/component_versions.py deleted file mode 100644 index 90b458aae..000000000 --- a/python/vyos/component_versions.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2017 VyOS maintainers and contributors -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see . - -""" -The version data looks like: - -/* Warning: Do not remove the following line. */ -/* === vyatta-config-version: -"cluster@1:config-management@1:conntrack-sync@1:conntrack@1:dhcp-relay@1:dhcp-server@4:firewall@5:ipsec@4:nat@4:qos@1:quagga@2:system@8:vrrp@1:wanloadbalance@3:webgui@1:webproxy@1:zone-policy@1" -=== */ -/* Release version: 1.2.0-rolling+201806131737 */ -""" - -import re - -def get_component_version(string_line): - """ - Get component version dictionary from string - return empty dictionary if string contains no config information - or raise error if component version string malformed - """ - return_value = {} - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', string_line): - - if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', string_line): - raise ValueError("malformed configuration string: " + str(string_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', string_line): - if pair[0] in return_value.keys(): - raise ValueError("duplicate unit name: \"" + str(pair[0]) + "\" in string: \"" + string_line + "\"") - return_value[pair[0]] = int(pair[1]) - - return return_value - - -def get_component_versions_from_file(config_file_name='/opt/vyatta/etc/config/config.boot'): - """ - Get component version dictionary parsing config file line by line - """ - f = open(config_file_name, 'r') - for line_in_config in f: - component_version = get_component_version(line_in_config) - if component_version: - return component_version - raise ValueError("no config string in file:", config_file_name) diff --git a/python/vyos/formatversions.py b/python/vyos/formatversions.py deleted file mode 100644 index 29117a5d3..000000000 --- a/python/vyos/formatversions.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2019 VyOS maintainers and contributors -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see . - -import sys -import os -import re -import fileinput - -def read_vyatta_versions(config_file): - config_file_versions = {} - - with open(config_file, 'r') as config_file_handle: - for config_line in config_file_handle: - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', config_line): - if not re.match(r'/\* === vyatta-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s+=== \*/$', config_line): - raise ValueError("malformed configuration string: " - "{}".format(config_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', config_line): - config_file_versions[pair[0]] = int(pair[1]) - - - return config_file_versions - -def read_vyos_versions(config_file): - config_file_versions = {} - - with open(config_file, 'r') as config_file_handle: - for config_line in config_file_handle: - if re.match(r'// vyos-config-version:.+', config_line): - if not re.match(r'// vyos-config-version:\s+"([\w,-]+@\d+:)+([\w,-]+@\d+)"\s*', config_line): - raise ValueError("malformed configuration string: " - "{}".format(config_line)) - - for pair in re.findall(r'([\w,-]+)@(\d+)', config_line): - config_file_versions[pair[0]] = int(pair[1]) - - return config_file_versions - -def remove_versions(config_file): - """ - Remove old version string. - """ - for line in fileinput.input(config_file, inplace=True): - if re.match(r'/\* Warning:.+ \*/$', line): - continue - if re.match(r'/\* === vyatta-config-version:.+=== \*/$', line): - continue - if re.match(r'/\* Release version:.+ \*/$', line): - continue - if re.match('// vyos-config-version:.+', line): - continue - if re.match('// Warning:.+', line): - continue - if re.match('// Release version:.+', line): - continue - sys.stdout.write(line) - -def format_versions_string(config_versions): - cfg_keys = list(config_versions.keys()) - cfg_keys.sort() - - component_version_strings = [] - - for key in cfg_keys: - cfg_vers = config_versions[key] - component_version_strings.append('{}@{}'.format(key, cfg_vers)) - - separator = ":" - component_version_string = separator.join(component_version_strings) - - return component_version_string - -def write_vyatta_versions_foot(config_file, component_version_string, - os_version_string): - if config_file: - with open(config_file, 'a') as config_file_handle: - config_file_handle.write('/* Warning: Do not remove the following line. */\n') - config_file_handle.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string)) - config_file_handle.write('/* Release version: {} */\n'.format(os_version_string)) - else: - sys.stdout.write('/* Warning: Do not remove the following line. */\n') - sys.stdout.write('/* === vyatta-config-version: "{}" === */\n'.format(component_version_string)) - sys.stdout.write('/* Release version: {} */\n'.format(os_version_string)) - -def write_vyos_versions_foot(config_file, component_version_string, - os_version_string): - if config_file: - with open(config_file, 'a') as config_file_handle: - config_file_handle.write('// Warning: Do not remove the following line.\n') - config_file_handle.write('// vyos-config-version: "{}"\n'.format(component_version_string)) - config_file_handle.write('// Release version: {}\n'.format(os_version_string)) - else: - sys.stdout.write('// Warning: Do not remove the following line.\n') - sys.stdout.write('// vyos-config-version: "{}"\n'.format(component_version_string)) - sys.stdout.write('// Release version: {}\n'.format(os_version_string)) - diff --git a/python/vyos/migrator.py b/python/vyos/migrator.py index c6e3435ca..45ea8b0eb 100644 --- a/python/vyos/migrator.py +++ b/python/vyos/migrator.py @@ -1,4 +1,4 @@ -# Copyright 2019 VyOS maintainers and contributors +# Copyright 2019-2022 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -17,10 +17,8 @@ import sys import os import json import subprocess -import vyos.version import vyos.defaults -import vyos.systemversions as systemversions -import vyos.formatversions as formatversions +import vyos.component_version as component_version class MigratorError(Exception): pass @@ -42,13 +40,13 @@ class Migrator(object): cfg_file = self._config_file component_versions = {} - cfg_versions = formatversions.read_vyatta_versions(cfg_file) + cfg_versions = component_version.from_file(cfg_file, vintage='vyatta') if cfg_versions: self._config_file_vintage = 'vyatta' component_versions = cfg_versions - cfg_versions = formatversions.read_vyos_versions(cfg_file) + cfg_versions = component_version.from_file(cfg_file, vintage='vyos') if cfg_versions: self._config_file_vintage = 'vyos' @@ -157,19 +155,15 @@ class Migrator(object): """ Write new versions string. """ - versions_string = formatversions.format_versions_string(cfg_versions) - - os_version_string = vyos.version.get_version() - if self._config_file_vintage == 'vyatta': - formatversions.write_vyatta_versions_foot(self._config_file, - versions_string, - os_version_string) + component_version.write_version_footer(cfg_versions, + self._config_file, + vintage='vyatta') if self._config_file_vintage == 'vyos': - formatversions.write_vyos_versions_foot(self._config_file, - versions_string, - os_version_string) + component_version.write_version_footer(cfg_versions, + self._config_file, + vintage='vyos') def save_json_record(self, component_versions: dict): """ @@ -200,7 +194,7 @@ class Migrator(object): # This will force calling all migration scripts: cfg_versions = {} - sys_versions = systemversions.get_system_component_version() + sys_versions = component_version.from_system() # save system component versions in json file for easy reference self.save_json_record(sys_versions) @@ -216,7 +210,7 @@ class Migrator(object): if not self._changed: return - formatversions.remove_versions(cfg_file) + component_version.remove_footer(cfg_file) self.write_config_file_versions(rev_versions) @@ -237,7 +231,7 @@ class VirtualMigrator(Migrator): if not self._changed: return - formatversions.remove_versions(cfg_file) + component_version.remove_footer(cfg_file) self.write_config_file_versions(cfg_versions) diff --git a/python/vyos/systemversions.py b/python/vyos/systemversions.py deleted file mode 100644 index f2da76d4f..000000000 --- a/python/vyos/systemversions.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2019 VyOS maintainers and contributors -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this library. If not, see . - -import os -import re -import sys -import vyos.defaults -from vyos.xml import component_version - -# legacy version, reading from the file names in -# /opt/vyatta/etc/config-migrate/current -def get_system_versions(): - """ - Get component versions from running system; critical failure if - unable to read migration directory. - """ - system_versions = {} - - try: - version_info = os.listdir(vyos.defaults.directories['current']) - except OSError as err: - print("OS error: {}".format(err)) - sys.exit(1) - - for info in version_info: - if re.match(r'[\w,-]+@\d+', info): - pair = info.split('@') - system_versions[pair[0]] = int(pair[1]) - - return system_versions - -# read from xml cache -def get_system_component_version(): - return component_version() diff --git a/smoketest/scripts/cli/test_component_version.py b/smoketest/scripts/cli/test_component_version.py index 1355c1f94..7b1b12c53 100755 --- a/smoketest/scripts/cli/test_component_version.py +++ b/smoketest/scripts/cli/test_component_version.py @@ -16,7 +16,7 @@ import unittest -from vyos.systemversions import get_system_versions, get_system_component_version +import vyos.component_version as component_version # After T3474, component versions should be updated in the files in # vyos-1x/interface-definitions/include/version/ @@ -24,8 +24,8 @@ from vyos.systemversions import get_system_versions, get_system_component_versio # that in the xml cache. class TestComponentVersion(unittest.TestCase): def setUp(self): - self.legacy_d = get_system_versions() - self.xml_d = get_system_component_version() + self.legacy_d = component_version.legacy_from_system() + self.xml_d = component_version.from_system() self.set_legacy_d = set(self.legacy_d) self.set_xml_d = set(self.xml_d) diff --git a/src/helpers/system-versions-foot.py b/src/helpers/system-versions-foot.py index 2aa687221..9614f0d28 100755 --- a/src/helpers/system-versions-foot.py +++ b/src/helpers/system-versions-foot.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -# Copyright 2019 VyOS maintainers and contributors +# Copyright 2019, 2022 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,24 +16,13 @@ # along with this library. If not, see . import sys -import vyos.formatversions as formatversions -import vyos.systemversions as systemversions import vyos.defaults -import vyos.version - -sys_versions = systemversions.get_system_component_version() - -component_string = formatversions.format_versions_string(sys_versions) - -os_version_string = vyos.version.get_version() +from vyos.component_version import write_system_footer sys.stdout.write("\n\n") if vyos.defaults.cfg_vintage == 'vyos': - formatversions.write_vyos_versions_foot(None, component_string, - os_version_string) + write_system_footer(None, vintage='vyos') elif vyos.defaults.cfg_vintage == 'vyatta': - formatversions.write_vyatta_versions_foot(None, component_string, - os_version_string) + write_system_footer(None, vintage='vyatta') else: - formatversions.write_vyatta_versions_foot(None, component_string, - os_version_string) + write_system_footer(None, vintage='vyos') -- cgit v1.2.3 From b8b752d5b3503f2874a490582e212edd38c902fc Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Fri, 28 Oct 2022 10:39:17 -0400 Subject: T4779: switch raw output of "show system storage" to bytes --- src/op_mode/storage.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'src') diff --git a/src/op_mode/storage.py b/src/op_mode/storage.py index 75964c493..d16e271bd 100755 --- a/src/op_mode/storage.py +++ b/src/op_mode/storage.py @@ -20,6 +20,16 @@ import sys import vyos.opmode from vyos.util import cmd +# FIY: As of coreutils from Debian Buster and Bullseye, +# the outpt looks like this: +# +# $ df -h -t ext4 --output=source,size,used,avail,pcent +# Filesystem Size Used Avail Use% +# /dev/sda1 16G 7.6G 7.3G 51% +# +# Those field names are automatically normalized by vyos.opmode.run, +# so we don't touch them here, +# and only normalize values. def _get_system_storage(only_persistent=False): if not only_persistent: @@ -32,11 +42,19 @@ def _get_system_storage(only_persistent=False): return res def _get_raw_data(): + from re import sub as re_sub + from vyos.util import human_to_bytes + out = _get_system_storage(only_persistent=True) lines = out.splitlines() lists = [l.split() for l in lines] res = {lists[0][i]: lists[1][i] for i in range(len(lists[0]))} + res["Size"] = human_to_bytes(res["Size"]) + res["Used"] = human_to_bytes(res["Used"]) + res["Avail"] = human_to_bytes(res["Avail"]) + res["Use%"] = re_sub(r'%', '', res["Use%"]) + return res def _get_formatted_output(): -- cgit v1.2.3 From f9c1277f5cf56fba2fc773d133de0221b06fa511 Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Fri, 28 Oct 2022 21:27:37 +0200 Subject: containers: T3903: Use systemd units for containers * ExecStop action with defined timeout allows for quicker reboot/shutdown with containers --- data/templates/container/systemd-unit.j2 | 17 +++ src/conf_mode/container.py | 172 ++++++++++++++++--------------- 2 files changed, 108 insertions(+), 81 deletions(-) create mode 100644 data/templates/container/systemd-unit.j2 (limited to 'src') diff --git a/data/templates/container/systemd-unit.j2 b/data/templates/container/systemd-unit.j2 new file mode 100644 index 000000000..fa48384ab --- /dev/null +++ b/data/templates/container/systemd-unit.j2 @@ -0,0 +1,17 @@ +### Autogenerated by container.py ### +[Unit] +Description=VyOS Container {{ name }} + +[Service] +Environment=PODMAN_SYSTEMD_UNIT=%n +Restart=on-failure +ExecStartPre=/bin/rm -f %t/%n.pid %t/%n.cid +ExecStart=/usr/bin/podman run \ + --conmon-pidfile %t/%n.pid --cidfile %t/%n.cid --cgroups=no-conmon \ + {{ run_args }} +ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%n.cid -t 5 +ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%n.cid +ExecStopPost=/bin/rm -f %t/%n.cid +PIDFile=%t/%n.pid +KillMode=none +Type=forking diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py index ac3dc536b..70d149f0d 100755 --- a/src/conf_mode/container.py +++ b/src/conf_mode/container.py @@ -40,20 +40,7 @@ airbag.enable() config_containers_registry = '/etc/containers/registries.conf' config_containers_storage = '/etc/containers/storage.conf' - -def _run_rerun(container_cmd): - counter = 0 - while True: - if counter >= 10: - break - try: - _cmd(container_cmd) - break - except: - counter = counter +1 - sleep(0.5) - - return None +systemd_unit_path = '/run/systemd/system' def _cmd(command): if os.path.exists('/tmp/vyos.container.debug'): @@ -122,7 +109,7 @@ def verify(container): # of image upgrade and deletion. image = container_config['image'] if run(f'podman image exists {image}') != 0: - Warning(f'Image "{image}" used in contianer "{name}" does not exist '\ + Warning(f'Image "{image}" used in container "{name}" does not exist '\ f'locally. Please use "add container image {image}" to add it '\ f'to the system! Container "{name}" will not be started!') @@ -136,9 +123,6 @@ def verify(container): raise ConfigError(f'Container network "{network_name}" does not exist!') if 'address' in container_config['network'][network_name]: - if 'network' not in container_config: - raise ConfigError(f'Can not use "address" without "network" for container "{name}"!') - address = container_config['network'][network_name]['address'] network = None if is_ipv4(address): @@ -220,6 +204,71 @@ def verify(container): return None +def generate_run_arguments(name, container_config): + image = container_config['image'] + memory = container_config['memory'] + restart = container_config['restart'] + + # Add capability options. Should be in uppercase + cap_add = '' + if 'cap_add' in container_config: + for c in container_config['cap_add']: + c = c.upper() + c = c.replace('-', '_') + cap_add += f' --cap-add={c}' + + # Add a host device to the container /dev/x:/dev/x + device = '' + if 'device' in container_config: + for dev, dev_config in container_config['device'].items(): + source_dev = dev_config['source'] + dest_dev = dev_config['destination'] + device += f' --device={source_dev}:{dest_dev}' + + # Check/set environment options "-e foo=bar" + env_opt = '' + if 'environment' in container_config: + for k, v in container_config['environment'].items(): + env_opt += f" -e \"{k}={v['value']}\"" + + # Publish ports + port = '' + if 'port' in container_config: + protocol = '' + for portmap in container_config['port']: + if 'protocol' in container_config['port'][portmap]: + protocol = container_config['port'][portmap]['protocol'] + protocol = f'/{protocol}' + else: + protocol = '/tcp' + sport = container_config['port'][portmap]['source'] + dport = container_config['port'][portmap]['destination'] + port += f' -p {sport}:{dport}{protocol}' + + # Bind volume + volume = '' + if 'volume' in container_config: + for vol, vol_config in container_config['volume'].items(): + svol = vol_config['source'] + dvol = vol_config['destination'] + volume += f' -v {svol}:{dvol}' + + container_base_cmd = f'--detach --interactive --tty --replace {cap_add} ' \ + f'--memory {memory}m --memory-swap 0 --restart {restart} ' \ + f'--name {name} {device} {port} {volume} {env_opt}' + + if 'allow_host_networks' in container_config: + return f'{container_base_cmd} --net host {image}' + + ip_param = '' + networks = ",".join(container_config['network']) + for network in container_config['network']: + if 'address' in container_config['network'][network]: + address = container_config['network'][network]['address'] + ip_param = f'--ip {address}' + + return f'{container_base_cmd} --net {networks} {ip_param} {image}' + def generate(container): # bail out early - looks like removal from running config if not container: @@ -263,6 +312,15 @@ def generate(container): render(config_containers_registry, 'container/registries.conf.j2', container) render(config_containers_storage, 'container/storage.conf.j2', container) + if 'name' in container: + for name, container_config in container['name'].items(): + if 'disable' in container_config: + continue + + file_path = os.path.join(systemd_unit_path, f'vyos-container-{name}.service') + run_args = generate_run_arguments(name, container_config) + render(file_path, 'container/systemd-unit.j2', {'name': name, 'run_args': run_args}) + return None def apply(container): @@ -270,8 +328,12 @@ def apply(container): # Option "--force" allows to delete containers with any status if 'container_remove' in container: for name in container['container_remove']: - call(f'podman stop --time 3 {name}') - call(f'podman rm --force {name}') + file_path = os.path.join(systemd_unit_path, f'vyos-container-{name}.service') + call(f'systemctl stop vyos-container-{name}.service') + if os.path.exists(file_path): + os.unlink(file_path) + + call('systemctl daemon-reload') # Delete old networks if needed if 'network_remove' in container: @@ -282,6 +344,7 @@ def apply(container): os.unlink(tmp) # Add container + disabled_new = False if 'name' in container: for name, container_config in container['name'].items(): image = container_config['image'] @@ -295,70 +358,17 @@ def apply(container): # check if there is a container by that name running tmp = _cmd('podman ps -a --format "{{.Names}}"') if name in tmp: - _cmd(f'podman stop --time 3 {name}') - _cmd(f'podman rm --force {name}') + file_path = os.path.join(systemd_unit_path, f'vyos-container-{name}.service') + call(f'systemctl stop vyos-container-{name}.service') + if os.path.exists(file_path): + disabled_new = True + os.unlink(file_path) continue - memory = container_config['memory'] - restart = container_config['restart'] - - # Add capability options. Should be in uppercase - cap_add = '' - if 'cap_add' in container_config: - for c in container_config['cap_add']: - c = c.upper() - c = c.replace('-', '_') - cap_add += f' --cap-add={c}' - - # Add a host device to the container /dev/x:/dev/x - device = '' - if 'device' in container_config: - for dev, dev_config in container_config['device'].items(): - source_dev = dev_config['source'] - dest_dev = dev_config['destination'] - device += f' --device={source_dev}:{dest_dev}' - - # Check/set environment options "-e foo=bar" - env_opt = '' - if 'environment' in container_config: - for k, v in container_config['environment'].items(): - env_opt += f" -e \"{k}={v['value']}\"" - - # Publish ports - port = '' - if 'port' in container_config: - protocol = '' - for portmap in container_config['port']: - if 'protocol' in container_config['port'][portmap]: - protocol = container_config['port'][portmap]['protocol'] - protocol = f'/{protocol}' - else: - protocol = '/tcp' - sport = container_config['port'][portmap]['source'] - dport = container_config['port'][portmap]['destination'] - port += f' -p {sport}:{dport}{protocol}' - - # Bind volume - volume = '' - if 'volume' in container_config: - for vol, vol_config in container_config['volume'].items(): - svol = vol_config['source'] - dvol = vol_config['destination'] - volume += f' -v {svol}:{dvol}' - - container_base_cmd = f'podman run --detach --interactive --tty --replace {cap_add} ' \ - f'--memory {memory}m --memory-swap 0 --restart {restart} ' \ - f'--name {name} {device} {port} {volume} {env_opt}' - if 'allow_host_networks' in container_config: - _run_rerun(f'{container_base_cmd} --net host {image}') - else: - for network in container_config['network']: - ipparam = '' - if 'address' in container_config['network'][network]: - address = container_config['network'][network]['address'] - ipparam = f'--ip {address}' + cmd(f'systemctl restart vyos-container-{name}.service') - _run_rerun(f'{container_base_cmd} --net {network} {ipparam} {image}') + if disabled_new: + call('systemctl daemon-reload') return None -- cgit v1.2.3 From 1afb3f8bd5de3748c5b37462eb42235d721d4963 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Fri, 28 Oct 2022 13:07:30 +0000 Subject: T4771: Ability to get raw format for op-mode BGP commands --- data/op-mode-standardized.json | 1 + src/op_mode/bgp.py | 120 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100755 src/op_mode/bgp.py (limited to 'src') diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index 9500d3aa7..c5e9f9243 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -1,4 +1,5 @@ [ +"bgp.py", "bridge.py", "conntrack.py", "container.py", diff --git a/src/op_mode/bgp.py b/src/op_mode/bgp.py new file mode 100755 index 000000000..23001a9d7 --- /dev/null +++ b/src/op_mode/bgp.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Purpose: +# Displays bgp neighbors information. +# Used by the "show bgp (vrf ) ipv4|ipv6 neighbors" commands. + +import re +import sys +import typing + +import jmespath +from jinja2 import Template +from humps import decamelize + +from vyos.configquery import ConfigTreeQuery + +import vyos.opmode + + +frr_command_template = Template(""" +{% if family %} + show bgp + {{ 'vrf ' ~ vrf if vrf else '' }} + {{ 'ipv6' if family == 'inet6' else 'ipv4'}} + {{ 'neighbor ' ~ peer if peer else 'summary' }} +{% endif %} + +{% if raw %} + json +{% endif %} +""") + + +def _verify(func): + """Decorator checks if BGP config exists + BGP configuration can be present under vrf + If we do npt get arg 'peer' then it can be 'bgp summary' + """ + from functools import wraps + + @wraps(func) + def _wrapper(*args, **kwargs): + config = ConfigTreeQuery() + afi = 'ipv6' if kwargs.get('family') == 'inet6' else 'ipv4' + global_vrfs = ['all', 'default'] + peer = kwargs.get('peer') + vrf = kwargs.get('vrf') + unconf_message = f'BGP or neighbor is not configured' + # Add option to check the specific neighbor if we have arg 'peer' + peer_opt = f'neighbor {peer} address-family {afi}-unicast' if peer else '' + vrf_opt = '' + if vrf and vrf not in global_vrfs: + vrf_opt = f'vrf name {vrf}' + # Check if config does not exist + if not config.exists(f'{vrf_opt} protocols bgp {peer_opt}'): + raise vyos.opmode.UnconfiguredSubsystem(unconf_message) + return func(*args, **kwargs) + + return _wrapper + + +@_verify +def show_neighbors(raw: bool, + family: str, + peer: typing.Optional[str], + vrf: typing.Optional[str]): + kwargs = dict(locals()) + frr_command = frr_command_template.render(kwargs) + frr_command = re.sub(r'\s+', ' ', frr_command) + + from vyos.util import cmd + output = cmd(f"vtysh -c '{frr_command}'") + + if raw: + from json import loads + data = loads(output) + # Get list of the peers + peers = jmespath.search('*.peers | [0]', data) + if peers: + # Create new dict, delete old key 'peers' + # add key 'peers' neighbors to the list + list_peers = [] + new_dict = jmespath.search('* | [0]', data) + if 'peers' in new_dict: + new_dict.pop('peers') + + for neighbor, neighbor_options in peers.items(): + neighbor_options['neighbor'] = neighbor + list_peers.append(neighbor_options) + new_dict['peers'] = list_peers + return decamelize(new_dict) + data = jmespath.search('* | [0]', data) + return decamelize(data) + + else: + return output + + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) -- cgit v1.2.3 From 61bc664137d73c6b73e9db5157e1a7a79ab5cebd Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Mon, 31 Oct 2022 08:14:05 -0400 Subject: T4526: use informative error messages for keepalived-fifo with commit in progress --- src/system/keepalived-fifo.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/system/keepalived-fifo.py b/src/system/keepalived-fifo.py index a0fccd1d0..864ee8419 100755 --- a/src/system/keepalived-fifo.py +++ b/src/system/keepalived-fifo.py @@ -67,13 +67,13 @@ class KeepalivedFifo: # For VRRP configuration to be read, the commit must be finished count = 1 while commit_in_progress(): - if ( count <= 40 ): - logger.debug(f'commit in progress try: {count}') + if ( count <= 20 ): + logger.debug(f'Attempt to load keepalived configuration aborted due to a commit in progress (attempt {count}/20)') else: - logger.error(f'commit still in progress after {count} continuing anyway') + logger.error(f'Forced keepalived configuration loading despite a commit in progress ({count} wait time expired, not waiting further)') break count += 1 - time.sleep(0.5) + time.sleep(1) try: base = ['high-availability', 'vrrp'] -- cgit v1.2.3 From 22c3dcbb01d731f0dab0ffefa2e5a0be7009baf1 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Mon, 31 Oct 2022 15:09:58 +0100 Subject: ipsec: T4787: add support for road-warrior/remote-access RADIUS timeout This enabled users to also use 2FA/MFA authentication with a radius backend as there is enough time to enter the second factor. --- data/templates/ipsec/charon/eap-radius.conf.j2 | 4 +++- interface-definitions/include/radius-timeout.xml.i | 16 ++++++++++++++++ interface-definitions/vpn-ipsec.xml.in | 1 + interface-definitions/vpn-openconnect.xml.in | 15 +-------------- src/conf_mode/vpn_ipsec.py | 17 +++++++++++++++-- 5 files changed, 36 insertions(+), 17 deletions(-) create mode 100644 interface-definitions/include/radius-timeout.xml.i (limited to 'src') diff --git a/data/templates/ipsec/charon/eap-radius.conf.j2 b/data/templates/ipsec/charon/eap-radius.conf.j2 index 8495011fe..364377473 100644 --- a/data/templates/ipsec/charon/eap-radius.conf.j2 +++ b/data/templates/ipsec/charon/eap-radius.conf.j2 @@ -49,8 +49,10 @@ eap-radius { # Base to use for calculating exponential back off. # retransmit_base = 1.4 +{% if remote_access.radius.timeout is vyos_defined %} # Timeout in seconds before sending first retransmit. - # retransmit_timeout = 2.0 + retransmit_timeout = {{ remote_access.radius.timeout | float }} +{% endif %} # Number of times to retransmit a packet before giving up. # retransmit_tries = 4 diff --git a/interface-definitions/include/radius-timeout.xml.i b/interface-definitions/include/radius-timeout.xml.i new file mode 100644 index 000000000..22bb6d312 --- /dev/null +++ b/interface-definitions/include/radius-timeout.xml.i @@ -0,0 +1,16 @@ + + + + Session timeout + + u32:1-240 + Session timeout in seconds (default: 2) + + + + + Timeout must be between 1 and 240 seconds + + 2 + + diff --git a/interface-definitions/vpn-ipsec.xml.in b/interface-definitions/vpn-ipsec.xml.in index 4776c53dc..64966b540 100644 --- a/interface-definitions/vpn-ipsec.xml.in +++ b/interface-definitions/vpn-ipsec.xml.in @@ -888,6 +888,7 @@ #include + #include #include diff --git a/interface-definitions/vpn-openconnect.xml.in b/interface-definitions/vpn-openconnect.xml.in index 3b3a83bd4..8b60f2e6e 100644 --- a/interface-definitions/vpn-openconnect.xml.in +++ b/interface-definitions/vpn-openconnect.xml.in @@ -140,20 +140,7 @@ #include - - - Session timeout - - u32:1-240 - Session timeout in seconds (default: 2) - - - - - Timeout must be between 1 and 240 seconds - - 2 - + #include If the groupconfig option is set, then config-per-user will be overriden, and all configuration will be read from RADIUS. diff --git a/src/conf_mode/vpn_ipsec.py b/src/conf_mode/vpn_ipsec.py index 77a425f8b..cfefcfbe8 100755 --- a/src/conf_mode/vpn_ipsec.py +++ b/src/conf_mode/vpn_ipsec.py @@ -117,13 +117,26 @@ def get_config(config=None): ipsec['ike_group'][group]['proposal'][proposal] = dict_merge(default_values, ipsec['ike_group'][group]['proposal'][proposal]) - if 'remote_access' in ipsec and 'connection' in ipsec['remote_access']: + # XXX: T2665: we can not safely rely on the defaults() when there are + # tagNodes in place, it is better to blend in the defaults manually. + if dict_search('remote_access.connection', ipsec): default_values = defaults(base + ['remote-access', 'connection']) for rw in ipsec['remote_access']['connection']: ipsec['remote_access']['connection'][rw] = dict_merge(default_values, ipsec['remote_access']['connection'][rw]) - if 'remote_access' in ipsec and 'radius' in ipsec['remote_access'] and 'server' in ipsec['remote_access']['radius']: + # XXX: T2665: we can not safely rely on the defaults() when there are + # tagNodes in place, it is better to blend in the defaults manually. + if dict_search('remote_access.radius.server', ipsec): + # Fist handle the "base" stuff like RADIUS timeout + default_values = defaults(base + ['remote-access', 'radius']) + if 'server' in default_values: + del default_values['server'] + ipsec['remote_access']['radius'] = dict_merge(default_values, + ipsec['remote_access']['radius']) + + # Take care about individual RADIUS servers implemented as tagNodes - this + # requires special treatment default_values = defaults(base + ['remote-access', 'radius', 'server']) for server in ipsec['remote_access']['radius']['server']: ipsec['remote_access']['radius']['server'][server] = dict_merge(default_values, -- cgit v1.2.3 From f489c5ecdab5bdd8a5faa130f4c79a6f4559353b Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Tue, 1 Nov 2022 17:07:24 +0000 Subject: T4777: Ability to get logs in machine-readable format Ability to get logs in JSON format Possible filter by unit. Options for count lines, UTC time, facility or logs since boot --- data/op-mode-standardized.json | 1 + src/op_mode/log.py | 94 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100755 src/op_mode/log.py (limited to 'src') diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index 9500d3aa7..a34c3f481 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -3,6 +3,7 @@ "conntrack.py", "container.py", "cpu.py", +"log.py", "memory.py", "nat.py", "neighbor.py", diff --git a/src/op_mode/log.py b/src/op_mode/log.py new file mode 100755 index 000000000..b0abd6191 --- /dev/null +++ b/src/op_mode/log.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import json +import re +import sys +import typing + +from jinja2 import Template + +from vyos.util import rc_cmd + +import vyos.opmode + +journalctl_command_template = Template(""" +--no-hostname +--quiet + +{% if boot %} + --boot +{% endif %} + +{% if count %} + --lines={{ count }} +{% endif %} + +{% if reverse %} + --reverse +{% endif %} + +{% if since %} + --since={{ since }} +{% endif %} + +{% if unit %} + --unit={{ unit }} +{% endif %} + +{% if utc %} + --utc +{% endif %} + +{% if raw %} +{# By default show 100 only lines for raw option if count does not set #} +{# Protection from parsing the full log by default #} +{% if not boot %} + --lines={{ '' ~ count if count else '100' }} +{% endif %} + --no-pager + --output=json +{% endif %} +""") + + +def show(raw: bool, + boot: typing.Optional[bool], + count: typing.Optional[int], + facility: typing.Optional[str], + reverse: typing.Optional[bool], + utc: typing.Optional[bool], + unit: typing.Optional[str]): + kwargs = dict(locals()) + + journalctl_options = journalctl_command_template.render(kwargs) + journalctl_options = re.sub(r'\s+', ' ', journalctl_options) + rc, output = rc_cmd(f'journalctl {journalctl_options}') + if raw: + # Each 'journalctl --output json' line is a separate JSON object + # So we should return list of dict + return [json.loads(line) for line in output.split('\n')] + return output + + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) -- cgit v1.2.3 From db0791238c9cccaf73aac61844a6c0293c513a54 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 1 Nov 2022 11:21:05 -0500 Subject: graphql: T4791: decamelize/normalize result of op-mode queries --- src/services/api/graphql/libs/op_mode.py | 6 ++++++ src/services/api/graphql/session/session.py | 3 +++ 2 files changed, 9 insertions(+) (limited to 'src') diff --git a/src/services/api/graphql/libs/op_mode.py b/src/services/api/graphql/libs/op_mode.py index da2bcdb5b..97a26520e 100644 --- a/src/services/api/graphql/libs/op_mode.py +++ b/src/services/api/graphql/libs/op_mode.py @@ -17,8 +17,11 @@ import os import re import typing import importlib.util +from typing import Union +from humps import decamelize from vyos.defaults import directories +from vyos.opmode import _normalize_field_names def load_as_module(name: str, path: str): spec = importlib.util.spec_from_file_location(name, path) @@ -98,3 +101,6 @@ def map_type_name(type_name: type, optional: bool = False) -> str: # scalar 'Generic' is defined in schema.graphql return 'Generic' + +def normalize_output(result: Union[dict, list]) -> Union[dict, list]: + return _normalize_field_names(decamelize(result)) diff --git a/src/services/api/graphql/session/session.py b/src/services/api/graphql/session/session.py index c2c1db1df..0b77b1433 100644 --- a/src/services/api/graphql/session/session.py +++ b/src/services/api/graphql/session/session.py @@ -25,6 +25,7 @@ from vyos.template import render from vyos.opmode import Error as OpModeError from api.graphql.libs.op_mode import load_op_mode_as_module, split_compound_op_mode_name +from api.graphql.libs.op_mode import normalize_output op_mode_include_file = os.path.join(directories['data'], 'op-mode-standardized.json') @@ -149,6 +150,8 @@ class Session: except OpModeError as e: raise e + res = normalize_output(res) + return res def gen_op_mutation(self): -- cgit v1.2.3 From 551cb56cf752776aa4f9d2336c887ea3127b530e Mon Sep 17 00:00:00 2001 From: goodNETnick Date: Sun, 16 Oct 2022 02:37:26 -0400 Subject: login: T4751: 2FA OTP key generator in VyOS CLI --- .../generate-system-login-user.xml.in | 90 ++++++++++++++++++++++ src/op_mode/generate_system_login_user.py | 77 ++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100755 op-mode-definitions/generate-system-login-user.xml.in create mode 100755 src/op_mode/generate_system_login_user.py (limited to 'src') diff --git a/op-mode-definitions/generate-system-login-user.xml.in b/op-mode-definitions/generate-system-login-user.xml.in new file mode 100755 index 000000000..d0519b6bd --- /dev/null +++ b/op-mode-definitions/generate-system-login-user.xml.in @@ -0,0 +1,90 @@ + + + + + + + Generate system related parameters + + + + + Generate system login related parameters + + + + + Username used for authentication + + <username> + + + + + + Generate OpenConnect OTP token + + + + + HOTP time-based token + + sudo ${vyos_op_scripts_dir}/generate_system_login_user.py --username "$5" + + + + Duration of single time interval + + sudo ${vyos_op_scripts_dir}/generate_system_login_user.py --username "$5" --rate_limit "$9" + + + + The number of digits in the one-time password + + sudo ${vyos_op_scripts_dir}/generate_system_login_user.py --username "$5" --rate_limit "$9" --rate_time "${11}" + + + + The number of digits in the one-time password + + sudo ${vyos_op_scripts_dir}/generate_system_login_user.py --username "$5" --rate_limit "$9" --rate_time "${11}" --window_size "${13}" + + + + + + + + The number of digits in the one-time password + + sudo ${vyos_op_scripts_dir}/generate_system_login_user.py --username "$5" --window_size "${9}" + + + + Duration of single time interval + + sudo ${vyos_op_scripts_dir}/generate_system_login_user.py --username "$5" --rate_limit "${11}" --window_size "${9}" + + + + Duration of single time interval + + sudo ${vyos_op_scripts_dir}/generate_system_login_user.py --username "$5" --rate_limit "${11}" --rate_time "${13}" --window_size "${9}" + + + + + + + + + + + + + + + + + + diff --git a/src/op_mode/generate_system_login_user.py b/src/op_mode/generate_system_login_user.py new file mode 100755 index 000000000..8f8827b1b --- /dev/null +++ b/src/op_mode/generate_system_login_user.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import argparse +import os + +from vyos.util import popen +from secrets import token_hex +from base64 import b32encode + +if os.geteuid() != 0: + exit("You need to have root privileges to run this script.\nPlease try again, this time using 'sudo'. Exiting.") + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("-u", "--username", type=str, help='Username used for authentication', required=True) + parser.add_argument("-l", "--rate_limit", type=str, help='Limit number of logins (rate-limit) per rate-time (default: 3)', default="3", required=False) + parser.add_argument("-t", "--rate_time", type=str, help='Limit number of logins (rate-limit) per rate-time (default: 30)', default="30", required=False) + parser.add_argument("-w", "--window_size", type=str, help='Set window of concurrently valid codes (default: 3)', default="3", required=False) + parser.add_argument("-i", "--interval", type=str, help='Duration of single time interval', default="30", required=False) + parser.add_argument("-d", "--digits", type=str, help='The number of digits in the one-time password', default="6", required=False) + args = parser.parse_args() + + hostname = os.uname()[1] + username = args.username + rate_limit = args.rate_limit + rate_time = args.rate_time + window_size = args.window_size + digits = args.digits + period = args.interval + + # check variables: + if int(rate_limit) < 1 or int(rate_limit) > 10: + print("") + quit("Number of logins (rate-limit) must be between '1' and '10'") + + if int(rate_time) < 15 or int(rate_time) > 600: + print("") + quit("The rate-time must be between '15' and '600' seconds") + + if int(window_size) < 1 or int(window_size) > 21: + print("") + quit("Window of concurrently valid codes must be between '1' and '21' seconds") + + # generate OTP key, URL & QR: + key_hex = token_hex(20) + key_base32 = b32encode(bytes.fromhex(key_hex)).decode() + + otp_url=''.join(["otpauth://totp/",username,"@",hostname,"?secret=",key_base32,"&digits=",digits,"&period=",period]) + qrcode,err = popen('qrencode -t ansiutf8', input=otp_url) + + print("# You can share it with the user, he just needs to scan the QR in his OTP app") + print("# username: ", username) + print("# OTP KEY: ", key_base32) + print("# OTP URL: ", otp_url) + print(qrcode) + print('# To add this OTP key to configuration, run the following commands:') + print(f"set system login user {username} authentication otp key '{key_base32}'") + if rate_limit != "3": + print(f"set system login user {username} authentication otp rate-limit '{rate_limit}'") + if rate_time != "30": + print(f"set system login user {username} authentication otp rate-time '{rate_time}'") + if window_size != "3": + print(f"set system login user {username} authentication otp window-size '{window_size}'") -- cgit v1.2.3 From 738641a6c66d22c09b8c028ee3d8a90527d9701f Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Wed, 19 Oct 2022 14:16:05 +0000 Subject: T4758: Rewrite show DHCP(v6) server leases to vyos.opmode format Rewrite op-mode DHCP and DHCPv6 leases to vyos.opmode format Abbility to show 'raw' format show dhcp server leases show dhcpv6 server leases --- data/op-mode-standardized.json | 2 + op-mode-definitions/dhcp.xml.in | 4 +- src/op_mode/dhcp.py | 278 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 282 insertions(+), 2 deletions(-) create mode 100755 src/op_mode/dhcp.py (limited to 'src') diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index 9500d3aa7..03b85b50f 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -3,6 +3,8 @@ "conntrack.py", "container.py", "cpu.py", +"dhcp.py", +"log.py", "memory.py", "nat.py", "neighbor.py", diff --git a/op-mode-definitions/dhcp.xml.in b/op-mode-definitions/dhcp.xml.in index 241cca0ce..ce4026ff4 100644 --- a/op-mode-definitions/dhcp.xml.in +++ b/op-mode-definitions/dhcp.xml.in @@ -16,7 +16,7 @@ Show DHCP server leases - sudo ${vyos_op_scripts_dir}/show_dhcp.py --leases + sudo ${vyos_op_scripts_dir}/dhcp.py show_server_leases --family inet @@ -82,7 +82,7 @@ Show DHCPv6 server leases - sudo ${vyos_op_scripts_dir}/show_dhcpv6.py --leases + sudo ${vyos_op_scripts_dir}/dhcp.py show_server_leases --family inet6 diff --git a/src/op_mode/dhcp.py b/src/op_mode/dhcp.py new file mode 100755 index 000000000..07e9b7d6c --- /dev/null +++ b/src/op_mode/dhcp.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys +from ipaddress import ip_address +import typing + +from datetime import datetime +from sys import exit +from tabulate import tabulate +from isc_dhcp_leases import IscDhcpLeases + +from vyos.base import Warning +from vyos.configquery import ConfigTreeQuery + +from vyos.util import cmd +from vyos.util import dict_search +from vyos.util import is_systemd_service_running + +import vyos.opmode + + +config = ConfigTreeQuery() +pool_key = "shared-networkname" + + +def _in_pool(lease, pool): + if pool_key in lease.sets: + if lease.sets[pool_key] == pool: + return True + return False + + +def _utc_to_local(utc_dt): + return datetime.fromtimestamp((datetime.fromtimestamp(utc_dt) - datetime(1970, 1, 1)).total_seconds()) + + +def _format_hex_string(in_str): + out_str = "" + # if input is divisible by 2, add : every 2 chars + if len(in_str) > 0 and len(in_str) % 2 == 0: + out_str = ':'.join(a+b for a,b in zip(in_str[::2], in_str[1::2])) + else: + out_str = in_str + + return out_str + + +def _find_list_of_dict_index(lst, key='ip', value='') -> int: + """ + Find the index entry of list of dict matching the dict value + Exampe: + % lst = [{'ip': '192.0.2.1'}, {'ip': '192.0.2.2'}] + % _find_list_of_dict_index(lst, key='ip', value='192.0.2.2') + % 1 + """ + idx = next((index for (index, d) in enumerate(lst) if d[key] == value), None) + return idx + + +def _get_raw_server_leases(family, pool=None) -> list: + """ + Get DHCP server leases + :return list + """ + lease_file = '/config/dhcpdv6.leases' if family == 'inet6' else '/config/dhcpd.leases' + data = [] + leases = IscDhcpLeases(lease_file).get() + if pool is not None: + if config.exists(f'service dhcp-server shared-network-name {pool}'): + leases = list(filter(lambda x: _in_pool(x, pool), leases)) + for lease in leases: + data_lease = {} + data_lease['ip'] = lease.ip + data_lease['state'] = lease.binding_state + data_lease['pool'] = lease.sets.get('shared-networkname', '') + data_lease['end'] = lease.end.timestamp() + + if family == 'inet': + data_lease['hardware'] = lease.ethernet + data_lease['start'] = lease.start.timestamp() + data_lease['hostname'] = lease.hostname + + if family == 'inet6': + data_lease['last_communication'] = lease.last_communication.timestamp() + data_lease['iaid_duid'] = _format_hex_string(lease.host_identifier_string) + lease_types_long = {'na': 'non-temporary', 'ta': 'temporary', 'pd': 'prefix delegation'} + data_lease['type'] = lease_types_long[lease.type] + + data_lease['remaining'] = lease.end - datetime.utcnow() + + if data_lease['remaining'].days >= 0: + # substraction gives us a timedelta object which can't be formatted with strftime + # so we use str(), split gets rid of the microseconds + data_lease['remaining'] = str(data_lease["remaining"]).split('.')[0] + else: + data_lease['remaining'] = '' + + # Do not add old leases + if data_lease['remaining'] != '': + data.append(data_lease) + + # deduplicate + checked = [] + for entry in data: + addr = entry.get('ip') + if addr not in checked: + checked.append(addr) + else: + idx = _find_list_of_dict_index(data, key='ip', value=addr) + data.pop(idx) + + return data + + +def _get_formatted_server_leases(raw_data, family): + data_entries = [] + if family == 'inet': + for lease in raw_data: + ipaddr = lease.get('ip') + hw_addr = lease.get('hardware') + state = lease.get('state') + start = lease.get('start') + start = _utc_to_local(start).strftime('%Y/%m/%d %H:%M:%S') + end = lease.get('end') + end = _utc_to_local(end).strftime('%Y/%m/%d %H:%M:%S') + remain = lease.get('remaining') + pool = lease.get('pool') + hostname = lease.get('hostname') + data_entries.append([ipaddr, hw_addr, state, start, end, remain, pool, hostname]) + + headers = ['IP Address', 'Hardware address', 'State', 'Lease start', 'Lease expiration', 'Remaining', 'Pool', + 'Hostname'] + + if family == 'inet6': + for lease in raw_data: + ipaddr = lease.get('ip') + state = lease.get('state') + start = lease.get('last_communication') + start = _utc_to_local(start).strftime('%Y/%m/%d %H:%M:%S') + end = lease.get('end') + end = _utc_to_local(end).strftime('%Y/%m/%d %H:%M:%S') + remain = lease.get('remaining') + lease_type = lease.get('type') + pool = lease.get('pool') + host_identifier = lease.get('iaid_duid') + data_entries.append([ipaddr, state, start, end, remain, lease_type, pool, host_identifier]) + + headers = ['IPv6 address', 'State', 'Last communication', 'Lease expiration', 'Remaining', 'Type', 'Pool', + 'IAID_DUID'] + + output = tabulate(data_entries, headers, numalign='left') + return output + + +def _get_dhcp_pools(family='inet') -> list: + v = 'v6' if family == 'inet6' else '' + pools = config.list_nodes(f'service dhcp{v}-server shared-network-name') + return pools + + +def _get_pool_size(pool, family='inet'): + v = 'v6' if family == 'inet6' else '' + base = f'service dhcp{v}-server shared-network-name {pool}' + size = 0 + subnets = config.list_nodes(f'{base} subnet') + for subnet in subnets: + if family == 'inet6': + ranges = config.list_nodes(f'{base} subnet {subnet} address-range start') + else: + ranges = config.list_nodes(f'{base} subnet {subnet} range') + for range in ranges: + if family == 'inet6': + start = config.list_nodes(f'{base} subnet {subnet} address-range start')[0] + stop = config.value(f'{base} subnet {subnet} address-range start {start} stop') + else: + start = config.value(f'{base} subnet {subnet} range {range} start') + stop = config.value(f'{base} subnet {subnet} range {range} stop') + # Add +1 because both range boundaries are inclusive + size += int(ip_address(stop)) - int(ip_address(start)) + 1 + return size + + +def _get_raw_pool_statistics(family='inet', pool=None): + if pool is None: + pool = _get_dhcp_pools(family=family) + else: + pool = [pool] + + v = 'v6' if family == 'inet6' else '' + stats = [] + for p in pool: + subnet = config.list_nodes(f'service dhcp{v}-server shared-network-name {p} subnet') + size = _get_pool_size(family=family, pool=p) + leases = len(_get_raw_server_leases(family=family, pool=p)) + use_percentage = round(leases / size * 100) if size != 0 else 0 + pool_stats = {'pool': p, 'size': size, 'leases': leases, + 'available': (size - leases), 'use_percentage': use_percentage, 'subnet': subnet} + stats.append(pool_stats) + return stats + + +def _get_formatted_pool_statistics(pool_data, family='inet'): + data_entries = [] + for entry in pool_data: + pool = entry.get('pool') + size = entry.get('size') + leases = entry.get('leases') + available = entry.get('available') + use_percentage = entry.get('use_percentage') + use_percentage = f'{use_percentage}%' + data_entries.append([pool, size, leases, available, use_percentage]) + + headers = ['Pool', 'Size','Leases', 'Available', 'Usage'] + output = tabulate(data_entries, headers, numalign='left') + return output + + +def _verify(func): + """Decorator checks if DHCP(v6) config exists""" + from functools import wraps + + @wraps(func) + def _wrapper(*args, **kwargs): + config = ConfigTreeQuery() + family = kwargs.get('family') + v = 'v6' if family == 'inet6' else '' + unconf_message = f'DHCP{v} server is not configured' + # Check if config does not exist + if not config.exists(f'service dhcp{v}-server'): + raise vyos.opmode.UnconfiguredSubsystem(unconf_message) + return func(*args, **kwargs) + return _wrapper + + +@_verify +def show_pool_statistics(raw: bool, family: str, pool: typing.Optional[str]): + pool_data = _get_raw_pool_statistics(family=family, pool=pool) + if raw: + return pool_data + else: + return _get_formatted_pool_statistics(pool_data, family=family) + + +@_verify +def show_server_leases(raw: bool, family: str): + # if dhcp server is down, inactive leases may still be shown as active, so warn the user. + if not is_systemd_service_running('isc-dhcp-server.service'): + Warning('DHCP server is configured but not started. Data may be stale.') + + leases = _get_raw_server_leases(family) + if raw: + return leases + else: + return _get_formatted_server_leases(leases, family) + + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) -- cgit v1.2.3 From 2e83c1eb53e9378873eb2192f85e717df84b9cd0 Mon Sep 17 00:00:00 2001 From: aapostoliuk Date: Tue, 18 Oct 2022 15:27:27 +0300 Subject: T4496: Added lists of values in the help of op-mode ping command Added list of possible VRFs in the help of the ping command Added list of possible interfaces in the help of the ping command Changed, if an option was selected before in the ping command, it does not appear in possible completion. Added error message when an unexpected option was selected. --- src/op_mode/ping.py | 90 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 73 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/op_mode/ping.py b/src/op_mode/ping.py index 60bbc0c78..5e5b95a0a 100755 --- a/src/op_mode/ping.py +++ b/src/op_mode/ping.py @@ -18,6 +18,32 @@ import os import sys import socket import ipaddress +import json +from vyos.util import cmd, rc_cmd +from vyos.ifconfig import Section + + +def interface_list() -> list: + """ + Get list of interfaces in system + :rtype: list + """ + return Section.interfaces() + + +def vrf_list() -> list: + """ + Get list of VRFs in system + :rtype: list + """ + result = cmd(f'sudo ip --json --brief link show type vrf') + data = json.loads(result) + vrflist: list = [] + for o in data: + if 'ifname' in o: + vrflist.append(o['ifname']) + return vrflist + options = { 'audible': { @@ -63,6 +89,7 @@ options = { 'interface': { 'ping': '{command} -I {value}', 'type': '', + 'helpfunction': interface_list, 'help': 'Source interface' }, 'interval': { @@ -128,6 +155,7 @@ options = { 'ping': 'sudo ip vrf exec {value} {command}', 'type': '', 'help': 'Use specified VRF table', + 'helpfunction': vrf_list, 'dflt': 'default', }, 'verbose': { @@ -142,20 +170,33 @@ ping = { } -class List (list): - def first (self): +class List(list): + def first(self): return self.pop(0) if self else '' def last(self): return self.pop() if self else '' - def prepend(self,value): - self.insert(0,value) + def prepend(self, value): + self.insert(0, value) + + +def completion_failure(option: str) -> None: + """ + Shows failure message after TAB when option is wrong + :param option: failure option + :type str: + """ + sys.stderr.write('\n\n Invalid option: {}\n\n'.format(option)) + sys.stdout.write('') + sys.exit(1) def expension_failure(option, completions): reason = 'Ambiguous' if completions else 'Invalid' - sys.stderr.write('\n\n {} command: {} [{}]\n\n'.format(reason,' '.join(sys.argv), option)) + sys.stderr.write( + '\n\n {} command: {} [{}]\n\n'.format(reason, ' '.join(sys.argv), + option)) if completions: sys.stderr.write(' Possible completions:\n ') sys.stderr.write('\n '.join(completions)) @@ -196,28 +237,44 @@ if __name__ == '__main__': if host == '--get-options': args.first() # pop ping args.first() # pop IP + usedoptionslist = [] while args: - option = args.first() - - matched = complete(option) + option = args.first() # pop option + matched = complete(option) # get option parameters + usedoptionslist.append(option) # list of used options + # Select options if not args: + # remove from Possible completions used options + for o in usedoptionslist: + if o in matched: + matched.remove(o) sys.stdout.write(' '.join(matched)) sys.exit(0) - if len(matched) > 1 : + if len(matched) > 1: sys.stdout.write(' '.join(matched)) sys.exit(0) + # If option doesn't have value + if matched: + if options[matched[0]]['type'] == 'noarg': + continue + else: + # Unexpected option + completion_failure(option) - if options[matched[0]]['type'] == 'noarg': - continue - - value = args.first() + value = args.first() # pop option's value if not args: matched = complete(option) - sys.stdout.write(options[matched[0]]['type']) + helplines = options[matched[0]]['type'] + # Run helpfunction to get list of possible values + if 'helpfunction' in options[matched[0]]: + result = options[matched[0]]['helpfunction']() + if result: + helplines = '\n' + ' '.join(result) + sys.stdout.write(helplines) sys.exit(0) - for name,option in options.items(): + for name, option in options.items(): if 'dflt' in option and name not in args: args.append(name) args.append(option['dflt']) @@ -234,8 +291,7 @@ if __name__ == '__main__': except ValueError: sys.exit(f'ping: Unknown host: {host}') - command = convert(ping[version],args) + command = convert(ping[version], args) # print(f'{command} {host}') os.system(f'{command} {host}') - -- cgit v1.2.3 From 6f37744ad45a5f4cd585fa7c981833a9ebe5e437 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Thu, 3 Nov 2022 17:40:12 +0100 Subject: xml: T4795: superseed allowed-vlan validator by numeric range validator Reduce CPU time when spawning the python interpreter. Same can be done by the numeric validator. --- interface-definitions/interfaces-bridge.xml.in | 2 +- src/validators/allowed-vlan | 19 ------------------- 2 files changed, 1 insertion(+), 20 deletions(-) delete mode 100755 src/validators/allowed-vlan (limited to 'src') diff --git a/interface-definitions/interfaces-bridge.xml.in b/interface-definitions/interfaces-bridge.xml.in index 1e11cd4c6..d633077d9 100644 --- a/interface-definitions/interfaces-bridge.xml.in +++ b/interface-definitions/interfaces-bridge.xml.in @@ -151,7 +151,7 @@ VLAN id range allowed on this interface (use '-' as delimiter) - + not a valid VLAN ID value or range diff --git a/src/validators/allowed-vlan b/src/validators/allowed-vlan deleted file mode 100755 index 11389390b..000000000 --- a/src/validators/allowed-vlan +++ /dev/null @@ -1,19 +0,0 @@ -#! /usr/bin/python3 - -import sys -import re - -if __name__ == '__main__': - if len(sys.argv)>1: - allowed_vlan = sys.argv[1] - if re.search('[0-9]{1,4}-[0-9]{1,4}', allowed_vlan): - for tmp in allowed_vlan.split('-'): - if int(tmp) not in range(1, 4095): - sys.exit(1) - else: - if int(allowed_vlan) not in range(1, 4095): - sys.exit(1) - else: - sys.exit(2) - - sys.exit(0) -- cgit v1.2.3 From 81a70033cc9552e7bcc9f6aa6cc092b4e64b6b7c Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Thu, 3 Nov 2022 17:41:53 +0100 Subject: validators: T4795: drop unused Python validators --- src/validators/dotted-decimal | 33 --------------------------------- src/validators/tcp-flag | 17 ----------------- 2 files changed, 50 deletions(-) delete mode 100755 src/validators/dotted-decimal delete mode 100755 src/validators/tcp-flag (limited to 'src') diff --git a/src/validators/dotted-decimal b/src/validators/dotted-decimal deleted file mode 100755 index 652110346..000000000 --- a/src/validators/dotted-decimal +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2020 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import re -import sys - -area = sys.argv[1] - -res = re.match(r'^(\d+)\.(\d+)\.(\d+)\.(\d+)$', area) -if not res: - print("\'{0}\' is not a valid dotted decimal value".format(area)) - sys.exit(1) -else: - components = res.groups() - for n in range(0, 4): - if (int(components[n]) > 255): - print("Invalid component of a dotted decimal value: {0} exceeds 255".format(components[n])) - sys.exit(1) - -sys.exit(0) diff --git a/src/validators/tcp-flag b/src/validators/tcp-flag deleted file mode 100755 index 1496b904a..000000000 --- a/src/validators/tcp-flag +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/python3 - -import sys -import re - -if __name__ == '__main__': - if len(sys.argv)>1: - flag = sys.argv[1] - if flag and flag[0] == '!': - flag = flag[1:] - if flag not in ['syn', 'ack', 'rst', 'fin', 'urg', 'psh', 'ecn', 'cwr']: - print(f'Error: {flag} is not a valid TCP flag') - sys.exit(1) - else: - sys.exit(2) - - sys.exit(0) -- cgit v1.2.3 From 3f5464d0ee857d204dc58867065380340008f79b Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Thu, 3 Nov 2022 17:47:55 +0100 Subject: validators: T4795: migrate mac-address python validator to validate-value Instead of spawning the Python interpreter for every mac-address to validate, rather use the base validate-value OCaml implementation which is much faster. This removes redundant code and also makes the CLI more responsive. Validator is moved out to a dedicated file instead of using XML inlined for the reason of re-usability. So if that regex needs to be touched again - it can all happen in one single file. --- .../include/firewall/mac-address.xml.i | 5 ++-- src/validators/mac-address | 29 ++-------------------- src/validators/mac-address-exclude | 2 ++ src/validators/mac-address-firewall | 27 -------------------- 4 files changed, 7 insertions(+), 56 deletions(-) create mode 100755 src/validators/mac-address-exclude delete mode 100755 src/validators/mac-address-firewall (limited to 'src') diff --git a/interface-definitions/include/firewall/mac-address.xml.i b/interface-definitions/include/firewall/mac-address.xml.i index 83aaf1ce1..db3e1e312 100644 --- a/interface-definitions/include/firewall/mac-address.xml.i +++ b/interface-definitions/include/firewall/mac-address.xml.i @@ -3,7 +3,7 @@ MAC address - macaddr; + macaddr MAC address to match @@ -11,7 +11,8 @@ Match everything except the specified MAC address - + + diff --git a/src/validators/mac-address b/src/validators/mac-address index 7d020f387..bb859a603 100755 --- a/src/validators/mac-address +++ b/src/validators/mac-address @@ -1,27 +1,2 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2018-2020 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import re -import sys - -pattern = "^([0-9A-Fa-f]{2}:){5}([0-9A-Fa-f]{2})$" - -if __name__ == '__main__': - if len(sys.argv) != 2: - sys.exit(1) - if not re.match(pattern, sys.argv[1]): - sys.exit(1) - sys.exit(0) +#!/usr/bin/env sh +${vyos_libexec_dir}/validate-value --regex "([0-9A-Fa-f]{2}:){5}([0-9A-Fa-f]{2})" --value "$1" diff --git a/src/validators/mac-address-exclude b/src/validators/mac-address-exclude new file mode 100755 index 000000000..c44913023 --- /dev/null +++ b/src/validators/mac-address-exclude @@ -0,0 +1,2 @@ +#!/usr/bin/env sh +${vyos_libexec_dir}/validate-value --regex "!([0-9A-Fa-f]{2}:){5}([0-9A-Fa-f]{2})" --value "$1" diff --git a/src/validators/mac-address-firewall b/src/validators/mac-address-firewall deleted file mode 100755 index 70551f86d..000000000 --- a/src/validators/mac-address-firewall +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2018-2022 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import re -import sys - -pattern = "^!?([0-9A-Fa-f]{2}:){5}([0-9A-Fa-f]{2})$" - -if __name__ == '__main__': - if len(sys.argv) != 2: - sys.exit(1) - if not re.match(pattern, sys.argv[1]): - sys.exit(1) - sys.exit(0) -- cgit v1.2.3 From d4cb20e1cef23ba3adec543536ef5dfdf143d392 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Thu, 3 Nov 2022 20:58:56 +0100 Subject: validators: T4795: migrate fqdn python validator to validate-value --- src/validators/fqdn | 29 ++--------------------------- 1 file changed, 2 insertions(+), 27 deletions(-) (limited to 'src') diff --git a/src/validators/fqdn b/src/validators/fqdn index a4027e4ca..a65d2d5d4 100755 --- a/src/validators/fqdn +++ b/src/validators/fqdn @@ -1,27 +1,2 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2020-2021 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import re -import sys - -pattern = '[A-Za-z0-9][-.A-Za-z0-9]*' - -if __name__ == '__main__': - if len(sys.argv) != 2: - sys.exit(1) - if not re.match(pattern, sys.argv[1]): - sys.exit(1) - sys.exit(0) +#!/usr/bin/env sh +${vyos_libexec_dir}/validate-value --regex "[A-Za-z0-9][-.A-Za-z0-9]*" --value "$1" -- cgit v1.2.3 From 051e063fdf2e459a0716a35778b33ea6bb2fdcb6 Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Mon, 31 Oct 2022 14:26:51 +0100 Subject: firewall: T970: Refactor domain resolver, add firewall source/destination `fqdn` node --- data/templates/firewall/nftables-defines.j2 | 8 + data/templates/firewall/nftables.j2 | 14 +- interface-definitions/firewall.xml.in | 25 ++- interface-definitions/include/firewall/fqdn.xml.i | 14 ++ .../firewall/source-destination-group-ipv6.xml.i | 8 + python/vyos/firewall.py | 90 ++++------ smoketest/scripts/cli/test_firewall.py | 16 ++ src/conf_mode/firewall.py | 60 +++---- src/helpers/vyos-domain-group-resolve.py | 60 ------- src/helpers/vyos-domain-resolver.py | 182 +++++++++++++++++++++ src/systemd/vyos-domain-group-resolve.service | 11 -- src/systemd/vyos-domain-resolver.service | 13 ++ 12 files changed, 328 insertions(+), 173 deletions(-) create mode 100644 interface-definitions/include/firewall/fqdn.xml.i delete mode 100755 src/helpers/vyos-domain-group-resolve.py create mode 100755 src/helpers/vyos-domain-resolver.py delete mode 100644 src/systemd/vyos-domain-group-resolve.service create mode 100644 src/systemd/vyos-domain-resolver.service (limited to 'src') diff --git a/data/templates/firewall/nftables-defines.j2 b/data/templates/firewall/nftables-defines.j2 index 5336f7ee6..dd06dee28 100644 --- a/data/templates/firewall/nftables-defines.j2 +++ b/data/templates/firewall/nftables-defines.j2 @@ -27,6 +27,14 @@ } {% endfor %} {% endif %} +{% if group.domain_group is vyos_defined %} +{% for name, name_config in group.domain_group.items() %} + set D_{{ name }} { + type {{ ip_type }} + flags interval + } +{% endfor %} +{% endif %} {% if group.mac_group is vyos_defined %} {% for group_name, group_conf in group.mac_group.items() %} {% set includes = group_conf.include if group_conf.include is vyos_defined else [] %} diff --git a/data/templates/firewall/nftables.j2 b/data/templates/firewall/nftables.j2 index a0f0b8c11..2c7115134 100644 --- a/data/templates/firewall/nftables.j2 +++ b/data/templates/firewall/nftables.j2 @@ -67,14 +67,12 @@ table ip vyos_filter { {{ conf | nft_default_rule(name_text) }} } {% endfor %} -{% if group is vyos_defined and group.domain_group is vyos_defined %} -{% for name, name_config in group.domain_group.items() %} - set D_{{ name }} { +{% for set_name in ip_fqdn %} + set FQDN_{{ set_name }} { type ipv4_addr flags interval } -{% endfor %} -{% endif %} +{% endfor %} {% for set_name in ns.sets %} set RECENT_{{ set_name }} { type ipv4_addr @@ -178,6 +176,12 @@ table ip6 vyos_filter { {{ conf | nft_default_rule(name_text, ipv6=True) }} } {% endfor %} +{% for set_name in ip6_fqdn %} + set FQDN_{{ set_name }} { + type ipv6_addr + flags interval + } +{% endfor %} {% for set_name in ns.sets %} set RECENT6_{{ set_name }} { type ipv6_addr diff --git a/interface-definitions/firewall.xml.in b/interface-definitions/firewall.xml.in index 673461036..2d8f17351 100644 --- a/interface-definitions/firewall.xml.in +++ b/interface-definitions/firewall.xml.in @@ -126,7 +126,7 @@ Domain address to match - [a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,99}?(\/.*)? + @@ -408,6 +408,7 @@ #include + #include #include #include #include @@ -419,6 +420,7 @@ #include + #include #include #include #include @@ -572,6 +574,7 @@ #include + #include #include #include #include @@ -583,6 +586,7 @@ #include + #include #include #include #include @@ -656,6 +660,25 @@ disable + + + Retains last successful value if domain resolution fails + + + + + + Domain resolver update interval + + u32:10-3600 + Interval (seconds) + + + + + + 300 + Policy for sending IPv4 ICMP redirect messages diff --git a/interface-definitions/include/firewall/fqdn.xml.i b/interface-definitions/include/firewall/fqdn.xml.i new file mode 100644 index 000000000..9eb3925b5 --- /dev/null +++ b/interface-definitions/include/firewall/fqdn.xml.i @@ -0,0 +1,14 @@ + + + + Fully qualified domain name + + <fqdn> + Fully qualified domain name + + + + + + + diff --git a/interface-definitions/include/firewall/source-destination-group-ipv6.xml.i b/interface-definitions/include/firewall/source-destination-group-ipv6.xml.i index c2cc7edb3..2a42d236c 100644 --- a/interface-definitions/include/firewall/source-destination-group-ipv6.xml.i +++ b/interface-definitions/include/firewall/source-destination-group-ipv6.xml.i @@ -12,6 +12,14 @@ + + + Group of domains + + firewall group domain-group + + + #include diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index 4075e55b0..db4878c9d 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -20,6 +20,9 @@ import os import re from pathlib import Path +from socket import AF_INET +from socket import AF_INET6 +from socket import getaddrinfo from time import strftime from vyos.remote import download @@ -31,65 +34,29 @@ from vyos.util import dict_search_args from vyos.util import dict_search_recursive from vyos.util import run +def fqdn_config_parse(firewall): + firewall['ip_fqdn'] = {} + firewall['ip6_fqdn'] = {} + + for domain, path in dict_search_recursive(firewall, 'fqdn'): + fw_name = path[1] # name/ipv6-name + rule = path[3] # rule id + suffix = path[4][0] # source/destination (1 char) + set_name = f'{fw_name}_{rule}_{suffix}' + + if path[0] == 'name': + firewall['ip_fqdn'][set_name] = domain + elif path[0] == 'ipv6_name': + firewall['ip6_fqdn'][set_name] = domain + +def fqdn_resolve(fqdn, ipv6=False): + try: + res = getaddrinfo(fqdn, None, AF_INET6 if ipv6 else AF_INET) + return set(item[4][0] for item in res) + except: + return None -# Functions for firewall group domain-groups -def get_ips_domains_dict(list_domains): - """ - Get list of IPv4 addresses by list of domains - Ex: get_ips_domains_dict(['ex1.com', 'ex2.com']) - {'ex1.com': ['192.0.2.1'], 'ex2.com': ['192.0.2.2', '192.0.2.3']} - """ - from socket import gethostbyname_ex - from socket import gaierror - - ip_dict = {} - for domain in list_domains: - try: - _, _, ips = gethostbyname_ex(domain) - ip_dict[domain] = ips - except gaierror: - pass - - return ip_dict - -def nft_init_set(group_name, table="vyos_filter", family="ip"): - """ - table ip vyos_filter { - set GROUP_NAME - type ipv4_addr - flags interval - } - """ - return call(f'nft add set ip {table} {group_name} {{ type ipv4_addr\\; flags interval\\; }}') - - -def nft_add_set_elements(group_name, elements, table="vyos_filter", family="ip"): - """ - table ip vyos_filter { - set GROUP_NAME { - type ipv4_addr - flags interval - elements = { 192.0.2.1, 192.0.2.2 } - } - """ - elements = ", ".join(elements) - return call(f'nft add element {family} {table} {group_name} {{ {elements} }} ') - -def nft_flush_set(group_name, table="vyos_filter", family="ip"): - """ - Flush elements of nft set - """ - return call(f'nft flush set {family} {table} {group_name}') - -def nft_update_set_elements(group_name, elements, table="vyos_filter", family="ip"): - """ - Update elements of nft set - """ - flush_set = nft_flush_set(group_name, table="vyos_filter", family="ip") - nft_add_set = nft_add_set_elements(group_name, elements, table="vyos_filter", family="ip") - return flush_set, nft_add_set - -# END firewall group domain-group (sets) +# End Domain Resolver def find_nftables_rule(table, chain, rule_matches=[]): # Find rule in table/chain that matches all criteria and return the handle @@ -151,6 +118,13 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name): suffix = f'!= {suffix[1:]}' output.append(f'{ip_name} {prefix}addr {suffix}') + if 'fqdn' in side_conf: + fqdn = side_conf['fqdn'] + operator = '' + if fqdn[0] == '!': + operator = '!=' + output.append(f'{ip_name} {prefix}addr {operator} @FQDN_{fw_name}_{rule_id}_{prefix}') + if dict_search_args(side_conf, 'geoip', 'country_code'): operator = '' if dict_search_args(side_conf, 'geoip', 'inverse_match') != None: diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py index 821925bcd..e172e086d 100755 --- a/smoketest/scripts/cli/test_firewall.py +++ b/smoketest/scripts/cli/test_firewall.py @@ -17,11 +17,13 @@ import unittest from glob import glob +from time import sleep from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.util import cmd +from vyos.util import run sysfs_config = { 'all_ping': {'sysfs': '/proc/sys/net/ipv4/icmp_echo_ignore_all', 'default': '0', 'test_value': 'disable'}, @@ -76,6 +78,17 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase): break self.assertTrue(not matched if inverse else matched, msg=search) + def wait_for_domain_resolver(self, table, set_name, element, max_wait=10): + # Resolver no longer blocks commit, need to wait for daemon to populate set + count = 0 + while count < max_wait: + code = run(f'sudo nft get element {table} {set_name} {{ {element} }}') + if code == 0: + return True + count += 1 + sleep(1) + return False + def test_geoip(self): self.cli_set(['firewall', 'name', 'smoketest', 'rule', '1', 'action', 'drop']) self.cli_set(['firewall', 'name', 'smoketest', 'rule', '1', 'source', 'geoip', 'country-code', 'se']) @@ -125,6 +138,9 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase): self.cli_set(['firewall', 'interface', 'eth0', 'in', 'name', 'smoketest']) self.cli_commit() + + self.wait_for_domain_resolver('ip vyos_filter', 'D_smoketest_domain', '192.0.2.5') + nftables_search = [ ['iifname "eth0"', 'jump NAME_smoketest'], ['ip saddr @N_smoketest_network', 'ip daddr 172.16.10.10', 'th dport @P_smoketest_port', 'return'], diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index cbd9cbe90..2bb765e65 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -27,12 +27,8 @@ from vyos.configdict import dict_merge from vyos.configdict import node_changed from vyos.configdiff import get_config_diff, Diff # from vyos.configverify import verify_interface_exists +from vyos.firewall import fqdn_config_parse from vyos.firewall import geoip_update -from vyos.firewall import get_ips_domains_dict -from vyos.firewall import nft_add_set_elements -from vyos.firewall import nft_flush_set -from vyos.firewall import nft_init_set -from vyos.firewall import nft_update_set_elements from vyos.template import render from vyos.util import call from vyos.util import cmd @@ -173,6 +169,8 @@ def get_config(config=None): firewall['geoip_updated'] = geoip_updated(conf, firewall) + fqdn_config_parse(firewall) + return firewall def verify_rule(firewall, rule_conf, ipv6): @@ -232,29 +230,28 @@ def verify_rule(firewall, rule_conf, ipv6): if side in rule_conf: side_conf = rule_conf[side] - if dict_search_args(side_conf, 'geoip', 'country_code'): - if 'address' in side_conf: - raise ConfigError('Address and GeoIP cannot both be defined') - - if dict_search_args(side_conf, 'group', 'address_group'): - raise ConfigError('Address-group and GeoIP cannot both be defined') - - if dict_search_args(side_conf, 'group', 'network_group'): - raise ConfigError('Network-group and GeoIP cannot both be defined') + if len({'address', 'fqdn', 'geoip'} & set(side_conf)) > 1: + raise ConfigError('Only one of address, fqdn or geoip can be specified') if 'group' in side_conf: - if {'address_group', 'network_group'} <= set(side_conf['group']): - raise ConfigError('Only one address-group or network-group can be specified') + if len({'address_group', 'network_group', 'domain_group'} & set(side_conf['group'])) > 1: + raise ConfigError('Only one address-group, network-group or domain-group can be specified') for group in valid_groups: if group in side_conf['group']: group_name = side_conf['group'][group] + fw_group = f'ipv6_{group}' if ipv6 and group in ['address_group', 'network_group'] else group + error_group = fw_group.replace("_", "-") + + if group in ['address_group', 'network_group', 'domain_group']: + types = [t for t in ['address', 'fqdn', 'geoip'] if t in side_conf] + if types: + raise ConfigError(f'{error_group} and {types[0]} cannot both be defined') + if group_name and group_name[0] == '!': group_name = group_name[1:] - fw_group = f'ipv6_{group}' if ipv6 and group in ['address_group', 'network_group'] else group - error_group = fw_group.replace("_", "-") group_obj = dict_search_args(firewall, 'group', fw_group, group_name) if group_obj is None: @@ -477,26 +474,13 @@ def apply(firewall): if install_result == 1: raise ConfigError(f'Failed to apply firewall: {output}') - # set firewall group domain-group xxx - if 'group' in firewall: - if 'domain_group' in firewall['group']: - # T970 Enable a resolver (systemd daemon) that checks - # domain-group addresses and update entries for domains by timeout - # If router loaded without internet connection or for synchronization - call('systemctl restart vyos-domain-group-resolve.service') - for group, group_config in firewall['group']['domain_group'].items(): - domains = [] - if group_config.get('address') is not None: - for address in group_config.get('address'): - domains.append(address) - # Add elements to domain-group, try to resolve domain => ip - # and add elements to nft set - ip_dict = get_ips_domains_dict(domains) - elements = sum(ip_dict.values(), []) - nft_init_set(f'D_{group}') - nft_add_set_elements(f'D_{group}', elements) - else: - call('systemctl stop vyos-domain-group-resolve.service') + # T970 Enable a resolver (systemd daemon) that checks + # domain-group addresses and update entries for domains by timeout + # If router loaded without internet connection or for synchronization + domain_action = 'stop' + if dict_search_args(firewall, 'group', 'domain_group') or firewall['ip_fqdn'] or firewall['ip6_fqdn']: + domain_action = 'restart' + call(f'systemctl {domain_action} vyos-domain-resolver.service') apply_sysfs(firewall) diff --git a/src/helpers/vyos-domain-group-resolve.py b/src/helpers/vyos-domain-group-resolve.py deleted file mode 100755 index 6b677670b..000000000 --- a/src/helpers/vyos-domain-group-resolve.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2022 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -import time - -from vyos.configquery import ConfigTreeQuery -from vyos.firewall import get_ips_domains_dict -from vyos.firewall import nft_add_set_elements -from vyos.firewall import nft_flush_set -from vyos.firewall import nft_init_set -from vyos.firewall import nft_update_set_elements -from vyos.util import call - - -base = ['firewall', 'group', 'domain-group'] -check_required = True -# count_failed = 0 -# Timeout in sec between checks -timeout = 300 - -domain_state = {} - -if __name__ == '__main__': - - while check_required: - config = ConfigTreeQuery() - if config.exists(base): - domain_groups = config.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - for set_name, domain_config in domain_groups.items(): - list_domains = domain_config['address'] - elements = [] - ip_dict = get_ips_domains_dict(list_domains) - - for domain in list_domains: - # Resolution succeeded, update domain state - if domain in ip_dict: - domain_state[domain] = ip_dict[domain] - elements += ip_dict[domain] - # Resolution failed, use previous domain state - elif domain in domain_state: - elements += domain_state[domain] - - # Resolve successful - if elements: - nft_update_set_elements(f'D_{set_name}', elements) - time.sleep(timeout) diff --git a/src/helpers/vyos-domain-resolver.py b/src/helpers/vyos-domain-resolver.py new file mode 100755 index 000000000..2f71f15db --- /dev/null +++ b/src/helpers/vyos-domain-resolver.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import json +import os +import time + +from vyos.configdict import dict_merge +from vyos.configquery import ConfigTreeQuery +from vyos.firewall import fqdn_config_parse +from vyos.firewall import fqdn_resolve +from vyos.util import cmd +from vyos.util import commit_in_progress +from vyos.util import dict_search_args +from vyos.util import run +from vyos.xml import defaults + +base = ['firewall'] +timeout = 300 +cache = False + +domain_state = {} + +ipv4_tables = { + 'ip mangle', + 'ip vyos_filter', +} + +ipv6_tables = { + 'ip6 mangle', + 'ip6 vyos_filter' +} + +def get_config(conf): + firewall = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True, + no_tag_node_value_mangle=True) + + default_values = defaults(base) + for tmp in ['name', 'ipv6_name']: + if tmp in default_values: + del default_values[tmp] + + if 'zone' in default_values: + del default_values['zone'] + + firewall = dict_merge(default_values, firewall) + + global timeout, cache + + if 'resolver_interval' in firewall: + timeout = int(firewall['resolver_interval']) + + if 'resolver_cache' in firewall: + cache = True + + fqdn_config_parse(firewall) + + return firewall + +def resolve(domains, ipv6=False): + global domain_state + + ip_list = set() + + for domain in domains: + resolved = fqdn_resolve(domain, ipv6=ipv6) + + if resolved and cache: + domain_state[domain] = resolved + elif not resolved: + if domain not in domain_state: + continue + resolved = domain_state[domain] + + ip_list = ip_list | resolved + return ip_list + +def nft_output(table, set_name, ip_list): + output = [f'flush set {table} {set_name}'] + if ip_list: + ip_str = ','.join(ip_list) + output.append(f'add element {table} {set_name} {{ {ip_str} }}') + return output + +def nft_valid_sets(): + try: + valid_sets = [] + sets_json = cmd('nft -j list sets') + sets_obj = json.loads(sets_json) + + for obj in sets_obj['nftables']: + if 'set' in obj: + family = obj['set']['family'] + table = obj['set']['table'] + name = obj['set']['name'] + valid_sets.append((f'{family} {table}', name)) + + return valid_sets + except: + return [] + +def update(firewall): + conf_lines = [] + count = 0 + + valid_sets = nft_valid_sets() + + domain_groups = dict_search_args(firewall, 'group', 'domain_group') + if domain_groups: + for set_name, domain_config in domain_groups.items(): + if 'address' not in domain_config: + continue + + nft_set_name = f'D_{set_name}' + domains = domain_config['address'] + + ip_list = resolve(domains, ipv6=False) + for table in ipv4_tables: + if (table, nft_set_name) in valid_sets: + conf_lines += nft_output(table, nft_set_name, ip_list) + + ip6_list = resolve(domains, ipv6=True) + for table in ipv6_tables: + if (table, nft_set_name) in valid_sets: + conf_lines += nft_output(table, nft_set_name, ip6_list) + count += 1 + + for set_name, domain in firewall['ip_fqdn'].items(): + table = 'ip vyos_filter' + nft_set_name = f'FQDN_{set_name}' + + ip_list = resolve([domain], ipv6=False) + + if (table, nft_set_name) in valid_sets: + conf_lines += nft_output(table, nft_set_name, ip_list) + count += 1 + + for set_name, domain in firewall['ip6_fqdn'].items(): + table = 'ip6 vyos_filter' + nft_set_name = f'FQDN_{set_name}' + + ip_list = resolve([domain], ipv6=True) + if (table, nft_set_name) in valid_sets: + conf_lines += nft_output(table, nft_set_name, ip_list) + count += 1 + + nft_conf_str = "\n".join(conf_lines) + "\n" + code = run(f'nft -f -', input=nft_conf_str) + + print(f'Updated {count} sets - result: {code}') + +if __name__ == '__main__': + print(f'VyOS domain resolver') + + count = 1 + while commit_in_progress(): + if ( count % 60 == 0 ): + print(f'Commit still in progress after {count}s - waiting') + count += 1 + time.sleep(1) + + conf = ConfigTreeQuery() + firewall = get_config(conf) + + print(f'interval: {timeout}s - cache: {cache}') + + while True: + update(firewall) + time.sleep(timeout) diff --git a/src/systemd/vyos-domain-group-resolve.service b/src/systemd/vyos-domain-group-resolve.service deleted file mode 100644 index 29628fddb..000000000 --- a/src/systemd/vyos-domain-group-resolve.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=VyOS firewall domain-group resolver -After=vyos-router.service - -[Service] -Type=simple -Restart=always -ExecStart=/usr/bin/python3 /usr/libexec/vyos/vyos-domain-group-resolve.py - -[Install] -WantedBy=multi-user.target diff --git a/src/systemd/vyos-domain-resolver.service b/src/systemd/vyos-domain-resolver.service new file mode 100644 index 000000000..c56b51f0c --- /dev/null +++ b/src/systemd/vyos-domain-resolver.service @@ -0,0 +1,13 @@ +[Unit] +Description=VyOS firewall domain resolver +After=vyos-router.service + +[Service] +Type=simple +Restart=always +ExecStart=/usr/bin/python3 -u /usr/libexec/vyos/vyos-domain-resolver.py +StandardError=journal +StandardOutput=journal + +[Install] +WantedBy=multi-user.target -- cgit v1.2.3 From b4b491d424fba6f3d417135adc1865e338a480a1 Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Mon, 31 Oct 2022 21:08:42 +0100 Subject: nat: T1877: T970: Add firewall groups to NAT --- data/templates/firewall/nftables-nat.j2 | 4 ++ interface-definitions/include/nat-rule.xml.i | 2 + python/vyos/firewall.py | 2 + python/vyos/nat.py | 56 +++++++++++++++++++-- smoketest/scripts/cli/test_nat.py | 35 +++++++++++++ src/conf_mode/firewall.py | 22 ++++++--- src/conf_mode/nat.py | 73 +++++++++++++++++++++++----- src/helpers/vyos-domain-resolver.py | 1 + 8 files changed, 174 insertions(+), 21 deletions(-) (limited to 'src') diff --git a/data/templates/firewall/nftables-nat.j2 b/data/templates/firewall/nftables-nat.j2 index c5c0a2c86..f0be3cf5d 100644 --- a/data/templates/firewall/nftables-nat.j2 +++ b/data/templates/firewall/nftables-nat.j2 @@ -1,5 +1,7 @@ #!/usr/sbin/nft -f +{% import 'firewall/nftables-defines.j2' as group_tmpl %} + {% if helper_functions is vyos_defined('remove') %} {# NAT if going to be disabled - remove rules and targets from nftables #} {% set base_command = 'delete rule ip raw' %} @@ -59,5 +61,7 @@ table ip vyos_nat { chain VYOS_PRE_SNAT_HOOK { return } + +{{ group_tmpl.groups(firewall_group, False) }} } {% endif %} diff --git a/interface-definitions/include/nat-rule.xml.i b/interface-definitions/include/nat-rule.xml.i index 84941aa6a..8f2029388 100644 --- a/interface-definitions/include/nat-rule.xml.i +++ b/interface-definitions/include/nat-rule.xml.i @@ -20,6 +20,7 @@ #include #include + #include #include @@ -285,6 +286,7 @@ #include #include + #include diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index db4878c9d..59ec4948f 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -34,6 +34,8 @@ from vyos.util import dict_search_args from vyos.util import dict_search_recursive from vyos.util import run +# Domain Resolver + def fqdn_config_parse(firewall): firewall['ip_fqdn'] = {} firewall['ip6_fqdn'] = {} diff --git a/python/vyos/nat.py b/python/vyos/nat.py index 31bbdc386..3d01829a7 100644 --- a/python/vyos/nat.py +++ b/python/vyos/nat.py @@ -85,8 +85,13 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): translation_str += f' {",".join(options)}' for target in ['source', 'destination']: + if target not in rule_conf: + continue + + side_conf = rule_conf[target] prefix = target[:1] - addr = dict_search_args(rule_conf, target, 'address') + + addr = dict_search_args(side_conf, 'address') if addr and not (ignore_type_addr and target == nat_type): operator = '' if addr[:1] == '!': @@ -94,7 +99,7 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): addr = addr[1:] output.append(f'{ip_prefix} {prefix}addr {operator} {addr}') - addr_prefix = dict_search_args(rule_conf, target, 'prefix') + addr_prefix = dict_search_args(side_conf, 'prefix') if addr_prefix and ipv6: operator = '' if addr_prefix[:1] == '!': @@ -102,7 +107,7 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): addr_prefix = addr[1:] output.append(f'ip6 {prefix}addr {operator} {addr_prefix}') - port = dict_search_args(rule_conf, target, 'port') + port = dict_search_args(side_conf, 'port') if port: protocol = rule_conf['protocol'] if protocol == 'tcp_udp': @@ -113,6 +118,51 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): port = port[1:] output.append(f'{protocol} {prefix}port {operator} {{ {port} }}') + if 'group' in side_conf: + group = side_conf['group'] + if 'address_group' in group and not (ignore_type_addr and target == nat_type): + group_name = group['address_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_prefix} {prefix}addr {operator} @A_{group_name}') + # Generate firewall group domain-group + elif 'domain_group' in group and not (ignore_type_addr and target == nat_type): + group_name = group['domain_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_prefix} {prefix}addr {operator} @D_{group_name}') + elif 'network_group' in group and not (ignore_type_addr and target == nat_type): + group_name = group['network_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_prefix} {prefix}addr {operator} @N_{group_name}') + if 'mac_group' in group: + group_name = group['mac_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'ether {prefix}addr {operator} @M_{group_name}') + if 'port_group' in group: + proto = rule_conf['protocol'] + group_name = group['port_group'] + + if proto == 'tcp_udp': + proto = 'th' + + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + + output.append(f'{proto} {prefix}port {operator} @P_{group_name}') + output.append('counter') if 'log' in rule_conf: diff --git a/smoketest/scripts/cli/test_nat.py b/smoketest/scripts/cli/test_nat.py index 2ae90fcaf..9f4e3b831 100755 --- a/smoketest/scripts/cli/test_nat.py +++ b/smoketest/scripts/cli/test_nat.py @@ -58,6 +58,17 @@ class TestNAT(VyOSUnitTestSHIM.TestCase): break self.assertTrue(not matched if inverse else matched, msg=search) + def wait_for_domain_resolver(self, table, set_name, element, max_wait=10): + # Resolver no longer blocks commit, need to wait for daemon to populate set + count = 0 + while count < max_wait: + code = run(f'sudo nft get element {table} {set_name} {{ {element} }}') + if code == 0: + return True + count += 1 + sleep(1) + return False + def test_snat(self): rules = ['100', '110', '120', '130', '200', '210', '220', '230'] outbound_iface_100 = 'eth0' @@ -84,6 +95,30 @@ class TestNAT(VyOSUnitTestSHIM.TestCase): self.verify_nftables(nftables_search, 'ip vyos_nat') + def test_snat_groups(self): + address_group = 'smoketest_addr' + address_group_member = '192.0.2.1' + rule = '100' + outbound_iface = 'eth0' + + self.cli_set(['firewall', 'group', 'address-group', address_group, 'address', address_group_member]) + + self.cli_set(src_path + ['rule', rule, 'source', 'group', 'address-group', address_group]) + self.cli_set(src_path + ['rule', rule, 'outbound-interface', outbound_iface]) + self.cli_set(src_path + ['rule', rule, 'translation', 'address', 'masquerade']) + + self.cli_commit() + + nftables_search = [ + [f'set A_{address_group}'], + [f'elements = {{ {address_group_member} }}'], + [f'ip saddr @A_{address_group}', f'oifname "{outbound_iface}"', 'masquerade'] + ] + + self.verify_nftables(nftables_search, 'ip vyos_nat') + + self.cli_delete(['firewall']) + def test_dnat(self): rules = ['100', '110', '120', '130', '200', '210', '220', '230'] inbound_iface_100 = 'eth0' diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index 2bb765e65..783adec46 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -41,6 +41,7 @@ from vyos import ConfigError from vyos import airbag airbag.enable() +nat_conf_script = '/usr/libexec/vyos/conf_mode/nat.py' policy_route_conf_script = '/usr/libexec/vyos/conf_mode/policy-route.py' nftables_conf = '/run/nftables.conf' @@ -158,7 +159,7 @@ def get_config(config=None): for zone in firewall['zone']: firewall['zone'][zone] = dict_merge(default_values, firewall['zone'][zone]) - firewall['policy_resync'] = bool('group' in firewall or node_changed(conf, base + ['group'])) + firewall['group_resync'] = bool('group' in firewall or node_changed(conf, base + ['group'])) if 'config_trap' in firewall and firewall['config_trap'] == 'enable': diff = get_config_diff(conf) @@ -463,6 +464,12 @@ def post_apply_trap(firewall): cmd(base_cmd + ' '.join(objects)) +def resync_nat(): + # Update nat as firewall groups were updated + tmp, out = rc_cmd(nat_conf_script) + if tmp > 0: + Warning(f'Failed to re-apply nat configuration! {out}') + def resync_policy_route(): # Update policy route as firewall groups were updated tmp, out = rc_cmd(policy_route_conf_script) @@ -474,19 +481,20 @@ def apply(firewall): if install_result == 1: raise ConfigError(f'Failed to apply firewall: {output}') + apply_sysfs(firewall) + + if firewall['group_resync']: + resync_nat() + resync_policy_route() + # T970 Enable a resolver (systemd daemon) that checks - # domain-group addresses and update entries for domains by timeout + # domain-group/fqdn addresses and update entries for domains by timeout # If router loaded without internet connection or for synchronization domain_action = 'stop' if dict_search_args(firewall, 'group', 'domain_group') or firewall['ip_fqdn'] or firewall['ip6_fqdn']: domain_action = 'restart' call(f'systemctl {domain_action} vyos-domain-resolver.service') - apply_sysfs(firewall) - - if firewall['policy_resync']: - resync_policy_route() - if firewall['geoip_updated']: # Call helper script to Update set contents if 'name' in firewall['geoip_updated'] or 'ipv6_name' in firewall['geoip_updated']: diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py index 978c043e9..9f8221514 100755 --- a/src/conf_mode/nat.py +++ b/src/conf_mode/nat.py @@ -32,6 +32,7 @@ from vyos.util import cmd from vyos.util import run from vyos.util import check_kmod from vyos.util import dict_search +from vyos.util import dict_search_args from vyos.validate import is_addr_assigned from vyos.xml import defaults from vyos import ConfigError @@ -47,6 +48,13 @@ else: nftables_nat_config = '/run/nftables_nat.conf' nftables_static_nat_conf = '/run/nftables_static-nat-rules.nft' +valid_groups = [ + 'address_group', + 'domain_group', + 'network_group', + 'port_group' +] + def get_handler(json, chain, target): """ Get nftable rule handler number of given chain/target combination. Handler is required when adding NAT/Conntrack helper targets """ @@ -60,7 +68,7 @@ def get_handler(json, chain, target): return None -def verify_rule(config, err_msg): +def verify_rule(config, err_msg, groups_dict): """ Common verify steps used for both source and destination NAT """ if (dict_search('translation.port', config) != None or @@ -78,6 +86,45 @@ def verify_rule(config, err_msg): 'statically maps a whole network of addresses onto another\n' \ 'network of addresses') + for side in ['destination', 'source']: + if side in config: + side_conf = config[side] + + if len({'address', 'fqdn'} & set(side_conf)) > 1: + raise ConfigError('Only one of address, fqdn or geoip can be specified') + + if 'group' in side_conf: + if len({'address_group', 'network_group', 'domain_group'} & set(side_conf['group'])) > 1: + raise ConfigError('Only one address-group, network-group or domain-group can be specified') + + for group in valid_groups: + if group in side_conf['group']: + group_name = side_conf['group'][group] + error_group = group.replace("_", "-") + + if group in ['address_group', 'network_group', 'domain_group']: + types = [t for t in ['address', 'fqdn'] if t in side_conf] + if types: + raise ConfigError(f'{error_group} and {types[0]} cannot both be defined') + + if group_name and group_name[0] == '!': + group_name = group_name[1:] + + group_obj = dict_search_args(groups_dict, group, group_name) + + if group_obj is None: + raise ConfigError(f'Invalid {error_group} "{group_name}" on firewall rule') + + if not group_obj: + Warning(f'{error_group} "{group_name}" has no members!') + + if dict_search_args(side_conf, 'group', 'port_group'): + if 'protocol' not in config: + raise ConfigError('Protocol must be defined if specifying a port-group') + + if config['protocol'] not in ['tcp', 'udp', 'tcp_udp']: + raise ConfigError('Protocol must be tcp, udp, or tcp_udp when specifying a port-group') + def get_config(config=None): if config: conf = config @@ -105,16 +152,20 @@ def get_config(config=None): condensed_json = jmespath.search(pattern, nftable_json) if not conf.exists(base): - nat['helper_functions'] = 'remove' - - # Retrieve current table handler positions - nat['pre_ct_ignore'] = get_handler(condensed_json, 'PREROUTING', 'VYOS_CT_HELPER') - nat['pre_ct_conntrack'] = get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK') - nat['out_ct_ignore'] = get_handler(condensed_json, 'OUTPUT', 'VYOS_CT_HELPER') - nat['out_ct_conntrack'] = get_handler(condensed_json, 'OUTPUT', 'NAT_CONNTRACK') + if get_handler(condensed_json, 'PREROUTING', 'VYOS_CT_HELPER'): + nat['helper_functions'] = 'remove' + + # Retrieve current table handler positions + nat['pre_ct_ignore'] = get_handler(condensed_json, 'PREROUTING', 'VYOS_CT_HELPER') + nat['pre_ct_conntrack'] = get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK') + nat['out_ct_ignore'] = get_handler(condensed_json, 'OUTPUT', 'VYOS_CT_HELPER') + nat['out_ct_conntrack'] = get_handler(condensed_json, 'OUTPUT', 'NAT_CONNTRACK') nat['deleted'] = '' return nat + nat['firewall_group'] = conf.get_config_dict(['firewall', 'group'], key_mangling=('-', '_'), get_first_key=True, + no_tag_node_value_mangle=True) + # check if NAT connection tracking helpers need to be set up - this has to # be done only once if not get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK'): @@ -157,7 +208,7 @@ def verify(nat): Warning(f'IP address {ip} does not exist on the system!') # common rule verification - verify_rule(config, err_msg) + verify_rule(config, err_msg, nat['firewall_group']) if dict_search('destination.rule', nat): @@ -175,7 +226,7 @@ def verify(nat): raise ConfigError(f'{err_msg} translation requires address and/or port') # common rule verification - verify_rule(config, err_msg) + verify_rule(config, err_msg, nat['firewall_group']) if dict_search('static.rule', nat): for rule, config in dict_search('static.rule', nat).items(): @@ -186,7 +237,7 @@ def verify(nat): 'inbound-interface not specified') # common rule verification - verify_rule(config, err_msg) + verify_rule(config, err_msg, nat['firewall_group']) return None diff --git a/src/helpers/vyos-domain-resolver.py b/src/helpers/vyos-domain-resolver.py index 2f71f15db..035c208b2 100755 --- a/src/helpers/vyos-domain-resolver.py +++ b/src/helpers/vyos-domain-resolver.py @@ -37,6 +37,7 @@ domain_state = {} ipv4_tables = { 'ip mangle', 'ip vyos_filter', + 'ip vyos_nat' } ipv6_tables = { -- cgit v1.2.3 From ff09d4f47e5f54fad8258cd27fb0adfaa4c552b3 Mon Sep 17 00:00:00 2001 From: initramfs Date: Sat, 5 Nov 2022 08:51:18 +0800 Subject: dns: T4799: fix bug with not reloading powerdns config PowerDNS version 4.7 and above has changed the main process name from 'pdns-r/worker' to 'pdns_recursor'. This commit updates the process name check to use the new name. --- src/services/vyos-hostsd | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src') diff --git a/src/services/vyos-hostsd b/src/services/vyos-hostsd index 9ae7b1ea9..a380f2e66 100755 --- a/src/services/vyos-hostsd +++ b/src/services/vyos-hostsd @@ -406,8 +406,7 @@ def validate_schema(data): def pdns_rec_control(command): - # pdns-r process name is NOT equal to the name shown in ps - if not process_named_running('pdns-r/worker'): + if not process_named_running('pdns_recursor'): logger.info(f'pdns_recursor not running, not sending "{command}"') return -- cgit v1.2.3 From acf2c24f8eec0918ca419db68196c76dc827f197 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sat, 5 Nov 2022 19:47:42 +0100 Subject: container: T4802: support per container shared-memory size configuration Size of /dev/shm within a container can be defined via --shm-size when invoking the container. Add corresponding CLI node. --- interface-definitions/container.xml.in | 20 +++++++++++++++++++- src/conf_mode/container.py | 5 +++-- 2 files changed, 22 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/interface-definitions/container.xml.in b/interface-definitions/container.xml.in index 51171d881..f84c94a40 100644 --- a/interface-definitions/container.xml.in +++ b/interface-definitions/container.xml.in @@ -111,7 +111,7 @@ - Constrain the memory available to a container + Memory (RAM) available to this container u32:0 Unlimited @@ -127,6 +127,24 @@ 512 + + + Shared memory available to this container + + u32:0 + Unlimited + + + u32:1-8192 + Container memory in megabytes (MB) + + + + + Container memory must be in range 0 to 8192 MB + + 64 + Attach user defined network to container diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py index 70d149f0d..8efeaed54 100755 --- a/src/conf_mode/container.py +++ b/src/conf_mode/container.py @@ -207,6 +207,7 @@ def verify(container): def generate_run_arguments(name, container_config): image = container_config['image'] memory = container_config['memory'] + shared_memory = container_config['shared_memory'] restart = container_config['restart'] # Add capability options. Should be in uppercase @@ -254,9 +255,9 @@ def generate_run_arguments(name, container_config): volume += f' -v {svol}:{dvol}' container_base_cmd = f'--detach --interactive --tty --replace {cap_add} ' \ - f'--memory {memory}m --memory-swap 0 --restart {restart} ' \ + f'--memory {memory}m --shm-size {shared_memory}m --memory-swap 0 --restart {restart} ' \ f'--name {name} {device} {port} {volume} {env_opt}' - + if 'allow_host_networks' in container_config: return f'{container_base_cmd} --net host {image}' -- cgit v1.2.3 From e1d9982c7b463b173cc8c261f61a9447ace62898 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 6 Nov 2022 09:31:13 -0600 Subject: graphql: T4803: allow 'Authorization' header in CORS middleware --- src/services/vyos-http-api-server | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server index 3c390d9dc..60ea9a5ee 100755 --- a/src/services/vyos-http-api-server +++ b/src/services/vyos-http-api-server @@ -659,10 +659,18 @@ def graphql_init(fast_api_app): if app.state.vyos_origins: origins = app.state.vyos_origins - app.add_route('/graphql', CORSMiddleware(GraphQL(schema, context_value=get_user_context, debug=True, introspection=in_spec), allow_origins=origins, allow_methods=("GET", "POST", "OPTIONS"))) + app.add_route('/graphql', CORSMiddleware(GraphQL(schema, + context_value=get_user_context, + debug=True, + introspection=in_spec), + allow_origins=origins, + allow_methods=("GET", "POST", "OPTIONS"), + allow_headers=("Authorization",))) else: - app.add_route('/graphql', GraphQL(schema, context_value=get_user_context, debug=True, introspection=in_spec)) - + app.add_route('/graphql', GraphQL(schema, + context_value=get_user_context, + debug=True, + introspection=in_spec)) ### if __name__ == '__main__': -- cgit v1.2.3 From 4245fd8fb1059d0a356e0fcb293fcbb923b09b68 Mon Sep 17 00:00:00 2001 From: "m.korobeinikov" Date: Tue, 8 Nov 2022 08:56:42 +1000 Subject: T4767: Rewrite generate ipsec archive to python Made the following changes: 1) made changes to the "XML" file to replace the script from "sh" to "py" 2) changed the extension of the main script from "sh" to "py" 3) changed the script to "py" --- .../generate-ipsec-debug-archive.xml.in | 2 +- src/op_mode/generate_ipsec_debug_archive.py | 90 ++++++++++++++++++++++ src/op_mode/generate_ipsec_debug_archive.sh | 36 --------- 3 files changed, 91 insertions(+), 37 deletions(-) create mode 100755 src/op_mode/generate_ipsec_debug_archive.py delete mode 100755 src/op_mode/generate_ipsec_debug_archive.sh (limited to 'src') diff --git a/op-mode-definitions/generate-ipsec-debug-archive.xml.in b/op-mode-definitions/generate-ipsec-debug-archive.xml.in index f268d5ae5..dcbed0c42 100644 --- a/op-mode-definitions/generate-ipsec-debug-archive.xml.in +++ b/op-mode-definitions/generate-ipsec-debug-archive.xml.in @@ -8,7 +8,7 @@ Generate IPSec debug-archive - ${vyos_op_scripts_dir}/generate_ipsec_debug_archive.sh + ${vyos_op_scripts_dir}/generate_ipsec_debug_archive.py diff --git a/src/op_mode/generate_ipsec_debug_archive.py b/src/op_mode/generate_ipsec_debug_archive.py new file mode 100755 index 000000000..933dd4e1a --- /dev/null +++ b/src/op_mode/generate_ipsec_debug_archive.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from datetime import datetime +from pathlib import Path +from shutil import rmtree +from socket import gethostname +from sys import exit +from tarfile import open as tar_open +from vyos.util import rc_cmd + +# define a list of commands that needs to be executed +CMD_LIST: list[str] = [ + 'sudo ipsec status', + 'sudo swanctl -L', + 'sudo swanctl -l', + 'sudo swanctl -P', + 'sudo ip x sa show', + 'sudo ip x policy show', + 'sudo ip tunnel show', + 'sudo ip address', + 'sudo ip rule show', + 'sudo ip route | head -100', + 'sudo ip route show table 220' +] +JOURNALCTL_CMD: str = 'sudo journalctl -b -n 10000 /usr/lib/ipsec/charon' + + +# execute a command and save the output to a file +def save_stdout(command: str, file: Path) -> None: + rc, stdout = rc_cmd(command) + body: str = f'''### {command} ### +Command: {command} +Exit code: {rc} +Stdout: +{stdout} + +''' + with file.open(mode='a') as f: + f.write(body) + + +# get local host name +hostname: str = gethostname() +# get current time +time_now: str = datetime.now().isoformat(timespec='seconds') + +# define a temporary directory for logs and collected data +tmp_dir: Path = Path(f'/tmp/ipsec_debug_{time_now}') +# set file paths +ipsec_status_file: Path = Path(f'{tmp_dir}/ipsec_status.txt') +journalctl_charon_file: Path = Path(f'{tmp_dir}/journalctl_charon.txt') +archive_file: str = f'/tmp/ipsec_debug_{time_now}.tar.bz2' + +# create files +tmp_dir.mkdir() +ipsec_status_file.touch() +journalctl_charon_file.touch() + +try: + # execute all commands + for command in CMD_LIST: + save_stdout(command, ipsec_status_file) + save_stdout(JOURNALCTL_CMD, journalctl_charon_file) + + # create an archive + with tar_open(name=archive_file, mode='x:bz2') as tar_file: + tar_file.add(tmp_dir) + + # inform user about success + print(f'Debug file is generated and located in {archive_file}') +except Exception as err: + print(f'Error during generating a debug file: {err}') +finally: + # cleanup + rmtree(tmp_dir) + exit() diff --git a/src/op_mode/generate_ipsec_debug_archive.sh b/src/op_mode/generate_ipsec_debug_archive.sh deleted file mode 100755 index 53d0a6eaa..000000000 --- a/src/op_mode/generate_ipsec_debug_archive.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -# Collecting IPSec Debug Information - -DATE=`date +%d-%m-%Y` - -a_CMD=( - "sudo ipsec status" - "sudo swanctl -L" - "sudo swanctl -l" - "sudo swanctl -P" - "sudo ip x sa show" - "sudo ip x policy show" - "sudo ip tunnel show" - "sudo ip address" - "sudo ip rule show" - "sudo ip route" - "sudo ip route show table 220" - ) - - -echo "DEBUG: ${DATE} on host \"$(hostname)\"" > /tmp/ipsec-status-${DATE}.txt -date >> /tmp/ipsec-status-${DATE}.txt - -# Execute all DEBUG commands and save it to file -for cmd in "${a_CMD[@]}"; do - echo -e "\n### ${cmd} ###" >> /tmp/ipsec-status-${DATE}.txt - ${cmd} >> /tmp/ipsec-status-${DATE}.txt 2>/dev/null -done - -# Collect charon logs, build .tgz archive -sudo journalctl /usr/lib/ipsec/charon > /tmp/journalctl-charon-${DATE}.txt && \ -sudo tar -zcvf /tmp/ipsec-debug-${DATE}.tgz /tmp/journalctl-charon-${DATE}.txt /tmp/ipsec-status-${DATE}.txt >& /dev/null -sudo rm -f /tmp/journalctl-charon-${DATE}.txt /tmp/ipsec-status-${DATE}.txt - -echo "Debug file is generated and located in /tmp/ipsec-debug-${DATE}.tgz" -- cgit v1.2.3 From 44df1cea1ebc3296844c5c35cf053a92cda4b944 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Wed, 9 Nov 2022 06:55:00 +0100 Subject: Revert "dns: T4799: fix bug with not reloading powerdns config" This reverts commit ff09d4f47e5f54fad8258cd27fb0adfaa4c552b3. Process name is actually: --- src/services/vyos-hostsd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/services/vyos-hostsd b/src/services/vyos-hostsd index a380f2e66..9ae7b1ea9 100755 --- a/src/services/vyos-hostsd +++ b/src/services/vyos-hostsd @@ -406,7 +406,8 @@ def validate_schema(data): def pdns_rec_control(command): - if not process_named_running('pdns_recursor'): + # pdns-r process name is NOT equal to the name shown in ps + if not process_named_running('pdns-r/worker'): logger.info(f'pdns_recursor not running, not sending "{command}"') return -- cgit v1.2.3 From a76c46ab0cdc2aa068aa85860767372f0c9550eb Mon Sep 17 00:00:00 2001 From: aapostoliuk Date: Tue, 8 Nov 2022 15:59:59 +0200 Subject: T4807: Fixed traceroute help completion Changes in traceroute command: Added list of possible VRFs in the help. Added list of possible interfaces in the help. Changed, if an option was selected before, it does not appear in possible completion. Added error message when an unexpected option was selected --- src/op_mode/traceroute.py | 85 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 67 insertions(+), 18 deletions(-) (limited to 'src') diff --git a/src/op_mode/traceroute.py b/src/op_mode/traceroute.py index 4299d6e5f..6c7030ea0 100755 --- a/src/op_mode/traceroute.py +++ b/src/op_mode/traceroute.py @@ -18,6 +18,25 @@ import os import sys import socket import ipaddress +from vyos.util import get_all_vrfs +from vyos.ifconfig import Section + + +def interface_list() -> list: + """ + Get list of interfaces in system + :rtype: list + """ + return Section.interfaces() + + +def vrf_list() -> list: + """ + Get list of VRFs in system + :rtype: list + """ + return list(get_all_vrfs().keys()) + options = { 'backward-hops': { @@ -48,6 +67,7 @@ options = { 'interface': { 'traceroute': '{command} -i {value}', 'type': '', + 'helpfunction': interface_list, 'help': 'Source interface' }, 'lookup-as': { @@ -99,6 +119,7 @@ options = { 'traceroute': 'sudo ip vrf exec {value} {command}', 'type': '', 'help': 'Use specified VRF table', + 'helpfunction': vrf_list, 'dflt': 'default'} } @@ -108,20 +129,33 @@ traceroute = { } -class List (list): - def first (self): +class List(list): + def first(self): return self.pop(0) if self else '' def last(self): return self.pop() if self else '' - def prepend(self,value): - self.insert(0,value) + def prepend(self, value): + self.insert(0, value) + + +def completion_failure(option: str) -> None: + """ + Shows failure message after TAB when option is wrong + :param option: failure option + :type str: + """ + sys.stderr.write('\n\n Invalid option: {}\n\n'.format(option)) + sys.stdout.write('') + sys.exit(1) def expension_failure(option, completions): reason = 'Ambiguous' if completions else 'Invalid' - sys.stderr.write('\n\n {} command: {} [{}]\n\n'.format(reason,' '.join(sys.argv), option)) + sys.stderr.write( + '\n\n {} command: {} [{}]\n\n'.format(reason, ' '.join(sys.argv), + option)) if completions: sys.stderr.write(' Possible completions:\n ') sys.stderr.write('\n '.join(completions)) @@ -160,30 +194,46 @@ if __name__ == '__main__': sys.exit("traceroute: Missing host") if host == '--get-options': - args.first() # pop traceroute + args.first() # pop ping args.first() # pop IP + usedoptionslist = [] while args: - option = args.first() - - matched = complete(option) + option = args.first() # pop option + matched = complete(option) # get option parameters + usedoptionslist.append(option) # list of used options + # Select options if not args: + # remove from Possible completions used options + for o in usedoptionslist: + if o in matched: + matched.remove(o) sys.stdout.write(' '.join(matched)) sys.exit(0) - if len(matched) > 1 : + if len(matched) > 1: sys.stdout.write(' '.join(matched)) sys.exit(0) + # If option doesn't have value + if matched: + if options[matched[0]]['type'] == 'noarg': + continue + else: + # Unexpected option + completion_failure(option) - if options[matched[0]]['type'] == 'noarg': - continue - - value = args.first() + value = args.first() # pop option's value if not args: matched = complete(option) - sys.stdout.write(options[matched[0]]['type']) + helplines = options[matched[0]]['type'] + # Run helpfunction to get list of possible values + if 'helpfunction' in options[matched[0]]: + result = options[matched[0]]['helpfunction']() + if result: + helplines = '\n' + ' '.join(result) + sys.stdout.write(helplines) sys.exit(0) - for name,option in options.items(): + for name, option in options.items(): if 'dflt' in option and name not in args: args.append(name) args.append(option['dflt']) @@ -200,8 +250,7 @@ if __name__ == '__main__': except ValueError: sys.exit(f'traceroute: Unknown host: {host}') - command = convert(traceroute[version],args) + command = convert(traceroute[version], args) # print(f'{command} {host}') os.system(f'{command} {host}') - -- cgit v1.2.3 From a34d189f3218199e329892b3ce88b367337a6de8 Mon Sep 17 00:00:00 2001 From: aapostoliuk Date: Thu, 10 Nov 2022 16:27:23 +0200 Subject: T4496: Refactoring vrf_list function in ping command Changed the function code of vrf_list to using the function from vyos.util --- src/op_mode/ping.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/op_mode/ping.py b/src/op_mode/ping.py index 5e5b95a0a..610e63cb3 100755 --- a/src/op_mode/ping.py +++ b/src/op_mode/ping.py @@ -18,8 +18,7 @@ import os import sys import socket import ipaddress -import json -from vyos.util import cmd, rc_cmd +from vyos.util import get_all_vrfs from vyos.ifconfig import Section @@ -36,13 +35,7 @@ def vrf_list() -> list: Get list of VRFs in system :rtype: list """ - result = cmd(f'sudo ip --json --brief link show type vrf') - data = json.loads(result) - vrflist: list = [] - for o in data: - if 'ifname' in o: - vrflist.append(o['ifname']) - return vrflist + return list(get_all_vrfs().keys()) options = { -- cgit v1.2.3 From ef365493aef665c20e27ff2de473624c14e1b521 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 7 Nov 2022 17:44:00 +0000 Subject: T4789: Ability to get op-mode raw data for PPPoE L2TP SSTP IPoE Ability to get 'raw' data sessions and statistics for accel-ppp protocols IPoE/PPPoE/L2TP/PPTP/SSTP server --- data/op-mode-standardized.json | 1 + python/vyos/accel_ppp.py | 74 +++++++++++++++++++++++ src/op_mode/accelppp.py | 133 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+) create mode 100644 python/vyos/accel_ppp.py create mode 100755 src/op_mode/accelppp.py (limited to 'src') diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index ec950765d..b162f4097 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -1,4 +1,5 @@ [ +"accelppp.py", "bgp.py", "bridge.py", "conntrack.py", diff --git a/python/vyos/accel_ppp.py b/python/vyos/accel_ppp.py new file mode 100644 index 000000000..bfc8ee5a9 --- /dev/null +++ b/python/vyos/accel_ppp.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import sys + +import vyos.opmode +from vyos.util import rc_cmd + + +def get_server_statistics(accel_statistics, pattern, sep=':') -> dict: + import re + + stat_dict = {'sessions': {}} + + cpu = re.search(r'cpu(.*)', accel_statistics).group(0) + # Find all lines with pattern, for example 'sstp:' + data = re.search(rf'{pattern}(.*)', accel_statistics, re.DOTALL).group(0) + session_starting = re.search(r'starting(.*)', data).group(0) + session_active = re.search(r'active(.*)', data).group(0) + + for entry in {cpu, session_starting, session_active}: + if sep in entry: + key, value = entry.split(sep) + if key in ['starting', 'active', 'finishing']: + stat_dict['sessions'][key] = value.strip() + continue + stat_dict[key] = value.strip() + return stat_dict + + +def accel_cmd(port: int, command: str) -> str: + _, output = rc_cmd(f'/usr/bin/accel-cmd -p{port} {command}') + return output + + +def accel_out_parse(accel_output: list[str]) -> list[dict[str, str]]: + """ Parse accel-cmd show sessions output """ + data_list: list[dict[str, str]] = list() + field_names: list[str] = list() + + field_names_unstripped: list[str] = accel_output.pop(0).split('|') + for field_name in field_names_unstripped: + field_names.append(field_name.strip()) + + while accel_output: + if '|' not in accel_output[0]: + accel_output.pop(0) + continue + + current_item: list[str] = accel_output.pop(0).split('|') + item_dict: dict[str, str] = {} + + for field_index in range(len(current_item)): + field_name: str = field_names[field_index] + field_value: str = current_item[field_index].strip() + item_dict[field_name] = field_value + + data_list.append(item_dict) + + return data_list diff --git a/src/op_mode/accelppp.py b/src/op_mode/accelppp.py new file mode 100755 index 000000000..2fd045dc3 --- /dev/null +++ b/src/op_mode/accelppp.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import sys + +import vyos.accel_ppp +import vyos.opmode + +from vyos.configquery import ConfigTreeQuery +from vyos.util import rc_cmd + + +accel_dict = { + 'ipoe': { + 'port': 2002, + 'path': 'service ipoe-server' + }, + 'pppoe': { + 'port': 2001, + 'path': 'service pppoe-server' + }, + 'pptp': { + 'port': 2003, + 'path': 'vpn pptp' + }, + 'l2tp': { + 'port': 2004, + 'path': 'vpn l2tp' + }, + 'sstp': { + 'port': 2005, + 'path': 'vpn sstp' + } +} + + +def _get_raw_statistics(accel_output, pattern): + return vyos.accel_ppp.get_server_statistics(accel_output, pattern, sep=':') + + +def _get_raw_sessions(port): + cmd_options = 'show sessions ifname,username,ip,ip6,ip6-dp,type,state,' \ + 'uptime-raw,calling-sid,called-sid,sid,comp,rx-bytes-raw,' \ + 'tx-bytes-raw,rx-pkts,tx-pkts' + output = vyos.accel_ppp.accel_cmd(port, cmd_options) + parsed_data: list[dict[str, str]] = vyos.accel_ppp.accel_out_parse( + output.splitlines()) + return parsed_data + + +def _verify(func): + """Decorator checks if accel-ppp protocol + ipoe/pppoe/pptp/l2tp/sstp is configured + + for example: + service ipoe-server + vpn sstp + """ + from functools import wraps + + @wraps(func) + def _wrapper(*args, **kwargs): + config = ConfigTreeQuery() + protocol_list = accel_dict.keys() + protocol = kwargs.get('protocol') + # unknown or incorrect protocol query + if protocol not in protocol_list: + unconf_message = f'unknown protocol "{protocol}"' + raise vyos.opmode.UnconfiguredSubsystem(unconf_message) + # Check if config does not exist + config_protocol_path = accel_dict[protocol]['path'] + if not config.exists(config_protocol_path): + unconf_message = f'"{config_protocol_path}" is not configured' + raise vyos.opmode.UnconfiguredSubsystem(unconf_message) + return func(*args, **kwargs) + + return _wrapper + + +@_verify +def show_statistics(raw: bool, protocol: str): + """show accel-cmd statistics + CPU utilization and amount of sessions + + protocol: ipoe/pppoe/ppptp/l2tp/sstp + """ + pattern = f'{protocol}:' + port = accel_dict[protocol]['port'] + rc, output = rc_cmd(f'/usr/bin/accel-cmd -p {port} show stat') + + if raw: + return _get_raw_statistics(output, pattern) + + return output + + +@_verify +def show_sessions(raw: bool, protocol: str): + """show accel-cmd sessions + + protocol: ipoe/pppoe/ppptp/l2tp/sstp + """ + port = accel_dict[protocol]['port'] + if raw: + return _get_raw_sessions(port) + + return vyos.accel_ppp.accel_cmd(port, + 'show sessions ifname,username,ip,ip6,ip6-dp,' + 'calling-sid,rate-limit,state,uptime,rx-bytes,tx-bytes') + + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) -- cgit v1.2.3 From 586b24e0af1ae57c47c772229fc94ab50dfc1e4f Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Wed, 2 Nov 2022 15:32:11 +0100 Subject: policy: T2199: T4605: Migrate policy route interface to `policy route|route6 interface ` * Include refactor to policy route to allow for deletion of mangle table instead of complex cleanup * T4605: Rename mangle table to vyos_mangle --- data/templates/firewall/nftables-policy.j2 | 31 +++-- .../include/interface/interface-policy-vif-c.xml.i | 26 ---- .../include/interface/interface-policy-vif.xml.i | 26 ---- .../include/interface/interface-policy.xml.i | 26 ---- .../include/interface/vif-s.xml.i | 2 - interface-definitions/include/interface/vif.xml.i | 1 - .../include/version/policy-version.xml.i | 2 +- interface-definitions/interfaces-bonding.xml.in | 1 - interface-definitions/interfaces-bridge.xml.in | 1 - interface-definitions/interfaces-dummy.xml.in | 1 - interface-definitions/interfaces-ethernet.xml.in | 1 - interface-definitions/interfaces-geneve.xml.in | 1 - interface-definitions/interfaces-input.xml.in | 1 - interface-definitions/interfaces-l2tpv3.xml.in | 1 - interface-definitions/interfaces-macsec.xml.in | 1 - interface-definitions/interfaces-openvpn.xml.in | 1 - interface-definitions/interfaces-pppoe.xml.in | 1 - .../interfaces-pseudo-ethernet.xml.in | 1 - interface-definitions/interfaces-tunnel.xml.in | 1 - interface-definitions/interfaces-vti.xml.in | 1 - interface-definitions/interfaces-vxlan.xml.in | 1 - interface-definitions/interfaces-wireguard.xml.in | 1 - interface-definitions/interfaces-wireless.xml.in | 1 - interface-definitions/interfaces-wwan.xml.in | 1 - interface-definitions/policy-route.xml.in | 2 + smoketest/scripts/cli/test_policy_route.py | 58 +++++---- src/conf_mode/policy-route-interface.py | 132 --------------------- src/conf_mode/policy-route.py | 106 +---------------- src/helpers/vyos-domain-resolver.py | 4 +- src/migration-scripts/policy/4-to-5 | 92 ++++++++++++++ src/op_mode/policy_route.py | 42 +------ 31 files changed, 154 insertions(+), 413 deletions(-) delete mode 100644 interface-definitions/include/interface/interface-policy-vif-c.xml.i delete mode 100644 interface-definitions/include/interface/interface-policy-vif.xml.i delete mode 100644 interface-definitions/include/interface/interface-policy.xml.i delete mode 100755 src/conf_mode/policy-route-interface.py create mode 100755 src/migration-scripts/policy/4-to-5 (limited to 'src') diff --git a/data/templates/firewall/nftables-policy.j2 b/data/templates/firewall/nftables-policy.j2 index 40118930b..6cb3b2f95 100644 --- a/data/templates/firewall/nftables-policy.j2 +++ b/data/templates/firewall/nftables-policy.j2 @@ -2,21 +2,24 @@ {% import 'firewall/nftables-defines.j2' as group_tmpl %} -{% if cleanup_commands is vyos_defined %} -{% for command in cleanup_commands %} -{{ command }} -{% endfor %} +{% if first_install is not vyos_defined %} +delete table ip vyos_mangle +delete table ip6 vyos_mangle {% endif %} - -table ip mangle { -{% if first_install is vyos_defined %} +table ip vyos_mangle { chain VYOS_PBR_PREROUTING { type filter hook prerouting priority -150; policy accept; +{% if route is vyos_defined %} +{% for route_text, conf in route.items() if conf.interface is vyos_defined %} + iifname { {{ ",".join(conf.interface) }} } counter jump VYOS_PBR_{{ route_text }} +{% endfor %} +{% endif %} } + chain VYOS_PBR_POSTROUTING { type filter hook postrouting priority -150; policy accept; } -{% endif %} + {% if route is vyos_defined %} {% for route_text, conf in route.items() %} chain VYOS_PBR_{{ route_text }} { @@ -32,15 +35,20 @@ table ip mangle { {{ group_tmpl.groups(firewall_group, False) }} } -table ip6 mangle { -{% if first_install is vyos_defined %} +table ip6 vyos_mangle { chain VYOS_PBR6_PREROUTING { type filter hook prerouting priority -150; policy accept; +{% if route6 is vyos_defined %} +{% for route_text, conf in route6.items() if conf.interface is vyos_defined %} + iifname { {{ ",".join(conf.interface) }} } counter jump VYOS_PBR6_{{ route_text }} +{% endfor %} +{% endif %} } + chain VYOS_PBR6_POSTROUTING { type filter hook postrouting priority -150; policy accept; } -{% endif %} + {% if route6 is vyos_defined %} {% for route_text, conf in route6.items() %} chain VYOS_PBR6_{{ route_text }} { @@ -52,5 +60,6 @@ table ip6 mangle { } {% endfor %} {% endif %} + {{ group_tmpl.groups(firewall_group, True) }} } diff --git a/interface-definitions/include/interface/interface-policy-vif-c.xml.i b/interface-definitions/include/interface/interface-policy-vif-c.xml.i deleted file mode 100644 index 866fcd5c0..000000000 --- a/interface-definitions/include/interface/interface-policy-vif-c.xml.i +++ /dev/null @@ -1,26 +0,0 @@ - - - - 620 - Policy route options - - - - - IPv4 policy route ruleset for interface - - policy route - - - - - - IPv6 policy route ruleset for interface - - policy route6 - - - - - - diff --git a/interface-definitions/include/interface/interface-policy-vif.xml.i b/interface-definitions/include/interface/interface-policy-vif.xml.i deleted file mode 100644 index 83510fe59..000000000 --- a/interface-definitions/include/interface/interface-policy-vif.xml.i +++ /dev/null @@ -1,26 +0,0 @@ - - - - 620 - Policy route options - - - - - IPv4 policy route ruleset for interface - - policy route - - - - - - IPv6 policy route ruleset for interface - - policy route6 - - - - - - diff --git a/interface-definitions/include/interface/interface-policy.xml.i b/interface-definitions/include/interface/interface-policy.xml.i deleted file mode 100644 index 42a8fd009..000000000 --- a/interface-definitions/include/interface/interface-policy.xml.i +++ /dev/null @@ -1,26 +0,0 @@ - - - - 620 - Policy route options - - - - - IPv4 policy route ruleset for interface - - policy route - - - - - - IPv6 policy route ruleset for interface - - policy route6 - - - - - - diff --git a/interface-definitions/include/interface/vif-s.xml.i b/interface-definitions/include/interface/vif-s.xml.i index 916349ade..6d50d7238 100644 --- a/interface-definitions/include/interface/vif-s.xml.i +++ b/interface-definitions/include/interface/vif-s.xml.i @@ -18,7 +18,6 @@ #include #include #include - #include Protocol used for service VLAN (default: 802.1ad) @@ -67,7 +66,6 @@ #include #include #include - #include #include diff --git a/interface-definitions/include/interface/vif.xml.i b/interface-definitions/include/interface/vif.xml.i index 73a8c98ff..3f8f113ea 100644 --- a/interface-definitions/include/interface/vif.xml.i +++ b/interface-definitions/include/interface/vif.xml.i @@ -18,7 +18,6 @@ #include #include #include - #include VLAN egress QoS diff --git a/interface-definitions/include/version/policy-version.xml.i b/interface-definitions/include/version/policy-version.xml.i index 89bde20c7..f1494eaa3 100644 --- a/interface-definitions/include/version/policy-version.xml.i +++ b/interface-definitions/include/version/policy-version.xml.i @@ -1,3 +1,3 @@ - + diff --git a/interface-definitions/interfaces-bonding.xml.in b/interface-definitions/interfaces-bonding.xml.in index 41e4a68a8..96e0e5d89 100644 --- a/interface-definitions/interfaces-bonding.xml.in +++ b/interface-definitions/interfaces-bonding.xml.in @@ -56,7 +56,6 @@ #include #include #include - #include Bonding transmit hash policy diff --git a/interface-definitions/interfaces-bridge.xml.in b/interface-definitions/interfaces-bridge.xml.in index d633077d9..d52e213b6 100644 --- a/interface-definitions/interfaces-bridge.xml.in +++ b/interface-definitions/interfaces-bridge.xml.in @@ -41,7 +41,6 @@ #include #include #include - #include Forwarding delay diff --git a/interface-definitions/interfaces-dummy.xml.in b/interface-definitions/interfaces-dummy.xml.in index fb36741f7..eb525b547 100644 --- a/interface-definitions/interfaces-dummy.xml.in +++ b/interface-definitions/interfaces-dummy.xml.in @@ -19,7 +19,6 @@ #include #include #include - #include IPv4 routing parameters diff --git a/interface-definitions/interfaces-ethernet.xml.in b/interface-definitions/interfaces-ethernet.xml.in index 77f130e1c..e9ae0acfe 100644 --- a/interface-definitions/interfaces-ethernet.xml.in +++ b/interface-definitions/interfaces-ethernet.xml.in @@ -31,7 +31,6 @@ #include #include - #include Duplex mode diff --git a/interface-definitions/interfaces-geneve.xml.in b/interface-definitions/interfaces-geneve.xml.in index b959c787d..f8e9909f8 100644 --- a/interface-definitions/interfaces-geneve.xml.in +++ b/interface-definitions/interfaces-geneve.xml.in @@ -23,7 +23,6 @@ #include #include #include - #include GENEVE tunnel parameters diff --git a/interface-definitions/interfaces-input.xml.in b/interface-definitions/interfaces-input.xml.in index d01c760f8..97502d954 100644 --- a/interface-definitions/interfaces-input.xml.in +++ b/interface-definitions/interfaces-input.xml.in @@ -19,7 +19,6 @@ #include #include - #include #include diff --git a/interface-definitions/interfaces-l2tpv3.xml.in b/interface-definitions/interfaces-l2tpv3.xml.in index bde68dd5a..0ebc3253d 100644 --- a/interface-definitions/interfaces-l2tpv3.xml.in +++ b/interface-definitions/interfaces-l2tpv3.xml.in @@ -32,7 +32,6 @@ 5000 #include - #include Encapsulation type diff --git a/interface-definitions/interfaces-macsec.xml.in b/interface-definitions/interfaces-macsec.xml.in index 5c9f4cd76..441236ec2 100644 --- a/interface-definitions/interfaces-macsec.xml.in +++ b/interface-definitions/interfaces-macsec.xml.in @@ -21,7 +21,6 @@ #include #include #include - #include #include diff --git a/interface-definitions/interfaces-openvpn.xml.in b/interface-definitions/interfaces-openvpn.xml.in index 3876e31da..7cfb9ee7a 100644 --- a/interface-definitions/interfaces-openvpn.xml.in +++ b/interface-definitions/interfaces-openvpn.xml.in @@ -34,7 +34,6 @@ #include - #include OpenVPN interface device-type diff --git a/interface-definitions/interfaces-pppoe.xml.in b/interface-definitions/interfaces-pppoe.xml.in index 84f76a7ee..719060fa9 100644 --- a/interface-definitions/interfaces-pppoe.xml.in +++ b/interface-definitions/interfaces-pppoe.xml.in @@ -19,7 +19,6 @@ #include #include #include - #include #include #include #include diff --git a/interface-definitions/interfaces-pseudo-ethernet.xml.in b/interface-definitions/interfaces-pseudo-ethernet.xml.in index 4eb9bf111..2fe07ffd5 100644 --- a/interface-definitions/interfaces-pseudo-ethernet.xml.in +++ b/interface-definitions/interfaces-pseudo-ethernet.xml.in @@ -28,7 +28,6 @@ #include #include #include - #include Receive mode (default: private) diff --git a/interface-definitions/interfaces-tunnel.xml.in b/interface-definitions/interfaces-tunnel.xml.in index fe49d337a..333a5b178 100644 --- a/interface-definitions/interfaces-tunnel.xml.in +++ b/interface-definitions/interfaces-tunnel.xml.in @@ -29,7 +29,6 @@ #include #include #include - #include 6rd network prefix diff --git a/interface-definitions/interfaces-vti.xml.in b/interface-definitions/interfaces-vti.xml.in index eeaea0dc3..11f001dc0 100644 --- a/interface-definitions/interfaces-vti.xml.in +++ b/interface-definitions/interfaces-vti.xml.in @@ -25,7 +25,6 @@ #include #include #include - #include diff --git a/interface-definitions/interfaces-vxlan.xml.in b/interface-definitions/interfaces-vxlan.xml.in index 4902ff36d..331f930d3 100644 --- a/interface-definitions/interfaces-vxlan.xml.in +++ b/interface-definitions/interfaces-vxlan.xml.in @@ -54,7 +54,6 @@ #include #include #include - #include 1450 diff --git a/interface-definitions/interfaces-wireguard.xml.in b/interface-definitions/interfaces-wireguard.xml.in index 23f50d146..35e223588 100644 --- a/interface-definitions/interfaces-wireguard.xml.in +++ b/interface-definitions/interfaces-wireguard.xml.in @@ -21,7 +21,6 @@ #include #include #include - #include #include 1420 diff --git a/interface-definitions/interfaces-wireless.xml.in b/interface-definitions/interfaces-wireless.xml.in index 9e7fc29bc..5271df624 100644 --- a/interface-definitions/interfaces-wireless.xml.in +++ b/interface-definitions/interfaces-wireless.xml.in @@ -20,7 +20,6 @@ #include - #include HT and VHT capabilities for your card diff --git a/interface-definitions/interfaces-wwan.xml.in b/interface-definitions/interfaces-wwan.xml.in index b0b8367dc..758784540 100644 --- a/interface-definitions/interfaces-wwan.xml.in +++ b/interface-definitions/interfaces-wwan.xml.in @@ -39,7 +39,6 @@ #include #include #include - #include #include #include diff --git a/interface-definitions/policy-route.xml.in b/interface-definitions/policy-route.xml.in index 44b96c2e6..48a5bf7d1 100644 --- a/interface-definitions/policy-route.xml.in +++ b/interface-definitions/policy-route.xml.in @@ -12,6 +12,7 @@ #include + #include #include @@ -65,6 +66,7 @@ #include + #include #include diff --git a/smoketest/scripts/cli/test_policy_route.py b/smoketest/scripts/cli/test_policy_route.py index 046e385bb..11b3c678e 100755 --- a/smoketest/scripts/cli/test_policy_route.py +++ b/smoketest/scripts/cli/test_policy_route.py @@ -42,18 +42,25 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): super(TestPolicyRoute, cls).tearDownClass() def tearDown(self): - self.cli_delete(['interfaces', 'ethernet', interface, 'policy']) self.cli_delete(['policy', 'route']) self.cli_delete(['policy', 'route6']) self.cli_commit() + # Verify nftables cleanup nftables_search = [ ['set N_smoketest_network'], ['set N_smoketest_network1'], ['chain VYOS_PBR_smoketest'] ] - self.verify_nftables(nftables_search, 'ip mangle', inverse=True) + self.verify_nftables(nftables_search, 'ip vyos_mangle', inverse=True) + + # Verify ip rule cleanup + ip_rule_search = [ + ['fwmark ' + hex(table_mark_offset - int(table_id)), 'lookup ' + table_id] + ] + + self.verify_rules(ip_rule_search, inverse=True) def verify_nftables(self, nftables_search, table, inverse=False): nftables_output = cmd(f'sudo nft list table {table}') @@ -66,6 +73,17 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): break self.assertTrue(not matched if inverse else matched, msg=search) + def verify_rules(self, rules_search, inverse=False): + rule_output = cmd('ip rule show') + + for search in rules_search: + matched = False + for line in rule_output.split("\n"): + if all(item in line for item in search): + matched = True + break + self.assertTrue(not matched if inverse else matched, msg=search) + def test_pbr_group(self): self.cli_set(['firewall', 'group', 'network-group', 'smoketest_network', 'network', '172.16.99.0/24']) self.cli_set(['firewall', 'group', 'network-group', 'smoketest_network1', 'network', '172.16.101.0/24']) @@ -74,8 +92,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'source', 'group', 'network-group', 'smoketest_network']) self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'destination', 'group', 'network-group', 'smoketest_network1']) self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'set', 'mark', mark]) - - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route', 'smoketest']) + self.cli_set(['policy', 'route', 'smoketest', 'interface', interface]) self.cli_commit() @@ -84,7 +101,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['ip daddr @N_smoketest_network1', 'ip saddr @N_smoketest_network'], ] - self.verify_nftables(nftables_search, 'ip mangle') + self.verify_nftables(nftables_search, 'ip vyos_mangle') self.cli_delete(['firewall']) @@ -92,8 +109,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'source', 'address', '172.16.20.10']) self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'destination', 'address', '172.16.10.10']) self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'set', 'mark', mark]) - - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route', 'smoketest']) + self.cli_set(['policy', 'route', 'smoketest', 'interface', interface]) self.cli_commit() @@ -104,7 +120,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['ip daddr 172.16.10.10', 'ip saddr 172.16.20.10', 'meta mark set ' + mark_hex], ] - self.verify_nftables(nftables_search, 'ip mangle') + self.verify_nftables(nftables_search, 'ip vyos_mangle') def test_pbr_table(self): self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'protocol', 'tcp']) @@ -116,8 +132,8 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '1', 'destination', 'port', '8888']) self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '1', 'set', 'table', table_id]) - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route', 'smoketest']) - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route6', 'smoketest6']) + self.cli_set(['policy', 'route', 'smoketest', 'interface', interface]) + self.cli_set(['policy', 'route6', 'smoketest6', 'interface', interface]) self.cli_commit() @@ -130,7 +146,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['tcp flags syn / syn,ack', 'tcp dport 8888', 'meta mark set ' + mark_hex] ] - self.verify_nftables(nftables_search, 'ip mangle') + self.verify_nftables(nftables_search, 'ip vyos_mangle') # IPv6 @@ -139,7 +155,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['meta l4proto { tcp, udp }', 'th dport 8888', 'meta mark set ' + mark_hex] ] - self.verify_nftables(nftables6_search, 'ip6 mangle') + self.verify_nftables(nftables6_search, 'ip6 vyos_mangle') # IP rule fwmark -> table @@ -147,15 +163,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['fwmark ' + hex(table_mark_offset - int(table_id)), 'lookup ' + table_id] ] - ip_rule_output = cmd('ip rule show') - - for search in ip_rule_search: - matched = False - for line in ip_rule_output.split("\n"): - if all(item in line for item in search): - matched = True - break - self.assertTrue(matched) + self.verify_rules(ip_rule_search) def test_pbr_matching_criteria(self): @@ -203,8 +211,8 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '5', 'dscp-exclude', '14-19']) self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '5', 'set', 'table', table_id]) - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route', 'smoketest']) - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route6', 'smoketest6']) + self.cli_set(['policy', 'route', 'smoketest', 'interface', interface]) + self.cli_set(['policy', 'route6', 'smoketest6', 'interface', interface]) self.cli_commit() @@ -220,7 +228,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['ip dscp { 0x29, 0x39-0x3b }', 'meta mark set ' + mark_hex] ] - self.verify_nftables(nftables_search, 'ip mangle') + self.verify_nftables(nftables_search, 'ip vyos_mangle') # IPv6 nftables6_search = [ @@ -232,7 +240,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['ip6 dscp != { 0x0e-0x13, 0x3d }', 'meta mark set ' + mark_hex] ] - self.verify_nftables(nftables6_search, 'ip6 mangle') + self.verify_nftables(nftables6_search, 'ip6 vyos_mangle') if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/policy-route-interface.py b/src/conf_mode/policy-route-interface.py deleted file mode 100755 index 58c5fd93d..000000000 --- a/src/conf_mode/policy-route-interface.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2021 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import re - -from sys import argv -from sys import exit - -from vyos.config import Config -from vyos.ifconfig import Section -from vyos.template import render -from vyos.util import cmd -from vyos.util import run -from vyos import ConfigError -from vyos import airbag -airbag.enable() - -def get_config(config=None): - if config: - conf = config - else: - conf = Config() - - ifname = argv[1] - ifpath = Section.get_config_path(ifname) - if_policy_path = f'interfaces {ifpath} policy' - - if_policy = conf.get_config_dict(if_policy_path, key_mangling=('-', '_'), get_first_key=True, - no_tag_node_value_mangle=True) - - if_policy['ifname'] = ifname - if_policy['policy'] = conf.get_config_dict(['policy'], key_mangling=('-', '_'), get_first_key=True, - no_tag_node_value_mangle=True) - - return if_policy - -def verify_chain(table, chain): - # Verify policy route applied - code = run(f'nft list chain {table} {chain}') - return code == 0 - -def verify(if_policy): - # bail out early - looks like removal from running config - if not if_policy: - return None - - for route in ['route', 'route6']: - if route in if_policy: - if route not in if_policy['policy']: - raise ConfigError('Policy route not configured') - - route_name = if_policy[route] - - if route_name not in if_policy['policy'][route]: - raise ConfigError(f'Invalid policy route name "{name}"') - - nft_prefix = 'VYOS_PBR6_' if route == 'route6' else 'VYOS_PBR_' - nft_table = 'ip6 mangle' if route == 'route6' else 'ip mangle' - - if not verify_chain(nft_table, nft_prefix + route_name): - raise ConfigError('Policy route did not apply') - - return None - -def generate(if_policy): - return None - -def cleanup_rule(table, chain, ifname, new_name=None): - results = cmd(f'nft -a list chain {table} {chain}').split("\n") - retval = None - for line in results: - if f'ifname "{ifname}"' in line: - if new_name and f'jump {new_name}' in line: - # new_name is used to clear rules for any previously referenced chains - # returns true when rule exists and doesn't need to be created - retval = True - continue - - handle_search = re.search('handle (\d+)', line) - if handle_search: - cmd(f'nft delete rule {table} {chain} handle {handle_search[1]}') - return retval - -def apply(if_policy): - ifname = if_policy['ifname'] - - route_chain = 'VYOS_PBR_PREROUTING' - ipv6_route_chain = 'VYOS_PBR6_PREROUTING' - - if 'route' in if_policy: - name = 'VYOS_PBR_' + if_policy['route'] - rule_exists = cleanup_rule('ip mangle', route_chain, ifname, name) - - if not rule_exists: - cmd(f'nft insert rule ip mangle {route_chain} iifname {ifname} counter jump {name}') - else: - cleanup_rule('ip mangle', route_chain, ifname) - - if 'route6' in if_policy: - name = 'VYOS_PBR6_' + if_policy['route6'] - rule_exists = cleanup_rule('ip6 mangle', ipv6_route_chain, ifname, name) - - if not rule_exists: - cmd(f'nft insert rule ip6 mangle {ipv6_route_chain} iifname {ifname} counter jump {name}') - else: - cleanup_rule('ip6 mangle', ipv6_route_chain, ifname) - - return None - -if __name__ == '__main__': - try: - c = get_config() - verify(c) - generate(c) - apply(c) - except ConfigError as e: - print(e) - exit(1) diff --git a/src/conf_mode/policy-route.py b/src/conf_mode/policy-route.py index 00539b9c7..1d016695e 100755 --- a/src/conf_mode/policy-route.py +++ b/src/conf_mode/policy-route.py @@ -15,7 +15,6 @@ # along with this program. If not, see . import os -import re from json import loads from sys import exit @@ -25,7 +24,6 @@ from vyos.config import Config from vyos.template import render from vyos.util import cmd from vyos.util import dict_search_args -from vyos.util import dict_search_recursive from vyos.util import run from vyos import ConfigError from vyos import airbag @@ -34,48 +32,13 @@ airbag.enable() mark_offset = 0x7FFFFFFF nftables_conf = '/run/nftables_policy.conf' -ROUTE_PREFIX = 'VYOS_PBR_' -ROUTE6_PREFIX = 'VYOS_PBR6_' - -preserve_chains = [ - 'VYOS_PBR_PREROUTING', - 'VYOS_PBR_POSTROUTING', - 'VYOS_PBR6_PREROUTING', - 'VYOS_PBR6_POSTROUTING' -] - valid_groups = [ 'address_group', + 'domain_group', 'network_group', 'port_group' ] -group_set_prefix = { - 'A_': 'address_group', - 'A6_': 'ipv6_address_group', -# 'D_': 'domain_group', - 'M_': 'mac_group', - 'N_': 'network_group', - 'N6_': 'ipv6_network_group', - 'P_': 'port_group' -} - -def get_policy_interfaces(conf): - out = {} - interfaces = conf.get_config_dict(['interfaces'], key_mangling=('-', '_'), get_first_key=True, - no_tag_node_value_mangle=True) - def find_interfaces(iftype_conf, output={}, prefix=''): - for ifname, if_conf in iftype_conf.items(): - if 'policy' in if_conf: - output[prefix + ifname] = if_conf['policy'] - for vif in ['vif', 'vif_s', 'vif_c']: - if vif in if_conf: - output.update(find_interfaces(if_conf[vif], output, f'{prefix}{ifname}.')) - return output - for iftype, iftype_conf in interfaces.items(): - out.update(find_interfaces(iftype_conf)) - return out - def get_config(config=None): if config: conf = config @@ -88,7 +51,6 @@ def get_config(config=None): policy['firewall_group'] = conf.get_config_dict(['firewall', 'group'], key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) - policy['interfaces'] = get_policy_interfaces(conf) return policy @@ -132,8 +94,8 @@ def verify_rule(policy, name, rule_conf, ipv6, rule_id): side_conf = rule_conf[side] if 'group' in side_conf: - if {'address_group', 'network_group'} <= set(side_conf['group']): - raise ConfigError('Only one address-group or network-group can be specified') + if len({'address_group', 'domain_group', 'network_group'} & set(side_conf['group'])) > 1: + raise ConfigError('Only one address-group, domain-group or network-group can be specified') for group in valid_groups: if group in side_conf['group']: @@ -168,73 +130,11 @@ def verify(policy): for rule_id, rule_conf in pol_conf['rule'].items(): verify_rule(policy, name, rule_conf, ipv6, rule_id) - for ifname, if_policy in policy['interfaces'].items(): - name = dict_search_args(if_policy, 'route') - ipv6_name = dict_search_args(if_policy, 'route6') - - if name and not dict_search_args(policy, 'route', name): - raise ConfigError(f'Policy route "{name}" is still referenced on interface {ifname}') - - if ipv6_name and not dict_search_args(policy, 'route6', ipv6_name): - raise ConfigError(f'Policy route6 "{ipv6_name}" is still referenced on interface {ifname}') - return None -def cleanup_commands(policy): - commands = [] - commands_chains = [] - commands_sets = [] - for table in ['ip mangle', 'ip6 mangle']: - route_node = 'route' if table == 'ip mangle' else 'route6' - chain_prefix = ROUTE_PREFIX if table == 'ip mangle' else ROUTE6_PREFIX - - json_str = cmd(f'nft -t -j list table {table}') - obj = loads(json_str) - if 'nftables' not in obj: - continue - for item in obj['nftables']: - if 'chain' in item: - chain = item['chain']['name'] - if chain in preserve_chains or not chain.startswith("VYOS_PBR"): - continue - - if dict_search_args(policy, route_node, chain.replace(chain_prefix, "", 1)) != None: - commands.append(f'flush chain {table} {chain}') - else: - commands_chains.append(f'delete chain {table} {chain}') - - if 'rule' in item: - rule = item['rule'] - chain = rule['chain'] - handle = rule['handle'] - - if chain not in preserve_chains: - continue - - target, _ = next(dict_search_recursive(rule['expr'], 'target')) - - if target.startswith(chain_prefix): - if dict_search_args(policy, route_node, target.replace(chain_prefix, "", 1)) == None: - commands.append(f'delete rule {table} {chain} handle {handle}') - - if 'set' in item: - set_name = item['set']['name'] - - for prefix, group_type in group_set_prefix.items(): - if set_name.startswith(prefix): - group_name = set_name.replace(prefix, "", 1) - if dict_search_args(policy, 'firewall_group', group_type, group_name) != None: - commands_sets.append(f'flush set {table} {set_name}') - else: - commands_sets.append(f'delete set {table} {set_name}') - - return commands + commands_chains + commands_sets - def generate(policy): if not os.path.exists(nftables_conf): policy['first_install'] = True - else: - policy['cleanup_commands'] = cleanup_commands(policy) render(nftables_conf, 'firewall/nftables-policy.j2', policy) return None diff --git a/src/helpers/vyos-domain-resolver.py b/src/helpers/vyos-domain-resolver.py index 035c208b2..e31d9238e 100755 --- a/src/helpers/vyos-domain-resolver.py +++ b/src/helpers/vyos-domain-resolver.py @@ -35,13 +35,13 @@ cache = False domain_state = {} ipv4_tables = { - 'ip mangle', + 'ip vyos_mangle', 'ip vyos_filter', 'ip vyos_nat' } ipv6_tables = { - 'ip6 mangle', + 'ip6 vyos_mangle', 'ip6 vyos_filter' } diff --git a/src/migration-scripts/policy/4-to-5 b/src/migration-scripts/policy/4-to-5 new file mode 100755 index 000000000..33c9e6ade --- /dev/null +++ b/src/migration-scripts/policy/4-to-5 @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# T2199: Migrate interface policy nodes to policy route interface + +import re + +from sys import argv +from sys import exit + +from vyos.configtree import ConfigTree +from vyos.ifconfig import Section + +if (len(argv) < 1): + print("Must specify file name!") + exit(1) + +file_name = argv[1] + +with open(file_name, 'r') as f: + config_file = f.read() + +base4 = ['policy', 'route'] +base6 = ['policy', 'route6'] +config = ConfigTree(config_file) + +if not config.exists(base4) and not config.exists(base6): + # Nothing to do + exit(0) + +def migrate_interface(config, iftype, ifname, vif=None, vifs=None, vifc=None): + if_path = ['interfaces', iftype, ifname] + ifname_full = ifname + + if vif: + if_path += ['vif', vif] + ifname_full = f'{ifname}.{vif}' + elif vifs: + if_path += ['vif-s', vifs] + ifname_full = f'{ifname}.{vifs}' + if vifc: + if_path += ['vif-c', vifc] + ifname_full = f'{ifname}.{vifs}.{vifc}' + + if not config.exists(if_path + ['policy']): + return + + if config.exists(if_path + ['policy', 'route']): + route_name = config.return_value(if_path + ['policy', 'route']) + config.set(base4 + [route_name, 'interface'], value=ifname_full, replace=False) + + if config.exists(if_path + ['policy', 'route6']): + route_name = config.return_value(if_path + ['policy', 'route6']) + config.set(base6 + [route_name, 'interface'], value=ifname_full, replace=False) + + config.delete(if_path + ['policy']) + +for iftype in config.list_nodes(['interfaces']): + for ifname in config.list_nodes(['interfaces', iftype]): + migrate_interface(config, iftype, ifname) + + if config.exists(['interfaces', iftype, ifname, 'vif']): + for vif in config.list_nodes(['interfaces', iftype, ifname, 'vif']): + migrate_interface(config, iftype, ifname, vif=vif) + + if config.exists(['interfaces', iftype, ifname, 'vif-s']): + for vifs in config.list_nodes(['interfaces', iftype, ifname, 'vif-s']): + migrate_interface(config, iftype, ifname, vifs=vifs) + + if config.exists(['interfaces', iftype, ifname, 'vif-s', vifs, 'vif-c']): + for vifc in config.list_nodes(['interfaces', iftype, ifname, 'vif-s', vifs, 'vif-c']): + migrate_interface(config, iftype, ifname, vifs=vifs, vifc=vifc) + +try: + with open(file_name, 'w') as f: + f.write(config.to_string()) +except OSError as e: + print("Failed to save the modified config: {}".format(e)) + exit(1) diff --git a/src/op_mode/policy_route.py b/src/op_mode/policy_route.py index 5be40082f..5953786f3 100755 --- a/src/op_mode/policy_route.py +++ b/src/op_mode/policy_route.py @@ -22,53 +22,13 @@ from vyos.config import Config from vyos.util import cmd from vyos.util import dict_search_args -def get_policy_interfaces(conf, policy, name=None, ipv6=False): - interfaces = conf.get_config_dict(['interfaces'], key_mangling=('-', '_'), - get_first_key=True, no_tag_node_value_mangle=True) - - routes = ['route', 'route6'] - - def parse_if(ifname, if_conf): - if 'policy' in if_conf: - for route in routes: - if route in if_conf['policy']: - route_name = if_conf['policy'][route] - name_str = f'({ifname},{route})' - - if not name: - policy[route][route_name]['interface'].append(name_str) - elif not ipv6 and name == route_name: - policy['interface'].append(name_str) - - for iftype in ['vif', 'vif_s', 'vif_c']: - if iftype in if_conf: - for vifname, vif_conf in if_conf[iftype].items(): - parse_if(f'{ifname}.{vifname}', vif_conf) - - for iftype, iftype_conf in interfaces.items(): - for ifname, if_conf in iftype_conf.items(): - parse_if(ifname, if_conf) - -def get_config_policy(conf, name=None, ipv6=False, interfaces=True): +def get_config_policy(conf, name=None, ipv6=False): config_path = ['policy'] if name: config_path += ['route6' if ipv6 else 'route', name] policy = conf.get_config_dict(config_path, key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) - if policy and interfaces: - if name: - policy['interface'] = [] - else: - if 'route' in policy: - for route_name, route_conf in policy['route'].items(): - route_conf['interface'] = [] - - if 'route6' in policy: - for route_name, route_conf in policy['route6'].items(): - route_conf['interface'] = [] - - get_policy_interfaces(conf, policy, name, ipv6) return policy -- cgit v1.2.3 From 00ec496877453cc37ceec0633821a47f128d9f4f Mon Sep 17 00:00:00 2001 From: Yuxiang Zhu Date: Mon, 14 Nov 2022 10:23:46 +0800 Subject: T4815: Fix various name server config issues 1. When a PPPoE session is connected, `pppd` will update `/etc/resolv.conf` regardless of `system name-server` option unless `no-peer-dns` is set. This is because `pppd` vendors scripts `/etc/ppp/ip-up.d/0000usepeerdns` and `/etc/ppp/ip-down.d/0000usepeerdns`, which updates `/etc/resolv.conf` on PPPoE connection and reverts the change on disconnection. This PR removes those scripts and adds custom scripts to update name server entries through `vyos-hostsd` instead. 2. There is a typo in `/etc/dhcp/dhclient-enter-hooks.d/04-vyos-resolvconf, which misspells variable name `new_dhcp6_name_servers` as `new_dhcpv6_name_servers`. This causes IPv6 name server entries in `vyos-hostsd` not updated when dhclient receives nameservers from DHCPv6. 3. Regular expressions in scripts under `/etc/dhcp/dhclient-enter-hooks.d` and `/etc/dhcp/dhclient-exit-hooks.d/` are not enclosed in `^$`, so those IPv4 related branches (like `BOUND`) could be mistakenly executed when an IPv6 reason (like `BOUND6`) is given. --- debian/vyos-1x.postinst | 3 ++- .../dhcp/dhclient-enter-hooks.d/04-vyos-resolvconf | 4 ++-- src/etc/dhcp/dhclient-exit-hooks.d/01-vyos-cleanup | 4 ++-- .../dhclient-exit-hooks.d/vyatta-dhclient-hook | 6 +++--- .../ip-down.d/98-vyos-pppoe-cleanup-nameservers | 15 ++++++++++++++ .../ppp/ip-up.d/98-vyos-pppoe-setup-nameservers | 24 ++++++++++++++++++++++ 6 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 src/etc/ppp/ip-down.d/98-vyos-pppoe-cleanup-nameservers create mode 100644 src/etc/ppp/ip-up.d/98-vyos-pppoe-setup-nameservers (limited to 'src') diff --git a/debian/vyos-1x.postinst b/debian/vyos-1x.postinst index d92fd8233..d5f5cbbc7 100644 --- a/debian/vyos-1x.postinst +++ b/debian/vyos-1x.postinst @@ -103,7 +103,8 @@ DELETE="/etc/logrotate.d/conntrackd.distrib /etc/init.d/conntrackd /etc/default/ /etc/default/pmacctd /etc/pmacct /etc/networks_list /etc/networks_whitelist /etc/fastnetmon.conf /etc/ntp.conf /etc/default/ssh - /etc/powerdns /etc/default/pdns-recursor" + /etc/powerdns /etc/default/pdns-recursor + /etc/ppp/ip-up.d/0000usepeerdns /etc/ppp/ip-down.d/0000usepeerdns" for tmp in $DELETE; do if [ -e ${tmp} ]; then rm -rf ${tmp} diff --git a/src/etc/dhcp/dhclient-enter-hooks.d/04-vyos-resolvconf b/src/etc/dhcp/dhclient-enter-hooks.d/04-vyos-resolvconf index b1902b585..518abeaec 100644 --- a/src/etc/dhcp/dhclient-enter-hooks.d/04-vyos-resolvconf +++ b/src/etc/dhcp/dhclient-enter-hooks.d/04-vyos-resolvconf @@ -33,8 +33,8 @@ if /usr/bin/systemctl -q is-active vyos-hostsd; then if [ -n "$new_dhcp6_name_servers" ]; then logmsg info "Deleting nameservers with tag \"dhcpv6-$interface\" via vyos-hostsd-client" $hostsd_client --delete-name-servers --tag "dhcpv6-$interface" - logmsg info "Adding nameservers \"$new_dhcpv6_name_servers\" with tag \"dhcpv6-$interface\" via vyos-hostsd-client" - $hostsd_client --add-name-servers $new_dhcpv6_name_servers --tag "dhcpv6-$interface" + logmsg info "Adding nameservers \"$new_dhcp6_name_servers\" with tag \"dhcpv6-$interface\" via vyos-hostsd-client" + $hostsd_client --add-name-servers $new_dhcp6_name_servers --tag "dhcpv6-$interface" hostsd_changes=y fi diff --git a/src/etc/dhcp/dhclient-exit-hooks.d/01-vyos-cleanup b/src/etc/dhcp/dhclient-exit-hooks.d/01-vyos-cleanup index ad6a1d5eb..da1bda137 100644 --- a/src/etc/dhcp/dhclient-exit-hooks.d/01-vyos-cleanup +++ b/src/etc/dhcp/dhclient-exit-hooks.d/01-vyos-cleanup @@ -8,7 +8,7 @@ hostsd_changes= /usr/bin/systemctl -q is-active vyos-hostsd hostsd_status=$? -if [[ $reason =~ (EXPIRE|FAIL|RELEASE|STOP) ]]; then +if [[ $reason =~ ^(EXPIRE|FAIL|RELEASE|STOP)$ ]]; then if [[ $hostsd_status -eq 0 ]]; then # delete search domains and nameservers via vyos-hostsd logmsg info "Deleting search domains with tag \"dhcp-$interface\" via vyos-hostsd-client" @@ -96,7 +96,7 @@ if [[ $reason =~ (EXPIRE|FAIL|RELEASE|STOP) ]]; then fi fi -if [[ $reason =~ (EXPIRE6|RELEASE6|STOP6) ]]; then +if [[ $reason =~ ^(EXPIRE6|RELEASE6|STOP6)$ ]]; then if [[ $hostsd_status -eq 0 ]]; then # delete search domains and nameservers via vyos-hostsd logmsg info "Deleting search domains with tag \"dhcpv6-$interface\" via vyos-hostsd-client" diff --git a/src/etc/dhcp/dhclient-exit-hooks.d/vyatta-dhclient-hook b/src/etc/dhcp/dhclient-exit-hooks.d/vyatta-dhclient-hook index eeb8b0782..49bb18372 100644 --- a/src/etc/dhcp/dhclient-exit-hooks.d/vyatta-dhclient-hook +++ b/src/etc/dhcp/dhclient-exit-hooks.d/vyatta-dhclient-hook @@ -8,12 +8,12 @@ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -# +# # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. -# +# # This code was originally developed by Vyatta, Inc. # Portions created by Vyatta are Copyright (C) 2006, 2007, 2008 Vyatta, Inc. # All Rights Reserved. @@ -23,7 +23,7 @@ RUN="yes" proto="" -if [[ $reason =~ (REBOOT6|INIT6|EXPIRE6|RELEASE6|STOP6|INFORM6|BOUND6|REBIND6|DELEGATED6) ]]; then +if [[ $reason =~ ^(REBOOT6|INIT6|EXPIRE6|RELEASE6|STOP6|INFORM6|BOUND6|REBIND6|DELEGATED6)$ ]]; then proto="v6" fi diff --git a/src/etc/ppp/ip-down.d/98-vyos-pppoe-cleanup-nameservers b/src/etc/ppp/ip-down.d/98-vyos-pppoe-cleanup-nameservers new file mode 100644 index 000000000..222c75f21 --- /dev/null +++ b/src/etc/ppp/ip-down.d/98-vyos-pppoe-cleanup-nameservers @@ -0,0 +1,15 @@ +#!/bin/bash +### Autogenerated by interfaces-pppoe.py ### + +interface=$6 +if [ -z "$interface" ]; then + exit +fi + +if ! /usr/bin/systemctl -q is-active vyos-hostsd; then + exit # vyos-hostsd is not running +fi + +hostsd_client="/usr/bin/vyos-hostsd-client" +$hostsd_client --delete-name-servers --tag "dhcp-$interface" +$hostsd_client --apply diff --git a/src/etc/ppp/ip-up.d/98-vyos-pppoe-setup-nameservers b/src/etc/ppp/ip-up.d/98-vyos-pppoe-setup-nameservers new file mode 100644 index 000000000..0fcedbedc --- /dev/null +++ b/src/etc/ppp/ip-up.d/98-vyos-pppoe-setup-nameservers @@ -0,0 +1,24 @@ +#!/bin/bash +### Autogenerated by interfaces-pppoe.py ### + +interface=$6 +if [ -z "$interface" ]; then + exit +fi + +if ! /usr/bin/systemctl -q is-active vyos-hostsd; then + exit # vyos-hostsd is not running +fi + +hostsd_client="/usr/bin/vyos-hostsd-client" + +$hostsd_client --delete-name-servers --tag "dhcp-$interface" + +if [ "$USEPEERDNS" ] && [ -n "$DNS1" ]; then +$hostsd_client --add-name-servers "$DNS1" --tag "dhcp-$interface" +fi +if [ "$USEPEERDNS" ] && [ -n "$DNS2" ]; then +$hostsd_client --add-name-servers "$DNS2" --tag "dhcp-$interface" +fi + +$hostsd_client --apply -- cgit v1.2.3 From b8a4442c898727eaf7def8c42ee850fbb9dd60be Mon Sep 17 00:00:00 2001 From: Yuxiang Zhu Date: Tue, 15 Nov 2022 13:52:14 +0800 Subject: T4815: ip-up/down scripts needs the executable bit ip-up/down scripts added in https://github.com/vyos/vyos-1x/pull/1656 need the executable bit. --- src/etc/ppp/ip-down.d/98-vyos-pppoe-cleanup-nameservers | 0 src/etc/ppp/ip-up.d/98-vyos-pppoe-setup-nameservers | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 src/etc/ppp/ip-down.d/98-vyos-pppoe-cleanup-nameservers mode change 100644 => 100755 src/etc/ppp/ip-up.d/98-vyos-pppoe-setup-nameservers (limited to 'src') diff --git a/src/etc/ppp/ip-down.d/98-vyos-pppoe-cleanup-nameservers b/src/etc/ppp/ip-down.d/98-vyos-pppoe-cleanup-nameservers old mode 100644 new mode 100755 diff --git a/src/etc/ppp/ip-up.d/98-vyos-pppoe-setup-nameservers b/src/etc/ppp/ip-up.d/98-vyos-pppoe-setup-nameservers old mode 100644 new mode 100755 -- cgit v1.2.3 From 2c4b6b74317240eedbbae9d2816f86eda464ae68 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 14 Nov 2022 16:54:24 +0000 Subject: T4812: Add op-mode Show vpn ipsec connections Add op-mode CLI "show vpn ipsec connections" Add the ability to show all configured connections/tunnels and their states. Ability to get --raw data --- op-mode-definitions/vpn-ipsec.xml.in | 6 + src/op_mode/ipsec.py | 283 +++++++++++++++++++++++++++++++++++ 2 files changed, 289 insertions(+) (limited to 'src') diff --git a/op-mode-definitions/vpn-ipsec.xml.in b/op-mode-definitions/vpn-ipsec.xml.in index f1af65fcb..803ce4cc2 100644 --- a/op-mode-definitions/vpn-ipsec.xml.in +++ b/op-mode-definitions/vpn-ipsec.xml.in @@ -137,6 +137,12 @@ Show Internet Protocol Security (IPsec) information + + + Show VPN connections + + sudo ${vyos_op_scripts_dir}/ipsec.py show_connections + Show the in-kernel crypto policies diff --git a/src/op_mode/ipsec.py b/src/op_mode/ipsec.py index aaa0cec5a..75579a447 100755 --- a/src/op_mode/ipsec.py +++ b/src/op_mode/ipsec.py @@ -135,6 +135,272 @@ def _get_formatted_output_sas(sas): return output +# Connections block +def _get_vici_connections(): + from vici import Session as vici_session + + try: + session = vici_session() + except Exception: + raise vyos.opmode.UnconfiguredSubsystem("IPsec not initialized") + connections = list(session.list_conns()) + return connections + + +def _get_convert_data_connections(): + get_connections = _get_vici_connections() + connections = convert_data(get_connections) + return connections + + +def _get_parent_sa_proposal(connection_name: str, data: list) -> dict: + """Get parent SA proposals by connection name + if connections not in the 'down' state + + Args: + connection_name (str): Connection name + data (list): List of current SAs from vici + + Returns: + str: Parent SA connection proposal + AES_CBC/256/HMAC_SHA2_256_128/MODP_1024 + """ + if not data: + return {} + for sa in data: + # check if parent SA exist + if connection_name not in sa.keys(): + return {} + if 'encr-alg' in sa[connection_name]: + encr_alg = sa.get(connection_name, '').get('encr-alg') + cipher = encr_alg.split('_')[0] + mode = encr_alg.split('_')[1] + encr_keysize = sa.get(connection_name, '').get('encr-keysize') + integ_alg = sa.get(connection_name, '').get('integ-alg') + # prf_alg = sa.get(connection_name, '').get('prf-alg') + dh_group = sa.get(connection_name, '').get('dh-group') + proposal = { + 'cipher': cipher, + 'mode': mode, + 'key_size': encr_keysize, + 'hash': integ_alg, + 'dh': dh_group + } + return proposal + return {} + + +def _get_parent_sa_state(connection_name: str, data: list) -> str: + """Get parent SA state by connection name + + Args: + connection_name (str): Connection name + data (list): List of current SAs from vici + + Returns: + Parent SA connection state + """ + if not data: + return 'down' + for sa in data: + # check if parent SA exist + if connection_name not in sa.keys(): + return 'down' + if sa[connection_name]['state'].lower() == 'established': + return 'up' + else: + return 'down' + + +def _get_child_sa_state(connection_name: str, tunnel_name: str, + data: list) -> str: + """Get child SA state by connection and tunnel name + + Args: + connection_name (str): Connection name + tunnel_name (str): Tunnel name + data (list): List of current SAs from vici + + Returns: + str: `up` if child SA state is 'installed' otherwise `down` + """ + if not data: + return 'down' + for sa in data: + # check if parent SA exist + if connection_name not in sa.keys(): + return 'down' + child_sas = sa[connection_name]['child-sas'] + # Get all child SA states + # there can be multiple SAs per tunnel + child_sa_states = [ + v['state'] for k, v in child_sas.items() if v['name'] == tunnel_name + ] + return 'up' if 'INSTALLED' in child_sa_states else 'down' + + +def _get_child_sa_info(connection_name: str, tunnel_name: str, + data: list) -> dict: + """Get child SA installed info by connection and tunnel name + + Args: + connection_name (str): Connection name + tunnel_name (str): Tunnel name + data (list): List of current SAs from vici + + Returns: + dict: Info of the child SA in the dictionary format + """ + for sa in data: + # check if parent SA exist + if connection_name not in sa.keys(): + return {} + child_sas = sa[connection_name]['child-sas'] + # Get all child SA data + # Skip temp SA name (first key), get only SA values as dict + # {'OFFICE-B-tunnel-0-46': {'name': 'OFFICE-B-tunnel-0'}...} + # i.e get all data after 'OFFICE-B-tunnel-0-46' + child_sa_info = [ + v for k, v in child_sas.items() if 'name' in v and + v['name'] == tunnel_name and v['state'] == 'INSTALLED' + ] + return child_sa_info[-1] if child_sa_info else {} + + +def _get_child_sa_proposal(child_sa_data: dict) -> dict: + if child_sa_data and 'encr-alg' in child_sa_data: + encr_alg = child_sa_data.get('encr-alg') + cipher = encr_alg.split('_')[0] + mode = encr_alg.split('_')[1] + key_size = child_sa_data.get('encr-keysize') + integ_alg = child_sa_data.get('integ-alg') + dh_group = child_sa_data.get('dh-group') + proposal = { + 'cipher': cipher, + 'mode': mode, + 'key_size': key_size, + 'hash': integ_alg, + 'dh': dh_group + } + return proposal + return {} + + +def _get_raw_data_connections(list_connections: list, list_sas: list) -> list: + """Get configured VPN IKE connections and IPsec states + + Args: + list_connections (list): List of configured connections from vici + list_sas (list): List of current SAs from vici + + Returns: + list: List and status of IKE/IPsec connections/tunnels + """ + base_dict = [] + for connections in list_connections: + base_list = {} + for connection, conn_conf in connections.items(): + base_list['ike_connection_name'] = connection + base_list['ike_connection_state'] = _get_parent_sa_state( + connection, list_sas) + base_list['ike_remote_address'] = conn_conf['remote_addrs'] + base_list['ike_proposal'] = _get_parent_sa_proposal( + connection, list_sas) + base_list['local_id'] = conn_conf.get('local-1', '').get('id') + base_list['remote_id'] = conn_conf.get('remote-1', '').get('id') + base_list['version'] = conn_conf.get('version', 'IKE') + base_list['children'] = [] + children = conn_conf['children'] + for tunnel, tun_options in children.items(): + state = _get_child_sa_state(connection, tunnel, list_sas) + local_ts = tun_options.get('local-ts') + remote_ts = tun_options.get('remote-ts') + dpd_action = tun_options.get('dpd_action') + close_action = tun_options.get('close_action') + sa_info = _get_child_sa_info(connection, tunnel, list_sas) + esp_proposal = _get_child_sa_proposal(sa_info) + base_list['children'].append({ + 'name': tunnel, + 'state': state, + 'local_ts': local_ts, + 'remote_ts': remote_ts, + 'dpd_action': dpd_action, + 'close_action': close_action, + 'sa': sa_info, + 'esp_proposal': esp_proposal + }) + base_dict.append(base_list) + return base_dict + + +def _get_raw_connections_summary(list_conn, list_sas): + import jmespath + data = _get_raw_data_connections(list_conn, list_sas) + match = '[*].children[]' + child = jmespath.search(match, data) + tunnels_down = len([k for k in child if k['state'] == 'down']) + tunnels_up = len([k for k in child if k['state'] == 'up']) + tun_dict = { + 'tunnels': child, + 'total': len(child), + 'down': tunnels_down, + 'up': tunnels_up + } + return tun_dict + + +def _get_formatted_output_conections(data): + from tabulate import tabulate + data_entries = '' + connections = [] + for entry in data: + tunnels = [] + ike_name = entry['ike_connection_name'] + ike_state = entry['ike_connection_state'] + conn_type = entry.get('version', 'IKE') + remote_addrs = ','.join(entry['ike_remote_address']) + local_ts, remote_ts = '-', '-' + local_id = entry['local_id'] + remote_id = entry['remote_id'] + proposal = '-' + if entry.get('ike_proposal'): + proposal = (f'{entry["ike_proposal"]["cipher"]}_' + f'{entry["ike_proposal"]["mode"]}/' + f'{entry["ike_proposal"]["key_size"]}/' + f'{entry["ike_proposal"]["hash"]}/' + f'{entry["ike_proposal"]["dh"]}') + connections.append([ + ike_name, ike_state, conn_type, remote_addrs, local_ts, remote_ts, + local_id, remote_id, proposal + ]) + for tun in entry['children']: + tun_name = tun.get('name') + tun_state = tun.get('state') + conn_type = 'IPsec' + local_ts = '\n'.join(tun.get('local_ts')) + remote_ts = '\n'.join(tun.get('remote_ts')) + proposal = '-' + if tun.get('esp_proposal'): + proposal = (f'{tun["esp_proposal"]["cipher"]}_' + f'{tun["esp_proposal"]["mode"]}/' + f'{tun["esp_proposal"]["key_size"]}/' + f'{tun["esp_proposal"]["hash"]}/' + f'{tun["esp_proposal"]["dh"]}') + connections.append([ + tun_name, tun_state, conn_type, remote_addrs, local_ts, + remote_ts, local_id, remote_id, proposal + ]) + connection_headers = [ + 'Connection', 'State', 'Type', 'Remote address', 'Local TS', + 'Remote TS', 'Local id', 'Remote id', 'Proposal' + ] + output = tabulate(connections, connection_headers, numalign='left') + return output + + +# Connections block end + + def get_peer_connections(peer, tunnel, return_all = False): search = rf'^[\s]*({peer}-(tunnel-[\d]+|vti)).*' matches = [] @@ -179,6 +445,23 @@ def show_sa(raw: bool): return _get_formatted_output_sas(sa_data) +def show_connections(raw: bool): + list_conns = _get_convert_data_connections() + list_sas = _get_raw_data_sas() + if raw: + return _get_raw_data_connections(list_conns, list_sas) + + connections = _get_raw_data_connections(list_conns, list_sas) + return _get_formatted_output_conections(connections) + + +def show_connections_summary(raw: bool): + list_conns = _get_convert_data_connections() + list_sas = _get_raw_data_sas() + if raw: + return _get_raw_connections_summary(list_conns, list_sas) + + if __name__ == '__main__': try: res = vyos.opmode.run(sys.modules[__name__]) -- cgit v1.2.3 From 612005604d14b70f953e07e5b4f04e60192fb2e1 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Wed, 16 Nov 2022 05:51:45 +0100 Subject: Revert "Revert "dns: T4799: fix bug with not reloading powerdns config"" This reverts commit 44df1cea1ebc3296844c5c35cf053a92cda4b944. --- src/services/vyos-hostsd | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src') diff --git a/src/services/vyos-hostsd b/src/services/vyos-hostsd index 9ae7b1ea9..a380f2e66 100755 --- a/src/services/vyos-hostsd +++ b/src/services/vyos-hostsd @@ -406,8 +406,7 @@ def validate_schema(data): def pdns_rec_control(command): - # pdns-r process name is NOT equal to the name shown in ps - if not process_named_running('pdns-r/worker'): + if not process_named_running('pdns_recursor'): logger.info(f'pdns_recursor not running, not sending "{command}"') return -- cgit v1.2.3 From 00639f6c84457eff08760dcdd66b654eb39629d1 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Wed, 16 Nov 2022 16:39:49 +0100 Subject: bridge: T4673: remove "sudo" as there is no need to elevate permissions --- src/op_mode/bridge.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/op_mode/bridge.py b/src/op_mode/bridge.py index 5a821a287..d6098c158 100755 --- a/src/op_mode/bridge.py +++ b/src/op_mode/bridge.py @@ -32,7 +32,7 @@ def _get_json_data(): """ Get bridge data format JSON """ - return cmd(f'sudo bridge --json link show') + return cmd(f'bridge --json link show') def _get_raw_data_summary(): @@ -48,7 +48,7 @@ def _get_raw_data_vlan(): """ :returns dict """ - json_data = cmd('sudo bridge --json --compressvlans vlan show') + json_data = cmd('bridge --json --compressvlans vlan show') data_dict = json.loads(json_data) return data_dict @@ -57,7 +57,7 @@ def _get_raw_data_fdb(bridge): """Get MAC-address for the bridge brX :returns list """ - code, json_data = rc_cmd(f'sudo bridge --json fdb show br {bridge}') + code, json_data = rc_cmd(f'bridge --json fdb show br {bridge}') # From iproute2 fdb.c, fdb_show() will only exit(-1) in case of # non-existent bridge device; raise error. if code == 255: -- cgit v1.2.3 From 9faa60a78d8fae85798a7219a5de82dd0af0e3c2 Mon Sep 17 00:00:00 2001 From: Sander Klein Date: Wed, 16 Nov 2022 20:25:19 +0100 Subject: T4794: Fix show show firewall name show firewall name will output an error as explained in https://phabricator.vyos.net/T4794 --- src/op_mode/firewall.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/op_mode/firewall.py b/src/op_mode/firewall.py index 950feb625..46bda5f7e 100755 --- a/src/op_mode/firewall.py +++ b/src/op_mode/firewall.py @@ -63,7 +63,7 @@ def get_config_firewall(conf, name=None, ipv6=False, interfaces=True): get_first_key=True, no_tag_node_value_mangle=True) if firewall and interfaces: if name: - firewall['interface'] = [] + firewall['interface'] = {} else: if 'name' in firewall: for fw_name, name_conf in firewall['name'].items(): -- cgit v1.2.3 From b8bda7c8d54fb500716c78ca39107e33988311ea Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 16 Nov 2022 08:43:27 -0600 Subject: firewall: T4821: correct calling of conf_mode script dependencies --- src/conf_mode/firewall.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) (limited to 'src') diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index 783adec46..9fee20358 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -26,6 +26,7 @@ from vyos.config import Config from vyos.configdict import dict_merge from vyos.configdict import node_changed from vyos.configdiff import get_config_diff, Diff +from vyos.configdep import set_dependent, call_dependents # from vyos.configverify import verify_interface_exists from vyos.firewall import fqdn_config_parse from vyos.firewall import geoip_update @@ -41,8 +42,8 @@ from vyos import ConfigError from vyos import airbag airbag.enable() -nat_conf_script = '/usr/libexec/vyos/conf_mode/nat.py' -policy_route_conf_script = '/usr/libexec/vyos/conf_mode/policy-route.py' +nat_conf_script = 'nat.py' +policy_route_conf_script = 'policy-route.py' nftables_conf = '/run/nftables.conf' @@ -160,6 +161,12 @@ def get_config(config=None): firewall['zone'][zone] = dict_merge(default_values, firewall['zone'][zone]) firewall['group_resync'] = bool('group' in firewall or node_changed(conf, base + ['group'])) + if firewall['group_resync']: + # Update nat as firewall groups were updated + set_dependent(nat_conf_script, conf) + # Update policy route as firewall groups were updated + set_dependent(policy_route_conf_script, conf) + if 'config_trap' in firewall and firewall['config_trap'] == 'enable': diff = get_config_diff(conf) @@ -464,18 +471,6 @@ def post_apply_trap(firewall): cmd(base_cmd + ' '.join(objects)) -def resync_nat(): - # Update nat as firewall groups were updated - tmp, out = rc_cmd(nat_conf_script) - if tmp > 0: - Warning(f'Failed to re-apply nat configuration! {out}') - -def resync_policy_route(): - # Update policy route as firewall groups were updated - tmp, out = rc_cmd(policy_route_conf_script) - if tmp > 0: - Warning(f'Failed to re-apply policy route configuration! {out}') - def apply(firewall): install_result, output = rc_cmd(f'nft -f {nftables_conf}') if install_result == 1: @@ -484,8 +479,7 @@ def apply(firewall): apply_sysfs(firewall) if firewall['group_resync']: - resync_nat() - resync_policy_route() + call_dependents() # T970 Enable a resolver (systemd daemon) that checks # domain-group/fqdn addresses and update entries for domains by timeout -- cgit v1.2.3 From c3be3f0a127819b4b922331f307a89afaaf7cef3 Mon Sep 17 00:00:00 2001 From: aapostoliuk Date: Fri, 4 Nov 2022 18:11:33 +0200 Subject: T4793: Added warning about disable-route-autoinstall Added warning message about disable-route-autoinstall when ipsec vti is used. --- src/conf_mode/vpn_ipsec.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src') diff --git a/src/conf_mode/vpn_ipsec.py b/src/conf_mode/vpn_ipsec.py index cfefcfbe8..b79e9847a 100755 --- a/src/conf_mode/vpn_ipsec.py +++ b/src/conf_mode/vpn_ipsec.py @@ -22,6 +22,7 @@ from sys import exit from time import sleep from time import time +from vyos.base import Warning from vyos.config import Config from vyos.configdict import leaf_node_changed from vyos.configverify import verify_interface_exists @@ -438,6 +439,10 @@ def verify(ipsec): if 'local_address' in peer_conf and 'dhcp_interface' in peer_conf: raise ConfigError(f"A single local-address or dhcp-interface is required when using VTI on site-to-site peer {peer}") + if dict_search('options.disable_route_autoinstall', + ipsec) == None: + Warning('It\'s recommended to use ipsec vty with the next command\n[set vpn ipsec option disable-route-autoinstall]') + if 'bind' in peer_conf['vti']: vti_interface = peer_conf['vti']['bind'] if not os.path.exists(f'/sys/class/net/{vti_interface}'): -- cgit v1.2.3 From 58057480e22712dc6d04396f8805d3db338bddfa Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 18 Nov 2022 16:08:47 -0600 Subject: IPsec: T4828: raise op-mode error on incorrect value --- python/vyos/opmode.py | 6 ++++++ src/op_mode/ipsec.py | 17 ++++++----------- .../api/graphql/session/errors/op_mode_errors.py | 6 ++++-- 3 files changed, 16 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/python/vyos/opmode.py b/python/vyos/opmode.py index 2e896c8e6..9dba8d30f 100644 --- a/python/vyos/opmode.py +++ b/python/vyos/opmode.py @@ -45,6 +45,12 @@ class PermissionDenied(Error): """ pass +class IncorrectValue(Error): + """ Requested operation is valid, but an argument provided has an + incorrect value, preventing successful completion. + """ + pass + class InternalError(Error): """ Any situation when VyOS detects that it could not perform an operation correctly due to logic errors in its own code diff --git a/src/op_mode/ipsec.py b/src/op_mode/ipsec.py index aaa0cec5a..83e4241d7 100755 --- a/src/op_mode/ipsec.py +++ b/src/op_mode/ipsec.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import os import re import sys @@ -138,6 +139,8 @@ def _get_formatted_output_sas(sas): def get_peer_connections(peer, tunnel, return_all = False): search = rf'^[\s]*({peer}-(tunnel-[\d]+|vti)).*' matches = [] + if not os.path.exists(SWANCTL_CONF): + raise vyos.opmode.UnconfiguredSubsystem("IPsec not initialized") with open(SWANCTL_CONF, 'r') as f: for line in f.readlines(): result = re.match(search, line) @@ -149,27 +152,19 @@ def get_peer_connections(peer, tunnel, return_all = False): def reset_peer(peer: str, tunnel:str): - if not peer: - print('Invalid peer, aborting') - return - conns = get_peer_connections(peer, tunnel, return_all = (not tunnel or tunnel == 'all')) if not conns: - print('Tunnel(s) not found, aborting') - return + raise vyos.opmode.IncorrectValue('Peer or tunnel(s) not found, aborting') - result = True for conn in conns: try: call(f'sudo /usr/sbin/ipsec down {conn}{{*}}', timeout = 10) call(f'sudo /usr/sbin/ipsec up {conn}', timeout = 10) except TimeoutExpired as e: - print(f'Timed out while resetting {conn}') - result = False - + raise vyos.opmode.InternalError(f'Timed out while resetting {conn}') - print('Peer reset result: ' + ('success' if result else 'failed')) + print('Peer reset result: success') def show_sa(raw: bool): diff --git a/src/services/api/graphql/session/errors/op_mode_errors.py b/src/services/api/graphql/session/errors/op_mode_errors.py index 7ba75455d..7bc1d1d81 100644 --- a/src/services/api/graphql/session/errors/op_mode_errors.py +++ b/src/services/api/graphql/session/errors/op_mode_errors.py @@ -3,11 +3,13 @@ op_mode_err_msg = { "UnconfiguredSubsystem": "subsystem is not configured or not running", "DataUnavailable": "data currently unavailable", - "PermissionDenied": "client does not have permission" + "PermissionDenied": "client does not have permission", + "IncorrectValue": "argument value is incorrect" } op_mode_err_code = { "UnconfiguredSubsystem": 2000, "DataUnavailable": 2001, - "PermissionDenied": 1003 + "PermissionDenied": 1003, + "IncorrectValue": 1002 } -- cgit v1.2.3 From 94287c30406894671a89b9b75dd5a163af714e6c Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sun, 20 Nov 2022 07:42:15 +0100 Subject: vrf: T4562: no need to invode "sudo" when retrieving VRf information --- src/op_mode/vrf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/op_mode/vrf.py b/src/op_mode/vrf.py index aeb50fe6e..a9a416761 100755 --- a/src/op_mode/vrf.py +++ b/src/op_mode/vrf.py @@ -31,14 +31,14 @@ def _get_raw_data(name=None): If vrf name is set - get only this name data If vrf name set and not found - return [] """ - output = cmd('sudo ip --json --brief link show type vrf') + output = cmd('ip --json --brief link show type vrf') data = json.loads(output) if not data: return [] if name: is_vrf_exists = True if [vrf for vrf in data if vrf.get('ifname') == name] else False if is_vrf_exists: - output = cmd(f'sudo ip --json --brief link show dev {name}') + output = cmd(f'ip --json --brief link show dev {name}') data = json.loads(output) return data return [] @@ -51,7 +51,7 @@ def _get_vrf_members(vrf: str) -> list: :param vrf: str :return: list """ - output = cmd(f'sudo ip --json --brief link show master {vrf}') + output = cmd(f'ip --json --brief link show master {vrf}') answer = json.loads(output) interfaces = [] for data in answer: -- cgit v1.2.3 From 42373334b1edae9553776bfbe8f21ba3bf4dd2a5 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Sun, 20 Nov 2022 09:17:06 +0000 Subject: T4827: Route-map state continue must be with action permit only route-map action 'deny' cannot be used for "continue" as FRR does not validate it r14(config)# route-map FOO permit 100 r14(config-route-map)# route-map FOO deny 50 r14(config-route-map)# on-match goto 100 % Configuration failed. Error type: validation r14(config-route-map)# --- src/conf_mode/policy.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src') diff --git a/src/conf_mode/policy.py b/src/conf_mode/policy.py index a0d288e91..331194fec 100755 --- a/src/conf_mode/policy.py +++ b/src/conf_mode/policy.py @@ -167,6 +167,11 @@ def verify(policy): continue for rule, rule_config in route_map_config['rule'].items(): + # Action 'deny' cannot be used with "continue" + # FRR does not validate it T4827 + if rule_config['action'] == 'deny' and 'continue' in rule_config: + raise ConfigError(f'rule {rule} "continue" cannot be used with action deny!') + # Specified community-list must exist tmp = dict_search('match.community.community_list', rule_config) -- cgit v1.2.3 From da5bff2e835a14997d7b176670376cbd8d1221ef Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sun, 20 Nov 2022 16:58:43 +0100 Subject: op-mode: dns-forwarding: T4578: drop sudo calls Commit 66288ccfee ("dns-forwarding: T4578: Rewrite show dns forwarding") added the implementation for the new standardized op-mode definitions/implementation. As the API daemon has the proper permissions and also the CLI op-mode calls the script already with "sudo", there is no need to call "sudo" inside this script, again. Also add dns.py to data/op-mode-standardized.json for the GraphQL schema to be generated. --- data/op-mode-standardized.json | 1 + src/op_mode/dns.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index b162f4097..1509975b4 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -6,6 +6,7 @@ "container.py", "cpu.py", "dhcp.py", +"dns.py", "log.py", "memory.py", "nat.py", diff --git a/src/op_mode/dns.py b/src/op_mode/dns.py index 9e5b1040c..a0e47d7ad 100755 --- a/src/op_mode/dns.py +++ b/src/op_mode/dns.py @@ -54,10 +54,10 @@ def _data_to_dict(data, sep="\t") -> dict: def _get_raw_forwarding_statistics() -> dict: - command = cmd('sudo /usr/bin/rec_control --socket-dir=/run/powerdns get-all') + command = cmd('rec_control --socket-dir=/run/powerdns get-all') data = _data_to_dict(command) data['cache-size'] = "{0:.2f}".format( int( - cmd('sudo /usr/bin/rec_control --socket-dir=/run/powerdns get cache-bytes')) / 1024 ) + cmd('rec_control --socket-dir=/run/powerdns get cache-bytes')) / 1024 ) return data -- cgit v1.2.3 From 11273157f6511d49c24d14516a44d581e9f3c126 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 20 Nov 2022 14:52:25 -0600 Subject: IPsec: T4829: add missing import TimeoutExpired --- src/op_mode/ipsec.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/op_mode/ipsec.py b/src/op_mode/ipsec.py index afe006834..3389254e3 100755 --- a/src/op_mode/ipsec.py +++ b/src/op_mode/ipsec.py @@ -22,6 +22,7 @@ from collections import OrderedDict from hurry import filesize from re import split as re_split from tabulate import tabulate +from subprocess import TimeoutExpired from vyos.util import call from vyos.util import convert_data -- cgit v1.2.3 From 97771d427c1660f16122da1260bf28e22e12612d Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 20 Nov 2022 15:00:22 -0600 Subject: IPsec: T4829: use type hint Optional for arg tunnel in reset_peer --- src/op_mode/ipsec.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/op_mode/ipsec.py b/src/op_mode/ipsec.py index 3389254e3..e0d204a0a 100755 --- a/src/op_mode/ipsec.py +++ b/src/op_mode/ipsec.py @@ -17,6 +17,7 @@ import os import re import sys +import typing from collections import OrderedDict from hurry import filesize @@ -403,23 +404,27 @@ def _get_formatted_output_conections(data): # Connections block end -def get_peer_connections(peer, tunnel, return_all = False): +def get_peer_connections(peer, tunnel): search = rf'^[\s]*({peer}-(tunnel-[\d]+|vti)).*' matches = [] if not os.path.exists(SWANCTL_CONF): raise vyos.opmode.UnconfiguredSubsystem("IPsec not initialized") + suffix = None if tunnel is None else (f'tunnel-{tunnel}' if + tunnel.isnumeric() else tunnel) with open(SWANCTL_CONF, 'r') as f: for line in f.readlines(): result = re.match(search, line) if result: - suffix = f'tunnel-{tunnel}' if tunnel.isnumeric() else tunnel - if return_all or (result[2] == suffix): + if tunnel is None: matches.append(result[1]) + else: + if result[2] == suffix: + matches.append(result[1]) return matches -def reset_peer(peer: str, tunnel:str): - conns = get_peer_connections(peer, tunnel, return_all = (not tunnel or tunnel == 'all')) +def reset_peer(peer: str, tunnel:typing.Optional[str]): + conns = get_peer_connections(peer, tunnel) if not conns: raise vyos.opmode.IncorrectValue('Peer or tunnel(s) not found, aborting') -- cgit v1.2.3 From 3d4dedd6709760ebf7bb9cff2b1a22ab5b212532 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Mon, 21 Nov 2022 12:45:46 -0600 Subject: graphql: T4544: use load_as_module from vyos.util load_as_module was added to util.py for T4821; prefer over local copy --- src/services/api/graphql/generate/schema_from_op_mode.py | 5 +++-- src/services/api/graphql/libs/op_mode.py | 7 +------ 2 files changed, 4 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/services/api/graphql/generate/schema_from_op_mode.py b/src/services/api/graphql/generate/schema_from_op_mode.py index 1fd198a37..fc63b0100 100755 --- a/src/services/api/graphql/generate/schema_from_op_mode.py +++ b/src/services/api/graphql/generate/schema_from_op_mode.py @@ -25,15 +25,16 @@ from inspect import signature, getmembers, isfunction, isclass, getmro from jinja2 import Template from vyos.defaults import directories +from vyos.util import load_as_module if __package__ is None or __package__ == '': sys.path.append("/usr/libexec/vyos/services/api") - from graphql.libs.op_mode import load_as_module, is_op_mode_function_name, is_show_function_name + from graphql.libs.op_mode import is_op_mode_function_name, is_show_function_name from graphql.libs.op_mode import snake_to_pascal_case, map_type_name from vyos.config import Config from vyos.configdict import dict_merge from vyos.xml import defaults else: - from .. libs.op_mode import load_as_module, is_op_mode_function_name, is_show_function_name + from .. libs.op_mode import is_op_mode_function_name, is_show_function_name from .. libs.op_mode import snake_to_pascal_case, map_type_name from .. import state diff --git a/src/services/api/graphql/libs/op_mode.py b/src/services/api/graphql/libs/op_mode.py index 97a26520e..6939ed5d6 100644 --- a/src/services/api/graphql/libs/op_mode.py +++ b/src/services/api/graphql/libs/op_mode.py @@ -21,14 +21,9 @@ from typing import Union from humps import decamelize from vyos.defaults import directories +from vyos.util import load_as_module from vyos.opmode import _normalize_field_names -def load_as_module(name: str, path: str): - spec = importlib.util.spec_from_file_location(name, path) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - def load_op_mode_as_module(name: str): path = os.path.join(directories['op_mode'], name) name = os.path.splitext(name)[0].replace('-', '_') -- cgit v1.2.3 From d70350f356c8a0df79600f8863addce104b0eba4 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Mon, 21 Nov 2022 12:52:01 -0600 Subject: graphql: T4574: use Optional in func_sig A misreading of the makefun docs seemed to indicate Optional was not supported; it is. --- src/services/api/graphql/graphql/mutations.py | 16 ++++------------ src/services/api/graphql/graphql/queries.py | 16 ++++------------ 2 files changed, 8 insertions(+), 24 deletions(-) (limited to 'src') diff --git a/src/services/api/graphql/graphql/mutations.py b/src/services/api/graphql/graphql/mutations.py index 2778feb69..31cb1afc4 100644 --- a/src/services/api/graphql/graphql/mutations.py +++ b/src/services/api/graphql/graphql/mutations.py @@ -14,7 +14,7 @@ # along with this library. If not, see . from importlib import import_module -from typing import Any, Dict +from typing import Any, Dict, Optional from ariadne import ObjectType, convert_kwargs_to_snake_case, convert_camel_case_to_snake from graphql import GraphQLResolveInfo from makefun import with_signature @@ -42,7 +42,7 @@ def make_mutation_resolver(mutation_name, class_name, session_func): func_base_name = convert_camel_case_to_snake(class_name) resolver_name = f'resolve_{func_base_name}' - func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Dict = {})' + func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Optional[Dict]=None)' @mutation.field(mutation_name) @convert_kwargs_to_snake_case @@ -67,17 +67,9 @@ def make_mutation_resolver(mutation_name, class_name, session_func): del data['key'] elif auth_type == 'token': - # there is a subtlety here: with the removal of the key entry, - # some requests will now have empty input, hence no data arg, so - # make it optional in the func_sig. However, it can not be None, - # as the makefun package provides accurate TypeError exceptions; - # hence set it to {}, but now it is a mutable default argument, - # so clear the key 'result', which is added at the end of - # this function. data = kwargs['data'] - if 'result' in data: - del data['result'] - + if data is None: + data = {} info = kwargs['info'] user = info.context.get('user') if user is None: diff --git a/src/services/api/graphql/graphql/queries.py b/src/services/api/graphql/graphql/queries.py index 9c8a4f064..3a88e3c80 100644 --- a/src/services/api/graphql/graphql/queries.py +++ b/src/services/api/graphql/graphql/queries.py @@ -14,7 +14,7 @@ # along with this library. If not, see . from importlib import import_module -from typing import Any, Dict +from typing import Any, Dict, Optional from ariadne import ObjectType, convert_kwargs_to_snake_case, convert_camel_case_to_snake from graphql import GraphQLResolveInfo from makefun import with_signature @@ -42,7 +42,7 @@ def make_query_resolver(query_name, class_name, session_func): func_base_name = convert_camel_case_to_snake(class_name) resolver_name = f'resolve_{func_base_name}' - func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Dict = {})' + func_sig = '(obj: Any, info: GraphQLResolveInfo, data: Optional[Dict]=None)' @query.field(query_name) @convert_kwargs_to_snake_case @@ -67,17 +67,9 @@ def make_query_resolver(query_name, class_name, session_func): del data['key'] elif auth_type == 'token': - # there is a subtlety here: with the removal of the key entry, - # some requests will now have empty input, hence no data arg, so - # make it optional in the func_sig. However, it can not be None, - # as the makefun package provides accurate TypeError exceptions; - # hence set it to {}, but now it is a mutable default argument, - # so clear the key 'result', which is added at the end of - # this function. data = kwargs['data'] - if 'result' in data: - del data['result'] - + if data is None: + data = {} info = kwargs['info'] user = info.context.get('user') if user is None: -- cgit v1.2.3 From 05b60b2dc6bd2187501b2583cdaa27a90c45b1d5 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Mon, 21 Nov 2022 14:06:23 -0600 Subject: graphql: T4574: add specific error message if token has expired Catch expiration error and return error-specific message instead of general 'not authenticated'. --- src/services/api/graphql/graphql/mutations.py | 6 ++++++ src/services/api/graphql/graphql/queries.py | 6 ++++++ src/services/api/graphql/libs/token_auth.py | 3 +++ 3 files changed, 15 insertions(+) (limited to 'src') diff --git a/src/services/api/graphql/graphql/mutations.py b/src/services/api/graphql/graphql/mutations.py index 31cb1afc4..87ea59c43 100644 --- a/src/services/api/graphql/graphql/mutations.py +++ b/src/services/api/graphql/graphql/mutations.py @@ -73,6 +73,12 @@ def make_mutation_resolver(mutation_name, class_name, session_func): info = kwargs['info'] user = info.context.get('user') if user is None: + error = info.context.get('error') + if error is not None: + return { + "success": False, + "errors": [error] + } return { "success": False, "errors": ['not authenticated'] diff --git a/src/services/api/graphql/graphql/queries.py b/src/services/api/graphql/graphql/queries.py index 3a88e3c80..1ad586428 100644 --- a/src/services/api/graphql/graphql/queries.py +++ b/src/services/api/graphql/graphql/queries.py @@ -73,6 +73,12 @@ def make_query_resolver(query_name, class_name, session_func): info = kwargs['info'] user = info.context.get('user') if user is None: + error = info.context.get('error') + if error is not None: + return { + "success": False, + "errors": [error] + } return { "success": False, "errors": ['not authenticated'] diff --git a/src/services/api/graphql/libs/token_auth.py b/src/services/api/graphql/libs/token_auth.py index 3ecd8b855..2100eba7f 100644 --- a/src/services/api/graphql/libs/token_auth.py +++ b/src/services/api/graphql/libs/token_auth.py @@ -54,6 +54,9 @@ def get_user_context(request): user_id: str = payload.get('sub') if user_id is None: return context + except jwt.exceptions.ExpiredSignatureError: + context['error'] = 'expired token' + return context except jwt.PyJWTError: return context try: -- cgit v1.2.3 From 43062d0b338a5aad7e3b158d1c83e0a4472b1420 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Thu, 24 Nov 2022 13:04:23 +0000 Subject: T4837: expose "show ip route summary" in the op mode API --- src/op_mode/route.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'src') diff --git a/src/op_mode/route.py b/src/op_mode/route.py index d11b00ba0..d07a34180 100755 --- a/src/op_mode/route.py +++ b/src/op_mode/route.py @@ -54,6 +54,18 @@ frr_command_template = Template(""" {% endif %} """) +def show_summary(raw: bool): + from vyos.util import cmd + + if raw: + from json import loads + + output = cmd(f"vtysh -c 'show ip route summary json'") + return loads(output) + else: + output = cmd(f"vtysh -c 'show ip route summary'") + return output + def show(raw: bool, family: str, net: typing.Optional[str], -- cgit v1.2.3 From 48ab6413cedb71934118143a7e095e946ac38bb7 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Sun, 20 Nov 2022 13:35:24 +0000 Subject: T4825: Add interface type veth Add interface type veth (Virtual ethernet) One of the usecases it's interconnect different vrf's and default vrf via bridge set interfaces virtual-ethernet veth0 peer-name 'veth1010' set interfaces virtual-ethernet veth1010 address '10.0.0.10/24' set interfaces virtual-ethernet veth1010 peer-name 'veth0' set interfaces virtual-ethernet veth1010 vrf 'foo' set interfaces bridge br0 address '10.0.0.1/24' set interfaces bridge br0 member interface veth0 --- .../interfaces-virtual-ethernet.xml.in | 36 ++++++++ python/vyos/ifconfig/__init__.py | 1 + python/vyos/ifconfig/veth.py | 54 ++++++++++++ src/conf_mode/interfaces-virtual-ethernet.py | 97 ++++++++++++++++++++++ 4 files changed, 188 insertions(+) create mode 100644 interface-definitions/interfaces-virtual-ethernet.xml.in create mode 100644 python/vyos/ifconfig/veth.py create mode 100755 src/conf_mode/interfaces-virtual-ethernet.py (limited to 'src') diff --git a/interface-definitions/interfaces-virtual-ethernet.xml.in b/interface-definitions/interfaces-virtual-ethernet.xml.in new file mode 100644 index 000000000..3b78b3637 --- /dev/null +++ b/interface-definitions/interfaces-virtual-ethernet.xml.in @@ -0,0 +1,36 @@ + + + + + + + Virtual Ethernet Interface (veth) + 300 + + veth[0-9]+ + + Virutal Ethernet interface must be named vethN + + vethN + Virtual Ethernet interface name + + + + #include + #include + #include + #include + + + Virtual ethernet peer interface name + + veth[0-9]+ + + Virutal Ethernet interface must be named vethN + + + + + + + diff --git a/python/vyos/ifconfig/__init__.py b/python/vyos/ifconfig/__init__.py index a37615c8f..d1ddaa13e 100644 --- a/python/vyos/ifconfig/__init__.py +++ b/python/vyos/ifconfig/__init__.py @@ -36,4 +36,5 @@ from vyos.ifconfig.tunnel import TunnelIf from vyos.ifconfig.wireless import WiFiIf from vyos.ifconfig.l2tpv3 import L2TPv3If from vyos.ifconfig.macsec import MACsecIf +from vyos.ifconfig.veth import VethIf from vyos.ifconfig.wwan import WWANIf diff --git a/python/vyos/ifconfig/veth.py b/python/vyos/ifconfig/veth.py new file mode 100644 index 000000000..aafbf226a --- /dev/null +++ b/python/vyos/ifconfig/veth.py @@ -0,0 +1,54 @@ +# Copyright 2022 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . + +from vyos.ifconfig.interface import Interface + + +@Interface.register +class VethIf(Interface): + """ + Abstraction of a Linux veth interface + """ + iftype = 'veth' + definition = { + **Interface.definition, + **{ + 'section': 'virtual-ethernet', + 'prefixes': ['veth', ], + 'bridgeable': True, + }, + } + + def _create(self): + """ + Create veth interface in OS kernel. Interface is administrative + down by default. + """ + # check before create, as we have 2 veth interfaces in our CLI + # interface virtual-ethernet veth0 peer-name 'veth1' + # interface virtual-ethernet veth1 peer-name 'veth0' + # + # but iproute2 creates the pair with one command: + # ip link add vet0 type veth peer name veth1 + if self.exists(self.config['peer_name']): + return + + # create virtual-ethernet interface + cmd = 'ip link add {ifname} type {type}'.format(**self.config) + cmd += f' peer name {self.config["peer_name"]}' + self._cmd(cmd) + + # interface is always A/D down. It needs to be enabled explicitly + self.set_admin_state('down') diff --git a/src/conf_mode/interfaces-virtual-ethernet.py b/src/conf_mode/interfaces-virtual-ethernet.py new file mode 100755 index 000000000..91609ded9 --- /dev/null +++ b/src/conf_mode/interfaces-virtual-ethernet.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from sys import exit + +from netifaces import interfaces +from vyos import ConfigError +from vyos import airbag +from vyos.config import Config +from vyos.configdict import get_interface_dict +from vyos.configverify import verify_address +from vyos.configverify import verify_bridge_delete +from vyos.configverify import verify_vrf +from vyos.ifconfig import VethIf + +airbag.enable() + + +def get_config(config=None): + """ + Retrive CLI config as dictionary. Dictionary can never be empty, as at + least the interface name will be added or a deleted flag + """ + if config: + conf = config + else: + conf = Config() + base = ['interfaces', 'virtual-ethernet'] + ifname, veth = get_interface_dict(conf, base) + + veth_dict = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, + no_tag_node_value_mangle=True) + veth['config_dict'] = veth_dict + + return veth + + +def verify(veth): + if 'deleted' in veth: + verify_bridge_delete(veth) + return None + + verify_vrf(veth) + verify_address(veth) + + if 'peer_name' not in veth: + raise ConfigError( + f'Remote peer name must be set for \"{veth["ifname"]}\"!') + + if veth['peer_name'] not in veth['config_dict'].keys(): + raise ConfigError( + f'Interface \"{veth["peer_name"]}\" is not configured!') + + return None + + +def generate(peth): + return None + + +def apply(veth): + # Check if the Veth interface already exists + if 'rebuild_required' in veth or 'deleted' in veth: + if veth['ifname'] in interfaces(): + p = VethIf(veth['ifname']) + p.remove() + + if 'deleted' not in veth: + p = VethIf(**veth) + p.update(veth) + + return None + + +if __name__ == '__main__': + try: + c = get_config() + verify(c) + generate(c) + apply(c) + except ConfigError as e: + print(e) + exit(1) -- cgit v1.2.3 From f9a5286f163b27b51eb5e9a801c5d646c07a7990 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Thu, 24 Nov 2022 20:35:31 +0100 Subject: veth: T4825: minor improvements on XML peer-name handling --- .../interfaces-virtual-ethernet.xml.in | 9 ++++++++- src/conf_mode/interfaces-virtual-ethernet.py | 23 +++++++++++----------- 2 files changed, 20 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/interface-definitions/interfaces-virtual-ethernet.xml.in b/interface-definitions/interfaces-virtual-ethernet.xml.in index 3b78b3637..d52e9ef80 100644 --- a/interface-definitions/interfaces-virtual-ethernet.xml.in +++ b/interface-definitions/interfaces-virtual-ethernet.xml.in @@ -4,7 +4,7 @@ - Virtual Ethernet Interface (veth) + Virtual Ethernet (veth) Interface 300 veth[0-9]+ @@ -23,6 +23,13 @@ Virtual ethernet peer interface name + + interfaces virtual-ethernet + + + txt + Name of peer interface + veth[0-9]+ diff --git a/src/conf_mode/interfaces-virtual-ethernet.py b/src/conf_mode/interfaces-virtual-ethernet.py index 91609ded9..b1819233c 100755 --- a/src/conf_mode/interfaces-virtual-ethernet.py +++ b/src/conf_mode/interfaces-virtual-ethernet.py @@ -28,7 +28,6 @@ from vyos.ifconfig import VethIf airbag.enable() - def get_config(config=None): """ Retrive CLI config as dictionary. Dictionary can never be empty, as at @@ -41,10 +40,12 @@ def get_config(config=None): base = ['interfaces', 'virtual-ethernet'] ifname, veth = get_interface_dict(conf, base) - veth_dict = conf.get_config_dict(base, key_mangling=('-', '_'), - get_first_key=True, - no_tag_node_value_mangle=True) - veth['config_dict'] = veth_dict + # We need to know all other veth related interfaces as veth requires a 1:1 + # mapping for the peer-names. The Linux kernel automatically creates both + # interfaces, the local one and the peer-name, but VyOS also needs a peer + # interfaces configrued on the CLI so we can assign proper IP addresses etc. + veth['other_interfaces'] = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, no_tag_node_value_mangle=True) return veth @@ -58,12 +59,13 @@ def verify(veth): verify_address(veth) if 'peer_name' not in veth: - raise ConfigError( - f'Remote peer name must be set for \"{veth["ifname"]}\"!') + raise ConfigError(f'Remote peer name must be set for "{veth["ifname"]}"!') - if veth['peer_name'] not in veth['config_dict'].keys(): - raise ConfigError( - f'Interface \"{veth["peer_name"]}\" is not configured!') + if veth['peer_name'] not in veth['other_interfaces']: + peer_name = veth['peer_name'] + ifname = veth['ifname'] + raise ConfigError(f'Used peer-name "{peer_name}" on interface "{ifname}" ' \ + 'is not configured!') return None @@ -71,7 +73,6 @@ def verify(veth): def generate(peth): return None - def apply(veth): # Check if the Veth interface already exists if 'rebuild_required' in veth or 'deleted' in veth: -- cgit v1.2.3 From ad8f5762234d6b482b40b574d97fc51f6557daff Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Fri, 25 Nov 2022 17:13:24 +0000 Subject: T4825: Verify if veth interface not used in conf before deleting Prevent to delete interface "vethX" which used for another interface as "vethY peer-name vethX" set interfaces virtual-ethernet veth0 peer-name 'veth1' set interfaces virtual-ethernet veth1 peer-name 'veth0' commit delete interfaces virtual-ethernet veth0 commit --- src/conf_mode/interfaces-virtual-ethernet.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src') diff --git a/src/conf_mode/interfaces-virtual-ethernet.py b/src/conf_mode/interfaces-virtual-ethernet.py index b1819233c..53422ad2d 100755 --- a/src/conf_mode/interfaces-virtual-ethernet.py +++ b/src/conf_mode/interfaces-virtual-ethernet.py @@ -53,6 +53,13 @@ def get_config(config=None): def verify(veth): if 'deleted' in veth: verify_bridge_delete(veth) + # Prevent to delete veth interface which used for another "vethX peer-name" + for iface, iface_config in veth['other_interfaces'].items(): + if veth['ifname'] in iface_config['peer_name']: + ifname = veth['ifname'] + raise ConfigError( + f'Cannot delete "{ifname}" used for "interface {iface} peer-name"' + ) return None verify_vrf(veth) -- cgit v1.2.3 From ba2074d25bd4afda4bf891b8e404ffb91d5769c7 Mon Sep 17 00:00:00 2001 From: aapostoliuk Date: Mon, 28 Nov 2022 17:26:19 +0200 Subject: T4844: Set DB directory rigths 755 in the update webproxy script Squidguard: Set DB directory rigths 755 in the update blacklist webproxy script --- src/op_mode/webproxy_update_blacklist.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/op_mode/webproxy_update_blacklist.sh b/src/op_mode/webproxy_update_blacklist.sh index 43a4b79fc..d5f301b75 100755 --- a/src/op_mode/webproxy_update_blacklist.sh +++ b/src/op_mode/webproxy_update_blacklist.sh @@ -88,7 +88,7 @@ if [[ -n $update ]] && [[ $update -eq "yes" ]]; then # fix permissions chown -R proxy:proxy ${db_dir} - chmod 2770 ${db_dir} + chmod 755 ${db_dir} logger --priority WARNING "webproxy blacklist entries updated (${count_before}/${count_after})" -- cgit v1.2.3 From 1d12b24aa5b4d61d59f847e0002f291b66c7a737 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Mon, 28 Nov 2022 11:10:18 -0600 Subject: conf-mode: T4845: add external file for dict of config-mode dependencies --- data/config-mode-dependencies.json | 1 + python/vyos/configdep.py | 38 +++++++++++++++++++++++++++++--------- src/conf_mode/firewall.py | 9 +++------ 3 files changed, 33 insertions(+), 15 deletions(-) create mode 100644 data/config-mode-dependencies.json (limited to 'src') diff --git a/data/config-mode-dependencies.json b/data/config-mode-dependencies.json new file mode 100644 index 000000000..dd0efda10 --- /dev/null +++ b/data/config-mode-dependencies.json @@ -0,0 +1 @@ +{"firewall": {"group_resync": ["nat", "policy-route"]}} diff --git a/python/vyos/configdep.py b/python/vyos/configdep.py index e6b82ca93..ca05cb092 100644 --- a/python/vyos/configdep.py +++ b/python/vyos/configdep.py @@ -14,11 +14,15 @@ # along with this library. If not, see . import os +import json from inspect import stack from vyos.util import load_as_module +from vyos.defaults import directories +from vyos.configsource import VyOSError +from vyos import ConfigError -dependents = {} +dependent_func = {} def canon_name(name: str) -> str: return os.path.splitext(name)[0].replace('-', '_') @@ -30,9 +34,22 @@ def canon_name_of_path(path: str) -> str: def caller_name() -> str: return stack()[-1].filename -def run_config_mode_script(script: str, config): - from vyos.defaults import directories +def read_dependency_dict(): + path = os.path.join(directories['data'], + 'config-mode-dependencies.json') + with open(path) as f: + d = json.load(f) + return d + +def get_dependency_dict(config): + if hasattr(config, 'cached_dependency_dict'): + d = getattr(config, 'cached_dependency_dict') + else: + d = read_dependency_dict() + setattr(config, 'cached_dependency_dict', d) + return d +def run_config_mode_script(script: str, config): path = os.path.join(directories['conf_mode'], script) name = canon_name(script) mod = load_as_module(name, path) @@ -46,20 +63,23 @@ def run_config_mode_script(script: str, config): except (VyOSError, ConfigError) as e: raise ConfigError(repr(e)) -def def_closure(script: str, config): +def def_closure(target: str, config): + script = target + '.py' def func_impl(): run_config_mode_script(script, config) return func_impl -def set_dependent(target: str, config): +def set_dependents(case, config): + d = get_dependency_dict(config) k = canon_name_of_path(caller_name()) - l = dependents.setdefault(k, []) - func = def_closure(target, config) - l.append(func) + l = dependent_func.setdefault(k, []) + for target in d[k][case]: + func = def_closure(target, config) + l.append(func) def call_dependents(): k = canon_name_of_path(caller_name()) - l = dependents.get(k, []) + l = dependent_func.get(k, []) while l: f = l.pop(0) f() diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index 9fee20358..38a332be3 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -26,7 +26,7 @@ from vyos.config import Config from vyos.configdict import dict_merge from vyos.configdict import node_changed from vyos.configdiff import get_config_diff, Diff -from vyos.configdep import set_dependent, call_dependents +from vyos.configdep import set_dependents, call_dependents # from vyos.configverify import verify_interface_exists from vyos.firewall import fqdn_config_parse from vyos.firewall import geoip_update @@ -162,11 +162,8 @@ def get_config(config=None): firewall['group_resync'] = bool('group' in firewall or node_changed(conf, base + ['group'])) if firewall['group_resync']: - # Update nat as firewall groups were updated - set_dependent(nat_conf_script, conf) - # Update policy route as firewall groups were updated - set_dependent(policy_route_conf_script, conf) - + # Update nat and policy-route as firewall groups were updated + set_dependents('group_resync', conf) if 'config_trap' in firewall and firewall['config_trap'] == 'enable': diff = get_config_diff(conf) -- cgit v1.2.3 From 97f36fe0c1f337d73e5f0af4e2fecefadc3325b4 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Tue, 29 Nov 2022 07:34:40 +0100 Subject: mpls: T915: verify interface actually exists on the system --- src/conf_mode/protocols_mpls.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src') diff --git a/src/conf_mode/protocols_mpls.py b/src/conf_mode/protocols_mpls.py index 5da8e7b06..73af6595b 100755 --- a/src/conf_mode/protocols_mpls.py +++ b/src/conf_mode/protocols_mpls.py @@ -24,6 +24,7 @@ from vyos.template import render_to_string from vyos.util import dict_search from vyos.util import read_file from vyos.util import sysctl_write +from vyos.configverify import verify_interface_exists from vyos import ConfigError from vyos import frr from vyos import airbag @@ -46,6 +47,10 @@ def verify(mpls): if not mpls: return None + if 'interface' in mpls: + for interface in mpls['interface']: + verify_interface_exists(interface) + # Checks to see if LDP is properly configured if 'ldp' in mpls: # If router ID not defined -- cgit v1.2.3 From a0c97e33f59adf68fa77222199143a6ba7a5c794 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 29 Nov 2022 16:21:54 -0600 Subject: pki: T4847: fix typos --- src/conf_mode/interfaces-ethernet.py | 2 +- src/conf_mode/pki.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/conf_mode/interfaces-ethernet.py b/src/conf_mode/interfaces-ethernet.py index e02841831..b49c945cd 100755 --- a/src/conf_mode/interfaces-ethernet.py +++ b/src/conf_mode/interfaces-ethernet.py @@ -175,7 +175,7 @@ def generate(ethernet): loaded_pki_cert = load_certificate(pki_cert['certificate']) loaded_ca_certs = {load_certificate(c['certificate']) - for c in ethernet['pki']['ca'].values()} + for c in ethernet['pki']['ca'].values()} if 'ca' in ethernet['pki'] else {} cert_full_chain = find_chain(loaded_pki_cert, loaded_ca_certs) diff --git a/src/conf_mode/pki.py b/src/conf_mode/pki.py index 29ed7b1b7..75807b86f 100755 --- a/src/conf_mode/pki.py +++ b/src/conf_mode/pki.py @@ -281,12 +281,13 @@ def apply(pki): for found_name, found_path in dict_search_recursive(search_dict, key): if found_name == item_name: - path_str = ' '.join(search['path'] + found_path) + path = search['path'] + path_str = ' '.join(path + found_path) print(f'pki: Updating config: {path_str} {found_name}') script = search['script'] - if found_path[0] == 'interfaces': - ifname = found_path[2] + if path[0] == 'interfaces': + ifname = found_path[0] call(f'VYOS_TAGNODE_VALUE={ifname} {script}') else: call(script) -- cgit v1.2.3 From 22635eb2c269ffdf0acb756622c1febf0a901e56 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 30 Nov 2022 09:12:17 -0600 Subject: pki: T4847: set and call dependent scripts --- src/conf_mode/pki.py | 72 +++++++++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 37 deletions(-) (limited to 'src') diff --git a/src/conf_mode/pki.py b/src/conf_mode/pki.py index 75807b86f..e8f3cc87a 100755 --- a/src/conf_mode/pki.py +++ b/src/conf_mode/pki.py @@ -16,20 +16,16 @@ from sys import exit -import jmespath - from vyos.config import Config +from vyos.configdep import set_dependents, call_dependents from vyos.configdict import dict_merge from vyos.configdict import node_changed from vyos.pki import is_ca_certificate from vyos.pki import load_certificate -from vyos.pki import load_certificate_request from vyos.pki import load_public_key from vyos.pki import load_private_key from vyos.pki import load_crl from vyos.pki import load_dh_parameters -from vyos.util import ask_input -from vyos.util import call from vyos.util import dict_search_args from vyos.util import dict_search_recursive from vyos.xml import defaults @@ -121,6 +117,39 @@ def get_config(config=None): get_first_key=True, no_tag_node_value_mangle=True) + if 'changed' in pki: + for search in sync_search: + for key in search['keys']: + changed_key = sync_translate[key] + + if changed_key not in pki['changed']: + continue + + for item_name in pki['changed'][changed_key]: + node_present = False + if changed_key == 'openvpn': + node_present = dict_search_args(pki, 'openvpn', 'shared_secret', item_name) + else: + node_present = dict_search_args(pki, changed_key, item_name) + + if node_present: + search_dict = dict_search_args(pki['system'], *search['path']) + + if not search_dict: + continue + + for found_name, found_path in dict_search_recursive(search_dict, key): + if found_name == item_name: + path = search['path'] + path_str = ' '.join(path + found_path) + print(f'pki: Updating config: {path_str} {found_name}') + + if path[0] == 'interfaces': + ifname = found_path[0] + set_dependents(path[1], conf, ifname) + else: + set_dependents(path[1], conf) + return pki def is_valid_certificate(raw_data): @@ -259,38 +288,7 @@ def apply(pki): return None if 'changed' in pki: - for search in sync_search: - for key in search['keys']: - changed_key = sync_translate[key] - - if changed_key not in pki['changed']: - continue - - for item_name in pki['changed'][changed_key]: - node_present = False - if changed_key == 'openvpn': - node_present = dict_search_args(pki, 'openvpn', 'shared_secret', item_name) - else: - node_present = dict_search_args(pki, changed_key, item_name) - - if node_present: - search_dict = dict_search_args(pki['system'], *search['path']) - - if not search_dict: - continue - - for found_name, found_path in dict_search_recursive(search_dict, key): - if found_name == item_name: - path = search['path'] - path_str = ' '.join(path + found_path) - print(f'pki: Updating config: {path_str} {found_name}') - - script = search['script'] - if path[0] == 'interfaces': - ifname = found_path[0] - call(f'VYOS_TAGNODE_VALUE={ifname} {script}') - else: - call(script) + call_dependents() return None -- cgit v1.2.3 From c2a5957a942c8f6070b8f6ff12764e90c1a4c237 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Fri, 2 Dec 2022 09:44:25 +0100 Subject: op-mode: T4767: drop sudo calls It's easier and more obvious if the script is called with sudo itself and not spawning a sudo sessionf or each individual command. --- .../generate-ipsec-debug-archive.xml.in | 2 +- src/op_mode/generate_ipsec_debug_archive.py | 25 +++++++++++----------- 2 files changed, 13 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/op-mode-definitions/generate-ipsec-debug-archive.xml.in b/op-mode-definitions/generate-ipsec-debug-archive.xml.in index dcbed0c42..a9ce113d1 100644 --- a/op-mode-definitions/generate-ipsec-debug-archive.xml.in +++ b/op-mode-definitions/generate-ipsec-debug-archive.xml.in @@ -8,7 +8,7 @@ Generate IPSec debug-archive - ${vyos_op_scripts_dir}/generate_ipsec_debug_archive.py + sudo ${vyos_op_scripts_dir}/generate_ipsec_debug_archive.py diff --git a/src/op_mode/generate_ipsec_debug_archive.py b/src/op_mode/generate_ipsec_debug_archive.py index 933dd4e1a..1422559a8 100755 --- a/src/op_mode/generate_ipsec_debug_archive.py +++ b/src/op_mode/generate_ipsec_debug_archive.py @@ -24,20 +24,19 @@ from vyos.util import rc_cmd # define a list of commands that needs to be executed CMD_LIST: list[str] = [ - 'sudo ipsec status', - 'sudo swanctl -L', - 'sudo swanctl -l', - 'sudo swanctl -P', - 'sudo ip x sa show', - 'sudo ip x policy show', - 'sudo ip tunnel show', - 'sudo ip address', - 'sudo ip rule show', - 'sudo ip route | head -100', - 'sudo ip route show table 220' + 'ipsec status', + 'swanctl -L', + 'swanctl -l', + 'swanctl -P', + 'ip x sa show', + 'ip x policy show', + 'ip tunnel show', + 'ip address', + 'ip rule show', + 'ip route | head -100', + 'ip route show table 220' ] -JOURNALCTL_CMD: str = 'sudo journalctl -b -n 10000 /usr/lib/ipsec/charon' - +JOURNALCTL_CMD: str = 'journalctl -b -n 10000 /usr/lib/ipsec/charon' # execute a command and save the output to a file def save_stdout(command: str, file: Path) -> None: -- cgit v1.2.3 From 63c18aefa85f5b5b749932eae4def675a7d23f51 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Thu, 1 Dec 2022 16:05:38 +0000 Subject: T4805: Restart pppoe-server if client pool was changed Some changes for 'service pppoe-server' require 'restart' the accel-ppp@pppoe.service But we use option 'reload-or-restart' that doesn't work correctly with 'accel-ppp' Restart pppoe-server if client pool was changed --- src/conf_mode/service_pppoe-server.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/conf_mode/service_pppoe-server.py b/src/conf_mode/service_pppoe-server.py index ba0249efd..600ba4e92 100755 --- a/src/conf_mode/service_pppoe-server.py +++ b/src/conf_mode/service_pppoe-server.py @@ -20,6 +20,7 @@ from sys import exit from vyos.config import Config from vyos.configdict import get_accel_dict +from vyos.configdict import is_node_changed from vyos.configverify import verify_accel_ppp_base_service from vyos.configverify import verify_interface_exists from vyos.template import render @@ -43,6 +44,13 @@ def get_config(config=None): # retrieve common dictionary keys pppoe = get_accel_dict(conf, base, pppoe_chap_secrets) + + # reload-or-restart does not implemented in accel-ppp + # use this workaround until it will be implemented + # https://phabricator.accel-ppp.org/T3 + if is_node_changed(conf, base + ['client-ip-pool']) or is_node_changed( + conf, base + ['client-ipv6-pool']): + pppoe.update({'restart_required': {}}) return pppoe def verify(pppoe): @@ -95,7 +103,10 @@ def apply(pppoe): os.unlink(file) return None - call(f'systemctl reload-or-restart {systemd_service}') + if 'restart_required' in pppoe: + call(f'systemctl restart {systemd_service}') + else: + call(f'systemctl reload-or-restart {systemd_service}') if __name__ == '__main__': try: -- cgit v1.2.3 From 94f345340f69333dbb06fcf5f59caa8176ebecc6 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Fri, 2 Dec 2022 14:16:18 +0000 Subject: T4825: Verify if you are trying to add a new vethX to exists pair Verify if you are trying to add a new vethX to exists pair: set int virtual-ethernet veth0 peer-name 'veth1' set int virtual-ethernet veth1 peer-name 'veth0' set int virtual-ethernet veth12 peer-name 'veth0' Verify veth-name and peer-name cannot be the same: set interfaces virtual-ethernet veth0 peer-name veth0 --- src/conf_mode/interfaces-virtual-ethernet.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/conf_mode/interfaces-virtual-ethernet.py b/src/conf_mode/interfaces-virtual-ethernet.py index 53422ad2d..8efe89c41 100755 --- a/src/conf_mode/interfaces-virtual-ethernet.py +++ b/src/conf_mode/interfaces-virtual-ethernet.py @@ -68,12 +68,21 @@ def verify(veth): if 'peer_name' not in veth: raise ConfigError(f'Remote peer name must be set for "{veth["ifname"]}"!') + peer_name = veth['peer_name'] + ifname = veth['ifname'] + if veth['peer_name'] not in veth['other_interfaces']: - peer_name = veth['peer_name'] - ifname = veth['ifname'] raise ConfigError(f'Used peer-name "{peer_name}" on interface "{ifname}" ' \ 'is not configured!') + if veth['other_interfaces'][peer_name]['peer_name'] != ifname: + raise ConfigError( + f'Configuration mismatch between "{ifname}" and "{peer_name}"!') + + if peer_name == ifname: + raise ConfigError( + f'Peer-name "{peer_name}" cannot be the same as interface "{ifname}"!') + return None -- cgit v1.2.3 From 579e9294075d82627997af41ac7895923c7e8d5d Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 2 Dec 2022 11:22:43 -0600 Subject: http-api: T4859: correct calling of script dependencies from http-api.py --- data/config-mode-dependencies.json | 1 + src/conf_mode/http-api.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/data/config-mode-dependencies.json b/data/config-mode-dependencies.json index ad12cff87..9e943ba2c 100644 --- a/data/config-mode-dependencies.json +++ b/data/config-mode-dependencies.json @@ -1,5 +1,6 @@ { "firewall": {"group_resync": ["nat", "policy-route"]}, + "http_api": {"https": ["https"]}, "pki": { "ethernet": ["interfaces-ethernet"], "openvpn": ["interfaces-openvpn"], diff --git a/src/conf_mode/http-api.py b/src/conf_mode/http-api.py index be80613c6..6328294c1 100755 --- a/src/conf_mode/http-api.py +++ b/src/conf_mode/http-api.py @@ -25,6 +25,7 @@ import vyos.defaults from vyos.config import Config from vyos.configdict import dict_merge +from vyos.configdep import set_dependents, call_dependents from vyos.template import render from vyos.util import cmd from vyos.util import call @@ -61,6 +62,11 @@ def get_config(config=None): else: conf = Config() + # reset on creation/deletion of 'api' node + https_base = ['service', 'https'] + if conf.exists(https_base): + set_dependents("https", conf) + base = ['service', 'https', 'api'] if not conf.exists(base): return None @@ -132,7 +138,7 @@ def apply(http_api): # Let uvicorn settle before restarting Nginx sleep(1) - cmd(f'{vyos_conf_scripts_dir}/https.py', raising=ConfigError) + call_dependents() if __name__ == '__main__': try: -- cgit v1.2.3 From f916f40ee9a726159c063952b879f8bce90c378c Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Sat, 3 Dec 2022 12:20:33 +0100 Subject: firewall: T478: Fix firewall group circular dependency check --- src/conf_mode/firewall.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index 9fee20358..ae3dd6414 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -279,6 +279,8 @@ def verify_nested_group(group_name, group, groups, seen): if 'include' not in group: return + seen.append(group_name) + for g in group['include']: if g not in groups: raise ConfigError(f'Nested group "{g}" does not exist') @@ -286,8 +288,6 @@ def verify_nested_group(group_name, group, groups, seen): if g in seen: raise ConfigError(f'Group "{group_name}" has a circular reference') - seen.append(g) - if 'include' in groups[g]: verify_nested_group(g, groups[g], groups, seen) -- cgit v1.2.3 From 8572e3d6e0d782a1474e97149a7f0c81edfc41da Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Sun, 4 Dec 2022 11:04:28 +0000 Subject: T4848: Fix for default route vpn openconnect ocserv template expects list of routes but gets str "default" it cause wrong routes like: route = d route = e route = f route = a route = u route = l route = t Fix it --- src/conf_mode/vpn_openconnect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/conf_mode/vpn_openconnect.py b/src/conf_mode/vpn_openconnect.py index c050b796b..7c86356e9 100755 --- a/src/conf_mode/vpn_openconnect.py +++ b/src/conf_mode/vpn_openconnect.py @@ -157,7 +157,7 @@ def verify(ocserv): ocserv["network_settings"]["push_route"].remove("0.0.0.0/0") ocserv["network_settings"]["push_route"].append("default") else: - ocserv["network_settings"]["push_route"] = "default" + ocserv["network_settings"]["push_route"] = ["default"] else: raise ConfigError('openconnect network settings required') -- cgit v1.2.3 From 7ce6ad4486912dc76c5999f420897efe2747cabb Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Sun, 4 Dec 2022 13:29:00 +0000 Subject: T4860: Verify if mode in openconnect ocserv dict openconnect authentication mode must be set check dict that 'mode' exists in openconnect authentication --- src/conf_mode/vpn_openconnect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/conf_mode/vpn_openconnect.py b/src/conf_mode/vpn_openconnect.py index c050b796b..845250dd4 100755 --- a/src/conf_mode/vpn_openconnect.py +++ b/src/conf_mode/vpn_openconnect.py @@ -58,7 +58,7 @@ def get_config(): default_values = defaults(base) ocserv = dict_merge(default_values, ocserv) - if "local" in ocserv["authentication"]["mode"]: + if 'mode' in ocserv["authentication"] and "local" in ocserv["authentication"]["mode"]: # workaround a "know limitation" - https://phabricator.vyos.net/T2665 del ocserv['authentication']['local_users']['username']['otp'] if not ocserv["authentication"]["local_users"]["username"]: -- cgit v1.2.3 From ecb245f13f8f4c28f13a0ea32cba19f2b56758cd Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Wed, 7 Dec 2022 12:31:53 +0000 Subject: T4861: Openconnect replace restart to reload-or-restart Every change in openconnect restarts the ocserv.service Replace "restart" to "reload-or-restart" to avoid disconnect clients during change configs --- src/conf_mode/vpn_openconnect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/conf_mode/vpn_openconnect.py b/src/conf_mode/vpn_openconnect.py index b336b0c03..af3c51efc 100755 --- a/src/conf_mode/vpn_openconnect.py +++ b/src/conf_mode/vpn_openconnect.py @@ -247,7 +247,7 @@ def apply(ocserv): if os.path.exists(file): os.unlink(file) else: - call('systemctl restart ocserv.service') + call('systemctl reload-or-restart ocserv.service') counter = 0 while True: # exit early when service runs -- cgit v1.2.3 From 9fa4b761d027e2eee8a6fac587857548292261fb Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Thu, 8 Dec 2022 13:04:56 +0000 Subject: T4117: Fix for L2TP DAE CoA server configuration Fix l2tp dae server template and python config dict for correctlly handling Dynamic Authorization Extension server configuration --- data/templates/accel-ppp/l2tp.config.j2 | 3 +++ interface-definitions/vpn-l2tp.xml.in | 1 + src/conf_mode/vpn_l2tp.py | 34 ++++++++++++++++++++++++--------- 3 files changed, 29 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/data/templates/accel-ppp/l2tp.config.j2 b/data/templates/accel-ppp/l2tp.config.j2 index 9eeaf7622..986f19656 100644 --- a/data/templates/accel-ppp/l2tp.config.j2 +++ b/data/templates/accel-ppp/l2tp.config.j2 @@ -88,6 +88,9 @@ verbose=1 {% for r in radius_server %} server={{ r.server }},{{ r.key }},auth-port={{ r.port }},acct-port={{ r.acct_port }},req-limit=0,fail-time={{ r.fail_time }} {% endfor %} +{% if radius_dynamic_author.server is vyos_defined %} +dae-server={{ radius_dynamic_author.server }}:{{ radius_dynamic_author.port }},{{ radius_dynamic_author.key }} +{% endif %} {% if radius_acct_inter_jitter %} acct-interim-jitter={{ radius_acct_inter_jitter }} {% endif %} diff --git a/interface-definitions/vpn-l2tp.xml.in b/interface-definitions/vpn-l2tp.xml.in index cb5900e0d..06ca4ece5 100644 --- a/interface-definitions/vpn-l2tp.xml.in +++ b/interface-definitions/vpn-l2tp.xml.in @@ -230,6 +230,7 @@ Port for Dynamic Authorization Extension server (DM/CoA) + 1700 diff --git a/src/conf_mode/vpn_l2tp.py b/src/conf_mode/vpn_l2tp.py index fd5a4acd8..c533ad404 100755 --- a/src/conf_mode/vpn_l2tp.py +++ b/src/conf_mode/vpn_l2tp.py @@ -26,7 +26,10 @@ from ipaddress import ip_network from vyos.config import Config from vyos.template import is_ipv4 from vyos.template import render -from vyos.util import call, get_half_cpus +from vyos.util import call +from vyos.util import get_half_cpus +from vyos.util import check_port_availability +from vyos.util import is_listen_port_bind_service from vyos import ConfigError from vyos import airbag @@ -64,7 +67,7 @@ default_config_data = { 'radius_source_address': '', 'radius_shaper_attr': '', 'radius_shaper_vendor': '', - 'radius_dynamic_author': '', + 'radius_dynamic_author': {}, 'wins': [], 'ip6_column': [], 'thread_cnt': get_half_cpus() @@ -205,21 +208,21 @@ def get_config(config=None): l2tp['radius_source_address'] = conf.return_value(['source-address']) # Dynamic Authorization Extensions (DOA)/Change Of Authentication (COA) - if conf.exists(['dynamic-author']): + if conf.exists(['dae-server']): dae = { 'port' : '', 'server' : '', 'key' : '' } - if conf.exists(['dynamic-author', 'server']): - dae['server'] = conf.return_value(['dynamic-author', 'server']) + if conf.exists(['dae-server', 'ip-address']): + dae['server'] = conf.return_value(['dae-server', 'ip-address']) - if conf.exists(['dynamic-author', 'port']): - dae['port'] = conf.return_value(['dynamic-author', 'port']) + if conf.exists(['dae-server', 'port']): + dae['port'] = conf.return_value(['dae-server', 'port']) - if conf.exists(['dynamic-author', 'key']): - dae['key'] = conf.return_value(['dynamic-author', 'key']) + if conf.exists(['dae-server', 'secret']): + dae['key'] = conf.return_value(['dae-server', 'secret']) l2tp['radius_dynamic_author'] = dae @@ -329,6 +332,19 @@ def verify(l2tp): if not radius['key']: raise ConfigError(f"Missing RADIUS secret for server { radius['key'] }") + if l2tp['radius_dynamic_author']: + if not l2tp['radius_dynamic_author']['server']: + raise ConfigError("Missing ip-address for dae-server") + if not l2tp['radius_dynamic_author']['key']: + raise ConfigError("Missing secret for dae-server") + address = l2tp['radius_dynamic_author']['server'] + port = l2tp['radius_dynamic_author']['port'] + proto = 'tcp' + # check if dae listen port is not used by another service + if check_port_availability(address, int(port), proto) is not True and \ + not is_listen_port_bind_service(int(port), 'accel-pppd'): + raise ConfigError(f'"{proto}" port "{port}" is used by another service') + # check for the existence of a client ip pool if not (l2tp['client_ip_pool'] or l2tp['client_ip_subnets']): raise ConfigError( -- cgit v1.2.3 From 056746bbbdc0cc139d20b9fecb807c78d04c6097 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Fri, 9 Dec 2022 10:04:34 +0000 Subject: T4868: Fix l2tp ppp IPv6 options in template and config get dict L2TP 'ppp-options ipv6 x' can work without declaring IPv6 pool As we can get addresses via RADIUS attributes: - Framed-IPv6-Prefix - Delegated-IPv6-Prefix --- data/templates/accel-ppp/l2tp.config.j2 | 6 ++++-- src/conf_mode/vpn_l2tp.py | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/data/templates/accel-ppp/l2tp.config.j2 b/data/templates/accel-ppp/l2tp.config.j2 index 986f19656..3d1e835a9 100644 --- a/data/templates/accel-ppp/l2tp.config.j2 +++ b/data/templates/accel-ppp/l2tp.config.j2 @@ -121,8 +121,10 @@ lcp-echo-failure={{ ppp_echo_failure }} {% if ccp_disable %} ccp=0 {% endif %} -{% if client_ipv6_pool %} -ipv6=allow +{% if ppp_ipv6 is vyos_defined %} +ipv6={{ ppp_ipv6 }} +{% else %} +{{ 'ipv6=allow' if client_ipv6_pool_configured else '' }} {% endif %} diff --git a/src/conf_mode/vpn_l2tp.py b/src/conf_mode/vpn_l2tp.py index c533ad404..27e78db99 100755 --- a/src/conf_mode/vpn_l2tp.py +++ b/src/conf_mode/vpn_l2tp.py @@ -46,6 +46,7 @@ default_config_data = { 'client_ip_pool': None, 'client_ip_subnets': [], 'client_ipv6_pool': [], + 'client_ipv6_pool_configured': False, 'client_ipv6_delegate_prefix': [], 'dnsv4': [], 'dnsv6': [], @@ -247,6 +248,7 @@ def get_config(config=None): l2tp['client_ip_subnets'] = conf.return_values(['client-ip-pool', 'subnet']) if conf.exists(['client-ipv6-pool', 'prefix']): + l2tp['client_ipv6_pool_configured'] = True l2tp['ip6_column'].append('ip6') for prefix in conf.list_nodes(['client-ipv6-pool', 'prefix']): tmp = { @@ -309,6 +311,9 @@ def get_config(config=None): if conf.exists(['ppp-options', 'lcp-echo-interval']): l2tp['ppp_echo_interval'] = conf.return_value(['ppp-options', 'lcp-echo-interval']) + if conf.exists(['ppp-options', 'ipv6']): + l2tp['ppp_ipv6'] = conf.return_value(['ppp-options', 'ipv6']) + return l2tp -- cgit v1.2.3 From 779f4001a4828f1af39a5b0b861d62635fcb3726 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 9 Dec 2022 16:08:57 -0600 Subject: openvpn: T4872: fix parsing of tunnel IP in 'show openvpn server' --- src/op_mode/show_openvpn.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/op_mode/show_openvpn.py b/src/op_mode/show_openvpn.py index 9a5adcffb..e29e594a5 100755 --- a/src/op_mode/show_openvpn.py +++ b/src/op_mode/show_openvpn.py @@ -59,7 +59,11 @@ def get_vpn_tunnel_address(peer, interface): for line in lines: if peer in line: lst.append(line) - tunnel_ip = lst[1].split(',')[0] + + # filter out subnet entries + lst = [l for l in lst[1:] if '/' not in l.split(',')[0]] + + tunnel_ip = lst[0].split(',')[0] return tunnel_ip -- cgit v1.2.3 From 3296a10dc27d2fb1c79db07b26d62c8abb6a51bc Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sun, 11 Dec 2022 19:36:03 +0100 Subject: pppoe: T4384: remove unused import of leaf_node_changed --- src/conf_mode/interfaces-pppoe.py | 1 - 1 file changed, 1 deletion(-) (limited to 'src') diff --git a/src/conf_mode/interfaces-pppoe.py b/src/conf_mode/interfaces-pppoe.py index e2fdc7a42..ee4defa0d 100755 --- a/src/conf_mode/interfaces-pppoe.py +++ b/src/conf_mode/interfaces-pppoe.py @@ -23,7 +23,6 @@ from netifaces import interfaces from vyos.config import Config from vyos.configdict import get_interface_dict from vyos.configdict import is_node_changed -from vyos.configdict import leaf_node_changed from vyos.configdict import get_pppoe_interfaces from vyos.configverify import verify_authentication from vyos.configverify import verify_source_interface -- cgit v1.2.3 From ff56aeefddaad2d37d3ea32626e1adf3960eaf26 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sun, 11 Dec 2022 19:38:28 +0100 Subject: sstp: T4384: initial implementation of SSTP client CLI vyos@vyos# show interfaces sstpc sstpc sstpc10 { authentication { password vyos user vyos } server sstp.vyos.net ssl { ca-certificate VyOS-CA } } --- data/configd-include.json | 1 + data/templates/sstp-client/peer.j2 | 46 ++++++++ interface-definitions/interfaces-sstpc.xml.in | 47 ++++++++ op-mode-definitions/connect.xml.in | 1 + op-mode-definitions/disconnect.xml.in | 1 + op-mode-definitions/monitor-log.xml.in | 17 +++ op-mode-definitions/show-interfaces-sstpc.xml.in | 51 ++++++++ op-mode-definitions/show-log.xml.in | 17 +++ python/vyos/ifconfig/__init__.py | 3 +- python/vyos/ifconfig/sstpc.py | 40 +++++++ src/conf_mode/interfaces-sstpc.py | 142 +++++++++++++++++++++++ src/etc/ppp/ip-up.d/96-vyos-sstpc-callback | 49 ++++++++ src/op_mode/connect_disconnect.py | 4 +- 13 files changed, 416 insertions(+), 3 deletions(-) create mode 100644 data/templates/sstp-client/peer.j2 create mode 100644 interface-definitions/interfaces-sstpc.xml.in create mode 100644 op-mode-definitions/show-interfaces-sstpc.xml.in create mode 100644 python/vyos/ifconfig/sstpc.py create mode 100755 src/conf_mode/interfaces-sstpc.py create mode 100755 src/etc/ppp/ip-up.d/96-vyos-sstpc-callback (limited to 'src') diff --git a/data/configd-include.json b/data/configd-include.json index 5a4912e30..648655a8b 100644 --- a/data/configd-include.json +++ b/data/configd-include.json @@ -28,6 +28,7 @@ "interfaces-openvpn.py", "interfaces-pppoe.py", "interfaces-pseudo-ethernet.py", +"interfaces-sstpc.py", "interfaces-tunnel.py", "interfaces-vti.py", "interfaces-vxlan.py", diff --git a/data/templates/sstp-client/peer.j2 b/data/templates/sstp-client/peer.j2 new file mode 100644 index 000000000..1127d0564 --- /dev/null +++ b/data/templates/sstp-client/peer.j2 @@ -0,0 +1,46 @@ +### Autogenerated by interfaces-sstpc.py ### +{{ '# ' ~ description if description is vyos_defined else '' }} + +# Require peer to provide the local IP address if it is not +# specified explicitly in the config file. +noipdefault + +# Don't show the password in logfiles: +hide-password + +remotename {{ ifname }} +linkname {{ ifname }} +ipparam {{ ifname }} +ifname {{ ifname }} +pty "sstpc --ipparam {{ ifname }} --nolaunchpppd {{ server }}:{{ port }} --ca-cert {{ ca_file_path }}" + +# Override any connect script that may have been set in /etc/ppp/options. +connect /bin/true + +# Don't try to authenticate the remote node +noauth + +# We won't want EAP +refuse-eap + +# Don't try to proxy ARP for the remote endpoint. User can set proxy +# arp entries up manually if they wish. More importantly, having +# the "proxyarp" parameter set disables the "defaultroute" option. +noproxyarp + +# Unlimited connection attempts +maxfail 0 + +plugin sstp-pppd-plugin.so +sstp-sock /var/run/sstpc/sstpc-{{ ifname }} + +persist +debug + +{% if authentication is vyos_defined %} +{{ 'user "' + authentication.user + '"' if authentication.user is vyos_defined }} +{{ 'password "' + authentication.password + '"' if authentication.password is vyos_defined }} +{% endif %} + +{{ "usepeerdns" if no_peer_dns is not vyos_defined }} + diff --git a/interface-definitions/interfaces-sstpc.xml.in b/interface-definitions/interfaces-sstpc.xml.in new file mode 100644 index 000000000..30b55a9fa --- /dev/null +++ b/interface-definitions/interfaces-sstpc.xml.in @@ -0,0 +1,47 @@ + + + + + + + Secure Socket Tunneling Protocol (SSTP) client Interface + 460 + + sstpc[0-9]+ + + Secure Socket Tunneling Protocol interface must be named sstpcN + + sstpcN + Secure Socket Tunneling Protocol interface name + + + + #include + #include + #include + #include + #include + #include + #include + + 1452 + + #include + #include + + 443 + + + + Secure Sockets Layer (SSL) configuration + + + #include + + + #include + + + + + diff --git a/op-mode-definitions/connect.xml.in b/op-mode-definitions/connect.xml.in index d0c93195c..116cd6231 100644 --- a/op-mode-definitions/connect.xml.in +++ b/op-mode-definitions/connect.xml.in @@ -20,6 +20,7 @@ Bring up a connection-oriented network interface interfaces pppoe + interfaces sstpc interfaces wwan diff --git a/op-mode-definitions/disconnect.xml.in b/op-mode-definitions/disconnect.xml.in index 4415c0ed2..843998c4f 100644 --- a/op-mode-definitions/disconnect.xml.in +++ b/op-mode-definitions/disconnect.xml.in @@ -10,6 +10,7 @@ Take down a connection-oriented network interface interfaces pppoe + interfaces sstpc interfaces wwan diff --git a/op-mode-definitions/monitor-log.xml.in b/op-mode-definitions/monitor-log.xml.in index dccdfaf9a..1b1f53dc2 100644 --- a/op-mode-definitions/monitor-log.xml.in +++ b/op-mode-definitions/monitor-log.xml.in @@ -224,6 +224,23 @@ journalctl --no-hostname --boot --follow --unit ssh.service + + + Monitor last lines of SSTP client log + + journalctl --no-hostname --boot --follow --unit "ppp@sstpc*.service" + + + + Monitor last lines of SSTP client log for specific interface + + + + + journalctl --no-hostname --boot --follow --unit "ppp@$5.service" + + + Show log for Virtual Private Network (VPN) diff --git a/op-mode-definitions/show-interfaces-sstpc.xml.in b/op-mode-definitions/show-interfaces-sstpc.xml.in new file mode 100644 index 000000000..e66d3a0ac --- /dev/null +++ b/op-mode-definitions/show-interfaces-sstpc.xml.in @@ -0,0 +1,51 @@ + + + + + + + + + Show specified SSTP client interface information + + interfaces sstpc + + + ${vyos_op_scripts_dir}/show_interfaces.py --intf="$4" + + + + Show specified SSTP client interface log + + journalctl --no-hostname --boot --follow --unit "ppp@$4".service + + + + Show specified SSTP client interface statistics + + interfaces sstpc + + + if [ -d "/sys/class/net/$4" ]; then /usr/sbin/pppstats "$4"; fi + + + + + + Show SSTP client interface information + + ${vyos_op_scripts_dir}/show_interfaces.py --intf-type=sstpc --action=show-brief + + + + Show detailed SSTP client interface information + + ${vyos_op_scripts_dir}/show_interfaces.py --intf-type=sstpc --action=show + + + + + + + + diff --git a/op-mode-definitions/show-log.xml.in b/op-mode-definitions/show-log.xml.in index 404de1913..64a54015b 100644 --- a/op-mode-definitions/show-log.xml.in +++ b/op-mode-definitions/show-log.xml.in @@ -356,6 +356,23 @@ journalctl --no-hostname --boot --unit ssh.service + + + Show log for SSTP client + + journalctl --no-hostname --boot --unit "ppp@sstpc*.service" + + + + Show SSTP client log on specific interface + + + + + journalctl --no-hostname --boot --unit "ppp@$5.service" + + + Show last n changes to messages diff --git a/python/vyos/ifconfig/__init__.py b/python/vyos/ifconfig/__init__.py index d1ddaa13e..206b2bba1 100644 --- a/python/vyos/ifconfig/__init__.py +++ b/python/vyos/ifconfig/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 VyOS maintainers and contributors +# Copyright 2019-2022 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -38,3 +38,4 @@ from vyos.ifconfig.l2tpv3 import L2TPv3If from vyos.ifconfig.macsec import MACsecIf from vyos.ifconfig.veth import VethIf from vyos.ifconfig.wwan import WWANIf +from vyos.ifconfig.sstpc import SSTPCIf diff --git a/python/vyos/ifconfig/sstpc.py b/python/vyos/ifconfig/sstpc.py new file mode 100644 index 000000000..50fc6ee6b --- /dev/null +++ b/python/vyos/ifconfig/sstpc.py @@ -0,0 +1,40 @@ +# Copyright 2022 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . + +from vyos.ifconfig.interface import Interface + +@Interface.register +class SSTPCIf(Interface): + iftype = 'sstpc' + definition = { + **Interface.definition, + **{ + 'section': 'sstpc', + 'prefixes': ['sstpc', ], + 'eternal': 'sstpc[0-9]+$', + }, + } + + def _create(self): + # we can not create this interface as it is managed outside + pass + + def _delete(self): + # we can not create this interface as it is managed outside + pass + + def get_mac(self): + """ Get a synthetic MAC address. """ + return self.get_mac_synthetic() diff --git a/src/conf_mode/interfaces-sstpc.py b/src/conf_mode/interfaces-sstpc.py new file mode 100755 index 000000000..6b8094c51 --- /dev/null +++ b/src/conf_mode/interfaces-sstpc.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +from sys import exit + +from vyos.config import Config +from vyos.configdict import get_interface_dict +from vyos.configdict import is_node_changed +from vyos.configverify import verify_authentication +from vyos.configverify import verify_vrf +from vyos.ifconfig import SSTPCIf +from vyos.pki import encode_certificate +from vyos.pki import find_chain +from vyos.pki import load_certificate +from vyos.template import render +from vyos.util import call +from vyos.util import dict_search +from vyos.util import is_systemd_service_running +from vyos.util import write_file +from vyos import ConfigError +from vyos import airbag +airbag.enable() + +def get_config(config=None): + """ + Retrive CLI config as dictionary. Dictionary can never be empty, as at least the + interface name will be added or a deleted flag + """ + if config: + conf = config + else: + conf = Config() + base = ['interfaces', 'sstpc'] + ifname, sstpc = get_interface_dict(conf, base) + + # We should only terminate the SSTP client session if critical parameters + # change. All parameters that can be changed on-the-fly (like interface + # description) should not lead to a reconnect! + for options in ['authentication', 'no_peer_dns', 'no_default_route', + 'server', 'ssl']: + if is_node_changed(conf, base + [ifname, options]): + sstpc.update({'shutdown_required': {}}) + # bail out early - no need to further process other nodes + break + + # Load PKI certificates for later processing + sstpc['pki'] = conf.get_config_dict(['pki'], key_mangling=('-', '_'), + get_first_key=True, + no_tag_node_value_mangle=True) + return sstpc + +def verify(sstpc): + if 'deleted' in sstpc: + return None + + verify_authentication(sstpc) + verify_vrf(sstpc) + + if dict_search('ssl.ca_certificate', sstpc) == None: + raise ConfigError('Missing mandatory CA certificate!') + + return None + +def generate(sstpc): + ifname = sstpc['ifname'] + config_sstpc = f'/etc/ppp/peers/{ifname}' + + sstpc['ca_file_path'] = f'/run/sstpc/{ifname}_ca-cert.pem' + + if 'deleted' in sstpc: + for file in [sstpc['ca_file_path'], config_sstpc]: + if os.path.exists(file): + os.unlink(file) + return None + + ca_name = sstpc['ssl']['ca_certificate'] + pki_ca_cert = sstpc['pki']['ca'][ca_name] + + loaded_ca_cert = load_certificate(pki_ca_cert['certificate']) + loaded_ca_certs = {load_certificate(c['certificate']) + for c in sstpc['pki']['ca'].values()} if 'ca' in sstpc['pki'] else {} + + ca_full_chain = find_chain(loaded_ca_cert, loaded_ca_certs) + + write_file(sstpc['ca_file_path'], '\n'.join(encode_certificate(c) for c in ca_full_chain)) + render(config_sstpc, 'sstp-client/peer.j2', sstpc, permission=0o640) + + return None + +def apply(sstpc): + ifname = sstpc['ifname'] + if 'deleted' in sstpc or 'disable' in sstpc: + if os.path.isdir(f'/sys/class/net/{ifname}'): + p = SSTPCIf(ifname) + p.remove() + call(f'systemctl stop ppp@{ifname}.service') + return None + + # reconnect should only be necessary when specific options change, + # like server, authentication ... (see get_config() for details) + if ((not is_systemd_service_running(f'ppp@{ifname}.service')) or + 'shutdown_required' in sstpc): + + # cleanup system (e.g. FRR routes first) + if os.path.isdir(f'/sys/class/net/{ifname}'): + p = SSTPCIf(ifname) + p.remove() + + call(f'systemctl restart ppp@{ifname}.service') + # When interface comes "live" a hook is called: + # /etc/ppp/ip-up.d/96-vyos-sstpc-callback + # which triggers SSTPCIf.update() + else: + if os.path.isdir(f'/sys/class/net/{ifname}'): + p = SSTPCIf(ifname) + p.update(sstpc) + + return None + +if __name__ == '__main__': + try: + c = get_config() + verify(c) + generate(c) + apply(c) + except ConfigError as e: + print(e) + exit(1) diff --git a/src/etc/ppp/ip-up.d/96-vyos-sstpc-callback b/src/etc/ppp/ip-up.d/96-vyos-sstpc-callback new file mode 100755 index 000000000..4e8804f29 --- /dev/null +++ b/src/etc/ppp/ip-up.d/96-vyos-sstpc-callback @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This is a Python hook script which is invoked whenever a SSTP client session +# goes "ip-up". It will call into our vyos.ifconfig library and will then +# execute common tasks for the SSTP interface. The reason we have to "hook" this +# is that we can not create a sstpcX interface in advance in linux and then +# connect pppd to this already existing interface. + +from sys import argv +from sys import exit + +from vyos.configquery import ConfigTreeQuery +from vyos.configdict import get_interface_dict +from vyos.ifconfig import SSTPCIf + +# When the ppp link comes up, this script is called with the following +# parameters +# $1 the interface name used by pppd (e.g. ppp3) +# $2 the tty device name +# $3 the tty device speed +# $4 the local IP address for the interface +# $5 the remote IP address +# $6 the parameter specified by the 'ipparam' option to pppd + +if (len(argv) < 7): + exit(1) + +interface = argv[6] + +conf = ConfigTreeQuery() +_, sstpc = get_interface_dict(conf.config, ['interfaces', 'sstpc'], interface) + +# Update the config +p = SSTPCIf(interface) +p.update(sstpc) diff --git a/src/op_mode/connect_disconnect.py b/src/op_mode/connect_disconnect.py index 936c20bcb..d39e88bf3 100755 --- a/src/op_mode/connect_disconnect.py +++ b/src/op_mode/connect_disconnect.py @@ -41,7 +41,7 @@ def check_ppp_running(interface): def connect(interface): """ Connect dialer interface """ - if interface.startswith('ppp'): + if interface.startswith('pppoe') or interface.startswith('sstpc'): check_ppp_interface(interface) # Check if interface is already dialed if os.path.isdir(f'/sys/class/net/{interface}'): @@ -62,7 +62,7 @@ def connect(interface): def disconnect(interface): """ Disconnect dialer interface """ - if interface.startswith('ppp'): + if interface.startswith('pppoe') or interface.startswith('sstpc'): check_ppp_interface(interface) # Check if interface is already down -- cgit v1.2.3 From 6a6493b11a33f878d7700b8e4ac7c65bb5edb8d2 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 6 Dec 2022 09:07:11 -0600 Subject: openvpn: T4770: add openvpn.py with standardized show command --- src/op_mode/openvpn.py | 210 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100755 src/op_mode/openvpn.py (limited to 'src') diff --git a/src/op_mode/openvpn.py b/src/op_mode/openvpn.py new file mode 100755 index 000000000..d66fa08fe --- /dev/null +++ b/src/op_mode/openvpn.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# + +import os +import sys +from tabulate import tabulate + +import vyos.opmode +from vyos.util import bytes_to_human +from vyos.config import Config + +def _get_tunnel_address(peer_host, peer_port, status_file): + peer = peer_host + ':' + peer_port + lst = [] + + with open(status_file, 'r') as f: + lines = f.readlines() + for line in lines: + if peer in line: + lst.append(line) + + # filter out subnet entries if iroute: + # in the case that one sets, say: + # [ ..., 'vtun10', 'server', 'client', 'client1', 'subnet','10.10.2.0/25'] + # the status file will have an entry: + # 10.10.2.0/25,client1,... + lst = [l for l in lst[1:] if '/' not in l.split(',')[0]] + + tunnel_ip = lst[0].split(',')[0] + + return tunnel_ip + +def _get_interface_status(mode: str, interface: str) -> dict: + status_file = f'/run/openvpn/{interface}.status' + + data = { + 'mode': mode, + 'intf': interface, + 'local_host': '', + 'local_port': '', + 'date': '', + 'clients': [], + } + + if not os.path.exists(status_file): + raise vyos.opmode.DataUnavailable('No information for interface {interface}') + + with open(status_file, 'r') as f: + lines = f.readlines() + for line_no, line in enumerate(lines): + # remove trailing newline character first + line = line.rstrip('\n') + + # check first line header + if line_no == 0: + if mode == 'server': + if not line == 'OpenVPN CLIENT LIST': + raise vyos.opmode.InternalError('Expected "OpenVPN CLIENT LIST"') + else: + if not line == 'OpenVPN STATISTICS': + raise vyos.opmode.InternalError('Expected "OpenVPN STATISTICS"') + + continue + + # second line informs us when the status file has been last updated + if line_no == 1: + data['date'] = line.lstrip('Updated,').rstrip('\n') + continue + + if mode == 'server': + # for line_no > 1, lines appear as follows: + # + # Common Name,Real Address,Bytes Received,Bytes Sent,Connected Since + # client1,172.18.202.10:55904,2880587,2882653,Fri Aug 23 16:25:48 2019 + # client3,172.18.204.10:41328,2850832,2869729,Fri Aug 23 16:25:43 2019 + # client2,172.18.203.10:48987,2856153,2871022,Fri Aug 23 16:25:45 2019 + # ... + # ROUTING TABLE + # ... + if line_no >= 3: + # indicator that there are no more clients + if line == 'ROUTING TABLE': + break + # otherwise, get client data + remote = (line.split(',')[1]).rsplit(':', maxsplit=1) + + client = { + 'name': line.split(',')[0], + 'remote_host': remote[0], + 'remote_port': remote[1], + 'tunnel': 'N/A', + 'rx_bytes': bytes_to_human(int(line.split(',')[2]), + precision=1), + 'tx_bytes': bytes_to_human(int(line.split(',')[3]), + precision=1), + 'online_since': line.split(',')[4] + } + client['tunnel'] = _get_tunnel_address(client['remote_host'], + client['remote_port'], + status_file) + data['clients'].append(client) + continue + else: # mode == 'client' or mode == 'site-to-site' + if line_no == 2: + client = { + 'name': 'N/A', + 'remote_host': 'N/A', + 'remote_port': 'N/A', + 'tunnel': 'N/A', + 'rx_bytes': bytes_to_human(int(line.split(',')[1]), + precision=1), + 'tx_bytes': '', + 'online_since': 'N/A' + } + continue + + if line_no == 3: + client['tx_bytes'] = bytes_to_human(int(line.split(',')[1]), + precision=1) + data['clients'].append(client) + break + + return data + +def _get_raw_data(mode: str) -> dict: + data = {} + conf = Config() + conf_dict = conf.get_config_dict(['interfaces', 'openvpn'], + get_first_key=True) + if not conf_dict: + return data + + interfaces = [x for x in list(conf_dict) if conf_dict[x]['mode'] == mode] + for intf in interfaces: + data[intf] = _get_interface_status(mode, intf) + d = data[intf] + d['local_host'] = conf_dict[intf].get('local-host', '') + d['local_port'] = conf_dict[intf].get('local-port', '') + if mode in ['client', 'site-to-site']: + for client in d['clients']: + if 'shared-secret-key-file' in list(conf_dict[intf]): + client['name'] = 'None (PSK)' + client['remote_host'] = conf_dict[intf].get('remote-host', [''])[0] + client['remote_port'] = conf_dict[intf].get('remote-port', '1194') + + return data + +def _format_openvpn(data: dict) -> str: + if not data: + out = 'No OpenVPN interfaces configured' + return out + + headers = ['Client CN', 'Remote Host', 'Tunnel IP', 'Local Host', + 'TX bytes', 'RX bytes', 'Connected Since'] + + out = '' + data_out = [] + for intf in list(data): + l_host = data[intf]['local_host'] + l_port = data[intf]['local_port'] + for client in list(data[intf]['clients']): + r_host = client['remote_host'] + r_port = client['remote_port'] + + out += f'\nOpenVPN status on {intf}\n\n' + name = client['name'] + remote = r_host + ':' + r_port if r_host and r_port else 'N/A' + tunnel = client['tunnel'] + local = l_host + ':' + l_port if l_host and l_port else 'N/A' + tx_bytes = client['tx_bytes'] + rx_bytes = client['rx_bytes'] + online_since = client['online_since'] + data_out.append([name, remote, tunnel, local, tx_bytes, + rx_bytes, online_since]) + + out += tabulate(data_out, headers) + + return out + +def show(raw: bool, mode: str) -> str: + openvpn_data = _get_raw_data(mode) + + if raw: + return openvpn_data + + return _format_openvpn(openvpn_data) + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) -- cgit v1.2.3 From 9af23d814aa37e90aca350375eeb7b9e83069eee Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 6 Dec 2022 14:10:21 -0600 Subject: openvpn: T4770: add reset function to openvpn.py --- src/op_mode/openvpn.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'src') diff --git a/src/op_mode/openvpn.py b/src/op_mode/openvpn.py index d66fa08fe..3797a7153 100755 --- a/src/op_mode/openvpn.py +++ b/src/op_mode/openvpn.py @@ -22,6 +22,8 @@ from tabulate import tabulate import vyos.opmode from vyos.util import bytes_to_human +from vyos.util import commit_in_progress +from vyos.util import call from vyos.config import Config def _get_tunnel_address(peer_host, peer_port, status_file): @@ -200,6 +202,14 @@ def show(raw: bool, mode: str) -> str: return _format_openvpn(openvpn_data) +def reset(interface: str): + if os.path.isfile(f'/run/openvpn/{interface}.conf'): + if commit_in_progress(): + raise vyos.opmode.CommitInProgress('Retry OpenVPN reset: commit in progress.') + call(f'systemctl restart openvpn@{interface}.service') + else: + raise vyos.opmode.IncorrectValue(f'OpenVPN interface "{interface}" does not exist!') + if __name__ == '__main__': try: res = vyos.opmode.run(sys.modules[__name__]) -- cgit v1.2.3 From 046bb9ccd56ac5e97c638bb4a9ca856d3d36026a Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 13 Dec 2022 12:09:03 -0600 Subject: validators: T4798: replace python file-exists validator with file-path --- interface-definitions/include/certificate-ca.xml.i | 2 +- .../include/certificate-key.xml.i | 2 +- interface-definitions/include/certificate.xml.i | 2 +- interface-definitions/protocols-rpki.xml.in | 6 +-- src/validators/file-exists | 61 ---------------------- 5 files changed, 6 insertions(+), 67 deletions(-) delete mode 100755 src/validators/file-exists (limited to 'src') diff --git a/interface-definitions/include/certificate-ca.xml.i b/interface-definitions/include/certificate-ca.xml.i index b97378658..3cde2a48d 100644 --- a/interface-definitions/include/certificate-ca.xml.i +++ b/interface-definitions/include/certificate-ca.xml.i @@ -7,7 +7,7 @@ File in /config/auth directory - + diff --git a/interface-definitions/include/certificate-key.xml.i b/interface-definitions/include/certificate-key.xml.i index 1db9dd069..2c4d81fbb 100644 --- a/interface-definitions/include/certificate-key.xml.i +++ b/interface-definitions/include/certificate-key.xml.i @@ -7,7 +7,7 @@ File in /config/auth directory - + diff --git a/interface-definitions/include/certificate.xml.i b/interface-definitions/include/certificate.xml.i index fb5be45cc..6a5b2936c 100644 --- a/interface-definitions/include/certificate.xml.i +++ b/interface-definitions/include/certificate.xml.i @@ -7,7 +7,7 @@ File in /config/auth directory - + diff --git a/interface-definitions/protocols-rpki.xml.in b/interface-definitions/protocols-rpki.xml.in index 4535d3990..0098cacb6 100644 --- a/interface-definitions/protocols-rpki.xml.in +++ b/interface-definitions/protocols-rpki.xml.in @@ -51,7 +51,7 @@ RPKI SSH known hosts file - + @@ -59,7 +59,7 @@ RPKI SSH private key file - + @@ -67,7 +67,7 @@ RPKI SSH public key file path - + diff --git a/src/validators/file-exists b/src/validators/file-exists deleted file mode 100755 index 5cef6b199..000000000 --- a/src/validators/file-exists +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2019 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# Description: -# Check if a given file exists on the system. Used for files that -# are referenced from the CLI and need to be preserved during an image upgrade. -# Warn the user if these aren't under /config - -import os -import sys -import argparse - - -def exit(strict, message): - if strict: - sys.exit(f'ERROR: {message}') - print(f'WARNING: {message}', file=sys.stderr) - sys.exit() - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("-d", "--directory", type=str, help="File must be present in this directory.") - parser.add_argument("-e", "--error", action="store_true", help="Tread warnings as errors - change exit code to '1'") - parser.add_argument("file", type=str, help="Path of file to validate") - - args = parser.parse_args() - - # - # Always check if the given file exists - # - if not os.path.exists(args.file): - exit(args.error, f"File '{args.file}' not found") - - # - # Optional check if the file is under a certain directory path - # - if args.directory: - # remove directory path from path to verify - rel_filename = args.file.replace(args.directory, '').lstrip('/') - - if not os.path.exists(args.directory + '/' + rel_filename): - exit(args.error, - f"'{args.file}' lies outside of '{args.directory}' directory.\n" - "It will not get preserved during image upgrade!" - ) - - sys.exit() -- cgit v1.2.3 From f0bc6c62016d285f0645c4b3ba8b1451c40c637f Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Mon, 12 Dec 2022 15:06:08 -0600 Subject: validators: T4875: use file-path to replace validator 'interface-name' --- interface-definitions/dns-domain-name.xml.in | 2 +- interface-definitions/high-availability.xml.in | 2 +- .../include/bgp/neighbor-update-source.xml.i | 2 +- .../include/bgp/protocol-common-config.xml.i | 2 +- .../include/constraint/interface-name.xml.in | 4 +++ interface-definitions/include/dhcp-interface.xml.i | 2 +- .../include/generic-interface-broadcast.xml.i | 2 +- .../generic-interface-multi-broadcast.xml.i | 2 +- .../include/generic-interface-multi.xml.i | 2 +- .../include/generic-interface.xml.i | 2 +- .../include/interface/redirect.xml.i | 2 +- .../include/ospf/protocol-common-config.xml.i | 2 +- .../include/ospfv3/protocol-common-config.xml.i | 2 +- interface-definitions/include/rip/interface.xml.i | 2 +- .../include/routing-passive-interface.xml.i | 2 +- .../include/source-interface.xml.i | 2 +- .../include/static/static-route-interface.xml.i | 2 +- .../include/static/static-route.xml.i | 2 +- .../include/static/static-route6.xml.i | 2 +- interface-definitions/interfaces-bonding.xml.in | 4 +-- interface-definitions/protocols-rip.xml.in | 2 +- interface-definitions/protocols-ripng.xml.in | 2 +- interface-definitions/protocols-static-arp.xml.in | 2 +- interface-definitions/qos.xml.in | 2 +- interface-definitions/service-upnp.xml.in | 4 +-- src/validators/interface-name | 34 ---------------------- 26 files changed, 30 insertions(+), 60 deletions(-) create mode 100644 interface-definitions/include/constraint/interface-name.xml.in delete mode 100755 src/validators/interface-name (limited to 'src') diff --git a/interface-definitions/dns-domain-name.xml.in b/interface-definitions/dns-domain-name.xml.in index 70b2fb271..9aca38735 100644 --- a/interface-definitions/dns-domain-name.xml.in +++ b/interface-definitions/dns-domain-name.xml.in @@ -25,7 +25,7 @@ - + #include diff --git a/interface-definitions/high-availability.xml.in b/interface-definitions/high-availability.xml.in index 0631acdda..784e51151 100644 --- a/interface-definitions/high-availability.xml.in +++ b/interface-definitions/high-availability.xml.in @@ -199,7 +199,7 @@ Interface name - + #include diff --git a/interface-definitions/include/bgp/neighbor-update-source.xml.i b/interface-definitions/include/bgp/neighbor-update-source.xml.i index 37faf2cce..60c127e8f 100644 --- a/interface-definitions/include/bgp/neighbor-update-source.xml.i +++ b/interface-definitions/include/bgp/neighbor-update-source.xml.i @@ -22,7 +22,7 @@ - + #include diff --git a/interface-definitions/include/bgp/protocol-common-config.xml.i b/interface-definitions/include/bgp/protocol-common-config.xml.i index fe192434d..366630f78 100644 --- a/interface-definitions/include/bgp/protocol-common-config.xml.i +++ b/interface-definitions/include/bgp/protocol-common-config.xml.i @@ -926,7 +926,7 @@ - + #include diff --git a/interface-definitions/include/constraint/interface-name.xml.in b/interface-definitions/include/constraint/interface-name.xml.in new file mode 100644 index 000000000..2d1f7b757 --- /dev/null +++ b/interface-definitions/include/constraint/interface-name.xml.in @@ -0,0 +1,4 @@ + +(bond|br|dum|en|ersp|eth|gnv|lan|l2tp|l2tpeth|macsec|peth|ppp|pppoe|pptp|sstp|tun|veth|vti|vtun|vxlan|wg|wlan|wwan)[0-9]+(.\d+)?|lo + + diff --git a/interface-definitions/include/dhcp-interface.xml.i b/interface-definitions/include/dhcp-interface.xml.i index 939b45f15..f5107ba2b 100644 --- a/interface-definitions/include/dhcp-interface.xml.i +++ b/interface-definitions/include/dhcp-interface.xml.i @@ -9,7 +9,7 @@ DHCP interface name - + #include diff --git a/interface-definitions/include/generic-interface-broadcast.xml.i b/interface-definitions/include/generic-interface-broadcast.xml.i index 6f76dde1a..af35a888b 100644 --- a/interface-definitions/include/generic-interface-broadcast.xml.i +++ b/interface-definitions/include/generic-interface-broadcast.xml.i @@ -10,7 +10,7 @@ Interface name - + #include diff --git a/interface-definitions/include/generic-interface-multi-broadcast.xml.i b/interface-definitions/include/generic-interface-multi-broadcast.xml.i index 00638f3b7..1ae38fb43 100644 --- a/interface-definitions/include/generic-interface-multi-broadcast.xml.i +++ b/interface-definitions/include/generic-interface-multi-broadcast.xml.i @@ -10,7 +10,7 @@ Interface name - + #include diff --git a/interface-definitions/include/generic-interface-multi.xml.i b/interface-definitions/include/generic-interface-multi.xml.i index 65aae28ae..16916ff54 100644 --- a/interface-definitions/include/generic-interface-multi.xml.i +++ b/interface-definitions/include/generic-interface-multi.xml.i @@ -10,7 +10,7 @@ Interface name - + #include diff --git a/interface-definitions/include/generic-interface.xml.i b/interface-definitions/include/generic-interface.xml.i index 8b4cf1d65..36ddee417 100644 --- a/interface-definitions/include/generic-interface.xml.i +++ b/interface-definitions/include/generic-interface.xml.i @@ -10,7 +10,7 @@ Interface name - + #include diff --git a/interface-definitions/include/interface/redirect.xml.i b/interface-definitions/include/interface/redirect.xml.i index 3be9ee16b..8df8957ac 100644 --- a/interface-definitions/include/interface/redirect.xml.i +++ b/interface-definitions/include/interface/redirect.xml.i @@ -10,7 +10,7 @@ Interface name - + #include diff --git a/interface-definitions/include/ospf/protocol-common-config.xml.i b/interface-definitions/include/ospf/protocol-common-config.xml.i index 0615063af..06609c10e 100644 --- a/interface-definitions/include/ospf/protocol-common-config.xml.i +++ b/interface-definitions/include/ospf/protocol-common-config.xml.i @@ -358,7 +358,7 @@ Interface name - + #include diff --git a/interface-definitions/include/ospfv3/protocol-common-config.xml.i b/interface-definitions/include/ospfv3/protocol-common-config.xml.i index 630534eea..c0aab912d 100644 --- a/interface-definitions/include/ospfv3/protocol-common-config.xml.i +++ b/interface-definitions/include/ospfv3/protocol-common-config.xml.i @@ -118,7 +118,7 @@ Interface used for routing information exchange - + #include diff --git a/interface-definitions/include/rip/interface.xml.i b/interface-definitions/include/rip/interface.xml.i index baeceac1c..e0792cdc1 100644 --- a/interface-definitions/include/rip/interface.xml.i +++ b/interface-definitions/include/rip/interface.xml.i @@ -10,7 +10,7 @@ Interface name - + #include diff --git a/interface-definitions/include/routing-passive-interface.xml.i b/interface-definitions/include/routing-passive-interface.xml.i index 095b683de..fe229aebe 100644 --- a/interface-definitions/include/routing-passive-interface.xml.i +++ b/interface-definitions/include/routing-passive-interface.xml.i @@ -16,7 +16,7 @@ (default) - + #include diff --git a/interface-definitions/include/source-interface.xml.i b/interface-definitions/include/source-interface.xml.i index a9c2a0f9d..4c1fddb57 100644 --- a/interface-definitions/include/source-interface.xml.i +++ b/interface-definitions/include/source-interface.xml.i @@ -10,7 +10,7 @@ - + #include diff --git a/interface-definitions/include/static/static-route-interface.xml.i b/interface-definitions/include/static/static-route-interface.xml.i index ed4f455e5..cc7a92612 100644 --- a/interface-definitions/include/static/static-route-interface.xml.i +++ b/interface-definitions/include/static/static-route-interface.xml.i @@ -10,7 +10,7 @@ Gateway interface name - + #include diff --git a/interface-definitions/include/static/static-route.xml.i b/interface-definitions/include/static/static-route.xml.i index 04ee999c7..aeb2044c9 100644 --- a/interface-definitions/include/static/static-route.xml.i +++ b/interface-definitions/include/static/static-route.xml.i @@ -26,7 +26,7 @@ Gateway interface name - + #include diff --git a/interface-definitions/include/static/static-route6.xml.i b/interface-definitions/include/static/static-route6.xml.i index 6131ac7fe..d5e7a25bc 100644 --- a/interface-definitions/include/static/static-route6.xml.i +++ b/interface-definitions/include/static/static-route6.xml.i @@ -25,7 +25,7 @@ Gateway interface name - + #include diff --git a/interface-definitions/interfaces-bonding.xml.in b/interface-definitions/interfaces-bonding.xml.in index 96e0e5d89..a8a558348 100644 --- a/interface-definitions/interfaces-bonding.xml.in +++ b/interface-definitions/interfaces-bonding.xml.in @@ -199,7 +199,7 @@ Interface name - + #include @@ -218,7 +218,7 @@ Interface name - + #include diff --git a/interface-definitions/protocols-rip.xml.in b/interface-definitions/protocols-rip.xml.in index 2195b0316..33aae5015 100644 --- a/interface-definitions/protocols-rip.xml.in +++ b/interface-definitions/protocols-rip.xml.in @@ -39,7 +39,7 @@ - + #include diff --git a/interface-definitions/protocols-ripng.xml.in b/interface-definitions/protocols-ripng.xml.in index d7e4b2514..cd35dbf53 100644 --- a/interface-definitions/protocols-ripng.xml.in +++ b/interface-definitions/protocols-ripng.xml.in @@ -40,7 +40,7 @@ - + #include diff --git a/interface-definitions/protocols-static-arp.xml.in b/interface-definitions/protocols-static-arp.xml.in index 8b1b3b5e1..52caf435a 100644 --- a/interface-definitions/protocols-static-arp.xml.in +++ b/interface-definitions/protocols-static-arp.xml.in @@ -20,7 +20,7 @@ Interface name - + #include diff --git a/interface-definitions/qos.xml.in b/interface-definitions/qos.xml.in index e2dbcbeef..dc807781e 100644 --- a/interface-definitions/qos.xml.in +++ b/interface-definitions/qos.xml.in @@ -16,7 +16,7 @@ Interface name - + #include diff --git a/interface-definitions/service-upnp.xml.in b/interface-definitions/service-upnp.xml.in index ec23d87df..79d8ae42e 100644 --- a/interface-definitions/service-upnp.xml.in +++ b/interface-definitions/service-upnp.xml.in @@ -24,7 +24,7 @@ - + #include @@ -119,7 +119,7 @@ - + #include diff --git a/src/validators/interface-name b/src/validators/interface-name deleted file mode 100755 index 105815eee..000000000 --- a/src/validators/interface-name +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2021 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import re - -from sys import argv -from sys import exit - -pattern = '^(bond|br|dum|en|ersp|eth|gnv|lan|l2tp|l2tpeth|macsec|peth|ppp|pppoe|pptp|sstp|tun|vti|vtun|vxlan|wg|wlan|wwan)[0-9]+(.\d+)?|lo$' - -if __name__ == '__main__': - if len(argv) != 2: - exit(1) - interface = argv[1] - - if re.match(pattern, interface): - exit(0) - if os.path.exists(f'/sys/class/net/{interface}'): - exit(0) - exit(1) -- cgit v1.2.3 From c4e56fe4f4d725592a873c1a77b0d11d347a2530 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 14 Dec 2022 11:47:41 -0600 Subject: ocserv: T4881: return vyos.opmode.Errors on failure --- src/op_mode/openconnect.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/op_mode/openconnect.py b/src/op_mode/openconnect.py index 00992c66a..b21890728 100755 --- a/src/op_mode/openconnect.py +++ b/src/op_mode/openconnect.py @@ -31,14 +31,7 @@ occtl_socket = '/run/ocserv/occtl.socket' def _get_raw_data_sessions(): rc, out = rc_cmd(f'sudo {occtl} --json --socket-file {occtl_socket} show users') if rc != 0: - output = {'openconnect': - { - 'configured': False, - 'return_code': rc, - 'reason': out - } - } - return output + raise vyos.opmode.DataUnavailable(out) sessions = json.loads(out) return sessions @@ -61,9 +54,8 @@ def _get_formatted_sessions(data): def show_sessions(raw: bool): config = ConfigTreeQuery() - if not config.exists('vpn openconnect') and not raw: - print('Openconnect is not configured') - exit(0) + if not config.exists('vpn openconnect'): + raise vyos.opmode.UnconfiguredSubsystem('Openconnect is not configured') openconnect_data = _get_raw_data_sessions() if raw: -- cgit v1.2.3 From 932af7f098808009f626c788deb9e1d1c8bf3426 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 13 Jun 2022 15:40:11 +0000 Subject: routing: T1237: Add new feature failover route Failover route allows to install static routes to the kernel routing table only if required target or gateway is alive When target or gateway doesn't respond for ICMP/ARP checks this route deleted from the routing table Routes are marked as protocol 'failover' (rt_protos) cat /etc/iproute2/rt_protos.d/failover.conf 111 failover ip route add 203.0.113.1 metric 2 via 192.0.2.1 dev eth0 proto failover $ sudo ip route show proto failover 203.0.113.1 via 192.0.2.1 dev eth0 metric 1 So we can safely flush such routes --- .../protocols/systemd_vyos_failover_service.j2 | 11 ++ interface-definitions/protocols-failover.xml.in | 114 +++++++++++++ src/conf_mode/protocols_failover.py | 121 ++++++++++++++ src/helpers/vyos-failover.py | 184 +++++++++++++++++++++ 4 files changed, 430 insertions(+) create mode 100644 data/templates/protocols/systemd_vyos_failover_service.j2 create mode 100644 interface-definitions/protocols-failover.xml.in create mode 100755 src/conf_mode/protocols_failover.py create mode 100755 src/helpers/vyos-failover.py (limited to 'src') diff --git a/data/templates/protocols/systemd_vyos_failover_service.j2 b/data/templates/protocols/systemd_vyos_failover_service.j2 new file mode 100644 index 000000000..e6501e0f5 --- /dev/null +++ b/data/templates/protocols/systemd_vyos_failover_service.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Failover route service +After=vyos-router.service + +[Service] +Type=simple +Restart=always +ExecStart=/usr/bin/python3 /usr/libexec/vyos/vyos-failover.py --config /run/vyos-failover.conf + +[Install] +WantedBy=multi-user.target diff --git a/interface-definitions/protocols-failover.xml.in b/interface-definitions/protocols-failover.xml.in new file mode 100644 index 000000000..900c76eab --- /dev/null +++ b/interface-definitions/protocols-failover.xml.in @@ -0,0 +1,114 @@ + + + + + + + Failover Routing + 490 + + + + + Failover IPv4 route + + ipv4net + IPv4 failover route + + + + + + + + + Next-hop IPv4 router address + + ipv4 + Next-hop router address + + + + + + + + + Check target options + + + #include + + + Check target address + + ipv4 + Address to check + + + + + + + + + Timeout between checks + + u32:1-300 + Timeout in seconds between checks + + + + + + 10 + + + + Check type + + arp icmp tcp + + + arp + Check target by ARP + + + icmp + Check target by ICMP + + + tcp + Check target by TCP + + + (arp|icmp|tcp) + + + icmp + + + + #include + + + Route metric for this gateway + + u32:1-255 + Route metric + + + + + + 1 + + + + + + + + + + diff --git a/src/conf_mode/protocols_failover.py b/src/conf_mode/protocols_failover.py new file mode 100755 index 000000000..048ba7a89 --- /dev/null +++ b/src/conf_mode/protocols_failover.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import json + +from pathlib import Path + +from vyos.config import Config +from vyos.configdict import dict_merge +from vyos.template import render +from vyos.util import call +from vyos.xml import defaults +from vyos import ConfigError +from vyos import airbag + +airbag.enable() + + +service_name = 'vyos-failover' +service_conf = Path(f'/run/{service_name}.conf') +systemd_service = '/etc/systemd/system/vyos-failover.service' +rt_proto_failover = '/etc/iproute2/rt_protos.d/failover.conf' + + +def get_config(config=None): + if config: + conf = config + else: + conf = Config() + + base = ['protocols', 'failover'] + failover = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) + + # Set default values only if we set config + if failover.get('route'): + for route, route_config in failover.get('route').items(): + for next_hop, next_hop_config in route_config.get('next_hop').items(): + default_values = defaults(base + ['route']) + failover['route'][route]['next_hop'][next_hop] = dict_merge( + default_values['next_hop'], failover['route'][route]['next_hop'][next_hop]) + + return failover + +def verify(failover): + # bail out early - looks like removal from running config + if not failover: + return None + + if 'route' not in failover: + raise ConfigError(f'Failover "route" is mandatory!') + + for route, route_config in failover['route'].items(): + if not route_config.get('next_hop'): + raise ConfigError(f'Next-hop for "{route}" is mandatory!') + + for next_hop, next_hop_config in route_config.get('next_hop').items(): + if 'interface' not in next_hop_config: + raise ConfigError(f'Interface for route "{route}" next-hop "{next_hop}" is mandatory!') + + if not next_hop_config.get('check'): + raise ConfigError(f'Check target for next-hop "{next_hop}" is mandatory!') + + if 'target' not in next_hop_config['check']: + raise ConfigError(f'Check target for next-hop "{next_hop}" is mandatory!') + + check_type = next_hop_config['check']['type'] + if check_type == 'tcp' and 'port' not in next_hop_config['check']: + raise ConfigError(f'Check port for next-hop "{next_hop}" and type TCP is mandatory!') + + return None + +def generate(failover): + if not failover: + service_conf.unlink(missing_ok=True) + return None + + # Add own rt_proto 'failover' + # Helps to detect all own routes 'proto failover' + with open(rt_proto_failover, 'w') as f: + f.write('111 failover\n') + + # Write configuration file + conf_json = json.dumps(failover, indent=4) + service_conf.write_text(conf_json) + render(systemd_service, 'protocols/systemd_vyos_failover_service.j2', failover) + + return None + +def apply(failover): + if not failover: + call(f'systemctl stop {service_name}.service') + call('ip route flush protocol failover') + else: + call('systemctl daemon-reload') + call(f'systemctl restart {service_name}.service') + call(f'ip route flush protocol failover') + + return None + +if __name__ == '__main__': + try: + c = get_config() + verify(c) + generate(c) + apply(c) + except ConfigError as e: + print(e) + exit(1) diff --git a/src/helpers/vyos-failover.py b/src/helpers/vyos-failover.py new file mode 100755 index 000000000..1ac193423 --- /dev/null +++ b/src/helpers/vyos-failover.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import argparse +import json +import subprocess +import socket +import time + +from vyos.util import rc_cmd +from pathlib import Path +from systemd import journal + + +my_name = Path(__file__).stem + + +def get_best_route_options(route, debug=False): + """ + Return current best route ('gateway, interface, metric) + + % get_best_route_options('203.0.113.1') + ('192.168.0.1', 'eth1', 1) + + % get_best_route_options('203.0.113.254') + (None, None, None) + """ + rc, data = rc_cmd(f'ip --detail --json route show protocol failover {route}') + if rc == 0: + data = json.loads(data) + if len(data) == 0: + print(f'\nRoute {route} for protocol failover was not found') + return None, None, None + # Fake metric 999 by default + # Search route with the lowest metric + best_metric = 999 + for entry in data: + if debug: print('\n', entry) + metric = entry.get('metric') + gateway = entry.get('gateway') + iface = entry.get('dev') + if metric < best_metric: + best_metric = metric + best_gateway = gateway + best_interface = iface + if debug: + print(f'### Best_route exists: {route}, best_gateway: {best_gateway}, ' + f'best_metric: {best_metric}, best_iface: {best_interface}') + return best_gateway, best_interface, best_metric + +def is_port_open(ip, port): + """ + Check connection to remote host and port + Return True if host alive + + % is_port_open('example.com', 8080) + True + """ + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) + s.settimeout(2) + try: + s.connect((ip, int(port))) + s.shutdown(socket.SHUT_RDWR) + return True + except: + return False + finally: + s.close() + +def is_target_alive(target=None, iface='', proto='icmp', port=None, debug=False): + """ + Host availability check by ICMP, ARP, TCP + Return True if target checks is successful + + % is_target_alive('192.0.2.1', 'eth1', proto='arp') + True + """ + if iface != '': + iface = f'-I {iface}' + if proto == 'icmp': + command = f'/usr/bin/ping -q {target} {iface} -n -c 2 -W 1' + rc, response = rc_cmd(command) + if debug: print(f' [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]') + if rc == 0: + return True + elif proto == 'arp': + command = f'/usr/bin/arping -b -c 2 -f -w 1 -i 1 {iface} {target}' + rc, response = rc_cmd(command) + if debug: print(f' [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]') + if rc == 0: + return True + elif proto == 'tcp' and port is not None: + return True if is_port_open(target, port) else False + else: + return False + + +if __name__ == '__main__': + # Parse command arguments and get config + parser = argparse.ArgumentParser() + parser.add_argument('-c', + '--config', + action='store', + help='Path to protocols failover configuration', + required=True, + type=Path) + + args = parser.parse_args() + try: + config_path = Path(args.config) + config = json.loads(config_path.read_text()) + except Exception as err: + print( + f'Configuration file "{config_path}" does not exist or malformed: {err}' + ) + exit(1) + + # Useful debug info to console, use debug = True + # sudo systemctl stop vyos-failover.service + # sudo /usr/libexec/vyos/vyos-failover.py --config /run/vyos-failover.conf + debug = False + + while(True): + + for route, route_config in config.get('route').items(): + + exists_route = exists_gateway, exists_iface, exists_metric = get_best_route_options(route, debug=debug) + + for next_hop, nexthop_config in route_config.get('next_hop').items(): + conf_iface = nexthop_config.get('interface') + conf_metric = int(nexthop_config.get('metric')) + port = nexthop_config.get('check').get('port') + port_opt = f'port {port}' if port else '' + proto = nexthop_config.get('check').get('type') + target = nexthop_config.get('check').get('target') + timeout = nexthop_config.get('check').get('timeout') + + # Best route not fonund in the current routing table + if exists_route == (None, None, None): + if debug: print(f" [NEW_ROUTE_DETECTED] route: [{route}]") + # Add route if check-target alive + if is_target_alive(target, conf_iface, proto, port, debug=debug): + if debug: print(f' [ ADD ] -- ip route add {route} via {next_hop} dev {conf_iface} ' + f'metric {conf_metric} proto failover\n###') + rc, command = rc_cmd(f'ip route add {route} via {next_hop} dev {conf_iface} ' + f'metric {conf_metric} proto failover') + # If something is wrong and gateway not added + # Example: Error: Next-hop has invalid gateway. + if rc !=0: + if debug: print(f'{command} -- return-code [RC: {rc}] {next_hop} dev {conf_iface}') + else: + journal.send(f'ip route add {route} via {next_hop} dev {conf_iface} ' + f'metric {conf_metric} proto failover', SYSLOG_IDENTIFIER=my_name) + else: + if debug: print(f' [ TARGET_FAIL ] target checks fails for [{target}], do nothing') + journal.send(f'Check fail for route {route} target {target} proto {proto} ' + f'{port_opt}', SYSLOG_IDENTIFIER=my_name) + + # Route was added, check if the target is alive + # We should delete route if check fails only if route exists in the routing table + if not is_target_alive(target, conf_iface, proto, port, debug=debug) and \ + exists_route != (None, None, None): + if debug: + print(f'Nexh_hop {next_hop} fail, target not response') + print(f' [ DEL ] -- ip route del {route} via {next_hop} dev {conf_iface} ' + f'metric {conf_metric} proto failover [DELETE]') + rc_cmd(f'ip route del {route} via {next_hop} dev {conf_iface} metric {conf_metric} proto failover') + journal.send(f'ip route del {route} via {next_hop} dev {conf_iface} ' + f'metric {conf_metric} proto failover', SYSLOG_IDENTIFIER=my_name) + + time.sleep(int(timeout)) -- cgit v1.2.3 From 837357071135b87532796dfab88d53ae7cba9e45 Mon Sep 17 00:00:00 2001 From: zsdc Date: Thu, 15 Dec 2022 11:13:02 +0200 Subject: bonding: T4878: Fixed unnecessary bonding flapping during commit There was a mistake in a config level that caused triggering the `shutdown_required` flag, even if there were no new interfaces added to a bonding. This commit sets the proper config level to avoid the problem. --- src/conf_mode/interfaces-bonding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/conf_mode/interfaces-bonding.py b/src/conf_mode/interfaces-bonding.py index 21cf204fc..b883ebef2 100755 --- a/src/conf_mode/interfaces-bonding.py +++ b/src/conf_mode/interfaces-bonding.py @@ -116,7 +116,7 @@ def get_config(config=None): if dict_search('member.interface', bond): for interface, interface_config in bond['member']['interface'].items(): # Check if member interface is a new member - if not conf.exists_effective(['member', 'interface', interface]): + if not conf.exists_effective(base + [ifname, 'member', 'interface', interface]): bond['shutdown_required'] = {} # Check if member interface is disabled -- cgit v1.2.3 From e78235213c7409ae0ddb50edc1ba83095d1c9080 Mon Sep 17 00:00:00 2001 From: aapostoliuk <108394744+aapostoliuk@users.noreply.github.com> Date: Sat, 17 Dec 2022 09:20:56 +0200 Subject: webproxy: T3810: multiple squidGuard fixes 1. Added in script update webproxy blacklists generation of all DBs 2. Fixed: if the blacklist category does not have generated db, the template generates an empty dest category in squidGuard.conf and a Warning message. 3. Added template generation for local's categories in the rule section. 4. Changed syntax in the generation dest section for blacklist's categories 4. Fixed generation dest local sections in squidGuard.conf 5. Fixed bug in syntax. The word 'allow' changed to the word 'any' in acl squidGuard.conf --- data/templates/squid/sg_acl.conf.j2 | 1 - data/templates/squid/squidGuard.conf.j2 | 122 ++++++++++++++++++++++++++----- src/conf_mode/service_webproxy.py | 100 +++++++++++++++++++------ src/op_mode/webproxy_update_blacklist.sh | 27 +++++++ 4 files changed, 205 insertions(+), 45 deletions(-) (limited to 'src') diff --git a/data/templates/squid/sg_acl.conf.j2 b/data/templates/squid/sg_acl.conf.j2 index ce72b173a..78297a2b8 100644 --- a/data/templates/squid/sg_acl.conf.j2 +++ b/data/templates/squid/sg_acl.conf.j2 @@ -1,6 +1,5 @@ ### generated by service_webproxy.py ### dbhome {{ squidguard_db_dir }} - dest {{ category }}-{{ rule }} { {% if list_type == 'domains' %} domainlist {{ category }}/domains diff --git a/data/templates/squid/squidGuard.conf.j2 b/data/templates/squid/squidGuard.conf.j2 index 1bc4c984f..a93f878df 100644 --- a/data/templates/squid/squidGuard.conf.j2 +++ b/data/templates/squid/squidGuard.conf.j2 @@ -1,10 +1,16 @@ ### generated by service_webproxy.py ### -{% macro sg_rule(category, log, db_dir) %} +{% macro sg_rule(category, rule, log, db_dir) %} +{% set domains = db_dir + '/' + category + '/domains' %} +{% set urls = db_dir + '/' + category + '/urls' %} {% set expressions = db_dir + '/' + category + '/expressions' %} -dest {{ category }}-default { +dest {{ category }}-{{ rule }}{ +{% if domains | is_file %} domainlist {{ category }}/domains +{% endif %} +{% if urls | is_file %} urllist {{ category }}/urls +{% endif %} {% if expressions | is_file %} expressionlist {{ category }}/expressions {% endif %} @@ -17,8 +23,9 @@ dest {{ category }}-default { {% if url_filtering is vyos_defined and url_filtering.disable is not vyos_defined %} {% if url_filtering.squidguard is vyos_defined %} {% set sg_config = url_filtering.squidguard %} -{% set acl = namespace(value='local-ok-default') %} +{% set acl = namespace(value='') %} {% set acl.value = acl.value + ' !in-addr' if sg_config.allow_ipaddr_url is not defined else acl.value %} +{% set ruleacls = {} %} dbhome {{ squidguard_db_dir }} logdir /var/log/squid @@ -38,24 +45,28 @@ dest local-ok-default { domainlist local-ok-default/domains } {% endif %} + {% if sg_config.local_ok_url is vyos_defined %} {% set acl.value = acl.value + ' local-ok-url-default' %} dest local-ok-url-default { urllist local-ok-url-default/urls } {% endif %} + {% if sg_config.local_block is vyos_defined %} {% set acl.value = acl.value + ' !local-block-default' %} dest local-block-default { domainlist local-block-default/domains } {% endif %} + {% if sg_config.local_block_url is vyos_defined %} {% set acl.value = acl.value + ' !local-block-url-default' %} dest local-block-url-default { urllist local-block-url-default/urls } {% endif %} + {% if sg_config.local_block_keyword is vyos_defined %} {% set acl.value = acl.value + ' !local-block-keyword-default' %} dest local-block-keyword-default { @@ -65,16 +76,100 @@ dest local-block-keyword-default { {% if sg_config.block_category is vyos_defined %} {% for category in sg_config.block_category %} -{{ sg_rule(category, sg_config.log, squidguard_db_dir) }} +{{ sg_rule(category, 'default', sg_config.log, squidguard_db_dir) }} {% set acl.value = acl.value + ' !' + category + '-default' %} {% endfor %} {% endif %} {% if sg_config.allow_category is vyos_defined %} {% for category in sg_config.allow_category %} -{{ sg_rule(category, False, squidguard_db_dir) }} +{{ sg_rule(category, 'default', False, squidguard_db_dir) }} {% set acl.value = acl.value + ' ' + category + '-default' %} {% endfor %} {% endif %} + + +{% if sg_config.rule is vyos_defined %} +{% for rule, rule_config in sg_config.rule.items() %} +{% if rule_config.local_ok is vyos_defined %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' local-ok-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'local-ok-' + rule}) %} +{% endif %} +dest local-ok-{{ rule }} { + domainlist local-ok-{{ rule }}/domains +} +{% endif %} + +{% if rule_config.local_ok_url is vyos_defined %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' local-ok-url-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'local-ok-url-' + rule}) %} +{% endif %} +dest local-ok-url-{{ rule }} { + urllist local-ok-url-{{ rule }}/urls +} +{% endif %} + +{% if rule_config.local_block is vyos_defined %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'!local-block-' + rule}) %} +{% endif %} +dest local-block-{{ rule }} { + domainlist local-block-{{ rule }}/domains +} +{% endif %} + +{% if rule_config.local_block_url is vyos_defined %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-url-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'!ocal-block-url-' + rule}) %} +{% endif %} +dest local-block-url-{{ rule }} { + urllist local-block-url-{{ rule }}/urls +} +{% endif %} + +{% if rule_config.local_block_keyword is vyos_defined %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-keyword-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'!local-block-keyword-' + rule}) %} +{% endif %} +dest local-block-keyword-{{ rule }} { + expressionlist local-block-keyword-{{ rule }}/expressions +} +{% endif %} + +{% if rule_config.block_category is vyos_defined %} +{% for b_category in rule_config.block_category %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !' + b_category + '-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'!' + b_category + '-' + rule}) %} +{% endif %} +{{ sg_rule(b_category, rule, sg_config.log, squidguard_db_dir) }} +{% endfor %} +{% endif %} + +{% if rule_config.allow_category is vyos_defined %} +{% for a_category in rule_config.allow_category %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' ' + a_category + '-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:a_category + '-' + rule}) %} +{% endif %} +{{ sg_rule(a_category, rule, sg_config.log, squidguard_db_dir) }} +{% endfor %} +{% endif %} +{% endfor %} +{% endif %} + + {% if sg_config.source_group is vyos_defined %} {% for sgroup, sg_config in sg_config.source_group.items() %} {% if sg_config.address is vyos_defined %} @@ -83,28 +178,15 @@ src {{ sgroup }} { ip {{ address }} {% endfor %} } - {% endif %} {% endfor %} {% endif %} -{% if sg_config.rule is vyos_defined %} -{% for rule, rule_config in sg_config.rule.items() %} -{% for b_category in rule_config.block_category %} -dest {{ b_category }} { - domainlist {{ b_category }}/domains - urllist {{ b_category }}/urls -} -{% endfor %} -{% endfor %} -{% endif %} acl { {% if sg_config.rule is vyos_defined %} {% for rule, rule_config in sg_config.rule.items() %} {{ rule_config.source_group }} { -{% for b_category in rule_config.block_category %} - pass local-ok-1 !in-addr !{{ b_category }} all -{% endfor %} + pass {{ ruleacls[rule] }} {{ 'none' if rule_config.default_action is vyos_defined('block') else 'any' }} } {% endfor %} {% endif %} @@ -113,7 +195,7 @@ acl { {% if sg_config.enable_safe_search is vyos_defined %} rewrite safesearch {% endif %} - pass {{ acl.value }} {{ 'none' if sg_config.default_action is vyos_defined('block') else 'allow' }} + pass {{ acl.value }} {{ 'none' if sg_config.default_action is vyos_defined('block') else 'any' }} redirect 302:http://{{ sg_config.redirect_url }} {% if sg_config.log is vyos_defined %} log blacklist.log diff --git a/src/conf_mode/service_webproxy.py b/src/conf_mode/service_webproxy.py index 32af31bde..41a1deaa3 100755 --- a/src/conf_mode/service_webproxy.py +++ b/src/conf_mode/service_webproxy.py @@ -28,8 +28,10 @@ from vyos.util import dict_search from vyos.util import write_file from vyos.validate import is_addr_assigned from vyos.xml import defaults +from vyos.base import Warning from vyos import ConfigError from vyos import airbag + airbag.enable() squid_config_file = '/etc/squid/squid.conf' @@ -37,24 +39,57 @@ squidguard_config_file = '/etc/squidguard/squidGuard.conf' squidguard_db_dir = '/opt/vyatta/etc/config/url-filtering/squidguard/db' user_group = 'proxy' -def generate_sg_localdb(category, list_type, role, proxy): + +def check_blacklist_categorydb(config_section): + if 'block_category' in config_section: + for category in config_section['block_category']: + check_categorydb(category) + if 'allow_category' in config_section: + for category in config_section['allow_category']: + check_categorydb(category) + + +def check_categorydb(category: str): + """ + Check if category's db exist + :param category: + :type str: + """ + path_to_cat: str = f'{squidguard_db_dir}/{category}' + if not os.path.exists(f'{path_to_cat}/domains.db') \ + and not os.path.exists(f'{path_to_cat}/urls.db') \ + and not os.path.exists(f'{path_to_cat}/expressions.db'): + Warning(f'DB of category {category} does not exist.\n ' + f'Use [update webproxy blacklists] ' + f'or delete undefined category!') + + +def generate_sg_rule_localdb(category, list_type, role, proxy): + if not category or not list_type or not role: + return None + cat_ = category.replace('-', '_') - if isinstance(dict_search(f'url_filtering.squidguard.{cat_}', proxy), - list): + if role == 'default': + path_to_cat = f'{cat_}' + else: + path_to_cat = f'rule.{role}.{cat_}' + if isinstance( + dict_search(f'url_filtering.squidguard.{path_to_cat}', proxy), + list): # local block databases must be generated "on-the-fly" tmp = { - 'squidguard_db_dir' : squidguard_db_dir, - 'category' : f'{category}-default', - 'list_type' : list_type, - 'rule' : role + 'squidguard_db_dir': squidguard_db_dir, + 'category': f'{category}-{role}', + 'list_type': list_type, + 'rule': role } sg_tmp_file = '/tmp/sg.conf' - db_file = f'{category}-default/{list_type}' - domains = '\n'.join(dict_search(f'url_filtering.squidguard.{cat_}', proxy)) - + db_file = f'{category}-{role}/{list_type}' + domains = '\n'.join( + dict_search(f'url_filtering.squidguard.{path_to_cat}', proxy)) # local file - write_file(f'{squidguard_db_dir}/{category}-default/local', '', + write_file(f'{squidguard_db_dir}/{category}-{role}/local', '', user=user_group, group=user_group) # database input file write_file(f'{squidguard_db_dir}/{db_file}', domains, @@ -64,17 +99,18 @@ def generate_sg_localdb(category, list_type, role, proxy): render(sg_tmp_file, 'squid/sg_acl.conf.j2', tmp, user=user_group, group=user_group) - call(f'su - {user_group} -c "squidGuard -d -c {sg_tmp_file} -C {db_file}"') + call( + f'su - {user_group} -c "squidGuard -d -c {sg_tmp_file} -C {db_file}"') if os.path.exists(sg_tmp_file): os.unlink(sg_tmp_file) - else: # if category is not part of our configuration, clean out the # squidguard lists - tmp = f'{squidguard_db_dir}/{category}-default' + tmp = f'{squidguard_db_dir}/{category}-{role}' if os.path.exists(tmp): - rmtree(f'{squidguard_db_dir}/{category}-default') + rmtree(f'{squidguard_db_dir}/{category}-{role}') + def get_config(config=None): if config: @@ -85,7 +121,8 @@ def get_config(config=None): if not conf.exists(base): return None - proxy = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) + proxy = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True) # We have gathered the dict representation of the CLI, but there are default # options which we need to update into the dictionary retrived. default_values = defaults(base) @@ -110,10 +147,11 @@ def get_config(config=None): default_values = defaults(base + ['cache-peer']) for peer in proxy['cache_peer']: proxy['cache_peer'][peer] = dict_merge(default_values, - proxy['cache_peer'][peer]) + proxy['cache_peer'][peer]) return proxy + def verify(proxy): if not proxy: return None @@ -170,17 +208,30 @@ def generate(proxy): render(squidguard_config_file, 'squid/squidGuard.conf.j2', proxy) cat_dict = { - 'local-block' : 'domains', - 'local-block-keyword' : 'expressions', - 'local-block-url' : 'urls', - 'local-ok' : 'domains', - 'local-ok-url' : 'urls' + 'local-block': 'domains', + 'local-block-keyword': 'expressions', + 'local-block-url': 'urls', + 'local-ok': 'domains', + 'local-ok-url': 'urls' } - for category, list_type in cat_dict.items(): - generate_sg_localdb(category, list_type, 'default', proxy) + if dict_search(f'url_filtering.squidguard', proxy) is not None: + squidgard_config_section = proxy['url_filtering']['squidguard'] + + for category, list_type in cat_dict.items(): + generate_sg_rule_localdb(category, list_type, 'default', proxy) + check_blacklist_categorydb(squidgard_config_section) + + if 'rule' in squidgard_config_section: + for rule in squidgard_config_section['rule']: + rule_config_section = squidgard_config_section['rule'][ + rule] + for category, list_type in cat_dict.items(): + generate_sg_rule_localdb(category, list_type, rule, proxy) + check_blacklist_categorydb(rule_config_section) return None + def apply(proxy): if not proxy: # proxy is removed in the commit @@ -198,6 +249,7 @@ def apply(proxy): call('systemctl restart squid.service') return None + if __name__ == '__main__': try: c = get_config() diff --git a/src/op_mode/webproxy_update_blacklist.sh b/src/op_mode/webproxy_update_blacklist.sh index d5f301b75..4fb9a54c6 100755 --- a/src/op_mode/webproxy_update_blacklist.sh +++ b/src/op_mode/webproxy_update_blacklist.sh @@ -18,6 +18,23 @@ blacklist_url='ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/black data_dir="/opt/vyatta/etc/config/url-filtering" archive="${data_dir}/squidguard/archive" db_dir="${data_dir}/squidguard/db" +conf_file="/etc/squidguard/squidGuard.conf" +tmp_conf_file="/tmp/sg_update_db.conf" + +#$1-category +#$2-type +#$3-list +create_sg_db () +{ + FILE=$db_dir/$1/$2 + if test -f "$FILE"; then + rm -f ${tmp_conf_file} + printf "dbhome $db_dir\ndest $1 {\n $3 $1/$2\n}\nacl {\n default {\n pass any\n }\n}" >> ${tmp_conf_file} + /usr/bin/squidGuard -b -c ${tmp_conf_file} -C $FILE + rm -f ${tmp_conf_file} + fi + +} while [ $# -gt 0 ] do @@ -88,6 +105,16 @@ if [[ -n $update ]] && [[ $update -eq "yes" ]]; then # fix permissions chown -R proxy:proxy ${db_dir} + + #create db + category_list=(`find $db_dir -type d -exec basename {} \; `) + for category in ${category_list[@]} + do + create_sg_db $category "domains" "domainlist" + create_sg_db $category "urls" "urllist" + create_sg_db $category "expressions" "expressionlist" + done + chown -R proxy:proxy ${db_dir} chmod 755 ${db_dir} logger --priority WARNING "webproxy blacklist entries updated (${count_before}/${count_after})" -- cgit v1.2.3