From 0e63712195465c9bf0bf55c369b86961d54dfaac Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 27 Oct 2022 12:24:44 -0500 Subject: T4291: consolidate component version string read/write functions --- src/helpers/system-versions-foot.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) (limited to 'src/helpers') diff --git a/src/helpers/system-versions-foot.py b/src/helpers/system-versions-foot.py index 2aa687221..9614f0d28 100755 --- a/src/helpers/system-versions-foot.py +++ b/src/helpers/system-versions-foot.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -# Copyright 2019 VyOS maintainers and contributors +# Copyright 2019, 2022 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,24 +16,13 @@ # along with this library. If not, see . import sys -import vyos.formatversions as formatversions -import vyos.systemversions as systemversions import vyos.defaults -import vyos.version - -sys_versions = systemversions.get_system_component_version() - -component_string = formatversions.format_versions_string(sys_versions) - -os_version_string = vyos.version.get_version() +from vyos.component_version import write_system_footer sys.stdout.write("\n\n") if vyos.defaults.cfg_vintage == 'vyos': - formatversions.write_vyos_versions_foot(None, component_string, - os_version_string) + write_system_footer(None, vintage='vyos') elif vyos.defaults.cfg_vintage == 'vyatta': - formatversions.write_vyatta_versions_foot(None, component_string, - os_version_string) + write_system_footer(None, vintage='vyatta') else: - formatversions.write_vyatta_versions_foot(None, component_string, - os_version_string) + write_system_footer(None, vintage='vyos') -- cgit v1.2.3 From 051e063fdf2e459a0716a35778b33ea6bb2fdcb6 Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Mon, 31 Oct 2022 14:26:51 +0100 Subject: firewall: T970: Refactor domain resolver, add firewall source/destination `fqdn` node --- data/templates/firewall/nftables-defines.j2 | 8 + data/templates/firewall/nftables.j2 | 14 +- interface-definitions/firewall.xml.in | 25 ++- interface-definitions/include/firewall/fqdn.xml.i | 14 ++ .../firewall/source-destination-group-ipv6.xml.i | 8 + python/vyos/firewall.py | 90 ++++------ smoketest/scripts/cli/test_firewall.py | 16 ++ src/conf_mode/firewall.py | 60 +++---- src/helpers/vyos-domain-group-resolve.py | 60 ------- src/helpers/vyos-domain-resolver.py | 182 +++++++++++++++++++++ src/systemd/vyos-domain-group-resolve.service | 11 -- src/systemd/vyos-domain-resolver.service | 13 ++ 12 files changed, 328 insertions(+), 173 deletions(-) create mode 100644 interface-definitions/include/firewall/fqdn.xml.i delete mode 100755 src/helpers/vyos-domain-group-resolve.py create mode 100755 src/helpers/vyos-domain-resolver.py delete mode 100644 src/systemd/vyos-domain-group-resolve.service create mode 100644 src/systemd/vyos-domain-resolver.service (limited to 'src/helpers') diff --git a/data/templates/firewall/nftables-defines.j2 b/data/templates/firewall/nftables-defines.j2 index 5336f7ee6..dd06dee28 100644 --- a/data/templates/firewall/nftables-defines.j2 +++ b/data/templates/firewall/nftables-defines.j2 @@ -27,6 +27,14 @@ } {% endfor %} {% endif %} +{% if group.domain_group is vyos_defined %} +{% for name, name_config in group.domain_group.items() %} + set D_{{ name }} { + type {{ ip_type }} + flags interval + } +{% endfor %} +{% endif %} {% if group.mac_group is vyos_defined %} {% for group_name, group_conf in group.mac_group.items() %} {% set includes = group_conf.include if group_conf.include is vyos_defined else [] %} diff --git a/data/templates/firewall/nftables.j2 b/data/templates/firewall/nftables.j2 index a0f0b8c11..2c7115134 100644 --- a/data/templates/firewall/nftables.j2 +++ b/data/templates/firewall/nftables.j2 @@ -67,14 +67,12 @@ table ip vyos_filter { {{ conf | nft_default_rule(name_text) }} } {% endfor %} -{% if group is vyos_defined and group.domain_group is vyos_defined %} -{% for name, name_config in group.domain_group.items() %} - set D_{{ name }} { +{% for set_name in ip_fqdn %} + set FQDN_{{ set_name }} { type ipv4_addr flags interval } -{% endfor %} -{% endif %} +{% endfor %} {% for set_name in ns.sets %} set RECENT_{{ set_name }} { type ipv4_addr @@ -178,6 +176,12 @@ table ip6 vyos_filter { {{ conf | nft_default_rule(name_text, ipv6=True) }} } {% endfor %} +{% for set_name in ip6_fqdn %} + set FQDN_{{ set_name }} { + type ipv6_addr + flags interval + } +{% endfor %} {% for set_name in ns.sets %} set RECENT6_{{ set_name }} { type ipv6_addr diff --git a/interface-definitions/firewall.xml.in b/interface-definitions/firewall.xml.in index 673461036..2d8f17351 100644 --- a/interface-definitions/firewall.xml.in +++ b/interface-definitions/firewall.xml.in @@ -126,7 +126,7 @@ Domain address to match - [a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,99}?(\/.*)? + @@ -408,6 +408,7 @@ #include + #include #include #include #include @@ -419,6 +420,7 @@ #include + #include #include #include #include @@ -572,6 +574,7 @@ #include + #include #include #include #include @@ -583,6 +586,7 @@ #include + #include #include #include #include @@ -656,6 +660,25 @@ disable + + + Retains last successful value if domain resolution fails + + + + + + Domain resolver update interval + + u32:10-3600 + Interval (seconds) + + + + + + 300 + Policy for sending IPv4 ICMP redirect messages diff --git a/interface-definitions/include/firewall/fqdn.xml.i b/interface-definitions/include/firewall/fqdn.xml.i new file mode 100644 index 000000000..9eb3925b5 --- /dev/null +++ b/interface-definitions/include/firewall/fqdn.xml.i @@ -0,0 +1,14 @@ + + + + Fully qualified domain name + + <fqdn> + Fully qualified domain name + + + + + + + diff --git a/interface-definitions/include/firewall/source-destination-group-ipv6.xml.i b/interface-definitions/include/firewall/source-destination-group-ipv6.xml.i index c2cc7edb3..2a42d236c 100644 --- a/interface-definitions/include/firewall/source-destination-group-ipv6.xml.i +++ b/interface-definitions/include/firewall/source-destination-group-ipv6.xml.i @@ -12,6 +12,14 @@ + + + Group of domains + + firewall group domain-group + + + #include diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index 4075e55b0..db4878c9d 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -20,6 +20,9 @@ import os import re from pathlib import Path +from socket import AF_INET +from socket import AF_INET6 +from socket import getaddrinfo from time import strftime from vyos.remote import download @@ -31,65 +34,29 @@ from vyos.util import dict_search_args from vyos.util import dict_search_recursive from vyos.util import run +def fqdn_config_parse(firewall): + firewall['ip_fqdn'] = {} + firewall['ip6_fqdn'] = {} + + for domain, path in dict_search_recursive(firewall, 'fqdn'): + fw_name = path[1] # name/ipv6-name + rule = path[3] # rule id + suffix = path[4][0] # source/destination (1 char) + set_name = f'{fw_name}_{rule}_{suffix}' + + if path[0] == 'name': + firewall['ip_fqdn'][set_name] = domain + elif path[0] == 'ipv6_name': + firewall['ip6_fqdn'][set_name] = domain + +def fqdn_resolve(fqdn, ipv6=False): + try: + res = getaddrinfo(fqdn, None, AF_INET6 if ipv6 else AF_INET) + return set(item[4][0] for item in res) + except: + return None -# Functions for firewall group domain-groups -def get_ips_domains_dict(list_domains): - """ - Get list of IPv4 addresses by list of domains - Ex: get_ips_domains_dict(['ex1.com', 'ex2.com']) - {'ex1.com': ['192.0.2.1'], 'ex2.com': ['192.0.2.2', '192.0.2.3']} - """ - from socket import gethostbyname_ex - from socket import gaierror - - ip_dict = {} - for domain in list_domains: - try: - _, _, ips = gethostbyname_ex(domain) - ip_dict[domain] = ips - except gaierror: - pass - - return ip_dict - -def nft_init_set(group_name, table="vyos_filter", family="ip"): - """ - table ip vyos_filter { - set GROUP_NAME - type ipv4_addr - flags interval - } - """ - return call(f'nft add set ip {table} {group_name} {{ type ipv4_addr\\; flags interval\\; }}') - - -def nft_add_set_elements(group_name, elements, table="vyos_filter", family="ip"): - """ - table ip vyos_filter { - set GROUP_NAME { - type ipv4_addr - flags interval - elements = { 192.0.2.1, 192.0.2.2 } - } - """ - elements = ", ".join(elements) - return call(f'nft add element {family} {table} {group_name} {{ {elements} }} ') - -def nft_flush_set(group_name, table="vyos_filter", family="ip"): - """ - Flush elements of nft set - """ - return call(f'nft flush set {family} {table} {group_name}') - -def nft_update_set_elements(group_name, elements, table="vyos_filter", family="ip"): - """ - Update elements of nft set - """ - flush_set = nft_flush_set(group_name, table="vyos_filter", family="ip") - nft_add_set = nft_add_set_elements(group_name, elements, table="vyos_filter", family="ip") - return flush_set, nft_add_set - -# END firewall group domain-group (sets) +# End Domain Resolver def find_nftables_rule(table, chain, rule_matches=[]): # Find rule in table/chain that matches all criteria and return the handle @@ -151,6 +118,13 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name): suffix = f'!= {suffix[1:]}' output.append(f'{ip_name} {prefix}addr {suffix}') + if 'fqdn' in side_conf: + fqdn = side_conf['fqdn'] + operator = '' + if fqdn[0] == '!': + operator = '!=' + output.append(f'{ip_name} {prefix}addr {operator} @FQDN_{fw_name}_{rule_id}_{prefix}') + if dict_search_args(side_conf, 'geoip', 'country_code'): operator = '' if dict_search_args(side_conf, 'geoip', 'inverse_match') != None: diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py index 821925bcd..e172e086d 100755 --- a/smoketest/scripts/cli/test_firewall.py +++ b/smoketest/scripts/cli/test_firewall.py @@ -17,11 +17,13 @@ import unittest from glob import glob +from time import sleep from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.util import cmd +from vyos.util import run sysfs_config = { 'all_ping': {'sysfs': '/proc/sys/net/ipv4/icmp_echo_ignore_all', 'default': '0', 'test_value': 'disable'}, @@ -76,6 +78,17 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase): break self.assertTrue(not matched if inverse else matched, msg=search) + def wait_for_domain_resolver(self, table, set_name, element, max_wait=10): + # Resolver no longer blocks commit, need to wait for daemon to populate set + count = 0 + while count < max_wait: + code = run(f'sudo nft get element {table} {set_name} {{ {element} }}') + if code == 0: + return True + count += 1 + sleep(1) + return False + def test_geoip(self): self.cli_set(['firewall', 'name', 'smoketest', 'rule', '1', 'action', 'drop']) self.cli_set(['firewall', 'name', 'smoketest', 'rule', '1', 'source', 'geoip', 'country-code', 'se']) @@ -125,6 +138,9 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase): self.cli_set(['firewall', 'interface', 'eth0', 'in', 'name', 'smoketest']) self.cli_commit() + + self.wait_for_domain_resolver('ip vyos_filter', 'D_smoketest_domain', '192.0.2.5') + nftables_search = [ ['iifname "eth0"', 'jump NAME_smoketest'], ['ip saddr @N_smoketest_network', 'ip daddr 172.16.10.10', 'th dport @P_smoketest_port', 'return'], diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index cbd9cbe90..2bb765e65 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -27,12 +27,8 @@ from vyos.configdict import dict_merge from vyos.configdict import node_changed from vyos.configdiff import get_config_diff, Diff # from vyos.configverify import verify_interface_exists +from vyos.firewall import fqdn_config_parse from vyos.firewall import geoip_update -from vyos.firewall import get_ips_domains_dict -from vyos.firewall import nft_add_set_elements -from vyos.firewall import nft_flush_set -from vyos.firewall import nft_init_set -from vyos.firewall import nft_update_set_elements from vyos.template import render from vyos.util import call from vyos.util import cmd @@ -173,6 +169,8 @@ def get_config(config=None): firewall['geoip_updated'] = geoip_updated(conf, firewall) + fqdn_config_parse(firewall) + return firewall def verify_rule(firewall, rule_conf, ipv6): @@ -232,29 +230,28 @@ def verify_rule(firewall, rule_conf, ipv6): if side in rule_conf: side_conf = rule_conf[side] - if dict_search_args(side_conf, 'geoip', 'country_code'): - if 'address' in side_conf: - raise ConfigError('Address and GeoIP cannot both be defined') - - if dict_search_args(side_conf, 'group', 'address_group'): - raise ConfigError('Address-group and GeoIP cannot both be defined') - - if dict_search_args(side_conf, 'group', 'network_group'): - raise ConfigError('Network-group and GeoIP cannot both be defined') + if len({'address', 'fqdn', 'geoip'} & set(side_conf)) > 1: + raise ConfigError('Only one of address, fqdn or geoip can be specified') if 'group' in side_conf: - if {'address_group', 'network_group'} <= set(side_conf['group']): - raise ConfigError('Only one address-group or network-group can be specified') + if len({'address_group', 'network_group', 'domain_group'} & set(side_conf['group'])) > 1: + raise ConfigError('Only one address-group, network-group or domain-group can be specified') for group in valid_groups: if group in side_conf['group']: group_name = side_conf['group'][group] + fw_group = f'ipv6_{group}' if ipv6 and group in ['address_group', 'network_group'] else group + error_group = fw_group.replace("_", "-") + + if group in ['address_group', 'network_group', 'domain_group']: + types = [t for t in ['address', 'fqdn', 'geoip'] if t in side_conf] + if types: + raise ConfigError(f'{error_group} and {types[0]} cannot both be defined') + if group_name and group_name[0] == '!': group_name = group_name[1:] - fw_group = f'ipv6_{group}' if ipv6 and group in ['address_group', 'network_group'] else group - error_group = fw_group.replace("_", "-") group_obj = dict_search_args(firewall, 'group', fw_group, group_name) if group_obj is None: @@ -477,26 +474,13 @@ def apply(firewall): if install_result == 1: raise ConfigError(f'Failed to apply firewall: {output}') - # set firewall group domain-group xxx - if 'group' in firewall: - if 'domain_group' in firewall['group']: - # T970 Enable a resolver (systemd daemon) that checks - # domain-group addresses and update entries for domains by timeout - # If router loaded without internet connection or for synchronization - call('systemctl restart vyos-domain-group-resolve.service') - for group, group_config in firewall['group']['domain_group'].items(): - domains = [] - if group_config.get('address') is not None: - for address in group_config.get('address'): - domains.append(address) - # Add elements to domain-group, try to resolve domain => ip - # and add elements to nft set - ip_dict = get_ips_domains_dict(domains) - elements = sum(ip_dict.values(), []) - nft_init_set(f'D_{group}') - nft_add_set_elements(f'D_{group}', elements) - else: - call('systemctl stop vyos-domain-group-resolve.service') + # T970 Enable a resolver (systemd daemon) that checks + # domain-group addresses and update entries for domains by timeout + # If router loaded without internet connection or for synchronization + domain_action = 'stop' + if dict_search_args(firewall, 'group', 'domain_group') or firewall['ip_fqdn'] or firewall['ip6_fqdn']: + domain_action = 'restart' + call(f'systemctl {domain_action} vyos-domain-resolver.service') apply_sysfs(firewall) diff --git a/src/helpers/vyos-domain-group-resolve.py b/src/helpers/vyos-domain-group-resolve.py deleted file mode 100755 index 6b677670b..000000000 --- a/src/helpers/vyos-domain-group-resolve.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2022 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -import time - -from vyos.configquery import ConfigTreeQuery -from vyos.firewall import get_ips_domains_dict -from vyos.firewall import nft_add_set_elements -from vyos.firewall import nft_flush_set -from vyos.firewall import nft_init_set -from vyos.firewall import nft_update_set_elements -from vyos.util import call - - -base = ['firewall', 'group', 'domain-group'] -check_required = True -# count_failed = 0 -# Timeout in sec between checks -timeout = 300 - -domain_state = {} - -if __name__ == '__main__': - - while check_required: - config = ConfigTreeQuery() - if config.exists(base): - domain_groups = config.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - for set_name, domain_config in domain_groups.items(): - list_domains = domain_config['address'] - elements = [] - ip_dict = get_ips_domains_dict(list_domains) - - for domain in list_domains: - # Resolution succeeded, update domain state - if domain in ip_dict: - domain_state[domain] = ip_dict[domain] - elements += ip_dict[domain] - # Resolution failed, use previous domain state - elif domain in domain_state: - elements += domain_state[domain] - - # Resolve successful - if elements: - nft_update_set_elements(f'D_{set_name}', elements) - time.sleep(timeout) diff --git a/src/helpers/vyos-domain-resolver.py b/src/helpers/vyos-domain-resolver.py new file mode 100755 index 000000000..2f71f15db --- /dev/null +++ b/src/helpers/vyos-domain-resolver.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import json +import os +import time + +from vyos.configdict import dict_merge +from vyos.configquery import ConfigTreeQuery +from vyos.firewall import fqdn_config_parse +from vyos.firewall import fqdn_resolve +from vyos.util import cmd +from vyos.util import commit_in_progress +from vyos.util import dict_search_args +from vyos.util import run +from vyos.xml import defaults + +base = ['firewall'] +timeout = 300 +cache = False + +domain_state = {} + +ipv4_tables = { + 'ip mangle', + 'ip vyos_filter', +} + +ipv6_tables = { + 'ip6 mangle', + 'ip6 vyos_filter' +} + +def get_config(conf): + firewall = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True, + no_tag_node_value_mangle=True) + + default_values = defaults(base) + for tmp in ['name', 'ipv6_name']: + if tmp in default_values: + del default_values[tmp] + + if 'zone' in default_values: + del default_values['zone'] + + firewall = dict_merge(default_values, firewall) + + global timeout, cache + + if 'resolver_interval' in firewall: + timeout = int(firewall['resolver_interval']) + + if 'resolver_cache' in firewall: + cache = True + + fqdn_config_parse(firewall) + + return firewall + +def resolve(domains, ipv6=False): + global domain_state + + ip_list = set() + + for domain in domains: + resolved = fqdn_resolve(domain, ipv6=ipv6) + + if resolved and cache: + domain_state[domain] = resolved + elif not resolved: + if domain not in domain_state: + continue + resolved = domain_state[domain] + + ip_list = ip_list | resolved + return ip_list + +def nft_output(table, set_name, ip_list): + output = [f'flush set {table} {set_name}'] + if ip_list: + ip_str = ','.join(ip_list) + output.append(f'add element {table} {set_name} {{ {ip_str} }}') + return output + +def nft_valid_sets(): + try: + valid_sets = [] + sets_json = cmd('nft -j list sets') + sets_obj = json.loads(sets_json) + + for obj in sets_obj['nftables']: + if 'set' in obj: + family = obj['set']['family'] + table = obj['set']['table'] + name = obj['set']['name'] + valid_sets.append((f'{family} {table}', name)) + + return valid_sets + except: + return [] + +def update(firewall): + conf_lines = [] + count = 0 + + valid_sets = nft_valid_sets() + + domain_groups = dict_search_args(firewall, 'group', 'domain_group') + if domain_groups: + for set_name, domain_config in domain_groups.items(): + if 'address' not in domain_config: + continue + + nft_set_name = f'D_{set_name}' + domains = domain_config['address'] + + ip_list = resolve(domains, ipv6=False) + for table in ipv4_tables: + if (table, nft_set_name) in valid_sets: + conf_lines += nft_output(table, nft_set_name, ip_list) + + ip6_list = resolve(domains, ipv6=True) + for table in ipv6_tables: + if (table, nft_set_name) in valid_sets: + conf_lines += nft_output(table, nft_set_name, ip6_list) + count += 1 + + for set_name, domain in firewall['ip_fqdn'].items(): + table = 'ip vyos_filter' + nft_set_name = f'FQDN_{set_name}' + + ip_list = resolve([domain], ipv6=False) + + if (table, nft_set_name) in valid_sets: + conf_lines += nft_output(table, nft_set_name, ip_list) + count += 1 + + for set_name, domain in firewall['ip6_fqdn'].items(): + table = 'ip6 vyos_filter' + nft_set_name = f'FQDN_{set_name}' + + ip_list = resolve([domain], ipv6=True) + if (table, nft_set_name) in valid_sets: + conf_lines += nft_output(table, nft_set_name, ip_list) + count += 1 + + nft_conf_str = "\n".join(conf_lines) + "\n" + code = run(f'nft -f -', input=nft_conf_str) + + print(f'Updated {count} sets - result: {code}') + +if __name__ == '__main__': + print(f'VyOS domain resolver') + + count = 1 + while commit_in_progress(): + if ( count % 60 == 0 ): + print(f'Commit still in progress after {count}s - waiting') + count += 1 + time.sleep(1) + + conf = ConfigTreeQuery() + firewall = get_config(conf) + + print(f'interval: {timeout}s - cache: {cache}') + + while True: + update(firewall) + time.sleep(timeout) diff --git a/src/systemd/vyos-domain-group-resolve.service b/src/systemd/vyos-domain-group-resolve.service deleted file mode 100644 index 29628fddb..000000000 --- a/src/systemd/vyos-domain-group-resolve.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=VyOS firewall domain-group resolver -After=vyos-router.service - -[Service] -Type=simple -Restart=always -ExecStart=/usr/bin/python3 /usr/libexec/vyos/vyos-domain-group-resolve.py - -[Install] -WantedBy=multi-user.target diff --git a/src/systemd/vyos-domain-resolver.service b/src/systemd/vyos-domain-resolver.service new file mode 100644 index 000000000..c56b51f0c --- /dev/null +++ b/src/systemd/vyos-domain-resolver.service @@ -0,0 +1,13 @@ +[Unit] +Description=VyOS firewall domain resolver +After=vyos-router.service + +[Service] +Type=simple +Restart=always +ExecStart=/usr/bin/python3 -u /usr/libexec/vyos/vyos-domain-resolver.py +StandardError=journal +StandardOutput=journal + +[Install] +WantedBy=multi-user.target -- cgit v1.2.3 From b4b491d424fba6f3d417135adc1865e338a480a1 Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Mon, 31 Oct 2022 21:08:42 +0100 Subject: nat: T1877: T970: Add firewall groups to NAT --- data/templates/firewall/nftables-nat.j2 | 4 ++ interface-definitions/include/nat-rule.xml.i | 2 + python/vyos/firewall.py | 2 + python/vyos/nat.py | 56 +++++++++++++++++++-- smoketest/scripts/cli/test_nat.py | 35 +++++++++++++ src/conf_mode/firewall.py | 22 ++++++--- src/conf_mode/nat.py | 73 +++++++++++++++++++++++----- src/helpers/vyos-domain-resolver.py | 1 + 8 files changed, 174 insertions(+), 21 deletions(-) (limited to 'src/helpers') diff --git a/data/templates/firewall/nftables-nat.j2 b/data/templates/firewall/nftables-nat.j2 index c5c0a2c86..f0be3cf5d 100644 --- a/data/templates/firewall/nftables-nat.j2 +++ b/data/templates/firewall/nftables-nat.j2 @@ -1,5 +1,7 @@ #!/usr/sbin/nft -f +{% import 'firewall/nftables-defines.j2' as group_tmpl %} + {% if helper_functions is vyos_defined('remove') %} {# NAT if going to be disabled - remove rules and targets from nftables #} {% set base_command = 'delete rule ip raw' %} @@ -59,5 +61,7 @@ table ip vyos_nat { chain VYOS_PRE_SNAT_HOOK { return } + +{{ group_tmpl.groups(firewall_group, False) }} } {% endif %} diff --git a/interface-definitions/include/nat-rule.xml.i b/interface-definitions/include/nat-rule.xml.i index 84941aa6a..8f2029388 100644 --- a/interface-definitions/include/nat-rule.xml.i +++ b/interface-definitions/include/nat-rule.xml.i @@ -20,6 +20,7 @@ #include #include + #include #include @@ -285,6 +286,7 @@ #include #include + #include diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index db4878c9d..59ec4948f 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -34,6 +34,8 @@ from vyos.util import dict_search_args from vyos.util import dict_search_recursive from vyos.util import run +# Domain Resolver + def fqdn_config_parse(firewall): firewall['ip_fqdn'] = {} firewall['ip6_fqdn'] = {} diff --git a/python/vyos/nat.py b/python/vyos/nat.py index 31bbdc386..3d01829a7 100644 --- a/python/vyos/nat.py +++ b/python/vyos/nat.py @@ -85,8 +85,13 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): translation_str += f' {",".join(options)}' for target in ['source', 'destination']: + if target not in rule_conf: + continue + + side_conf = rule_conf[target] prefix = target[:1] - addr = dict_search_args(rule_conf, target, 'address') + + addr = dict_search_args(side_conf, 'address') if addr and not (ignore_type_addr and target == nat_type): operator = '' if addr[:1] == '!': @@ -94,7 +99,7 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): addr = addr[1:] output.append(f'{ip_prefix} {prefix}addr {operator} {addr}') - addr_prefix = dict_search_args(rule_conf, target, 'prefix') + addr_prefix = dict_search_args(side_conf, 'prefix') if addr_prefix and ipv6: operator = '' if addr_prefix[:1] == '!': @@ -102,7 +107,7 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): addr_prefix = addr[1:] output.append(f'ip6 {prefix}addr {operator} {addr_prefix}') - port = dict_search_args(rule_conf, target, 'port') + port = dict_search_args(side_conf, 'port') if port: protocol = rule_conf['protocol'] if protocol == 'tcp_udp': @@ -113,6 +118,51 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): port = port[1:] output.append(f'{protocol} {prefix}port {operator} {{ {port} }}') + if 'group' in side_conf: + group = side_conf['group'] + if 'address_group' in group and not (ignore_type_addr and target == nat_type): + group_name = group['address_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_prefix} {prefix}addr {operator} @A_{group_name}') + # Generate firewall group domain-group + elif 'domain_group' in group and not (ignore_type_addr and target == nat_type): + group_name = group['domain_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_prefix} {prefix}addr {operator} @D_{group_name}') + elif 'network_group' in group and not (ignore_type_addr and target == nat_type): + group_name = group['network_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_prefix} {prefix}addr {operator} @N_{group_name}') + if 'mac_group' in group: + group_name = group['mac_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'ether {prefix}addr {operator} @M_{group_name}') + if 'port_group' in group: + proto = rule_conf['protocol'] + group_name = group['port_group'] + + if proto == 'tcp_udp': + proto = 'th' + + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + + output.append(f'{proto} {prefix}port {operator} @P_{group_name}') + output.append('counter') if 'log' in rule_conf: diff --git a/smoketest/scripts/cli/test_nat.py b/smoketest/scripts/cli/test_nat.py index 2ae90fcaf..9f4e3b831 100755 --- a/smoketest/scripts/cli/test_nat.py +++ b/smoketest/scripts/cli/test_nat.py @@ -58,6 +58,17 @@ class TestNAT(VyOSUnitTestSHIM.TestCase): break self.assertTrue(not matched if inverse else matched, msg=search) + def wait_for_domain_resolver(self, table, set_name, element, max_wait=10): + # Resolver no longer blocks commit, need to wait for daemon to populate set + count = 0 + while count < max_wait: + code = run(f'sudo nft get element {table} {set_name} {{ {element} }}') + if code == 0: + return True + count += 1 + sleep(1) + return False + def test_snat(self): rules = ['100', '110', '120', '130', '200', '210', '220', '230'] outbound_iface_100 = 'eth0' @@ -84,6 +95,30 @@ class TestNAT(VyOSUnitTestSHIM.TestCase): self.verify_nftables(nftables_search, 'ip vyos_nat') + def test_snat_groups(self): + address_group = 'smoketest_addr' + address_group_member = '192.0.2.1' + rule = '100' + outbound_iface = 'eth0' + + self.cli_set(['firewall', 'group', 'address-group', address_group, 'address', address_group_member]) + + self.cli_set(src_path + ['rule', rule, 'source', 'group', 'address-group', address_group]) + self.cli_set(src_path + ['rule', rule, 'outbound-interface', outbound_iface]) + self.cli_set(src_path + ['rule', rule, 'translation', 'address', 'masquerade']) + + self.cli_commit() + + nftables_search = [ + [f'set A_{address_group}'], + [f'elements = {{ {address_group_member} }}'], + [f'ip saddr @A_{address_group}', f'oifname "{outbound_iface}"', 'masquerade'] + ] + + self.verify_nftables(nftables_search, 'ip vyos_nat') + + self.cli_delete(['firewall']) + def test_dnat(self): rules = ['100', '110', '120', '130', '200', '210', '220', '230'] inbound_iface_100 = 'eth0' diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index 2bb765e65..783adec46 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -41,6 +41,7 @@ from vyos import ConfigError from vyos import airbag airbag.enable() +nat_conf_script = '/usr/libexec/vyos/conf_mode/nat.py' policy_route_conf_script = '/usr/libexec/vyos/conf_mode/policy-route.py' nftables_conf = '/run/nftables.conf' @@ -158,7 +159,7 @@ def get_config(config=None): for zone in firewall['zone']: firewall['zone'][zone] = dict_merge(default_values, firewall['zone'][zone]) - firewall['policy_resync'] = bool('group' in firewall or node_changed(conf, base + ['group'])) + firewall['group_resync'] = bool('group' in firewall or node_changed(conf, base + ['group'])) if 'config_trap' in firewall and firewall['config_trap'] == 'enable': diff = get_config_diff(conf) @@ -463,6 +464,12 @@ def post_apply_trap(firewall): cmd(base_cmd + ' '.join(objects)) +def resync_nat(): + # Update nat as firewall groups were updated + tmp, out = rc_cmd(nat_conf_script) + if tmp > 0: + Warning(f'Failed to re-apply nat configuration! {out}') + def resync_policy_route(): # Update policy route as firewall groups were updated tmp, out = rc_cmd(policy_route_conf_script) @@ -474,19 +481,20 @@ def apply(firewall): if install_result == 1: raise ConfigError(f'Failed to apply firewall: {output}') + apply_sysfs(firewall) + + if firewall['group_resync']: + resync_nat() + resync_policy_route() + # T970 Enable a resolver (systemd daemon) that checks - # domain-group addresses and update entries for domains by timeout + # domain-group/fqdn addresses and update entries for domains by timeout # If router loaded without internet connection or for synchronization domain_action = 'stop' if dict_search_args(firewall, 'group', 'domain_group') or firewall['ip_fqdn'] or firewall['ip6_fqdn']: domain_action = 'restart' call(f'systemctl {domain_action} vyos-domain-resolver.service') - apply_sysfs(firewall) - - if firewall['policy_resync']: - resync_policy_route() - if firewall['geoip_updated']: # Call helper script to Update set contents if 'name' in firewall['geoip_updated'] or 'ipv6_name' in firewall['geoip_updated']: diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py index 978c043e9..9f8221514 100755 --- a/src/conf_mode/nat.py +++ b/src/conf_mode/nat.py @@ -32,6 +32,7 @@ from vyos.util import cmd from vyos.util import run from vyos.util import check_kmod from vyos.util import dict_search +from vyos.util import dict_search_args from vyos.validate import is_addr_assigned from vyos.xml import defaults from vyos import ConfigError @@ -47,6 +48,13 @@ else: nftables_nat_config = '/run/nftables_nat.conf' nftables_static_nat_conf = '/run/nftables_static-nat-rules.nft' +valid_groups = [ + 'address_group', + 'domain_group', + 'network_group', + 'port_group' +] + def get_handler(json, chain, target): """ Get nftable rule handler number of given chain/target combination. Handler is required when adding NAT/Conntrack helper targets """ @@ -60,7 +68,7 @@ def get_handler(json, chain, target): return None -def verify_rule(config, err_msg): +def verify_rule(config, err_msg, groups_dict): """ Common verify steps used for both source and destination NAT """ if (dict_search('translation.port', config) != None or @@ -78,6 +86,45 @@ def verify_rule(config, err_msg): 'statically maps a whole network of addresses onto another\n' \ 'network of addresses') + for side in ['destination', 'source']: + if side in config: + side_conf = config[side] + + if len({'address', 'fqdn'} & set(side_conf)) > 1: + raise ConfigError('Only one of address, fqdn or geoip can be specified') + + if 'group' in side_conf: + if len({'address_group', 'network_group', 'domain_group'} & set(side_conf['group'])) > 1: + raise ConfigError('Only one address-group, network-group or domain-group can be specified') + + for group in valid_groups: + if group in side_conf['group']: + group_name = side_conf['group'][group] + error_group = group.replace("_", "-") + + if group in ['address_group', 'network_group', 'domain_group']: + types = [t for t in ['address', 'fqdn'] if t in side_conf] + if types: + raise ConfigError(f'{error_group} and {types[0]} cannot both be defined') + + if group_name and group_name[0] == '!': + group_name = group_name[1:] + + group_obj = dict_search_args(groups_dict, group, group_name) + + if group_obj is None: + raise ConfigError(f'Invalid {error_group} "{group_name}" on firewall rule') + + if not group_obj: + Warning(f'{error_group} "{group_name}" has no members!') + + if dict_search_args(side_conf, 'group', 'port_group'): + if 'protocol' not in config: + raise ConfigError('Protocol must be defined if specifying a port-group') + + if config['protocol'] not in ['tcp', 'udp', 'tcp_udp']: + raise ConfigError('Protocol must be tcp, udp, or tcp_udp when specifying a port-group') + def get_config(config=None): if config: conf = config @@ -105,16 +152,20 @@ def get_config(config=None): condensed_json = jmespath.search(pattern, nftable_json) if not conf.exists(base): - nat['helper_functions'] = 'remove' - - # Retrieve current table handler positions - nat['pre_ct_ignore'] = get_handler(condensed_json, 'PREROUTING', 'VYOS_CT_HELPER') - nat['pre_ct_conntrack'] = get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK') - nat['out_ct_ignore'] = get_handler(condensed_json, 'OUTPUT', 'VYOS_CT_HELPER') - nat['out_ct_conntrack'] = get_handler(condensed_json, 'OUTPUT', 'NAT_CONNTRACK') + if get_handler(condensed_json, 'PREROUTING', 'VYOS_CT_HELPER'): + nat['helper_functions'] = 'remove' + + # Retrieve current table handler positions + nat['pre_ct_ignore'] = get_handler(condensed_json, 'PREROUTING', 'VYOS_CT_HELPER') + nat['pre_ct_conntrack'] = get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK') + nat['out_ct_ignore'] = get_handler(condensed_json, 'OUTPUT', 'VYOS_CT_HELPER') + nat['out_ct_conntrack'] = get_handler(condensed_json, 'OUTPUT', 'NAT_CONNTRACK') nat['deleted'] = '' return nat + nat['firewall_group'] = conf.get_config_dict(['firewall', 'group'], key_mangling=('-', '_'), get_first_key=True, + no_tag_node_value_mangle=True) + # check if NAT connection tracking helpers need to be set up - this has to # be done only once if not get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK'): @@ -157,7 +208,7 @@ def verify(nat): Warning(f'IP address {ip} does not exist on the system!') # common rule verification - verify_rule(config, err_msg) + verify_rule(config, err_msg, nat['firewall_group']) if dict_search('destination.rule', nat): @@ -175,7 +226,7 @@ def verify(nat): raise ConfigError(f'{err_msg} translation requires address and/or port') # common rule verification - verify_rule(config, err_msg) + verify_rule(config, err_msg, nat['firewall_group']) if dict_search('static.rule', nat): for rule, config in dict_search('static.rule', nat).items(): @@ -186,7 +237,7 @@ def verify(nat): 'inbound-interface not specified') # common rule verification - verify_rule(config, err_msg) + verify_rule(config, err_msg, nat['firewall_group']) return None diff --git a/src/helpers/vyos-domain-resolver.py b/src/helpers/vyos-domain-resolver.py index 2f71f15db..035c208b2 100755 --- a/src/helpers/vyos-domain-resolver.py +++ b/src/helpers/vyos-domain-resolver.py @@ -37,6 +37,7 @@ domain_state = {} ipv4_tables = { 'ip mangle', 'ip vyos_filter', + 'ip vyos_nat' } ipv6_tables = { -- cgit v1.2.3 From 586b24e0af1ae57c47c772229fc94ab50dfc1e4f Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Wed, 2 Nov 2022 15:32:11 +0100 Subject: policy: T2199: T4605: Migrate policy route interface to `policy route|route6 interface ` * Include refactor to policy route to allow for deletion of mangle table instead of complex cleanup * T4605: Rename mangle table to vyos_mangle --- data/templates/firewall/nftables-policy.j2 | 31 +++-- .../include/interface/interface-policy-vif-c.xml.i | 26 ---- .../include/interface/interface-policy-vif.xml.i | 26 ---- .../include/interface/interface-policy.xml.i | 26 ---- .../include/interface/vif-s.xml.i | 2 - interface-definitions/include/interface/vif.xml.i | 1 - .../include/version/policy-version.xml.i | 2 +- interface-definitions/interfaces-bonding.xml.in | 1 - interface-definitions/interfaces-bridge.xml.in | 1 - interface-definitions/interfaces-dummy.xml.in | 1 - interface-definitions/interfaces-ethernet.xml.in | 1 - interface-definitions/interfaces-geneve.xml.in | 1 - interface-definitions/interfaces-input.xml.in | 1 - interface-definitions/interfaces-l2tpv3.xml.in | 1 - interface-definitions/interfaces-macsec.xml.in | 1 - interface-definitions/interfaces-openvpn.xml.in | 1 - interface-definitions/interfaces-pppoe.xml.in | 1 - .../interfaces-pseudo-ethernet.xml.in | 1 - interface-definitions/interfaces-tunnel.xml.in | 1 - interface-definitions/interfaces-vti.xml.in | 1 - interface-definitions/interfaces-vxlan.xml.in | 1 - interface-definitions/interfaces-wireguard.xml.in | 1 - interface-definitions/interfaces-wireless.xml.in | 1 - interface-definitions/interfaces-wwan.xml.in | 1 - interface-definitions/policy-route.xml.in | 2 + smoketest/scripts/cli/test_policy_route.py | 58 +++++---- src/conf_mode/policy-route-interface.py | 132 --------------------- src/conf_mode/policy-route.py | 106 +---------------- src/helpers/vyos-domain-resolver.py | 4 +- src/migration-scripts/policy/4-to-5 | 92 ++++++++++++++ src/op_mode/policy_route.py | 42 +------ 31 files changed, 154 insertions(+), 413 deletions(-) delete mode 100644 interface-definitions/include/interface/interface-policy-vif-c.xml.i delete mode 100644 interface-definitions/include/interface/interface-policy-vif.xml.i delete mode 100644 interface-definitions/include/interface/interface-policy.xml.i delete mode 100755 src/conf_mode/policy-route-interface.py create mode 100755 src/migration-scripts/policy/4-to-5 (limited to 'src/helpers') diff --git a/data/templates/firewall/nftables-policy.j2 b/data/templates/firewall/nftables-policy.j2 index 40118930b..6cb3b2f95 100644 --- a/data/templates/firewall/nftables-policy.j2 +++ b/data/templates/firewall/nftables-policy.j2 @@ -2,21 +2,24 @@ {% import 'firewall/nftables-defines.j2' as group_tmpl %} -{% if cleanup_commands is vyos_defined %} -{% for command in cleanup_commands %} -{{ command }} -{% endfor %} +{% if first_install is not vyos_defined %} +delete table ip vyos_mangle +delete table ip6 vyos_mangle {% endif %} - -table ip mangle { -{% if first_install is vyos_defined %} +table ip vyos_mangle { chain VYOS_PBR_PREROUTING { type filter hook prerouting priority -150; policy accept; +{% if route is vyos_defined %} +{% for route_text, conf in route.items() if conf.interface is vyos_defined %} + iifname { {{ ",".join(conf.interface) }} } counter jump VYOS_PBR_{{ route_text }} +{% endfor %} +{% endif %} } + chain VYOS_PBR_POSTROUTING { type filter hook postrouting priority -150; policy accept; } -{% endif %} + {% if route is vyos_defined %} {% for route_text, conf in route.items() %} chain VYOS_PBR_{{ route_text }} { @@ -32,15 +35,20 @@ table ip mangle { {{ group_tmpl.groups(firewall_group, False) }} } -table ip6 mangle { -{% if first_install is vyos_defined %} +table ip6 vyos_mangle { chain VYOS_PBR6_PREROUTING { type filter hook prerouting priority -150; policy accept; +{% if route6 is vyos_defined %} +{% for route_text, conf in route6.items() if conf.interface is vyos_defined %} + iifname { {{ ",".join(conf.interface) }} } counter jump VYOS_PBR6_{{ route_text }} +{% endfor %} +{% endif %} } + chain VYOS_PBR6_POSTROUTING { type filter hook postrouting priority -150; policy accept; } -{% endif %} + {% if route6 is vyos_defined %} {% for route_text, conf in route6.items() %} chain VYOS_PBR6_{{ route_text }} { @@ -52,5 +60,6 @@ table ip6 mangle { } {% endfor %} {% endif %} + {{ group_tmpl.groups(firewall_group, True) }} } diff --git a/interface-definitions/include/interface/interface-policy-vif-c.xml.i b/interface-definitions/include/interface/interface-policy-vif-c.xml.i deleted file mode 100644 index 866fcd5c0..000000000 --- a/interface-definitions/include/interface/interface-policy-vif-c.xml.i +++ /dev/null @@ -1,26 +0,0 @@ - - - - 620 - Policy route options - - - - - IPv4 policy route ruleset for interface - - policy route - - - - - - IPv6 policy route ruleset for interface - - policy route6 - - - - - - diff --git a/interface-definitions/include/interface/interface-policy-vif.xml.i b/interface-definitions/include/interface/interface-policy-vif.xml.i deleted file mode 100644 index 83510fe59..000000000 --- a/interface-definitions/include/interface/interface-policy-vif.xml.i +++ /dev/null @@ -1,26 +0,0 @@ - - - - 620 - Policy route options - - - - - IPv4 policy route ruleset for interface - - policy route - - - - - - IPv6 policy route ruleset for interface - - policy route6 - - - - - - diff --git a/interface-definitions/include/interface/interface-policy.xml.i b/interface-definitions/include/interface/interface-policy.xml.i deleted file mode 100644 index 42a8fd009..000000000 --- a/interface-definitions/include/interface/interface-policy.xml.i +++ /dev/null @@ -1,26 +0,0 @@ - - - - 620 - Policy route options - - - - - IPv4 policy route ruleset for interface - - policy route - - - - - - IPv6 policy route ruleset for interface - - policy route6 - - - - - - diff --git a/interface-definitions/include/interface/vif-s.xml.i b/interface-definitions/include/interface/vif-s.xml.i index 916349ade..6d50d7238 100644 --- a/interface-definitions/include/interface/vif-s.xml.i +++ b/interface-definitions/include/interface/vif-s.xml.i @@ -18,7 +18,6 @@ #include #include #include - #include Protocol used for service VLAN (default: 802.1ad) @@ -67,7 +66,6 @@ #include #include #include - #include #include diff --git a/interface-definitions/include/interface/vif.xml.i b/interface-definitions/include/interface/vif.xml.i index 73a8c98ff..3f8f113ea 100644 --- a/interface-definitions/include/interface/vif.xml.i +++ b/interface-definitions/include/interface/vif.xml.i @@ -18,7 +18,6 @@ #include #include #include - #include VLAN egress QoS diff --git a/interface-definitions/include/version/policy-version.xml.i b/interface-definitions/include/version/policy-version.xml.i index 89bde20c7..f1494eaa3 100644 --- a/interface-definitions/include/version/policy-version.xml.i +++ b/interface-definitions/include/version/policy-version.xml.i @@ -1,3 +1,3 @@ - + diff --git a/interface-definitions/interfaces-bonding.xml.in b/interface-definitions/interfaces-bonding.xml.in index 41e4a68a8..96e0e5d89 100644 --- a/interface-definitions/interfaces-bonding.xml.in +++ b/interface-definitions/interfaces-bonding.xml.in @@ -56,7 +56,6 @@ #include #include #include - #include Bonding transmit hash policy diff --git a/interface-definitions/interfaces-bridge.xml.in b/interface-definitions/interfaces-bridge.xml.in index d633077d9..d52e213b6 100644 --- a/interface-definitions/interfaces-bridge.xml.in +++ b/interface-definitions/interfaces-bridge.xml.in @@ -41,7 +41,6 @@ #include #include #include - #include Forwarding delay diff --git a/interface-definitions/interfaces-dummy.xml.in b/interface-definitions/interfaces-dummy.xml.in index fb36741f7..eb525b547 100644 --- a/interface-definitions/interfaces-dummy.xml.in +++ b/interface-definitions/interfaces-dummy.xml.in @@ -19,7 +19,6 @@ #include #include #include - #include IPv4 routing parameters diff --git a/interface-definitions/interfaces-ethernet.xml.in b/interface-definitions/interfaces-ethernet.xml.in index 77f130e1c..e9ae0acfe 100644 --- a/interface-definitions/interfaces-ethernet.xml.in +++ b/interface-definitions/interfaces-ethernet.xml.in @@ -31,7 +31,6 @@ #include #include - #include Duplex mode diff --git a/interface-definitions/interfaces-geneve.xml.in b/interface-definitions/interfaces-geneve.xml.in index b959c787d..f8e9909f8 100644 --- a/interface-definitions/interfaces-geneve.xml.in +++ b/interface-definitions/interfaces-geneve.xml.in @@ -23,7 +23,6 @@ #include #include #include - #include GENEVE tunnel parameters diff --git a/interface-definitions/interfaces-input.xml.in b/interface-definitions/interfaces-input.xml.in index d01c760f8..97502d954 100644 --- a/interface-definitions/interfaces-input.xml.in +++ b/interface-definitions/interfaces-input.xml.in @@ -19,7 +19,6 @@ #include #include - #include #include diff --git a/interface-definitions/interfaces-l2tpv3.xml.in b/interface-definitions/interfaces-l2tpv3.xml.in index bde68dd5a..0ebc3253d 100644 --- a/interface-definitions/interfaces-l2tpv3.xml.in +++ b/interface-definitions/interfaces-l2tpv3.xml.in @@ -32,7 +32,6 @@ 5000 #include - #include Encapsulation type diff --git a/interface-definitions/interfaces-macsec.xml.in b/interface-definitions/interfaces-macsec.xml.in index 5c9f4cd76..441236ec2 100644 --- a/interface-definitions/interfaces-macsec.xml.in +++ b/interface-definitions/interfaces-macsec.xml.in @@ -21,7 +21,6 @@ #include #include #include - #include #include diff --git a/interface-definitions/interfaces-openvpn.xml.in b/interface-definitions/interfaces-openvpn.xml.in index 3876e31da..7cfb9ee7a 100644 --- a/interface-definitions/interfaces-openvpn.xml.in +++ b/interface-definitions/interfaces-openvpn.xml.in @@ -34,7 +34,6 @@ #include - #include OpenVPN interface device-type diff --git a/interface-definitions/interfaces-pppoe.xml.in b/interface-definitions/interfaces-pppoe.xml.in index 84f76a7ee..719060fa9 100644 --- a/interface-definitions/interfaces-pppoe.xml.in +++ b/interface-definitions/interfaces-pppoe.xml.in @@ -19,7 +19,6 @@ #include #include #include - #include #include #include #include diff --git a/interface-definitions/interfaces-pseudo-ethernet.xml.in b/interface-definitions/interfaces-pseudo-ethernet.xml.in index 4eb9bf111..2fe07ffd5 100644 --- a/interface-definitions/interfaces-pseudo-ethernet.xml.in +++ b/interface-definitions/interfaces-pseudo-ethernet.xml.in @@ -28,7 +28,6 @@ #include #include #include - #include Receive mode (default: private) diff --git a/interface-definitions/interfaces-tunnel.xml.in b/interface-definitions/interfaces-tunnel.xml.in index fe49d337a..333a5b178 100644 --- a/interface-definitions/interfaces-tunnel.xml.in +++ b/interface-definitions/interfaces-tunnel.xml.in @@ -29,7 +29,6 @@ #include #include #include - #include 6rd network prefix diff --git a/interface-definitions/interfaces-vti.xml.in b/interface-definitions/interfaces-vti.xml.in index eeaea0dc3..11f001dc0 100644 --- a/interface-definitions/interfaces-vti.xml.in +++ b/interface-definitions/interfaces-vti.xml.in @@ -25,7 +25,6 @@ #include #include #include - #include diff --git a/interface-definitions/interfaces-vxlan.xml.in b/interface-definitions/interfaces-vxlan.xml.in index 4902ff36d..331f930d3 100644 --- a/interface-definitions/interfaces-vxlan.xml.in +++ b/interface-definitions/interfaces-vxlan.xml.in @@ -54,7 +54,6 @@ #include #include #include - #include 1450 diff --git a/interface-definitions/interfaces-wireguard.xml.in b/interface-definitions/interfaces-wireguard.xml.in index 23f50d146..35e223588 100644 --- a/interface-definitions/interfaces-wireguard.xml.in +++ b/interface-definitions/interfaces-wireguard.xml.in @@ -21,7 +21,6 @@ #include #include #include - #include #include 1420 diff --git a/interface-definitions/interfaces-wireless.xml.in b/interface-definitions/interfaces-wireless.xml.in index 9e7fc29bc..5271df624 100644 --- a/interface-definitions/interfaces-wireless.xml.in +++ b/interface-definitions/interfaces-wireless.xml.in @@ -20,7 +20,6 @@ #include - #include HT and VHT capabilities for your card diff --git a/interface-definitions/interfaces-wwan.xml.in b/interface-definitions/interfaces-wwan.xml.in index b0b8367dc..758784540 100644 --- a/interface-definitions/interfaces-wwan.xml.in +++ b/interface-definitions/interfaces-wwan.xml.in @@ -39,7 +39,6 @@ #include #include #include - #include #include #include diff --git a/interface-definitions/policy-route.xml.in b/interface-definitions/policy-route.xml.in index 44b96c2e6..48a5bf7d1 100644 --- a/interface-definitions/policy-route.xml.in +++ b/interface-definitions/policy-route.xml.in @@ -12,6 +12,7 @@ #include + #include #include @@ -65,6 +66,7 @@ #include + #include #include diff --git a/smoketest/scripts/cli/test_policy_route.py b/smoketest/scripts/cli/test_policy_route.py index 046e385bb..11b3c678e 100755 --- a/smoketest/scripts/cli/test_policy_route.py +++ b/smoketest/scripts/cli/test_policy_route.py @@ -42,18 +42,25 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): super(TestPolicyRoute, cls).tearDownClass() def tearDown(self): - self.cli_delete(['interfaces', 'ethernet', interface, 'policy']) self.cli_delete(['policy', 'route']) self.cli_delete(['policy', 'route6']) self.cli_commit() + # Verify nftables cleanup nftables_search = [ ['set N_smoketest_network'], ['set N_smoketest_network1'], ['chain VYOS_PBR_smoketest'] ] - self.verify_nftables(nftables_search, 'ip mangle', inverse=True) + self.verify_nftables(nftables_search, 'ip vyos_mangle', inverse=True) + + # Verify ip rule cleanup + ip_rule_search = [ + ['fwmark ' + hex(table_mark_offset - int(table_id)), 'lookup ' + table_id] + ] + + self.verify_rules(ip_rule_search, inverse=True) def verify_nftables(self, nftables_search, table, inverse=False): nftables_output = cmd(f'sudo nft list table {table}') @@ -66,6 +73,17 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): break self.assertTrue(not matched if inverse else matched, msg=search) + def verify_rules(self, rules_search, inverse=False): + rule_output = cmd('ip rule show') + + for search in rules_search: + matched = False + for line in rule_output.split("\n"): + if all(item in line for item in search): + matched = True + break + self.assertTrue(not matched if inverse else matched, msg=search) + def test_pbr_group(self): self.cli_set(['firewall', 'group', 'network-group', 'smoketest_network', 'network', '172.16.99.0/24']) self.cli_set(['firewall', 'group', 'network-group', 'smoketest_network1', 'network', '172.16.101.0/24']) @@ -74,8 +92,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'source', 'group', 'network-group', 'smoketest_network']) self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'destination', 'group', 'network-group', 'smoketest_network1']) self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'set', 'mark', mark]) - - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route', 'smoketest']) + self.cli_set(['policy', 'route', 'smoketest', 'interface', interface]) self.cli_commit() @@ -84,7 +101,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['ip daddr @N_smoketest_network1', 'ip saddr @N_smoketest_network'], ] - self.verify_nftables(nftables_search, 'ip mangle') + self.verify_nftables(nftables_search, 'ip vyos_mangle') self.cli_delete(['firewall']) @@ -92,8 +109,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'source', 'address', '172.16.20.10']) self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'destination', 'address', '172.16.10.10']) self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'set', 'mark', mark]) - - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route', 'smoketest']) + self.cli_set(['policy', 'route', 'smoketest', 'interface', interface]) self.cli_commit() @@ -104,7 +120,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['ip daddr 172.16.10.10', 'ip saddr 172.16.20.10', 'meta mark set ' + mark_hex], ] - self.verify_nftables(nftables_search, 'ip mangle') + self.verify_nftables(nftables_search, 'ip vyos_mangle') def test_pbr_table(self): self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'protocol', 'tcp']) @@ -116,8 +132,8 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '1', 'destination', 'port', '8888']) self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '1', 'set', 'table', table_id]) - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route', 'smoketest']) - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route6', 'smoketest6']) + self.cli_set(['policy', 'route', 'smoketest', 'interface', interface]) + self.cli_set(['policy', 'route6', 'smoketest6', 'interface', interface]) self.cli_commit() @@ -130,7 +146,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['tcp flags syn / syn,ack', 'tcp dport 8888', 'meta mark set ' + mark_hex] ] - self.verify_nftables(nftables_search, 'ip mangle') + self.verify_nftables(nftables_search, 'ip vyos_mangle') # IPv6 @@ -139,7 +155,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['meta l4proto { tcp, udp }', 'th dport 8888', 'meta mark set ' + mark_hex] ] - self.verify_nftables(nftables6_search, 'ip6 mangle') + self.verify_nftables(nftables6_search, 'ip6 vyos_mangle') # IP rule fwmark -> table @@ -147,15 +163,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['fwmark ' + hex(table_mark_offset - int(table_id)), 'lookup ' + table_id] ] - ip_rule_output = cmd('ip rule show') - - for search in ip_rule_search: - matched = False - for line in ip_rule_output.split("\n"): - if all(item in line for item in search): - matched = True - break - self.assertTrue(matched) + self.verify_rules(ip_rule_search) def test_pbr_matching_criteria(self): @@ -203,8 +211,8 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '5', 'dscp-exclude', '14-19']) self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '5', 'set', 'table', table_id]) - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route', 'smoketest']) - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route6', 'smoketest6']) + self.cli_set(['policy', 'route', 'smoketest', 'interface', interface]) + self.cli_set(['policy', 'route6', 'smoketest6', 'interface', interface]) self.cli_commit() @@ -220,7 +228,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['ip dscp { 0x29, 0x39-0x3b }', 'meta mark set ' + mark_hex] ] - self.verify_nftables(nftables_search, 'ip mangle') + self.verify_nftables(nftables_search, 'ip vyos_mangle') # IPv6 nftables6_search = [ @@ -232,7 +240,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['ip6 dscp != { 0x0e-0x13, 0x3d }', 'meta mark set ' + mark_hex] ] - self.verify_nftables(nftables6_search, 'ip6 mangle') + self.verify_nftables(nftables6_search, 'ip6 vyos_mangle') if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/policy-route-interface.py b/src/conf_mode/policy-route-interface.py deleted file mode 100755 index 58c5fd93d..000000000 --- a/src/conf_mode/policy-route-interface.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2021 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import re - -from sys import argv -from sys import exit - -from vyos.config import Config -from vyos.ifconfig import Section -from vyos.template import render -from vyos.util import cmd -from vyos.util import run -from vyos import ConfigError -from vyos import airbag -airbag.enable() - -def get_config(config=None): - if config: - conf = config - else: - conf = Config() - - ifname = argv[1] - ifpath = Section.get_config_path(ifname) - if_policy_path = f'interfaces {ifpath} policy' - - if_policy = conf.get_config_dict(if_policy_path, key_mangling=('-', '_'), get_first_key=True, - no_tag_node_value_mangle=True) - - if_policy['ifname'] = ifname - if_policy['policy'] = conf.get_config_dict(['policy'], key_mangling=('-', '_'), get_first_key=True, - no_tag_node_value_mangle=True) - - return if_policy - -def verify_chain(table, chain): - # Verify policy route applied - code = run(f'nft list chain {table} {chain}') - return code == 0 - -def verify(if_policy): - # bail out early - looks like removal from running config - if not if_policy: - return None - - for route in ['route', 'route6']: - if route in if_policy: - if route not in if_policy['policy']: - raise ConfigError('Policy route not configured') - - route_name = if_policy[route] - - if route_name not in if_policy['policy'][route]: - raise ConfigError(f'Invalid policy route name "{name}"') - - nft_prefix = 'VYOS_PBR6_' if route == 'route6' else 'VYOS_PBR_' - nft_table = 'ip6 mangle' if route == 'route6' else 'ip mangle' - - if not verify_chain(nft_table, nft_prefix + route_name): - raise ConfigError('Policy route did not apply') - - return None - -def generate(if_policy): - return None - -def cleanup_rule(table, chain, ifname, new_name=None): - results = cmd(f'nft -a list chain {table} {chain}').split("\n") - retval = None - for line in results: - if f'ifname "{ifname}"' in line: - if new_name and f'jump {new_name}' in line: - # new_name is used to clear rules for any previously referenced chains - # returns true when rule exists and doesn't need to be created - retval = True - continue - - handle_search = re.search('handle (\d+)', line) - if handle_search: - cmd(f'nft delete rule {table} {chain} handle {handle_search[1]}') - return retval - -def apply(if_policy): - ifname = if_policy['ifname'] - - route_chain = 'VYOS_PBR_PREROUTING' - ipv6_route_chain = 'VYOS_PBR6_PREROUTING' - - if 'route' in if_policy: - name = 'VYOS_PBR_' + if_policy['route'] - rule_exists = cleanup_rule('ip mangle', route_chain, ifname, name) - - if not rule_exists: - cmd(f'nft insert rule ip mangle {route_chain} iifname {ifname} counter jump {name}') - else: - cleanup_rule('ip mangle', route_chain, ifname) - - if 'route6' in if_policy: - name = 'VYOS_PBR6_' + if_policy['route6'] - rule_exists = cleanup_rule('ip6 mangle', ipv6_route_chain, ifname, name) - - if not rule_exists: - cmd(f'nft insert rule ip6 mangle {ipv6_route_chain} iifname {ifname} counter jump {name}') - else: - cleanup_rule('ip6 mangle', ipv6_route_chain, ifname) - - return None - -if __name__ == '__main__': - try: - c = get_config() - verify(c) - generate(c) - apply(c) - except ConfigError as e: - print(e) - exit(1) diff --git a/src/conf_mode/policy-route.py b/src/conf_mode/policy-route.py index 00539b9c7..1d016695e 100755 --- a/src/conf_mode/policy-route.py +++ b/src/conf_mode/policy-route.py @@ -15,7 +15,6 @@ # along with this program. If not, see . import os -import re from json import loads from sys import exit @@ -25,7 +24,6 @@ from vyos.config import Config from vyos.template import render from vyos.util import cmd from vyos.util import dict_search_args -from vyos.util import dict_search_recursive from vyos.util import run from vyos import ConfigError from vyos import airbag @@ -34,48 +32,13 @@ airbag.enable() mark_offset = 0x7FFFFFFF nftables_conf = '/run/nftables_policy.conf' -ROUTE_PREFIX = 'VYOS_PBR_' -ROUTE6_PREFIX = 'VYOS_PBR6_' - -preserve_chains = [ - 'VYOS_PBR_PREROUTING', - 'VYOS_PBR_POSTROUTING', - 'VYOS_PBR6_PREROUTING', - 'VYOS_PBR6_POSTROUTING' -] - valid_groups = [ 'address_group', + 'domain_group', 'network_group', 'port_group' ] -group_set_prefix = { - 'A_': 'address_group', - 'A6_': 'ipv6_address_group', -# 'D_': 'domain_group', - 'M_': 'mac_group', - 'N_': 'network_group', - 'N6_': 'ipv6_network_group', - 'P_': 'port_group' -} - -def get_policy_interfaces(conf): - out = {} - interfaces = conf.get_config_dict(['interfaces'], key_mangling=('-', '_'), get_first_key=True, - no_tag_node_value_mangle=True) - def find_interfaces(iftype_conf, output={}, prefix=''): - for ifname, if_conf in iftype_conf.items(): - if 'policy' in if_conf: - output[prefix + ifname] = if_conf['policy'] - for vif in ['vif', 'vif_s', 'vif_c']: - if vif in if_conf: - output.update(find_interfaces(if_conf[vif], output, f'{prefix}{ifname}.')) - return output - for iftype, iftype_conf in interfaces.items(): - out.update(find_interfaces(iftype_conf)) - return out - def get_config(config=None): if config: conf = config @@ -88,7 +51,6 @@ def get_config(config=None): policy['firewall_group'] = conf.get_config_dict(['firewall', 'group'], key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) - policy['interfaces'] = get_policy_interfaces(conf) return policy @@ -132,8 +94,8 @@ def verify_rule(policy, name, rule_conf, ipv6, rule_id): side_conf = rule_conf[side] if 'group' in side_conf: - if {'address_group', 'network_group'} <= set(side_conf['group']): - raise ConfigError('Only one address-group or network-group can be specified') + if len({'address_group', 'domain_group', 'network_group'} & set(side_conf['group'])) > 1: + raise ConfigError('Only one address-group, domain-group or network-group can be specified') for group in valid_groups: if group in side_conf['group']: @@ -168,73 +130,11 @@ def verify(policy): for rule_id, rule_conf in pol_conf['rule'].items(): verify_rule(policy, name, rule_conf, ipv6, rule_id) - for ifname, if_policy in policy['interfaces'].items(): - name = dict_search_args(if_policy, 'route') - ipv6_name = dict_search_args(if_policy, 'route6') - - if name and not dict_search_args(policy, 'route', name): - raise ConfigError(f'Policy route "{name}" is still referenced on interface {ifname}') - - if ipv6_name and not dict_search_args(policy, 'route6', ipv6_name): - raise ConfigError(f'Policy route6 "{ipv6_name}" is still referenced on interface {ifname}') - return None -def cleanup_commands(policy): - commands = [] - commands_chains = [] - commands_sets = [] - for table in ['ip mangle', 'ip6 mangle']: - route_node = 'route' if table == 'ip mangle' else 'route6' - chain_prefix = ROUTE_PREFIX if table == 'ip mangle' else ROUTE6_PREFIX - - json_str = cmd(f'nft -t -j list table {table}') - obj = loads(json_str) - if 'nftables' not in obj: - continue - for item in obj['nftables']: - if 'chain' in item: - chain = item['chain']['name'] - if chain in preserve_chains or not chain.startswith("VYOS_PBR"): - continue - - if dict_search_args(policy, route_node, chain.replace(chain_prefix, "", 1)) != None: - commands.append(f'flush chain {table} {chain}') - else: - commands_chains.append(f'delete chain {table} {chain}') - - if 'rule' in item: - rule = item['rule'] - chain = rule['chain'] - handle = rule['handle'] - - if chain not in preserve_chains: - continue - - target, _ = next(dict_search_recursive(rule['expr'], 'target')) - - if target.startswith(chain_prefix): - if dict_search_args(policy, route_node, target.replace(chain_prefix, "", 1)) == None: - commands.append(f'delete rule {table} {chain} handle {handle}') - - if 'set' in item: - set_name = item['set']['name'] - - for prefix, group_type in group_set_prefix.items(): - if set_name.startswith(prefix): - group_name = set_name.replace(prefix, "", 1) - if dict_search_args(policy, 'firewall_group', group_type, group_name) != None: - commands_sets.append(f'flush set {table} {set_name}') - else: - commands_sets.append(f'delete set {table} {set_name}') - - return commands + commands_chains + commands_sets - def generate(policy): if not os.path.exists(nftables_conf): policy['first_install'] = True - else: - policy['cleanup_commands'] = cleanup_commands(policy) render(nftables_conf, 'firewall/nftables-policy.j2', policy) return None diff --git a/src/helpers/vyos-domain-resolver.py b/src/helpers/vyos-domain-resolver.py index 035c208b2..e31d9238e 100755 --- a/src/helpers/vyos-domain-resolver.py +++ b/src/helpers/vyos-domain-resolver.py @@ -35,13 +35,13 @@ cache = False domain_state = {} ipv4_tables = { - 'ip mangle', + 'ip vyos_mangle', 'ip vyos_filter', 'ip vyos_nat' } ipv6_tables = { - 'ip6 mangle', + 'ip6 vyos_mangle', 'ip6 vyos_filter' } diff --git a/src/migration-scripts/policy/4-to-5 b/src/migration-scripts/policy/4-to-5 new file mode 100755 index 000000000..33c9e6ade --- /dev/null +++ b/src/migration-scripts/policy/4-to-5 @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# T2199: Migrate interface policy nodes to policy route interface + +import re + +from sys import argv +from sys import exit + +from vyos.configtree import ConfigTree +from vyos.ifconfig import Section + +if (len(argv) < 1): + print("Must specify file name!") + exit(1) + +file_name = argv[1] + +with open(file_name, 'r') as f: + config_file = f.read() + +base4 = ['policy', 'route'] +base6 = ['policy', 'route6'] +config = ConfigTree(config_file) + +if not config.exists(base4) and not config.exists(base6): + # Nothing to do + exit(0) + +def migrate_interface(config, iftype, ifname, vif=None, vifs=None, vifc=None): + if_path = ['interfaces', iftype, ifname] + ifname_full = ifname + + if vif: + if_path += ['vif', vif] + ifname_full = f'{ifname}.{vif}' + elif vifs: + if_path += ['vif-s', vifs] + ifname_full = f'{ifname}.{vifs}' + if vifc: + if_path += ['vif-c', vifc] + ifname_full = f'{ifname}.{vifs}.{vifc}' + + if not config.exists(if_path + ['policy']): + return + + if config.exists(if_path + ['policy', 'route']): + route_name = config.return_value(if_path + ['policy', 'route']) + config.set(base4 + [route_name, 'interface'], value=ifname_full, replace=False) + + if config.exists(if_path + ['policy', 'route6']): + route_name = config.return_value(if_path + ['policy', 'route6']) + config.set(base6 + [route_name, 'interface'], value=ifname_full, replace=False) + + config.delete(if_path + ['policy']) + +for iftype in config.list_nodes(['interfaces']): + for ifname in config.list_nodes(['interfaces', iftype]): + migrate_interface(config, iftype, ifname) + + if config.exists(['interfaces', iftype, ifname, 'vif']): + for vif in config.list_nodes(['interfaces', iftype, ifname, 'vif']): + migrate_interface(config, iftype, ifname, vif=vif) + + if config.exists(['interfaces', iftype, ifname, 'vif-s']): + for vifs in config.list_nodes(['interfaces', iftype, ifname, 'vif-s']): + migrate_interface(config, iftype, ifname, vifs=vifs) + + if config.exists(['interfaces', iftype, ifname, 'vif-s', vifs, 'vif-c']): + for vifc in config.list_nodes(['interfaces', iftype, ifname, 'vif-s', vifs, 'vif-c']): + migrate_interface(config, iftype, ifname, vifs=vifs, vifc=vifc) + +try: + with open(file_name, 'w') as f: + f.write(config.to_string()) +except OSError as e: + print("Failed to save the modified config: {}".format(e)) + exit(1) diff --git a/src/op_mode/policy_route.py b/src/op_mode/policy_route.py index 5be40082f..5953786f3 100755 --- a/src/op_mode/policy_route.py +++ b/src/op_mode/policy_route.py @@ -22,53 +22,13 @@ from vyos.config import Config from vyos.util import cmd from vyos.util import dict_search_args -def get_policy_interfaces(conf, policy, name=None, ipv6=False): - interfaces = conf.get_config_dict(['interfaces'], key_mangling=('-', '_'), - get_first_key=True, no_tag_node_value_mangle=True) - - routes = ['route', 'route6'] - - def parse_if(ifname, if_conf): - if 'policy' in if_conf: - for route in routes: - if route in if_conf['policy']: - route_name = if_conf['policy'][route] - name_str = f'({ifname},{route})' - - if not name: - policy[route][route_name]['interface'].append(name_str) - elif not ipv6 and name == route_name: - policy['interface'].append(name_str) - - for iftype in ['vif', 'vif_s', 'vif_c']: - if iftype in if_conf: - for vifname, vif_conf in if_conf[iftype].items(): - parse_if(f'{ifname}.{vifname}', vif_conf) - - for iftype, iftype_conf in interfaces.items(): - for ifname, if_conf in iftype_conf.items(): - parse_if(ifname, if_conf) - -def get_config_policy(conf, name=None, ipv6=False, interfaces=True): +def get_config_policy(conf, name=None, ipv6=False): config_path = ['policy'] if name: config_path += ['route6' if ipv6 else 'route', name] policy = conf.get_config_dict(config_path, key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) - if policy and interfaces: - if name: - policy['interface'] = [] - else: - if 'route' in policy: - for route_name, route_conf in policy['route'].items(): - route_conf['interface'] = [] - - if 'route6' in policy: - for route_name, route_conf in policy['route6'].items(): - route_conf['interface'] = [] - - get_policy_interfaces(conf, policy, name, ipv6) return policy -- cgit v1.2.3 From 932af7f098808009f626c788deb9e1d1c8bf3426 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 13 Jun 2022 15:40:11 +0000 Subject: routing: T1237: Add new feature failover route Failover route allows to install static routes to the kernel routing table only if required target or gateway is alive When target or gateway doesn't respond for ICMP/ARP checks this route deleted from the routing table Routes are marked as protocol 'failover' (rt_protos) cat /etc/iproute2/rt_protos.d/failover.conf 111 failover ip route add 203.0.113.1 metric 2 via 192.0.2.1 dev eth0 proto failover $ sudo ip route show proto failover 203.0.113.1 via 192.0.2.1 dev eth0 metric 1 So we can safely flush such routes --- .../protocols/systemd_vyos_failover_service.j2 | 11 ++ interface-definitions/protocols-failover.xml.in | 114 +++++++++++++ src/conf_mode/protocols_failover.py | 121 ++++++++++++++ src/helpers/vyos-failover.py | 184 +++++++++++++++++++++ 4 files changed, 430 insertions(+) create mode 100644 data/templates/protocols/systemd_vyos_failover_service.j2 create mode 100644 interface-definitions/protocols-failover.xml.in create mode 100755 src/conf_mode/protocols_failover.py create mode 100755 src/helpers/vyos-failover.py (limited to 'src/helpers') diff --git a/data/templates/protocols/systemd_vyos_failover_service.j2 b/data/templates/protocols/systemd_vyos_failover_service.j2 new file mode 100644 index 000000000..e6501e0f5 --- /dev/null +++ b/data/templates/protocols/systemd_vyos_failover_service.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Failover route service +After=vyos-router.service + +[Service] +Type=simple +Restart=always +ExecStart=/usr/bin/python3 /usr/libexec/vyos/vyos-failover.py --config /run/vyos-failover.conf + +[Install] +WantedBy=multi-user.target diff --git a/interface-definitions/protocols-failover.xml.in b/interface-definitions/protocols-failover.xml.in new file mode 100644 index 000000000..900c76eab --- /dev/null +++ b/interface-definitions/protocols-failover.xml.in @@ -0,0 +1,114 @@ + + + + + + + Failover Routing + 490 + + + + + Failover IPv4 route + + ipv4net + IPv4 failover route + + + + + + + + + Next-hop IPv4 router address + + ipv4 + Next-hop router address + + + + + + + + + Check target options + + + #include + + + Check target address + + ipv4 + Address to check + + + + + + + + + Timeout between checks + + u32:1-300 + Timeout in seconds between checks + + + + + + 10 + + + + Check type + + arp icmp tcp + + + arp + Check target by ARP + + + icmp + Check target by ICMP + + + tcp + Check target by TCP + + + (arp|icmp|tcp) + + + icmp + + + + #include + + + Route metric for this gateway + + u32:1-255 + Route metric + + + + + + 1 + + + + + + + + + + diff --git a/src/conf_mode/protocols_failover.py b/src/conf_mode/protocols_failover.py new file mode 100755 index 000000000..048ba7a89 --- /dev/null +++ b/src/conf_mode/protocols_failover.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import json + +from pathlib import Path + +from vyos.config import Config +from vyos.configdict import dict_merge +from vyos.template import render +from vyos.util import call +from vyos.xml import defaults +from vyos import ConfigError +from vyos import airbag + +airbag.enable() + + +service_name = 'vyos-failover' +service_conf = Path(f'/run/{service_name}.conf') +systemd_service = '/etc/systemd/system/vyos-failover.service' +rt_proto_failover = '/etc/iproute2/rt_protos.d/failover.conf' + + +def get_config(config=None): + if config: + conf = config + else: + conf = Config() + + base = ['protocols', 'failover'] + failover = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) + + # Set default values only if we set config + if failover.get('route'): + for route, route_config in failover.get('route').items(): + for next_hop, next_hop_config in route_config.get('next_hop').items(): + default_values = defaults(base + ['route']) + failover['route'][route]['next_hop'][next_hop] = dict_merge( + default_values['next_hop'], failover['route'][route]['next_hop'][next_hop]) + + return failover + +def verify(failover): + # bail out early - looks like removal from running config + if not failover: + return None + + if 'route' not in failover: + raise ConfigError(f'Failover "route" is mandatory!') + + for route, route_config in failover['route'].items(): + if not route_config.get('next_hop'): + raise ConfigError(f'Next-hop for "{route}" is mandatory!') + + for next_hop, next_hop_config in route_config.get('next_hop').items(): + if 'interface' not in next_hop_config: + raise ConfigError(f'Interface for route "{route}" next-hop "{next_hop}" is mandatory!') + + if not next_hop_config.get('check'): + raise ConfigError(f'Check target for next-hop "{next_hop}" is mandatory!') + + if 'target' not in next_hop_config['check']: + raise ConfigError(f'Check target for next-hop "{next_hop}" is mandatory!') + + check_type = next_hop_config['check']['type'] + if check_type == 'tcp' and 'port' not in next_hop_config['check']: + raise ConfigError(f'Check port for next-hop "{next_hop}" and type TCP is mandatory!') + + return None + +def generate(failover): + if not failover: + service_conf.unlink(missing_ok=True) + return None + + # Add own rt_proto 'failover' + # Helps to detect all own routes 'proto failover' + with open(rt_proto_failover, 'w') as f: + f.write('111 failover\n') + + # Write configuration file + conf_json = json.dumps(failover, indent=4) + service_conf.write_text(conf_json) + render(systemd_service, 'protocols/systemd_vyos_failover_service.j2', failover) + + return None + +def apply(failover): + if not failover: + call(f'systemctl stop {service_name}.service') + call('ip route flush protocol failover') + else: + call('systemctl daemon-reload') + call(f'systemctl restart {service_name}.service') + call(f'ip route flush protocol failover') + + return None + +if __name__ == '__main__': + try: + c = get_config() + verify(c) + generate(c) + apply(c) + except ConfigError as e: + print(e) + exit(1) diff --git a/src/helpers/vyos-failover.py b/src/helpers/vyos-failover.py new file mode 100755 index 000000000..1ac193423 --- /dev/null +++ b/src/helpers/vyos-failover.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import argparse +import json +import subprocess +import socket +import time + +from vyos.util import rc_cmd +from pathlib import Path +from systemd import journal + + +my_name = Path(__file__).stem + + +def get_best_route_options(route, debug=False): + """ + Return current best route ('gateway, interface, metric) + + % get_best_route_options('203.0.113.1') + ('192.168.0.1', 'eth1', 1) + + % get_best_route_options('203.0.113.254') + (None, None, None) + """ + rc, data = rc_cmd(f'ip --detail --json route show protocol failover {route}') + if rc == 0: + data = json.loads(data) + if len(data) == 0: + print(f'\nRoute {route} for protocol failover was not found') + return None, None, None + # Fake metric 999 by default + # Search route with the lowest metric + best_metric = 999 + for entry in data: + if debug: print('\n', entry) + metric = entry.get('metric') + gateway = entry.get('gateway') + iface = entry.get('dev') + if metric < best_metric: + best_metric = metric + best_gateway = gateway + best_interface = iface + if debug: + print(f'### Best_route exists: {route}, best_gateway: {best_gateway}, ' + f'best_metric: {best_metric}, best_iface: {best_interface}') + return best_gateway, best_interface, best_metric + +def is_port_open(ip, port): + """ + Check connection to remote host and port + Return True if host alive + + % is_port_open('example.com', 8080) + True + """ + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) + s.settimeout(2) + try: + s.connect((ip, int(port))) + s.shutdown(socket.SHUT_RDWR) + return True + except: + return False + finally: + s.close() + +def is_target_alive(target=None, iface='', proto='icmp', port=None, debug=False): + """ + Host availability check by ICMP, ARP, TCP + Return True if target checks is successful + + % is_target_alive('192.0.2.1', 'eth1', proto='arp') + True + """ + if iface != '': + iface = f'-I {iface}' + if proto == 'icmp': + command = f'/usr/bin/ping -q {target} {iface} -n -c 2 -W 1' + rc, response = rc_cmd(command) + if debug: print(f' [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]') + if rc == 0: + return True + elif proto == 'arp': + command = f'/usr/bin/arping -b -c 2 -f -w 1 -i 1 {iface} {target}' + rc, response = rc_cmd(command) + if debug: print(f' [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]') + if rc == 0: + return True + elif proto == 'tcp' and port is not None: + return True if is_port_open(target, port) else False + else: + return False + + +if __name__ == '__main__': + # Parse command arguments and get config + parser = argparse.ArgumentParser() + parser.add_argument('-c', + '--config', + action='store', + help='Path to protocols failover configuration', + required=True, + type=Path) + + args = parser.parse_args() + try: + config_path = Path(args.config) + config = json.loads(config_path.read_text()) + except Exception as err: + print( + f'Configuration file "{config_path}" does not exist or malformed: {err}' + ) + exit(1) + + # Useful debug info to console, use debug = True + # sudo systemctl stop vyos-failover.service + # sudo /usr/libexec/vyos/vyos-failover.py --config /run/vyos-failover.conf + debug = False + + while(True): + + for route, route_config in config.get('route').items(): + + exists_route = exists_gateway, exists_iface, exists_metric = get_best_route_options(route, debug=debug) + + for next_hop, nexthop_config in route_config.get('next_hop').items(): + conf_iface = nexthop_config.get('interface') + conf_metric = int(nexthop_config.get('metric')) + port = nexthop_config.get('check').get('port') + port_opt = f'port {port}' if port else '' + proto = nexthop_config.get('check').get('type') + target = nexthop_config.get('check').get('target') + timeout = nexthop_config.get('check').get('timeout') + + # Best route not fonund in the current routing table + if exists_route == (None, None, None): + if debug: print(f" [NEW_ROUTE_DETECTED] route: [{route}]") + # Add route if check-target alive + if is_target_alive(target, conf_iface, proto, port, debug=debug): + if debug: print(f' [ ADD ] -- ip route add {route} via {next_hop} dev {conf_iface} ' + f'metric {conf_metric} proto failover\n###') + rc, command = rc_cmd(f'ip route add {route} via {next_hop} dev {conf_iface} ' + f'metric {conf_metric} proto failover') + # If something is wrong and gateway not added + # Example: Error: Next-hop has invalid gateway. + if rc !=0: + if debug: print(f'{command} -- return-code [RC: {rc}] {next_hop} dev {conf_iface}') + else: + journal.send(f'ip route add {route} via {next_hop} dev {conf_iface} ' + f'metric {conf_metric} proto failover', SYSLOG_IDENTIFIER=my_name) + else: + if debug: print(f' [ TARGET_FAIL ] target checks fails for [{target}], do nothing') + journal.send(f'Check fail for route {route} target {target} proto {proto} ' + f'{port_opt}', SYSLOG_IDENTIFIER=my_name) + + # Route was added, check if the target is alive + # We should delete route if check fails only if route exists in the routing table + if not is_target_alive(target, conf_iface, proto, port, debug=debug) and \ + exists_route != (None, None, None): + if debug: + print(f'Nexh_hop {next_hop} fail, target not response') + print(f' [ DEL ] -- ip route del {route} via {next_hop} dev {conf_iface} ' + f'metric {conf_metric} proto failover [DELETE]') + rc_cmd(f'ip route del {route} via {next_hop} dev {conf_iface} metric {conf_metric} proto failover') + journal.send(f'ip route del {route} via {next_hop} dev {conf_iface} ' + f'metric {conf_metric} proto failover', SYSLOG_IDENTIFIER=my_name) + + time.sleep(int(timeout)) -- cgit v1.2.3