From 85f04237160a6ea98eea4ec58f1ccab9f6bfc31a Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 17 Oct 2022 12:15:22 +0000 Subject: ssh: T4720: Ability to configure SSH-server HostKeyAlgorithms Ability to configure SSH-server HostKeyAlgorithms. Specifies the host key signature algorithms that the server offers. Can accept multiple values. --- data/templates/ssh/sshd_config.j2 | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'data') diff --git a/data/templates/ssh/sshd_config.j2 b/data/templates/ssh/sshd_config.j2 index 5bbfdeb88..93735020c 100644 --- a/data/templates/ssh/sshd_config.j2 +++ b/data/templates/ssh/sshd_config.j2 @@ -62,6 +62,11 @@ ListenAddress {{ address }} Ciphers {{ ciphers | join(',') }} {% endif %} +{% if hostkey_algorithm is vyos_defined %} +# Specifies the available Host Key signature algorithms +HostKeyAlgorithms {{ hostkey_algorithm | join(',') }} +{% endif %} + {% if mac is vyos_defined %} # Specifies the available MAC (message authentication code) algorithms MACs {{ mac | join(',') }} -- cgit v1.2.3 From 2a5273e650ce1242bc22e992e5a3104961ec1295 Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Tue, 25 Oct 2022 12:29:03 +0200 Subject: nat: T4764: Remove tables on NAT deletion --- data/templates/firewall/nftables-nat.j2 | 18 ++++++++++-------- data/templates/firewall/nftables-static-nat.j2 | 18 ++++++++++-------- smoketest/scripts/cli/test_nat.py | 6 ++++++ src/conf_mode/nat.py | 4 ++++ 4 files changed, 30 insertions(+), 16 deletions(-) (limited to 'data') diff --git a/data/templates/firewall/nftables-nat.j2 b/data/templates/firewall/nftables-nat.j2 index 55fe6024b..c5c0a2c86 100644 --- a/data/templates/firewall/nftables-nat.j2 +++ b/data/templates/firewall/nftables-nat.j2 @@ -24,6 +24,7 @@ add rule ip raw NAT_CONNTRACK counter accept {% if first_install is not vyos_defined %} delete table ip vyos_nat {% endif %} +{% if deleted is not vyos_defined %} table ip vyos_nat { # # Destination NAT rules build up here @@ -31,11 +32,11 @@ table ip vyos_nat { chain PREROUTING { type nat hook prerouting priority -100; policy accept; counter jump VYOS_PRE_DNAT_HOOK -{% if destination.rule is vyos_defined %} -{% for rule, config in destination.rule.items() if config.disable is not vyos_defined %} +{% if destination.rule is vyos_defined %} +{% for rule, config in destination.rule.items() if config.disable is not vyos_defined %} {{ config | nat_rule(rule, 'destination') }} -{% endfor %} -{% endif %} +{% endfor %} +{% endif %} } # @@ -44,11 +45,11 @@ table ip vyos_nat { chain POSTROUTING { type nat hook postrouting priority 100; policy accept; counter jump VYOS_PRE_SNAT_HOOK -{% if source.rule is vyos_defined %} -{% for rule, config in source.rule.items() if config.disable is not vyos_defined %} +{% if source.rule is vyos_defined %} +{% for rule, config in source.rule.items() if config.disable is not vyos_defined %} {{ config | nat_rule(rule, 'source') }} -{% endfor %} -{% endif %} +{% endfor %} +{% endif %} } chain VYOS_PRE_DNAT_HOOK { @@ -59,3 +60,4 @@ table ip vyos_nat { return } } +{% endif %} diff --git a/data/templates/firewall/nftables-static-nat.j2 b/data/templates/firewall/nftables-static-nat.j2 index 790c33ce9..e5e3da867 100644 --- a/data/templates/firewall/nftables-static-nat.j2 +++ b/data/templates/firewall/nftables-static-nat.j2 @@ -3,6 +3,7 @@ {% if first_install is not vyos_defined %} delete table ip vyos_static_nat {% endif %} +{% if deleted is not vyos_defined %} table ip vyos_static_nat { # # Destination NAT rules build up here @@ -10,11 +11,11 @@ table ip vyos_static_nat { chain PREROUTING { type nat hook prerouting priority -100; policy accept; -{% if static.rule is vyos_defined %} -{% for rule, config in static.rule.items() if config.disable is not vyos_defined %} +{% if static.rule is vyos_defined %} +{% for rule, config in static.rule.items() if config.disable is not vyos_defined %} {{ config | nat_static_rule(rule, 'destination') }} -{% endfor %} -{% endif %} +{% endfor %} +{% endif %} } # @@ -22,10 +23,11 @@ table ip vyos_static_nat { # chain POSTROUTING { type nat hook postrouting priority 100; policy accept; -{% if static.rule is vyos_defined %} -{% for rule, config in static.rule.items() if config.disable is not vyos_defined %} +{% if static.rule is vyos_defined %} +{% for rule, config in static.rule.items() if config.disable is not vyos_defined %} {{ config | nat_static_rule(rule, 'source') }} -{% endfor %} -{% endif %} +{% endfor %} +{% endif %} } } +{% endif %} diff --git a/smoketest/scripts/cli/test_nat.py b/smoketest/scripts/cli/test_nat.py index f824838c0..2ae90fcaf 100755 --- a/smoketest/scripts/cli/test_nat.py +++ b/smoketest/scripts/cli/test_nat.py @@ -16,6 +16,7 @@ import jmespath import json +import os import unittest from base_vyostest_shim import VyOSUnitTestSHIM @@ -28,6 +29,9 @@ src_path = base_path + ['source'] dst_path = base_path + ['destination'] static_path = base_path + ['static'] +nftables_nat_config = '/run/nftables_nat.conf' +nftables_static_nat_conf = '/run/nftables_static-nat-rules.nft' + class TestNAT(VyOSUnitTestSHIM.TestCase): @classmethod def setUpClass(cls): @@ -40,6 +44,8 @@ class TestNAT(VyOSUnitTestSHIM.TestCase): def tearDown(self): self.cli_delete(base_path) self.cli_commit() + self.assertFalse(os.path.exists(nftables_nat_config)) + self.assertFalse(os.path.exists(nftables_static_nat_conf)) def verify_nftables(self, nftables_search, table, inverse=False, args=''): nftables_output = cmd(f'sudo nft {args} list table {table}') diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py index 8b1a5a720..1e807753d 100755 --- a/src/conf_mode/nat.py +++ b/src/conf_mode/nat.py @@ -204,6 +204,10 @@ def apply(nat): cmd(f'nft -f {nftables_nat_config}') cmd(f'nft -f {nftables_static_nat_conf}') + if not nat or 'deleted' in nat: + os.unlink(nftables_nat_config) + os.unlink(nftables_static_nat_conf) + return None if __name__ == '__main__': -- cgit v1.2.3 From f9c1277f5cf56fba2fc773d133de0221b06fa511 Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Fri, 28 Oct 2022 21:27:37 +0200 Subject: containers: T3903: Use systemd units for containers * ExecStop action with defined timeout allows for quicker reboot/shutdown with containers --- data/templates/container/systemd-unit.j2 | 17 +++ src/conf_mode/container.py | 172 ++++++++++++++++--------------- 2 files changed, 108 insertions(+), 81 deletions(-) create mode 100644 data/templates/container/systemd-unit.j2 (limited to 'data') diff --git a/data/templates/container/systemd-unit.j2 b/data/templates/container/systemd-unit.j2 new file mode 100644 index 000000000..fa48384ab --- /dev/null +++ b/data/templates/container/systemd-unit.j2 @@ -0,0 +1,17 @@ +### Autogenerated by container.py ### +[Unit] +Description=VyOS Container {{ name }} + +[Service] +Environment=PODMAN_SYSTEMD_UNIT=%n +Restart=on-failure +ExecStartPre=/bin/rm -f %t/%n.pid %t/%n.cid +ExecStart=/usr/bin/podman run \ + --conmon-pidfile %t/%n.pid --cidfile %t/%n.cid --cgroups=no-conmon \ + {{ run_args }} +ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%n.cid -t 5 +ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%n.cid +ExecStopPost=/bin/rm -f %t/%n.cid +PIDFile=%t/%n.pid +KillMode=none +Type=forking diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py index ac3dc536b..70d149f0d 100755 --- a/src/conf_mode/container.py +++ b/src/conf_mode/container.py @@ -40,20 +40,7 @@ airbag.enable() config_containers_registry = '/etc/containers/registries.conf' config_containers_storage = '/etc/containers/storage.conf' - -def _run_rerun(container_cmd): - counter = 0 - while True: - if counter >= 10: - break - try: - _cmd(container_cmd) - break - except: - counter = counter +1 - sleep(0.5) - - return None +systemd_unit_path = '/run/systemd/system' def _cmd(command): if os.path.exists('/tmp/vyos.container.debug'): @@ -122,7 +109,7 @@ def verify(container): # of image upgrade and deletion. image = container_config['image'] if run(f'podman image exists {image}') != 0: - Warning(f'Image "{image}" used in contianer "{name}" does not exist '\ + Warning(f'Image "{image}" used in container "{name}" does not exist '\ f'locally. Please use "add container image {image}" to add it '\ f'to the system! Container "{name}" will not be started!') @@ -136,9 +123,6 @@ def verify(container): raise ConfigError(f'Container network "{network_name}" does not exist!') if 'address' in container_config['network'][network_name]: - if 'network' not in container_config: - raise ConfigError(f'Can not use "address" without "network" for container "{name}"!') - address = container_config['network'][network_name]['address'] network = None if is_ipv4(address): @@ -220,6 +204,71 @@ def verify(container): return None +def generate_run_arguments(name, container_config): + image = container_config['image'] + memory = container_config['memory'] + restart = container_config['restart'] + + # Add capability options. Should be in uppercase + cap_add = '' + if 'cap_add' in container_config: + for c in container_config['cap_add']: + c = c.upper() + c = c.replace('-', '_') + cap_add += f' --cap-add={c}' + + # Add a host device to the container /dev/x:/dev/x + device = '' + if 'device' in container_config: + for dev, dev_config in container_config['device'].items(): + source_dev = dev_config['source'] + dest_dev = dev_config['destination'] + device += f' --device={source_dev}:{dest_dev}' + + # Check/set environment options "-e foo=bar" + env_opt = '' + if 'environment' in container_config: + for k, v in container_config['environment'].items(): + env_opt += f" -e \"{k}={v['value']}\"" + + # Publish ports + port = '' + if 'port' in container_config: + protocol = '' + for portmap in container_config['port']: + if 'protocol' in container_config['port'][portmap]: + protocol = container_config['port'][portmap]['protocol'] + protocol = f'/{protocol}' + else: + protocol = '/tcp' + sport = container_config['port'][portmap]['source'] + dport = container_config['port'][portmap]['destination'] + port += f' -p {sport}:{dport}{protocol}' + + # Bind volume + volume = '' + if 'volume' in container_config: + for vol, vol_config in container_config['volume'].items(): + svol = vol_config['source'] + dvol = vol_config['destination'] + volume += f' -v {svol}:{dvol}' + + container_base_cmd = f'--detach --interactive --tty --replace {cap_add} ' \ + f'--memory {memory}m --memory-swap 0 --restart {restart} ' \ + f'--name {name} {device} {port} {volume} {env_opt}' + + if 'allow_host_networks' in container_config: + return f'{container_base_cmd} --net host {image}' + + ip_param = '' + networks = ",".join(container_config['network']) + for network in container_config['network']: + if 'address' in container_config['network'][network]: + address = container_config['network'][network]['address'] + ip_param = f'--ip {address}' + + return f'{container_base_cmd} --net {networks} {ip_param} {image}' + def generate(container): # bail out early - looks like removal from running config if not container: @@ -263,6 +312,15 @@ def generate(container): render(config_containers_registry, 'container/registries.conf.j2', container) render(config_containers_storage, 'container/storage.conf.j2', container) + if 'name' in container: + for name, container_config in container['name'].items(): + if 'disable' in container_config: + continue + + file_path = os.path.join(systemd_unit_path, f'vyos-container-{name}.service') + run_args = generate_run_arguments(name, container_config) + render(file_path, 'container/systemd-unit.j2', {'name': name, 'run_args': run_args}) + return None def apply(container): @@ -270,8 +328,12 @@ def apply(container): # Option "--force" allows to delete containers with any status if 'container_remove' in container: for name in container['container_remove']: - call(f'podman stop --time 3 {name}') - call(f'podman rm --force {name}') + file_path = os.path.join(systemd_unit_path, f'vyos-container-{name}.service') + call(f'systemctl stop vyos-container-{name}.service') + if os.path.exists(file_path): + os.unlink(file_path) + + call('systemctl daemon-reload') # Delete old networks if needed if 'network_remove' in container: @@ -282,6 +344,7 @@ def apply(container): os.unlink(tmp) # Add container + disabled_new = False if 'name' in container: for name, container_config in container['name'].items(): image = container_config['image'] @@ -295,70 +358,17 @@ def apply(container): # check if there is a container by that name running tmp = _cmd('podman ps -a --format "{{.Names}}"') if name in tmp: - _cmd(f'podman stop --time 3 {name}') - _cmd(f'podman rm --force {name}') + file_path = os.path.join(systemd_unit_path, f'vyos-container-{name}.service') + call(f'systemctl stop vyos-container-{name}.service') + if os.path.exists(file_path): + disabled_new = True + os.unlink(file_path) continue - memory = container_config['memory'] - restart = container_config['restart'] - - # Add capability options. Should be in uppercase - cap_add = '' - if 'cap_add' in container_config: - for c in container_config['cap_add']: - c = c.upper() - c = c.replace('-', '_') - cap_add += f' --cap-add={c}' - - # Add a host device to the container /dev/x:/dev/x - device = '' - if 'device' in container_config: - for dev, dev_config in container_config['device'].items(): - source_dev = dev_config['source'] - dest_dev = dev_config['destination'] - device += f' --device={source_dev}:{dest_dev}' - - # Check/set environment options "-e foo=bar" - env_opt = '' - if 'environment' in container_config: - for k, v in container_config['environment'].items(): - env_opt += f" -e \"{k}={v['value']}\"" - - # Publish ports - port = '' - if 'port' in container_config: - protocol = '' - for portmap in container_config['port']: - if 'protocol' in container_config['port'][portmap]: - protocol = container_config['port'][portmap]['protocol'] - protocol = f'/{protocol}' - else: - protocol = '/tcp' - sport = container_config['port'][portmap]['source'] - dport = container_config['port'][portmap]['destination'] - port += f' -p {sport}:{dport}{protocol}' - - # Bind volume - volume = '' - if 'volume' in container_config: - for vol, vol_config in container_config['volume'].items(): - svol = vol_config['source'] - dvol = vol_config['destination'] - volume += f' -v {svol}:{dvol}' - - container_base_cmd = f'podman run --detach --interactive --tty --replace {cap_add} ' \ - f'--memory {memory}m --memory-swap 0 --restart {restart} ' \ - f'--name {name} {device} {port} {volume} {env_opt}' - if 'allow_host_networks' in container_config: - _run_rerun(f'{container_base_cmd} --net host {image}') - else: - for network in container_config['network']: - ipparam = '' - if 'address' in container_config['network'][network]: - address = container_config['network'][network]['address'] - ipparam = f'--ip {address}' + cmd(f'systemctl restart vyos-container-{name}.service') - _run_rerun(f'{container_base_cmd} --net {network} {ipparam} {image}') + if disabled_new: + call('systemctl daemon-reload') return None -- cgit v1.2.3 From 1afb3f8bd5de3748c5b37462eb42235d721d4963 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Fri, 28 Oct 2022 13:07:30 +0000 Subject: T4771: Ability to get raw format for op-mode BGP commands --- data/op-mode-standardized.json | 1 + src/op_mode/bgp.py | 120 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100755 src/op_mode/bgp.py (limited to 'data') diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index 9500d3aa7..c5e9f9243 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -1,4 +1,5 @@ [ +"bgp.py", "bridge.py", "conntrack.py", "container.py", diff --git a/src/op_mode/bgp.py b/src/op_mode/bgp.py new file mode 100755 index 000000000..23001a9d7 --- /dev/null +++ b/src/op_mode/bgp.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Purpose: +# Displays bgp neighbors information. +# Used by the "show bgp (vrf ) ipv4|ipv6 neighbors" commands. + +import re +import sys +import typing + +import jmespath +from jinja2 import Template +from humps import decamelize + +from vyos.configquery import ConfigTreeQuery + +import vyos.opmode + + +frr_command_template = Template(""" +{% if family %} + show bgp + {{ 'vrf ' ~ vrf if vrf else '' }} + {{ 'ipv6' if family == 'inet6' else 'ipv4'}} + {{ 'neighbor ' ~ peer if peer else 'summary' }} +{% endif %} + +{% if raw %} + json +{% endif %} +""") + + +def _verify(func): + """Decorator checks if BGP config exists + BGP configuration can be present under vrf + If we do npt get arg 'peer' then it can be 'bgp summary' + """ + from functools import wraps + + @wraps(func) + def _wrapper(*args, **kwargs): + config = ConfigTreeQuery() + afi = 'ipv6' if kwargs.get('family') == 'inet6' else 'ipv4' + global_vrfs = ['all', 'default'] + peer = kwargs.get('peer') + vrf = kwargs.get('vrf') + unconf_message = f'BGP or neighbor is not configured' + # Add option to check the specific neighbor if we have arg 'peer' + peer_opt = f'neighbor {peer} address-family {afi}-unicast' if peer else '' + vrf_opt = '' + if vrf and vrf not in global_vrfs: + vrf_opt = f'vrf name {vrf}' + # Check if config does not exist + if not config.exists(f'{vrf_opt} protocols bgp {peer_opt}'): + raise vyos.opmode.UnconfiguredSubsystem(unconf_message) + return func(*args, **kwargs) + + return _wrapper + + +@_verify +def show_neighbors(raw: bool, + family: str, + peer: typing.Optional[str], + vrf: typing.Optional[str]): + kwargs = dict(locals()) + frr_command = frr_command_template.render(kwargs) + frr_command = re.sub(r'\s+', ' ', frr_command) + + from vyos.util import cmd + output = cmd(f"vtysh -c '{frr_command}'") + + if raw: + from json import loads + data = loads(output) + # Get list of the peers + peers = jmespath.search('*.peers | [0]', data) + if peers: + # Create new dict, delete old key 'peers' + # add key 'peers' neighbors to the list + list_peers = [] + new_dict = jmespath.search('* | [0]', data) + if 'peers' in new_dict: + new_dict.pop('peers') + + for neighbor, neighbor_options in peers.items(): + neighbor_options['neighbor'] = neighbor + list_peers.append(neighbor_options) + new_dict['peers'] = list_peers + return decamelize(new_dict) + data = jmespath.search('* | [0]', data) + return decamelize(data) + + else: + return output + + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) -- cgit v1.2.3 From 22c3dcbb01d731f0dab0ffefa2e5a0be7009baf1 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Mon, 31 Oct 2022 15:09:58 +0100 Subject: ipsec: T4787: add support for road-warrior/remote-access RADIUS timeout This enabled users to also use 2FA/MFA authentication with a radius backend as there is enough time to enter the second factor. --- data/templates/ipsec/charon/eap-radius.conf.j2 | 4 +++- interface-definitions/include/radius-timeout.xml.i | 16 ++++++++++++++++ interface-definitions/vpn-ipsec.xml.in | 1 + interface-definitions/vpn-openconnect.xml.in | 15 +-------------- src/conf_mode/vpn_ipsec.py | 17 +++++++++++++++-- 5 files changed, 36 insertions(+), 17 deletions(-) create mode 100644 interface-definitions/include/radius-timeout.xml.i (limited to 'data') diff --git a/data/templates/ipsec/charon/eap-radius.conf.j2 b/data/templates/ipsec/charon/eap-radius.conf.j2 index 8495011fe..364377473 100644 --- a/data/templates/ipsec/charon/eap-radius.conf.j2 +++ b/data/templates/ipsec/charon/eap-radius.conf.j2 @@ -49,8 +49,10 @@ eap-radius { # Base to use for calculating exponential back off. # retransmit_base = 1.4 +{% if remote_access.radius.timeout is vyos_defined %} # Timeout in seconds before sending first retransmit. - # retransmit_timeout = 2.0 + retransmit_timeout = {{ remote_access.radius.timeout | float }} +{% endif %} # Number of times to retransmit a packet before giving up. # retransmit_tries = 4 diff --git a/interface-definitions/include/radius-timeout.xml.i b/interface-definitions/include/radius-timeout.xml.i new file mode 100644 index 000000000..22bb6d312 --- /dev/null +++ b/interface-definitions/include/radius-timeout.xml.i @@ -0,0 +1,16 @@ + + + + Session timeout + + u32:1-240 + Session timeout in seconds (default: 2) + + + + + Timeout must be between 1 and 240 seconds + + 2 + + diff --git a/interface-definitions/vpn-ipsec.xml.in b/interface-definitions/vpn-ipsec.xml.in index 4776c53dc..64966b540 100644 --- a/interface-definitions/vpn-ipsec.xml.in +++ b/interface-definitions/vpn-ipsec.xml.in @@ -888,6 +888,7 @@ #include + #include #include diff --git a/interface-definitions/vpn-openconnect.xml.in b/interface-definitions/vpn-openconnect.xml.in index 3b3a83bd4..8b60f2e6e 100644 --- a/interface-definitions/vpn-openconnect.xml.in +++ b/interface-definitions/vpn-openconnect.xml.in @@ -140,20 +140,7 @@ #include - - - Session timeout - - u32:1-240 - Session timeout in seconds (default: 2) - - - - - Timeout must be between 1 and 240 seconds - - 2 - + #include If the groupconfig option is set, then config-per-user will be overriden, and all configuration will be read from RADIUS. diff --git a/src/conf_mode/vpn_ipsec.py b/src/conf_mode/vpn_ipsec.py index 77a425f8b..cfefcfbe8 100755 --- a/src/conf_mode/vpn_ipsec.py +++ b/src/conf_mode/vpn_ipsec.py @@ -117,13 +117,26 @@ def get_config(config=None): ipsec['ike_group'][group]['proposal'][proposal] = dict_merge(default_values, ipsec['ike_group'][group]['proposal'][proposal]) - if 'remote_access' in ipsec and 'connection' in ipsec['remote_access']: + # XXX: T2665: we can not safely rely on the defaults() when there are + # tagNodes in place, it is better to blend in the defaults manually. + if dict_search('remote_access.connection', ipsec): default_values = defaults(base + ['remote-access', 'connection']) for rw in ipsec['remote_access']['connection']: ipsec['remote_access']['connection'][rw] = dict_merge(default_values, ipsec['remote_access']['connection'][rw]) - if 'remote_access' in ipsec and 'radius' in ipsec['remote_access'] and 'server' in ipsec['remote_access']['radius']: + # XXX: T2665: we can not safely rely on the defaults() when there are + # tagNodes in place, it is better to blend in the defaults manually. + if dict_search('remote_access.radius.server', ipsec): + # Fist handle the "base" stuff like RADIUS timeout + default_values = defaults(base + ['remote-access', 'radius']) + if 'server' in default_values: + del default_values['server'] + ipsec['remote_access']['radius'] = dict_merge(default_values, + ipsec['remote_access']['radius']) + + # Take care about individual RADIUS servers implemented as tagNodes - this + # requires special treatment default_values = defaults(base + ['remote-access', 'radius', 'server']) for server in ipsec['remote_access']['radius']['server']: ipsec['remote_access']['radius']['server'][server] = dict_merge(default_values, -- cgit v1.2.3 From f489c5ecdab5bdd8a5faa130f4c79a6f4559353b Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Tue, 1 Nov 2022 17:07:24 +0000 Subject: T4777: Ability to get logs in machine-readable format Ability to get logs in JSON format Possible filter by unit. Options for count lines, UTC time, facility or logs since boot --- data/op-mode-standardized.json | 1 + src/op_mode/log.py | 94 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100755 src/op_mode/log.py (limited to 'data') diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index 9500d3aa7..a34c3f481 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -3,6 +3,7 @@ "conntrack.py", "container.py", "cpu.py", +"log.py", "memory.py", "nat.py", "neighbor.py", diff --git a/src/op_mode/log.py b/src/op_mode/log.py new file mode 100755 index 000000000..b0abd6191 --- /dev/null +++ b/src/op_mode/log.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import json +import re +import sys +import typing + +from jinja2 import Template + +from vyos.util import rc_cmd + +import vyos.opmode + +journalctl_command_template = Template(""" +--no-hostname +--quiet + +{% if boot %} + --boot +{% endif %} + +{% if count %} + --lines={{ count }} +{% endif %} + +{% if reverse %} + --reverse +{% endif %} + +{% if since %} + --since={{ since }} +{% endif %} + +{% if unit %} + --unit={{ unit }} +{% endif %} + +{% if utc %} + --utc +{% endif %} + +{% if raw %} +{# By default show 100 only lines for raw option if count does not set #} +{# Protection from parsing the full log by default #} +{% if not boot %} + --lines={{ '' ~ count if count else '100' }} +{% endif %} + --no-pager + --output=json +{% endif %} +""") + + +def show(raw: bool, + boot: typing.Optional[bool], + count: typing.Optional[int], + facility: typing.Optional[str], + reverse: typing.Optional[bool], + utc: typing.Optional[bool], + unit: typing.Optional[str]): + kwargs = dict(locals()) + + journalctl_options = journalctl_command_template.render(kwargs) + journalctl_options = re.sub(r'\s+', ' ', journalctl_options) + rc, output = rc_cmd(f'journalctl {journalctl_options}') + if raw: + # Each 'journalctl --output json' line is a separate JSON object + # So we should return list of dict + return [json.loads(line) for line in output.split('\n')] + return output + + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) -- cgit v1.2.3 From 738641a6c66d22c09b8c028ee3d8a90527d9701f Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Wed, 19 Oct 2022 14:16:05 +0000 Subject: T4758: Rewrite show DHCP(v6) server leases to vyos.opmode format Rewrite op-mode DHCP and DHCPv6 leases to vyos.opmode format Abbility to show 'raw' format show dhcp server leases show dhcpv6 server leases --- data/op-mode-standardized.json | 2 + op-mode-definitions/dhcp.xml.in | 4 +- src/op_mode/dhcp.py | 278 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 282 insertions(+), 2 deletions(-) create mode 100755 src/op_mode/dhcp.py (limited to 'data') diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index 9500d3aa7..03b85b50f 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -3,6 +3,8 @@ "conntrack.py", "container.py", "cpu.py", +"dhcp.py", +"log.py", "memory.py", "nat.py", "neighbor.py", diff --git a/op-mode-definitions/dhcp.xml.in b/op-mode-definitions/dhcp.xml.in index 241cca0ce..ce4026ff4 100644 --- a/op-mode-definitions/dhcp.xml.in +++ b/op-mode-definitions/dhcp.xml.in @@ -16,7 +16,7 @@ Show DHCP server leases - sudo ${vyos_op_scripts_dir}/show_dhcp.py --leases + sudo ${vyos_op_scripts_dir}/dhcp.py show_server_leases --family inet @@ -82,7 +82,7 @@ Show DHCPv6 server leases - sudo ${vyos_op_scripts_dir}/show_dhcpv6.py --leases + sudo ${vyos_op_scripts_dir}/dhcp.py show_server_leases --family inet6 diff --git a/src/op_mode/dhcp.py b/src/op_mode/dhcp.py new file mode 100755 index 000000000..07e9b7d6c --- /dev/null +++ b/src/op_mode/dhcp.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys +from ipaddress import ip_address +import typing + +from datetime import datetime +from sys import exit +from tabulate import tabulate +from isc_dhcp_leases import IscDhcpLeases + +from vyos.base import Warning +from vyos.configquery import ConfigTreeQuery + +from vyos.util import cmd +from vyos.util import dict_search +from vyos.util import is_systemd_service_running + +import vyos.opmode + + +config = ConfigTreeQuery() +pool_key = "shared-networkname" + + +def _in_pool(lease, pool): + if pool_key in lease.sets: + if lease.sets[pool_key] == pool: + return True + return False + + +def _utc_to_local(utc_dt): + return datetime.fromtimestamp((datetime.fromtimestamp(utc_dt) - datetime(1970, 1, 1)).total_seconds()) + + +def _format_hex_string(in_str): + out_str = "" + # if input is divisible by 2, add : every 2 chars + if len(in_str) > 0 and len(in_str) % 2 == 0: + out_str = ':'.join(a+b for a,b in zip(in_str[::2], in_str[1::2])) + else: + out_str = in_str + + return out_str + + +def _find_list_of_dict_index(lst, key='ip', value='') -> int: + """ + Find the index entry of list of dict matching the dict value + Exampe: + % lst = [{'ip': '192.0.2.1'}, {'ip': '192.0.2.2'}] + % _find_list_of_dict_index(lst, key='ip', value='192.0.2.2') + % 1 + """ + idx = next((index for (index, d) in enumerate(lst) if d[key] == value), None) + return idx + + +def _get_raw_server_leases(family, pool=None) -> list: + """ + Get DHCP server leases + :return list + """ + lease_file = '/config/dhcpdv6.leases' if family == 'inet6' else '/config/dhcpd.leases' + data = [] + leases = IscDhcpLeases(lease_file).get() + if pool is not None: + if config.exists(f'service dhcp-server shared-network-name {pool}'): + leases = list(filter(lambda x: _in_pool(x, pool), leases)) + for lease in leases: + data_lease = {} + data_lease['ip'] = lease.ip + data_lease['state'] = lease.binding_state + data_lease['pool'] = lease.sets.get('shared-networkname', '') + data_lease['end'] = lease.end.timestamp() + + if family == 'inet': + data_lease['hardware'] = lease.ethernet + data_lease['start'] = lease.start.timestamp() + data_lease['hostname'] = lease.hostname + + if family == 'inet6': + data_lease['last_communication'] = lease.last_communication.timestamp() + data_lease['iaid_duid'] = _format_hex_string(lease.host_identifier_string) + lease_types_long = {'na': 'non-temporary', 'ta': 'temporary', 'pd': 'prefix delegation'} + data_lease['type'] = lease_types_long[lease.type] + + data_lease['remaining'] = lease.end - datetime.utcnow() + + if data_lease['remaining'].days >= 0: + # substraction gives us a timedelta object which can't be formatted with strftime + # so we use str(), split gets rid of the microseconds + data_lease['remaining'] = str(data_lease["remaining"]).split('.')[0] + else: + data_lease['remaining'] = '' + + # Do not add old leases + if data_lease['remaining'] != '': + data.append(data_lease) + + # deduplicate + checked = [] + for entry in data: + addr = entry.get('ip') + if addr not in checked: + checked.append(addr) + else: + idx = _find_list_of_dict_index(data, key='ip', value=addr) + data.pop(idx) + + return data + + +def _get_formatted_server_leases(raw_data, family): + data_entries = [] + if family == 'inet': + for lease in raw_data: + ipaddr = lease.get('ip') + hw_addr = lease.get('hardware') + state = lease.get('state') + start = lease.get('start') + start = _utc_to_local(start).strftime('%Y/%m/%d %H:%M:%S') + end = lease.get('end') + end = _utc_to_local(end).strftime('%Y/%m/%d %H:%M:%S') + remain = lease.get('remaining') + pool = lease.get('pool') + hostname = lease.get('hostname') + data_entries.append([ipaddr, hw_addr, state, start, end, remain, pool, hostname]) + + headers = ['IP Address', 'Hardware address', 'State', 'Lease start', 'Lease expiration', 'Remaining', 'Pool', + 'Hostname'] + + if family == 'inet6': + for lease in raw_data: + ipaddr = lease.get('ip') + state = lease.get('state') + start = lease.get('last_communication') + start = _utc_to_local(start).strftime('%Y/%m/%d %H:%M:%S') + end = lease.get('end') + end = _utc_to_local(end).strftime('%Y/%m/%d %H:%M:%S') + remain = lease.get('remaining') + lease_type = lease.get('type') + pool = lease.get('pool') + host_identifier = lease.get('iaid_duid') + data_entries.append([ipaddr, state, start, end, remain, lease_type, pool, host_identifier]) + + headers = ['IPv6 address', 'State', 'Last communication', 'Lease expiration', 'Remaining', 'Type', 'Pool', + 'IAID_DUID'] + + output = tabulate(data_entries, headers, numalign='left') + return output + + +def _get_dhcp_pools(family='inet') -> list: + v = 'v6' if family == 'inet6' else '' + pools = config.list_nodes(f'service dhcp{v}-server shared-network-name') + return pools + + +def _get_pool_size(pool, family='inet'): + v = 'v6' if family == 'inet6' else '' + base = f'service dhcp{v}-server shared-network-name {pool}' + size = 0 + subnets = config.list_nodes(f'{base} subnet') + for subnet in subnets: + if family == 'inet6': + ranges = config.list_nodes(f'{base} subnet {subnet} address-range start') + else: + ranges = config.list_nodes(f'{base} subnet {subnet} range') + for range in ranges: + if family == 'inet6': + start = config.list_nodes(f'{base} subnet {subnet} address-range start')[0] + stop = config.value(f'{base} subnet {subnet} address-range start {start} stop') + else: + start = config.value(f'{base} subnet {subnet} range {range} start') + stop = config.value(f'{base} subnet {subnet} range {range} stop') + # Add +1 because both range boundaries are inclusive + size += int(ip_address(stop)) - int(ip_address(start)) + 1 + return size + + +def _get_raw_pool_statistics(family='inet', pool=None): + if pool is None: + pool = _get_dhcp_pools(family=family) + else: + pool = [pool] + + v = 'v6' if family == 'inet6' else '' + stats = [] + for p in pool: + subnet = config.list_nodes(f'service dhcp{v}-server shared-network-name {p} subnet') + size = _get_pool_size(family=family, pool=p) + leases = len(_get_raw_server_leases(family=family, pool=p)) + use_percentage = round(leases / size * 100) if size != 0 else 0 + pool_stats = {'pool': p, 'size': size, 'leases': leases, + 'available': (size - leases), 'use_percentage': use_percentage, 'subnet': subnet} + stats.append(pool_stats) + return stats + + +def _get_formatted_pool_statistics(pool_data, family='inet'): + data_entries = [] + for entry in pool_data: + pool = entry.get('pool') + size = entry.get('size') + leases = entry.get('leases') + available = entry.get('available') + use_percentage = entry.get('use_percentage') + use_percentage = f'{use_percentage}%' + data_entries.append([pool, size, leases, available, use_percentage]) + + headers = ['Pool', 'Size','Leases', 'Available', 'Usage'] + output = tabulate(data_entries, headers, numalign='left') + return output + + +def _verify(func): + """Decorator checks if DHCP(v6) config exists""" + from functools import wraps + + @wraps(func) + def _wrapper(*args, **kwargs): + config = ConfigTreeQuery() + family = kwargs.get('family') + v = 'v6' if family == 'inet6' else '' + unconf_message = f'DHCP{v} server is not configured' + # Check if config does not exist + if not config.exists(f'service dhcp{v}-server'): + raise vyos.opmode.UnconfiguredSubsystem(unconf_message) + return func(*args, **kwargs) + return _wrapper + + +@_verify +def show_pool_statistics(raw: bool, family: str, pool: typing.Optional[str]): + pool_data = _get_raw_pool_statistics(family=family, pool=pool) + if raw: + return pool_data + else: + return _get_formatted_pool_statistics(pool_data, family=family) + + +@_verify +def show_server_leases(raw: bool, family: str): + # if dhcp server is down, inactive leases may still be shown as active, so warn the user. + if not is_systemd_service_running('isc-dhcp-server.service'): + Warning('DHCP server is configured but not started. Data may be stale.') + + leases = _get_raw_server_leases(family) + if raw: + return leases + else: + return _get_formatted_server_leases(leases, family) + + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) -- cgit v1.2.3 From 051e063fdf2e459a0716a35778b33ea6bb2fdcb6 Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Mon, 31 Oct 2022 14:26:51 +0100 Subject: firewall: T970: Refactor domain resolver, add firewall source/destination `fqdn` node --- data/templates/firewall/nftables-defines.j2 | 8 + data/templates/firewall/nftables.j2 | 14 +- interface-definitions/firewall.xml.in | 25 ++- interface-definitions/include/firewall/fqdn.xml.i | 14 ++ .../firewall/source-destination-group-ipv6.xml.i | 8 + python/vyos/firewall.py | 90 ++++------ smoketest/scripts/cli/test_firewall.py | 16 ++ src/conf_mode/firewall.py | 60 +++---- src/helpers/vyos-domain-group-resolve.py | 60 ------- src/helpers/vyos-domain-resolver.py | 182 +++++++++++++++++++++ src/systemd/vyos-domain-group-resolve.service | 11 -- src/systemd/vyos-domain-resolver.service | 13 ++ 12 files changed, 328 insertions(+), 173 deletions(-) create mode 100644 interface-definitions/include/firewall/fqdn.xml.i delete mode 100755 src/helpers/vyos-domain-group-resolve.py create mode 100755 src/helpers/vyos-domain-resolver.py delete mode 100644 src/systemd/vyos-domain-group-resolve.service create mode 100644 src/systemd/vyos-domain-resolver.service (limited to 'data') diff --git a/data/templates/firewall/nftables-defines.j2 b/data/templates/firewall/nftables-defines.j2 index 5336f7ee6..dd06dee28 100644 --- a/data/templates/firewall/nftables-defines.j2 +++ b/data/templates/firewall/nftables-defines.j2 @@ -27,6 +27,14 @@ } {% endfor %} {% endif %} +{% if group.domain_group is vyos_defined %} +{% for name, name_config in group.domain_group.items() %} + set D_{{ name }} { + type {{ ip_type }} + flags interval + } +{% endfor %} +{% endif %} {% if group.mac_group is vyos_defined %} {% for group_name, group_conf in group.mac_group.items() %} {% set includes = group_conf.include if group_conf.include is vyos_defined else [] %} diff --git a/data/templates/firewall/nftables.j2 b/data/templates/firewall/nftables.j2 index a0f0b8c11..2c7115134 100644 --- a/data/templates/firewall/nftables.j2 +++ b/data/templates/firewall/nftables.j2 @@ -67,14 +67,12 @@ table ip vyos_filter { {{ conf | nft_default_rule(name_text) }} } {% endfor %} -{% if group is vyos_defined and group.domain_group is vyos_defined %} -{% for name, name_config in group.domain_group.items() %} - set D_{{ name }} { +{% for set_name in ip_fqdn %} + set FQDN_{{ set_name }} { type ipv4_addr flags interval } -{% endfor %} -{% endif %} +{% endfor %} {% for set_name in ns.sets %} set RECENT_{{ set_name }} { type ipv4_addr @@ -178,6 +176,12 @@ table ip6 vyos_filter { {{ conf | nft_default_rule(name_text, ipv6=True) }} } {% endfor %} +{% for set_name in ip6_fqdn %} + set FQDN_{{ set_name }} { + type ipv6_addr + flags interval + } +{% endfor %} {% for set_name in ns.sets %} set RECENT6_{{ set_name }} { type ipv6_addr diff --git a/interface-definitions/firewall.xml.in b/interface-definitions/firewall.xml.in index 673461036..2d8f17351 100644 --- a/interface-definitions/firewall.xml.in +++ b/interface-definitions/firewall.xml.in @@ -126,7 +126,7 @@ Domain address to match - [a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,99}?(\/.*)? + @@ -408,6 +408,7 @@ #include + #include #include #include #include @@ -419,6 +420,7 @@ #include + #include #include #include #include @@ -572,6 +574,7 @@ #include + #include #include #include #include @@ -583,6 +586,7 @@ #include + #include #include #include #include @@ -656,6 +660,25 @@ disable + + + Retains last successful value if domain resolution fails + + + + + + Domain resolver update interval + + u32:10-3600 + Interval (seconds) + + + + + + 300 + Policy for sending IPv4 ICMP redirect messages diff --git a/interface-definitions/include/firewall/fqdn.xml.i b/interface-definitions/include/firewall/fqdn.xml.i new file mode 100644 index 000000000..9eb3925b5 --- /dev/null +++ b/interface-definitions/include/firewall/fqdn.xml.i @@ -0,0 +1,14 @@ + + + + Fully qualified domain name + + <fqdn> + Fully qualified domain name + + + + + + + diff --git a/interface-definitions/include/firewall/source-destination-group-ipv6.xml.i b/interface-definitions/include/firewall/source-destination-group-ipv6.xml.i index c2cc7edb3..2a42d236c 100644 --- a/interface-definitions/include/firewall/source-destination-group-ipv6.xml.i +++ b/interface-definitions/include/firewall/source-destination-group-ipv6.xml.i @@ -12,6 +12,14 @@ + + + Group of domains + + firewall group domain-group + + + #include diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index 4075e55b0..db4878c9d 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -20,6 +20,9 @@ import os import re from pathlib import Path +from socket import AF_INET +from socket import AF_INET6 +from socket import getaddrinfo from time import strftime from vyos.remote import download @@ -31,65 +34,29 @@ from vyos.util import dict_search_args from vyos.util import dict_search_recursive from vyos.util import run +def fqdn_config_parse(firewall): + firewall['ip_fqdn'] = {} + firewall['ip6_fqdn'] = {} + + for domain, path in dict_search_recursive(firewall, 'fqdn'): + fw_name = path[1] # name/ipv6-name + rule = path[3] # rule id + suffix = path[4][0] # source/destination (1 char) + set_name = f'{fw_name}_{rule}_{suffix}' + + if path[0] == 'name': + firewall['ip_fqdn'][set_name] = domain + elif path[0] == 'ipv6_name': + firewall['ip6_fqdn'][set_name] = domain + +def fqdn_resolve(fqdn, ipv6=False): + try: + res = getaddrinfo(fqdn, None, AF_INET6 if ipv6 else AF_INET) + return set(item[4][0] for item in res) + except: + return None -# Functions for firewall group domain-groups -def get_ips_domains_dict(list_domains): - """ - Get list of IPv4 addresses by list of domains - Ex: get_ips_domains_dict(['ex1.com', 'ex2.com']) - {'ex1.com': ['192.0.2.1'], 'ex2.com': ['192.0.2.2', '192.0.2.3']} - """ - from socket import gethostbyname_ex - from socket import gaierror - - ip_dict = {} - for domain in list_domains: - try: - _, _, ips = gethostbyname_ex(domain) - ip_dict[domain] = ips - except gaierror: - pass - - return ip_dict - -def nft_init_set(group_name, table="vyos_filter", family="ip"): - """ - table ip vyos_filter { - set GROUP_NAME - type ipv4_addr - flags interval - } - """ - return call(f'nft add set ip {table} {group_name} {{ type ipv4_addr\\; flags interval\\; }}') - - -def nft_add_set_elements(group_name, elements, table="vyos_filter", family="ip"): - """ - table ip vyos_filter { - set GROUP_NAME { - type ipv4_addr - flags interval - elements = { 192.0.2.1, 192.0.2.2 } - } - """ - elements = ", ".join(elements) - return call(f'nft add element {family} {table} {group_name} {{ {elements} }} ') - -def nft_flush_set(group_name, table="vyos_filter", family="ip"): - """ - Flush elements of nft set - """ - return call(f'nft flush set {family} {table} {group_name}') - -def nft_update_set_elements(group_name, elements, table="vyos_filter", family="ip"): - """ - Update elements of nft set - """ - flush_set = nft_flush_set(group_name, table="vyos_filter", family="ip") - nft_add_set = nft_add_set_elements(group_name, elements, table="vyos_filter", family="ip") - return flush_set, nft_add_set - -# END firewall group domain-group (sets) +# End Domain Resolver def find_nftables_rule(table, chain, rule_matches=[]): # Find rule in table/chain that matches all criteria and return the handle @@ -151,6 +118,13 @@ def parse_rule(rule_conf, fw_name, rule_id, ip_name): suffix = f'!= {suffix[1:]}' output.append(f'{ip_name} {prefix}addr {suffix}') + if 'fqdn' in side_conf: + fqdn = side_conf['fqdn'] + operator = '' + if fqdn[0] == '!': + operator = '!=' + output.append(f'{ip_name} {prefix}addr {operator} @FQDN_{fw_name}_{rule_id}_{prefix}') + if dict_search_args(side_conf, 'geoip', 'country_code'): operator = '' if dict_search_args(side_conf, 'geoip', 'inverse_match') != None: diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py index 821925bcd..e172e086d 100755 --- a/smoketest/scripts/cli/test_firewall.py +++ b/smoketest/scripts/cli/test_firewall.py @@ -17,11 +17,13 @@ import unittest from glob import glob +from time import sleep from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError from vyos.util import cmd +from vyos.util import run sysfs_config = { 'all_ping': {'sysfs': '/proc/sys/net/ipv4/icmp_echo_ignore_all', 'default': '0', 'test_value': 'disable'}, @@ -76,6 +78,17 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase): break self.assertTrue(not matched if inverse else matched, msg=search) + def wait_for_domain_resolver(self, table, set_name, element, max_wait=10): + # Resolver no longer blocks commit, need to wait for daemon to populate set + count = 0 + while count < max_wait: + code = run(f'sudo nft get element {table} {set_name} {{ {element} }}') + if code == 0: + return True + count += 1 + sleep(1) + return False + def test_geoip(self): self.cli_set(['firewall', 'name', 'smoketest', 'rule', '1', 'action', 'drop']) self.cli_set(['firewall', 'name', 'smoketest', 'rule', '1', 'source', 'geoip', 'country-code', 'se']) @@ -125,6 +138,9 @@ class TestFirewall(VyOSUnitTestSHIM.TestCase): self.cli_set(['firewall', 'interface', 'eth0', 'in', 'name', 'smoketest']) self.cli_commit() + + self.wait_for_domain_resolver('ip vyos_filter', 'D_smoketest_domain', '192.0.2.5') + nftables_search = [ ['iifname "eth0"', 'jump NAME_smoketest'], ['ip saddr @N_smoketest_network', 'ip daddr 172.16.10.10', 'th dport @P_smoketest_port', 'return'], diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index cbd9cbe90..2bb765e65 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -27,12 +27,8 @@ from vyos.configdict import dict_merge from vyos.configdict import node_changed from vyos.configdiff import get_config_diff, Diff # from vyos.configverify import verify_interface_exists +from vyos.firewall import fqdn_config_parse from vyos.firewall import geoip_update -from vyos.firewall import get_ips_domains_dict -from vyos.firewall import nft_add_set_elements -from vyos.firewall import nft_flush_set -from vyos.firewall import nft_init_set -from vyos.firewall import nft_update_set_elements from vyos.template import render from vyos.util import call from vyos.util import cmd @@ -173,6 +169,8 @@ def get_config(config=None): firewall['geoip_updated'] = geoip_updated(conf, firewall) + fqdn_config_parse(firewall) + return firewall def verify_rule(firewall, rule_conf, ipv6): @@ -232,29 +230,28 @@ def verify_rule(firewall, rule_conf, ipv6): if side in rule_conf: side_conf = rule_conf[side] - if dict_search_args(side_conf, 'geoip', 'country_code'): - if 'address' in side_conf: - raise ConfigError('Address and GeoIP cannot both be defined') - - if dict_search_args(side_conf, 'group', 'address_group'): - raise ConfigError('Address-group and GeoIP cannot both be defined') - - if dict_search_args(side_conf, 'group', 'network_group'): - raise ConfigError('Network-group and GeoIP cannot both be defined') + if len({'address', 'fqdn', 'geoip'} & set(side_conf)) > 1: + raise ConfigError('Only one of address, fqdn or geoip can be specified') if 'group' in side_conf: - if {'address_group', 'network_group'} <= set(side_conf['group']): - raise ConfigError('Only one address-group or network-group can be specified') + if len({'address_group', 'network_group', 'domain_group'} & set(side_conf['group'])) > 1: + raise ConfigError('Only one address-group, network-group or domain-group can be specified') for group in valid_groups: if group in side_conf['group']: group_name = side_conf['group'][group] + fw_group = f'ipv6_{group}' if ipv6 and group in ['address_group', 'network_group'] else group + error_group = fw_group.replace("_", "-") + + if group in ['address_group', 'network_group', 'domain_group']: + types = [t for t in ['address', 'fqdn', 'geoip'] if t in side_conf] + if types: + raise ConfigError(f'{error_group} and {types[0]} cannot both be defined') + if group_name and group_name[0] == '!': group_name = group_name[1:] - fw_group = f'ipv6_{group}' if ipv6 and group in ['address_group', 'network_group'] else group - error_group = fw_group.replace("_", "-") group_obj = dict_search_args(firewall, 'group', fw_group, group_name) if group_obj is None: @@ -477,26 +474,13 @@ def apply(firewall): if install_result == 1: raise ConfigError(f'Failed to apply firewall: {output}') - # set firewall group domain-group xxx - if 'group' in firewall: - if 'domain_group' in firewall['group']: - # T970 Enable a resolver (systemd daemon) that checks - # domain-group addresses and update entries for domains by timeout - # If router loaded without internet connection or for synchronization - call('systemctl restart vyos-domain-group-resolve.service') - for group, group_config in firewall['group']['domain_group'].items(): - domains = [] - if group_config.get('address') is not None: - for address in group_config.get('address'): - domains.append(address) - # Add elements to domain-group, try to resolve domain => ip - # and add elements to nft set - ip_dict = get_ips_domains_dict(domains) - elements = sum(ip_dict.values(), []) - nft_init_set(f'D_{group}') - nft_add_set_elements(f'D_{group}', elements) - else: - call('systemctl stop vyos-domain-group-resolve.service') + # T970 Enable a resolver (systemd daemon) that checks + # domain-group addresses and update entries for domains by timeout + # If router loaded without internet connection or for synchronization + domain_action = 'stop' + if dict_search_args(firewall, 'group', 'domain_group') or firewall['ip_fqdn'] or firewall['ip6_fqdn']: + domain_action = 'restart' + call(f'systemctl {domain_action} vyos-domain-resolver.service') apply_sysfs(firewall) diff --git a/src/helpers/vyos-domain-group-resolve.py b/src/helpers/vyos-domain-group-resolve.py deleted file mode 100755 index 6b677670b..000000000 --- a/src/helpers/vyos-domain-group-resolve.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2022 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -import time - -from vyos.configquery import ConfigTreeQuery -from vyos.firewall import get_ips_domains_dict -from vyos.firewall import nft_add_set_elements -from vyos.firewall import nft_flush_set -from vyos.firewall import nft_init_set -from vyos.firewall import nft_update_set_elements -from vyos.util import call - - -base = ['firewall', 'group', 'domain-group'] -check_required = True -# count_failed = 0 -# Timeout in sec between checks -timeout = 300 - -domain_state = {} - -if __name__ == '__main__': - - while check_required: - config = ConfigTreeQuery() - if config.exists(base): - domain_groups = config.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - for set_name, domain_config in domain_groups.items(): - list_domains = domain_config['address'] - elements = [] - ip_dict = get_ips_domains_dict(list_domains) - - for domain in list_domains: - # Resolution succeeded, update domain state - if domain in ip_dict: - domain_state[domain] = ip_dict[domain] - elements += ip_dict[domain] - # Resolution failed, use previous domain state - elif domain in domain_state: - elements += domain_state[domain] - - # Resolve successful - if elements: - nft_update_set_elements(f'D_{set_name}', elements) - time.sleep(timeout) diff --git a/src/helpers/vyos-domain-resolver.py b/src/helpers/vyos-domain-resolver.py new file mode 100755 index 000000000..2f71f15db --- /dev/null +++ b/src/helpers/vyos-domain-resolver.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import json +import os +import time + +from vyos.configdict import dict_merge +from vyos.configquery import ConfigTreeQuery +from vyos.firewall import fqdn_config_parse +from vyos.firewall import fqdn_resolve +from vyos.util import cmd +from vyos.util import commit_in_progress +from vyos.util import dict_search_args +from vyos.util import run +from vyos.xml import defaults + +base = ['firewall'] +timeout = 300 +cache = False + +domain_state = {} + +ipv4_tables = { + 'ip mangle', + 'ip vyos_filter', +} + +ipv6_tables = { + 'ip6 mangle', + 'ip6 vyos_filter' +} + +def get_config(conf): + firewall = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True, + no_tag_node_value_mangle=True) + + default_values = defaults(base) + for tmp in ['name', 'ipv6_name']: + if tmp in default_values: + del default_values[tmp] + + if 'zone' in default_values: + del default_values['zone'] + + firewall = dict_merge(default_values, firewall) + + global timeout, cache + + if 'resolver_interval' in firewall: + timeout = int(firewall['resolver_interval']) + + if 'resolver_cache' in firewall: + cache = True + + fqdn_config_parse(firewall) + + return firewall + +def resolve(domains, ipv6=False): + global domain_state + + ip_list = set() + + for domain in domains: + resolved = fqdn_resolve(domain, ipv6=ipv6) + + if resolved and cache: + domain_state[domain] = resolved + elif not resolved: + if domain not in domain_state: + continue + resolved = domain_state[domain] + + ip_list = ip_list | resolved + return ip_list + +def nft_output(table, set_name, ip_list): + output = [f'flush set {table} {set_name}'] + if ip_list: + ip_str = ','.join(ip_list) + output.append(f'add element {table} {set_name} {{ {ip_str} }}') + return output + +def nft_valid_sets(): + try: + valid_sets = [] + sets_json = cmd('nft -j list sets') + sets_obj = json.loads(sets_json) + + for obj in sets_obj['nftables']: + if 'set' in obj: + family = obj['set']['family'] + table = obj['set']['table'] + name = obj['set']['name'] + valid_sets.append((f'{family} {table}', name)) + + return valid_sets + except: + return [] + +def update(firewall): + conf_lines = [] + count = 0 + + valid_sets = nft_valid_sets() + + domain_groups = dict_search_args(firewall, 'group', 'domain_group') + if domain_groups: + for set_name, domain_config in domain_groups.items(): + if 'address' not in domain_config: + continue + + nft_set_name = f'D_{set_name}' + domains = domain_config['address'] + + ip_list = resolve(domains, ipv6=False) + for table in ipv4_tables: + if (table, nft_set_name) in valid_sets: + conf_lines += nft_output(table, nft_set_name, ip_list) + + ip6_list = resolve(domains, ipv6=True) + for table in ipv6_tables: + if (table, nft_set_name) in valid_sets: + conf_lines += nft_output(table, nft_set_name, ip6_list) + count += 1 + + for set_name, domain in firewall['ip_fqdn'].items(): + table = 'ip vyos_filter' + nft_set_name = f'FQDN_{set_name}' + + ip_list = resolve([domain], ipv6=False) + + if (table, nft_set_name) in valid_sets: + conf_lines += nft_output(table, nft_set_name, ip_list) + count += 1 + + for set_name, domain in firewall['ip6_fqdn'].items(): + table = 'ip6 vyos_filter' + nft_set_name = f'FQDN_{set_name}' + + ip_list = resolve([domain], ipv6=True) + if (table, nft_set_name) in valid_sets: + conf_lines += nft_output(table, nft_set_name, ip_list) + count += 1 + + nft_conf_str = "\n".join(conf_lines) + "\n" + code = run(f'nft -f -', input=nft_conf_str) + + print(f'Updated {count} sets - result: {code}') + +if __name__ == '__main__': + print(f'VyOS domain resolver') + + count = 1 + while commit_in_progress(): + if ( count % 60 == 0 ): + print(f'Commit still in progress after {count}s - waiting') + count += 1 + time.sleep(1) + + conf = ConfigTreeQuery() + firewall = get_config(conf) + + print(f'interval: {timeout}s - cache: {cache}') + + while True: + update(firewall) + time.sleep(timeout) diff --git a/src/systemd/vyos-domain-group-resolve.service b/src/systemd/vyos-domain-group-resolve.service deleted file mode 100644 index 29628fddb..000000000 --- a/src/systemd/vyos-domain-group-resolve.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=VyOS firewall domain-group resolver -After=vyos-router.service - -[Service] -Type=simple -Restart=always -ExecStart=/usr/bin/python3 /usr/libexec/vyos/vyos-domain-group-resolve.py - -[Install] -WantedBy=multi-user.target diff --git a/src/systemd/vyos-domain-resolver.service b/src/systemd/vyos-domain-resolver.service new file mode 100644 index 000000000..c56b51f0c --- /dev/null +++ b/src/systemd/vyos-domain-resolver.service @@ -0,0 +1,13 @@ +[Unit] +Description=VyOS firewall domain resolver +After=vyos-router.service + +[Service] +Type=simple +Restart=always +ExecStart=/usr/bin/python3 -u /usr/libexec/vyos/vyos-domain-resolver.py +StandardError=journal +StandardOutput=journal + +[Install] +WantedBy=multi-user.target -- cgit v1.2.3 From b4b491d424fba6f3d417135adc1865e338a480a1 Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Mon, 31 Oct 2022 21:08:42 +0100 Subject: nat: T1877: T970: Add firewall groups to NAT --- data/templates/firewall/nftables-nat.j2 | 4 ++ interface-definitions/include/nat-rule.xml.i | 2 + python/vyos/firewall.py | 2 + python/vyos/nat.py | 56 +++++++++++++++++++-- smoketest/scripts/cli/test_nat.py | 35 +++++++++++++ src/conf_mode/firewall.py | 22 ++++++--- src/conf_mode/nat.py | 73 +++++++++++++++++++++++----- src/helpers/vyos-domain-resolver.py | 1 + 8 files changed, 174 insertions(+), 21 deletions(-) (limited to 'data') diff --git a/data/templates/firewall/nftables-nat.j2 b/data/templates/firewall/nftables-nat.j2 index c5c0a2c86..f0be3cf5d 100644 --- a/data/templates/firewall/nftables-nat.j2 +++ b/data/templates/firewall/nftables-nat.j2 @@ -1,5 +1,7 @@ #!/usr/sbin/nft -f +{% import 'firewall/nftables-defines.j2' as group_tmpl %} + {% if helper_functions is vyos_defined('remove') %} {# NAT if going to be disabled - remove rules and targets from nftables #} {% set base_command = 'delete rule ip raw' %} @@ -59,5 +61,7 @@ table ip vyos_nat { chain VYOS_PRE_SNAT_HOOK { return } + +{{ group_tmpl.groups(firewall_group, False) }} } {% endif %} diff --git a/interface-definitions/include/nat-rule.xml.i b/interface-definitions/include/nat-rule.xml.i index 84941aa6a..8f2029388 100644 --- a/interface-definitions/include/nat-rule.xml.i +++ b/interface-definitions/include/nat-rule.xml.i @@ -20,6 +20,7 @@ #include #include + #include #include @@ -285,6 +286,7 @@ #include #include + #include diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index db4878c9d..59ec4948f 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -34,6 +34,8 @@ from vyos.util import dict_search_args from vyos.util import dict_search_recursive from vyos.util import run +# Domain Resolver + def fqdn_config_parse(firewall): firewall['ip_fqdn'] = {} firewall['ip6_fqdn'] = {} diff --git a/python/vyos/nat.py b/python/vyos/nat.py index 31bbdc386..3d01829a7 100644 --- a/python/vyos/nat.py +++ b/python/vyos/nat.py @@ -85,8 +85,13 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): translation_str += f' {",".join(options)}' for target in ['source', 'destination']: + if target not in rule_conf: + continue + + side_conf = rule_conf[target] prefix = target[:1] - addr = dict_search_args(rule_conf, target, 'address') + + addr = dict_search_args(side_conf, 'address') if addr and not (ignore_type_addr and target == nat_type): operator = '' if addr[:1] == '!': @@ -94,7 +99,7 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): addr = addr[1:] output.append(f'{ip_prefix} {prefix}addr {operator} {addr}') - addr_prefix = dict_search_args(rule_conf, target, 'prefix') + addr_prefix = dict_search_args(side_conf, 'prefix') if addr_prefix and ipv6: operator = '' if addr_prefix[:1] == '!': @@ -102,7 +107,7 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): addr_prefix = addr[1:] output.append(f'ip6 {prefix}addr {operator} {addr_prefix}') - port = dict_search_args(rule_conf, target, 'port') + port = dict_search_args(side_conf, 'port') if port: protocol = rule_conf['protocol'] if protocol == 'tcp_udp': @@ -113,6 +118,51 @@ def parse_nat_rule(rule_conf, rule_id, nat_type, ipv6=False): port = port[1:] output.append(f'{protocol} {prefix}port {operator} {{ {port} }}') + if 'group' in side_conf: + group = side_conf['group'] + if 'address_group' in group and not (ignore_type_addr and target == nat_type): + group_name = group['address_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_prefix} {prefix}addr {operator} @A_{group_name}') + # Generate firewall group domain-group + elif 'domain_group' in group and not (ignore_type_addr and target == nat_type): + group_name = group['domain_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_prefix} {prefix}addr {operator} @D_{group_name}') + elif 'network_group' in group and not (ignore_type_addr and target == nat_type): + group_name = group['network_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'{ip_prefix} {prefix}addr {operator} @N_{group_name}') + if 'mac_group' in group: + group_name = group['mac_group'] + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + output.append(f'ether {prefix}addr {operator} @M_{group_name}') + if 'port_group' in group: + proto = rule_conf['protocol'] + group_name = group['port_group'] + + if proto == 'tcp_udp': + proto = 'th' + + operator = '' + if group_name[0] == '!': + operator = '!=' + group_name = group_name[1:] + + output.append(f'{proto} {prefix}port {operator} @P_{group_name}') + output.append('counter') if 'log' in rule_conf: diff --git a/smoketest/scripts/cli/test_nat.py b/smoketest/scripts/cli/test_nat.py index 2ae90fcaf..9f4e3b831 100755 --- a/smoketest/scripts/cli/test_nat.py +++ b/smoketest/scripts/cli/test_nat.py @@ -58,6 +58,17 @@ class TestNAT(VyOSUnitTestSHIM.TestCase): break self.assertTrue(not matched if inverse else matched, msg=search) + def wait_for_domain_resolver(self, table, set_name, element, max_wait=10): + # Resolver no longer blocks commit, need to wait for daemon to populate set + count = 0 + while count < max_wait: + code = run(f'sudo nft get element {table} {set_name} {{ {element} }}') + if code == 0: + return True + count += 1 + sleep(1) + return False + def test_snat(self): rules = ['100', '110', '120', '130', '200', '210', '220', '230'] outbound_iface_100 = 'eth0' @@ -84,6 +95,30 @@ class TestNAT(VyOSUnitTestSHIM.TestCase): self.verify_nftables(nftables_search, 'ip vyos_nat') + def test_snat_groups(self): + address_group = 'smoketest_addr' + address_group_member = '192.0.2.1' + rule = '100' + outbound_iface = 'eth0' + + self.cli_set(['firewall', 'group', 'address-group', address_group, 'address', address_group_member]) + + self.cli_set(src_path + ['rule', rule, 'source', 'group', 'address-group', address_group]) + self.cli_set(src_path + ['rule', rule, 'outbound-interface', outbound_iface]) + self.cli_set(src_path + ['rule', rule, 'translation', 'address', 'masquerade']) + + self.cli_commit() + + nftables_search = [ + [f'set A_{address_group}'], + [f'elements = {{ {address_group_member} }}'], + [f'ip saddr @A_{address_group}', f'oifname "{outbound_iface}"', 'masquerade'] + ] + + self.verify_nftables(nftables_search, 'ip vyos_nat') + + self.cli_delete(['firewall']) + def test_dnat(self): rules = ['100', '110', '120', '130', '200', '210', '220', '230'] inbound_iface_100 = 'eth0' diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index 2bb765e65..783adec46 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -41,6 +41,7 @@ from vyos import ConfigError from vyos import airbag airbag.enable() +nat_conf_script = '/usr/libexec/vyos/conf_mode/nat.py' policy_route_conf_script = '/usr/libexec/vyos/conf_mode/policy-route.py' nftables_conf = '/run/nftables.conf' @@ -158,7 +159,7 @@ def get_config(config=None): for zone in firewall['zone']: firewall['zone'][zone] = dict_merge(default_values, firewall['zone'][zone]) - firewall['policy_resync'] = bool('group' in firewall or node_changed(conf, base + ['group'])) + firewall['group_resync'] = bool('group' in firewall or node_changed(conf, base + ['group'])) if 'config_trap' in firewall and firewall['config_trap'] == 'enable': diff = get_config_diff(conf) @@ -463,6 +464,12 @@ def post_apply_trap(firewall): cmd(base_cmd + ' '.join(objects)) +def resync_nat(): + # Update nat as firewall groups were updated + tmp, out = rc_cmd(nat_conf_script) + if tmp > 0: + Warning(f'Failed to re-apply nat configuration! {out}') + def resync_policy_route(): # Update policy route as firewall groups were updated tmp, out = rc_cmd(policy_route_conf_script) @@ -474,19 +481,20 @@ def apply(firewall): if install_result == 1: raise ConfigError(f'Failed to apply firewall: {output}') + apply_sysfs(firewall) + + if firewall['group_resync']: + resync_nat() + resync_policy_route() + # T970 Enable a resolver (systemd daemon) that checks - # domain-group addresses and update entries for domains by timeout + # domain-group/fqdn addresses and update entries for domains by timeout # If router loaded without internet connection or for synchronization domain_action = 'stop' if dict_search_args(firewall, 'group', 'domain_group') or firewall['ip_fqdn'] or firewall['ip6_fqdn']: domain_action = 'restart' call(f'systemctl {domain_action} vyos-domain-resolver.service') - apply_sysfs(firewall) - - if firewall['policy_resync']: - resync_policy_route() - if firewall['geoip_updated']: # Call helper script to Update set contents if 'name' in firewall['geoip_updated'] or 'ipv6_name' in firewall['geoip_updated']: diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py index 978c043e9..9f8221514 100755 --- a/src/conf_mode/nat.py +++ b/src/conf_mode/nat.py @@ -32,6 +32,7 @@ from vyos.util import cmd from vyos.util import run from vyos.util import check_kmod from vyos.util import dict_search +from vyos.util import dict_search_args from vyos.validate import is_addr_assigned from vyos.xml import defaults from vyos import ConfigError @@ -47,6 +48,13 @@ else: nftables_nat_config = '/run/nftables_nat.conf' nftables_static_nat_conf = '/run/nftables_static-nat-rules.nft' +valid_groups = [ + 'address_group', + 'domain_group', + 'network_group', + 'port_group' +] + def get_handler(json, chain, target): """ Get nftable rule handler number of given chain/target combination. Handler is required when adding NAT/Conntrack helper targets """ @@ -60,7 +68,7 @@ def get_handler(json, chain, target): return None -def verify_rule(config, err_msg): +def verify_rule(config, err_msg, groups_dict): """ Common verify steps used for both source and destination NAT """ if (dict_search('translation.port', config) != None or @@ -78,6 +86,45 @@ def verify_rule(config, err_msg): 'statically maps a whole network of addresses onto another\n' \ 'network of addresses') + for side in ['destination', 'source']: + if side in config: + side_conf = config[side] + + if len({'address', 'fqdn'} & set(side_conf)) > 1: + raise ConfigError('Only one of address, fqdn or geoip can be specified') + + if 'group' in side_conf: + if len({'address_group', 'network_group', 'domain_group'} & set(side_conf['group'])) > 1: + raise ConfigError('Only one address-group, network-group or domain-group can be specified') + + for group in valid_groups: + if group in side_conf['group']: + group_name = side_conf['group'][group] + error_group = group.replace("_", "-") + + if group in ['address_group', 'network_group', 'domain_group']: + types = [t for t in ['address', 'fqdn'] if t in side_conf] + if types: + raise ConfigError(f'{error_group} and {types[0]} cannot both be defined') + + if group_name and group_name[0] == '!': + group_name = group_name[1:] + + group_obj = dict_search_args(groups_dict, group, group_name) + + if group_obj is None: + raise ConfigError(f'Invalid {error_group} "{group_name}" on firewall rule') + + if not group_obj: + Warning(f'{error_group} "{group_name}" has no members!') + + if dict_search_args(side_conf, 'group', 'port_group'): + if 'protocol' not in config: + raise ConfigError('Protocol must be defined if specifying a port-group') + + if config['protocol'] not in ['tcp', 'udp', 'tcp_udp']: + raise ConfigError('Protocol must be tcp, udp, or tcp_udp when specifying a port-group') + def get_config(config=None): if config: conf = config @@ -105,16 +152,20 @@ def get_config(config=None): condensed_json = jmespath.search(pattern, nftable_json) if not conf.exists(base): - nat['helper_functions'] = 'remove' - - # Retrieve current table handler positions - nat['pre_ct_ignore'] = get_handler(condensed_json, 'PREROUTING', 'VYOS_CT_HELPER') - nat['pre_ct_conntrack'] = get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK') - nat['out_ct_ignore'] = get_handler(condensed_json, 'OUTPUT', 'VYOS_CT_HELPER') - nat['out_ct_conntrack'] = get_handler(condensed_json, 'OUTPUT', 'NAT_CONNTRACK') + if get_handler(condensed_json, 'PREROUTING', 'VYOS_CT_HELPER'): + nat['helper_functions'] = 'remove' + + # Retrieve current table handler positions + nat['pre_ct_ignore'] = get_handler(condensed_json, 'PREROUTING', 'VYOS_CT_HELPER') + nat['pre_ct_conntrack'] = get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK') + nat['out_ct_ignore'] = get_handler(condensed_json, 'OUTPUT', 'VYOS_CT_HELPER') + nat['out_ct_conntrack'] = get_handler(condensed_json, 'OUTPUT', 'NAT_CONNTRACK') nat['deleted'] = '' return nat + nat['firewall_group'] = conf.get_config_dict(['firewall', 'group'], key_mangling=('-', '_'), get_first_key=True, + no_tag_node_value_mangle=True) + # check if NAT connection tracking helpers need to be set up - this has to # be done only once if not get_handler(condensed_json, 'PREROUTING', 'NAT_CONNTRACK'): @@ -157,7 +208,7 @@ def verify(nat): Warning(f'IP address {ip} does not exist on the system!') # common rule verification - verify_rule(config, err_msg) + verify_rule(config, err_msg, nat['firewall_group']) if dict_search('destination.rule', nat): @@ -175,7 +226,7 @@ def verify(nat): raise ConfigError(f'{err_msg} translation requires address and/or port') # common rule verification - verify_rule(config, err_msg) + verify_rule(config, err_msg, nat['firewall_group']) if dict_search('static.rule', nat): for rule, config in dict_search('static.rule', nat).items(): @@ -186,7 +237,7 @@ def verify(nat): 'inbound-interface not specified') # common rule verification - verify_rule(config, err_msg) + verify_rule(config, err_msg, nat['firewall_group']) return None diff --git a/src/helpers/vyos-domain-resolver.py b/src/helpers/vyos-domain-resolver.py index 2f71f15db..035c208b2 100755 --- a/src/helpers/vyos-domain-resolver.py +++ b/src/helpers/vyos-domain-resolver.py @@ -37,6 +37,7 @@ domain_state = {} ipv4_tables = { 'ip mangle', 'ip vyos_filter', + 'ip vyos_nat' } ipv6_tables = { -- cgit v1.2.3 From 2f105b1b22de382927699d2f3a1ec6f00cb4ecbe Mon Sep 17 00:00:00 2001 From: Zen3515 <7106408+Zen3515@users.noreply.github.com> Date: Thu, 10 Nov 2022 16:23:48 +0700 Subject: dns: T738: add CLI option for PowerDNS local-port --- data/templates/dns-forwarding/recursor.conf.j2 | 3 +++ interface-definitions/dns-forwarding.xml.in | 4 ++++ .../scripts/cli/test_service_dns_forwarding.py | 21 +++++++++++++++++++++ 3 files changed, 28 insertions(+) (limited to 'data') diff --git a/data/templates/dns-forwarding/recursor.conf.j2 b/data/templates/dns-forwarding/recursor.conf.j2 index ce1b676d1..e02e6c13d 100644 --- a/data/templates/dns-forwarding/recursor.conf.j2 +++ b/data/templates/dns-forwarding/recursor.conf.j2 @@ -29,6 +29,9 @@ export-etc-hosts={{ 'no' if ignore_hosts_file is vyos_defined else 'yes' }} # listen-address local-address={{ listen_address | join(',') }} +# listen-port +local-port={{ port }} + # dnssec dnssec={{ dnssec }} diff --git a/interface-definitions/dns-forwarding.xml.in b/interface-definitions/dns-forwarding.xml.in index 3de0dc0eb..409028572 100644 --- a/interface-definitions/dns-forwarding.xml.in +++ b/interface-definitions/dns-forwarding.xml.in @@ -605,6 +605,10 @@ #include + #include + + 53 + Maximum amount of time negative entries are cached diff --git a/smoketest/scripts/cli/test_service_dns_forwarding.py b/smoketest/scripts/cli/test_service_dns_forwarding.py index 65b676451..8e9b7ef43 100755 --- a/smoketest/scripts/cli/test_service_dns_forwarding.py +++ b/smoketest/scripts/cli/test_service_dns_forwarding.py @@ -32,6 +32,7 @@ base_path = ['service', 'dns', 'forwarding'] allow_from = ['192.0.2.0/24', '2001:db8::/32'] listen_adress = ['127.0.0.1', '::1'] +listen_ports = ['53', '5353'] def get_config_value(key, file=CONFIG_FILE): tmp = read_file(file) @@ -224,5 +225,25 @@ class TestServicePowerDNS(VyOSUnitTestSHIM.TestCase): tmp = get_config_value('dns64-prefix') self.assertEqual(tmp, dns_prefix) + def test_listening_port(self): + # only one port can be listen + for port in listen_ports: + self.cli_set(base_path + ['port', port]) + for network in allow_from: + self.cli_set(base_path + ['allow-from', network]) + for address in listen_adress: + self.cli_set(base_path + ['listen-address', address]) + + # commit changes + self.cli_commit() + + # verify local-port configuration + tmp = get_config_value('local-port') + self.assertEqual(tmp, port) + + # reset to test differnt port + self.cli_delete(base_path) + self.cli_commit() + if __name__ == '__main__': unittest.main(verbosity=2) -- cgit v1.2.3 From ef365493aef665c20e27ff2de473624c14e1b521 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 7 Nov 2022 17:44:00 +0000 Subject: T4789: Ability to get op-mode raw data for PPPoE L2TP SSTP IPoE Ability to get 'raw' data sessions and statistics for accel-ppp protocols IPoE/PPPoE/L2TP/PPTP/SSTP server --- data/op-mode-standardized.json | 1 + python/vyos/accel_ppp.py | 74 +++++++++++++++++++++++ src/op_mode/accelppp.py | 133 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+) create mode 100644 python/vyos/accel_ppp.py create mode 100755 src/op_mode/accelppp.py (limited to 'data') diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index ec950765d..b162f4097 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -1,4 +1,5 @@ [ +"accelppp.py", "bgp.py", "bridge.py", "conntrack.py", diff --git a/python/vyos/accel_ppp.py b/python/vyos/accel_ppp.py new file mode 100644 index 000000000..bfc8ee5a9 --- /dev/null +++ b/python/vyos/accel_ppp.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import sys + +import vyos.opmode +from vyos.util import rc_cmd + + +def get_server_statistics(accel_statistics, pattern, sep=':') -> dict: + import re + + stat_dict = {'sessions': {}} + + cpu = re.search(r'cpu(.*)', accel_statistics).group(0) + # Find all lines with pattern, for example 'sstp:' + data = re.search(rf'{pattern}(.*)', accel_statistics, re.DOTALL).group(0) + session_starting = re.search(r'starting(.*)', data).group(0) + session_active = re.search(r'active(.*)', data).group(0) + + for entry in {cpu, session_starting, session_active}: + if sep in entry: + key, value = entry.split(sep) + if key in ['starting', 'active', 'finishing']: + stat_dict['sessions'][key] = value.strip() + continue + stat_dict[key] = value.strip() + return stat_dict + + +def accel_cmd(port: int, command: str) -> str: + _, output = rc_cmd(f'/usr/bin/accel-cmd -p{port} {command}') + return output + + +def accel_out_parse(accel_output: list[str]) -> list[dict[str, str]]: + """ Parse accel-cmd show sessions output """ + data_list: list[dict[str, str]] = list() + field_names: list[str] = list() + + field_names_unstripped: list[str] = accel_output.pop(0).split('|') + for field_name in field_names_unstripped: + field_names.append(field_name.strip()) + + while accel_output: + if '|' not in accel_output[0]: + accel_output.pop(0) + continue + + current_item: list[str] = accel_output.pop(0).split('|') + item_dict: dict[str, str] = {} + + for field_index in range(len(current_item)): + field_name: str = field_names[field_index] + field_value: str = current_item[field_index].strip() + item_dict[field_name] = field_value + + data_list.append(item_dict) + + return data_list diff --git a/src/op_mode/accelppp.py b/src/op_mode/accelppp.py new file mode 100755 index 000000000..2fd045dc3 --- /dev/null +++ b/src/op_mode/accelppp.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import sys + +import vyos.accel_ppp +import vyos.opmode + +from vyos.configquery import ConfigTreeQuery +from vyos.util import rc_cmd + + +accel_dict = { + 'ipoe': { + 'port': 2002, + 'path': 'service ipoe-server' + }, + 'pppoe': { + 'port': 2001, + 'path': 'service pppoe-server' + }, + 'pptp': { + 'port': 2003, + 'path': 'vpn pptp' + }, + 'l2tp': { + 'port': 2004, + 'path': 'vpn l2tp' + }, + 'sstp': { + 'port': 2005, + 'path': 'vpn sstp' + } +} + + +def _get_raw_statistics(accel_output, pattern): + return vyos.accel_ppp.get_server_statistics(accel_output, pattern, sep=':') + + +def _get_raw_sessions(port): + cmd_options = 'show sessions ifname,username,ip,ip6,ip6-dp,type,state,' \ + 'uptime-raw,calling-sid,called-sid,sid,comp,rx-bytes-raw,' \ + 'tx-bytes-raw,rx-pkts,tx-pkts' + output = vyos.accel_ppp.accel_cmd(port, cmd_options) + parsed_data: list[dict[str, str]] = vyos.accel_ppp.accel_out_parse( + output.splitlines()) + return parsed_data + + +def _verify(func): + """Decorator checks if accel-ppp protocol + ipoe/pppoe/pptp/l2tp/sstp is configured + + for example: + service ipoe-server + vpn sstp + """ + from functools import wraps + + @wraps(func) + def _wrapper(*args, **kwargs): + config = ConfigTreeQuery() + protocol_list = accel_dict.keys() + protocol = kwargs.get('protocol') + # unknown or incorrect protocol query + if protocol not in protocol_list: + unconf_message = f'unknown protocol "{protocol}"' + raise vyos.opmode.UnconfiguredSubsystem(unconf_message) + # Check if config does not exist + config_protocol_path = accel_dict[protocol]['path'] + if not config.exists(config_protocol_path): + unconf_message = f'"{config_protocol_path}" is not configured' + raise vyos.opmode.UnconfiguredSubsystem(unconf_message) + return func(*args, **kwargs) + + return _wrapper + + +@_verify +def show_statistics(raw: bool, protocol: str): + """show accel-cmd statistics + CPU utilization and amount of sessions + + protocol: ipoe/pppoe/ppptp/l2tp/sstp + """ + pattern = f'{protocol}:' + port = accel_dict[protocol]['port'] + rc, output = rc_cmd(f'/usr/bin/accel-cmd -p {port} show stat') + + if raw: + return _get_raw_statistics(output, pattern) + + return output + + +@_verify +def show_sessions(raw: bool, protocol: str): + """show accel-cmd sessions + + protocol: ipoe/pppoe/ppptp/l2tp/sstp + """ + port = accel_dict[protocol]['port'] + if raw: + return _get_raw_sessions(port) + + return vyos.accel_ppp.accel_cmd(port, + 'show sessions ifname,username,ip,ip6,ip6-dp,' + 'calling-sid,rate-limit,state,uptime,rx-bytes,tx-bytes') + + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) -- cgit v1.2.3 From 586b24e0af1ae57c47c772229fc94ab50dfc1e4f Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Wed, 2 Nov 2022 15:32:11 +0100 Subject: policy: T2199: T4605: Migrate policy route interface to `policy route|route6 interface ` * Include refactor to policy route to allow for deletion of mangle table instead of complex cleanup * T4605: Rename mangle table to vyos_mangle --- data/templates/firewall/nftables-policy.j2 | 31 +++-- .../include/interface/interface-policy-vif-c.xml.i | 26 ---- .../include/interface/interface-policy-vif.xml.i | 26 ---- .../include/interface/interface-policy.xml.i | 26 ---- .../include/interface/vif-s.xml.i | 2 - interface-definitions/include/interface/vif.xml.i | 1 - .../include/version/policy-version.xml.i | 2 +- interface-definitions/interfaces-bonding.xml.in | 1 - interface-definitions/interfaces-bridge.xml.in | 1 - interface-definitions/interfaces-dummy.xml.in | 1 - interface-definitions/interfaces-ethernet.xml.in | 1 - interface-definitions/interfaces-geneve.xml.in | 1 - interface-definitions/interfaces-input.xml.in | 1 - interface-definitions/interfaces-l2tpv3.xml.in | 1 - interface-definitions/interfaces-macsec.xml.in | 1 - interface-definitions/interfaces-openvpn.xml.in | 1 - interface-definitions/interfaces-pppoe.xml.in | 1 - .../interfaces-pseudo-ethernet.xml.in | 1 - interface-definitions/interfaces-tunnel.xml.in | 1 - interface-definitions/interfaces-vti.xml.in | 1 - interface-definitions/interfaces-vxlan.xml.in | 1 - interface-definitions/interfaces-wireguard.xml.in | 1 - interface-definitions/interfaces-wireless.xml.in | 1 - interface-definitions/interfaces-wwan.xml.in | 1 - interface-definitions/policy-route.xml.in | 2 + smoketest/scripts/cli/test_policy_route.py | 58 +++++---- src/conf_mode/policy-route-interface.py | 132 --------------------- src/conf_mode/policy-route.py | 106 +---------------- src/helpers/vyos-domain-resolver.py | 4 +- src/migration-scripts/policy/4-to-5 | 92 ++++++++++++++ src/op_mode/policy_route.py | 42 +------ 31 files changed, 154 insertions(+), 413 deletions(-) delete mode 100644 interface-definitions/include/interface/interface-policy-vif-c.xml.i delete mode 100644 interface-definitions/include/interface/interface-policy-vif.xml.i delete mode 100644 interface-definitions/include/interface/interface-policy.xml.i delete mode 100755 src/conf_mode/policy-route-interface.py create mode 100755 src/migration-scripts/policy/4-to-5 (limited to 'data') diff --git a/data/templates/firewall/nftables-policy.j2 b/data/templates/firewall/nftables-policy.j2 index 40118930b..6cb3b2f95 100644 --- a/data/templates/firewall/nftables-policy.j2 +++ b/data/templates/firewall/nftables-policy.j2 @@ -2,21 +2,24 @@ {% import 'firewall/nftables-defines.j2' as group_tmpl %} -{% if cleanup_commands is vyos_defined %} -{% for command in cleanup_commands %} -{{ command }} -{% endfor %} +{% if first_install is not vyos_defined %} +delete table ip vyos_mangle +delete table ip6 vyos_mangle {% endif %} - -table ip mangle { -{% if first_install is vyos_defined %} +table ip vyos_mangle { chain VYOS_PBR_PREROUTING { type filter hook prerouting priority -150; policy accept; +{% if route is vyos_defined %} +{% for route_text, conf in route.items() if conf.interface is vyos_defined %} + iifname { {{ ",".join(conf.interface) }} } counter jump VYOS_PBR_{{ route_text }} +{% endfor %} +{% endif %} } + chain VYOS_PBR_POSTROUTING { type filter hook postrouting priority -150; policy accept; } -{% endif %} + {% if route is vyos_defined %} {% for route_text, conf in route.items() %} chain VYOS_PBR_{{ route_text }} { @@ -32,15 +35,20 @@ table ip mangle { {{ group_tmpl.groups(firewall_group, False) }} } -table ip6 mangle { -{% if first_install is vyos_defined %} +table ip6 vyos_mangle { chain VYOS_PBR6_PREROUTING { type filter hook prerouting priority -150; policy accept; +{% if route6 is vyos_defined %} +{% for route_text, conf in route6.items() if conf.interface is vyos_defined %} + iifname { {{ ",".join(conf.interface) }} } counter jump VYOS_PBR6_{{ route_text }} +{% endfor %} +{% endif %} } + chain VYOS_PBR6_POSTROUTING { type filter hook postrouting priority -150; policy accept; } -{% endif %} + {% if route6 is vyos_defined %} {% for route_text, conf in route6.items() %} chain VYOS_PBR6_{{ route_text }} { @@ -52,5 +60,6 @@ table ip6 mangle { } {% endfor %} {% endif %} + {{ group_tmpl.groups(firewall_group, True) }} } diff --git a/interface-definitions/include/interface/interface-policy-vif-c.xml.i b/interface-definitions/include/interface/interface-policy-vif-c.xml.i deleted file mode 100644 index 866fcd5c0..000000000 --- a/interface-definitions/include/interface/interface-policy-vif-c.xml.i +++ /dev/null @@ -1,26 +0,0 @@ - - - - 620 - Policy route options - - - - - IPv4 policy route ruleset for interface - - policy route - - - - - - IPv6 policy route ruleset for interface - - policy route6 - - - - - - diff --git a/interface-definitions/include/interface/interface-policy-vif.xml.i b/interface-definitions/include/interface/interface-policy-vif.xml.i deleted file mode 100644 index 83510fe59..000000000 --- a/interface-definitions/include/interface/interface-policy-vif.xml.i +++ /dev/null @@ -1,26 +0,0 @@ - - - - 620 - Policy route options - - - - - IPv4 policy route ruleset for interface - - policy route - - - - - - IPv6 policy route ruleset for interface - - policy route6 - - - - - - diff --git a/interface-definitions/include/interface/interface-policy.xml.i b/interface-definitions/include/interface/interface-policy.xml.i deleted file mode 100644 index 42a8fd009..000000000 --- a/interface-definitions/include/interface/interface-policy.xml.i +++ /dev/null @@ -1,26 +0,0 @@ - - - - 620 - Policy route options - - - - - IPv4 policy route ruleset for interface - - policy route - - - - - - IPv6 policy route ruleset for interface - - policy route6 - - - - - - diff --git a/interface-definitions/include/interface/vif-s.xml.i b/interface-definitions/include/interface/vif-s.xml.i index 916349ade..6d50d7238 100644 --- a/interface-definitions/include/interface/vif-s.xml.i +++ b/interface-definitions/include/interface/vif-s.xml.i @@ -18,7 +18,6 @@ #include #include #include - #include Protocol used for service VLAN (default: 802.1ad) @@ -67,7 +66,6 @@ #include #include #include - #include #include diff --git a/interface-definitions/include/interface/vif.xml.i b/interface-definitions/include/interface/vif.xml.i index 73a8c98ff..3f8f113ea 100644 --- a/interface-definitions/include/interface/vif.xml.i +++ b/interface-definitions/include/interface/vif.xml.i @@ -18,7 +18,6 @@ #include #include #include - #include VLAN egress QoS diff --git a/interface-definitions/include/version/policy-version.xml.i b/interface-definitions/include/version/policy-version.xml.i index 89bde20c7..f1494eaa3 100644 --- a/interface-definitions/include/version/policy-version.xml.i +++ b/interface-definitions/include/version/policy-version.xml.i @@ -1,3 +1,3 @@ - + diff --git a/interface-definitions/interfaces-bonding.xml.in b/interface-definitions/interfaces-bonding.xml.in index 41e4a68a8..96e0e5d89 100644 --- a/interface-definitions/interfaces-bonding.xml.in +++ b/interface-definitions/interfaces-bonding.xml.in @@ -56,7 +56,6 @@ #include #include #include - #include Bonding transmit hash policy diff --git a/interface-definitions/interfaces-bridge.xml.in b/interface-definitions/interfaces-bridge.xml.in index d633077d9..d52e213b6 100644 --- a/interface-definitions/interfaces-bridge.xml.in +++ b/interface-definitions/interfaces-bridge.xml.in @@ -41,7 +41,6 @@ #include #include #include - #include Forwarding delay diff --git a/interface-definitions/interfaces-dummy.xml.in b/interface-definitions/interfaces-dummy.xml.in index fb36741f7..eb525b547 100644 --- a/interface-definitions/interfaces-dummy.xml.in +++ b/interface-definitions/interfaces-dummy.xml.in @@ -19,7 +19,6 @@ #include #include #include - #include IPv4 routing parameters diff --git a/interface-definitions/interfaces-ethernet.xml.in b/interface-definitions/interfaces-ethernet.xml.in index 77f130e1c..e9ae0acfe 100644 --- a/interface-definitions/interfaces-ethernet.xml.in +++ b/interface-definitions/interfaces-ethernet.xml.in @@ -31,7 +31,6 @@ #include #include - #include Duplex mode diff --git a/interface-definitions/interfaces-geneve.xml.in b/interface-definitions/interfaces-geneve.xml.in index b959c787d..f8e9909f8 100644 --- a/interface-definitions/interfaces-geneve.xml.in +++ b/interface-definitions/interfaces-geneve.xml.in @@ -23,7 +23,6 @@ #include #include #include - #include GENEVE tunnel parameters diff --git a/interface-definitions/interfaces-input.xml.in b/interface-definitions/interfaces-input.xml.in index d01c760f8..97502d954 100644 --- a/interface-definitions/interfaces-input.xml.in +++ b/interface-definitions/interfaces-input.xml.in @@ -19,7 +19,6 @@ #include #include - #include #include diff --git a/interface-definitions/interfaces-l2tpv3.xml.in b/interface-definitions/interfaces-l2tpv3.xml.in index bde68dd5a..0ebc3253d 100644 --- a/interface-definitions/interfaces-l2tpv3.xml.in +++ b/interface-definitions/interfaces-l2tpv3.xml.in @@ -32,7 +32,6 @@ 5000 #include - #include Encapsulation type diff --git a/interface-definitions/interfaces-macsec.xml.in b/interface-definitions/interfaces-macsec.xml.in index 5c9f4cd76..441236ec2 100644 --- a/interface-definitions/interfaces-macsec.xml.in +++ b/interface-definitions/interfaces-macsec.xml.in @@ -21,7 +21,6 @@ #include #include #include - #include #include diff --git a/interface-definitions/interfaces-openvpn.xml.in b/interface-definitions/interfaces-openvpn.xml.in index 3876e31da..7cfb9ee7a 100644 --- a/interface-definitions/interfaces-openvpn.xml.in +++ b/interface-definitions/interfaces-openvpn.xml.in @@ -34,7 +34,6 @@ #include - #include OpenVPN interface device-type diff --git a/interface-definitions/interfaces-pppoe.xml.in b/interface-definitions/interfaces-pppoe.xml.in index 84f76a7ee..719060fa9 100644 --- a/interface-definitions/interfaces-pppoe.xml.in +++ b/interface-definitions/interfaces-pppoe.xml.in @@ -19,7 +19,6 @@ #include #include #include - #include #include #include #include diff --git a/interface-definitions/interfaces-pseudo-ethernet.xml.in b/interface-definitions/interfaces-pseudo-ethernet.xml.in index 4eb9bf111..2fe07ffd5 100644 --- a/interface-definitions/interfaces-pseudo-ethernet.xml.in +++ b/interface-definitions/interfaces-pseudo-ethernet.xml.in @@ -28,7 +28,6 @@ #include #include #include - #include Receive mode (default: private) diff --git a/interface-definitions/interfaces-tunnel.xml.in b/interface-definitions/interfaces-tunnel.xml.in index fe49d337a..333a5b178 100644 --- a/interface-definitions/interfaces-tunnel.xml.in +++ b/interface-definitions/interfaces-tunnel.xml.in @@ -29,7 +29,6 @@ #include #include #include - #include 6rd network prefix diff --git a/interface-definitions/interfaces-vti.xml.in b/interface-definitions/interfaces-vti.xml.in index eeaea0dc3..11f001dc0 100644 --- a/interface-definitions/interfaces-vti.xml.in +++ b/interface-definitions/interfaces-vti.xml.in @@ -25,7 +25,6 @@ #include #include #include - #include diff --git a/interface-definitions/interfaces-vxlan.xml.in b/interface-definitions/interfaces-vxlan.xml.in index 4902ff36d..331f930d3 100644 --- a/interface-definitions/interfaces-vxlan.xml.in +++ b/interface-definitions/interfaces-vxlan.xml.in @@ -54,7 +54,6 @@ #include #include #include - #include 1450 diff --git a/interface-definitions/interfaces-wireguard.xml.in b/interface-definitions/interfaces-wireguard.xml.in index 23f50d146..35e223588 100644 --- a/interface-definitions/interfaces-wireguard.xml.in +++ b/interface-definitions/interfaces-wireguard.xml.in @@ -21,7 +21,6 @@ #include #include #include - #include #include 1420 diff --git a/interface-definitions/interfaces-wireless.xml.in b/interface-definitions/interfaces-wireless.xml.in index 9e7fc29bc..5271df624 100644 --- a/interface-definitions/interfaces-wireless.xml.in +++ b/interface-definitions/interfaces-wireless.xml.in @@ -20,7 +20,6 @@ #include - #include HT and VHT capabilities for your card diff --git a/interface-definitions/interfaces-wwan.xml.in b/interface-definitions/interfaces-wwan.xml.in index b0b8367dc..758784540 100644 --- a/interface-definitions/interfaces-wwan.xml.in +++ b/interface-definitions/interfaces-wwan.xml.in @@ -39,7 +39,6 @@ #include #include #include - #include #include #include diff --git a/interface-definitions/policy-route.xml.in b/interface-definitions/policy-route.xml.in index 44b96c2e6..48a5bf7d1 100644 --- a/interface-definitions/policy-route.xml.in +++ b/interface-definitions/policy-route.xml.in @@ -12,6 +12,7 @@ #include + #include #include @@ -65,6 +66,7 @@ #include + #include #include diff --git a/smoketest/scripts/cli/test_policy_route.py b/smoketest/scripts/cli/test_policy_route.py index 046e385bb..11b3c678e 100755 --- a/smoketest/scripts/cli/test_policy_route.py +++ b/smoketest/scripts/cli/test_policy_route.py @@ -42,18 +42,25 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): super(TestPolicyRoute, cls).tearDownClass() def tearDown(self): - self.cli_delete(['interfaces', 'ethernet', interface, 'policy']) self.cli_delete(['policy', 'route']) self.cli_delete(['policy', 'route6']) self.cli_commit() + # Verify nftables cleanup nftables_search = [ ['set N_smoketest_network'], ['set N_smoketest_network1'], ['chain VYOS_PBR_smoketest'] ] - self.verify_nftables(nftables_search, 'ip mangle', inverse=True) + self.verify_nftables(nftables_search, 'ip vyos_mangle', inverse=True) + + # Verify ip rule cleanup + ip_rule_search = [ + ['fwmark ' + hex(table_mark_offset - int(table_id)), 'lookup ' + table_id] + ] + + self.verify_rules(ip_rule_search, inverse=True) def verify_nftables(self, nftables_search, table, inverse=False): nftables_output = cmd(f'sudo nft list table {table}') @@ -66,6 +73,17 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): break self.assertTrue(not matched if inverse else matched, msg=search) + def verify_rules(self, rules_search, inverse=False): + rule_output = cmd('ip rule show') + + for search in rules_search: + matched = False + for line in rule_output.split("\n"): + if all(item in line for item in search): + matched = True + break + self.assertTrue(not matched if inverse else matched, msg=search) + def test_pbr_group(self): self.cli_set(['firewall', 'group', 'network-group', 'smoketest_network', 'network', '172.16.99.0/24']) self.cli_set(['firewall', 'group', 'network-group', 'smoketest_network1', 'network', '172.16.101.0/24']) @@ -74,8 +92,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'source', 'group', 'network-group', 'smoketest_network']) self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'destination', 'group', 'network-group', 'smoketest_network1']) self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'set', 'mark', mark]) - - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route', 'smoketest']) + self.cli_set(['policy', 'route', 'smoketest', 'interface', interface]) self.cli_commit() @@ -84,7 +101,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['ip daddr @N_smoketest_network1', 'ip saddr @N_smoketest_network'], ] - self.verify_nftables(nftables_search, 'ip mangle') + self.verify_nftables(nftables_search, 'ip vyos_mangle') self.cli_delete(['firewall']) @@ -92,8 +109,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'source', 'address', '172.16.20.10']) self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'destination', 'address', '172.16.10.10']) self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'set', 'mark', mark]) - - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route', 'smoketest']) + self.cli_set(['policy', 'route', 'smoketest', 'interface', interface]) self.cli_commit() @@ -104,7 +120,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['ip daddr 172.16.10.10', 'ip saddr 172.16.20.10', 'meta mark set ' + mark_hex], ] - self.verify_nftables(nftables_search, 'ip mangle') + self.verify_nftables(nftables_search, 'ip vyos_mangle') def test_pbr_table(self): self.cli_set(['policy', 'route', 'smoketest', 'rule', '1', 'protocol', 'tcp']) @@ -116,8 +132,8 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '1', 'destination', 'port', '8888']) self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '1', 'set', 'table', table_id]) - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route', 'smoketest']) - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route6', 'smoketest6']) + self.cli_set(['policy', 'route', 'smoketest', 'interface', interface]) + self.cli_set(['policy', 'route6', 'smoketest6', 'interface', interface]) self.cli_commit() @@ -130,7 +146,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['tcp flags syn / syn,ack', 'tcp dport 8888', 'meta mark set ' + mark_hex] ] - self.verify_nftables(nftables_search, 'ip mangle') + self.verify_nftables(nftables_search, 'ip vyos_mangle') # IPv6 @@ -139,7 +155,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['meta l4proto { tcp, udp }', 'th dport 8888', 'meta mark set ' + mark_hex] ] - self.verify_nftables(nftables6_search, 'ip6 mangle') + self.verify_nftables(nftables6_search, 'ip6 vyos_mangle') # IP rule fwmark -> table @@ -147,15 +163,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['fwmark ' + hex(table_mark_offset - int(table_id)), 'lookup ' + table_id] ] - ip_rule_output = cmd('ip rule show') - - for search in ip_rule_search: - matched = False - for line in ip_rule_output.split("\n"): - if all(item in line for item in search): - matched = True - break - self.assertTrue(matched) + self.verify_rules(ip_rule_search) def test_pbr_matching_criteria(self): @@ -203,8 +211,8 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '5', 'dscp-exclude', '14-19']) self.cli_set(['policy', 'route6', 'smoketest6', 'rule', '5', 'set', 'table', table_id]) - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route', 'smoketest']) - self.cli_set(['interfaces', 'ethernet', interface, 'policy', 'route6', 'smoketest6']) + self.cli_set(['policy', 'route', 'smoketest', 'interface', interface]) + self.cli_set(['policy', 'route6', 'smoketest6', 'interface', interface]) self.cli_commit() @@ -220,7 +228,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['ip dscp { 0x29, 0x39-0x3b }', 'meta mark set ' + mark_hex] ] - self.verify_nftables(nftables_search, 'ip mangle') + self.verify_nftables(nftables_search, 'ip vyos_mangle') # IPv6 nftables6_search = [ @@ -232,7 +240,7 @@ class TestPolicyRoute(VyOSUnitTestSHIM.TestCase): ['ip6 dscp != { 0x0e-0x13, 0x3d }', 'meta mark set ' + mark_hex] ] - self.verify_nftables(nftables6_search, 'ip6 mangle') + self.verify_nftables(nftables6_search, 'ip6 vyos_mangle') if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/policy-route-interface.py b/src/conf_mode/policy-route-interface.py deleted file mode 100755 index 58c5fd93d..000000000 --- a/src/conf_mode/policy-route-interface.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2021 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import re - -from sys import argv -from sys import exit - -from vyos.config import Config -from vyos.ifconfig import Section -from vyos.template import render -from vyos.util import cmd -from vyos.util import run -from vyos import ConfigError -from vyos import airbag -airbag.enable() - -def get_config(config=None): - if config: - conf = config - else: - conf = Config() - - ifname = argv[1] - ifpath = Section.get_config_path(ifname) - if_policy_path = f'interfaces {ifpath} policy' - - if_policy = conf.get_config_dict(if_policy_path, key_mangling=('-', '_'), get_first_key=True, - no_tag_node_value_mangle=True) - - if_policy['ifname'] = ifname - if_policy['policy'] = conf.get_config_dict(['policy'], key_mangling=('-', '_'), get_first_key=True, - no_tag_node_value_mangle=True) - - return if_policy - -def verify_chain(table, chain): - # Verify policy route applied - code = run(f'nft list chain {table} {chain}') - return code == 0 - -def verify(if_policy): - # bail out early - looks like removal from running config - if not if_policy: - return None - - for route in ['route', 'route6']: - if route in if_policy: - if route not in if_policy['policy']: - raise ConfigError('Policy route not configured') - - route_name = if_policy[route] - - if route_name not in if_policy['policy'][route]: - raise ConfigError(f'Invalid policy route name "{name}"') - - nft_prefix = 'VYOS_PBR6_' if route == 'route6' else 'VYOS_PBR_' - nft_table = 'ip6 mangle' if route == 'route6' else 'ip mangle' - - if not verify_chain(nft_table, nft_prefix + route_name): - raise ConfigError('Policy route did not apply') - - return None - -def generate(if_policy): - return None - -def cleanup_rule(table, chain, ifname, new_name=None): - results = cmd(f'nft -a list chain {table} {chain}').split("\n") - retval = None - for line in results: - if f'ifname "{ifname}"' in line: - if new_name and f'jump {new_name}' in line: - # new_name is used to clear rules for any previously referenced chains - # returns true when rule exists and doesn't need to be created - retval = True - continue - - handle_search = re.search('handle (\d+)', line) - if handle_search: - cmd(f'nft delete rule {table} {chain} handle {handle_search[1]}') - return retval - -def apply(if_policy): - ifname = if_policy['ifname'] - - route_chain = 'VYOS_PBR_PREROUTING' - ipv6_route_chain = 'VYOS_PBR6_PREROUTING' - - if 'route' in if_policy: - name = 'VYOS_PBR_' + if_policy['route'] - rule_exists = cleanup_rule('ip mangle', route_chain, ifname, name) - - if not rule_exists: - cmd(f'nft insert rule ip mangle {route_chain} iifname {ifname} counter jump {name}') - else: - cleanup_rule('ip mangle', route_chain, ifname) - - if 'route6' in if_policy: - name = 'VYOS_PBR6_' + if_policy['route6'] - rule_exists = cleanup_rule('ip6 mangle', ipv6_route_chain, ifname, name) - - if not rule_exists: - cmd(f'nft insert rule ip6 mangle {ipv6_route_chain} iifname {ifname} counter jump {name}') - else: - cleanup_rule('ip6 mangle', ipv6_route_chain, ifname) - - return None - -if __name__ == '__main__': - try: - c = get_config() - verify(c) - generate(c) - apply(c) - except ConfigError as e: - print(e) - exit(1) diff --git a/src/conf_mode/policy-route.py b/src/conf_mode/policy-route.py index 00539b9c7..1d016695e 100755 --- a/src/conf_mode/policy-route.py +++ b/src/conf_mode/policy-route.py @@ -15,7 +15,6 @@ # along with this program. If not, see . import os -import re from json import loads from sys import exit @@ -25,7 +24,6 @@ from vyos.config import Config from vyos.template import render from vyos.util import cmd from vyos.util import dict_search_args -from vyos.util import dict_search_recursive from vyos.util import run from vyos import ConfigError from vyos import airbag @@ -34,48 +32,13 @@ airbag.enable() mark_offset = 0x7FFFFFFF nftables_conf = '/run/nftables_policy.conf' -ROUTE_PREFIX = 'VYOS_PBR_' -ROUTE6_PREFIX = 'VYOS_PBR6_' - -preserve_chains = [ - 'VYOS_PBR_PREROUTING', - 'VYOS_PBR_POSTROUTING', - 'VYOS_PBR6_PREROUTING', - 'VYOS_PBR6_POSTROUTING' -] - valid_groups = [ 'address_group', + 'domain_group', 'network_group', 'port_group' ] -group_set_prefix = { - 'A_': 'address_group', - 'A6_': 'ipv6_address_group', -# 'D_': 'domain_group', - 'M_': 'mac_group', - 'N_': 'network_group', - 'N6_': 'ipv6_network_group', - 'P_': 'port_group' -} - -def get_policy_interfaces(conf): - out = {} - interfaces = conf.get_config_dict(['interfaces'], key_mangling=('-', '_'), get_first_key=True, - no_tag_node_value_mangle=True) - def find_interfaces(iftype_conf, output={}, prefix=''): - for ifname, if_conf in iftype_conf.items(): - if 'policy' in if_conf: - output[prefix + ifname] = if_conf['policy'] - for vif in ['vif', 'vif_s', 'vif_c']: - if vif in if_conf: - output.update(find_interfaces(if_conf[vif], output, f'{prefix}{ifname}.')) - return output - for iftype, iftype_conf in interfaces.items(): - out.update(find_interfaces(iftype_conf)) - return out - def get_config(config=None): if config: conf = config @@ -88,7 +51,6 @@ def get_config(config=None): policy['firewall_group'] = conf.get_config_dict(['firewall', 'group'], key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) - policy['interfaces'] = get_policy_interfaces(conf) return policy @@ -132,8 +94,8 @@ def verify_rule(policy, name, rule_conf, ipv6, rule_id): side_conf = rule_conf[side] if 'group' in side_conf: - if {'address_group', 'network_group'} <= set(side_conf['group']): - raise ConfigError('Only one address-group or network-group can be specified') + if len({'address_group', 'domain_group', 'network_group'} & set(side_conf['group'])) > 1: + raise ConfigError('Only one address-group, domain-group or network-group can be specified') for group in valid_groups: if group in side_conf['group']: @@ -168,73 +130,11 @@ def verify(policy): for rule_id, rule_conf in pol_conf['rule'].items(): verify_rule(policy, name, rule_conf, ipv6, rule_id) - for ifname, if_policy in policy['interfaces'].items(): - name = dict_search_args(if_policy, 'route') - ipv6_name = dict_search_args(if_policy, 'route6') - - if name and not dict_search_args(policy, 'route', name): - raise ConfigError(f'Policy route "{name}" is still referenced on interface {ifname}') - - if ipv6_name and not dict_search_args(policy, 'route6', ipv6_name): - raise ConfigError(f'Policy route6 "{ipv6_name}" is still referenced on interface {ifname}') - return None -def cleanup_commands(policy): - commands = [] - commands_chains = [] - commands_sets = [] - for table in ['ip mangle', 'ip6 mangle']: - route_node = 'route' if table == 'ip mangle' else 'route6' - chain_prefix = ROUTE_PREFIX if table == 'ip mangle' else ROUTE6_PREFIX - - json_str = cmd(f'nft -t -j list table {table}') - obj = loads(json_str) - if 'nftables' not in obj: - continue - for item in obj['nftables']: - if 'chain' in item: - chain = item['chain']['name'] - if chain in preserve_chains or not chain.startswith("VYOS_PBR"): - continue - - if dict_search_args(policy, route_node, chain.replace(chain_prefix, "", 1)) != None: - commands.append(f'flush chain {table} {chain}') - else: - commands_chains.append(f'delete chain {table} {chain}') - - if 'rule' in item: - rule = item['rule'] - chain = rule['chain'] - handle = rule['handle'] - - if chain not in preserve_chains: - continue - - target, _ = next(dict_search_recursive(rule['expr'], 'target')) - - if target.startswith(chain_prefix): - if dict_search_args(policy, route_node, target.replace(chain_prefix, "", 1)) == None: - commands.append(f'delete rule {table} {chain} handle {handle}') - - if 'set' in item: - set_name = item['set']['name'] - - for prefix, group_type in group_set_prefix.items(): - if set_name.startswith(prefix): - group_name = set_name.replace(prefix, "", 1) - if dict_search_args(policy, 'firewall_group', group_type, group_name) != None: - commands_sets.append(f'flush set {table} {set_name}') - else: - commands_sets.append(f'delete set {table} {set_name}') - - return commands + commands_chains + commands_sets - def generate(policy): if not os.path.exists(nftables_conf): policy['first_install'] = True - else: - policy['cleanup_commands'] = cleanup_commands(policy) render(nftables_conf, 'firewall/nftables-policy.j2', policy) return None diff --git a/src/helpers/vyos-domain-resolver.py b/src/helpers/vyos-domain-resolver.py index 035c208b2..e31d9238e 100755 --- a/src/helpers/vyos-domain-resolver.py +++ b/src/helpers/vyos-domain-resolver.py @@ -35,13 +35,13 @@ cache = False domain_state = {} ipv4_tables = { - 'ip mangle', + 'ip vyos_mangle', 'ip vyos_filter', 'ip vyos_nat' } ipv6_tables = { - 'ip6 mangle', + 'ip6 vyos_mangle', 'ip6 vyos_filter' } diff --git a/src/migration-scripts/policy/4-to-5 b/src/migration-scripts/policy/4-to-5 new file mode 100755 index 000000000..33c9e6ade --- /dev/null +++ b/src/migration-scripts/policy/4-to-5 @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# T2199: Migrate interface policy nodes to policy route interface + +import re + +from sys import argv +from sys import exit + +from vyos.configtree import ConfigTree +from vyos.ifconfig import Section + +if (len(argv) < 1): + print("Must specify file name!") + exit(1) + +file_name = argv[1] + +with open(file_name, 'r') as f: + config_file = f.read() + +base4 = ['policy', 'route'] +base6 = ['policy', 'route6'] +config = ConfigTree(config_file) + +if not config.exists(base4) and not config.exists(base6): + # Nothing to do + exit(0) + +def migrate_interface(config, iftype, ifname, vif=None, vifs=None, vifc=None): + if_path = ['interfaces', iftype, ifname] + ifname_full = ifname + + if vif: + if_path += ['vif', vif] + ifname_full = f'{ifname}.{vif}' + elif vifs: + if_path += ['vif-s', vifs] + ifname_full = f'{ifname}.{vifs}' + if vifc: + if_path += ['vif-c', vifc] + ifname_full = f'{ifname}.{vifs}.{vifc}' + + if not config.exists(if_path + ['policy']): + return + + if config.exists(if_path + ['policy', 'route']): + route_name = config.return_value(if_path + ['policy', 'route']) + config.set(base4 + [route_name, 'interface'], value=ifname_full, replace=False) + + if config.exists(if_path + ['policy', 'route6']): + route_name = config.return_value(if_path + ['policy', 'route6']) + config.set(base6 + [route_name, 'interface'], value=ifname_full, replace=False) + + config.delete(if_path + ['policy']) + +for iftype in config.list_nodes(['interfaces']): + for ifname in config.list_nodes(['interfaces', iftype]): + migrate_interface(config, iftype, ifname) + + if config.exists(['interfaces', iftype, ifname, 'vif']): + for vif in config.list_nodes(['interfaces', iftype, ifname, 'vif']): + migrate_interface(config, iftype, ifname, vif=vif) + + if config.exists(['interfaces', iftype, ifname, 'vif-s']): + for vifs in config.list_nodes(['interfaces', iftype, ifname, 'vif-s']): + migrate_interface(config, iftype, ifname, vifs=vifs) + + if config.exists(['interfaces', iftype, ifname, 'vif-s', vifs, 'vif-c']): + for vifc in config.list_nodes(['interfaces', iftype, ifname, 'vif-s', vifs, 'vif-c']): + migrate_interface(config, iftype, ifname, vifs=vifs, vifc=vifc) + +try: + with open(file_name, 'w') as f: + f.write(config.to_string()) +except OSError as e: + print("Failed to save the modified config: {}".format(e)) + exit(1) diff --git a/src/op_mode/policy_route.py b/src/op_mode/policy_route.py index 5be40082f..5953786f3 100755 --- a/src/op_mode/policy_route.py +++ b/src/op_mode/policy_route.py @@ -22,53 +22,13 @@ from vyos.config import Config from vyos.util import cmd from vyos.util import dict_search_args -def get_policy_interfaces(conf, policy, name=None, ipv6=False): - interfaces = conf.get_config_dict(['interfaces'], key_mangling=('-', '_'), - get_first_key=True, no_tag_node_value_mangle=True) - - routes = ['route', 'route6'] - - def parse_if(ifname, if_conf): - if 'policy' in if_conf: - for route in routes: - if route in if_conf['policy']: - route_name = if_conf['policy'][route] - name_str = f'({ifname},{route})' - - if not name: - policy[route][route_name]['interface'].append(name_str) - elif not ipv6 and name == route_name: - policy['interface'].append(name_str) - - for iftype in ['vif', 'vif_s', 'vif_c']: - if iftype in if_conf: - for vifname, vif_conf in if_conf[iftype].items(): - parse_if(f'{ifname}.{vifname}', vif_conf) - - for iftype, iftype_conf in interfaces.items(): - for ifname, if_conf in iftype_conf.items(): - parse_if(ifname, if_conf) - -def get_config_policy(conf, name=None, ipv6=False, interfaces=True): +def get_config_policy(conf, name=None, ipv6=False): config_path = ['policy'] if name: config_path += ['route6' if ipv6 else 'route', name] policy = conf.get_config_dict(config_path, key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) - if policy and interfaces: - if name: - policy['interface'] = [] - else: - if 'route' in policy: - for route_name, route_conf in policy['route'].items(): - route_conf['interface'] = [] - - if 'route6' in policy: - for route_name, route_conf in policy['route6'].items(): - route_conf['interface'] = [] - - get_policy_interfaces(conf, policy, name, ipv6) return policy -- cgit v1.2.3 From a8daba9549668c45abf33c55bc5e68f4b9320f5e Mon Sep 17 00:00:00 2001 From: fett0 Date: Sun, 13 Nov 2022 17:05:00 +0000 Subject: l3VPN : T4182: add l3vpn over gre option from route-map --- data/templates/frr/policy.frr.j2 | 3 +++ interface-definitions/policy.xml.in | 22 +++++++++++++++++++++- smoketest/scripts/cli/test_policy.py | 5 +++++ 3 files changed, 29 insertions(+), 1 deletion(-) (limited to 'data') diff --git a/data/templates/frr/policy.frr.j2 b/data/templates/frr/policy.frr.j2 index 5ad4bd28c..9b5e80aed 100644 --- a/data/templates/frr/policy.frr.j2 +++ b/data/templates/frr/policy.frr.j2 @@ -322,6 +322,9 @@ route-map {{ route_map }} {{ rule_config.action }} {{ rule }} {% if rule_config.set.ipv6_next_hop.prefer_global is vyos_defined %} set ipv6 next-hop prefer-global {% endif %} +{% if rule_config.set.l3vpn_nexthop.encapsulation.gre is vyos_defined %} +set l3vpn next-hop encapsulation gre +{% endif %} {% if rule_config.set.large_community.replace is vyos_defined %} set large-community {{ rule_config.set.large_community.replace | join(' ') }} {% endif %} diff --git a/interface-definitions/policy.xml.in b/interface-definitions/policy.xml.in index 6c60276d5..ac774dc1f 100644 --- a/interface-definitions/policy.xml.in +++ b/interface-definitions/policy.xml.in @@ -1341,7 +1341,7 @@ - + Use peer address (for BGP only) @@ -1356,6 +1356,26 @@ + + + Next hop Information + + + + + Encapsulation options (for BGP only) + + + + + Accept L3VPN traffic over GRE encapsulation + + + + + + + BGP local preference attribute diff --git a/smoketest/scripts/cli/test_policy.py b/smoketest/scripts/cli/test_policy.py index 2166e63ec..3a4ef666a 100755 --- a/smoketest/scripts/cli/test_policy.py +++ b/smoketest/scripts/cli/test_policy.py @@ -1030,6 +1030,7 @@ class TestPolicy(VyOSUnitTestSHIM.TestCase): 'metric' : '150', 'metric-type' : 'type-1', 'origin' : 'incomplete', + 'l3vpn' : '', 'originator-id' : '172.16.10.1', 'src' : '100.0.0.1', 'tag' : '65530', @@ -1229,6 +1230,8 @@ class TestPolicy(VyOSUnitTestSHIM.TestCase): self.cli_set(path + ['rule', rule, 'set', 'ipv6-next-hop', 'local', rule_config['set']['ipv6-next-hop-local']]) if 'ip-next-hop' in rule_config['set']: self.cli_set(path + ['rule', rule, 'set', 'ip-next-hop', rule_config['set']['ip-next-hop']]) + if 'l3vpn' in rule_config['set']: + self.cli_set(path + ['rule', rule, 'set', 'l3vpn-nexthop', 'encapsulation', 'gre']) if 'local-preference' in rule_config['set']: self.cli_set(path + ['rule', rule, 'set', 'local-preference', rule_config['set']['local-preference']]) if 'metric' in rule_config['set']: @@ -1408,6 +1411,8 @@ class TestPolicy(VyOSUnitTestSHIM.TestCase): tmp += 'ipv6 next-hop global ' + rule_config['set']['ipv6-next-hop-global'] elif 'ipv6-next-hop-local' in rule_config['set']: tmp += 'ipv6 next-hop local ' + rule_config['set']['ipv6-next-hop-local'] + elif 'l3vpn' in rule_config['set']: + tmp += 'l3vpn next-hop encapsulation gre' elif 'local-preference' in rule_config['set']: tmp += 'local-preference ' + rule_config['set']['local-preference'] elif 'metric' in rule_config['set']: -- cgit v1.2.3 From da5bff2e835a14997d7b176670376cbd8d1221ef Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sun, 20 Nov 2022 16:58:43 +0100 Subject: op-mode: dns-forwarding: T4578: drop sudo calls Commit 66288ccfee ("dns-forwarding: T4578: Rewrite show dns forwarding") added the implementation for the new standardized op-mode definitions/implementation. As the API daemon has the proper permissions and also the CLI op-mode calls the script already with "sudo", there is no need to call "sudo" inside this script, again. Also add dns.py to data/op-mode-standardized.json for the GraphQL schema to be generated. --- data/op-mode-standardized.json | 1 + src/op_mode/dns.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'data') diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index b162f4097..1509975b4 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -6,6 +6,7 @@ "container.py", "cpu.py", "dhcp.py", +"dns.py", "log.py", "memory.py", "nat.py", diff --git a/src/op_mode/dns.py b/src/op_mode/dns.py index 9e5b1040c..a0e47d7ad 100755 --- a/src/op_mode/dns.py +++ b/src/op_mode/dns.py @@ -54,10 +54,10 @@ def _data_to_dict(data, sep="\t") -> dict: def _get_raw_forwarding_statistics() -> dict: - command = cmd('sudo /usr/bin/rec_control --socket-dir=/run/powerdns get-all') + command = cmd('rec_control --socket-dir=/run/powerdns get-all') data = _data_to_dict(command) data['cache-size'] = "{0:.2f}".format( int( - cmd('sudo /usr/bin/rec_control --socket-dir=/run/powerdns get cache-bytes')) / 1024 ) + cmd('rec_control --socket-dir=/run/powerdns get cache-bytes')) / 1024 ) return data -- cgit v1.2.3 From 2ac4a8a5fed9db471b7ffac0f54e6741c6f87834 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 21 Nov 2022 18:42:41 +0000 Subject: T4823: Fix IPsec transport mode remote TS Remote TS for transport mode GRE must be remote-address and not peer name --- data/templates/ipsec/swanctl/peer.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'data') diff --git a/data/templates/ipsec/swanctl/peer.j2 b/data/templates/ipsec/swanctl/peer.j2 index d097a04fc..837fa263c 100644 --- a/data/templates/ipsec/swanctl/peer.j2 +++ b/data/templates/ipsec/swanctl/peer.j2 @@ -124,7 +124,7 @@ {% endif %} {% elif tunnel_esp.mode == 'transport' %} local_ts = {{ peer_conf.local_address }}{{ local_suffix }} - remote_ts = {{ peer }}{{ remote_suffix }} + remote_ts = {{ peer_conf.remote_address | join(",") }}{{ remote_suffix }} {% endif %} ipcomp = {{ 'yes' if tunnel_esp.compression is vyos_defined else 'no' }} mode = {{ tunnel_esp.mode }} -- cgit v1.2.3 From 96b8107b437925b1b02399dedab8c3b887cf5093 Mon Sep 17 00:00:00 2001 From: Sander Klein Date: Wed, 23 Nov 2022 19:17:14 +0100 Subject: T4835: snmpd: Fix copy/paste error in snmpd.conf The variable 'client' was accidently used where 'network should have been used. This lead to missing community6 string when an IPv6 network was defined instead of an IPv6 client. --- data/templates/snmp/etc.snmpd.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'data') diff --git a/data/templates/snmp/etc.snmpd.conf.j2 b/data/templates/snmp/etc.snmpd.conf.j2 index d7dc0ba5d..57ad704c0 100644 --- a/data/templates/snmp/etc.snmpd.conf.j2 +++ b/data/templates/snmp/etc.snmpd.conf.j2 @@ -69,7 +69,7 @@ agentaddress unix:/run/snmpd.socket{{ ',' ~ options | join(',') if options is vy {% for network in comm_config.network %} {% if network | is_ipv4 %} {{ comm_config.authorization }}community {{ comm }} {{ network }} -{% elif client | is_ipv6 %} +{% elif network | is_ipv6 %} {{ comm_config.authorization }}community6 {{ comm }} {{ network }} {% endif %} {% endfor %} -- cgit v1.2.3 From f96731947f2f53a65171aace0bfafe497f523d59 Mon Sep 17 00:00:00 2001 From: Cheeze-It <16260577+Cheeze-It@users.noreply.github.com> Date: Sat, 26 Nov 2022 10:28:57 -0700 Subject: ospf: T4739: Adding missing OSPF FRR template Adding the parameters that were missing to the OSPF FRR template. --- data/templates/frr/ospfd.frr.j2 | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'data') diff --git a/data/templates/frr/ospfd.frr.j2 b/data/templates/frr/ospfd.frr.j2 index 2a8afefbc..882ec8f97 100644 --- a/data/templates/frr/ospfd.frr.j2 +++ b/data/templates/frr/ospfd.frr.j2 @@ -161,6 +161,12 @@ router ospf {{ 'vrf ' ~ vrf if vrf is vyos_defined }} {% if parameters.abr_type is vyos_defined %} ospf abr-type {{ parameters.abr_type }} {% endif %} +{% if parameters.opaque_lsa is vyos_defined %} + ospf opaque-lsa +{% endif %} +{% if parameters.rfc1583_compatibility is vyos_defined %} + ospf rfc1583compatibility +{% endif %} {% if parameters.router_id is vyos_defined %} ospf router-id {{ parameters.router_id }} {% endif %} -- cgit v1.2.3 From 1d12b24aa5b4d61d59f847e0002f291b66c7a737 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Mon, 28 Nov 2022 11:10:18 -0600 Subject: conf-mode: T4845: add external file for dict of config-mode dependencies --- data/config-mode-dependencies.json | 1 + python/vyos/configdep.py | 38 +++++++++++++++++++++++++++++--------- src/conf_mode/firewall.py | 9 +++------ 3 files changed, 33 insertions(+), 15 deletions(-) create mode 100644 data/config-mode-dependencies.json (limited to 'data') diff --git a/data/config-mode-dependencies.json b/data/config-mode-dependencies.json new file mode 100644 index 000000000..dd0efda10 --- /dev/null +++ b/data/config-mode-dependencies.json @@ -0,0 +1 @@ +{"firewall": {"group_resync": ["nat", "policy-route"]}} diff --git a/python/vyos/configdep.py b/python/vyos/configdep.py index e6b82ca93..ca05cb092 100644 --- a/python/vyos/configdep.py +++ b/python/vyos/configdep.py @@ -14,11 +14,15 @@ # along with this library. If not, see . import os +import json from inspect import stack from vyos.util import load_as_module +from vyos.defaults import directories +from vyos.configsource import VyOSError +from vyos import ConfigError -dependents = {} +dependent_func = {} def canon_name(name: str) -> str: return os.path.splitext(name)[0].replace('-', '_') @@ -30,9 +34,22 @@ def canon_name_of_path(path: str) -> str: def caller_name() -> str: return stack()[-1].filename -def run_config_mode_script(script: str, config): - from vyos.defaults import directories +def read_dependency_dict(): + path = os.path.join(directories['data'], + 'config-mode-dependencies.json') + with open(path) as f: + d = json.load(f) + return d + +def get_dependency_dict(config): + if hasattr(config, 'cached_dependency_dict'): + d = getattr(config, 'cached_dependency_dict') + else: + d = read_dependency_dict() + setattr(config, 'cached_dependency_dict', d) + return d +def run_config_mode_script(script: str, config): path = os.path.join(directories['conf_mode'], script) name = canon_name(script) mod = load_as_module(name, path) @@ -46,20 +63,23 @@ def run_config_mode_script(script: str, config): except (VyOSError, ConfigError) as e: raise ConfigError(repr(e)) -def def_closure(script: str, config): +def def_closure(target: str, config): + script = target + '.py' def func_impl(): run_config_mode_script(script, config) return func_impl -def set_dependent(target: str, config): +def set_dependents(case, config): + d = get_dependency_dict(config) k = canon_name_of_path(caller_name()) - l = dependents.setdefault(k, []) - func = def_closure(target, config) - l.append(func) + l = dependent_func.setdefault(k, []) + for target in d[k][case]: + func = def_closure(target, config) + l.append(func) def call_dependents(): k = canon_name_of_path(caller_name()) - l = dependents.get(k, []) + l = dependent_func.get(k, []) while l: f = l.pop(0) f() diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index 9fee20358..38a332be3 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -26,7 +26,7 @@ from vyos.config import Config from vyos.configdict import dict_merge from vyos.configdict import node_changed from vyos.configdiff import get_config_diff, Diff -from vyos.configdep import set_dependent, call_dependents +from vyos.configdep import set_dependents, call_dependents # from vyos.configverify import verify_interface_exists from vyos.firewall import fqdn_config_parse from vyos.firewall import geoip_update @@ -162,11 +162,8 @@ def get_config(config=None): firewall['group_resync'] = bool('group' in firewall or node_changed(conf, base + ['group'])) if firewall['group_resync']: - # Update nat as firewall groups were updated - set_dependent(nat_conf_script, conf) - # Update policy route as firewall groups were updated - set_dependent(policy_route_conf_script, conf) - + # Update nat and policy-route as firewall groups were updated + set_dependents('group_resync', conf) if 'config_trap' in firewall and firewall['config_trap'] == 'enable': diff = get_config_diff(conf) -- cgit v1.2.3 From 2ef945cbb00fb43ee60e442fd75798b513f34491 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 29 Nov 2022 16:23:53 -0600 Subject: pki: T4847: add config-mode script dependencies --- data/config-mode-dependencies.json | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'data') diff --git a/data/config-mode-dependencies.json b/data/config-mode-dependencies.json index dd0efda10..ad12cff87 100644 --- a/data/config-mode-dependencies.json +++ b/data/config-mode-dependencies.json @@ -1 +1,11 @@ -{"firewall": {"group_resync": ["nat", "policy-route"]}} +{ + "firewall": {"group_resync": ["nat", "policy-route"]}, + "pki": { + "ethernet": ["interfaces-ethernet"], + "openvpn": ["interfaces-openvpn"], + "https": ["https"], + "ipsec": ["vpn_ipsec"], + "openconnect": ["vpn_openconnect"], + "sstp": ["vpn_sstp"] + } +} -- cgit v1.2.3 From 579e9294075d82627997af41ac7895923c7e8d5d Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 2 Dec 2022 11:22:43 -0600 Subject: http-api: T4859: correct calling of script dependencies from http-api.py --- data/config-mode-dependencies.json | 1 + src/conf_mode/http-api.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'data') diff --git a/data/config-mode-dependencies.json b/data/config-mode-dependencies.json index ad12cff87..9e943ba2c 100644 --- a/data/config-mode-dependencies.json +++ b/data/config-mode-dependencies.json @@ -1,5 +1,6 @@ { "firewall": {"group_resync": ["nat", "policy-route"]}, + "http_api": {"https": ["https"]}, "pki": { "ethernet": ["interfaces-ethernet"], "openvpn": ["interfaces-openvpn"], diff --git a/src/conf_mode/http-api.py b/src/conf_mode/http-api.py index be80613c6..6328294c1 100755 --- a/src/conf_mode/http-api.py +++ b/src/conf_mode/http-api.py @@ -25,6 +25,7 @@ import vyos.defaults from vyos.config import Config from vyos.configdict import dict_merge +from vyos.configdep import set_dependents, call_dependents from vyos.template import render from vyos.util import cmd from vyos.util import call @@ -61,6 +62,11 @@ def get_config(config=None): else: conf = Config() + # reset on creation/deletion of 'api' node + https_base = ['service', 'https'] + if conf.exists(https_base): + set_dependents("https", conf) + base = ['service', 'https', 'api'] if not conf.exists(base): return None @@ -132,7 +138,7 @@ def apply(http_api): # Let uvicorn settle before restarting Nginx sleep(1) - cmd(f'{vyos_conf_scripts_dir}/https.py', raising=ConfigError) + call_dependents() if __name__ == '__main__': try: -- cgit v1.2.3 From e4befa4987404aecc83e3e48b3d52dd4b64f7d99 Mon Sep 17 00:00:00 2001 From: fett0 Date: Fri, 2 Dec 2022 21:38:36 +0000 Subject: T4854: route reflector allows to apply route-maps --- data/templates/frr/bgpd.frr.j2 | 3 +++ interface-definitions/include/bgp/protocol-common-config.xml.i | 6 ++++++ smoketest/scripts/cli/test_protocols_bgp.py | 2 ++ 3 files changed, 11 insertions(+) (limited to 'data') diff --git a/data/templates/frr/bgpd.frr.j2 b/data/templates/frr/bgpd.frr.j2 index e8d135c78..5febd7c66 100644 --- a/data/templates/frr/bgpd.frr.j2 +++ b/data/templates/frr/bgpd.frr.j2 @@ -517,6 +517,9 @@ router bgp {{ system_as }} {{ 'vrf ' ~ vrf if vrf is vyos_defined }} {% if parameters.network_import_check is vyos_defined %} bgp network import-check {% endif %} +{% if parameters.route_reflector_allow_outbound_policy is vyos_defined %} +bgp route-reflector allow-outbound-policy +{% endif %} {% if parameters.no_client_to_client_reflection is vyos_defined %} no bgp client-to-client reflection {% endif %} diff --git a/interface-definitions/include/bgp/protocol-common-config.xml.i b/interface-definitions/include/bgp/protocol-common-config.xml.i index 70176144d..fe192434d 100644 --- a/interface-definitions/include/bgp/protocol-common-config.xml.i +++ b/interface-definitions/include/bgp/protocol-common-config.xml.i @@ -1431,6 +1431,12 @@ + + + Route reflector client allow policy outbound + + + Disable client to client route reflection diff --git a/smoketest/scripts/cli/test_protocols_bgp.py b/smoketest/scripts/cli/test_protocols_bgp.py index d2dad8c1a..debc8270c 100755 --- a/smoketest/scripts/cli/test_protocols_bgp.py +++ b/smoketest/scripts/cli/test_protocols_bgp.py @@ -294,6 +294,7 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): self.cli_set(base_path + ['parameters', 'minimum-holdtime', min_hold_time]) self.cli_set(base_path + ['parameters', 'no-suppress-duplicates']) self.cli_set(base_path + ['parameters', 'reject-as-sets']) + self.cli_set(base_path + ['parameters', 'route-reflector-allow-outbound-policy']) self.cli_set(base_path + ['parameters', 'shutdown']) self.cli_set(base_path + ['parameters', 'suppress-fib-pending']) @@ -322,6 +323,7 @@ class TestProtocolsBGP(VyOSUnitTestSHIM.TestCase): self.assertIn(f' bgp bestpath peer-type multipath-relax', frrconfig) self.assertIn(f' bgp minimum-holdtime {min_hold_time}', frrconfig) self.assertIn(f' bgp reject-as-sets', frrconfig) + self.assertIn(f' bgp route-reflector allow-outbound-policy', frrconfig) self.assertIn(f' bgp shutdown', frrconfig) self.assertIn(f' bgp suppress-fib-pending', frrconfig) self.assertNotIn(f'bgp ebgp-requires-policy', frrconfig) -- cgit v1.2.3 From d846f000424522bc2e26d554ada61d0ae7e10ecc Mon Sep 17 00:00:00 2001 From: aapostoliuk Date: Tue, 6 Dec 2022 13:57:42 +0200 Subject: T4862: Added the generation config for webproxy domain-block Added the generation in the config file /etc/squid/squid.conf for command: set service webroxy domain-block --- data/templates/squid/squid.conf.j2 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'data') diff --git a/data/templates/squid/squid.conf.j2 b/data/templates/squid/squid.conf.j2 index 5781c883f..b953c8b18 100644 --- a/data/templates/squid/squid.conf.j2 +++ b/data/templates/squid/squid.conf.j2 @@ -24,7 +24,12 @@ acl Safe_ports port {{ port }} {% endfor %} {% endif %} acl CONNECT method CONNECT - +{% if domain_block is vyos_defined %} +{% for domain in domain_block %} +acl BLOCKDOMAIN dstdomain {{ domain }} +{% endfor %} +http_access deny BLOCKDOMAIN +{% endif %} {% if authentication is vyos_defined %} {% if authentication.children is vyos_defined %} auth_param basic children {{ authentication.children }} -- cgit v1.2.3 From 9fa4b761d027e2eee8a6fac587857548292261fb Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Thu, 8 Dec 2022 13:04:56 +0000 Subject: T4117: Fix for L2TP DAE CoA server configuration Fix l2tp dae server template and python config dict for correctlly handling Dynamic Authorization Extension server configuration --- data/templates/accel-ppp/l2tp.config.j2 | 3 +++ interface-definitions/vpn-l2tp.xml.in | 1 + src/conf_mode/vpn_l2tp.py | 34 ++++++++++++++++++++++++--------- 3 files changed, 29 insertions(+), 9 deletions(-) (limited to 'data') diff --git a/data/templates/accel-ppp/l2tp.config.j2 b/data/templates/accel-ppp/l2tp.config.j2 index 9eeaf7622..986f19656 100644 --- a/data/templates/accel-ppp/l2tp.config.j2 +++ b/data/templates/accel-ppp/l2tp.config.j2 @@ -88,6 +88,9 @@ verbose=1 {% for r in radius_server %} server={{ r.server }},{{ r.key }},auth-port={{ r.port }},acct-port={{ r.acct_port }},req-limit=0,fail-time={{ r.fail_time }} {% endfor %} +{% if radius_dynamic_author.server is vyos_defined %} +dae-server={{ radius_dynamic_author.server }}:{{ radius_dynamic_author.port }},{{ radius_dynamic_author.key }} +{% endif %} {% if radius_acct_inter_jitter %} acct-interim-jitter={{ radius_acct_inter_jitter }} {% endif %} diff --git a/interface-definitions/vpn-l2tp.xml.in b/interface-definitions/vpn-l2tp.xml.in index cb5900e0d..06ca4ece5 100644 --- a/interface-definitions/vpn-l2tp.xml.in +++ b/interface-definitions/vpn-l2tp.xml.in @@ -230,6 +230,7 @@ Port for Dynamic Authorization Extension server (DM/CoA) + 1700 diff --git a/src/conf_mode/vpn_l2tp.py b/src/conf_mode/vpn_l2tp.py index fd5a4acd8..c533ad404 100755 --- a/src/conf_mode/vpn_l2tp.py +++ b/src/conf_mode/vpn_l2tp.py @@ -26,7 +26,10 @@ from ipaddress import ip_network from vyos.config import Config from vyos.template import is_ipv4 from vyos.template import render -from vyos.util import call, get_half_cpus +from vyos.util import call +from vyos.util import get_half_cpus +from vyos.util import check_port_availability +from vyos.util import is_listen_port_bind_service from vyos import ConfigError from vyos import airbag @@ -64,7 +67,7 @@ default_config_data = { 'radius_source_address': '', 'radius_shaper_attr': '', 'radius_shaper_vendor': '', - 'radius_dynamic_author': '', + 'radius_dynamic_author': {}, 'wins': [], 'ip6_column': [], 'thread_cnt': get_half_cpus() @@ -205,21 +208,21 @@ def get_config(config=None): l2tp['radius_source_address'] = conf.return_value(['source-address']) # Dynamic Authorization Extensions (DOA)/Change Of Authentication (COA) - if conf.exists(['dynamic-author']): + if conf.exists(['dae-server']): dae = { 'port' : '', 'server' : '', 'key' : '' } - if conf.exists(['dynamic-author', 'server']): - dae['server'] = conf.return_value(['dynamic-author', 'server']) + if conf.exists(['dae-server', 'ip-address']): + dae['server'] = conf.return_value(['dae-server', 'ip-address']) - if conf.exists(['dynamic-author', 'port']): - dae['port'] = conf.return_value(['dynamic-author', 'port']) + if conf.exists(['dae-server', 'port']): + dae['port'] = conf.return_value(['dae-server', 'port']) - if conf.exists(['dynamic-author', 'key']): - dae['key'] = conf.return_value(['dynamic-author', 'key']) + if conf.exists(['dae-server', 'secret']): + dae['key'] = conf.return_value(['dae-server', 'secret']) l2tp['radius_dynamic_author'] = dae @@ -329,6 +332,19 @@ def verify(l2tp): if not radius['key']: raise ConfigError(f"Missing RADIUS secret for server { radius['key'] }") + if l2tp['radius_dynamic_author']: + if not l2tp['radius_dynamic_author']['server']: + raise ConfigError("Missing ip-address for dae-server") + if not l2tp['radius_dynamic_author']['key']: + raise ConfigError("Missing secret for dae-server") + address = l2tp['radius_dynamic_author']['server'] + port = l2tp['radius_dynamic_author']['port'] + proto = 'tcp' + # check if dae listen port is not used by another service + if check_port_availability(address, int(port), proto) is not True and \ + not is_listen_port_bind_service(int(port), 'accel-pppd'): + raise ConfigError(f'"{proto}" port "{port}" is used by another service') + # check for the existence of a client ip pool if not (l2tp['client_ip_pool'] or l2tp['client_ip_subnets']): raise ConfigError( -- cgit v1.2.3 From 056746bbbdc0cc139d20b9fecb807c78d04c6097 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Fri, 9 Dec 2022 10:04:34 +0000 Subject: T4868: Fix l2tp ppp IPv6 options in template and config get dict L2TP 'ppp-options ipv6 x' can work without declaring IPv6 pool As we can get addresses via RADIUS attributes: - Framed-IPv6-Prefix - Delegated-IPv6-Prefix --- data/templates/accel-ppp/l2tp.config.j2 | 6 ++++-- src/conf_mode/vpn_l2tp.py | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) (limited to 'data') diff --git a/data/templates/accel-ppp/l2tp.config.j2 b/data/templates/accel-ppp/l2tp.config.j2 index 986f19656..3d1e835a9 100644 --- a/data/templates/accel-ppp/l2tp.config.j2 +++ b/data/templates/accel-ppp/l2tp.config.j2 @@ -121,8 +121,10 @@ lcp-echo-failure={{ ppp_echo_failure }} {% if ccp_disable %} ccp=0 {% endif %} -{% if client_ipv6_pool %} -ipv6=allow +{% if ppp_ipv6 is vyos_defined %} +ipv6={{ ppp_ipv6 }} +{% else %} +{{ 'ipv6=allow' if client_ipv6_pool_configured else '' }} {% endif %} diff --git a/src/conf_mode/vpn_l2tp.py b/src/conf_mode/vpn_l2tp.py index c533ad404..27e78db99 100755 --- a/src/conf_mode/vpn_l2tp.py +++ b/src/conf_mode/vpn_l2tp.py @@ -46,6 +46,7 @@ default_config_data = { 'client_ip_pool': None, 'client_ip_subnets': [], 'client_ipv6_pool': [], + 'client_ipv6_pool_configured': False, 'client_ipv6_delegate_prefix': [], 'dnsv4': [], 'dnsv6': [], @@ -247,6 +248,7 @@ def get_config(config=None): l2tp['client_ip_subnets'] = conf.return_values(['client-ip-pool', 'subnet']) if conf.exists(['client-ipv6-pool', 'prefix']): + l2tp['client_ipv6_pool_configured'] = True l2tp['ip6_column'].append('ip6') for prefix in conf.list_nodes(['client-ipv6-pool', 'prefix']): tmp = { @@ -309,6 +311,9 @@ def get_config(config=None): if conf.exists(['ppp-options', 'lcp-echo-interval']): l2tp['ppp_echo_interval'] = conf.return_value(['ppp-options', 'lcp-echo-interval']) + if conf.exists(['ppp-options', 'ipv6']): + l2tp['ppp_ipv6'] = conf.return_value(['ppp-options', 'ipv6']) + return l2tp -- cgit v1.2.3 From ff56aeefddaad2d37d3ea32626e1adf3960eaf26 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sun, 11 Dec 2022 19:38:28 +0100 Subject: sstp: T4384: initial implementation of SSTP client CLI vyos@vyos# show interfaces sstpc sstpc sstpc10 { authentication { password vyos user vyos } server sstp.vyos.net ssl { ca-certificate VyOS-CA } } --- data/configd-include.json | 1 + data/templates/sstp-client/peer.j2 | 46 ++++++++ interface-definitions/interfaces-sstpc.xml.in | 47 ++++++++ op-mode-definitions/connect.xml.in | 1 + op-mode-definitions/disconnect.xml.in | 1 + op-mode-definitions/monitor-log.xml.in | 17 +++ op-mode-definitions/show-interfaces-sstpc.xml.in | 51 ++++++++ op-mode-definitions/show-log.xml.in | 17 +++ python/vyos/ifconfig/__init__.py | 3 +- python/vyos/ifconfig/sstpc.py | 40 +++++++ src/conf_mode/interfaces-sstpc.py | 142 +++++++++++++++++++++++ src/etc/ppp/ip-up.d/96-vyos-sstpc-callback | 49 ++++++++ src/op_mode/connect_disconnect.py | 4 +- 13 files changed, 416 insertions(+), 3 deletions(-) create mode 100644 data/templates/sstp-client/peer.j2 create mode 100644 interface-definitions/interfaces-sstpc.xml.in create mode 100644 op-mode-definitions/show-interfaces-sstpc.xml.in create mode 100644 python/vyos/ifconfig/sstpc.py create mode 100755 src/conf_mode/interfaces-sstpc.py create mode 100755 src/etc/ppp/ip-up.d/96-vyos-sstpc-callback (limited to 'data') diff --git a/data/configd-include.json b/data/configd-include.json index 5a4912e30..648655a8b 100644 --- a/data/configd-include.json +++ b/data/configd-include.json @@ -28,6 +28,7 @@ "interfaces-openvpn.py", "interfaces-pppoe.py", "interfaces-pseudo-ethernet.py", +"interfaces-sstpc.py", "interfaces-tunnel.py", "interfaces-vti.py", "interfaces-vxlan.py", diff --git a/data/templates/sstp-client/peer.j2 b/data/templates/sstp-client/peer.j2 new file mode 100644 index 000000000..1127d0564 --- /dev/null +++ b/data/templates/sstp-client/peer.j2 @@ -0,0 +1,46 @@ +### Autogenerated by interfaces-sstpc.py ### +{{ '# ' ~ description if description is vyos_defined else '' }} + +# Require peer to provide the local IP address if it is not +# specified explicitly in the config file. +noipdefault + +# Don't show the password in logfiles: +hide-password + +remotename {{ ifname }} +linkname {{ ifname }} +ipparam {{ ifname }} +ifname {{ ifname }} +pty "sstpc --ipparam {{ ifname }} --nolaunchpppd {{ server }}:{{ port }} --ca-cert {{ ca_file_path }}" + +# Override any connect script that may have been set in /etc/ppp/options. +connect /bin/true + +# Don't try to authenticate the remote node +noauth + +# We won't want EAP +refuse-eap + +# Don't try to proxy ARP for the remote endpoint. User can set proxy +# arp entries up manually if they wish. More importantly, having +# the "proxyarp" parameter set disables the "defaultroute" option. +noproxyarp + +# Unlimited connection attempts +maxfail 0 + +plugin sstp-pppd-plugin.so +sstp-sock /var/run/sstpc/sstpc-{{ ifname }} + +persist +debug + +{% if authentication is vyos_defined %} +{{ 'user "' + authentication.user + '"' if authentication.user is vyos_defined }} +{{ 'password "' + authentication.password + '"' if authentication.password is vyos_defined }} +{% endif %} + +{{ "usepeerdns" if no_peer_dns is not vyos_defined }} + diff --git a/interface-definitions/interfaces-sstpc.xml.in b/interface-definitions/interfaces-sstpc.xml.in new file mode 100644 index 000000000..30b55a9fa --- /dev/null +++ b/interface-definitions/interfaces-sstpc.xml.in @@ -0,0 +1,47 @@ + + + + + + + Secure Socket Tunneling Protocol (SSTP) client Interface + 460 + + sstpc[0-9]+ + + Secure Socket Tunneling Protocol interface must be named sstpcN + + sstpcN + Secure Socket Tunneling Protocol interface name + + + + #include + #include + #include + #include + #include + #include + #include + + 1452 + + #include + #include + + 443 + + + + Secure Sockets Layer (SSL) configuration + + + #include + + + #include + + + + + diff --git a/op-mode-definitions/connect.xml.in b/op-mode-definitions/connect.xml.in index d0c93195c..116cd6231 100644 --- a/op-mode-definitions/connect.xml.in +++ b/op-mode-definitions/connect.xml.in @@ -20,6 +20,7 @@ Bring up a connection-oriented network interface interfaces pppoe + interfaces sstpc interfaces wwan diff --git a/op-mode-definitions/disconnect.xml.in b/op-mode-definitions/disconnect.xml.in index 4415c0ed2..843998c4f 100644 --- a/op-mode-definitions/disconnect.xml.in +++ b/op-mode-definitions/disconnect.xml.in @@ -10,6 +10,7 @@ Take down a connection-oriented network interface interfaces pppoe + interfaces sstpc interfaces wwan diff --git a/op-mode-definitions/monitor-log.xml.in b/op-mode-definitions/monitor-log.xml.in index dccdfaf9a..1b1f53dc2 100644 --- a/op-mode-definitions/monitor-log.xml.in +++ b/op-mode-definitions/monitor-log.xml.in @@ -224,6 +224,23 @@ journalctl --no-hostname --boot --follow --unit ssh.service + + + Monitor last lines of SSTP client log + + journalctl --no-hostname --boot --follow --unit "ppp@sstpc*.service" + + + + Monitor last lines of SSTP client log for specific interface + + + + + journalctl --no-hostname --boot --follow --unit "ppp@$5.service" + + + Show log for Virtual Private Network (VPN) diff --git a/op-mode-definitions/show-interfaces-sstpc.xml.in b/op-mode-definitions/show-interfaces-sstpc.xml.in new file mode 100644 index 000000000..e66d3a0ac --- /dev/null +++ b/op-mode-definitions/show-interfaces-sstpc.xml.in @@ -0,0 +1,51 @@ + + + + + + + + + Show specified SSTP client interface information + + interfaces sstpc + + + ${vyos_op_scripts_dir}/show_interfaces.py --intf="$4" + + + + Show specified SSTP client interface log + + journalctl --no-hostname --boot --follow --unit "ppp@$4".service + + + + Show specified SSTP client interface statistics + + interfaces sstpc + + + if [ -d "/sys/class/net/$4" ]; then /usr/sbin/pppstats "$4"; fi + + + + + + Show SSTP client interface information + + ${vyos_op_scripts_dir}/show_interfaces.py --intf-type=sstpc --action=show-brief + + + + Show detailed SSTP client interface information + + ${vyos_op_scripts_dir}/show_interfaces.py --intf-type=sstpc --action=show + + + + + + + + diff --git a/op-mode-definitions/show-log.xml.in b/op-mode-definitions/show-log.xml.in index 404de1913..64a54015b 100644 --- a/op-mode-definitions/show-log.xml.in +++ b/op-mode-definitions/show-log.xml.in @@ -356,6 +356,23 @@ journalctl --no-hostname --boot --unit ssh.service + + + Show log for SSTP client + + journalctl --no-hostname --boot --unit "ppp@sstpc*.service" + + + + Show SSTP client log on specific interface + + + + + journalctl --no-hostname --boot --unit "ppp@$5.service" + + + Show last n changes to messages diff --git a/python/vyos/ifconfig/__init__.py b/python/vyos/ifconfig/__init__.py index d1ddaa13e..206b2bba1 100644 --- a/python/vyos/ifconfig/__init__.py +++ b/python/vyos/ifconfig/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 VyOS maintainers and contributors +# Copyright 2019-2022 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -38,3 +38,4 @@ from vyos.ifconfig.l2tpv3 import L2TPv3If from vyos.ifconfig.macsec import MACsecIf from vyos.ifconfig.veth import VethIf from vyos.ifconfig.wwan import WWANIf +from vyos.ifconfig.sstpc import SSTPCIf diff --git a/python/vyos/ifconfig/sstpc.py b/python/vyos/ifconfig/sstpc.py new file mode 100644 index 000000000..50fc6ee6b --- /dev/null +++ b/python/vyos/ifconfig/sstpc.py @@ -0,0 +1,40 @@ +# Copyright 2022 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . + +from vyos.ifconfig.interface import Interface + +@Interface.register +class SSTPCIf(Interface): + iftype = 'sstpc' + definition = { + **Interface.definition, + **{ + 'section': 'sstpc', + 'prefixes': ['sstpc', ], + 'eternal': 'sstpc[0-9]+$', + }, + } + + def _create(self): + # we can not create this interface as it is managed outside + pass + + def _delete(self): + # we can not create this interface as it is managed outside + pass + + def get_mac(self): + """ Get a synthetic MAC address. """ + return self.get_mac_synthetic() diff --git a/src/conf_mode/interfaces-sstpc.py b/src/conf_mode/interfaces-sstpc.py new file mode 100755 index 000000000..6b8094c51 --- /dev/null +++ b/src/conf_mode/interfaces-sstpc.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +from sys import exit + +from vyos.config import Config +from vyos.configdict import get_interface_dict +from vyos.configdict import is_node_changed +from vyos.configverify import verify_authentication +from vyos.configverify import verify_vrf +from vyos.ifconfig import SSTPCIf +from vyos.pki import encode_certificate +from vyos.pki import find_chain +from vyos.pki import load_certificate +from vyos.template import render +from vyos.util import call +from vyos.util import dict_search +from vyos.util import is_systemd_service_running +from vyos.util import write_file +from vyos import ConfigError +from vyos import airbag +airbag.enable() + +def get_config(config=None): + """ + Retrive CLI config as dictionary. Dictionary can never be empty, as at least the + interface name will be added or a deleted flag + """ + if config: + conf = config + else: + conf = Config() + base = ['interfaces', 'sstpc'] + ifname, sstpc = get_interface_dict(conf, base) + + # We should only terminate the SSTP client session if critical parameters + # change. All parameters that can be changed on-the-fly (like interface + # description) should not lead to a reconnect! + for options in ['authentication', 'no_peer_dns', 'no_default_route', + 'server', 'ssl']: + if is_node_changed(conf, base + [ifname, options]): + sstpc.update({'shutdown_required': {}}) + # bail out early - no need to further process other nodes + break + + # Load PKI certificates for later processing + sstpc['pki'] = conf.get_config_dict(['pki'], key_mangling=('-', '_'), + get_first_key=True, + no_tag_node_value_mangle=True) + return sstpc + +def verify(sstpc): + if 'deleted' in sstpc: + return None + + verify_authentication(sstpc) + verify_vrf(sstpc) + + if dict_search('ssl.ca_certificate', sstpc) == None: + raise ConfigError('Missing mandatory CA certificate!') + + return None + +def generate(sstpc): + ifname = sstpc['ifname'] + config_sstpc = f'/etc/ppp/peers/{ifname}' + + sstpc['ca_file_path'] = f'/run/sstpc/{ifname}_ca-cert.pem' + + if 'deleted' in sstpc: + for file in [sstpc['ca_file_path'], config_sstpc]: + if os.path.exists(file): + os.unlink(file) + return None + + ca_name = sstpc['ssl']['ca_certificate'] + pki_ca_cert = sstpc['pki']['ca'][ca_name] + + loaded_ca_cert = load_certificate(pki_ca_cert['certificate']) + loaded_ca_certs = {load_certificate(c['certificate']) + for c in sstpc['pki']['ca'].values()} if 'ca' in sstpc['pki'] else {} + + ca_full_chain = find_chain(loaded_ca_cert, loaded_ca_certs) + + write_file(sstpc['ca_file_path'], '\n'.join(encode_certificate(c) for c in ca_full_chain)) + render(config_sstpc, 'sstp-client/peer.j2', sstpc, permission=0o640) + + return None + +def apply(sstpc): + ifname = sstpc['ifname'] + if 'deleted' in sstpc or 'disable' in sstpc: + if os.path.isdir(f'/sys/class/net/{ifname}'): + p = SSTPCIf(ifname) + p.remove() + call(f'systemctl stop ppp@{ifname}.service') + return None + + # reconnect should only be necessary when specific options change, + # like server, authentication ... (see get_config() for details) + if ((not is_systemd_service_running(f'ppp@{ifname}.service')) or + 'shutdown_required' in sstpc): + + # cleanup system (e.g. FRR routes first) + if os.path.isdir(f'/sys/class/net/{ifname}'): + p = SSTPCIf(ifname) + p.remove() + + call(f'systemctl restart ppp@{ifname}.service') + # When interface comes "live" a hook is called: + # /etc/ppp/ip-up.d/96-vyos-sstpc-callback + # which triggers SSTPCIf.update() + else: + if os.path.isdir(f'/sys/class/net/{ifname}'): + p = SSTPCIf(ifname) + p.update(sstpc) + + return None + +if __name__ == '__main__': + try: + c = get_config() + verify(c) + generate(c) + apply(c) + except ConfigError as e: + print(e) + exit(1) diff --git a/src/etc/ppp/ip-up.d/96-vyos-sstpc-callback b/src/etc/ppp/ip-up.d/96-vyos-sstpc-callback new file mode 100755 index 000000000..4e8804f29 --- /dev/null +++ b/src/etc/ppp/ip-up.d/96-vyos-sstpc-callback @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This is a Python hook script which is invoked whenever a SSTP client session +# goes "ip-up". It will call into our vyos.ifconfig library and will then +# execute common tasks for the SSTP interface. The reason we have to "hook" this +# is that we can not create a sstpcX interface in advance in linux and then +# connect pppd to this already existing interface. + +from sys import argv +from sys import exit + +from vyos.configquery import ConfigTreeQuery +from vyos.configdict import get_interface_dict +from vyos.ifconfig import SSTPCIf + +# When the ppp link comes up, this script is called with the following +# parameters +# $1 the interface name used by pppd (e.g. ppp3) +# $2 the tty device name +# $3 the tty device speed +# $4 the local IP address for the interface +# $5 the remote IP address +# $6 the parameter specified by the 'ipparam' option to pppd + +if (len(argv) < 7): + exit(1) + +interface = argv[6] + +conf = ConfigTreeQuery() +_, sstpc = get_interface_dict(conf.config, ['interfaces', 'sstpc'], interface) + +# Update the config +p = SSTPCIf(interface) +p.update(sstpc) diff --git a/src/op_mode/connect_disconnect.py b/src/op_mode/connect_disconnect.py index 936c20bcb..d39e88bf3 100755 --- a/src/op_mode/connect_disconnect.py +++ b/src/op_mode/connect_disconnect.py @@ -41,7 +41,7 @@ def check_ppp_running(interface): def connect(interface): """ Connect dialer interface """ - if interface.startswith('ppp'): + if interface.startswith('pppoe') or interface.startswith('sstpc'): check_ppp_interface(interface) # Check if interface is already dialed if os.path.isdir(f'/sys/class/net/{interface}'): @@ -62,7 +62,7 @@ def connect(interface): def disconnect(interface): """ Disconnect dialer interface """ - if interface.startswith('ppp'): + if interface.startswith('pppoe') or interface.startswith('sstpc'): check_ppp_interface(interface) # Check if interface is already down -- cgit v1.2.3 From 6fb7c09670a80af2f839ccfd4bb94c8b0f745b95 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 8 Dec 2022 07:59:37 -0600 Subject: openvpn: T4770: add openvpn.py to op-mode-standardized.json --- data/op-mode-standardized.json | 1 + 1 file changed, 1 insertion(+) (limited to 'data') diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index 1509975b4..a69cf55e9 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -12,6 +12,7 @@ "nat.py", "neighbor.py", "openconnect.py", +"openvpn.py", "route.py", "system.py", "ipsec.py", -- cgit v1.2.3 From 932af7f098808009f626c788deb9e1d1c8bf3426 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 13 Jun 2022 15:40:11 +0000 Subject: routing: T1237: Add new feature failover route Failover route allows to install static routes to the kernel routing table only if required target or gateway is alive When target or gateway doesn't respond for ICMP/ARP checks this route deleted from the routing table Routes are marked as protocol 'failover' (rt_protos) cat /etc/iproute2/rt_protos.d/failover.conf 111 failover ip route add 203.0.113.1 metric 2 via 192.0.2.1 dev eth0 proto failover $ sudo ip route show proto failover 203.0.113.1 via 192.0.2.1 dev eth0 metric 1 So we can safely flush such routes --- .../protocols/systemd_vyos_failover_service.j2 | 11 ++ interface-definitions/protocols-failover.xml.in | 114 +++++++++++++ src/conf_mode/protocols_failover.py | 121 ++++++++++++++ src/helpers/vyos-failover.py | 184 +++++++++++++++++++++ 4 files changed, 430 insertions(+) create mode 100644 data/templates/protocols/systemd_vyos_failover_service.j2 create mode 100644 interface-definitions/protocols-failover.xml.in create mode 100755 src/conf_mode/protocols_failover.py create mode 100755 src/helpers/vyos-failover.py (limited to 'data') diff --git a/data/templates/protocols/systemd_vyos_failover_service.j2 b/data/templates/protocols/systemd_vyos_failover_service.j2 new file mode 100644 index 000000000..e6501e0f5 --- /dev/null +++ b/data/templates/protocols/systemd_vyos_failover_service.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Failover route service +After=vyos-router.service + +[Service] +Type=simple +Restart=always +ExecStart=/usr/bin/python3 /usr/libexec/vyos/vyos-failover.py --config /run/vyos-failover.conf + +[Install] +WantedBy=multi-user.target diff --git a/interface-definitions/protocols-failover.xml.in b/interface-definitions/protocols-failover.xml.in new file mode 100644 index 000000000..900c76eab --- /dev/null +++ b/interface-definitions/protocols-failover.xml.in @@ -0,0 +1,114 @@ + + + + + + + Failover Routing + 490 + + + + + Failover IPv4 route + + ipv4net + IPv4 failover route + + + + + + + + + Next-hop IPv4 router address + + ipv4 + Next-hop router address + + + + + + + + + Check target options + + + #include + + + Check target address + + ipv4 + Address to check + + + + + + + + + Timeout between checks + + u32:1-300 + Timeout in seconds between checks + + + + + + 10 + + + + Check type + + arp icmp tcp + + + arp + Check target by ARP + + + icmp + Check target by ICMP + + + tcp + Check target by TCP + + + (arp|icmp|tcp) + + + icmp + + + + #include + + + Route metric for this gateway + + u32:1-255 + Route metric + + + + + + 1 + + + + + + + + + + diff --git a/src/conf_mode/protocols_failover.py b/src/conf_mode/protocols_failover.py new file mode 100755 index 000000000..048ba7a89 --- /dev/null +++ b/src/conf_mode/protocols_failover.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import json + +from pathlib import Path + +from vyos.config import Config +from vyos.configdict import dict_merge +from vyos.template import render +from vyos.util import call +from vyos.xml import defaults +from vyos import ConfigError +from vyos import airbag + +airbag.enable() + + +service_name = 'vyos-failover' +service_conf = Path(f'/run/{service_name}.conf') +systemd_service = '/etc/systemd/system/vyos-failover.service' +rt_proto_failover = '/etc/iproute2/rt_protos.d/failover.conf' + + +def get_config(config=None): + if config: + conf = config + else: + conf = Config() + + base = ['protocols', 'failover'] + failover = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) + + # Set default values only if we set config + if failover.get('route'): + for route, route_config in failover.get('route').items(): + for next_hop, next_hop_config in route_config.get('next_hop').items(): + default_values = defaults(base + ['route']) + failover['route'][route]['next_hop'][next_hop] = dict_merge( + default_values['next_hop'], failover['route'][route]['next_hop'][next_hop]) + + return failover + +def verify(failover): + # bail out early - looks like removal from running config + if not failover: + return None + + if 'route' not in failover: + raise ConfigError(f'Failover "route" is mandatory!') + + for route, route_config in failover['route'].items(): + if not route_config.get('next_hop'): + raise ConfigError(f'Next-hop for "{route}" is mandatory!') + + for next_hop, next_hop_config in route_config.get('next_hop').items(): + if 'interface' not in next_hop_config: + raise ConfigError(f'Interface for route "{route}" next-hop "{next_hop}" is mandatory!') + + if not next_hop_config.get('check'): + raise ConfigError(f'Check target for next-hop "{next_hop}" is mandatory!') + + if 'target' not in next_hop_config['check']: + raise ConfigError(f'Check target for next-hop "{next_hop}" is mandatory!') + + check_type = next_hop_config['check']['type'] + if check_type == 'tcp' and 'port' not in next_hop_config['check']: + raise ConfigError(f'Check port for next-hop "{next_hop}" and type TCP is mandatory!') + + return None + +def generate(failover): + if not failover: + service_conf.unlink(missing_ok=True) + return None + + # Add own rt_proto 'failover' + # Helps to detect all own routes 'proto failover' + with open(rt_proto_failover, 'w') as f: + f.write('111 failover\n') + + # Write configuration file + conf_json = json.dumps(failover, indent=4) + service_conf.write_text(conf_json) + render(systemd_service, 'protocols/systemd_vyos_failover_service.j2', failover) + + return None + +def apply(failover): + if not failover: + call(f'systemctl stop {service_name}.service') + call('ip route flush protocol failover') + else: + call('systemctl daemon-reload') + call(f'systemctl restart {service_name}.service') + call(f'ip route flush protocol failover') + + return None + +if __name__ == '__main__': + try: + c = get_config() + verify(c) + generate(c) + apply(c) + except ConfigError as e: + print(e) + exit(1) diff --git a/src/helpers/vyos-failover.py b/src/helpers/vyos-failover.py new file mode 100755 index 000000000..1ac193423 --- /dev/null +++ b/src/helpers/vyos-failover.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import argparse +import json +import subprocess +import socket +import time + +from vyos.util import rc_cmd +from pathlib import Path +from systemd import journal + + +my_name = Path(__file__).stem + + +def get_best_route_options(route, debug=False): + """ + Return current best route ('gateway, interface, metric) + + % get_best_route_options('203.0.113.1') + ('192.168.0.1', 'eth1', 1) + + % get_best_route_options('203.0.113.254') + (None, None, None) + """ + rc, data = rc_cmd(f'ip --detail --json route show protocol failover {route}') + if rc == 0: + data = json.loads(data) + if len(data) == 0: + print(f'\nRoute {route} for protocol failover was not found') + return None, None, None + # Fake metric 999 by default + # Search route with the lowest metric + best_metric = 999 + for entry in data: + if debug: print('\n', entry) + metric = entry.get('metric') + gateway = entry.get('gateway') + iface = entry.get('dev') + if metric < best_metric: + best_metric = metric + best_gateway = gateway + best_interface = iface + if debug: + print(f'### Best_route exists: {route}, best_gateway: {best_gateway}, ' + f'best_metric: {best_metric}, best_iface: {best_interface}') + return best_gateway, best_interface, best_metric + +def is_port_open(ip, port): + """ + Check connection to remote host and port + Return True if host alive + + % is_port_open('example.com', 8080) + True + """ + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) + s.settimeout(2) + try: + s.connect((ip, int(port))) + s.shutdown(socket.SHUT_RDWR) + return True + except: + return False + finally: + s.close() + +def is_target_alive(target=None, iface='', proto='icmp', port=None, debug=False): + """ + Host availability check by ICMP, ARP, TCP + Return True if target checks is successful + + % is_target_alive('192.0.2.1', 'eth1', proto='arp') + True + """ + if iface != '': + iface = f'-I {iface}' + if proto == 'icmp': + command = f'/usr/bin/ping -q {target} {iface} -n -c 2 -W 1' + rc, response = rc_cmd(command) + if debug: print(f' [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]') + if rc == 0: + return True + elif proto == 'arp': + command = f'/usr/bin/arping -b -c 2 -f -w 1 -i 1 {iface} {target}' + rc, response = rc_cmd(command) + if debug: print(f' [ CHECK-TARGET ]: [{command}] -- return-code [RC: {rc}]') + if rc == 0: + return True + elif proto == 'tcp' and port is not None: + return True if is_port_open(target, port) else False + else: + return False + + +if __name__ == '__main__': + # Parse command arguments and get config + parser = argparse.ArgumentParser() + parser.add_argument('-c', + '--config', + action='store', + help='Path to protocols failover configuration', + required=True, + type=Path) + + args = parser.parse_args() + try: + config_path = Path(args.config) + config = json.loads(config_path.read_text()) + except Exception as err: + print( + f'Configuration file "{config_path}" does not exist or malformed: {err}' + ) + exit(1) + + # Useful debug info to console, use debug = True + # sudo systemctl stop vyos-failover.service + # sudo /usr/libexec/vyos/vyos-failover.py --config /run/vyos-failover.conf + debug = False + + while(True): + + for route, route_config in config.get('route').items(): + + exists_route = exists_gateway, exists_iface, exists_metric = get_best_route_options(route, debug=debug) + + for next_hop, nexthop_config in route_config.get('next_hop').items(): + conf_iface = nexthop_config.get('interface') + conf_metric = int(nexthop_config.get('metric')) + port = nexthop_config.get('check').get('port') + port_opt = f'port {port}' if port else '' + proto = nexthop_config.get('check').get('type') + target = nexthop_config.get('check').get('target') + timeout = nexthop_config.get('check').get('timeout') + + # Best route not fonund in the current routing table + if exists_route == (None, None, None): + if debug: print(f" [NEW_ROUTE_DETECTED] route: [{route}]") + # Add route if check-target alive + if is_target_alive(target, conf_iface, proto, port, debug=debug): + if debug: print(f' [ ADD ] -- ip route add {route} via {next_hop} dev {conf_iface} ' + f'metric {conf_metric} proto failover\n###') + rc, command = rc_cmd(f'ip route add {route} via {next_hop} dev {conf_iface} ' + f'metric {conf_metric} proto failover') + # If something is wrong and gateway not added + # Example: Error: Next-hop has invalid gateway. + if rc !=0: + if debug: print(f'{command} -- return-code [RC: {rc}] {next_hop} dev {conf_iface}') + else: + journal.send(f'ip route add {route} via {next_hop} dev {conf_iface} ' + f'metric {conf_metric} proto failover', SYSLOG_IDENTIFIER=my_name) + else: + if debug: print(f' [ TARGET_FAIL ] target checks fails for [{target}], do nothing') + journal.send(f'Check fail for route {route} target {target} proto {proto} ' + f'{port_opt}', SYSLOG_IDENTIFIER=my_name) + + # Route was added, check if the target is alive + # We should delete route if check fails only if route exists in the routing table + if not is_target_alive(target, conf_iface, proto, port, debug=debug) and \ + exists_route != (None, None, None): + if debug: + print(f'Nexh_hop {next_hop} fail, target not response') + print(f' [ DEL ] -- ip route del {route} via {next_hop} dev {conf_iface} ' + f'metric {conf_metric} proto failover [DELETE]') + rc_cmd(f'ip route del {route} via {next_hop} dev {conf_iface} metric {conf_metric} proto failover') + journal.send(f'ip route del {route} via {next_hop} dev {conf_iface} ' + f'metric {conf_metric} proto failover', SYSLOG_IDENTIFIER=my_name) + + time.sleep(int(timeout)) -- cgit v1.2.3 From 03d49fe0fc5020b851bd639a2cb94f911f48cbea Mon Sep 17 00:00:00 2001 From: Sander Klein Date: Fri, 16 Dec 2022 21:58:06 +0100 Subject: T4884: snmpd: add community6 fallback If no client and network is defined only a `community` config is created. This also adds the `community6` part --- data/templates/snmp/etc.snmpd.conf.j2 | 1 + 1 file changed, 1 insertion(+) (limited to 'data') diff --git a/data/templates/snmp/etc.snmpd.conf.j2 b/data/templates/snmp/etc.snmpd.conf.j2 index 57ad704c0..47bf6878f 100644 --- a/data/templates/snmp/etc.snmpd.conf.j2 +++ b/data/templates/snmp/etc.snmpd.conf.j2 @@ -76,6 +76,7 @@ agentaddress unix:/run/snmpd.socket{{ ',' ~ options | join(',') if options is vy {% endif %} {% if comm_config.client is not vyos_defined and comm_config.network is not vyos_defined %} {{ comm_config.authorization }}community {{ comm }} +{{ comm_config.authorization }}community6 {{ comm }} {% endif %} {% endfor %} {% endif %} -- cgit v1.2.3 From 13071a4a534b2101120e0c007f0fbb5174b79052 Mon Sep 17 00:00:00 2001 From: Sander Klein Date: Fri, 16 Dec 2022 22:55:57 +0100 Subject: T4809: radvd: Allow the use of AdvRASrcAddress This add the AdvRASrcAddress configuration option to configure a source address for the router advertisements. The source address still must be configured on the system. This is useful for VRRP setups where you want fe80::1 on the VRRP interface for cleaner VRRP failovers. --- data/templates/router-advert/radvd.conf.j2 | 7 +++++++ interface-definitions/service-router-advert.xml.in | 13 +++++++++++++ 2 files changed, 20 insertions(+) (limited to 'data') diff --git a/data/templates/router-advert/radvd.conf.j2 b/data/templates/router-advert/radvd.conf.j2 index a464795ad..f4b384958 100644 --- a/data/templates/router-advert/radvd.conf.j2 +++ b/data/templates/router-advert/radvd.conf.j2 @@ -43,6 +43,13 @@ interface {{ iface }} { }; {% endfor %} {% endif %} +{% if iface_config.source_address is vyos_defined %} + AdvRASrcAddress { +{% for source_address in iface_config.source_address %} + {{ source_address }} +{% endfor %} + }; +{% endif %} {% if iface_config.prefix is vyos_defined %} {% for prefix, prefix_options in iface_config.prefix.items() %} prefix {{ prefix }} { diff --git a/interface-definitions/service-router-advert.xml.in b/interface-definitions/service-router-advert.xml.in index 87ec512d6..8b7364a8c 100644 --- a/interface-definitions/service-router-advert.xml.in +++ b/interface-definitions/service-router-advert.xml.in @@ -305,6 +305,19 @@ + + + Use IPv6 address as source address. Useful with VRRP. + + ipv6 + IPv6 address to be advertized (must be configured on interface) + + + + + + + Time, in milliseconds, that a node assumes a neighbor is reachable after having received a reachability confirmation -- cgit v1.2.3 From e78235213c7409ae0ddb50edc1ba83095d1c9080 Mon Sep 17 00:00:00 2001 From: aapostoliuk <108394744+aapostoliuk@users.noreply.github.com> Date: Sat, 17 Dec 2022 09:20:56 +0200 Subject: webproxy: T3810: multiple squidGuard fixes 1. Added in script update webproxy blacklists generation of all DBs 2. Fixed: if the blacklist category does not have generated db, the template generates an empty dest category in squidGuard.conf and a Warning message. 3. Added template generation for local's categories in the rule section. 4. Changed syntax in the generation dest section for blacklist's categories 4. Fixed generation dest local sections in squidGuard.conf 5. Fixed bug in syntax. The word 'allow' changed to the word 'any' in acl squidGuard.conf --- data/templates/squid/sg_acl.conf.j2 | 1 - data/templates/squid/squidGuard.conf.j2 | 122 ++++++++++++++++++++++++++----- src/conf_mode/service_webproxy.py | 100 +++++++++++++++++++------ src/op_mode/webproxy_update_blacklist.sh | 27 +++++++ 4 files changed, 205 insertions(+), 45 deletions(-) (limited to 'data') diff --git a/data/templates/squid/sg_acl.conf.j2 b/data/templates/squid/sg_acl.conf.j2 index ce72b173a..78297a2b8 100644 --- a/data/templates/squid/sg_acl.conf.j2 +++ b/data/templates/squid/sg_acl.conf.j2 @@ -1,6 +1,5 @@ ### generated by service_webproxy.py ### dbhome {{ squidguard_db_dir }} - dest {{ category }}-{{ rule }} { {% if list_type == 'domains' %} domainlist {{ category }}/domains diff --git a/data/templates/squid/squidGuard.conf.j2 b/data/templates/squid/squidGuard.conf.j2 index 1bc4c984f..a93f878df 100644 --- a/data/templates/squid/squidGuard.conf.j2 +++ b/data/templates/squid/squidGuard.conf.j2 @@ -1,10 +1,16 @@ ### generated by service_webproxy.py ### -{% macro sg_rule(category, log, db_dir) %} +{% macro sg_rule(category, rule, log, db_dir) %} +{% set domains = db_dir + '/' + category + '/domains' %} +{% set urls = db_dir + '/' + category + '/urls' %} {% set expressions = db_dir + '/' + category + '/expressions' %} -dest {{ category }}-default { +dest {{ category }}-{{ rule }}{ +{% if domains | is_file %} domainlist {{ category }}/domains +{% endif %} +{% if urls | is_file %} urllist {{ category }}/urls +{% endif %} {% if expressions | is_file %} expressionlist {{ category }}/expressions {% endif %} @@ -17,8 +23,9 @@ dest {{ category }}-default { {% if url_filtering is vyos_defined and url_filtering.disable is not vyos_defined %} {% if url_filtering.squidguard is vyos_defined %} {% set sg_config = url_filtering.squidguard %} -{% set acl = namespace(value='local-ok-default') %} +{% set acl = namespace(value='') %} {% set acl.value = acl.value + ' !in-addr' if sg_config.allow_ipaddr_url is not defined else acl.value %} +{% set ruleacls = {} %} dbhome {{ squidguard_db_dir }} logdir /var/log/squid @@ -38,24 +45,28 @@ dest local-ok-default { domainlist local-ok-default/domains } {% endif %} + {% if sg_config.local_ok_url is vyos_defined %} {% set acl.value = acl.value + ' local-ok-url-default' %} dest local-ok-url-default { urllist local-ok-url-default/urls } {% endif %} + {% if sg_config.local_block is vyos_defined %} {% set acl.value = acl.value + ' !local-block-default' %} dest local-block-default { domainlist local-block-default/domains } {% endif %} + {% if sg_config.local_block_url is vyos_defined %} {% set acl.value = acl.value + ' !local-block-url-default' %} dest local-block-url-default { urllist local-block-url-default/urls } {% endif %} + {% if sg_config.local_block_keyword is vyos_defined %} {% set acl.value = acl.value + ' !local-block-keyword-default' %} dest local-block-keyword-default { @@ -65,16 +76,100 @@ dest local-block-keyword-default { {% if sg_config.block_category is vyos_defined %} {% for category in sg_config.block_category %} -{{ sg_rule(category, sg_config.log, squidguard_db_dir) }} +{{ sg_rule(category, 'default', sg_config.log, squidguard_db_dir) }} {% set acl.value = acl.value + ' !' + category + '-default' %} {% endfor %} {% endif %} {% if sg_config.allow_category is vyos_defined %} {% for category in sg_config.allow_category %} -{{ sg_rule(category, False, squidguard_db_dir) }} +{{ sg_rule(category, 'default', False, squidguard_db_dir) }} {% set acl.value = acl.value + ' ' + category + '-default' %} {% endfor %} {% endif %} + + +{% if sg_config.rule is vyos_defined %} +{% for rule, rule_config in sg_config.rule.items() %} +{% if rule_config.local_ok is vyos_defined %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' local-ok-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'local-ok-' + rule}) %} +{% endif %} +dest local-ok-{{ rule }} { + domainlist local-ok-{{ rule }}/domains +} +{% endif %} + +{% if rule_config.local_ok_url is vyos_defined %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' local-ok-url-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'local-ok-url-' + rule}) %} +{% endif %} +dest local-ok-url-{{ rule }} { + urllist local-ok-url-{{ rule }}/urls +} +{% endif %} + +{% if rule_config.local_block is vyos_defined %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'!local-block-' + rule}) %} +{% endif %} +dest local-block-{{ rule }} { + domainlist local-block-{{ rule }}/domains +} +{% endif %} + +{% if rule_config.local_block_url is vyos_defined %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-url-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'!ocal-block-url-' + rule}) %} +{% endif %} +dest local-block-url-{{ rule }} { + urllist local-block-url-{{ rule }}/urls +} +{% endif %} + +{% if rule_config.local_block_keyword is vyos_defined %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-keyword-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'!local-block-keyword-' + rule}) %} +{% endif %} +dest local-block-keyword-{{ rule }} { + expressionlist local-block-keyword-{{ rule }}/expressions +} +{% endif %} + +{% if rule_config.block_category is vyos_defined %} +{% for b_category in rule_config.block_category %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !' + b_category + '-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'!' + b_category + '-' + rule}) %} +{% endif %} +{{ sg_rule(b_category, rule, sg_config.log, squidguard_db_dir) }} +{% endfor %} +{% endif %} + +{% if rule_config.allow_category is vyos_defined %} +{% for a_category in rule_config.allow_category %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' ' + a_category + '-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:a_category + '-' + rule}) %} +{% endif %} +{{ sg_rule(a_category, rule, sg_config.log, squidguard_db_dir) }} +{% endfor %} +{% endif %} +{% endfor %} +{% endif %} + + {% if sg_config.source_group is vyos_defined %} {% for sgroup, sg_config in sg_config.source_group.items() %} {% if sg_config.address is vyos_defined %} @@ -83,28 +178,15 @@ src {{ sgroup }} { ip {{ address }} {% endfor %} } - {% endif %} {% endfor %} {% endif %} -{% if sg_config.rule is vyos_defined %} -{% for rule, rule_config in sg_config.rule.items() %} -{% for b_category in rule_config.block_category %} -dest {{ b_category }} { - domainlist {{ b_category }}/domains - urllist {{ b_category }}/urls -} -{% endfor %} -{% endfor %} -{% endif %} acl { {% if sg_config.rule is vyos_defined %} {% for rule, rule_config in sg_config.rule.items() %} {{ rule_config.source_group }} { -{% for b_category in rule_config.block_category %} - pass local-ok-1 !in-addr !{{ b_category }} all -{% endfor %} + pass {{ ruleacls[rule] }} {{ 'none' if rule_config.default_action is vyos_defined('block') else 'any' }} } {% endfor %} {% endif %} @@ -113,7 +195,7 @@ acl { {% if sg_config.enable_safe_search is vyos_defined %} rewrite safesearch {% endif %} - pass {{ acl.value }} {{ 'none' if sg_config.default_action is vyos_defined('block') else 'allow' }} + pass {{ acl.value }} {{ 'none' if sg_config.default_action is vyos_defined('block') else 'any' }} redirect 302:http://{{ sg_config.redirect_url }} {% if sg_config.log is vyos_defined %} log blacklist.log diff --git a/src/conf_mode/service_webproxy.py b/src/conf_mode/service_webproxy.py index 32af31bde..41a1deaa3 100755 --- a/src/conf_mode/service_webproxy.py +++ b/src/conf_mode/service_webproxy.py @@ -28,8 +28,10 @@ from vyos.util import dict_search from vyos.util import write_file from vyos.validate import is_addr_assigned from vyos.xml import defaults +from vyos.base import Warning from vyos import ConfigError from vyos import airbag + airbag.enable() squid_config_file = '/etc/squid/squid.conf' @@ -37,24 +39,57 @@ squidguard_config_file = '/etc/squidguard/squidGuard.conf' squidguard_db_dir = '/opt/vyatta/etc/config/url-filtering/squidguard/db' user_group = 'proxy' -def generate_sg_localdb(category, list_type, role, proxy): + +def check_blacklist_categorydb(config_section): + if 'block_category' in config_section: + for category in config_section['block_category']: + check_categorydb(category) + if 'allow_category' in config_section: + for category in config_section['allow_category']: + check_categorydb(category) + + +def check_categorydb(category: str): + """ + Check if category's db exist + :param category: + :type str: + """ + path_to_cat: str = f'{squidguard_db_dir}/{category}' + if not os.path.exists(f'{path_to_cat}/domains.db') \ + and not os.path.exists(f'{path_to_cat}/urls.db') \ + and not os.path.exists(f'{path_to_cat}/expressions.db'): + Warning(f'DB of category {category} does not exist.\n ' + f'Use [update webproxy blacklists] ' + f'or delete undefined category!') + + +def generate_sg_rule_localdb(category, list_type, role, proxy): + if not category or not list_type or not role: + return None + cat_ = category.replace('-', '_') - if isinstance(dict_search(f'url_filtering.squidguard.{cat_}', proxy), - list): + if role == 'default': + path_to_cat = f'{cat_}' + else: + path_to_cat = f'rule.{role}.{cat_}' + if isinstance( + dict_search(f'url_filtering.squidguard.{path_to_cat}', proxy), + list): # local block databases must be generated "on-the-fly" tmp = { - 'squidguard_db_dir' : squidguard_db_dir, - 'category' : f'{category}-default', - 'list_type' : list_type, - 'rule' : role + 'squidguard_db_dir': squidguard_db_dir, + 'category': f'{category}-{role}', + 'list_type': list_type, + 'rule': role } sg_tmp_file = '/tmp/sg.conf' - db_file = f'{category}-default/{list_type}' - domains = '\n'.join(dict_search(f'url_filtering.squidguard.{cat_}', proxy)) - + db_file = f'{category}-{role}/{list_type}' + domains = '\n'.join( + dict_search(f'url_filtering.squidguard.{path_to_cat}', proxy)) # local file - write_file(f'{squidguard_db_dir}/{category}-default/local', '', + write_file(f'{squidguard_db_dir}/{category}-{role}/local', '', user=user_group, group=user_group) # database input file write_file(f'{squidguard_db_dir}/{db_file}', domains, @@ -64,17 +99,18 @@ def generate_sg_localdb(category, list_type, role, proxy): render(sg_tmp_file, 'squid/sg_acl.conf.j2', tmp, user=user_group, group=user_group) - call(f'su - {user_group} -c "squidGuard -d -c {sg_tmp_file} -C {db_file}"') + call( + f'su - {user_group} -c "squidGuard -d -c {sg_tmp_file} -C {db_file}"') if os.path.exists(sg_tmp_file): os.unlink(sg_tmp_file) - else: # if category is not part of our configuration, clean out the # squidguard lists - tmp = f'{squidguard_db_dir}/{category}-default' + tmp = f'{squidguard_db_dir}/{category}-{role}' if os.path.exists(tmp): - rmtree(f'{squidguard_db_dir}/{category}-default') + rmtree(f'{squidguard_db_dir}/{category}-{role}') + def get_config(config=None): if config: @@ -85,7 +121,8 @@ def get_config(config=None): if not conf.exists(base): return None - proxy = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) + proxy = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True) # We have gathered the dict representation of the CLI, but there are default # options which we need to update into the dictionary retrived. default_values = defaults(base) @@ -110,10 +147,11 @@ def get_config(config=None): default_values = defaults(base + ['cache-peer']) for peer in proxy['cache_peer']: proxy['cache_peer'][peer] = dict_merge(default_values, - proxy['cache_peer'][peer]) + proxy['cache_peer'][peer]) return proxy + def verify(proxy): if not proxy: return None @@ -170,17 +208,30 @@ def generate(proxy): render(squidguard_config_file, 'squid/squidGuard.conf.j2', proxy) cat_dict = { - 'local-block' : 'domains', - 'local-block-keyword' : 'expressions', - 'local-block-url' : 'urls', - 'local-ok' : 'domains', - 'local-ok-url' : 'urls' + 'local-block': 'domains', + 'local-block-keyword': 'expressions', + 'local-block-url': 'urls', + 'local-ok': 'domains', + 'local-ok-url': 'urls' } - for category, list_type in cat_dict.items(): - generate_sg_localdb(category, list_type, 'default', proxy) + if dict_search(f'url_filtering.squidguard', proxy) is not None: + squidgard_config_section = proxy['url_filtering']['squidguard'] + + for category, list_type in cat_dict.items(): + generate_sg_rule_localdb(category, list_type, 'default', proxy) + check_blacklist_categorydb(squidgard_config_section) + + if 'rule' in squidgard_config_section: + for rule in squidgard_config_section['rule']: + rule_config_section = squidgard_config_section['rule'][ + rule] + for category, list_type in cat_dict.items(): + generate_sg_rule_localdb(category, list_type, rule, proxy) + check_blacklist_categorydb(rule_config_section) return None + def apply(proxy): if not proxy: # proxy is removed in the commit @@ -198,6 +249,7 @@ def apply(proxy): call('systemctl restart squid.service') return None + if __name__ == '__main__': try: c = get_config() diff --git a/src/op_mode/webproxy_update_blacklist.sh b/src/op_mode/webproxy_update_blacklist.sh index d5f301b75..4fb9a54c6 100755 --- a/src/op_mode/webproxy_update_blacklist.sh +++ b/src/op_mode/webproxy_update_blacklist.sh @@ -18,6 +18,23 @@ blacklist_url='ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/black data_dir="/opt/vyatta/etc/config/url-filtering" archive="${data_dir}/squidguard/archive" db_dir="${data_dir}/squidguard/db" +conf_file="/etc/squidguard/squidGuard.conf" +tmp_conf_file="/tmp/sg_update_db.conf" + +#$1-category +#$2-type +#$3-list +create_sg_db () +{ + FILE=$db_dir/$1/$2 + if test -f "$FILE"; then + rm -f ${tmp_conf_file} + printf "dbhome $db_dir\ndest $1 {\n $3 $1/$2\n}\nacl {\n default {\n pass any\n }\n}" >> ${tmp_conf_file} + /usr/bin/squidGuard -b -c ${tmp_conf_file} -C $FILE + rm -f ${tmp_conf_file} + fi + +} while [ $# -gt 0 ] do @@ -88,6 +105,16 @@ if [[ -n $update ]] && [[ $update -eq "yes" ]]; then # fix permissions chown -R proxy:proxy ${db_dir} + + #create db + category_list=(`find $db_dir -type d -exec basename {} \; `) + for category in ${category_list[@]} + do + create_sg_db $category "domains" "domainlist" + create_sg_db $category "urls" "urllist" + create_sg_db $category "expressions" "expressionlist" + done + chown -R proxy:proxy ${db_dir} chmod 755 ${db_dir} logger --priority WARNING "webproxy blacklist entries updated (${count_before}/${count_after})" -- cgit v1.2.3