From cbb6c944fea616547cec43f7f1ed6ea3cc4beb54 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Tue, 22 Apr 2025 16:31:09 +0200 Subject: vyos.utils: T7122: fix IPv6 support in check_port_availability() Commit 4523e9c897b3 ("wireguard: T3763: Added check for listening port availability") added a function to check if a port is free to use or already occupied by a different running service. This has been done by trying to bind a socket to said given port. Unfortunately there is no support for IPv6 address-fdamily in both socketserver.TCPServer or socketserver.UDPServer. This must be done manually by deriving TCPServer and setting self.address_family for IPv6. The new implementation gets rid of both TCPServer and UDPServer and replaces it with a simple socket binding to a given IPv4/IPv6 address or any interface/ address if unspecified. In addition build time tests are added for the function to check for proper behavior during build time of vyos-1x. --- python/vyos/utils/network.py | 60 +++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 20 deletions(-) (limited to 'python') diff --git a/python/vyos/utils/network.py b/python/vyos/utils/network.py index 2f666f0ee..67d247fba 100644 --- a/python/vyos/utils/network.py +++ b/python/vyos/utils/network.py @@ -256,40 +256,60 @@ def mac2eui64(mac, prefix=None): except: # pylint: disable=bare-except return -def check_port_availability(ipaddress, port, protocol): +def check_port_availability(address: str=None, port: int=0, protocol: str='tcp') -> bool: """ - Check if port is available and not used by any service - Return False if a port is busy or IP address does not exists + Check if given port is available and not used by any service. + Should be used carefully for services that can start listening dynamically, because IP address may be dynamic too + + Args: + address: IPv4 or IPv6 address - if None, checks on all interfaces + port: TCP/UDP port number. + + + Returns: + False if a port is busy or IP address does not exists + True if a port is free and IP address exists """ - from socketserver import TCPServer, UDPServer + import socket from ipaddress import ip_address + # treat None as "any address" + address = address or '::' + # verify arguments try: - ipaddress = ip_address(ipaddress).compressed - except: - raise ValueError(f'The {ipaddress} is not a valid IPv4 or IPv6 address') + address = ip_address(address).compressed + except ValueError: + raise ValueError(f'{address} is not a valid IPv4 or IPv6 address') if port not in range(1, 65536): - raise ValueError(f'The port number {port} is not in the 1-65535 range') + raise ValueError(f'Port {port} is not in range 1-65535') if protocol not in ['tcp', 'udp']: - raise ValueError(f'The protocol {protocol} is not supported. Only tcp and udp are allowed') + raise ValueError(f'{protocol} is not supported - only tcp and udp are allowed') - # check port availability + protocol = socket.SOCK_STREAM if protocol == 'tcp' else socket.SOCK_DGRAM try: - if protocol == 'tcp': - server = TCPServer((ipaddress, port), None, bind_and_activate=True) - if protocol == 'udp': - server = UDPServer((ipaddress, port), None, bind_and_activate=True) - server.server_close() - except Exception as e: - # errno.h: - #define EADDRINUSE 98 /* Address already in use */ - if e.errno == 98: + addr_info = socket.getaddrinfo(address, port, socket.AF_UNSPEC, protocol) + except socket.gaierror as e: + print(f'Invalid address: {address}') + return False + + for family, socktype, proto, canonname, sockaddr in addr_info: + try: + with socket.socket(family, socktype, proto) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(sockaddr) + # port is free to use + return True + except OSError: + # port is already in use return False - return True + # if we reach this point, no socket was tested and we assume the port is + # already in use - better safe then sorry + return False + def is_listen_port_bind_service(port: int, service: str) -> bool: """Check if listen port bound to expected program name -- cgit v1.2.3 From a41398d9289457794f3f2e66e1dded7804f3c080 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Mon, 28 Apr 2025 21:51:07 +0200 Subject: T7122: remove trailing chars and add new line for every template.render() call --- python/vyos/template.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'python') diff --git a/python/vyos/template.py b/python/vyos/template.py index d79e1183f..513490963 100755 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -150,6 +150,8 @@ def render( # As we are opening the file with 'w', we are performing the rendering before # calling open() to not accidentally erase the file if rendering fails rendered = render_to_string(template, content, formater, location) + # Remove any trailing character and always add a new line at the end + rendered = rendered.rstrip() + "\n" # Write to file with open(destination, "w") as file: -- cgit v1.2.3 From b93427874a0e502f83c3cc450663e079af214ea9 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Mon, 28 Apr 2025 22:06:40 +0200 Subject: pki: T7122: place certbot behind reverse-proxy if cert used by haproxy If we detect that an ACME issued certificate is consumed by haproxy service, we will move the certbot webserver to localhost and a highport, to proxy the request via haproxy which is already using port 80. --- python/vyos/defaults.py | 4 ++++ src/conf_mode/pki.py | 37 +++++++++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) (limited to 'python') diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index 7efccded6..1e6be6241 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -47,6 +47,10 @@ systemd_services = { 'snmpd' : 'snmpd.service', } +internal_ports = { + 'certbot_haproxy' : 65080, # Certbot running behing haproxy +} + config_status = '/tmp/vyos-config-status' api_config_state = '/run/http-api-state' frr_debug_enable = '/tmp/vyos.frr.debug' diff --git a/src/conf_mode/pki.py b/src/conf_mode/pki.py index 521af61df..c1ff80d8a 100755 --- a/src/conf_mode/pki.py +++ b/src/conf_mode/pki.py @@ -27,6 +27,7 @@ from vyos.configdict import node_changed from vyos.configdiff import Diff from vyos.configdiff import get_config_diff from vyos.defaults import directories +from vyos.defaults import internal_ports from vyos.pki import encode_certificate from vyos.pki import is_ca_certificate from vyos.pki import load_certificate @@ -42,6 +43,7 @@ from vyos.utils.dict import dict_search from vyos.utils.dict import dict_search_args from vyos.utils.dict import dict_search_recursive from vyos.utils.file import read_file +from vyos.utils.network import check_port_availability from vyos.utils.process import call from vyos.utils.process import cmd from vyos.utils.process import is_systemd_service_active @@ -128,8 +130,13 @@ def certbot_request(name: str, config: dict, dry_run: bool=True): f'--standalone --agree-tos --no-eff-email --expand --server {config["url"]} '\ f'--email {config["email"]} --key-type rsa --rsa-key-size {config["rsa_key_size"]} '\ f'{domains}' - if 'listen_address' in config: + # When ACME is used behind a reverse proxy, we always bind to localhost + # whatever the CLI listen-address is configured for. + if 'used_by' in config and 'haproxy' in config['used_by']: + tmp += f' --http-01-address 127.0.0.1 --http-01-port {internal_ports["certbot_haproxy"]}' + elif 'listen_address' in config: tmp += f' --http-01-address {config["listen_address"]}' + # verify() does not need to actually request a cert but only test for plausability if dry_run: tmp += ' --dry-run' @@ -223,7 +230,7 @@ def get_config(config=None): continue path = search['path'] - path_str = ' '.join(path + found_path) + path_str = ' '.join(path + found_path).replace('_','-') #print(f'PKI: Updating config: {path_str} {item_name}') if path[0] == 'interfaces': @@ -234,6 +241,24 @@ def get_config(config=None): if not D.node_changed_presence(path): set_dependents(path[1], conf) + # Check PKI certificates if they are generated by ACME. If they are, traverse + # the current configutration and determine the service where the certificate + # is used by. This is needed to check if we might need to start ACME behing + # a reverse proxy. + if 'certificate' in pki: + for name, cert_config in pki['certificate'].items(): + if 'acme' not in cert_config: + continue + if not dict_search('system.load_balancing.haproxy', pki): + continue + used_by = [] + for cert_list, cli_path in dict_search_recursive( + pki['system']['load_balancing']['haproxy'], 'certificate'): + if name in cert_list: + used_by.append('haproxy') + if used_by: + pki['certificate'][name]['acme'].update({'used_by': used_by}) + return pki def is_valid_certificate(raw_data): @@ -325,6 +350,14 @@ def verify(pki): raise ConfigError(f'An email address is required to request '\ f'certificate for "{name}" via ACME!') + listen_address = None + if 'listen_address' in cert_conf['acme']: + listen_address = cert_conf['acme']['listen_address'] + + if 'used_by' not in cert_conf['acme']: + if not check_port_availability(listen_address, 80): + raise ConfigError(f'Port 80 is not available for ACME challenge for certificate "{name}"!') + if 'certbot_renew' not in pki: # Only run the ACME command if something on this entity changed, # as this is time intensive -- cgit v1.2.3 From aff2835d7b6226e4b89f51e3f6133da26f3a07bf Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sun, 4 May 2025 11:23:54 +0200 Subject: vyos.template: T7122: add Jinja2 clever function helper to read vyos.defaults Add a new category if Jinja2 operands. We already have filters and tests, but sometimes we would like to call a Python function without and data "|" piped to it - that's what they call a clever-function. {{ get_default_port(NAME) }} can be used to retrieve the value from vyos.defaults.internal_ports[NAME] within Jinja2. We no longer need to extend the dictionary with arbitrary data retrieved from vyos.defaults, we can now simply register another clever-function to the Jinja2 backend. --- python/vyos/template.py | 44 ++++++++++++++++++++++++++++++++++++++++++-- src/tests/test_template.py | 9 +++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) (limited to 'python') diff --git a/python/vyos/template.py b/python/vyos/template.py index 513490963..11e1cc50f 100755 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -36,6 +36,7 @@ DEFAULT_TEMPLATE_DIR = directories["templates"] # Holds template filters registered via register_filter() _FILTERS = {} _TESTS = {} +_CLEVER_FUNCTIONS = {} # reuse Environments with identical settings to improve performance @functools.lru_cache(maxsize=2) @@ -58,6 +59,7 @@ def _get_environment(location=None): ) env.filters.update(_FILTERS) env.tests.update(_TESTS) + env.globals.update(_CLEVER_FUNCTIONS) return env @@ -77,7 +79,7 @@ def register_filter(name, func=None): "Filters can only be registered before rendering the first template" ) if name in _FILTERS: - raise ValueError(f"A filter with name {name!r} was registered already") + raise ValueError(f"A filter with name {name!r} was already registered") _FILTERS[name] = func return func @@ -97,10 +99,30 @@ def register_test(name, func=None): "Tests can only be registered before rendering the first template" ) if name in _TESTS: - raise ValueError(f"A test with name {name!r} was registered already") + raise ValueError(f"A test with name {name!r} was already registered") _TESTS[name] = func return func +def register_clever_function(name, func=None): + """Register a function to be available as test in templates under given name. + + It can also be used as a decorator, see below in this module for examples. + + :raise RuntimeError: + when trying to register a test after a template has been rendered already + :raise ValueError: when trying to register a name which was taken already + """ + if func is None: + return functools.partial(register_clever_function, name) + if _get_environment.cache_info().currsize: + raise RuntimeError( + "Clever functions can only be registered before rendering the" \ + "first template") + if name in _CLEVER_FUNCTIONS: + raise ValueError(f"A clever function with name {name!r} was already "\ + "registered") + _CLEVER_FUNCTIONS[name] = func + return func def render_to_string(template, content, formater=None, location=None): """Render a template from the template directory, raise on any errors. @@ -1052,3 +1074,21 @@ def vyos_defined(value, test_value=None, var_type=None): else: # Valid value and is matching optional argument if provided - return true return True + +@register_clever_function('get_default_port') +def get_default_port(service): + """ + Jinja2 plugin to retrieve common service port number from vyos.defaults + class form a Jinja2 template. This removes the need to hardcode, or pass in + the data using the general dictionary. + + Added to remove code complexity and make it easier to read. + + Example: + {{ get_default_port('certbot_haproxy') }} + """ + from vyos.defaults import internal_ports + if service not in internal_ports: + raise RuntimeError(f'Service "{service}" not found in internal ' \ + 'vyos.defaults.internal_ports dict!') + return internal_ports[service] diff --git a/src/tests/test_template.py b/src/tests/test_template.py index 6377f6da5..7cae867a0 100644 --- a/src/tests/test_template.py +++ b/src/tests/test_template.py @@ -190,3 +190,12 @@ class TestVyOSTemplate(TestCase): for group_name, group_config in data['ike_group'].items(): ciphers = vyos.template.get_esp_ike_cipher(group_config) self.assertIn(IKEv2_DEFAULT, ','.join(ciphers)) + + def test_get_default_port(self): + from vyos.defaults import internal_ports + + with self.assertRaises(RuntimeError): + vyos.template.get_default_port('UNKNOWN') + + self.assertEqual(vyos.template.get_default_port('certbot_haproxy'), + internal_ports['certbot_haproxy']) -- cgit v1.2.3 From 40a99b1d00d822ef6f1a3772768919d14c3500d9 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sun, 4 May 2025 12:17:06 +0200 Subject: vyos.base: T7122: add new Message() helper wrapper for print() This will wrap the messages at 72 characters in the same way as Warning() and DeprecationWarning() would do. We now have simple wrappers for it! Example: vyos@vyos# commit [ pki ] Updating configuration: "load-balancing haproxy service frontend ssl certificate LE_cloud" Add/replace automatically imported CA certificate for "LE_cloud" --- python/vyos/base.py | 21 +++++++++++++-------- src/conf_mode/pki.py | 5 +++-- 2 files changed, 16 insertions(+), 10 deletions(-) (limited to 'python') diff --git a/python/vyos/base.py b/python/vyos/base.py index ca96d96ce..3173ddc20 100644 --- a/python/vyos/base.py +++ b/python/vyos/base.py @@ -1,4 +1,4 @@ -# Copyright 2018-2022 VyOS maintainers and contributors +# Copyright 2018-2025 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -15,8 +15,7 @@ from textwrap import fill - -class BaseWarning: +class UserMessage: def __init__(self, header, message, **kwargs): self.message = message self.kwargs = kwargs @@ -33,7 +32,6 @@ class BaseWarning: messages = self.message.split('\n') isfirstmessage = True initial_indent = self.textinitindent - print('') for mes in messages: mes = fill(mes, initial_indent=initial_indent, subsequent_indent=self.standardindent, **self.kwargs) @@ -44,17 +42,24 @@ class BaseWarning: print('', flush=True) +class Message(): + def __init__(self, message, **kwargs): + self.Message = UserMessage('', message, **kwargs) + self.Message.print() + class Warning(): def __init__(self, message, **kwargs): - self.BaseWarn = BaseWarning('WARNING: ', message, **kwargs) - self.BaseWarn.print() + print('') + self.UserMessage = UserMessage('WARNING: ', message, **kwargs) + self.UserMessage.print() class DeprecationWarning(): def __init__(self, message, **kwargs): # Reformat the message and trim it to 72 characters in length - self.BaseWarn = BaseWarning('DEPRECATION WARNING: ', message, **kwargs) - self.BaseWarn.print() + print('') + self.UserMessage = UserMessage('DEPRECATION WARNING: ', message, **kwargs) + self.UserMessage.print() class ConfigError(Exception): diff --git a/src/conf_mode/pki.py b/src/conf_mode/pki.py index 98922595c..6957248d2 100755 --- a/src/conf_mode/pki.py +++ b/src/conf_mode/pki.py @@ -19,6 +19,7 @@ import os from sys import argv from sys import exit +from vyos.base import Message from vyos.config import Config from vyos.config import config_dict_merge from vyos.configdep import set_dependents @@ -231,7 +232,7 @@ def get_config(config=None): path = search['path'] path_str = ' '.join(path + found_path).replace('_','-') - print(f'PKI: Updating config: {path_str} {item_name}') + Message(f'Updating configuration: "{path_str} {item_name}"') if path[0] == 'interfaces': ifname = found_path[0] @@ -528,7 +529,7 @@ def generate(pki): if not ca_cert_present: tmp = dict_search_args(pki, 'ca', f'{autochain_prefix}{cert}', 'certificate') if not bool(tmp) or tmp != cert_chain_base64: - print(f'Adding/replacing automatically imported CA certificate for "{cert}" ...') + Message(f'Add/replace automatically imported CA certificate for "{cert}"...') add_cli_node(['pki', 'ca', f'{autochain_prefix}{cert}', 'certificate'], value=cert_chain_base64) return None -- cgit v1.2.3 From 59d86826a2ffb2df6a0ce603c879e541a4fe88ba Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sun, 4 May 2025 22:08:13 +0200 Subject: haproxy: T7122: add ACME/certbot bootstrap support When both the CLI PKI node for an ACME-issued certificate and HAProxy are configured during initial setup, the certbot challenge cannot be served via the reverse proxy because HAProxy has not yet been configured at all. This commit introduces a special case to handle this bootstrap scenario, ensuring that the certbot challenge can still be served correctly in standalone mode on port 80 despite initial config dependencies/priorities between PKI and HAProxy. --- python/vyos/defaults.py | 1 + src/conf_mode/load-balancing_haproxy.py | 10 +++++----- src/conf_mode/pki.py | 5 ++++- 3 files changed, 10 insertions(+), 6 deletions(-) (limited to 'python') diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index 1e6be6241..c1e5ddc04 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -43,6 +43,7 @@ directories = { } systemd_services = { + 'haproxy' : 'haproxy.service', 'syslog' : 'syslog.service', 'snmpd' : 'snmpd.service', } diff --git a/src/conf_mode/load-balancing_haproxy.py b/src/conf_mode/load-balancing_haproxy.py index 0e959480c..504a90596 100644 --- a/src/conf_mode/load-balancing_haproxy.py +++ b/src/conf_mode/load-balancing_haproxy.py @@ -19,6 +19,7 @@ import os from sys import exit from shutil import rmtree +from vyos.defaults import systemd_services from vyos.config import Config from vyos.configverify import verify_pki_certificate from vyos.configverify import verify_pki_ca_certificate @@ -39,7 +40,6 @@ airbag.enable() load_balancing_dir = '/run/haproxy' load_balancing_conf_file = f'{load_balancing_dir}/haproxy.cfg' -systemd_service = 'haproxy.service' systemd_override = '/run/systemd/system/haproxy.service.d/10-override.conf' def get_config(config=None): @@ -191,11 +191,11 @@ def generate(lb): return None def apply(lb): + action = 'stop' + if lb: + action = 'reload-or-restart' call('systemctl daemon-reload') - if not lb: - call(f'systemctl stop {systemd_service}') - else: - call(f'systemctl reload-or-restart {systemd_service}') + call(f'systemctl {action} {systemd_services["haproxy"]}') return None diff --git a/src/conf_mode/pki.py b/src/conf_mode/pki.py index f53e5db8b..7ee1705c0 100755 --- a/src/conf_mode/pki.py +++ b/src/conf_mode/pki.py @@ -29,6 +29,7 @@ from vyos.configdiff import Diff from vyos.configdiff import get_config_diff from vyos.defaults import directories from vyos.defaults import internal_ports +from vyos.defaults import systemd_services from vyos.pki import encode_certificate from vyos.pki import is_ca_certificate from vyos.pki import load_certificate @@ -48,6 +49,7 @@ from vyos.utils.network import check_port_availability from vyos.utils.process import call from vyos.utils.process import cmd from vyos.utils.process import is_systemd_service_active +from vyos.utils.process import is_systemd_service_running from vyos import ConfigError from vyos import airbag airbag.enable() @@ -133,7 +135,8 @@ def certbot_request(name: str, config: dict, dry_run: bool=True): f'{domains}' # When ACME is used behind a reverse proxy, we always bind to localhost # whatever the CLI listen-address is configured for. - if 'used_by' in config and 'haproxy' in config['used_by']: + if ('haproxy' in dict_search('used_by', config) and + is_systemd_service_running(systemd_services['haproxy'])): tmp += f' --http-01-address 127.0.0.1 --http-01-port {internal_ports["certbot_haproxy"]}' elif 'listen_address' in config: tmp += f' --http-01-address {config["listen_address"]}' -- cgit v1.2.3