diff options
22 files changed, 455 insertions, 270 deletions
diff --git a/data/templates/dns-forwarding/recursor.conf.j2 b/data/templates/dns-forwarding/recursor.conf.j2 index deeb250f0..55b37732b 100644 --- a/data/templates/dns-forwarding/recursor.conf.j2 +++ b/data/templates/dns-forwarding/recursor.conf.j2 @@ -40,12 +40,12 @@ dnssec={{ dnssec }} dns64-prefix={{ dns64_prefix }} {% endif %} -{% if dont_throttle_netmasks is vyos_defined %} +{% if exclude_throttle_address is vyos_defined %} # dont-throttle-netmasks dont-throttle-netmasks={{ exclude_throttle_address | join(',') }} {% endif %} -{% if serve_stale_extensions is vyos_defined %} +{% if serve_stale_extension is vyos_defined %} # serve-stale-extensions serve-stale-extensions={{ serve_stale_extension }} {% endif %} diff --git a/data/templates/https/nginx.default.j2 b/data/templates/https/nginx.default.j2 index 80239ea56..a530c14ba 100644 --- a/data/templates/https/nginx.default.j2 +++ b/data/templates/https/nginx.default.j2 @@ -18,12 +18,7 @@ server { root /srv/localui; -{% if server.certbot %} - ssl_certificate {{ server.certbot_dir }}/live/{{ server.certbot_domain_dir }}/fullchain.pem; - ssl_certificate_key {{ server.certbot_dir }}/live/{{ server.certbot_domain_dir }}/privkey.pem; - include {{ server.certbot_dir }}/options-ssl-nginx.conf; - ssl_dhparam {{ server.certbot_dir }}/ssl-dhparams.pem; -{% elif server.vyos_cert %} +{% if server.vyos_cert %} ssl_certificate {{ server.vyos_cert.crt }}; ssl_certificate_key {{ server.vyos_cert.key }}; {% else %} @@ -33,7 +28,12 @@ server { # include snippets/snakeoil.conf; {% endif %} + ssl_session_cache shared:le_nginx_SSL:10m; + ssl_session_timeout 1440m; + ssl_session_tickets off; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!3DES:!MD5:!PSK'; # proxy settings for HTTP API, if enabled; 503, if not location ~ ^/(retrieve|configure|config-file|image|container-image|generate|show|reboot|reset|poweroff|docs|openapi.json|redoc|graphql) { diff --git a/debian/control b/debian/control index af42202fc..726a083f2 100644 --- a/debian/control +++ b/debian/control @@ -43,7 +43,6 @@ Depends: ## End of Fundamentals ## Python libraries used in multiple modules and scripts python3, - python3-certbot-nginx, python3-cryptography, python3-hurry.filesize, python3-inotify, @@ -146,6 +145,9 @@ Depends: # For "protocols igmp-proxy" igmpproxy, # End "protocols igmp-proxy" +# For "pki" + certbot, +# End "pki" # For "service console-server" conserver-client, conserver-server, diff --git a/interface-definitions/include/constraint/email.xml.i b/interface-definitions/include/constraint/email.xml.i new file mode 100644 index 000000000..b19a88d64 --- /dev/null +++ b/interface-definitions/include/constraint/email.xml.i @@ -0,0 +1,3 @@ +<!-- include start from constraint/email.xml.i --> +<regex>[^\s@]+@([^\s@.,]+\.)+[^\s@.,]{2,}</regex> +<!-- include end --> diff --git a/interface-definitions/include/version/https-version.xml.i b/interface-definitions/include/version/https-version.xml.i index fa18278f3..525314dbd 100644 --- a/interface-definitions/include/version/https-version.xml.i +++ b/interface-definitions/include/version/https-version.xml.i @@ -1,3 +1,3 @@ <!-- include start from include/version/https-version.xml.i --> -<syntaxVersion component='https' version='5'></syntaxVersion> +<syntaxVersion component='https' version='6'></syntaxVersion> <!-- include end --> diff --git a/interface-definitions/pki.xml.in b/interface-definitions/pki.xml.in index 097c541ac..0ed199539 100644 --- a/interface-definitions/pki.xml.in +++ b/interface-definitions/pki.xml.in @@ -81,6 +81,60 @@ <constraintErrorMessage>Certificate is not base64-encoded</constraintErrorMessage> </properties> </leafNode> + <node name="acme"> + <properties> + <help>Automatic Certificate Management Environment (ACME) request</help> + </properties> + <children> + #include <include/url-http-https.xml.i> + <leafNode name="url"> + <defaultValue>https://acme-v02.api.letsencrypt.org/directory</defaultValue> + </leafNode> + <leafNode name="domain-name"> + <properties> + <help>Domain Name</help> + <constraint> + <validator name="fqdn"/> + </constraint> + <constraintErrorMessage>Invalid domain name (RFC 1123 section 2).\nMay only contain letters, numbers and .-_</constraintErrorMessage> + <multi/> + </properties> + </leafNode> + <leafNode name="email"> + <properties> + <help>Email address to associate with certificate</help> + <constraint> + #include <include/constraint/email.xml.i> + </constraint> + </properties> + </leafNode> + #include <include/listen-address-ipv4-single.xml.i> + <leafNode name="rsa-key-size"> + <properties> + <help>Size of the RSA key</help> + <completionHelp> + <list>2048 3072 4096</list> + </completionHelp> + <valueHelp> + <format>2048</format> + <description>RSA key length 2048 bit</description> + </valueHelp> + <valueHelp> + <format>3072</format> + <description>RSA key length 3072 bit</description> + </valueHelp> + <valueHelp> + <format>4096</format> + <description>RSA key length 4096 bit</description> + </valueHelp> + <constraint> + <regex>(2048|3072|4096)</regex> + </constraint> + </properties> + <defaultValue>2048</defaultValue> + </leafNode> + </children> + </node> #include <include/generic-description.xml.i> <node name="private"> <properties> diff --git a/interface-definitions/service_https.xml.in b/interface-definitions/service_https.xml.in index 223f10962..57f36a982 100644 --- a/interface-definitions/service_https.xml.in +++ b/interface-definitions/service_https.xml.in @@ -192,24 +192,6 @@ <children> #include <include/pki/ca-certificate.xml.i> #include <include/pki/certificate.xml.i> - <node name="certbot" owner="${vyos_conf_scripts_dir}/service_https_certificates_certbot.py"> - <properties> - <help>Request or apply a letsencrypt certificate for domain-name</help> - </properties> - <children> - <leafNode name="domain-name"> - <properties> - <help>Domain name(s) for which to obtain certificate</help> - <multi/> - </properties> - </leafNode> - <leafNode name="email"> - <properties> - <help>Email address to associate with certificate</help> - </properties> - </leafNode> - </children> - </node> </children> </node> #include <include/interface/vrf.xml.i> diff --git a/op-mode-definitions/monitor-log.xml.in b/op-mode-definitions/monitor-log.xml.in index c03ec4cce..559952e25 100644 --- a/op-mode-definitions/monitor-log.xml.in +++ b/op-mode-definitions/monitor-log.xml.in @@ -30,6 +30,12 @@ </leafNode> </children> </node> + <leafNode name="certbot"> + <properties> + <help>Monitor last lines of certbot log</help> + </properties> + <command>if sudo test -f /var/log/letsencrypt/letsencrypt.log; then sudo tail --follow=name /var/log/letsencrypt/letsencrypt.log; else echo "Cerbot log does not exist"; fi</command> + </leafNode> <leafNode name="conntrack-sync"> <properties> <help>Monitor last lines of conntrack-sync log</help> diff --git a/op-mode-definitions/pki.xml.in b/op-mode-definitions/pki.xml.in index ca0eb3687..4b8d9c47a 100644 --- a/op-mode-definitions/pki.xml.in +++ b/op-mode-definitions/pki.xml.in @@ -574,4 +574,14 @@ </node> </children> </node> + <node name="renew"> + <children> + <leafNode name="certbot"> + <properties> + <help>Start manual certbot renewal</help> + </properties> + <command>sudo systemctl start certbot.service</command> + </leafNode> + </children> + </node> </interfaceDefinition> diff --git a/op-mode-definitions/show-ipv6-route.xml.in b/op-mode-definitions/show-ipv6-route.xml.in index 7df1a873a..d73fb46b4 100644 --- a/op-mode-definitions/show-ipv6-route.xml.in +++ b/op-mode-definitions/show-ipv6-route.xml.in @@ -82,6 +82,23 @@ </properties> <command>${vyos_op_scripts_dir}/route.py show_summary --family inet6 --vrf $5</command> </node> + <node name="node.tag"> + <properties> + <help>Show IPv6 routes of given address or prefix</help> + <completionHelp> + <list><h:h:h:h:h:h:h:h> <h:h:h:h:h:h:h:h/x></list> + </completionHelp> + </properties> + <command>${vyos_op_scripts_dir}/vtysh_wrapper.sh $@</command> + <children> + <node name="longer-prefixes"> + <properties> + <help>Show longer prefixes of routes for given prefix</help> + </properties> + <command>${vyos_op_scripts_dir}/vtysh_wrapper.sh $@</command> + </node> + </children> + </node> #include <include/show-route-bgp.xml.i> #include <include/show-route-connected.xml.i> #include <include/show-route-isis.xml.i> @@ -103,6 +120,7 @@ <list><h:h:h:h:h:h:h:h> <h:h:h:h:h:h:h:h/x></list> </completionHelp> </properties> + <command>${vyos_op_scripts_dir}/vtysh_wrapper.sh $@</command> <children> <node name="longer-prefixes"> <properties> @@ -111,7 +129,6 @@ <command>${vyos_op_scripts_dir}/vtysh_wrapper.sh $@</command> </node> </children> - <command>vtysh -c "show ipv6 route $4"</command> </tagNode> </children> </node> diff --git a/op-mode-definitions/show-log.xml.in b/op-mode-definitions/show-log.xml.in index b013bdfe4..a6ce04624 100644 --- a/op-mode-definitions/show-log.xml.in +++ b/op-mode-definitions/show-log.xml.in @@ -38,6 +38,12 @@ </properties> <command>journalctl --no-hostname --boot --quiet SYSLOG_FACILITY=10 SYSLOG_FACILITY=4</command> </leafNode> + <leafNode name="certbot"> + <properties> + <help>Show log for certbot</help> + </properties> + <command>if sudo test -f /var/log/letsencrypt/letsencrypt.log; then sudo cat /var/log/letsencrypt/letsencrypt.log; else echo "Cerbot log does not exist"; fi</command> + </leafNode> <leafNode name="cluster"> <properties> <help>Show log for Cluster</help> diff --git a/python/vyos/config.py b/python/vyos/config.py index ca7b035e5..bee85315d 100644 --- a/python/vyos/config.py +++ b/python/vyos/config.py @@ -92,6 +92,38 @@ def config_dict_merge(src: dict, dest: Union[dict, ConfigDict]) -> ConfigDict: dest = ConfigDict(dest) return ext_dict_merge(src, dest) +def config_dict_mangle_acme(name, cli_dict): + """ + Load CLI PKI dictionary and if an ACME certificate is used, load it's content + and place it into the CLI dictionary as it would be a "regular" CLI PKI based + certificate with private key + """ + from vyos.base import ConfigError + from vyos.defaults import directories + from vyos.utils.file import read_file + from vyos.pki import encode_certificate + from vyos.pki import encode_private_key + from vyos.pki import load_certificate + from vyos.pki import load_private_key + + try: + vyos_certbot_dir = directories['certbot'] + + if 'acme' in cli_dict: + tmp = read_file(f'{vyos_certbot_dir}/live/{name}/cert.pem') + tmp = load_certificate(tmp, wrap_tags=False) + cert_base64 = "".join(encode_certificate(tmp).strip().split("\n")[1:-1]) + + tmp = read_file(f'{vyos_certbot_dir}/live/{name}/privkey.pem') + tmp = load_private_key(tmp, wrap_tags=False) + key_base64 = "".join(encode_private_key(tmp).strip().split("\n")[1:-1]) + # install ACME based PEM keys into "regular" CLI config keys + cli_dict.update({'certificate' : cert_base64, 'private' : {'key' : key_base64}}) + except: + raise ConfigError(f'Unable to load ACME certificates for "{name}"!') + + return cli_dict + class Config(object): """ The class of config access objects. @@ -306,6 +338,11 @@ class Config(object): no_tag_node_value_mangle=True, get_first_key=True) if pki_dict: + if 'certificate' in pki_dict: + for certificate in pki_dict['certificate']: + pki_dict['certificate'][certificate] = config_dict_mangle_acme( + certificate, pki_dict['certificate'][certificate]) + conf_dict['pki'] = pki_dict # save optional args for a call to get_config_defaults diff --git a/python/vyos/utils/process.py b/python/vyos/utils/process.py index e09c7d86d..cd58b4be2 100644 --- a/python/vyos/utils/process.py +++ b/python/vyos/utils/process.py @@ -204,17 +204,32 @@ def process_running(pid_file): pid = f.read().strip() return pid_exists(int(pid)) -def process_named_running(name, cmdline: str=None): +def process_named_running(name, cmdline: str=None, timeout=0): """ Checks if process with given name is running and returns its PID. If Process is not running, return None """ from psutil import process_iter - for p in process_iter(['name', 'pid', 'cmdline']): - if cmdline: - if p.info['name'] == name and cmdline in p.info['cmdline']: + def check_process(name, cmdline): + for p in process_iter(['name', 'pid', 'cmdline']): + if cmdline: + if name in p.info['name'] and cmdline in p.info['cmdline']: + return p.info['pid'] + elif name in p.info['name']: return p.info['pid'] - elif p.info['name'] == name: - return p.info['pid'] + return None + if timeout: + import time + time_expire = time.time() + timeout + while True: + tmp = check_process(name, cmdline) + if not tmp: + if time.time() > time_expire: + break + time.sleep(0.100) # wait 250ms + continue + return tmp + else: + return check_process(name, cmdline) return None def is_systemd_service_active(service): diff --git a/smoketest/scripts/cli/base_interfaces_test.py b/smoketest/scripts/cli/base_interfaces_test.py index 3f42196f7..7219fe622 100644 --- a/smoketest/scripts/cli/base_interfaces_test.py +++ b/smoketest/scripts/cli/base_interfaces_test.py @@ -127,9 +127,9 @@ class BasicInterfaceTest: # by also checking the cmd arguments passed to the daemon if self._interfaces: for tmp in self._interfaces: - self.assertFalse(process_named_running(daemon, tmp)) + self.assertFalse(process_named_running(daemon, tmp, timeout=10)) else: - self.assertFalse(process_named_running(daemon)) + self.assertFalse(process_named_running(daemon, timeout=10)) def test_dhcp_disable_interface(self): if not self._test_dhcp: @@ -179,7 +179,7 @@ class BasicInterfaceTest: for interface in self._interfaces: # Check if dhclient process runs - dhclient_pid = process_named_running(dhclient_process_name, cmdline=interface) + dhclient_pid = process_named_running(dhclient_process_name, cmdline=interface, timeout=10) self.assertTrue(dhclient_pid) dhclient_config = read_file(f'{dhclient_base_dir}/dhclient_{interface}.conf') @@ -216,7 +216,7 @@ class BasicInterfaceTest: self.assertEqual(tmp, vrf_name) # Check if dhclient process runs - dhclient_pid = process_named_running(dhclient_process_name, cmdline=interface) + dhclient_pid = process_named_running(dhclient_process_name, cmdline=interface, timeout=10) self.assertTrue(dhclient_pid) # .. inside the appropriate VRF instance vrf_pids = cmd(f'ip vrf pids {vrf_name}') @@ -251,7 +251,7 @@ class BasicInterfaceTest: self.assertEqual(tmp, vrf_name) # Check if dhclient process runs - tmp = process_named_running(dhcp6c_process_name, cmdline=interface) + tmp = process_named_running(dhcp6c_process_name, cmdline=interface, timeout=10) self.assertTrue(tmp) # .. inside the appropriate VRF instance vrf_pids = cmd(f'ip vrf pids {vrf_name}') @@ -945,7 +945,7 @@ class BasicInterfaceTest: duid_base += 1 # Better ask the process about it's commandline in the future - pid = process_named_running(dhcp6c_process_name, cmdline=interface) + pid = process_named_running(dhcp6c_process_name, cmdline=interface, timeout=10) self.assertTrue(pid) dhcp6c_options = read_file(f'/proc/{pid}/cmdline') @@ -1004,7 +1004,7 @@ class BasicInterfaceTest: address = str(int(address) + 1) # Check for running process - self.assertTrue(process_named_running(dhcp6c_process_name, cmdline=interface)) + self.assertTrue(process_named_running(dhcp6c_process_name, cmdline=interface, timeout=10)) for delegatee in delegatees: # we can already cleanup the test delegatee interface here @@ -1070,7 +1070,7 @@ class BasicInterfaceTest: address = str(int(address) + 1) # Check for running process - self.assertTrue(process_named_running(dhcp6c_process_name, cmdline=interface)) + self.assertTrue(process_named_running(dhcp6c_process_name, cmdline=interface, timeout=10)) for delegatee in delegatees: # we can already cleanup the test delegatee interface here diff --git a/smoketest/scripts/cli/test_service_dns_forwarding.py b/smoketest/scripts/cli/test_service_dns_forwarding.py index 4f2f182e5..85a5f1448 100755 --- a/smoketest/scripts/cli/test_service_dns_forwarding.py +++ b/smoketest/scripts/cli/test_service_dns_forwarding.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2019-2022 VyOS maintainers and contributors +# Copyright (C) 2019-2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -43,7 +43,6 @@ class TestServicePowerDNS(VyOSUnitTestSHIM.TestCase): @classmethod def setUpClass(cls): super(TestServicePowerDNS, cls).setUpClass() - # ensure we can also run this test on a live system - so lets clean # out the current configuration :) cls.cli_delete(cls, base_path) @@ -259,23 +258,24 @@ class TestServicePowerDNS(VyOSUnitTestSHIM.TestCase): self.cli_commit() # verify dont-throttle-netmasks configuration - tmp = get_config_value('exclude-throttle-address') + tmp = get_config_value('dont-throttle-netmasks') self.assertEqual(tmp, ','.join(exclude_throttle_adress_examples)) def test_serve_stale_extension(self): + server_stale = '20' for network in allow_from: self.cli_set(base_path + ['allow-from', network]) for address in listen_adress: self.cli_set(base_path + ['listen-address', address]) - self.cli_set(base_path + ['serve-stale-extension', '20']) + self.cli_set(base_path + ['serve-stale-extension', server_stale]) # commit changes self.cli_commit() # verify configuration - tmp = get_config_value('serve-stale-extension') - self.assertEqual(tmp, '20') + tmp = get_config_value('serve-stale-extensions') + self.assertEqual(tmp, server_stale) def test_listening_port(self): # We can listen on a different port compared to '53' but only one at a time diff --git a/src/conf_mode/pki.py b/src/conf_mode/pki.py index f7e14aa16..310519abd 100755 --- a/src/conf_mode/pki.py +++ b/src/conf_mode/pki.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 VyOS maintainers and contributors +# Copyright (C) 2021-2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -14,59 +14,66 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import os + +from sys import argv from sys import exit from vyos.config import Config -from vyos.configdep import set_dependents, call_dependents +from vyos.config import config_dict_merge +from vyos.configdep import set_dependents +from vyos.configdep import call_dependents from vyos.configdict import node_changed +from vyos.configdiff import Diff +from vyos.defaults import directories from vyos.pki import is_ca_certificate from vyos.pki import load_certificate from vyos.pki import load_public_key from vyos.pki import load_private_key from vyos.pki import load_crl from vyos.pki import load_dh_parameters +from vyos.utils.boot import boot_configuration_complete +from vyos.utils.dict import dict_search from vyos.utils.dict import dict_search_args from vyos.utils.dict import dict_search_recursive +from vyos.utils.process import call +from vyos.utils.process import cmd +from vyos.utils.process import is_systemd_service_active from vyos import ConfigError from vyos import airbag airbag.enable() -# keys to recursively search for under specified path, script to call if update required +vyos_certbot_dir = directories['certbot'] + +# keys to recursively search for under specified path sync_search = [ { 'keys': ['certificate'], 'path': ['service', 'https'], - 'script': '/usr/libexec/vyos/conf_mode/service_https.py' }, { 'keys': ['certificate', 'ca_certificate'], 'path': ['interfaces', 'ethernet'], - 'script': '/usr/libexec/vyos/conf_mode/interfaces_ethernet.py' }, { 'keys': ['certificate', 'ca_certificate', 'dh_params', 'shared_secret_key', 'auth_key', 'crypt_key'], 'path': ['interfaces', 'openvpn'], - 'script': '/usr/libexec/vyos/conf_mode/interfaces_openvpn.py' }, { 'keys': ['ca_certificate'], 'path': ['interfaces', 'sstpc'], - 'script': '/usr/libexec/vyos/conf_mode/interfaces_sstpc.py' }, { 'keys': ['certificate', 'ca_certificate', 'local_key', 'remote_key'], 'path': ['vpn', 'ipsec'], - 'script': '/usr/libexec/vyos/conf_mode/vpn_ipsec.py' }, { 'keys': ['certificate', 'ca_certificate'], 'path': ['vpn', 'openconnect'], - 'script': '/usr/libexec/vyos/conf_mode/vpn_openconnect.py' }, { 'keys': ['certificate', 'ca_certificate'], 'path': ['vpn', 'sstp'], - 'script': '/usr/libexec/vyos/conf_mode/vpn_sstp.py' } ] @@ -82,6 +89,33 @@ sync_translate = { 'crypt_key': 'openvpn' } +def certbot_delete(certificate): + if not boot_configuration_complete(): + return + if os.path.exists(f'{vyos_certbot_dir}/renewal/{certificate}.conf'): + cmd(f'certbot delete --non-interactive --config-dir {vyos_certbot_dir} --cert-name {certificate}') + +def certbot_request(name: str, config: dict, dry_run: bool=True): + # We do not call certbot when booting the system - there is no need to do so and + # request new certificates during boot/image upgrade as the certbot configuration + # is stored persistent under /config - thus we do not open the door to transient + # errors + if not boot_configuration_complete(): + return + + domains = '--domains ' + ' --domains '.join(config['domain_name']) + tmp = f'certbot certonly --config-dir {vyos_certbot_dir} --cert-name {name} '\ + f'--non-interactive --standalone --agree-tos --no-eff-email --expand '\ + f'--server {config["url"]} --email {config["email"]} '\ + f'--key-type rsa --rsa-key-size {config["rsa_key_size"]} {domains}' + if 'listen_address' in config: + tmp += f' --http-01-address {config["listen_address"]}' + # verify() does not need to actually request a cert but only test for plausability + if dry_run: + tmp += ' --dry-run' + + cmd(tmp, raising=ConfigError, message=f'ACME certbot request failed for "{name}"!') + def get_config(config=None): if config: conf = config @@ -93,25 +127,62 @@ def get_config(config=None): get_first_key=True, no_tag_node_value_mangle=True) - pki['changed'] = {} + if len(argv) > 1 and argv[1] == 'certbot_renew': + pki['certbot_renew'] = {} + tmp = node_changed(conf, base + ['ca'], key_mangling=('-', '_'), recursive=True) - if tmp: pki['changed'].update({'ca' : tmp}) + if tmp: + if 'changed' not in pki: pki.update({'changed':{}}) + pki['changed'].update({'ca' : tmp}) - tmp = node_changed(conf, base + ['certificate'], key_mangling=('-', '_'), recursive=True) - if tmp: pki['changed'].update({'certificate' : tmp}) + tmp = node_changed(conf, base + ['certificate'], key_mangling=('-', '_'), + recursive=True, expand_nodes=Diff.ADD|Diff.DELETE) + if tmp: + if 'changed' not in pki: pki.update({'changed':{}}) + pki['changed'].update({'certificate' : tmp}) tmp = node_changed(conf, base + ['dh'], key_mangling=('-', '_'), recursive=True) - if tmp: pki['changed'].update({'dh' : tmp}) + if tmp: + if 'changed' not in pki: pki.update({'changed':{}}) + pki['changed'].update({'dh' : tmp}) tmp = node_changed(conf, base + ['key-pair'], key_mangling=('-', '_'), recursive=True) - if tmp: pki['changed'].update({'key_pair' : tmp}) + if tmp: + if 'changed' not in pki: pki.update({'changed':{}}) + pki['changed'].update({'key_pair' : tmp}) - tmp = node_changed(conf, base + ['openvpn', 'shared-secret'], key_mangling=('-', '_'), recursive=True) - if tmp: pki['changed'].update({'openvpn' : tmp}) + tmp = node_changed(conf, base + ['openvpn', 'shared-secret'], key_mangling=('-', '_'), + recursive=True) + if tmp: + if 'changed' not in pki: pki.update({'changed':{}}) + pki['changed'].update({'openvpn' : tmp}) # We only merge on the defaults of there is a configuration at all if conf.exists(base): - pki = conf.merge_defaults(pki, recursive=True) + # We have gathered the dict representation of the CLI, but there are default + # options which we need to update into the dictionary retrived. + default_values = conf.get_config_defaults(**pki.kwargs, recursive=True) + # remove ACME default configuration if unused by CLI + if 'certificate' in pki: + for name, cert_config in pki['certificate'].items(): + if 'acme' not in cert_config: + # Remove ACME default values + del default_values['certificate'][name]['acme'] + + # merge CLI and default dictionary + pki = config_dict_merge(default_values, pki) + + # Certbot triggered an external renew of the certificates. + # Mark all ACME based certificates as "changed" to trigger + # update of dependent services + if 'certificate' in pki and 'certbot_renew' in pki: + renew = [] + for name, cert_config in pki['certificate'].items(): + if 'acme' in cert_config: + renew.append(name) + # If triggered externally by certbot, certificate key is not present in changed + if 'changed' not in pki: pki.update({'changed':{}}) + pki['changed'].update({'certificate' : renew}) # We need to get the entire system configuration to verify that we are not # deleting a certificate that is still referenced somewhere! @@ -119,38 +190,34 @@ def get_config(config=None): get_first_key=True, no_tag_node_value_mangle=True) - if 'changed' in pki: - for search in sync_search: - for key in search['keys']: - changed_key = sync_translate[key] - - if changed_key not in pki['changed']: - continue - - for item_name in pki['changed'][changed_key]: - node_present = False - if changed_key == 'openvpn': - node_present = dict_search_args(pki, 'openvpn', 'shared_secret', item_name) - else: - node_present = dict_search_args(pki, changed_key, item_name) - - if node_present: - search_dict = dict_search_args(pki['system'], *search['path']) - - if not search_dict: - continue - - for found_name, found_path in dict_search_recursive(search_dict, key): - if found_name == item_name: - path = search['path'] - path_str = ' '.join(path + found_path) - print(f'pki: Updating config: {path_str} {found_name}') - - if path[0] == 'interfaces': - ifname = found_path[0] - set_dependents(path[1], conf, ifname) - else: - set_dependents(path[1], conf) + for search in sync_search: + for key in search['keys']: + changed_key = sync_translate[key] + if 'changed' not in pki or changed_key not in pki['changed']: + continue + + for item_name in pki['changed'][changed_key]: + node_present = False + if changed_key == 'openvpn': + node_present = dict_search_args(pki, 'openvpn', 'shared_secret', item_name) + else: + node_present = dict_search_args(pki, changed_key, item_name) + + if node_present: + search_dict = dict_search_args(pki['system'], *search['path']) + if not search_dict: + continue + for found_name, found_path in dict_search_recursive(search_dict, key): + if found_name == item_name: + path = search['path'] + path_str = ' '.join(path + found_path) + print(f'pki: Updating config: {path_str} {found_name}') + + if path[0] == 'interfaces': + ifname = found_path[0] + set_dependents(path[1], conf, ifname) + else: + set_dependents(path[1], conf) return pki @@ -223,6 +290,22 @@ def verify(pki): if not is_valid_private_key(private['key'], protected): raise ConfigError(f'Invalid private key on certificate "{name}"') + if 'acme' in cert_conf: + if 'domain_name' not in cert_conf['acme']: + raise ConfigError(f'At least one domain-name is required to request '\ + f'certificate for "{name}" via ACME!') + + if 'email' not in cert_conf['acme']: + raise ConfigError(f'An email address is required to request '\ + f'certificate for "{name}" via ACME!') + + if 'certbot_renew' not in pki: + # Only run the ACME command if something on this entity changed, + # as this is time intensive + tmp = dict_search('changed.certificate', pki) + if tmp != None and name in tmp: + certbot_request(name, cert_conf['acme']) + if 'dh' in pki: for name, dh_conf in pki['dh'].items(): if 'parameters' in dh_conf: @@ -283,12 +366,50 @@ def generate(pki): if not pki: return None + # Certbot renewal only needs to re-trigger the services to load up the + # new PEM file + if 'certbot_renew' in pki: + return None + + # list of certificates issued via certbot + certbot_list = [] + if 'certificate' in pki: + for name, cert_conf in pki['certificate'].items(): + if 'acme' in cert_conf: + certbot_list.append(name) + # when something for the certificate changed, we should delete it + if name in dict_search('changed.certificate', pki): + certbot_delete(name) + certbot_request(name, cert_conf['acme'], dry_run=False) + + # Cleanup certbot configuration and certificates if no longer in use by CLI + # Get foldernames under vyos_certbot_dir which each represent a certbot cert + if os.path.exists(f'{vyos_certbot_dir}/live'): + for cert in [f.path.split('/')[-1] for f in os.scandir(f'{vyos_certbot_dir}/live') if f.is_dir()]: + if cert not in certbot_list: + # certificate is no longer active on the CLI - remove it + certbot_delete(cert) + return None def apply(pki): + systemd_certbot_name = 'certbot.timer' if not pki: + call(f'systemctl stop {systemd_certbot_name}') return None + has_certbot = False + if 'certificate' in pki: + for name, cert_conf in pki['certificate'].items(): + if 'acme' in cert_conf: + has_certbot = True + break + + if not has_certbot: + call(f'systemctl stop {systemd_certbot_name}') + elif has_certbot and not is_systemd_service_active(systemd_certbot_name): + call(f'systemctl restart {systemd_certbot_name}') + if 'changed' in pki: call_dependents() diff --git a/src/conf_mode/service_https.py b/src/conf_mode/service_https.py index cb40acc9f..2e7ebda5a 100755 --- a/src/conf_mode/service_https.py +++ b/src/conf_mode/service_https.py @@ -22,7 +22,6 @@ from copy import deepcopy from time import sleep import vyos.defaults -import vyos.certbot_util from vyos.base import Warning from vyos.config import Config @@ -33,8 +32,6 @@ from vyos.pki import wrap_certificate from vyos.pki import wrap_private_key from vyos.template import render from vyos.utils.process import call -from vyos.utils.process import is_systemd_service_running -from vyos.utils.process import is_systemd_service_active from vyos.utils.network import check_port_availability from vyos.utils.network import is_listen_port_bind_service from vyos.utils.file import write_file @@ -46,12 +43,11 @@ config_file = '/etc/nginx/sites-available/default' systemd_override = r'/run/systemd/system/nginx.service.d/override.conf' cert_dir = '/etc/ssl/certs' key_dir = '/etc/ssl/private' -certbot_dir = vyos.defaults.directories['certbot'] api_config_state = '/run/http-api-state' systemd_service = '/run/systemd/system/vyos-http-api.service' -# https config needs to coordinate several subsystems: api, certbot, +# https config needs to coordinate several subsystems: api, # self-signed certificate, as well as the virtual hosts defined within the # https config definition itself. Consequently, one needs a general dict, # encompassing the https and other configs, and a list of such virtual hosts @@ -63,7 +59,6 @@ default_server_block = { 'name' : ['_'], 'api' : False, 'vyos_cert' : {}, - 'certbot' : False } def get_config(config=None): @@ -80,7 +75,6 @@ def get_config(config=None): https = conf.get_config_dict(base, get_first_key=True, with_pki=True) - https['children_changed'] = diff.node_changed_children(base) https['api_add_or_delete'] = diff.node_changed_presence(base + ['api']) if 'api' not in https: @@ -100,7 +94,6 @@ def get_config(config=None): http_api['vrf'] = conf.return_value(vrf_path) https['api'] = http_api - return https def verify(https): @@ -119,25 +112,17 @@ def verify(https): cert_name = certificates['certificate'] if cert_name not in https['pki']['certificate']: - raise ConfigError("Invalid certificate on https configuration") + raise ConfigError('Invalid certificate on https configuration') pki_cert = https['pki']['certificate'][cert_name] if 'certificate' not in pki_cert: - raise ConfigError("Missing certificate on https configuration") + raise ConfigError('Missing certificate on https configuration') if 'private' not in pki_cert or 'key' not in pki_cert['private']: raise ConfigError("Missing certificate private key on https configuration") - - if 'certbot' in https['certificates']: - vhost_names = [] - for _, vh_conf in https.get('virtual-host', {}).items(): - vhost_names += vh_conf.get('server-name', []) - domains = https['certificates']['certbot'].get('domain-name', []) - domains_found = [domain for domain in domains if domain in vhost_names] - if not domains_found: - raise ConfigError("At least one 'virtual-host <id> server-name' " - "matching the 'certbot domain-name' is required.") + else: + Warning('No certificate specified, using buildin self-signed certificates!') server_block_list = [] @@ -255,22 +240,6 @@ def generate(https): for block in server_block_list: block['vyos_cert'] = vyos_cert_data - # letsencrypt certificate using certbot - - certbot = False - cert_domains = cert_dict.get('certbot', {}).get('domain-name', []) - if cert_domains: - certbot = True - for domain in cert_domains: - sub_list = vyos.certbot_util.choose_server_block(server_block_list, - domain) - if sub_list: - for sb in sub_list: - sb['certbot'] = True - sb['certbot_dir'] = certbot_dir - # certbot organizes certificates by first domain - sb['certbot_domain_dir'] = cert_domains[0] - if 'api' in list(https): vhost_list = https.get('api-restrict', {}).get('virtual-host', []) if not vhost_list: @@ -283,7 +252,6 @@ def generate(https): data = { 'server_block_list': server_block_list, - 'certbot': certbot } render(config_file, 'https/nginx.default.j2', data) @@ -297,27 +265,18 @@ def apply(https): https_service_name = 'nginx.service' if https is None: - if is_systemd_service_active(f'{http_api_service_name}'): - call(f'systemctl stop {http_api_service_name}') + call(f'systemctl stop {http_api_service_name}') call(f'systemctl stop {https_service_name}') return - if 'api' in https['children_changed']: - if 'api' in https: - if is_systemd_service_running(f'{http_api_service_name}'): - call(f'systemctl reload {http_api_service_name}') - else: - call(f'systemctl restart {http_api_service_name}') - # Let uvicorn settle before (possibly) restarting nginx - sleep(1) - else: - if is_systemd_service_active(f'{http_api_service_name}'): - call(f'systemctl stop {http_api_service_name}') + if 'api' in https: + call(f'systemctl reload-or-restart {http_api_service_name}') + # Let uvicorn settle before (possibly) restarting nginx + sleep(1) + else: + call(f'systemctl stop {http_api_service_name}') - if (not is_systemd_service_running(f'{https_service_name}') or - https['api_add_or_delete'] or - set(https['children_changed']) - set(['api'])): - call(f'systemctl restart {https_service_name}') + call(f'systemctl reload-or-restart {https_service_name}') if __name__ == '__main__': try: diff --git a/src/conf_mode/service_https_certificates_certbot.py b/src/conf_mode/service_https_certificates_certbot.py deleted file mode 100755 index 1a6a498de..000000000 --- a/src/conf_mode/service_https_certificates_certbot.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2019-2020 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -import sys -import os - -import vyos.defaults -from vyos.config import Config -from vyos import ConfigError -from vyos.utils.process import cmd -from vyos.utils.process import call -from vyos.utils.process import is_systemd_service_running - -from vyos import airbag -airbag.enable() - -vyos_conf_scripts_dir = vyos.defaults.directories['conf_mode'] -vyos_certbot_dir = vyos.defaults.directories['certbot'] - -dependencies = [ - 'service_https.py', -] - -def request_certbot(cert): - email = cert.get('email') - if email is not None: - email_flag = '-m {0}'.format(email) - else: - email_flag = '' - - domains = cert.get('domains') - if domains is not None: - domain_flag = '-d ' + ' -d '.join(domains) - else: - domain_flag = '' - - certbot_cmd = f'certbot certonly --config-dir {vyos_certbot_dir} -n --nginx --agree-tos --no-eff-email --expand {email_flag} {domain_flag}' - - cmd(certbot_cmd, - raising=ConfigError, - message="The certbot request failed for the specified domains.") - -def get_config(): - conf = Config() - if not conf.exists('service https certificates certbot'): - return None - else: - conf.set_level('service https certificates certbot') - - cert = {} - - if conf.exists('domain-name'): - cert['domains'] = conf.return_values('domain-name') - - if conf.exists('email'): - cert['email'] = conf.return_value('email') - - return cert - -def verify(cert): - if cert is None: - return None - - if 'domains' not in cert: - raise ConfigError("At least one domain name is required to" - " request a letsencrypt certificate.") - - if 'email' not in cert: - raise ConfigError("An email address is required to request" - " a letsencrypt certificate.") - -def generate(cert): - if cert is None: - return None - - # certbot will attempt to reload nginx, even with 'certonly'; - # start nginx if not active - if not is_systemd_service_running('nginx.service'): - call('systemctl start nginx.service') - - request_certbot(cert) - -def apply(cert): - if cert is not None: - call('systemctl restart certbot.timer') - else: - call('systemctl stop certbot.timer') - return None - - for dep in dependencies: - cmd(f'{vyos_conf_scripts_dir}/{dep}', raising=ConfigError) - -if __name__ == '__main__': - try: - c = get_config() - verify(c) - generate(c) - apply(c) - except ConfigError as e: - print(e) - sys.exit(1) diff --git a/src/etc/systemd/system/certbot.service.d/10-override.conf b/src/etc/systemd/system/certbot.service.d/10-override.conf new file mode 100644 index 000000000..542f77eb2 --- /dev/null +++ b/src/etc/systemd/system/certbot.service.d/10-override.conf @@ -0,0 +1,7 @@ +[Unit] +After= +After=vyos-router.service + +[Service] +ExecStart= +ExecStart=/usr/bin/certbot renew --config-dir /config/auth/letsencrypt --no-random-sleep-on-renew --post-hook "/usr/libexec/vyos/vyos-certbot-renew-pki.sh" diff --git a/src/helpers/vyos-certbot-renew-pki.sh b/src/helpers/vyos-certbot-renew-pki.sh new file mode 100755 index 000000000..d0b663f7b --- /dev/null +++ b/src/helpers/vyos-certbot-renew-pki.sh @@ -0,0 +1,3 @@ +#!/bin/sh +source /opt/vyatta/etc/functions/script-template +/usr/libexec/vyos/conf_mode/pki.py certbot_renew diff --git a/src/migration-scripts/https/5-to-6 b/src/migration-scripts/https/5-to-6 new file mode 100755 index 000000000..b4159f02f --- /dev/null +++ b/src/migration-scripts/https/5-to-6 @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# T5886: Add support for ACME protocol (LetsEncrypt), migrate https certbot +# to new "pki certificate" CLI tree + +import os +import sys + +from vyos.configtree import ConfigTree +from vyos.defaults import directories + +vyos_certbot_dir = directories['certbot'] + +if len(sys.argv) < 2: + print("Must specify file name!") + sys.exit(1) + +file_name = sys.argv[1] + +with open(file_name, 'r') as f: + config_file = f.read() + +config = ConfigTree(config_file) + +base = ['service', 'https', 'certificates'] +if not config.exists(base): + # Nothing to do + sys.exit(0) + +# both domain-name and email must be set on CLI - ensured by previous verify() +domain_names = config.return_values(base + ['certbot', 'domain-name']) +email = config.return_value(base + ['certbot', 'email']) +config.delete(base) + +# Set default certname based on domain-name +cert_name = 'https-' + domain_names[0].split('.')[0] +# Overwrite certname from previous certbot calls if available +if os.path.exists(f'{vyos_certbot_dir}/live'): + for cert in [f.path.split('/')[-1] for f in os.scandir(f'{vyos_certbot_dir}/live') if f.is_dir()]: + cert_name = cert + break + +for domain in domain_names: + config.set(['pki', 'certificate', cert_name, 'acme', 'domain-name'], value=domain, replace=False) + config.set(['pki', 'certificate', cert_name, 'acme', 'email'], value=email) + +# Update Webserver certificate +config.set(base + ['certificate'], value=cert_name) + +try: + with open(file_name, 'w') as f: + f.write(config.to_string()) +except OSError as e: + print("Failed to save the modified config: {}".format(e)) + sys.exit(1) diff --git a/src/op_mode/pki.py b/src/op_mode/pki.py index 6c854afb5..ad2c1ada0 100755 --- a/src/op_mode/pki.py +++ b/src/op_mode/pki.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021-2023 VyOS maintainers and contributors +# Copyright (C) 2021-2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -25,6 +25,7 @@ from cryptography import x509 from cryptography.x509.oid import ExtendedKeyUsageOID from vyos.config import Config +from vyos.config import config_dict_mangle_acme from vyos.pki import encode_certificate, encode_public_key, encode_private_key, encode_dh_parameters from vyos.pki import get_certificate_fingerprint from vyos.pki import create_certificate, create_certificate_request, create_certificate_revocation_list @@ -79,9 +80,14 @@ def get_config_certificate(name=None): if not conf.exists(base + ['private', 'key']) or not conf.exists(base + ['certificate']): return False - return conf.get_config_dict(base, key_mangling=('-', '_'), + pki = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) + if pki: + for certificate in pki: + pki[certificate] = config_dict_mangle_acme(certificate, pki[certificate]) + + return pki def get_certificate_ca(cert, ca_certs): # Find CA certificate for given certificate @@ -1073,7 +1079,9 @@ if __name__ == '__main__': show_crl(None if args.crl == 'all' else args.crl, args.pem) else: show_certificate_authority() + print('\n') show_certificate() + print('\n') show_crl() except KeyboardInterrupt: print("Aborted") |