pki: https: T3642: Migrate HTTPS to use PKI configuration

8 files changed, 201 insertions, 206 deletions

diff --git a/data/configd-include.json b/data/configd-include.json
index a03360bdb..3b4e2925b 100644
--- a/data/configd-include.json
+++ b/data/configd-include.json
@@ -69,6 +69,5 @@
 "vpn_pptp.py",
 "vpn_sstp.py",
 "vrf.py",
-"vrrp.py",
-"vyos_cert.py"
+"vrrp.py"
 ]
diff --git a/data/templates/https/nginx.default.tmpl b/data/templates/https/nginx.default.tmpl
index 916764410..5459fe98d 100644
--- a/data/templates/https/nginx.default.tmpl
+++ b/data/templates/https/nginx.default.tmpl
@@ -30,7 +30,8 @@ server {
         include {{ server.certbot_dir }}/options-ssl-nginx.conf;
         ssl_dhparam {{ server.certbot_dir }}/ssl-dhparams.pem;
 {% elif server.vyos_cert %}
-        include {{ server.vyos_cert.conf }};
+        ssl_certificate {{ server.vyos_cert.crt }};
+        ssl_certificate_key {{ server.vyos_cert.key }};
 {% else %}
         #
         # Self signed certs generated by the ssl-cert package
diff --git a/interface-definitions/https.xml.in b/interface-definitions/https.xml.in
index b613e30c1..b65a89b56 100644
--- a/interface-definitions/https.xml.in
+++ b/interface-definitions/https.xml.in
@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 <!-- HTTPS configuration -->
 <interfaceDefinition>
-  <syntaxVersion component='https' version='2'></syntaxVersion>
+  <syntaxVersion component='https' version='3'></syntaxVersion>
   <node name="service">
     <children>
       <node name="https" owner="${vyos_conf_scripts_dir}/https.py">
@@ -123,22 +123,7 @@
               <help>TLS certificates</help>
             </properties>
             <children>
-              <node name="system-generated-certificate" owner="${vyos_conf_scripts_dir}/vyos_cert.py">
-                <properties>
-                  <help>Use an automatically generated self-signed certificate</help>
-                </properties>
-                <children>
-                  <leafNode name="lifetime">
-                    <properties>
-                      <help>Lifetime in days; default is 365</help>
-                      <valueHelp>
-                        <format>1-65535</format>
-                        <description>Number of days</description>
-                      </valueHelp>
-                    </properties>
-                  </leafNode>
-                </children>
-              </node>
+              #include <include/pki/certificate.xml.i>
               <node name="certbot" owner="${vyos_conf_scripts_dir}/le_cert.py">
                 <properties>
                   <help>Request or apply a letsencrypt certificate for domain-name</help>
diff --git a/smoketest/configs/pki-misc b/smoketest/configs/pki-misc
index 45e6dd9b2..c90226a2a 100644
--- a/smoketest/configs/pki-misc
+++ b/smoketest/configs/pki-misc
@@ -3,6 +3,15 @@ interfaces {
         address 192.168.150.1/24
     }
 }
+service {
+    https {
+        certificates {
+            system-generated-certificate {
+                lifetime 365
+            }
+        }
+    }
+}
 system {
     config-management {
         commit-revisions 100
diff --git a/smoketest/scripts/cli/test_service_https.py b/smoketest/scripts/cli/test_service_https.py
index 3ed7655e9..3af63636a 100755
--- a/smoketest/scripts/cli/test_service_https.py
+++ b/smoketest/scripts/cli/test_service_https.py
@@ -22,14 +22,20 @@ from vyos.util import run
 
 base_path = ['service', 'https']
 
+pki_base = ['pki']
+cert_data = 'MIICFDCCAbugAwIBAgIUfMbIsB/ozMXijYgUYG80T1ry+mcwCgYIKoZIzj0EAwIwWTELMAkGA1UEBhMCR0IxEzARBgNVBAgMClNvbWUtU3RhdGUxEjAQBgNVBAcMCVNvbWUtQ2l0eTENMAsGA1UECgwEVnlPUzESMBAGA1UEAwwJVnlPUyBUZXN0MB4XDTIxMDcyMDEyNDUxMloXDTI2MDcxOTEyNDUxMlowWTELMAkGA1UEBhMCR0IxEzARBgNVBAgMClNvbWUtU3RhdGUxEjAQBgNVBAcMCVNvbWUtQ2l0eTENMAsGA1UECgwEVnlPUzESMBAGA1UEAwwJVnlPUyBUZXN0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE01HrLcNttqq4/PtoMua8rMWEkOdBu7vP94xzDO7A8C92ls1v86eePy4QllKCzIw3QxBIoCuH2peGRfWgPRdFsKNhMF8wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMB0GA1UdDgQWBBSu+JnU5ZC4mkuEpqg2+Mk4K79oeDAKBggqhkjOPQQDAgNHADBEAiBEFdzQ/Bc3LftzngrY605UhA6UprHhAogKgROv7iR4QgIgEFUxTtW3xXJcnUPWhhUFhyZoqfn8dE93+dm/LDnp7C0='
+key_data = 'MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgPLpD0Ohhoq0g4nhx2KMIuze7ucKUt/lBEB2wc03IxXyhRANCAATTUestw222qrj8+2gy5rysxYSQ50G7u8/3jHMM7sDwL3aWzW/zp54/LhCWUoLMjDdDEEigK4fal4ZF9aA9F0Ww'
+
 class TestHTTPSService(VyOSUnitTestSHIM.TestCase):
     def setUp(self):
         # ensure we can also run this test on a live system - so lets clean
         # out the current configuration :)
         self.cli_delete(base_path)
+        self.cli_delete(pki_base)
 
     def tearDown(self):
         self.cli_delete(base_path)
+        self.cli_delete(pki_base)
         self.cli_commit()
 
     def test_default(self):
@@ -56,5 +62,16 @@ class TestHTTPSService(VyOSUnitTestSHIM.TestCase):
         ret = run('sudo /usr/sbin/nginx -t')
         self.assertEqual(ret, 0)
 
+    def test_certificate(self):
+        self.cli_set(pki_base + ['certificate', 'test_https', 'certificate', cert_data])
+        self.cli_set(pki_base + ['certificate', 'test_https', 'private', 'key', key_data])
+
+        self.cli_set(base_path + ['certificates', 'certificate', 'test_https'])
+
+        self.cli_commit()
+
+        ret = run('sudo /usr/sbin/nginx -t')
+        self.assertEqual(ret, 0)
+
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/src/conf_mode/https.py b/src/conf_mode/https.py
index a6e2d9c8c..be4380462 100755
--- a/src/conf_mode/https.py
+++ b/src/conf_mode/https.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2019-2020 VyOS maintainers and contributors
+# Copyright (C) 2019-2021 VyOS maintainers and contributors
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 or later as
@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
 import sys
 
 from copy import deepcopy
@@ -23,13 +24,17 @@ import vyos.certbot_util
 
 from vyos.config import Config
 from vyos import ConfigError
-from vyos.util import call
+from vyos.pki import wrap_certificate
+from vyos.pki import wrap_private_key
 from vyos.template import render
+from vyos.util import call
 
 from vyos import airbag
 airbag.enable()
 
 config_file = '/etc/nginx/sites-available/default'
+cert_dir = '/etc/ssl/certs'
+key_dir = '/etc/ssl/private'
 certbot_dir = vyos.defaults.directories['certbot']
 
 # https config needs to coordinate several subsystems: api, certbot,
@@ -56,12 +61,58 @@ def get_config(config=None):
     if not conf.exists('service https'):
         return None
 
+    https = conf.get_config_dict('service https', get_first_key=True)
+
+    if https:
+        https['pki'] = conf.get_config_dict(['pki'], key_mangling=('-', '_'),
+                                get_first_key=True, no_tag_node_value_mangle=True)
+
+    return https
+
+def verify(https):
+    if https is None:
+        return None
+
+    if 'certificates' in https:
+        certificates = https['certificates']
+
+        if 'certificate' in certificates:
+            if not https['pki']:
+                raise ConfigError("PKI is not configured")
+
+            cert_name = certificates['certificate']
+
+            if cert_name not in https['pki']['certificate']:
+                raise ConfigError("Invalid certificate on https configuration")
+
+            pki_cert = https['pki']['certificate'][cert_name]
+
+            if 'certificate' not in pki_cert:
+                raise ConfigError("Missing certificate on https configuration")
+
+            if 'private' not in pki_cert or 'key' not in pki_cert['private']:
+                raise ConfigError("Missing certificate private key on https configuration")
+
+        if 'certbot' in https['certificates']:
+            vhost_names = []
+            for vh, vh_conf in https.get('virtual-host', {}).items():
+                vhost_names += vh_conf.get('server-name', [])
+            domains = https['certificates']['certbot'].get('domain-name', [])
+            domains_found = [domain for domain in domains if domain in vhost_names]
+            if not domains_found:
+                raise ConfigError("At least one 'virtual-host <id> server-name' "
+                              "matching the 'certbot domain-name' is required.")
+    return None
+
+def generate(https):
+    if https is None:
+        return None
+
     server_block_list = []
-    https_dict = conf.get_config_dict('service https', get_first_key=True)
 
     # organize by vhosts
 
-    vhost_dict = https_dict.get('virtual-host', {})
+    vhost_dict = https.get('virtual-host', {})
 
     if not vhost_dict:
         # no specified virtual hosts (server blocks); use default
@@ -79,18 +130,30 @@ def get_config(config=None):
 
     # get certificate data
 
-    cert_dict = https_dict.get('certificates', {})
+    cert_dict = https.get('certificates', {})
+
+    if 'certificate' in cert_dict:
+        cert_name = cert_dict['certificate']
+        pki_cert = https['pki']['certificate'][cert_name]
+
+        cert_path = os.path.join(cert_dir, f'{cert_name}.pem')
+        key_path = os.path.join(key_dir, f'{cert_name}.pem')
+
+        with open(cert_path, 'w') as f:
+            f.write(wrap_certificate(pki_cert['certificate']))
+
+        with open(key_path, 'w') as f:
+            f.write(wrap_private_key(pki_cert['private']['key']))
 
-        # self-signed certificate
+        vyos_cert_data = {
+            "crt": cert_path,
+            "key": key_path
+        }
 
-    vyos_cert_data = {}
-    if 'system-generated-certificate' in list(cert_dict):
-        vyos_cert_data = vyos.defaults.vyos_cert_data
-    if vyos_cert_data:
         for block in server_block_list:
             block['vyos_cert'] = vyos_cert_data
 
-        # letsencrypt certificate using certbot
+    # letsencrypt certificate using certbot
 
     certbot = False
     cert_domains = cert_dict.get('certbot', {}).get('domain-name', [])
@@ -110,15 +173,15 @@ def get_config(config=None):
 
     api_set = False
     api_data = {}
-    if 'api' in list(https_dict):
+    if 'api' in list(https):
         api_set = True
         api_data = vyos.defaults.api_data
-    api_settings = https_dict.get('api', {})
+    api_settings = https.get('api', {})
     if api_settings:
         port = api_settings.get('port', '')
         if port:
             api_data['port'] = port
-        vhosts = https_dict.get('api-restrict', {}).get('virtual-host', [])
+        vhosts = https.get('api-restrict', {}).get('virtual-host', [])
         if vhosts:
             api_data['vhost'] = vhosts[:]
 
@@ -132,34 +195,16 @@ def get_config(config=None):
                 if block['id'] in vhost_list:
                     block['api'] = api_data
 
-    # return dict for use in template
-
-    https = {'server_block_list' : server_block_list,
-             'api_set': api_set,
-             'certbot': certbot}
-
-    return https
-
-def verify(https):
-    if https is None:
-        return None
-
-    if https['certbot']:
-        for sb in https['server_block_list']:
-            if sb['certbot']:
-                return None
-        raise ConfigError("At least one 'virtual-host <id> server-name' "
-                          "matching the 'certbot domain-name' is required.")
-    return None
-
-def generate(https):
-    if https is None:
-        return None
-
     if 'server_block_list' not in https or not https['server_block_list']:
         https['server_block_list'] = [default_server_block]
 
-    render(config_file, 'https/nginx.default.tmpl', https)
+    data = {
+        'server_block_list': server_block_list,
+        'api_set': api_set,
+        'certbot': certbot
+    }
+
+    render(config_file, 'https/nginx.default.tmpl', data)
 
     return None
 
diff --git a/src/conf_mode/vyos_cert.py b/src/conf_mode/vyos_cert.py
deleted file mode 100755
index dc7c64684..000000000
--- a/src/conf_mode/vyos_cert.py
+++ /dev/null
@@ -1,147 +0,0 @@
-#!/usr/bin/env python3
-#
-# Copyright (C) 2019 VyOS maintainers and contributors
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 or later as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-import sys
-import os
-import tempfile
-import pathlib
-import ssl
-
-import vyos.defaults
-from vyos.config import Config
-from vyos import ConfigError
-from vyos.util import cmd
-
-from vyos import airbag
-airbag.enable()
-
-vyos_conf_scripts_dir = vyos.defaults.directories['conf_mode']
-
-# XXX: this model will need to be extended for tag nodes
-dependencies = [
-    'https.py',
-]
-
-def status_self_signed(cert_data):
-# check existence and expiration date
-    path = pathlib.Path(cert_data['conf'])
-    if not path.is_file():
-        return False
-    path = pathlib.Path(cert_data['crt'])
-    if not path.is_file():
-        return False
-    path = pathlib.Path(cert_data['key'])
-    if not path.is_file():
-        return False
-
-    # check if certificate is 1/2 past lifetime, with openssl -checkend
-    end_days = int(cert_data['lifetime'])
-    end_seconds = int(0.5*60*60*24*end_days)
-    checkend_cmd = 'openssl x509 -checkend {end} -noout -in {crt}'.format(end=end_seconds, **cert_data)
-    try:
-        cmd(checkend_cmd, message='Called process error')
-        return True
-    except OSError as err:
-        if err.errno == 1:
-            return False
-        print(err)
-        # XXX: This seems wrong to continue on failure
-        # implicitely returning None
-
-def generate_self_signed(cert_data):
-    san_config = None
-
-    if ssl.OPENSSL_VERSION_INFO < (1, 1, 1, 0, 0):
-        san_config = tempfile.NamedTemporaryFile()
-        with open(san_config.name, 'w') as fd:
-            fd.write('[req]\n')
-            fd.write('distinguished_name=req\n')
-            fd.write('[san]\n')
-            fd.write('subjectAltName=DNS:vyos\n')
-
-        openssl_req_cmd = ('openssl req -x509 -nodes -days {lifetime} '
-                           '-newkey rsa:4096 -keyout {key} -out {crt} '
-                           '-subj "/O=Sentrium/OU=VyOS/CN=vyos" '
-                           '-extensions san -config {san_conf}'
-                           ''.format(san_conf=san_config.name,
-                                     **cert_data))
-
-    else:
-        openssl_req_cmd = ('openssl req -x509 -nodes -days {lifetime} '
-                           '-newkey rsa:4096 -keyout {key} -out {crt} '
-                           '-subj "/O=Sentrium/OU=VyOS/CN=vyos" '
-                           '-addext "subjectAltName=DNS:vyos"'
-                           ''.format(**cert_data))
-
-    try:
-        cmd(openssl_req_cmd, message='Called process error')
-    except OSError as err:
-        print(err)
-        # XXX: seems wrong to ignore the failure
-
-    os.chmod('{key}'.format(**cert_data), 0o400)
-
-    with open('{conf}'.format(**cert_data), 'w') as f:
-        f.write('ssl_certificate {crt};\n'.format(**cert_data))
-        f.write('ssl_certificate_key {key};\n'.format(**cert_data))
-
-    if san_config:
-        san_config.close()
-
-def get_config(config=None):
-    vyos_cert = vyos.defaults.vyos_cert_data
-
-    if config:
-        conf = config
-    else:
-        conf = Config()
-    if not conf.exists('service https certificates system-generated-certificate'):
-        return None
-    else:
-        conf.set_level('service https certificates system-generated-certificate')
-
-    if conf.exists('lifetime'):
-        lifetime = conf.return_value('lifetime')
-        vyos_cert['lifetime'] = lifetime
-
-    return vyos_cert
-
-def verify(vyos_cert):
-    return None
-
-def generate(vyos_cert):
-    if vyos_cert is None:
-        return None
-
-    if not status_self_signed(vyos_cert):
-        generate_self_signed(vyos_cert)
-
-def apply(vyos_cert):
-    for dep in dependencies:
-        command = '{0}/{1}'.format(vyos_conf_scripts_dir, dep)
-        cmd(command, raising=ConfigError)
-
-if __name__ == '__main__':
-    try:
-        c = get_config()
-        verify(c)
-        generate(c)
-        apply(c)
-    except ConfigError as e:
-        print(e)
-        sys.exit(1)
diff --git a/src/migration-scripts/https/2-to-3 b/src/migration-scripts/https/2-to-3
new file mode 100755
index 000000000..fa29fdd18
--- /dev/null
+++ b/src/migration-scripts/https/2-to-3
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2021 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# * Migrate system signed certificate to use PKI
+
+import sys
+
+from vyos.configtree import ConfigTree
+from vyos.pki import create_certificate
+from vyos.pki import create_certificate_request
+from vyos.pki import create_private_key
+from vyos.pki import encode_certificate
+from vyos.pki import encode_private_key
+
+if (len(sys.argv) < 2):
+    print("Must specify file name!")
+    sys.exit(1)
+
+file_name = sys.argv[1]
+
+with open(file_name, 'r') as f:
+    config_file = f.read()
+
+config = ConfigTree(config_file)
+
+base = ['service', 'https', 'certificates']
+pki_base = ['pki']
+
+if not config.exists(base + ['system-generated-certificate']):
+    sys.exit(0)
+
+def wrapped_pem_to_config_value(pem):
+    out = []
+    for line in pem.strip().split("\n"):
+        if not line or line.startswith("-----") or line[0] == '#':
+            continue
+        out.append(line)
+    return "".join(out)
+
+if not config.exists(pki_base + ['certificate']):
+    config.set(pki_base + ['certificate'])
+    config.set_tag(pki_base + ['certificate'])
+
+valid_days = 365
+if config.exists(base + ['system-generated-certificate', 'lifetime']):
+    valid_days = int(config.return_value(base + ['system-generated-certificate', 'lifetime']))
+
+key = create_private_key('rsa', 2048)
+subject = {'country': 'GB', 'state': 'N/A', 'locality': 'N/A', 'organization': 'VyOS', 'common_name': 'vyos'}
+cert_req = create_certificate_request(subject, key, ['vyos'])
+cert = create_certificate(cert_req, cert_req, key, valid_days)
+
+if cert:
+    cert_pem = encode_certificate(cert)
+    config.set(pki_base + ['certificate', 'generated_https', 'certificate'], value=wrapped_pem_to_config_value(cert_pem))
+
+if key:
+    key_pem = encode_private_key(key)
+    config.set(pki_base + ['certificate', 'generated_https', 'private', 'key'], value=wrapped_pem_to_config_value(key_pem))
+
+if cert and key:
+    config.set(base + ['certificate'], value='generated_https')
+else:
+    print('Failed to migrate system-generated-certificate from https service')
+
+config.delete(base + ['system-generated-certificate'])
+
+try:
+    with open(file_name, 'w') as f:
+        f.write(config.to_string())
+except OSError as e:
+    print("Failed to save the modified config: {}".format(e))
+    sys.exit(1)