Datasource for UpCloud (#743)

New datasource utilizing UpCloud metadata API, including relevant unit
tests and documentation.

2 files changed, 396 insertions, 0 deletions

diff --git a/cloudinit/sources/DataSourceUpCloud.py b/cloudinit/sources/DataSourceUpCloud.py
new file mode 100644
index 00000000..209b9672
--- /dev/null
+++ b/cloudinit/sources/DataSourceUpCloud.py
@@ -0,0 +1,165 @@
+# Author: Antti Myyrä <antti.myyra@upcloud.com>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+# UpCloud server metadata API:
+# https://developers.upcloud.com/1.3/8-servers/#metadata-service
+
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import util
+from cloudinit import net as cloudnet
+from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError
+
+
+from cloudinit.sources.helpers import upcloud as uc_helper
+
+LOG = logging.getLogger(__name__)
+
+BUILTIN_DS_CONFIG = {"metadata_url": "http://169.254.169.254/metadata/v1.json"}
+
+# Wait for a up to a minute, retrying the meta-data server
+# every 2 seconds.
+MD_RETRIES = 30
+MD_TIMEOUT = 2
+MD_WAIT_RETRY = 2
+
+
+class DataSourceUpCloud(sources.DataSource):
+
+    dsname = "UpCloud"
+
+    # We'll perform DHCP setup only in init-local, see DataSourceUpCloudLocal
+    perform_dhcp_setup = False
+
+    def __init__(self, sys_cfg, distro, paths):
+        sources.DataSource.__init__(self, sys_cfg, distro, paths)
+        self.distro = distro
+        self.metadata = dict()
+        self.ds_cfg = util.mergemanydict(
+            [
+                util.get_cfg_by_path(sys_cfg, ["datasource", "UpCloud"], {}),
+                BUILTIN_DS_CONFIG,
+            ]
+        )
+        self.metadata_address = self.ds_cfg["metadata_url"]
+        self.retries = self.ds_cfg.get("retries", MD_RETRIES)
+        self.timeout = self.ds_cfg.get("timeout", MD_TIMEOUT)
+        self.wait_retry = self.ds_cfg.get("wait_retry", MD_WAIT_RETRY)
+        self._network_config = None
+
+    def _get_sysinfo(self):
+        return uc_helper.read_sysinfo()
+
+    def _read_metadata(self):
+        return uc_helper.read_metadata(
+            self.metadata_address,
+            timeout=self.timeout,
+            sec_between=self.wait_retry,
+            retries=self.retries,
+        )
+
+    def _get_data(self):
+        (is_upcloud, server_uuid) = self._get_sysinfo()
+
+        # only proceed if we know we are on UpCloud
+        if not is_upcloud:
+            return False
+
+        LOG.info("Running on UpCloud. server_uuid=%s", server_uuid)
+
+        if self.perform_dhcp_setup:  # Setup networking in init-local stage.
+            try:
+                LOG.debug("Finding a fallback NIC")
+                nic = cloudnet.find_fallback_nic()
+                LOG.debug("Discovering metadata via DHCP interface %s", nic)
+                with EphemeralDHCPv4(nic):
+                    md = util.log_time(
+                        logfunc=LOG.debug,
+                        msg="Reading from metadata service",
+                        func=self._read_metadata,
+                    )
+            except (NoDHCPLeaseError, sources.InvalidMetaDataException) as e:
+                util.logexc(LOG, str(e))
+                return False
+        else:
+            try:
+                LOG.debug(
+                    "Discovering metadata without DHCP-configured networking"
+                )
+                md = util.log_time(
+                    logfunc=LOG.debug,
+                    msg="Reading from metadata service",
+                    func=self._read_metadata,
+                )
+            except sources.InvalidMetaDataException as e:
+                util.logexc(LOG, str(e))
+                LOG.info(
+                    "No DHCP-enabled interfaces available, "
+                    "unable to fetch metadata for %s",
+                    server_uuid,
+                )
+                return False
+
+        self.metadata_full = md
+        self.metadata["instance-id"] = md.get("instance_id", server_uuid)
+        self.metadata["local-hostname"] = md.get("hostname")
+        self.metadata["network"] = md.get("network")
+        self.metadata["public-keys"] = md.get("public_keys")
+        self.metadata["availability_zone"] = md.get("region", "default")
+        self.vendordata_raw = md.get("vendor_data", None)
+        self.userdata_raw = md.get("user_data", None)
+
+        return True
+
+    def check_instance_id(self, sys_cfg):
+        return sources.instance_id_matches_system_uuid(self.get_instance_id())
+
+    @property
+    def network_config(self):
+        """
+        Configure the networking. This needs to be done each boot,
+        since the IP and interface information might have changed
+        due to reconfiguration.
+        """
+
+        if self._network_config:
+            return self._network_config
+
+        raw_network_config = self.metadata.get("network")
+        if not raw_network_config:
+            raise Exception("Unable to get network meta-data from server....")
+
+        self._network_config = uc_helper.convert_network_config(
+            raw_network_config,
+        )
+
+        return self._network_config
+
+
+class DataSourceUpCloudLocal(DataSourceUpCloud):
+    """
+    Run in init-local using a DHCP discovery prior to metadata crawl.
+
+    In init-local, no network is available. This subclass sets up minimal
+    networking with dhclient on a viable nic so that it can talk to the
+    metadata service. If the metadata service provides network configuration
+    then render the network configuration for that instance based on metadata.
+    """
+
+    perform_dhcp_setup = True  # Get metadata network config if present
+
+
+# Used to match classes to dependencies
+datasources = [
+    (DataSourceUpCloudLocal, (sources.DEP_FILESYSTEM, )),
+    (DataSourceUpCloud, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/sources/helpers/upcloud.py b/cloudinit/sources/helpers/upcloud.py
new file mode 100644
index 00000000..199baa58
--- /dev/null
+++ b/cloudinit/sources/helpers/upcloud.py
@@ -0,0 +1,231 @@
+# Author: Antti Myyrä <antti.myyra@upcloud.com>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import json
+
+from cloudinit import dmi
+from cloudinit import log as logging
+from cloudinit import net as cloudnet
+from cloudinit import url_helper
+
+LOG = logging.getLogger(__name__)
+
+
+def convert_to_network_config_v1(config):
+    """
+    Convert the UpCloud network metadata description into
+    Cloud-init's version 1 netconfig format.
+
+    Example JSON:
+    {
+      "interfaces": [
+        {
+          "index": 1,
+          "ip_addresses": [
+            {
+              "address": "94.237.105.53",
+              "dhcp": true,
+              "dns": [
+                "94.237.127.9",
+                "94.237.40.9"
+              ],
+              "family": "IPv4",
+              "floating": false,
+              "gateway": "94.237.104.1",
+              "network": "94.237.104.0/22"
+            },
+            {
+              "address": "94.237.105.50",
+              "dhcp": false,
+              "dns": [],
+              "family": "IPv4",
+              "floating": true,
+              "gateway": "",
+              "network": "94.237.105.50/32"
+            }
+          ],
+          "mac": "32:d5:ba:4a:36:e7",
+          "network_id": "031457f4-0f8c-483c-96f2-eccede02909c",
+          "type": "public"
+        },
+        {
+          "index": 2,
+          "ip_addresses": [
+            {
+              "address": "10.6.3.27",
+              "dhcp": true,
+              "dns": [],
+              "family": "IPv4",
+              "floating": false,
+              "gateway": "10.6.0.1",
+              "network": "10.6.0.0/22"
+            }
+          ],
+          "mac": "32:d5:ba:4a:84:cc",
+          "network_id": "03d82553-5bea-4132-b29a-e1cf67ec2dd1",
+          "type": "utility"
+        },
+        {
+          "index": 3,
+          "ip_addresses": [
+            {
+              "address": "2a04:3545:1000:720:38d6:baff:fe4a:63e7",
+              "dhcp": true,
+              "dns": [
+                "2a04:3540:53::1",
+                "2a04:3544:53::1"
+              ],
+              "family": "IPv6",
+              "floating": false,
+              "gateway": "2a04:3545:1000:720::1",
+              "network": "2a04:3545:1000:720::/64"
+            }
+          ],
+          "mac": "32:d5:ba:4a:63:e7",
+          "network_id": "03000000-0000-4000-8046-000000000000",
+          "type": "public"
+        },
+        {
+          "index": 4,
+          "ip_addresses": [
+            {
+              "address": "172.30.1.10",
+              "dhcp": true,
+              "dns": [],
+              "family": "IPv4",
+              "floating": false,
+              "gateway": "172.30.1.1",
+              "network": "172.30.1.0/24"
+            }
+          ],
+          "mac": "32:d5:ba:4a:8a:e1",
+          "network_id": "035a0a4a-77b4-4de5-820d-189fc8135714",
+          "type": "private"
+        }
+      ],
+      "dns": [
+        "94.237.127.9",
+        "94.237.40.9"
+      ]
+    }
+    """
+
+    def _get_subnet_config(ip_addr, dns):
+        if ip_addr.get("dhcp"):
+            dhcp_type = "dhcp"
+            if ip_addr.get("family") == "IPv6":
+                # UpCloud currently passes IPv6 addresses via
+                # StateLess Address Auto Configuration (SLAAC)
+                dhcp_type = "ipv6_dhcpv6-stateless"
+            return {"type": dhcp_type}
+
+        static_type = "static"
+        if ip_addr.get("family") == "IPv6":
+            static_type = "static6"
+        subpart = {
+            "type": static_type,
+            "control": "auto",
+            "address": ip_addr.get("address"),
+        }
+
+        if ip_addr.get("gateway"):
+            subpart["gateway"] = ip_addr.get("gateway")
+
+        if "/" in ip_addr.get("network"):
+            subpart["netmask"] = ip_addr.get("network").split("/")[1]
+
+        if dns != ip_addr.get("dns") and ip_addr.get("dns"):
+            subpart["dns_nameservers"] = ip_addr.get("dns")
+
+        return subpart
+
+    nic_configs = []
+    macs_to_interfaces = cloudnet.get_interfaces_by_mac()
+    LOG.debug("NIC mapping: %s", macs_to_interfaces)
+
+    for raw_iface in config.get("interfaces"):
+        LOG.debug("Considering %s", raw_iface)
+
+        mac_address = raw_iface.get("mac")
+        if mac_address not in macs_to_interfaces:
+            raise RuntimeError(
+                "Did not find network interface on system "
+                "with mac '%s'. Cannot apply configuration: %s"
+                % (mac_address, raw_iface)
+            )
+
+        iface_type = raw_iface.get("type")
+        sysfs_name = macs_to_interfaces.get(mac_address)
+
+        LOG.debug(
+            "Found %s interface '%s' with address '%s' (index %d)",
+            iface_type,
+            sysfs_name,
+            mac_address,
+            raw_iface.get("index"),
+        )
+
+        interface = {
+            "type": "physical",
+            "name": sysfs_name,
+            "mac_address": mac_address
+        }
+
+        subnets = []
+        for ip_address in raw_iface.get("ip_addresses"):
+            sub_part = _get_subnet_config(ip_address, config.get("dns"))
+            subnets.append(sub_part)
+
+        interface["subnets"] = subnets
+        nic_configs.append(interface)
+
+    if config.get("dns"):
+        LOG.debug("Setting DNS nameservers to %s", config.get("dns"))
+        nic_configs.append({
+            "type": "nameserver",
+            "address": config.get("dns")
+        })
+
+    return {"version": 1, "config": nic_configs}
+
+
+def convert_network_config(config):
+    return convert_to_network_config_v1(config)
+
+
+def read_metadata(url, timeout=2, sec_between=2, retries=30):
+    response = url_helper.readurl(
+        url, timeout=timeout, sec_between=sec_between, retries=retries
+    )
+    if not response.ok():
+        raise RuntimeError("unable to read metadata at %s" % url)
+    return json.loads(response.contents.decode())
+
+
+def read_sysinfo():
+    # UpCloud embeds vendor ID and server UUID in the
+    # SMBIOS information
+
+    # Detect if we are on UpCloud and return the UUID
+
+    vendor_name = dmi.read_dmi_data("system-manufacturer")
+    if vendor_name != "UpCloud":
+        return False, None
+
+    server_uuid = dmi.read_dmi_data("system-uuid")
+    if server_uuid:
+        LOG.debug(
+            "system identified via SMBIOS as UpCloud server: %s",
+            server_uuid
+        )
+    else:
+        msg = (
+            "system identified via SMBIOS as a UpCloud server, but "
+            "did not provide an ID. Please contact support via"
+            "https://hub.upcloud.com or via email with support@upcloud.com"
+        )
+        LOG.critical(msg)
+        raise RuntimeError(msg)
+
+    return True, server_uuid