DigitalOcean: use the v1.json endpoint

Per [1], DigitalOcean provides the metadata in multiple formats. The JSON
document is the preferred endpoint.

Changes:
- Switch to the v1.json meta-data endpoint
- Identify droplet identity from SMBIOS
- Only poll for metadata when the instance is confirmed to be a droplet
- Removal of hard-coded mirrors

Additionally, centralize the gates on running 'dmidecode' on arm arches,
and update tests to address.

[1] https://developers.digitalocean.com/documentation/metadata/

5 files changed, 71 insertions, 62 deletions

diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index a3529609..48136f7c 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -110,12 +110,6 @@ class DataSourceAltCloud(sources.DataSource):
 
         '''
 
-        uname_arch = os.uname()[4]
-        if uname_arch.startswith("arm") or uname_arch == "aarch64":
-            # Disabling because dmi data is not available on ARM processors
-            LOG.debug("Disabling AltCloud datasource on arm (LP: #1243287)")
-            return 'UNKNOWN'
-
         system_name = util.read_dmi_data("system-product-name")
         if not system_name:
             return 'UNKNOWN'
diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py
index d1f806d6..be74503b 100644
--- a/cloudinit/sources/DataSourceCloudSigma.py
+++ b/cloudinit/sources/DataSourceCloudSigma.py
@@ -16,7 +16,6 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 from base64 import b64decode
-import os
 import re
 
 from cloudinit.cs_utils import Cepko
@@ -45,11 +44,6 @@ class DataSourceCloudSigma(sources.DataSource):
         Uses dmi data to detect if this instance of cloud-init is running
         in the CloudSigma's infrastructure.
         """
-        uname_arch = os.uname()[4]
-        if uname_arch.startswith("arm") or uname_arch == "aarch64":
-            # Disabling because dmi data on ARM processors
-            LOG.debug("Disabling CloudSigma datasource on arm (LP: #1243287)")
-            return False
 
         LOG.debug("determining hypervisor product name via dmi data")
         sys_product_name = util.read_dmi_data("system-product-name")
diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py
index 44a17a00..fc596e17 100644
--- a/cloudinit/sources/DataSourceDigitalOcean.py
+++ b/cloudinit/sources/DataSourceDigitalOcean.py
@@ -1,6 +1,7 @@
 # vi: ts=4 expandtab
 #
 #    Author: Neal Shrader <neal@digitalocean.com>
+#    Author: Ben Howard  <bh@digitalocean.com>
 #
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License version 3, as
@@ -14,22 +15,27 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from cloudinit import ec2_utils
+# DigitalOcean Droplet API:
+# https://developers.digitalocean.com/documentation/metadata/
+
+import json
+
 from cloudinit import log as logging
 from cloudinit import sources
+from cloudinit import url_helper
 from cloudinit import util
 
-import functools
-
-
 LOG = logging.getLogger(__name__)
 
 BUILTIN_DS_CONFIG = {
-    'metadata_url': 'http://169.254.169.254/metadata/v1/',
-    'mirrors_url': 'http://mirrors.digitalocean.com/'
+    'metadata_url': 'http://169.254.169.254/metadata/v1.json',
 }
-MD_RETRIES = 0
-MD_TIMEOUT = 1
+
+# Wait for a up to a minute, retrying the meta-data server
+# every 2 seconds.
+MD_RETRIES = 30
+MD_TIMEOUT = 2
+MD_WAIT_RETRY = 2
 
 
 class DataSourceDigitalOcean(sources.DataSource):
@@ -40,43 +46,61 @@ class DataSourceDigitalOcean(sources.DataSource):
             util.get_cfg_by_path(sys_cfg, ["datasource", "DigitalOcean"], {}),
             BUILTIN_DS_CONFIG])
         self.metadata_address = self.ds_cfg['metadata_url']
+        self.retries = self.ds_cfg.get('retries', MD_RETRIES)
+        self.timeout = self.ds_cfg.get('timeout', MD_TIMEOUT)
+        self.wait_retry = self.ds_cfg.get('wait_retry', MD_WAIT_RETRY)
 
-        if self.ds_cfg.get('retries'):
-            self.retries = self.ds_cfg['retries']
-        else:
-            self.retries = MD_RETRIES
+    def _get_sysinfo(self):
+        # DigitalOcean embeds vendor ID and instance/droplet_id in the
+        # SMBIOS information
 
-        if self.ds_cfg.get('timeout'):
-            self.timeout = self.ds_cfg['timeout']
-        else:
-            self.timeout = MD_TIMEOUT
+        LOG.debug("checking if instance is a DigitalOcean droplet")
+
+        # Detect if we are on DigitalOcean and return the Droplet's ID
+        vendor_name = util.read_dmi_data("system-manufacturer")
+        if vendor_name != "DigitalOcean":
+            return (False, None)
 
-    def get_data(self):
-        caller = functools.partial(util.read_file_or_url,
-                                   timeout=self.timeout, retries=self.retries)
+        LOG.info("running on DigitalOcean")
 
-        def mcaller(url):
-            return caller(url).contents
+        droplet_id = util.read_dmi_data("system-serial-number")
+        if droplet_id:
+            LOG.debug(("system identified via SMBIOS as DigitalOcean Droplet"
+                       "{}").format(droplet_id))
+        else:
+            LOG.critical(("system identified via SMBIOS as a DigitalOcean "
+                          "Droplet, but did not provide an ID. Please file a "
+                          "support ticket at: "
+                          "https://cloud.digitalocean.com/support/tickets/"
+                          "new"))
 
-        md = ec2_utils.MetadataMaterializer(mcaller(self.metadata_address),
-                                            base_url=self.metadata_address,
-                                            caller=mcaller)
+        return (True, droplet_id)
 
-        self.metadata = md.materialize()
+    def get_data(self, apply_filter=False):
+        (is_do, droplet_id) = self._get_sysinfo()
 
-        if self.metadata.get('id'):
-            return True
-        else:
+        # only proceed if we know we are on DigitalOcean
+        if not is_do:
             return False
 
-    def get_userdata_raw(self):
-        return "\n".join(self.metadata['user-data'])
+        LOG.debug("reading metadata from {}".format(self.metadata_address))
+        response = url_helper.readurl(self.metadata_address,
+                                      timeout=self.timeout,
+                                      sec_between=self.wait_retry,
+                                      retries=self.retries)
 
-    def get_vendordata_raw(self):
-        return "\n".join(self.metadata['vendor-data'])
+        contents = util.decode_binary(response.contents)
+        decoded = json.loads(contents)
+
+        self.metadata = decoded
+        self.metadata['instance-id'] = decoded.get('droplet_id', droplet_id)
+        self.metadata['local-hostname'] = decoded.get('hostname', droplet_id)
+        self.vendordata_raw = decoded.get("vendor_data", None)
+        self.userdata_raw = decoded.get("user_data", None)
+        return True
 
     def get_public_ssh_keys(self):
-        public_keys = self.metadata['public-keys']
+        public_keys = self.metadata.get('public_keys', [])
         if isinstance(public_keys, list):
             return public_keys
         else:
@@ -84,21 +108,17 @@ class DataSourceDigitalOcean(sources.DataSource):
 
     @property
     def availability_zone(self):
-        return self.metadata['region']
-
-    def get_instance_id(self):
-        return self.metadata['id']
-
-    def get_hostname(self, fqdn=False, resolve_ip=False):
-        return self.metadata['hostname']
-
-    def get_package_mirror_info(self):
-        return self.ds_cfg['mirrors_url']
+        return self.metadata.get('region', 'default')
 
     @property
     def launch_index(self):
         return None
 
+    def check_instance_id(self, sys_cfg):
+        return sources.instance_id_matches_system_uuid(
+            self.get_instance_id(), 'system-serial-number')
+
+
 # Used to match classes to dependencies
 datasources = [
     (DataSourceDigitalOcean, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 39e7bbd9..143ab368 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -653,14 +653,8 @@ def write_boot_content(content, content_f, link=None, shebang=False,
             util.logexc(LOG, "failed establishing content link: %s", e)
 
 
-def get_smartos_environ(uname_version=None, product_name=None,
-                        uname_arch=None):
+def get_smartos_environ(uname_version=None, product_name=None):
     uname = os.uname()
-    if uname_arch is None:
-        uname_arch = uname[4]
-
-    if uname_arch.startswith("arm") or uname_arch == "aarch64":
-        return None
 
     # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but
     # report 'BrandZ virtual linux' as the kernel version
diff --git a/cloudinit/util.py b/cloudinit/util.py
index e5dd61a0..226628cc 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -2227,10 +2227,17 @@ def read_dmi_data(key):
 
     If all of the above fail to find a value, None will be returned.
     """
+
     syspath_value = _read_dmi_syspath(key)
     if syspath_value is not None:
         return syspath_value
 
+    # running dmidecode can be problematic on some arches (LP: #1243287)
+    uname_arch = os.uname()[4]
+    if uname_arch.startswith("arm") or uname_arch == "aarch64":
+        LOG.debug("dmidata is not supported on %s", uname_arch)
+        return None
+
     dmidecode_path = which('dmidecode')
     if dmidecode_path:
         return _call_dmidecode(key, dmidecode_path)