DigitalOcean: use the v1.json endpoint

Per [1], DigitalOcean provides the metadata in multiple formats. The JSON
document is the preferred endpoint.

Changes:
- Switch to the v1.json meta-data endpoint
- Identify droplet identity from SMBIOS
- Only poll for metadata when the instance is confirmed to be a droplet
- Removal of hard-coded mirrors

Additionally, centralize the gates on running 'dmidecode' on arm arches,
and update tests to address.

[1] https://developers.digitalocean.com/documentation/metadata/

7 files changed, 122 insertions, 104 deletions

diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index a3529609..48136f7c 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -110,12 +110,6 @@ class DataSourceAltCloud(sources.DataSource):
 
         '''
 
-        uname_arch = os.uname()[4]
-        if uname_arch.startswith("arm") or uname_arch == "aarch64":
-            # Disabling because dmi data is not available on ARM processors
-            LOG.debug("Disabling AltCloud datasource on arm (LP: #1243287)")
-            return 'UNKNOWN'
-
         system_name = util.read_dmi_data("system-product-name")
         if not system_name:
             return 'UNKNOWN'
diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py
index d1f806d6..be74503b 100644
--- a/cloudinit/sources/DataSourceCloudSigma.py
+++ b/cloudinit/sources/DataSourceCloudSigma.py
@@ -16,7 +16,6 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 from base64 import b64decode
-import os
 import re
 
 from cloudinit.cs_utils import Cepko
@@ -45,11 +44,6 @@ class DataSourceCloudSigma(sources.DataSource):
         Uses dmi data to detect if this instance of cloud-init is running
         in the CloudSigma's infrastructure.
         """
-        uname_arch = os.uname()[4]
-        if uname_arch.startswith("arm") or uname_arch == "aarch64":
-            # Disabling because dmi data on ARM processors
-            LOG.debug("Disabling CloudSigma datasource on arm (LP: #1243287)")
-            return False
 
         LOG.debug("determining hypervisor product name via dmi data")
         sys_product_name = util.read_dmi_data("system-product-name")
diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py
index 44a17a00..fc596e17 100644
--- a/cloudinit/sources/DataSourceDigitalOcean.py
+++ b/cloudinit/sources/DataSourceDigitalOcean.py
@@ -1,6 +1,7 @@
 # vi: ts=4 expandtab
 #
 #    Author: Neal Shrader <neal@digitalocean.com>
+#    Author: Ben Howard  <bh@digitalocean.com>
 #
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License version 3, as
@@ -14,22 +15,27 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from cloudinit import ec2_utils
+# DigitalOcean Droplet API:
+# https://developers.digitalocean.com/documentation/metadata/
+
+import json
+
 from cloudinit import log as logging
 from cloudinit import sources
+from cloudinit import url_helper
 from cloudinit import util
 
-import functools
-
-
 LOG = logging.getLogger(__name__)
 
 BUILTIN_DS_CONFIG = {
-    'metadata_url': 'http://169.254.169.254/metadata/v1/',
-    'mirrors_url': 'http://mirrors.digitalocean.com/'
+    'metadata_url': 'http://169.254.169.254/metadata/v1.json',
 }
-MD_RETRIES = 0
-MD_TIMEOUT = 1
+
+# Wait for a up to a minute, retrying the meta-data server
+# every 2 seconds.
+MD_RETRIES = 30
+MD_TIMEOUT = 2
+MD_WAIT_RETRY = 2
 
 
 class DataSourceDigitalOcean(sources.DataSource):
@@ -40,43 +46,61 @@ class DataSourceDigitalOcean(sources.DataSource):
             util.get_cfg_by_path(sys_cfg, ["datasource", "DigitalOcean"], {}),
             BUILTIN_DS_CONFIG])
         self.metadata_address = self.ds_cfg['metadata_url']
+        self.retries = self.ds_cfg.get('retries', MD_RETRIES)
+        self.timeout = self.ds_cfg.get('timeout', MD_TIMEOUT)
+        self.wait_retry = self.ds_cfg.get('wait_retry', MD_WAIT_RETRY)
 
-        if self.ds_cfg.get('retries'):
-            self.retries = self.ds_cfg['retries']
-        else:
-            self.retries = MD_RETRIES
+    def _get_sysinfo(self):
+        # DigitalOcean embeds vendor ID and instance/droplet_id in the
+        # SMBIOS information
 
-        if self.ds_cfg.get('timeout'):
-            self.timeout = self.ds_cfg['timeout']
-        else:
-            self.timeout = MD_TIMEOUT
+        LOG.debug("checking if instance is a DigitalOcean droplet")
+
+        # Detect if we are on DigitalOcean and return the Droplet's ID
+        vendor_name = util.read_dmi_data("system-manufacturer")
+        if vendor_name != "DigitalOcean":
+            return (False, None)
 
-    def get_data(self):
-        caller = functools.partial(util.read_file_or_url,
-                                   timeout=self.timeout, retries=self.retries)
+        LOG.info("running on DigitalOcean")
 
-        def mcaller(url):
-            return caller(url).contents
+        droplet_id = util.read_dmi_data("system-serial-number")
+        if droplet_id:
+            LOG.debug(("system identified via SMBIOS as DigitalOcean Droplet"
+                       "{}").format(droplet_id))
+        else:
+            LOG.critical(("system identified via SMBIOS as a DigitalOcean "
+                          "Droplet, but did not provide an ID. Please file a "
+                          "support ticket at: "
+                          "https://cloud.digitalocean.com/support/tickets/"
+                          "new"))
 
-        md = ec2_utils.MetadataMaterializer(mcaller(self.metadata_address),
-                                            base_url=self.metadata_address,
-                                            caller=mcaller)
+        return (True, droplet_id)
 
-        self.metadata = md.materialize()
+    def get_data(self, apply_filter=False):
+        (is_do, droplet_id) = self._get_sysinfo()
 
-        if self.metadata.get('id'):
-            return True
-        else:
+        # only proceed if we know we are on DigitalOcean
+        if not is_do:
             return False
 
-    def get_userdata_raw(self):
-        return "\n".join(self.metadata['user-data'])
+        LOG.debug("reading metadata from {}".format(self.metadata_address))
+        response = url_helper.readurl(self.metadata_address,
+                                      timeout=self.timeout,
+                                      sec_between=self.wait_retry,
+                                      retries=self.retries)
 
-    def get_vendordata_raw(self):
-        return "\n".join(self.metadata['vendor-data'])
+        contents = util.decode_binary(response.contents)
+        decoded = json.loads(contents)
+
+        self.metadata = decoded
+        self.metadata['instance-id'] = decoded.get('droplet_id', droplet_id)
+        self.metadata['local-hostname'] = decoded.get('hostname', droplet_id)
+        self.vendordata_raw = decoded.get("vendor_data", None)
+        self.userdata_raw = decoded.get("user_data", None)
+        return True
 
     def get_public_ssh_keys(self):
-        public_keys = self.metadata['public-keys']
+        public_keys = self.metadata.get('public_keys', [])
         if isinstance(public_keys, list):
             return public_keys
         else:
@@ -84,21 +108,17 @@ class DataSourceDigitalOcean(sources.DataSource):
 
     @property
     def availability_zone(self):
-        return self.metadata['region']
-
-    def get_instance_id(self):
-        return self.metadata['id']
-
-    def get_hostname(self, fqdn=False, resolve_ip=False):
-        return self.metadata['hostname']
-
-    def get_package_mirror_info(self):
-        return self.ds_cfg['mirrors_url']
+        return self.metadata.get('region', 'default')
 
     @property
     def launch_index(self):
         return None
 
+    def check_instance_id(self, sys_cfg):
+        return sources.instance_id_matches_system_uuid(
+            self.get_instance_id(), 'system-serial-number')
+
+
 # Used to match classes to dependencies
 datasources = [
     (DataSourceDigitalOcean, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 39e7bbd9..143ab368 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -653,14 +653,8 @@ def write_boot_content(content, content_f, link=None, shebang=False,
             util.logexc(LOG, "failed establishing content link: %s", e)
 
 
-def get_smartos_environ(uname_version=None, product_name=None,
-                        uname_arch=None):
+def get_smartos_environ(uname_version=None, product_name=None):
     uname = os.uname()
-    if uname_arch is None:
-        uname_arch = uname[4]
-
-    if uname_arch.startswith("arm") or uname_arch == "aarch64":
-        return None
 
     # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but
     # report 'BrandZ virtual linux' as the kernel version
diff --git a/cloudinit/util.py b/cloudinit/util.py
index e5dd61a0..226628cc 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -2227,10 +2227,17 @@ def read_dmi_data(key):
 
     If all of the above fail to find a value, None will be returned.
     """
+
     syspath_value = _read_dmi_syspath(key)
     if syspath_value is not None:
         return syspath_value
 
+    # running dmidecode can be problematic on some arches (LP: #1243287)
+    uname_arch = os.uname()[4]
+    if uname_arch.startswith("arm") or uname_arch == "aarch64":
+        LOG.debug("dmidata is not supported on %s", uname_arch)
+        return None
+
     dmidecode_path = which('dmidecode')
     if dmidecode_path:
         return _call_dmidecode(key, dmidecode_path)
diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py
index 8936a1e3..f5d2ef35 100644
--- a/tests/unittests/test_datasource/test_digitalocean.py
+++ b/tests/unittests/test_datasource/test_digitalocean.py
@@ -15,68 +15,58 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import re
-
-from six.moves.urllib_parse import urlparse
+import json
 
 from cloudinit import helpers
 from cloudinit import settings
 from cloudinit.sources import DataSourceDigitalOcean
 
 from .. import helpers as test_helpers
+from ..helpers import HttprettyTestCase
 
 httpretty = test_helpers.import_httpretty()
 
-# Abbreviated for the test
-DO_INDEX = """id
-           hostname
-           user-data
-           vendor-data
-           public-keys
-           region"""
-
-DO_MULTIPLE_KEYS = """ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@digitalocean.com
-                   ssh-rsa AAAAB3NzaC1yc2EAAAA... neal2@digitalocean.com"""
-DO_SINGLE_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@digitalocean.com"
+DO_MULTIPLE_KEYS = ["ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@do.co",
+                    "ssh-rsa AAAAB3NzaC1yc2EAAAA... test2@do.co"]
+DO_SINGLE_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAA... test@do.co"
 
 DO_META = {
-    '': DO_INDEX,
-    'user-data': '#!/bin/bash\necho "user-data"',
-    'vendor-data': '#!/bin/bash\necho "vendor-data"',
-    'public-keys': DO_SINGLE_KEY,
+    'user_data': 'user_data_here',
+    'vendor_data': 'vendor_data_here',
+    'public_keys': DO_SINGLE_KEY,
     'region': 'nyc3',
     'id': '2000000',
     'hostname': 'cloudinit-test',
 }
 
-MD_URL_RE = re.compile(r'http://169.254.169.254/metadata/v1/.*')
+MD_URL = 'http://169.254.169.254/metadata/v1.json'
+
+
+def _mock_dmi():
+    return (True, DO_META.get('id'))
 
 
 def _request_callback(method, uri, headers):
-    url_path = urlparse(uri).path
-    if url_path.startswith('/metadata/v1/'):
-        path = url_path.split('/metadata/v1/')[1:][0]
-    else:
-        path = None
-    if path in DO_META:
-        return (200, headers, DO_META.get(path))
-    else:
-        return (404, headers, '')
+    return (200, headers, json.dumps(DO_META))
 
 
-class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase):
+class TestDataSourceDigitalOcean(HttprettyTestCase):
+    """
+    Test reading the meta-data
+    """
 
     def setUp(self):
         self.ds = DataSourceDigitalOcean.DataSourceDigitalOcean(
             settings.CFG_BUILTIN, None,
             helpers.Paths({}))
+        self.ds._get_sysinfo = _mock_dmi
         super(TestDataSourceDigitalOcean, self).setUp()
 
     @httpretty.activate
     def test_connection(self):
         httpretty.register_uri(
-            httpretty.GET, MD_URL_RE,
-            body=_request_callback)
+            httpretty.GET, MD_URL,
+            body=json.dumps(DO_META))
 
         success = self.ds.get_data()
         self.assertTrue(success)
@@ -84,14 +74,14 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase):
     @httpretty.activate
     def test_metadata(self):
         httpretty.register_uri(
-            httpretty.GET, MD_URL_RE,
+            httpretty.GET, MD_URL,
             body=_request_callback)
         self.ds.get_data()
 
-        self.assertEqual(DO_META.get('user-data'),
+        self.assertEqual(DO_META.get('user_data'),
                          self.ds.get_userdata_raw())
 
-        self.assertEqual(DO_META.get('vendor-data'),
+        self.assertEqual(DO_META.get('vendor_data'),
                          self.ds.get_vendordata_raw())
 
         self.assertEqual(DO_META.get('region'),
@@ -103,11 +93,8 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase):
         self.assertEqual(DO_META.get('hostname'),
                          self.ds.get_hostname())
 
-        self.assertEqual('http://mirrors.digitalocean.com/',
-                         self.ds.get_package_mirror_info())
-
         # Single key
-        self.assertEqual([DO_META.get('public-keys')],
+        self.assertEqual([DO_META.get('public_keys')],
                          self.ds.get_public_ssh_keys())
 
         self.assertIsInstance(self.ds.get_public_ssh_keys(), list)
@@ -116,12 +103,12 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase):
     def test_multiple_ssh_keys(self):
         DO_META['public_keys'] = DO_MULTIPLE_KEYS
         httpretty.register_uri(
-            httpretty.GET, MD_URL_RE,
+            httpretty.GET, MD_URL,
             body=_request_callback)
         self.ds.get_data()
 
         # Multiple keys
-        self.assertEqual(DO_META.get('public-keys').splitlines(),
+        self.assertEqual(DO_META.get('public_keys'),
                          self.ds.get_public_ssh_keys())
 
         self.assertIsInstance(self.ds.get_public_ssh_keys(), list)
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index 37a984ac..73369cd3 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -371,8 +371,30 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase):
         self._create_sysfs_parent_directory()
         expected_dmi_value = 'dmidecode-used'
         self._configure_dmidecode_return('use-dmidecode', expected_dmi_value)
-        self.assertEqual(expected_dmi_value,
-                         util.read_dmi_data('use-dmidecode'))
+        with mock.patch("cloudinit.util.os.uname") as m_uname:
+            m_uname.return_value = ('x-sysname', 'x-nodename',
+                                    'x-release', 'x-version', 'x86_64')
+            self.assertEqual(expected_dmi_value,
+                             util.read_dmi_data('use-dmidecode'))
+
+    def test_dmidecode_not_used_on_arm(self):
+        self.patch_mapping({})
+        self._create_sysfs_parent_directory()
+        dmi_val = 'from-dmidecode'
+        dmi_name = 'use-dmidecode'
+        self._configure_dmidecode_return(dmi_name, dmi_val)
+
+        expected = {'armel': None, 'aarch64': None, 'x86_64': dmi_val}
+        found = {}
+        # we do not run the 'dmi-decode' binary on some arches
+        # verify that anything requested that is not in the sysfs dir
+        # will return None on those arches.
+        with mock.patch("cloudinit.util.os.uname") as m_uname:
+            for arch in expected:
+                m_uname.return_value = ('x-sysname', 'x-nodename',
+                                        'x-release', 'x-version', arch)
+                found[arch] = util.read_dmi_data(dmi_name)
+        self.assertEqual(expected, found)
 
     def test_none_returned_if_neither_source_has_data(self):
         self.patch_mapping({})