diff options
| -rw-r--r-- | cloudinit/sources/DataSourceAltCloud.py | 6 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceCloudSigma.py | 6 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceDigitalOcean.py | 106 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceSmartOS.py | 8 | ||||
| -rw-r--r-- | cloudinit/util.py | 7 | ||||
| -rw-r--r-- | tests/unittests/test_datasource/test_digitalocean.py | 67 | ||||
| -rw-r--r-- | tests/unittests/test_util.py | 26 |
7 files changed, 122 insertions, 104 deletions
diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index a3529609..48136f7c 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -110,12 +110,6 @@ class DataSourceAltCloud(sources.DataSource): ''' - uname_arch = os.uname()[4] - if uname_arch.startswith("arm") or uname_arch == "aarch64": - # Disabling because dmi data is not available on ARM processors - LOG.debug("Disabling AltCloud datasource on arm (LP: #1243287)") - return 'UNKNOWN' - system_name = util.read_dmi_data("system-product-name") if not system_name: return 'UNKNOWN' diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index d1f806d6..be74503b 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -16,7 +16,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. from base64 import b64decode -import os import re from cloudinit.cs_utils import Cepko @@ -45,11 +44,6 @@ class DataSourceCloudSigma(sources.DataSource): Uses dmi data to detect if this instance of cloud-init is running in the CloudSigma's infrastructure. """ - uname_arch = os.uname()[4] - if uname_arch.startswith("arm") or uname_arch == "aarch64": - # Disabling because dmi data on ARM processors - LOG.debug("Disabling CloudSigma datasource on arm (LP: #1243287)") - return False LOG.debug("determining hypervisor product name via dmi data") sys_product_name = util.read_dmi_data("system-product-name") diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index 44a17a00..fc596e17 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -1,6 +1,7 @@ # vi: ts=4 expandtab # # Author: Neal Shrader <neal@digitalocean.com> +# Author: Ben Howard <bh@digitalocean.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -14,22 +15,27 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from cloudinit import ec2_utils +# DigitalOcean Droplet API: +# https://developers.digitalocean.com/documentation/metadata/ + +import json + from cloudinit import log as logging from cloudinit import sources +from cloudinit import url_helper from cloudinit import util -import functools - - LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://169.254.169.254/metadata/v1/', - 'mirrors_url': 'http://mirrors.digitalocean.com/' + 'metadata_url': 'http://169.254.169.254/metadata/v1.json', } -MD_RETRIES = 0 -MD_TIMEOUT = 1 + +# Wait for a up to a minute, retrying the meta-data server +# every 2 seconds. +MD_RETRIES = 30 +MD_TIMEOUT = 2 +MD_WAIT_RETRY = 2 class DataSourceDigitalOcean(sources.DataSource): @@ -40,43 +46,61 @@ class DataSourceDigitalOcean(sources.DataSource): util.get_cfg_by_path(sys_cfg, ["datasource", "DigitalOcean"], {}), BUILTIN_DS_CONFIG]) self.metadata_address = self.ds_cfg['metadata_url'] + self.retries = self.ds_cfg.get('retries', MD_RETRIES) + self.timeout = self.ds_cfg.get('timeout', MD_TIMEOUT) + self.wait_retry = self.ds_cfg.get('wait_retry', MD_WAIT_RETRY) - if self.ds_cfg.get('retries'): - self.retries = self.ds_cfg['retries'] - else: - self.retries = MD_RETRIES + def _get_sysinfo(self): + # DigitalOcean embeds vendor ID and instance/droplet_id in the + # SMBIOS information - if self.ds_cfg.get('timeout'): - self.timeout = self.ds_cfg['timeout'] - else: - self.timeout = MD_TIMEOUT + LOG.debug("checking if instance is a DigitalOcean droplet") + + # Detect if we are on DigitalOcean and return the Droplet's ID + vendor_name = util.read_dmi_data("system-manufacturer") + if vendor_name != "DigitalOcean": + return (False, None) - def get_data(self): - caller = functools.partial(util.read_file_or_url, - timeout=self.timeout, retries=self.retries) + LOG.info("running on DigitalOcean") - def mcaller(url): - return caller(url).contents + droplet_id = util.read_dmi_data("system-serial-number") + if droplet_id: + LOG.debug(("system identified via SMBIOS as DigitalOcean Droplet" + "{}").format(droplet_id)) + else: + LOG.critical(("system identified via SMBIOS as a DigitalOcean " + "Droplet, but did not provide an ID. Please file a " + "support ticket at: " + "https://cloud.digitalocean.com/support/tickets/" + "new")) - md = ec2_utils.MetadataMaterializer(mcaller(self.metadata_address), - base_url=self.metadata_address, - caller=mcaller) + return (True, droplet_id) - self.metadata = md.materialize() + def get_data(self, apply_filter=False): + (is_do, droplet_id) = self._get_sysinfo() - if self.metadata.get('id'): - return True - else: + # only proceed if we know we are on DigitalOcean + if not is_do: return False - def get_userdata_raw(self): - return "\n".join(self.metadata['user-data']) + LOG.debug("reading metadata from {}".format(self.metadata_address)) + response = url_helper.readurl(self.metadata_address, + timeout=self.timeout, + sec_between=self.wait_retry, + retries=self.retries) - def get_vendordata_raw(self): - return "\n".join(self.metadata['vendor-data']) + contents = util.decode_binary(response.contents) + decoded = json.loads(contents) + + self.metadata = decoded + self.metadata['instance-id'] = decoded.get('droplet_id', droplet_id) + self.metadata['local-hostname'] = decoded.get('hostname', droplet_id) + self.vendordata_raw = decoded.get("vendor_data", None) + self.userdata_raw = decoded.get("user_data", None) + return True def get_public_ssh_keys(self): - public_keys = self.metadata['public-keys'] + public_keys = self.metadata.get('public_keys', []) if isinstance(public_keys, list): return public_keys else: @@ -84,21 +108,17 @@ class DataSourceDigitalOcean(sources.DataSource): @property def availability_zone(self): - return self.metadata['region'] - - def get_instance_id(self): - return self.metadata['id'] - - def get_hostname(self, fqdn=False, resolve_ip=False): - return self.metadata['hostname'] - - def get_package_mirror_info(self): - return self.ds_cfg['mirrors_url'] + return self.metadata.get('region', 'default') @property def launch_index(self): return None + def check_instance_id(self, sys_cfg): + return sources.instance_id_matches_system_uuid( + self.get_instance_id(), 'system-serial-number') + + # Used to match classes to dependencies datasources = [ (DataSourceDigitalOcean, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 39e7bbd9..143ab368 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -653,14 +653,8 @@ def write_boot_content(content, content_f, link=None, shebang=False, util.logexc(LOG, "failed establishing content link: %s", e) -def get_smartos_environ(uname_version=None, product_name=None, - uname_arch=None): +def get_smartos_environ(uname_version=None, product_name=None): uname = os.uname() - if uname_arch is None: - uname_arch = uname[4] - - if uname_arch.startswith("arm") or uname_arch == "aarch64": - return None # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but # report 'BrandZ virtual linux' as the kernel version diff --git a/cloudinit/util.py b/cloudinit/util.py index e5dd61a0..226628cc 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2227,10 +2227,17 @@ def read_dmi_data(key): If all of the above fail to find a value, None will be returned. """ + syspath_value = _read_dmi_syspath(key) if syspath_value is not None: return syspath_value + # running dmidecode can be problematic on some arches (LP: #1243287) + uname_arch = os.uname()[4] + if uname_arch.startswith("arm") or uname_arch == "aarch64": + LOG.debug("dmidata is not supported on %s", uname_arch) + return None + dmidecode_path = which('dmidecode') if dmidecode_path: return _call_dmidecode(key, dmidecode_path) diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index 8936a1e3..f5d2ef35 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -15,68 +15,58 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import re - -from six.moves.urllib_parse import urlparse +import json from cloudinit import helpers from cloudinit import settings from cloudinit.sources import DataSourceDigitalOcean from .. import helpers as test_helpers +from ..helpers import HttprettyTestCase httpretty = test_helpers.import_httpretty() -# Abbreviated for the test -DO_INDEX = """id - hostname - user-data - vendor-data - public-keys - region""" - -DO_MULTIPLE_KEYS = """ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@digitalocean.com - ssh-rsa AAAAB3NzaC1yc2EAAAA... neal2@digitalocean.com""" -DO_SINGLE_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@digitalocean.com" +DO_MULTIPLE_KEYS = ["ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@do.co", + "ssh-rsa AAAAB3NzaC1yc2EAAAA... test2@do.co"] +DO_SINGLE_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAA... test@do.co" DO_META = { - '': DO_INDEX, - 'user-data': '#!/bin/bash\necho "user-data"', - 'vendor-data': '#!/bin/bash\necho "vendor-data"', - 'public-keys': DO_SINGLE_KEY, + 'user_data': 'user_data_here', + 'vendor_data': 'vendor_data_here', + 'public_keys': DO_SINGLE_KEY, 'region': 'nyc3', 'id': '2000000', 'hostname': 'cloudinit-test', } -MD_URL_RE = re.compile(r'http://169.254.169.254/metadata/v1/.*') +MD_URL = 'http://169.254.169.254/metadata/v1.json' + + +def _mock_dmi(): + return (True, DO_META.get('id')) def _request_callback(method, uri, headers): - url_path = urlparse(uri).path - if url_path.startswith('/metadata/v1/'): - path = url_path.split('/metadata/v1/')[1:][0] - else: - path = None - if path in DO_META: - return (200, headers, DO_META.get(path)) - else: - return (404, headers, '') + return (200, headers, json.dumps(DO_META)) -class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase): +class TestDataSourceDigitalOcean(HttprettyTestCase): + """ + Test reading the meta-data + """ def setUp(self): self.ds = DataSourceDigitalOcean.DataSourceDigitalOcean( settings.CFG_BUILTIN, None, helpers.Paths({})) + self.ds._get_sysinfo = _mock_dmi super(TestDataSourceDigitalOcean, self).setUp() @httpretty.activate def test_connection(self): httpretty.register_uri( - httpretty.GET, MD_URL_RE, - body=_request_callback) + httpretty.GET, MD_URL, + body=json.dumps(DO_META)) success = self.ds.get_data() self.assertTrue(success) @@ -84,14 +74,14 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase): @httpretty.activate def test_metadata(self): httpretty.register_uri( - httpretty.GET, MD_URL_RE, + httpretty.GET, MD_URL, body=_request_callback) self.ds.get_data() - self.assertEqual(DO_META.get('user-data'), + self.assertEqual(DO_META.get('user_data'), self.ds.get_userdata_raw()) - self.assertEqual(DO_META.get('vendor-data'), + self.assertEqual(DO_META.get('vendor_data'), self.ds.get_vendordata_raw()) self.assertEqual(DO_META.get('region'), @@ -103,11 +93,8 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase): self.assertEqual(DO_META.get('hostname'), self.ds.get_hostname()) - self.assertEqual('http://mirrors.digitalocean.com/', - self.ds.get_package_mirror_info()) - # Single key - self.assertEqual([DO_META.get('public-keys')], + self.assertEqual([DO_META.get('public_keys')], self.ds.get_public_ssh_keys()) self.assertIsInstance(self.ds.get_public_ssh_keys(), list) @@ -116,12 +103,12 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase): def test_multiple_ssh_keys(self): DO_META['public_keys'] = DO_MULTIPLE_KEYS httpretty.register_uri( - httpretty.GET, MD_URL_RE, + httpretty.GET, MD_URL, body=_request_callback) self.ds.get_data() # Multiple keys - self.assertEqual(DO_META.get('public-keys').splitlines(), + self.assertEqual(DO_META.get('public_keys'), self.ds.get_public_ssh_keys()) self.assertIsInstance(self.ds.get_public_ssh_keys(), list) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 37a984ac..73369cd3 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -371,8 +371,30 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): self._create_sysfs_parent_directory() expected_dmi_value = 'dmidecode-used' self._configure_dmidecode_return('use-dmidecode', expected_dmi_value) - self.assertEqual(expected_dmi_value, - util.read_dmi_data('use-dmidecode')) + with mock.patch("cloudinit.util.os.uname") as m_uname: + m_uname.return_value = ('x-sysname', 'x-nodename', + 'x-release', 'x-version', 'x86_64') + self.assertEqual(expected_dmi_value, + util.read_dmi_data('use-dmidecode')) + + def test_dmidecode_not_used_on_arm(self): + self.patch_mapping({}) + self._create_sysfs_parent_directory() + dmi_val = 'from-dmidecode' + dmi_name = 'use-dmidecode' + self._configure_dmidecode_return(dmi_name, dmi_val) + + expected = {'armel': None, 'aarch64': None, 'x86_64': dmi_val} + found = {} + # we do not run the 'dmi-decode' binary on some arches + # verify that anything requested that is not in the sysfs dir + # will return None on those arches. + with mock.patch("cloudinit.util.os.uname") as m_uname: + for arch in expected: + m_uname.return_value = ('x-sysname', 'x-nodename', + 'x-release', 'x-version', arch) + found[arch] = util.read_dmi_data(dmi_name) + self.assertEqual(expected, found) def test_none_returned_if_neither_source_has_data(self): self.patch_mapping({}) |