From 03cec66c495e57fdf29e7bab4b3938e24a8dec18 Mon Sep 17 00:00:00 2001 From: Neal Shrader Date: Thu, 16 Oct 2014 11:30:08 -0400 Subject: Add DigitalOcean DataSource The DigitalOcean metadata service is an AWS-style service available over HTTP via the link local address 169.254.169.254. The specifics of the API are documented at: https://developers.digitalocean.com/metadata/ --- cloudinit/sources/DataSourceDigitalOcean.py | 102 ++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 cloudinit/sources/DataSourceDigitalOcean.py (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py new file mode 100644 index 00000000..c580e2d5 --- /dev/null +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -0,0 +1,102 @@ +# vi: ts=4 expandtab +# +# Author: Neal Shrader +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit import log as logging +from cloudinit import util +from cloudinit import sources +from cloudinit import url_helper + +LOG = logging.getLogger(__name__) + +BUILTIN_DS_CONFIG = { + 'metadata_url': 'http://169.254.169.254/metadata/v1', + 'mirrors_url': 'http://mirrors.digitalocean.com/' +} + +class DataSourceDigitalOcean(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.metadata = dict() + self.ds_cfg = util.mergemanydict([ + util.get_cfg_by_path(sys_cfg, ["datasource", "DigitalOcean"], {}), + BUILTIN_DS_CONFIG]) + self.metadata_address = self.ds_cfg['metadata_url'] + self.retries = 3 + self.timeout = 1 + + def get_data(self): + url_map = [ + ('user-data', '/user-data'), + ('vendor-data', '/vendor-data'), + ('public-keys', '/public-keys'), + ('region', '/region'), + ('id', '/id'), + ('hostname', '/hostname'), + ] + + found = False + for (key, path) in url_map: + try: + resp = url_helper.readurl(url=self.metadata_address + path, + timeout=self.timeout, + retries=self.retries) + if resp.code == 200: + found = True + self.metadata[key] = resp.contents + else: + LOG.warn("Path: %s returned %s", path, resp.code) + return False + except url_helper.UrlError as e: + LOG.warn("Path: %s raised exception: %s", path, e) + return False + + return found + + def get_userdata_raw(self): + return self.metadata['user-data'] + + def get_vendordata_raw(self): + return self.metadata['vendor-data'] + + def get_public_ssh_keys(self): + return self.metadata['public-keys'].splitlines() + + @property + def availability_zone(self): + return self.metadata['region'] + + def get_instance_id(self): + return self.metadata['id'] + + def get_hostname(self, fqdn=False): + return self.metadata['hostname'] + + def get_package_mirror_info(self): + return self.ds_cfg['mirrors_url'] + + @property + def launch_index(self): + return None + +# Used to match classes to dependencies +datasources = [ + (DataSourceDigitalOcean, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + ] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) -- cgit v1.2.3 From 33b54a3ac2560b192f29ce1fbe797fdd3cb968aa Mon Sep 17 00:00:00 2001 From: Neal Shrader Date: Thu, 16 Oct 2014 18:20:19 -0400 Subject: Make metadata timeout/retries configurable Defaulting to only trying once. --- cloudinit/sources/DataSourceDigitalOcean.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index c580e2d5..985f1663 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -25,6 +25,8 @@ BUILTIN_DS_CONFIG = { 'metadata_url': 'http://169.254.169.254/metadata/v1', 'mirrors_url': 'http://mirrors.digitalocean.com/' } +MD_RETRIES = 0 +MD_TIMEOUT = 1 class DataSourceDigitalOcean(sources.DataSource): def __init__(self, sys_cfg, distro, paths): @@ -34,8 +36,16 @@ class DataSourceDigitalOcean(sources.DataSource): util.get_cfg_by_path(sys_cfg, ["datasource", "DigitalOcean"], {}), BUILTIN_DS_CONFIG]) self.metadata_address = self.ds_cfg['metadata_url'] - self.retries = 3 - self.timeout = 1 + + if self.ds_cfg.get('retries'): + self.retries = self.ds_cfg['retries'] + else: + self.retries = MD_RETRIES + + if self.ds_cfg.get('timeout'): + self.timeout = self.ds_cfg['timeout'] + else: + self.timeout = MD_TIMEOUT def get_data(self): url_map = [ -- cgit v1.2.3 From 01e8df0557098093a0e3444f41ba3f1861ded316 Mon Sep 17 00:00:00 2001 From: Neal Shrader Date: Thu, 16 Oct 2014 19:19:29 -0400 Subject: Use existing metadata crawler to populate datasource --- cloudinit/sources/DataSourceDigitalOcean.py | 53 +++++++++------------- .../unittests/test_datasource/test_digitalocean.py | 9 ++++ 2 files changed, 30 insertions(+), 32 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index 985f1663..b7afca93 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -18,11 +18,14 @@ from cloudinit import log as logging from cloudinit import util from cloudinit import sources from cloudinit import url_helper +from cloudinit import ec2_utils +import functools + LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://169.254.169.254/metadata/v1', + 'metadata_url': 'http://169.254.169.254/metadata/v1/', 'mirrors_url': 'http://mirrors.digitalocean.com/' } MD_RETRIES = 0 @@ -37,9 +40,9 @@ class DataSourceDigitalOcean(sources.DataSource): BUILTIN_DS_CONFIG]) self.metadata_address = self.ds_cfg['metadata_url'] - if self.ds_cfg.get('retries'): + if self.ds_cfg.get('retries'): self.retries = self.ds_cfg['retries'] - else: + else: self.retries = MD_RETRIES if self.ds_cfg.get('timeout'): @@ -48,41 +51,27 @@ class DataSourceDigitalOcean(sources.DataSource): self.timeout = MD_TIMEOUT def get_data(self): - url_map = [ - ('user-data', '/user-data'), - ('vendor-data', '/vendor-data'), - ('public-keys', '/public-keys'), - ('region', '/region'), - ('id', '/id'), - ('hostname', '/hostname'), - ] - - found = False - for (key, path) in url_map: - try: - resp = url_helper.readurl(url=self.metadata_address + path, - timeout=self.timeout, - retries=self.retries) - if resp.code == 200: - found = True - self.metadata[key] = resp.contents - else: - LOG.warn("Path: %s returned %s", path, resp.code) - return False - except url_helper.UrlError as e: - LOG.warn("Path: %s raised exception: %s", path, e) - return False - - return found + caller = functools.partial(util.read_file_or_url, timeout=self.timeout, + retries=self.retries) + md = ec2_utils.MetadataMaterializer(str(caller(self.metadata_address)), + base_url=self.metadata_address, + caller=caller) + + self.metadata = md.materialize() + + if self.metadata.get('id'): + return True + else: + return False def get_userdata_raw(self): - return self.metadata['user-data'] + return "\n".join(self.metadata['user-data']) def get_vendordata_raw(self): - return self.metadata['vendor-data'] + return "\n".join(self.metadata['vendor-data']) def get_public_ssh_keys(self): - return self.metadata['public-keys'].splitlines() + return self.metadata['public-keys'].splitlines() @property def availability_zone(self): diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index 9576e042..559a4f9f 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -26,7 +26,16 @@ from cloudinit.sources import DataSourceDigitalOcean from .. import helpers as test_helpers +# Abbreviated for the test +DO_INDEX = """id + hostname + user-data + vendor-data + public-keys + region""" + DO_META = { + '': DO_INDEX, 'user-data': '#!/bin/bash\necho "user-data"', 'vendor-data': '#!/bin/bash\necho "vendor-data"', 'public-keys': 'ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@digitalocean.com', -- cgit v1.2.3 From fdef6a9f84c5720bbd37f3eb98c9b7c58913bbfd Mon Sep 17 00:00:00 2001 From: Neal Shrader Date: Fri, 17 Oct 2014 16:26:46 -0400 Subject: Correct handling of single/multiple ssh keys --- cloudinit/sources/DataSourceDigitalOcean.py | 6 ++++- .../unittests/test_datasource/test_digitalocean.py | 30 +++++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index b7afca93..c59232ca 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -19,6 +19,7 @@ from cloudinit import util from cloudinit import sources from cloudinit import url_helper from cloudinit import ec2_utils +from types import * import functools @@ -71,7 +72,10 @@ class DataSourceDigitalOcean(sources.DataSource): return "\n".join(self.metadata['vendor-data']) def get_public_ssh_keys(self): - return self.metadata['public-keys'].splitlines() + if type(self.metadata['public-keys']) is StringType: + return [self.metadata['public-keys']] + else: + return self.metadata['public-keys'] @property def availability_zone(self): diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index 559a4f9f..0997cf38 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -18,6 +18,7 @@ import httpretty import re +from types import * from urlparse import urlparse from cloudinit import settings @@ -34,11 +35,15 @@ DO_INDEX = """id public-keys region""" +DO_MULTIPLE_KEYS = """ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@digitalocean.com + ssh-rsa AAAAB3NzaC1yc2EAAAA... neal2@digitalocean.com""" +DO_SINGLE_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@digitalocean.com" + DO_META = { '': DO_INDEX, 'user-data': '#!/bin/bash\necho "user-data"', 'vendor-data': '#!/bin/bash\necho "vendor-data"', - 'public-keys': 'ssh-rsa AAAAB3NzaC1yc2EAAAA... neal@digitalocean.com', + 'public-keys': DO_SINGLE_KEY, 'region': 'nyc3', 'id': '2000000', 'hostname': 'cloudinit-test', @@ -88,9 +93,6 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase): self.assertEqual(DO_META.get('vendor-data'), self.ds.get_vendordata_raw()) - self.assertEqual([DO_META.get('public-keys')], - self.ds.get_public_ssh_keys()) - self.assertEqual(DO_META.get('region'), self.ds.availability_zone) @@ -102,3 +104,23 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase): self.assertEqual('http://mirrors.digitalocean.com/', self.ds.get_package_mirror_info()) + + # Single key + self.assertEqual([DO_META.get('public-keys')], + self.ds.get_public_ssh_keys()) + + self.assertIs(type(self.ds.get_public_ssh_keys()), ListType) + + @httpretty.activate + def test_multiple_ssh_keys(self): + DO_META['public_keys'] = DO_MULTIPLE_KEYS + httpretty.register_uri( + httpretty.GET, MD_URL_RE, + body=_request_callback) + self.ds.get_data() + + # Multiple keys + self.assertEqual(DO_META.get('public-keys').splitlines(), + self.ds.get_public_ssh_keys()) + + self.assertIs(type(self.ds.get_public_ssh_keys()), ListType) -- cgit v1.2.3 From 216d3bf22414ca731a1eac2098e5883d2dab06b1 Mon Sep 17 00:00:00 2001 From: Neal Shrader Date: Fri, 17 Oct 2014 19:00:01 -0400 Subject: Explicitly import only types being compared --- cloudinit/sources/DataSourceDigitalOcean.py | 2 +- tests/unittests/test_datasource/test_digitalocean.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index c59232ca..b25dcb27 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -19,7 +19,7 @@ from cloudinit import util from cloudinit import sources from cloudinit import url_helper from cloudinit import ec2_utils -from types import * +from types import StringType import functools diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index 0997cf38..04bee340 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -18,7 +18,7 @@ import httpretty import re -from types import * +from types import ListType from urlparse import urlparse from cloudinit import settings -- cgit v1.2.3