From 046352fc69e94f4c2f2d209ff5638fc455ff4b97 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Thu, 13 Feb 2014 21:24:41 +0000 Subject: GCE: add unit tests, user-data support and few other fixes --- cloudinit/sources/DataSourceGCE.py | 49 ++++++------ tests/unittests/test_datasource/test_gce.py | 112 ++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 24 deletions(-) create mode 100644 tests/unittests/test_datasource/test_gce.py diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 95a410ba..2d5ccad5 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -23,7 +23,7 @@ from cloudinit import url_helper LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' + 'metadata_url': 'http://169.254.169.254/computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') @@ -31,7 +31,7 @@ REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') class DataSourceGCE(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.metadata = {} + self.metadata = dict() self.ds_cfg = util.mergemanydict([ util.get_cfg_by_path(sys_cfg, ["datasource", "GCE"], {}), BUILTIN_DS_CONFIG]) @@ -59,33 +59,31 @@ class DataSourceGCE(sources.DataSource): 'user-data': self.metadata_address + 'instance/attributes/user-data', } - # if we cannot resolve the metadata server, then no point in trying - if not util.is_resolvable(self.metadata_address): - LOG.debug("%s is not resolvable", self.metadata_address) + # try to connect to metadata service or fail otherwise + try: + url_helper.readurl(url=url_map['instance-id'], headers=headers) + except url_helper.UrlError as e: + LOG.debug('Failed to connect to metadata service. Reason: {0}'.format( + e)) return False + # iterate over url_map keys to get metadata items for mkey in url_map.iterkeys(): try: - resp = url_helper.readurl(url=url_map[mkey], headers=headers, - retries=0) - except IOError: - return False - if resp.ok(): - if mkey == 'public-keys': - pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] - self.metadata[mkey] = pub_keys + resp = url_helper.readurl( + url=url_map[mkey], headers=headers) + if resp.code == 200: + if mkey == 'public-keys': + pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] + self.metadata[mkey] = pub_keys + else: + self.metadata[mkey] = resp.contents else: - self.metadata[mkey] = resp.contents - else: - if mkey in REQUIRED_FIELDS: - LOG.warn("required metadata '%s' not found in metadata", - url_map[mkey]) - return False - + self.metadata[mkey] = None + except url_helper.UrlError as e: self.metadata[mkey] = None - return False - - self.user_data_raw = self.metadata['user-data'] + LOG.warn('Failed to get {0} metadata item. Reason {1}.'.format( + mkey, e)) return True @property @@ -102,9 +100,12 @@ class DataSourceGCE(sources.DataSource): def get_hostname(self, fqdn=False): return self.metadata['local-hostname'] + def get_userdata_raw(self): + return self.metadata['user-data'] + @property def availability_zone(self): - return self.metadata['instance-zone'] + return self.metadata['availability-zone'] # Used to match classes to dependencies datasources = [ diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py new file mode 100644 index 00000000..0c58be52 --- /dev/null +++ b/tests/unittests/test_datasource/test_gce.py @@ -0,0 +1,112 @@ +# +# Copyright (C) 2014 Vaidas Jablonskis +# +# Author: Vaidas Jablonskis +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import unittest +import httpretty +import re + +from urlparse import urlparse + +from cloudinit import settings +from cloudinit import helpers +from cloudinit.sources import DataSourceGCE + +GCE_META = { + 'instance/id': '123', + 'instance/zone': 'foo/bar', + 'project/attributes/sshKeys': 'user:ssh-rsa AA2..+aRD0fyVw== root@server', + 'instance/hostname': 'server.project-name.local', + 'instance/attributes/user-data': '/bin/echo foo\n', +} + +GCE_META_PARTIAL = { + 'instance/id': '123', + 'instance/hostname': 'server.project-name.local', +} + +HEADERS = {'X-Google-Metadata-Request': 'True'} +MD_URL_RE = re.compile(r'http://169.254.169.254/computeMetadata/v1/.*') + + +def _request_callback(method, uri, headers): + url_path = urlparse(uri).path + if url_path.startswith('/computeMetadata/v1/'): + path = url_path.split('/computeMetadata/v1/')[1:][0] + else: + path = None + if path in GCE_META: + #return (200, headers, GCE_META.get(path)) + return (200, headers, GCE_META.get(path)) + else: + return (404, headers, '') + + +class TestDataSourceGCE(unittest.TestCase): + + def setUp(self): + self.ds = DataSourceGCE.DataSourceGCE( + settings.CFG_BUILTIN, None, + helpers.Paths({})) + + @httpretty.activate + def test_connection(self): + httpretty.register_uri( + httpretty.GET, MD_URL_RE, + body=_request_callback) + + success = self.ds.get_data() + self.assertTrue(success) + + req_header = httpretty.last_request().headers + self.assertDictContainsSubset(HEADERS, req_header) + + @httpretty.activate + def test_metadata(self): + httpretty.register_uri( + httpretty.GET, MD_URL_RE, + body=_request_callback) + self.ds.get_data() + + self.assertEqual(GCE_META.get('instance/hostname'), + self.ds.get_hostname()) + + self.assertEqual(GCE_META.get('instance/id'), + self.ds.get_instance_id()) + + self.assertEqual(GCE_META.get('instance/zone'), + self.ds.availability_zone) + + self.assertEqual(GCE_META.get('instance/attributes/user-data'), + self.ds.get_userdata_raw()) + + # we expect a list of public ssh keys with user names stripped + self.assertEqual(['ssh-rsa AA2..+aRD0fyVw== root@server'], + self.ds.get_public_ssh_keys()) + + # test partial metadata (missing user-data in particular) + @httpretty.activate + def test_metadata_partial(self): + httpretty.register_uri( + httpretty.GET, MD_URL_RE, + body=_request_callback) + self.ds.get_data() + + self.assertEqual(GCE_META_PARTIAL.get('instance/id'), + self.ds.get_instance_id()) + + self.assertEqual(GCE_META_PARTIAL.get('instance/hostname'), + self.ds.get_hostname()) -- cgit v1.2.3 From f40bdb4869567124dbc48feaa0574cc83df26e3a Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Thu, 13 Feb 2014 22:03:12 +0000 Subject: GCE: use dns name instead of IP address --- cloudinit/sources/DataSourceGCE.py | 11 ++++------- tests/unittests/test_datasource/test_gce.py | 3 +-- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 2d5ccad5..e2be15b0 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -23,7 +23,7 @@ from cloudinit import url_helper LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://169.254.169.254/computeMetadata/v1/' + 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') @@ -59,12 +59,9 @@ class DataSourceGCE(sources.DataSource): 'user-data': self.metadata_address + 'instance/attributes/user-data', } - # try to connect to metadata service or fail otherwise - try: - url_helper.readurl(url=url_map['instance-id'], headers=headers) - except url_helper.UrlError as e: - LOG.debug('Failed to connect to metadata service. Reason: {0}'.format( - e)) + # if we cannot resolve the metadata server, then no point in trying + if not util.is_resolvable(self.metadata_address): + LOG.debug('{0} is not resolvable'.format(self.metadata_address)) return False # iterate over url_map keys to get metadata items diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 0c58be52..d91bd531 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -39,7 +39,7 @@ GCE_META_PARTIAL = { } HEADERS = {'X-Google-Metadata-Request': 'True'} -MD_URL_RE = re.compile(r'http://169.254.169.254/computeMetadata/v1/.*') +MD_URL_RE = re.compile(r'http://metadata.google.internal./computeMetadata/v1/.*') def _request_callback(method, uri, headers): @@ -49,7 +49,6 @@ def _request_callback(method, uri, headers): else: path = None if path in GCE_META: - #return (200, headers, GCE_META.get(path)) return (200, headers, GCE_META.get(path)) else: return (404, headers, '') -- cgit v1.2.3 From 6baa9166aa8b150566003d2d4cf56cfd03fe6952 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 21:09:47 -0500 Subject: some style changes, some pylint, be less noisy this changes url_map to a list and adds 'required' information. * If we've not already found an entry, and this is required, then debug log (ie, this is just not GCE). * if we already found an entry and this is required: warn split the keys fixing out of the loop. --- cloudinit/sources/DataSourceGCE.py | 65 ++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index e2be15b0..c993293f 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -51,37 +51,60 @@ class DataSourceGCE(sources.DataSource): # GCE metadata server requires a custom header since v1 headers = {'X-Google-Metadata-Request': True} - url_map = { - 'instance-id': self.metadata_address + 'instance/id', - 'availability-zone': self.metadata_address + 'instance/zone', - 'public-keys': self.metadata_address + 'project/attributes/sshKeys', - 'local-hostname': self.metadata_address + 'instance/hostname', - 'user-data': self.metadata_address + 'instance/attributes/user-data', - } + # url_map: (our-key, path, required) + url_map = [ + ('instance-id', 'instance/id', True), + ('availability-zone', 'instance/zone', True), + ('local-hostname', 'instance/hostname', True), + ('public-keys', 'project/attributes/sshKeys', False), + ('user-data', 'instance/attributes/user-data', False), + ] # if we cannot resolve the metadata server, then no point in trying if not util.is_resolvable(self.metadata_address): - LOG.debug('{0} is not resolvable'.format(self.metadata_address)) + LOG.debug("%s is not resolvable", self.metadata_address) return False # iterate over url_map keys to get metadata items - for mkey in url_map.iterkeys(): + found = False + for (mkey, path, required) in url_map: try: - resp = url_helper.readurl( - url=url_map[mkey], headers=headers) + resp = url_helper.readurl(url=self.metadata_address + path, + headers=headers) if resp.code == 200: - if mkey == 'public-keys': - pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] - self.metadata[mkey] = pub_keys - else: - self.metadata[mkey] = resp.contents + found = True + self.metadata[mkey] = resp.contents else: - self.metadata[mkey] = None + if required: + msg = "required url %s returned code %s. not GCE" + if not found: + LOG.debug(msg, path, resp.code) + else: + LOG.warn(msg, path, resp.code) + return False + else: + self.metadata[mkey] = None except url_helper.UrlError as e: + if required: + msg = "required url %s raised exception %s. not GCE" + if not found: + LOG.debug(msg, path, e) + else: + LOG.warn(msg, path, e) + return False + msg = "Failed to get %s metadata item: %s." + if found: + LOG.warn(msg, path, e) + else: + LOG.debug(msg, path, e) + self.metadata[mkey] = None - LOG.warn('Failed to get {0} metadata item. Reason {1}.'.format( - mkey, e)) - return True + + if self.metadata['public-keys']: + lines = self.metadata['public-keys'].splitlines() + self.metadata['public-keys'] = [self._trim_key(k) for k in lines] + + return found @property def launch_index(self): @@ -94,7 +117,7 @@ class DataSourceGCE(sources.DataSource): def get_public_ssh_keys(self): return self.metadata['public-keys'] - def get_hostname(self, fqdn=False): + def get_hostname(self, fqdn=False, _resolve_ip=False): return self.metadata['local-hostname'] def get_userdata_raw(self): -- cgit v1.2.3