From 3551333795539c2cec1195841d2a641046981ba6 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 20 Apr 2015 11:52:50 +0100 Subject: Add test that missing GCE required key returns False. --- tests/unittests/test_datasource/test_gce.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 4280abc4..540a55d0 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -141,3 +141,14 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase): decoded = b64decode( GCE_META_ENCODING.get('instance/attributes/user-data')) self.assertEqual(decoded, self.ds.get_userdata_raw()) + + @httpretty.activate + def test_missing_required_keys_return_false(self): + for required_key in ['instance/id', 'instance/zone', + 'instance/hostname']: + meta = GCE_META_PARTIAL.copy() + del meta[required_key] + httpretty.register_uri(httpretty.GET, MD_URL_RE, + body=_new_request_callback(meta)) + self.assertEqual(False, self.ds.get_data()) + httpretty.reset() -- cgit v1.2.3 From 844ebbee112143e85fb46b4b5ed649729f903d2c Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 20 Apr 2015 15:23:57 +0100 Subject: Refactor GCE metadata fetching to use a helper class. --- cloudinit/sources/DataSourceGCE.py | 69 ++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 608c07f1..255f5f45 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -30,6 +30,31 @@ BUILTIN_DS_CONFIG = { REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') +class GoogleMetadataFetcher(object): + headers = {'X-Google-Metadata-Request': True} + + def __init__(self, metadata_address): + self.metadata_address = metadata_address + + def get_value(self, path, is_text): + value = None + try: + resp = url_helper.readurl(url=self.metadata_address + path, + headers=self.headers) + except url_helper.UrlError as exc: + msg = "url %s raised exception %s" + LOG.debug(msg, path, exc) + else: + if resp.code == 200: + if is_text: + value = util.decode_binary(resp.contents) + else: + value = resp.contents + else: + LOG.debug("url %s returned code %s", path, resp.code) + return value + + class DataSourceGCE(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) @@ -50,9 +75,6 @@ class DataSourceGCE(sources.DataSource): return public_key def get_data(self): - # GCE metadata server requires a custom header since v1 - headers = {'X-Google-Metadata-Request': True} - # url_map: (our-key, path, required, is_text) url_map = [ ('instance-id', 'instance/id', True, True), @@ -69,40 +91,21 @@ class DataSourceGCE(sources.DataSource): LOG.debug("%s is not resolvable", self.metadata_address) return False + metadata_fetcher = GoogleMetadataFetcher(self.metadata_address) # iterate over url_map keys to get metadata items found = False for (mkey, path, required, is_text) in url_map: - try: - resp = url_helper.readurl(url=self.metadata_address + path, - headers=headers) - if resp.code == 200: - found = True - if is_text: - self.metadata[mkey] = util.decode_binary(resp.contents) - else: - self.metadata[mkey] = resp.contents + value = metadata_fetcher.get_value(path, is_text) + if value: + found = True + if required and value is None: + msg = "required url %s returned nothing. not GCE" + if not found: + LOG.debug(msg, path) else: - if required: - msg = "required url %s returned code %s. not GCE" - if not found: - LOG.debug(msg, path, resp.code) - else: - LOG.warn(msg, path, resp.code) - return False - else: - self.metadata[mkey] = None - except url_helper.UrlError as e: - if required: - msg = "required url %s raised exception %s. not GCE" - if not found: - LOG.debug(msg, path, e) - else: - LOG.warn(msg, path, e) - return False - msg = "Failed to get %s metadata item: %s." - LOG.debug(msg, path, e) - - self.metadata[mkey] = None + LOG.warn(msg, path) + return False + self.metadata[mkey] = value if self.metadata['public-keys']: lines = self.metadata['public-keys'].splitlines() -- cgit v1.2.3 From 47eb1c4b52a2f5f4f8ea657918acd94209668bd7 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 20 Apr 2015 15:24:00 +0100 Subject: Rename found variable in GCE data source. --- cloudinit/sources/DataSourceGCE.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 255f5f45..9cf2f56e 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -93,14 +93,14 @@ class DataSourceGCE(sources.DataSource): metadata_fetcher = GoogleMetadataFetcher(self.metadata_address) # iterate over url_map keys to get metadata items - found = False + running_on_gce = False for (mkey, path, required, is_text) in url_map: value = metadata_fetcher.get_value(path, is_text) if value: - found = True + running_on_gce = True if required and value is None: msg = "required url %s returned nothing. not GCE" - if not found: + if not running_on_gce: LOG.debug(msg, path) else: LOG.warn(msg, path) @@ -119,7 +119,7 @@ class DataSourceGCE(sources.DataSource): else: LOG.warn('unknown user-data-encoding: %s, ignoring', encoding) - return found + return running_on_gce @property def launch_index(self): -- cgit v1.2.3 From 6e84c05d2dc402de8cc4ae414af8657b97317218 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 20 Apr 2015 15:24:21 +0100 Subject: Support multiple metadata paths for metadata keys in GCE data source. --- cloudinit/sources/DataSourceGCE.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 9cf2f56e..1a133c28 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -77,12 +77,12 @@ class DataSourceGCE(sources.DataSource): def get_data(self): # url_map: (our-key, path, required, is_text) url_map = [ - ('instance-id', 'instance/id', True, True), - ('availability-zone', 'instance/zone', True, True), - ('local-hostname', 'instance/hostname', True, True), - ('public-keys', 'project/attributes/sshKeys', False, True), - ('user-data', 'instance/attributes/user-data', False, False), - ('user-data-encoding', 'instance/attributes/user-data-encoding', + ('instance-id', ('instance/id',), True, True), + ('availability-zone', ('instance/zone',), True, True), + ('local-hostname', ('instance/hostname',), True, True), + ('public-keys', ('project/attributes/sshKeys',), False, True), + ('user-data', ('instance/attributes/user-data',), False, False), + ('user-data-encoding', ('instance/attributes/user-data-encoding',), False, True), ] @@ -94,16 +94,20 @@ class DataSourceGCE(sources.DataSource): metadata_fetcher = GoogleMetadataFetcher(self.metadata_address) # iterate over url_map keys to get metadata items running_on_gce = False - for (mkey, path, required, is_text) in url_map: - value = metadata_fetcher.get_value(path, is_text) + for (mkey, paths, required, is_text) in url_map: + value = None + for path in paths: + new_value = metadata_fetcher.get_value(path, is_text) + if new_value is not None: + value = new_value if value: running_on_gce = True if required and value is None: - msg = "required url %s returned nothing. not GCE" + msg = "required key %s returned nothing. not GCE" if not running_on_gce: - LOG.debug(msg, path) + LOG.debug(msg, mkey) else: - LOG.warn(msg, path) + LOG.warn(msg, mkey) return False self.metadata[mkey] = value -- cgit v1.2.3 From 4fc65f02ae3fbf1a2062e6169ee39b5c5d5e23bc Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 20 Apr 2015 15:24:22 +0100 Subject: GCE instance-level SSH keys override project-level keys. (LP: #1403617) --- cloudinit/sources/DataSourceGCE.py | 3 ++- tests/unittests/test_datasource/test_gce.py | 38 ++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 1a133c28..f4ed915d 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -80,7 +80,8 @@ class DataSourceGCE(sources.DataSource): ('instance-id', ('instance/id',), True, True), ('availability-zone', ('instance/zone',), True, True), ('local-hostname', ('instance/hostname',), True, True), - ('public-keys', ('project/attributes/sshKeys',), False, True), + ('public-keys', ('project/attributes/sshKeys', + 'instance/attributes/sshKeys'), False, True), ('user-data', ('instance/attributes/user-data',), False, False), ('user-data-encoding', ('instance/attributes/user-data-encoding',), False, True), diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 540a55d0..1fb100f7 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -113,10 +113,6 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase): self.assertEqual(GCE_META.get('instance/attributes/user-data'), self.ds.get_userdata_raw()) - # we expect a list of public ssh keys with user names stripped - self.assertEqual(['ssh-rsa AA2..+aRD0fyVw== root@server'], - self.ds.get_public_ssh_keys()) - # test partial metadata (missing user-data in particular) @httpretty.activate def test_metadata_partial(self): @@ -152,3 +148,37 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase): body=_new_request_callback(meta)) self.assertEqual(False, self.ds.get_data()) httpretty.reset() + + @httpretty.activate + def test_project_level_ssh_keys_are_used(self): + httpretty.register_uri(httpretty.GET, MD_URL_RE, + body=_new_request_callback()) + self.ds.get_data() + + # we expect a list of public ssh keys with user names stripped + self.assertEqual(['ssh-rsa AA2..+aRD0fyVw== root@server'], + self.ds.get_public_ssh_keys()) + + @httpretty.activate + def test_instance_level_ssh_keys_are_used(self): + key_content = 'ssh-rsa JustAUser root@server' + meta = GCE_META.copy() + meta['instance/attributes/sshKeys'] = 'user:{0}'.format(key_content) + + httpretty.register_uri(httpretty.GET, MD_URL_RE, + body=_new_request_callback(meta)) + self.ds.get_data() + + self.assertIn(key_content, self.ds.get_public_ssh_keys()) + + @httpretty.activate + def test_instance_level_keys_replace_project_level_keys(self): + key_content = 'ssh-rsa JustAUser root@server' + meta = GCE_META.copy() + meta['instance/attributes/sshKeys'] = 'user:{0}'.format(key_content) + + httpretty.register_uri(httpretty.GET, MD_URL_RE, + body=_new_request_callback(meta)) + self.ds.get_data() + + self.assertEqual([key_content], self.ds.get_public_ssh_keys()) -- cgit v1.2.3