From 046352fc69e94f4c2f2d209ff5638fc455ff4b97 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Thu, 13 Feb 2014 21:24:41 +0000 Subject: GCE: add unit tests, user-data support and few other fixes --- cloudinit/sources/DataSourceGCE.py | 49 +++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 24 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 95a410ba..2d5ccad5 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -23,7 +23,7 @@ from cloudinit import url_helper LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' + 'metadata_url': 'http://169.254.169.254/computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') @@ -31,7 +31,7 @@ REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') class DataSourceGCE(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.metadata = {} + self.metadata = dict() self.ds_cfg = util.mergemanydict([ util.get_cfg_by_path(sys_cfg, ["datasource", "GCE"], {}), BUILTIN_DS_CONFIG]) @@ -59,33 +59,31 @@ class DataSourceGCE(sources.DataSource): 'user-data': self.metadata_address + 'instance/attributes/user-data', } - # if we cannot resolve the metadata server, then no point in trying - if not util.is_resolvable(self.metadata_address): - LOG.debug("%s is not resolvable", self.metadata_address) + # try to connect to metadata service or fail otherwise + try: + url_helper.readurl(url=url_map['instance-id'], headers=headers) + except url_helper.UrlError as e: + LOG.debug('Failed to connect to metadata service. Reason: {0}'.format( + e)) return False + # iterate over url_map keys to get metadata items for mkey in url_map.iterkeys(): try: - resp = url_helper.readurl(url=url_map[mkey], headers=headers, - retries=0) - except IOError: - return False - if resp.ok(): - if mkey == 'public-keys': - pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] - self.metadata[mkey] = pub_keys + resp = url_helper.readurl( + url=url_map[mkey], headers=headers) + if resp.code == 200: + if mkey == 'public-keys': + pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] + self.metadata[mkey] = pub_keys + else: + self.metadata[mkey] = resp.contents else: - self.metadata[mkey] = resp.contents - else: - if mkey in REQUIRED_FIELDS: - LOG.warn("required metadata '%s' not found in metadata", - url_map[mkey]) - return False - + self.metadata[mkey] = None + except url_helper.UrlError as e: self.metadata[mkey] = None - return False - - self.user_data_raw = self.metadata['user-data'] + LOG.warn('Failed to get {0} metadata item. Reason {1}.'.format( + mkey, e)) return True @property @@ -102,9 +100,12 @@ class DataSourceGCE(sources.DataSource): def get_hostname(self, fqdn=False): return self.metadata['local-hostname'] + def get_userdata_raw(self): + return self.metadata['user-data'] + @property def availability_zone(self): - return self.metadata['instance-zone'] + return self.metadata['availability-zone'] # Used to match classes to dependencies datasources = [ -- cgit v1.2.3 From f40bdb4869567124dbc48feaa0574cc83df26e3a Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Thu, 13 Feb 2014 22:03:12 +0000 Subject: GCE: use dns name instead of IP address --- cloudinit/sources/DataSourceGCE.py | 11 ++++------- tests/unittests/test_datasource/test_gce.py | 3 +-- 2 files changed, 5 insertions(+), 9 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 2d5ccad5..e2be15b0 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -23,7 +23,7 @@ from cloudinit import url_helper LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://169.254.169.254/computeMetadata/v1/' + 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') @@ -59,12 +59,9 @@ class DataSourceGCE(sources.DataSource): 'user-data': self.metadata_address + 'instance/attributes/user-data', } - # try to connect to metadata service or fail otherwise - try: - url_helper.readurl(url=url_map['instance-id'], headers=headers) - except url_helper.UrlError as e: - LOG.debug('Failed to connect to metadata service. Reason: {0}'.format( - e)) + # if we cannot resolve the metadata server, then no point in trying + if not util.is_resolvable(self.metadata_address): + LOG.debug('{0} is not resolvable'.format(self.metadata_address)) return False # iterate over url_map keys to get metadata items diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 0c58be52..d91bd531 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -39,7 +39,7 @@ GCE_META_PARTIAL = { } HEADERS = {'X-Google-Metadata-Request': 'True'} -MD_URL_RE = re.compile(r'http://169.254.169.254/computeMetadata/v1/.*') +MD_URL_RE = re.compile(r'http://metadata.google.internal./computeMetadata/v1/.*') def _request_callback(method, uri, headers): @@ -49,7 +49,6 @@ def _request_callback(method, uri, headers): else: path = None if path in GCE_META: - #return (200, headers, GCE_META.get(path)) return (200, headers, GCE_META.get(path)) else: return (404, headers, '') -- cgit v1.2.3 From 6baa9166aa8b150566003d2d4cf56cfd03fe6952 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 21:09:47 -0500 Subject: some style changes, some pylint, be less noisy this changes url_map to a list and adds 'required' information. * If we've not already found an entry, and this is required, then debug log (ie, this is just not GCE). * if we already found an entry and this is required: warn split the keys fixing out of the loop. --- cloudinit/sources/DataSourceGCE.py | 65 ++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 21 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index e2be15b0..c993293f 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -51,37 +51,60 @@ class DataSourceGCE(sources.DataSource): # GCE metadata server requires a custom header since v1 headers = {'X-Google-Metadata-Request': True} - url_map = { - 'instance-id': self.metadata_address + 'instance/id', - 'availability-zone': self.metadata_address + 'instance/zone', - 'public-keys': self.metadata_address + 'project/attributes/sshKeys', - 'local-hostname': self.metadata_address + 'instance/hostname', - 'user-data': self.metadata_address + 'instance/attributes/user-data', - } + # url_map: (our-key, path, required) + url_map = [ + ('instance-id', 'instance/id', True), + ('availability-zone', 'instance/zone', True), + ('local-hostname', 'instance/hostname', True), + ('public-keys', 'project/attributes/sshKeys', False), + ('user-data', 'instance/attributes/user-data', False), + ] # if we cannot resolve the metadata server, then no point in trying if not util.is_resolvable(self.metadata_address): - LOG.debug('{0} is not resolvable'.format(self.metadata_address)) + LOG.debug("%s is not resolvable", self.metadata_address) return False # iterate over url_map keys to get metadata items - for mkey in url_map.iterkeys(): + found = False + for (mkey, path, required) in url_map: try: - resp = url_helper.readurl( - url=url_map[mkey], headers=headers) + resp = url_helper.readurl(url=self.metadata_address + path, + headers=headers) if resp.code == 200: - if mkey == 'public-keys': - pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] - self.metadata[mkey] = pub_keys - else: - self.metadata[mkey] = resp.contents + found = True + self.metadata[mkey] = resp.contents else: - self.metadata[mkey] = None + if required: + msg = "required url %s returned code %s. not GCE" + if not found: + LOG.debug(msg, path, resp.code) + else: + LOG.warn(msg, path, resp.code) + return False + else: + self.metadata[mkey] = None except url_helper.UrlError as e: + if required: + msg = "required url %s raised exception %s. not GCE" + if not found: + LOG.debug(msg, path, e) + else: + LOG.warn(msg, path, e) + return False + msg = "Failed to get %s metadata item: %s." + if found: + LOG.warn(msg, path, e) + else: + LOG.debug(msg, path, e) + self.metadata[mkey] = None - LOG.warn('Failed to get {0} metadata item. Reason {1}.'.format( - mkey, e)) - return True + + if self.metadata['public-keys']: + lines = self.metadata['public-keys'].splitlines() + self.metadata['public-keys'] = [self._trim_key(k) for k in lines] + + return found @property def launch_index(self): @@ -94,7 +117,7 @@ class DataSourceGCE(sources.DataSource): def get_public_ssh_keys(self): return self.metadata['public-keys'] - def get_hostname(self, fqdn=False): + def get_hostname(self, fqdn=False, _resolve_ip=False): return self.metadata['local-hostname'] def get_userdata_raw(self): -- cgit v1.2.3