From 29b838af062549ef5e2f12c3df74721d294dea37 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Mon, 3 Feb 2014 11:28:50 +0000 Subject: Add Google Compute Engine data source support. --- cloudinit/sources/DataSourceGCE.py | 100 +++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 cloudinit/sources/DataSourceGCE.py (limited to 'cloudinit/sources/DataSourceGCE.py') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py new file mode 100644 index 00000000..b6a82245 --- /dev/null +++ b/cloudinit/sources/DataSourceGCE.py @@ -0,0 +1,100 @@ +# vi: ts=4 expandtab +# +# Author: Vaidas Jablonskis +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import requests + +from cloudinit import log as logging +from cloudinit import sources + +LOG = logging.getLogger(__name__) + +MD_URL = 'http://metadata/computeMetadata/v1/' + + +class DataSourceGCE(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.metadata_address = MD_URL + self.metadata = {} + + # GCE takes sshKeys attribute in the format of ':' + # so we have to trim each key to remove the username part + def _trim_key(self, public_key): + try: + index = public_key.index(':') + if index > 0: + return public_key[(index + 1):] + except: + return public_key + + def get_data(self): + # GCE metadata server requires a custom header since v1 + headers = {'X-Google-Metadata-Request': True} + + url_map = { + 'instance-id': self.metadata_address + 'instance/id', + 'availability-zone': self.metadata_address + 'instance/zone', + 'public-keys': self.metadata_address + 'project/attributes/sshKeys', + 'local-hostname': self.metadata_address + 'instance/hostname', + } + + with requests.Session() as s: + for mkey in url_map.iterkeys(): + try: + r = s.get(url_map[mkey], headers=headers) + except requests.exceptions.ConnectionError: + return False + if r.ok: + if mkey == 'public-keys': + pub_keys = [self._trim_key(k) for k in r.text.splitlines()] + self.metadata[mkey] = pub_keys + else: + self.metadata[mkey] = r.text + else: + self.metadata[mkey] = None + return False + return True + + @property + def launch_index(self): + # GCE does not provide lauch_index property + return None + + def get_instance_id(self): + return self.metadata['instance-id'] + + def get_public_ssh_keys(self): + return self.metadata['public-keys'] + + def get_hostname(self, fqdn=False): + return self.metadata['local-hostname'] + + def get_userdata_raw(self): + return None + + @property + def availability_zone(self): + return self.metadata['instance-zone'] + +# Used to match classes to dependencies +datasources = [ + (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) -- cgit v1.2.3 From e1710e06fd09c3f643fdca851c011da3a3eb7a98 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Fri, 7 Feb 2014 09:43:32 +0000 Subject: use url_helper instead of requests --- cloudinit/sources/DataSourceGCE.py | 38 ++++++++++++-------------------------- 1 file changed, 12 insertions(+), 26 deletions(-) (limited to 'cloudinit/sources/DataSourceGCE.py') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index b6a82245..f9cdc1da 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -14,10 +14,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import requests from cloudinit import log as logging from cloudinit import sources +from cloudinit import url_helper LOG = logging.getLogger(__name__) @@ -30,16 +30,6 @@ class DataSourceGCE(sources.DataSource): self.metadata_address = MD_URL self.metadata = {} - # GCE takes sshKeys attribute in the format of ':' - # so we have to trim each key to remove the username part - def _trim_key(self, public_key): - try: - index = public_key.index(':') - if index > 0: - return public_key[(index + 1):] - except: - return public_key - def get_data(self): # GCE metadata server requires a custom header since v1 headers = {'X-Google-Metadata-Request': True} @@ -51,22 +41,18 @@ class DataSourceGCE(sources.DataSource): 'local-hostname': self.metadata_address + 'instance/hostname', } - with requests.Session() as s: - for mkey in url_map.iterkeys(): - try: - r = s.get(url_map[mkey], headers=headers) - except requests.exceptions.ConnectionError: - return False - if r.ok: - if mkey == 'public-keys': - pub_keys = [self._trim_key(k) for k in r.text.splitlines()] - self.metadata[mkey] = pub_keys - else: - self.metadata[mkey] = r.text + for mkey in url_map.iterkeys(): + resp = url_helper.readurl(url=url_map[mkey], headers=headers) + if resp.ok(): + if mkey == 'public-keys': + pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] + self.metadata[mkey] = pub_keys else: - self.metadata[mkey] = None - return False - return True + self.metadata[mkey] = resp.contents + else: + self.metadata[mkey] = None + return False + return True @property def launch_index(self): -- cgit v1.2.3 From 64b35a9c3b201aed5073823fd4a15ecb15195aea Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Fri, 7 Feb 2014 13:12:48 +0000 Subject: Forgot to include _trim_key function Got removed somehow --- cloudinit/sources/DataSourceGCE.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'cloudinit/sources/DataSourceGCE.py') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index f9cdc1da..8a46f933 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -30,6 +30,16 @@ class DataSourceGCE(sources.DataSource): self.metadata_address = MD_URL self.metadata = {} + # GCE takes sshKeys attribute in the format of ':' + # so we have to trim each key to remove the username part + def _trim_key(self, public_key): + try: + index = public_key.index(':') + if index > 0: + return public_key[(index + 1):] + except: + return public_key + def get_data(self): # GCE metadata server requires a custom header since v1 headers = {'X-Google-Metadata-Request': True} -- cgit v1.2.3 From aa1d0328da6ae1ace03e327feafabcf84f91b2b5 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Sat, 8 Feb 2014 20:59:52 +0000 Subject: wrap url get call in try/except clause --- cloudinit/sources/DataSourceGCE.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'cloudinit/sources/DataSourceGCE.py') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 8a46f933..c96cfffd 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -52,7 +52,10 @@ class DataSourceGCE(sources.DataSource): } for mkey in url_map.iterkeys(): - resp = url_helper.readurl(url=url_map[mkey], headers=headers) + try: + resp = url_helper.readurl(url=url_map[mkey], headers=headers) + except IOError: + return False if resp.ok(): if mkey == 'public-keys': pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] -- cgit v1.2.3 From 065287bd56fe7f63a8dc41fa1457be4439f20efd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 12 Feb 2014 19:58:38 -0500 Subject: support configuration of MD_URL, disable if not resolvable. this allows the metadata url to be configured by setting: datasource: GCE: metadata_url: Then also, if its not resolvable, we just deactivate the datasource quickly. --- cloudinit/sources/DataSourceGCE.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources/DataSourceGCE.py') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index c96cfffd..6eb3da4d 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -16,19 +16,25 @@ from cloudinit import log as logging +from cloudinit import util from cloudinit import sources from cloudinit import url_helper LOG = logging.getLogger(__name__) -MD_URL = 'http://metadata/computeMetadata/v1/' +BUILTIN_DS_CONFIG = { + 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' +} class DataSourceGCE(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.metadata_address = MD_URL self.metadata = {} + self.ds_cfg = util.mergemanydict([ + util.get_cfg_by_path(sys_cfg, ["datasource", "GCE"], {}), + BUILTIN_DS_CONFIG]) + self.metadata_address = self.ds_cfg['metadata_url'] # GCE takes sshKeys attribute in the format of ':' # so we have to trim each key to remove the username part @@ -51,6 +57,11 @@ class DataSourceGCE(sources.DataSource): 'local-hostname': self.metadata_address + 'instance/hostname', } + # if we cannot resolve the metadata server, then no point in trying + if not util.is_resolvable(self.metadata_address): + LOG.debug("%s is not resolvable", self.metadata_address) + return False + for mkey in url_map.iterkeys(): try: resp = url_helper.readurl(url=url_map[mkey], headers=headers) -- cgit v1.2.3 From 64856d7f245cc329a11b263c4eef8bf76bdee0e6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 10:58:08 -0500 Subject: add 'user-data' support. This just adds user-data in 'instance/attributes/user-data'. Also turns retries to 0 on all other things. --- cloudinit/sources/DataSourceGCE.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'cloudinit/sources/DataSourceGCE.py') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 6eb3da4d..95a410ba 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -25,6 +25,7 @@ LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' } +REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') class DataSourceGCE(sources.DataSource): @@ -55,6 +56,7 @@ class DataSourceGCE(sources.DataSource): 'availability-zone': self.metadata_address + 'instance/zone', 'public-keys': self.metadata_address + 'project/attributes/sshKeys', 'local-hostname': self.metadata_address + 'instance/hostname', + 'user-data': self.metadata_address + 'instance/attributes/user-data', } # if we cannot resolve the metadata server, then no point in trying @@ -64,7 +66,8 @@ class DataSourceGCE(sources.DataSource): for mkey in url_map.iterkeys(): try: - resp = url_helper.readurl(url=url_map[mkey], headers=headers) + resp = url_helper.readurl(url=url_map[mkey], headers=headers, + retries=0) except IOError: return False if resp.ok(): @@ -74,8 +77,15 @@ class DataSourceGCE(sources.DataSource): else: self.metadata[mkey] = resp.contents else: + if mkey in REQUIRED_FIELDS: + LOG.warn("required metadata '%s' not found in metadata", + url_map[mkey]) + return False + self.metadata[mkey] = None return False + + self.user_data_raw = self.metadata['user-data'] return True @property @@ -92,9 +102,6 @@ class DataSourceGCE(sources.DataSource): def get_hostname(self, fqdn=False): return self.metadata['local-hostname'] - def get_userdata_raw(self): - return None - @property def availability_zone(self): return self.metadata['instance-zone'] -- cgit v1.2.3