From 29b838af062549ef5e2f12c3df74721d294dea37 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Mon, 3 Feb 2014 11:28:50 +0000 Subject: Add Google Compute Engine data source support. --- cloudinit/settings.py | 1 + cloudinit/sources/DataSourceGCE.py | 100 +++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 cloudinit/sources/DataSourceGCE.py diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 7be2199a..7c598bf9 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -36,6 +36,7 @@ CFG_BUILTIN = { 'AltCloud', 'OVF', 'MAAS', + 'GCE', 'Ec2', 'CloudStack', 'SmartOS', diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py new file mode 100644 index 00000000..b6a82245 --- /dev/null +++ b/cloudinit/sources/DataSourceGCE.py @@ -0,0 +1,100 @@ +# vi: ts=4 expandtab +# +# Author: Vaidas Jablonskis +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import requests + +from cloudinit import log as logging +from cloudinit import sources + +LOG = logging.getLogger(__name__) + +MD_URL = 'http://metadata/computeMetadata/v1/' + + +class DataSourceGCE(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.metadata_address = MD_URL + self.metadata = {} + + # GCE takes sshKeys attribute in the format of ':' + # so we have to trim each key to remove the username part + def _trim_key(self, public_key): + try: + index = public_key.index(':') + if index > 0: + return public_key[(index + 1):] + except: + return public_key + + def get_data(self): + # GCE metadata server requires a custom header since v1 + headers = {'X-Google-Metadata-Request': True} + + url_map = { + 'instance-id': self.metadata_address + 'instance/id', + 'availability-zone': self.metadata_address + 'instance/zone', + 'public-keys': self.metadata_address + 'project/attributes/sshKeys', + 'local-hostname': self.metadata_address + 'instance/hostname', + } + + with requests.Session() as s: + for mkey in url_map.iterkeys(): + try: + r = s.get(url_map[mkey], headers=headers) + except requests.exceptions.ConnectionError: + return False + if r.ok: + if mkey == 'public-keys': + pub_keys = [self._trim_key(k) for k in r.text.splitlines()] + self.metadata[mkey] = pub_keys + else: + self.metadata[mkey] = r.text + else: + self.metadata[mkey] = None + return False + return True + + @property + def launch_index(self): + # GCE does not provide lauch_index property + return None + + def get_instance_id(self): + return self.metadata['instance-id'] + + def get_public_ssh_keys(self): + return self.metadata['public-keys'] + + def get_hostname(self, fqdn=False): + return self.metadata['local-hostname'] + + def get_userdata_raw(self): + return None + + @property + def availability_zone(self): + return self.metadata['instance-zone'] + +# Used to match classes to dependencies +datasources = [ + (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) -- cgit v1.2.3 From e1710e06fd09c3f643fdca851c011da3a3eb7a98 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Fri, 7 Feb 2014 09:43:32 +0000 Subject: use url_helper instead of requests --- cloudinit/sources/DataSourceGCE.py | 38 ++++++++++++-------------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index b6a82245..f9cdc1da 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -14,10 +14,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import requests from cloudinit import log as logging from cloudinit import sources +from cloudinit import url_helper LOG = logging.getLogger(__name__) @@ -30,16 +30,6 @@ class DataSourceGCE(sources.DataSource): self.metadata_address = MD_URL self.metadata = {} - # GCE takes sshKeys attribute in the format of ':' - # so we have to trim each key to remove the username part - def _trim_key(self, public_key): - try: - index = public_key.index(':') - if index > 0: - return public_key[(index + 1):] - except: - return public_key - def get_data(self): # GCE metadata server requires a custom header since v1 headers = {'X-Google-Metadata-Request': True} @@ -51,22 +41,18 @@ class DataSourceGCE(sources.DataSource): 'local-hostname': self.metadata_address + 'instance/hostname', } - with requests.Session() as s: - for mkey in url_map.iterkeys(): - try: - r = s.get(url_map[mkey], headers=headers) - except requests.exceptions.ConnectionError: - return False - if r.ok: - if mkey == 'public-keys': - pub_keys = [self._trim_key(k) for k in r.text.splitlines()] - self.metadata[mkey] = pub_keys - else: - self.metadata[mkey] = r.text + for mkey in url_map.iterkeys(): + resp = url_helper.readurl(url=url_map[mkey], headers=headers) + if resp.ok(): + if mkey == 'public-keys': + pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] + self.metadata[mkey] = pub_keys else: - self.metadata[mkey] = None - return False - return True + self.metadata[mkey] = resp.contents + else: + self.metadata[mkey] = None + return False + return True @property def launch_index(self): -- cgit v1.2.3 From 64b35a9c3b201aed5073823fd4a15ecb15195aea Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Fri, 7 Feb 2014 13:12:48 +0000 Subject: Forgot to include _trim_key function Got removed somehow --- cloudinit/sources/DataSourceGCE.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index f9cdc1da..8a46f933 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -30,6 +30,16 @@ class DataSourceGCE(sources.DataSource): self.metadata_address = MD_URL self.metadata = {} + # GCE takes sshKeys attribute in the format of ':' + # so we have to trim each key to remove the username part + def _trim_key(self, public_key): + try: + index = public_key.index(':') + if index > 0: + return public_key[(index + 1):] + except: + return public_key + def get_data(self): # GCE metadata server requires a custom header since v1 headers = {'X-Google-Metadata-Request': True} -- cgit v1.2.3 From aa1d0328da6ae1ace03e327feafabcf84f91b2b5 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Sat, 8 Feb 2014 20:59:52 +0000 Subject: wrap url get call in try/except clause --- cloudinit/sources/DataSourceGCE.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 8a46f933..c96cfffd 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -52,7 +52,10 @@ class DataSourceGCE(sources.DataSource): } for mkey in url_map.iterkeys(): - resp = url_helper.readurl(url=url_map[mkey], headers=headers) + try: + resp = url_helper.readurl(url=url_map[mkey], headers=headers) + except IOError: + return False if resp.ok(): if mkey == 'public-keys': pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] -- cgit v1.2.3 From 065287bd56fe7f63a8dc41fa1457be4439f20efd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 12 Feb 2014 19:58:38 -0500 Subject: support configuration of MD_URL, disable if not resolvable. this allows the metadata url to be configured by setting: datasource: GCE: metadata_url: Then also, if its not resolvable, we just deactivate the datasource quickly. --- cloudinit/sources/DataSourceGCE.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index c96cfffd..6eb3da4d 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -16,19 +16,25 @@ from cloudinit import log as logging +from cloudinit import util from cloudinit import sources from cloudinit import url_helper LOG = logging.getLogger(__name__) -MD_URL = 'http://metadata/computeMetadata/v1/' +BUILTIN_DS_CONFIG = { + 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' +} class DataSourceGCE(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.metadata_address = MD_URL self.metadata = {} + self.ds_cfg = util.mergemanydict([ + util.get_cfg_by_path(sys_cfg, ["datasource", "GCE"], {}), + BUILTIN_DS_CONFIG]) + self.metadata_address = self.ds_cfg['metadata_url'] # GCE takes sshKeys attribute in the format of ':' # so we have to trim each key to remove the username part @@ -51,6 +57,11 @@ class DataSourceGCE(sources.DataSource): 'local-hostname': self.metadata_address + 'instance/hostname', } + # if we cannot resolve the metadata server, then no point in trying + if not util.is_resolvable(self.metadata_address): + LOG.debug("%s is not resolvable", self.metadata_address) + return False + for mkey in url_map.iterkeys(): try: resp = url_helper.readurl(url=url_map[mkey], headers=headers) -- cgit v1.2.3 From 18b8d40d96d2e19f1a949b06b55a3dac54595b23 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 10:39:39 -0500 Subject: cloudsigma: change default dsmode to 'net' Previously this had 'local' as the default datasource mode, meaning that user-data code such as boot hooks and such would not be guaranteed to have network access. That would be out of sync with the expectation on other platforms where the default is 'network up'. The user can still specify 'dsmode' as local if necessary and the local datasource will claim itself found. --- cloudinit/sources/DataSourceCloudSigma.py | 27 ++++++++++++++++++++------- doc/sources/cloudsigma/README.rst | 6 +++--- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index 78acd8a4..e734d7e5 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -45,18 +45,25 @@ class DataSourceCloudSigma(sources.DataSource): Metadata is the whole server context and /meta/cloud-config is used as userdata. """ + dsmode = None try: server_context = self.cepko.all().result server_meta = server_context['meta'] - self.userdata_raw = server_meta.get('cloudinit-user-data', "") - self.metadata = server_context - self.ssh_public_key = server_meta['ssh_public_key'] - - if server_meta.get('cloudinit-dsmode') in VALID_DSMODES: - self.dsmode = server_meta['cloudinit-dsmode'] except: util.logexc(LOG, "Failed reading from the serial port") return False + + dsmode = server_meta.get('cloudinit-dsmode', self.dsmode) + if dsmode not in VALID_DSMODES: + LOG.warn("Invalid dsmode %s, assuming default of 'net'", dsmode) + dsmode = 'net' + if dsmode == "disabled" or dsmode != self.dsmode: + return False + + self.userdata_raw = server_meta.get('cloudinit-user-data', "") + self.metadata = server_context + self.ssh_public_key = server_meta['ssh_public_key'] + return True def get_hostname(self, fqdn=False, resolve_ip=False): @@ -76,11 +83,17 @@ class DataSourceCloudSigma(sources.DataSource): return self.metadata['uuid'] +class DataSourceCloudSigmaNet(DataSourceCloudSigma): + def __init__(self, sys_cfg, distro, paths): + DataSourceCloudSigma.__init__(self, sys_cfg, distro, paths) + self.dsmode = 'net' + + # Used to match classes to dependencies. Since this datasource uses the serial # port network is not really required, so it's okay to load without it, too. datasources = [ (DataSourceCloudSigma, (sources.DEP_FILESYSTEM)), - (DataSourceCloudSigma, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + (DataSourceCloudSigmaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/doc/sources/cloudsigma/README.rst b/doc/sources/cloudsigma/README.rst index 8cb2b0fe..1d9160a2 100644 --- a/doc/sources/cloudsigma/README.rst +++ b/doc/sources/cloudsigma/README.rst @@ -23,9 +23,9 @@ You can provide user-data to the VM using the dedicated `meta field`_ in the `se header could be omitted. However since this is a raw-text field you could provide any of the valid `config formats`_. -If your user-data needs an internet connection you have to create a `meta field`_ in the `server context`_ -``cloudinit-dsmode`` and set "net" as value. If this field does not exist the default value is "local". - +If your user-data does not need an internet connection you can create a +`meta field`_ in the `server context`_ ``cloudinit-dsmode`` and set "local" as value. +If this field does not exist the default value is "net". .. _CloudSigma: http://cloudsigma.com/ -- cgit v1.2.3 From 64856d7f245cc329a11b263c4eef8bf76bdee0e6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 10:58:08 -0500 Subject: add 'user-data' support. This just adds user-data in 'instance/attributes/user-data'. Also turns retries to 0 on all other things. --- cloudinit/sources/DataSourceGCE.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 6eb3da4d..95a410ba 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -25,6 +25,7 @@ LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' } +REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') class DataSourceGCE(sources.DataSource): @@ -55,6 +56,7 @@ class DataSourceGCE(sources.DataSource): 'availability-zone': self.metadata_address + 'instance/zone', 'public-keys': self.metadata_address + 'project/attributes/sshKeys', 'local-hostname': self.metadata_address + 'instance/hostname', + 'user-data': self.metadata_address + 'instance/attributes/user-data', } # if we cannot resolve the metadata server, then no point in trying @@ -64,7 +66,8 @@ class DataSourceGCE(sources.DataSource): for mkey in url_map.iterkeys(): try: - resp = url_helper.readurl(url=url_map[mkey], headers=headers) + resp = url_helper.readurl(url=url_map[mkey], headers=headers, + retries=0) except IOError: return False if resp.ok(): @@ -74,8 +77,15 @@ class DataSourceGCE(sources.DataSource): else: self.metadata[mkey] = resp.contents else: + if mkey in REQUIRED_FIELDS: + LOG.warn("required metadata '%s' not found in metadata", + url_map[mkey]) + return False + self.metadata[mkey] = None return False + + self.user_data_raw = self.metadata['user-data'] return True @property @@ -92,9 +102,6 @@ class DataSourceGCE(sources.DataSource): def get_hostname(self, fqdn=False): return self.metadata['local-hostname'] - def get_userdata_raw(self): - return None - @property def availability_zone(self): return self.metadata['instance-zone'] -- cgit v1.2.3