From f0a6ec70d13ea771efee86b2544346731cd79991 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 11:58:52 -0500 Subject: Add initial DataSourceMaaS. Tests at this point seem to indicate that seed-dir would work. --- cloudinit/DataSourceMaaS.py | 308 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 308 insertions(+) create mode 100644 cloudinit/DataSourceMaaS.py (limited to 'cloudinit') diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py new file mode 100644 index 00000000..2bc1f71f --- /dev/null +++ b/cloudinit/DataSourceMaaS.py @@ -0,0 +1,308 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import errno +import oauth.oauth as oauth +import os.path +import socket +import urllib2 +import time + + +class DataSourceMaaS(DataSource.DataSource): + """ + DataSourceMaaS reads instance information from MaaS. + Given a config metadata_url, and oauth tokens, it expects to find + files under the root named: + instance-id + user-data + hostname + """ + seeddir = base_seeddir + '/maas' + baseurl = None + + def __str__(self): + return("DataSourceMaaS[%s]" % self.baseurl) + + def get_data(self): + mcfg = self.ds_cfg + + try: + (userdata, metadata) = read_maas_seed_dir(self.seeddir) + self.userdata_raw = userdata + self.metadata = metadata + self.baseurl = self.seeddir + return True + except MaasSeedDirNone: + pass + except MaasSeedDirMalformed as exc: + log.warn("%s was malformed: %s\n" % (self.seeddir, exc)) + raise + + try: + # if there is no metadata_url, then we're not configured + url = mcfg.get('metadata_url', None) + if url == None: + return False + + if not self.wait_for_metadata_service(url): + return False + + self.baseurl = url + + (userdata, metadata) = read_maas_seed_url(self.baseurl, + self.md_headers) + return True + except Exception: + util.logexc(log) + return False + + def md_headers(self, url): + mcfg = self.ds_cfg + + # if we are missing token_key, token_secret or consumer_key + # then just do non-authed requests + for required in ('token_key', 'token_secret', 'consumer_key'): + if required not in mcfg: + return({}) + + consumer_secret = mcfg.get('consumer_secret', "") + + return(oauth_headers(url=url, consumer_key=mcfg['consumer_key'], + token_key=mcfg['token_key'], token_secret=mcfg['token_secret'], + consumer_secret=consumer_secret)) + + def wait_for_metadata_service(self, url): + mcfg = self.ds_cfg + + max_wait = 120 + try: + max_wait = int(mcfg.get("max_wait", max_wait)) + except Exception: + util.logexc(log) + log.warn("Failed to get max wait. using %s" % max_wait) + + if max_wait == 0: + return False + + timeout = 50 + try: + timeout = int(mcfg.get("timeout", timeout)) + except Exception: + util.logexc(log) + log.warn("Failed to get timeout, using %s" % timeout) + + starttime = time.time() + check_url = "%s/instance-id" % url + url = wait_for_metadata_service(urls=[check_url], max_wait=max_wait, + timeout=timeout, status_cb=log.warn, + headers_cb=self.md_headers) + + if url: + log.debug("Using metadata source: '%s'" % url) + else: + log.critical("giving up on md after %i seconds\n" % + int(time.time() - starttime)) + + return (bool(url)) + + +def wait_for_metadata_service(urls, max_wait=None, timeout=None, + status_cb=None, headers_cb=None): + """ + urls: a list of urls to try + max_wait: roughly the maximum time to wait before giving up + The max time is *actually* len(urls)*timeout as each url will + be tried once and given the timeout provided. + timeout: the timeout provided to urllib2.urlopen + status_cb: call method with string message when a url is not available + + the idea of this routine is to wait for the EC2 metdata service to + come up. On both Eucalyptus and EC2 we have seen the case where + the instance hit the MD before the MD service was up. EC2 seems + to have permenantely fixed this, though. + + In openstack, the metadata service might be painfully slow, and + unable to avoid hitting a timeout of even up to 10 seconds or more + (LP: #894279) for a simple GET. + + Offset those needs with the need to not hang forever (and block boot) + on a system where cloud-init is configured to look for EC2 Metadata + service but is not going to find one. It is possible that the instance + data host (169.254.169.254) may be firewalled off Entirely for a sytem, + meaning that the connection will block forever unless a timeout is set. + """ + starttime = time.time() + + sleeptime = 1 + + def nullstatus_cb(msg): + return + + if status_cb == None: + status_cb = nullstatus_cb + + def timeup(max_wait, starttime): + return((max_wait <= 0 or max_wait == None) or + (time.time() - starttime > max_wait)) + + loop_n = 0 + while True: + sleeptime = int(loop_n / 5) + 1 + for url in urls: + now = time.time() + if loop_n != 0: + if timeup(max_wait, starttime): + break + if timeout and (now + timeout > (starttime + max_wait)): + # shorten timeout to not run way over max_time + timeout = int((starttime + max_wait) - now) + + reason = "" + try: + if headers_cb != None: + headers = headers_cb(url) + else: + headers = {} + + req = urllib2.Request(url, data=None, headers=headers) + resp = urllib2.urlopen(req, timeout=timeout) + if resp.read() != "": + return url + reason = "empty data [%s]" % resp.getcode() + except urllib2.HTTPError as e: + reason = "http error [%s]" % e.code + except urllib2.URLError as e: + reason = "url error [%s]" % e.reason + except socket.timeout as e: + reason = "socket timeout [%s]" % e + except Exception as e: + reason = "unexpected error [%s]" % e + + if log: + status_cb("'%s' failed [%s/%ss]: %s" % + (url, int(time.time() - starttime), max_wait, + reason)) + + if timeup(max_wait, starttime): + break + + loop_n = loop_n + 1 + time.sleep(sleeptime) + + return False + + +def read_maas_seed_dir(seed_d): + """ + Return user-data and metadata for a maas seed dir in seed_d. + Expected format of seed_d are the following files: + * instance-id + * hostname + * user-data + """ + md_required = set(('hostname', 'instance-id')) + files = md_required.union(set(('userdata',))) + userdata = None + md = {} + + if not os.path.isdir(seed_d): + raise MaasSeedDirNone("%s: not a directory") + + for fname in files: + try: + with open(os.path.join(seed_d, fname)) as fp: + if fname == 'userdata': + userdata = fp.read() + else: + md[fname] = fp.read() + fp.close() + except IOError as e: + if e.errno != errno.ENOENT: + raise + + if userdata == None and len(md) == 0: + raise MaasSeedDirNone("%s: no data files found" % seed_d) + + if userdata == None: + raise MaasSeedDirMalformed("%s: missing userdata" % seed_d) + + missing = md_required - set(md.keys()) + if len(missing): + raise MaasSeedDirMalformed("%s: missing files %s" % + (seed_d, str(missing))) + + return(userdata, md) + + +def read_maas_seed_url(seed_url, header_cb=None): + """ + Read the maas datasource at seed_url. + header_cb is a method that should return a headers dictionary that will + be given to urllib2.Request() + + Expected format of seed_url is are the following files: + * /instance-id + * /hostname + * /user-data + """ + userdata = "" + metadata = {'instance-id': 'i-maas-url', 'hostname': 'maas-url-hostname'} + + return(userdata, metadata) + + +def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): + consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) + token = oauth.OAuthToken(token_key, token_secret) + params = { + 'oauth_version': "1.0", + 'oauth_nonce': oauth.generate_nonce(), + 'oauth_timestamp': int(time.time()), + 'oauth_token': token.key, + 'oauth_consumer_key': consumer.key, + } + req = oauth.OAuthRequest(http_url=url, parameters=params) + req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(), + consumer, token) + return(req.to_header()) + + +class MaasSeedDirNone(Exception): + pass + + +class MaasSeedDirMalformed(Exception): + pass + + +datasources = [ + (DataSourceMaaS, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +] + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) -- cgit v1.2.3 From 2303079f3ad6c98cee8f27a0488057297cb049fb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 12:53:40 -0500 Subject: move wait_for_metadata_service for util, rename to wait_for_url Also, add in the headers_cb which will be required for oauth. --- cloudinit/DataSourceEc2.py | 85 ++----------------------------------------- cloudinit/DataSourceMaaS.py | 88 +-------------------------------------------- cloudinit/util.py | 87 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 170 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py index 06635746..4e06803d 100644 --- a/cloudinit/DataSourceEc2.py +++ b/cloudinit/DataSourceEc2.py @@ -134,8 +134,8 @@ class DataSourceEc2(DataSource.DataSource): url2base[cur] = url starttime = time.time() - url = wait_for_metadata_service(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=log.warn) + url = util.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, status_cb=log.warn) if url: log.debug("Using metadata source: '%s'" % url2base[url]) @@ -208,87 +208,6 @@ class DataSourceEc2(DataSource.DataSource): return False -def wait_for_metadata_service(urls, max_wait=None, timeout=None, - status_cb=None): - """ - urls: a list of urls to try - max_wait: roughly the maximum time to wait before giving up - The max time is *actually* len(urls)*timeout as each url will - be tried once and given the timeout provided. - timeout: the timeout provided to urllib2.urlopen - status_cb: call method with string message when a url is not available - - the idea of this routine is to wait for the EC2 metdata service to - come up. On both Eucalyptus and EC2 we have seen the case where - the instance hit the MD before the MD service was up. EC2 seems - to have permenantely fixed this, though. - - In openstack, the metadata service might be painfully slow, and - unable to avoid hitting a timeout of even up to 10 seconds or more - (LP: #894279) for a simple GET. - - Offset those needs with the need to not hang forever (and block boot) - on a system where cloud-init is configured to look for EC2 Metadata - service but is not going to find one. It is possible that the instance - data host (169.254.169.254) may be firewalled off Entirely for a sytem, - meaning that the connection will block forever unless a timeout is set. - """ - starttime = time.time() - - sleeptime = 1 - - def nullstatus_cb(msg): - return - - if status_cb == None: - status_cb = nullstatus_cb - - def timeup(max_wait, starttime): - return((max_wait <= 0 or max_wait == None) or - (time.time() - starttime > max_wait)) - - loop_n = 0 - while True: - sleeptime = int(loop_n / 5) + 1 - for url in urls: - now = time.time() - if loop_n != 0: - if timeup(max_wait, starttime): - break - if timeout and (now + timeout > (starttime + max_wait)): - # shorten timeout to not run way over max_time - timeout = int((starttime + max_wait) - now) - - reason = "" - try: - req = urllib2.Request(url) - resp = urllib2.urlopen(req, timeout=timeout) - if resp.read() != "": - return url - reason = "empty data [%s]" % resp.getcode() - except urllib2.HTTPError as e: - reason = "http error [%s]" % e.code - except urllib2.URLError as e: - reason = "url error [%s]" % e.reason - except socket.timeout as e: - reason = "socket timeout [%s]" % e - except Exception as e: - reason = "unexpected error [%s]" % e - - if log: - status_cb("'%s' failed [%s/%ss]: %s" % - (url, int(time.time() - starttime), max_wait, - reason)) - - if timeup(max_wait, starttime): - break - - loop_n = loop_n + 1 - time.sleep(sleeptime) - - return False - - datasources = [ (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), ] diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index 2bc1f71f..d902ccb4 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -116,7 +116,7 @@ class DataSourceMaaS(DataSource.DataSource): starttime = time.time() check_url = "%s/instance-id" % url - url = wait_for_metadata_service(urls=[check_url], max_wait=max_wait, + url = util.wait_for_url(urls=[check_url], max_wait=max_wait, timeout=timeout, status_cb=log.warn, headers_cb=self.md_headers) @@ -129,92 +129,6 @@ class DataSourceMaaS(DataSource.DataSource): return (bool(url)) -def wait_for_metadata_service(urls, max_wait=None, timeout=None, - status_cb=None, headers_cb=None): - """ - urls: a list of urls to try - max_wait: roughly the maximum time to wait before giving up - The max time is *actually* len(urls)*timeout as each url will - be tried once and given the timeout provided. - timeout: the timeout provided to urllib2.urlopen - status_cb: call method with string message when a url is not available - - the idea of this routine is to wait for the EC2 metdata service to - come up. On both Eucalyptus and EC2 we have seen the case where - the instance hit the MD before the MD service was up. EC2 seems - to have permenantely fixed this, though. - - In openstack, the metadata service might be painfully slow, and - unable to avoid hitting a timeout of even up to 10 seconds or more - (LP: #894279) for a simple GET. - - Offset those needs with the need to not hang forever (and block boot) - on a system where cloud-init is configured to look for EC2 Metadata - service but is not going to find one. It is possible that the instance - data host (169.254.169.254) may be firewalled off Entirely for a sytem, - meaning that the connection will block forever unless a timeout is set. - """ - starttime = time.time() - - sleeptime = 1 - - def nullstatus_cb(msg): - return - - if status_cb == None: - status_cb = nullstatus_cb - - def timeup(max_wait, starttime): - return((max_wait <= 0 or max_wait == None) or - (time.time() - starttime > max_wait)) - - loop_n = 0 - while True: - sleeptime = int(loop_n / 5) + 1 - for url in urls: - now = time.time() - if loop_n != 0: - if timeup(max_wait, starttime): - break - if timeout and (now + timeout > (starttime + max_wait)): - # shorten timeout to not run way over max_time - timeout = int((starttime + max_wait) - now) - - reason = "" - try: - if headers_cb != None: - headers = headers_cb(url) - else: - headers = {} - - req = urllib2.Request(url, data=None, headers=headers) - resp = urllib2.urlopen(req, timeout=timeout) - if resp.read() != "": - return url - reason = "empty data [%s]" % resp.getcode() - except urllib2.HTTPError as e: - reason = "http error [%s]" % e.code - except urllib2.URLError as e: - reason = "url error [%s]" % e.reason - except socket.timeout as e: - reason = "socket timeout [%s]" % e - except Exception as e: - reason = "unexpected error [%s]" % e - - if log: - status_cb("'%s' failed [%s/%ss]: %s" % - (url, int(time.time() - starttime), max_wait, - reason)) - - if timeup(max_wait, starttime): - break - - loop_n = loop_n + 1 - time.sleep(sleeptime) - - return False - - def read_maas_seed_dir(seed_d): """ Return user-data and metadata for a maas seed dir in seed_d. diff --git a/cloudinit/util.py b/cloudinit/util.py index c37f0316..882fd9fb 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -751,3 +751,90 @@ def mount_callback_umount(device, callback, data=None): _cleanup(umount, tmpd) return(ret) + +def wait_for_url(urls, max_wait=None, timeout=None, + status_cb=None, headers_cb=None): + """ + urls: a list of urls to try + max_wait: roughly the maximum time to wait before giving up + The max time is *actually* len(urls)*timeout as each url will + be tried once and given the timeout provided. + timeout: the timeout provided to urllib2.urlopen + status_cb: call method with string message when a url is not available + + the idea of this routine is to wait for the EC2 metdata service to + come up. On both Eucalyptus and EC2 we have seen the case where + the instance hit the MD before the MD service was up. EC2 seems + to have permenantely fixed this, though. + + In openstack, the metadata service might be painfully slow, and + unable to avoid hitting a timeout of even up to 10 seconds or more + (LP: #894279) for a simple GET. + + Offset those needs with the need to not hang forever (and block boot) + on a system where cloud-init is configured to look for EC2 Metadata + service but is not going to find one. It is possible that the instance + data host (169.254.169.254) may be firewalled off Entirely for a sytem, + meaning that the connection will block forever unless a timeout is set. + """ + starttime = time.time() + + sleeptime = 1 + + def nullstatus_cb(msg): + return + + if status_cb == None: + status_cb = nullstatus_cb + + def timeup(max_wait, starttime): + return((max_wait <= 0 or max_wait == None) or + (time.time() - starttime > max_wait)) + + loop_n = 0 + while True: + sleeptime = int(loop_n / 5) + 1 + for url in urls: + now = time.time() + if loop_n != 0: + if timeup(max_wait, starttime): + break + if timeout and (now + timeout > (starttime + max_wait)): + # shorten timeout to not run way over max_time + timeout = int((starttime + max_wait) - now) + + reason = "" + try: + if headers_cb != None: + headers = headers_cb(url) + else: + headers = {} + + req = urllib2.Request(url, data=None, headers=headers) + resp = urllib2.urlopen(req, timeout=timeout) + if resp.read() != "": + return url + reason = "empty data [%s]" % resp.getcode() + except urllib2.HTTPError as e: + reason = "http error [%s]" % e.code + except urllib2.URLError as e: + reason = "url error [%s]" % e.reason + except socket.timeout as e: + reason = "socket timeout [%s]" % e + except Exception as e: + reason = "unexpected error [%s]" % e + + if log: + status_cb("'%s' failed [%s/%ss]: %s" % + (url, int(time.time() - starttime), max_wait, + reason)) + + if timeup(max_wait, starttime): + break + + loop_n = loop_n + 1 + time.sleep(sleeptime) + + return False + + -- cgit v1.2.3 From bb072c5661ffcc8b1d69f9d8d15e8ff5839f5305 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 17:06:09 -0500 Subject: add headers_cb to doc for wait_for_url --- cloudinit/util.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/util.py b/cloudinit/util.py index 882fd9fb..897f0a0d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -761,6 +761,8 @@ def wait_for_url(urls, max_wait=None, timeout=None, be tried once and given the timeout provided. timeout: the timeout provided to urllib2.urlopen status_cb: call method with string message when a url is not available + headers_cb: call method with single argument of url to get headers + for request. the idea of this routine is to wait for the EC2 metdata service to come up. On both Eucalyptus and EC2 we have seen the case where -- cgit v1.2.3 From 0e2e852bb448553479e68e91896419f3e06a6a19 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 17:06:28 -0500 Subject: functional read_maas_seed_url This commits a generally functional read_maas_seed_url, and re-works how the content is checked, to share between read_maas_seed{url,dir}. --- cloudinit/DataSourceMaaS.py | 90 +++++++++++++++++++++++++++++++++------------ 1 file changed, 67 insertions(+), 23 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index d902ccb4..c25652b1 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -137,9 +137,7 @@ def read_maas_seed_dir(seed_d): * hostname * user-data """ - md_required = set(('hostname', 'instance-id')) - files = md_required.union(set(('userdata',))) - userdata = None + files = ('hostname', 'instance-id', 'userdata') md = {} if not os.path.isdir(seed_d): @@ -148,30 +146,16 @@ def read_maas_seed_dir(seed_d): for fname in files: try: with open(os.path.join(seed_d, fname)) as fp: - if fname == 'userdata': - userdata = fp.read() - else: - md[fname] = fp.read() + md[fname] = fp.read() fp.close() except IOError as e: if e.errno != errno.ENOENT: raise - if userdata == None and len(md) == 0: - raise MaasSeedDirNone("%s: no data files found" % seed_d) + return(check_seed_contents(md, seed_d)) - if userdata == None: - raise MaasSeedDirMalformed("%s: missing userdata" % seed_d) - missing = md_required - set(md.keys()) - if len(missing): - raise MaasSeedDirMalformed("%s: missing files %s" % - (seed_d, str(missing))) - - return(userdata, md) - - -def read_maas_seed_url(seed_url, header_cb=None): +def read_maas_seed_url(seed_url, header_cb=None, timeout=None): """ Read the maas datasource at seed_url. header_cb is a method that should return a headers dictionary that will @@ -182,10 +166,51 @@ def read_maas_seed_url(seed_url, header_cb=None): * /hostname * /user-data """ - userdata = "" - metadata = {'instance-id': 'i-maas-url', 'hostname': 'maas-url-hostname'} + files = ('hostname', 'instance-id', 'userdata') - return(userdata, metadata) + md = {} + for fname in files: + url = "%s/%s" % (seed_url, fname) + if header_cb: + headers = header_cb(url) + else: + headers = {} + + req = urllib2.Request(url, data=None, headers=headers) + resp = urllib2.urlopen(req, timeout=timeout) + + md[fname] = resp.read() + + return(check_seed_contents(md, seed_url)) + + +def check_seed_contents(content, seed): + """Validate if content is Is the content a dict that is valid as a + return for a datasource. + Either return a (userdata, metadata) tuple or + Raise MaasSeedDirMalformed or MaasSeedDirNone + """ + md_required = ('userdata', 'instance-id', 'hostname') + found = content.keys() + + if len(content) == 0: + raise MaasSeedDirNone("%s: no data files found" % seed) + + if 'userdata' not in content: + raise MaasSeedDirMalformed("%s: missing userdata" % seed) + + missing = [k for k in md_required if k not in found] + if len(missing): + raise MaasSeedDirMalformed("%s: missing files %s" % (seed, missing)) + + userdata = content['userdata'] + md = { } + for (key, val) in content.iteritems(): + if key == 'userdata': + continue + md[key] = val + + return(userdata, md) def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): @@ -220,3 +245,22 @@ datasources = [ # return a list of data sources that match this set of dependencies def get_datasource_list(depends): return(DataSource.list_from_depends(depends, datasources)) + + +if __name__ == "__main__": + def main(): + import sys + import pprint + seed = sys.argv[1] + if seed.startswith("http://") or seed.startswith("https://"): + (userdata, metadata) = read_maas_seed_url(seed) + else: + (userdata, metadata) = read_maas_seed_dir(seed) + + print "=== userdata ===" + print userdata + print "=== metadata ===" + pprint.pprint(metadata) + + main() + -- cgit v1.2.3 From 27ad683638a30e202a0def71485fade430976856 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 17:10:52 -0500 Subject: file for user-data should be 'user-data' (including the '-') --- cloudinit/DataSourceMaaS.py | 14 +++++++------- tests/unittests/test_datasource/test_maas.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index c25652b1..88686b13 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -137,7 +137,7 @@ def read_maas_seed_dir(seed_d): * hostname * user-data """ - files = ('hostname', 'instance-id', 'userdata') + files = ('hostname', 'instance-id', 'user-data') md = {} if not os.path.isdir(seed_d): @@ -166,7 +166,7 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None): * /hostname * /user-data """ - files = ('hostname', 'instance-id', 'userdata') + files = ('hostname', 'instance-id', 'user-data') md = {} for fname in files: @@ -190,23 +190,23 @@ def check_seed_contents(content, seed): Either return a (userdata, metadata) tuple or Raise MaasSeedDirMalformed or MaasSeedDirNone """ - md_required = ('userdata', 'instance-id', 'hostname') + md_required = ('user-data', 'instance-id', 'hostname') found = content.keys() if len(content) == 0: raise MaasSeedDirNone("%s: no data files found" % seed) - if 'userdata' not in content: - raise MaasSeedDirMalformed("%s: missing userdata" % seed) + if 'user-data' not in content: + raise MaasSeedDirMalformed("%s: missing user-data" % seed) missing = [k for k in md_required if k not in found] if len(missing): raise MaasSeedDirMalformed("%s: missing files %s" % (seed, missing)) - userdata = content['userdata'] + userdata = content['user-data'] md = { } for (key, val) in content.iteritems(): - if key == 'userdata': + if key == 'user-data': continue md[key] = val diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 22374415..dc8964d3 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -26,33 +26,33 @@ class TestMaasDataSource(TestCase): """Verify a valid seeddir is read as such""" data = {'instance-id': 'i-valid01', 'hostname': 'valid01-hostname', - 'userdata': 'valid01-userdata'} + 'user-data': 'valid01-userdata'} my_d = os.path.join(self.tmp, "valid") populate_dir(my_d, data) (userdata, metadata) = read_maas_seed_dir(my_d) - self.assertEqual(userdata, data['userdata']) + self.assertEqual(userdata, data['user-data']) for key in ('instance-id', 'hostname'): self.assertEqual(data[key], metadata[key]) # verify that 'userdata' is not returned as part of the metadata - self.assertFalse(('userdata' in metadata)) + self.assertFalse(('user-data' in metadata)) def test_seed_dir_valid_extra(self): """Verify extra files do not affect seed_dir validity """ data = {'instance-id': 'i-valid-extra', 'hostname': 'valid-extra-hostname', - 'userdata': 'valid-extra-userdata', 'foo': 'bar'} + 'user-data': 'valid-extra-userdata', 'foo': 'bar'} my_d = os.path.join(self.tmp, "valid_extra") populate_dir(my_d, data) (userdata, metadata) = read_maas_seed_dir(my_d) - self.assertEqual(userdata, data['userdata']) + self.assertEqual(userdata, data['user-data']) for key in ('instance-id', 'hostname'): self.assertEqual(data[key], metadata[key]) @@ -63,14 +63,14 @@ class TestMaasDataSource(TestCase): """Verify that invalid seed_dir raises MaasSeedDirMalformed""" valid = {'instance-id': 'i-instanceid', - 'hostname': 'test-hostname', 'userdata': ''} + 'hostname': 'test-hostname', 'user-data': ''} my_based = os.path.join(self.tmp, "valid_extra") # missing 'userdata' file my_d = "%s-01" % my_based invalid_data = copy(valid) - del invalid_data['userdata'] + del invalid_data['user-data'] populate_dir(my_d, invalid_data) self.assertRaises(MaasSeedDirMalformed, read_maas_seed_dir, my_d) -- cgit v1.2.3 From ea7ccdb7c118661b8e2487e5b6c7c00fda4e0c85 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 17:11:53 -0500 Subject: no need to check explicitly for user-data --- cloudinit/DataSourceMaaS.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index 88686b13..fc4e890d 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -196,9 +196,6 @@ def check_seed_contents(content, seed): if len(content) == 0: raise MaasSeedDirNone("%s: no data files found" % seed) - if 'user-data' not in content: - raise MaasSeedDirMalformed("%s: missing user-data" % seed) - missing = [k for k in md_required if k not in found] if len(missing): raise MaasSeedDirMalformed("%s: missing files %s" % (seed, missing)) -- cgit v1.2.3 From 3703ed74cea613e96f3d882e90af1cadae30a092 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 7 Mar 2012 09:54:04 -0500 Subject: fix pylint and pep8 warnings --- cloudinit/DataSourceEc2.py | 1 - cloudinit/DataSourceMaaS.py | 8 +++----- cloudinit/util.py | 10 ++++------ tests/unittests/test_datasource/test_maas.py | 2 +- 4 files changed, 8 insertions(+), 13 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py index 4e06803d..7051ecda 100644 --- a/cloudinit/DataSourceEc2.py +++ b/cloudinit/DataSourceEc2.py @@ -24,7 +24,6 @@ from cloudinit import seeddir as base_seeddir from cloudinit import log import cloudinit.util as util import socket -import urllib2 import time import boto.utils as boto_utils import os.path diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index fc4e890d..08a48443 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -26,7 +26,6 @@ import cloudinit.util as util import errno import oauth.oauth as oauth import os.path -import socket import urllib2 import time @@ -185,7 +184,7 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None): def check_seed_contents(content, seed): - """Validate if content is Is the content a dict that is valid as a + """Validate if content is Is the content a dict that is valid as a return for a datasource. Either return a (userdata, metadata) tuple or Raise MaasSeedDirMalformed or MaasSeedDirNone @@ -201,12 +200,12 @@ def check_seed_contents(content, seed): raise MaasSeedDirMalformed("%s: missing files %s" % (seed, missing)) userdata = content['user-data'] - md = { } + md = {} for (key, val) in content.iteritems(): if key == 'user-data': continue md[key] = val - + return(userdata, md) @@ -260,4 +259,3 @@ if __name__ == "__main__": pprint.pprint(metadata) main() - diff --git a/cloudinit/util.py b/cloudinit/util.py index 897f0a0d..35d194ba 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -752,6 +752,7 @@ def mount_callback_umount(device, callback, data=None): return(ret) + def wait_for_url(urls, max_wait=None, timeout=None, status_cb=None, headers_cb=None): """ @@ -826,10 +827,9 @@ def wait_for_url(urls, max_wait=None, timeout=None, except Exception as e: reason = "unexpected error [%s]" % e - if log: - status_cb("'%s' failed [%s/%ss]: %s" % - (url, int(time.time() - starttime), max_wait, - reason)) + status_cb("'%s' failed [%s/%ss]: %s" % + (url, int(time.time() - starttime), max_wait, + reason)) if timeup(max_wait, starttime): break @@ -838,5 +838,3 @@ def wait_for_url(urls, max_wait=None, timeout=None, time.sleep(sleeptime) return False - - diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index dc8964d3..4f896add 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -39,7 +39,7 @@ class TestMaasDataSource(TestCase): # verify that 'userdata' is not returned as part of the metadata self.assertFalse(('user-data' in metadata)) - + def test_seed_dir_valid_extra(self): """Verify extra files do not affect seed_dir validity """ -- cgit v1.2.3 From 8e705c8542719cf4d90083c9432497ff525247ce Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 7 Mar 2012 09:54:41 -0500 Subject: DataSourceMaaS: add test code for the oauth path This adds to the 'main' in cloudinit/DataSourceMaaS.py a method for testing oauth_headers. --- cloudinit/DataSourceMaaS.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index 08a48443..4c89c2eb 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -245,11 +245,27 @@ def get_datasource_list(depends): if __name__ == "__main__": def main(): + """ + Call with single argument of directory or http or https url. + If url is given additional arguments are allowed, which will be + interpreted as consumer_key, token_key, token_secret, consumer_secret + """ import sys import pprint seed = sys.argv[1] if seed.startswith("http://") or seed.startswith("https://"): - (userdata, metadata) = read_maas_seed_url(seed) + def my_headers(url): + headers = oauth_headers(url, c_key, t_key, t_sec, c_sec) + print "%s\n %s\n" % (url, headers) + return headers + cb = None + (c_key, t_key, t_sec, c_sec) = (sys.argv + ["", "", "", ""])[2:6] + if c_key: + print "oauth headers (%s)" % str((c_key, t_key, t_sec, c_sec,)) + (userdata, metadata) = read_maas_seed_url(seed, my_headers) + else: + (userdata, metadata) = read_maas_seed_url(seed) + else: (userdata, metadata) = read_maas_seed_dir(seed) -- cgit v1.2.3 From 78d33bcbabba8a38d659f54f92d703d3cc562a1f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 7 Mar 2012 09:58:54 -0500 Subject: pylint/pep8 cleanup --- cloudinit/DataSourceMaaS.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index 4c89c2eb..c168013b 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -258,14 +258,13 @@ if __name__ == "__main__": headers = oauth_headers(url, c_key, t_key, t_sec, c_sec) print "%s\n %s\n" % (url, headers) return headers - cb = None (c_key, t_key, t_sec, c_sec) = (sys.argv + ["", "", "", ""])[2:6] if c_key: print "oauth headers (%s)" % str((c_key, t_key, t_sec, c_sec,)) (userdata, metadata) = read_maas_seed_url(seed, my_headers) else: (userdata, metadata) = read_maas_seed_url(seed) - + else: (userdata, metadata) = read_maas_seed_dir(seed) -- cgit v1.2.3 From a352455707e192a4ea537578f3e11f221d378800 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Mar 2012 12:02:08 -0500 Subject: better 'main()', add and use version in api, do not require user-data main now is more useful for debugging. now it does: * get: just dump contents of a url provided after oauth * crawl: walk through using indexes, dumping content * check-seed: validate the seed is good uses MD_VERSION in the url, and appends that to the metadata url in the config file. (previously it assumed the url in the config was the full url) does not require user-data in the http seed. if the user did not specify user-data, it wont be there, so do not fail on that case. --- cloudinit/DataSourceMaaS.py | 135 +++++++++++++++++++++++++++++++++----------- 1 file changed, 102 insertions(+), 33 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index c168013b..c320cb34 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -30,6 +30,9 @@ import urllib2 import time +MD_VERSION = "2012-03-01" + + class DataSourceMaaS(DataSource.DataSource): """ DataSourceMaaS reads instance information from MaaS. @@ -133,10 +136,10 @@ def read_maas_seed_dir(seed_d): Return user-data and metadata for a maas seed dir in seed_d. Expected format of seed_d are the following files: * instance-id - * hostname + * local-hostname * user-data """ - files = ('hostname', 'instance-id', 'user-data') + files = ('local-hostname', 'instance-id', 'user-data') md = {} if not os.path.isdir(seed_d): @@ -154,31 +157,38 @@ def read_maas_seed_dir(seed_d): return(check_seed_contents(md, seed_d)) -def read_maas_seed_url(seed_url, header_cb=None, timeout=None): +def read_maas_seed_url(seed_url, header_cb=None, timeout=None, + version=MD_VERSION): """ Read the maas datasource at seed_url. header_cb is a method that should return a headers dictionary that will be given to urllib2.Request() Expected format of seed_url is are the following files: - * /instance-id - * /hostname - * /user-data + * //instance-id + * //local-hostname + * //user-data """ - files = ('hostname', 'instance-id', 'user-data') + files = ('meta-data/local-hostname', 'meta-data/instance-id', 'user-data') + base_url = "%s/%s" % (seed_url, version) + print "seed_url=%s version=%s" % (seed_url, version) md = {} for fname in files: - url = "%s/%s" % (seed_url, fname) + url = "%s/%s" % (base_url, fname) if header_cb: headers = header_cb(url) else: headers = {} - req = urllib2.Request(url, data=None, headers=headers) - resp = urllib2.urlopen(req, timeout=timeout) - - md[fname] = resp.read() + try: + print url + req = urllib2.Request(url, data=None, headers=headers) + resp = urllib2.urlopen(req, timeout=timeout) + md[os.path.basename(fname)] = resp.read() + except urllib2.HTTPError as e: + if e.code != 404: + raise return(check_seed_contents(md, seed_url)) @@ -189,7 +199,7 @@ def check_seed_contents(content, seed): Either return a (userdata, metadata) tuple or Raise MaasSeedDirMalformed or MaasSeedDirNone """ - md_required = ('user-data', 'instance-id', 'hostname') + md_required = ('instance-id', 'local-hostname') found = content.keys() if len(content) == 0: @@ -199,7 +209,7 @@ def check_seed_contents(content, seed): if len(missing): raise MaasSeedDirMalformed("%s: missing files %s" % (seed, missing)) - userdata = content['user-data'] + userdata = content.get('user-data', "") md = {} for (key, val) in content.iteritems(): if key == 'user-data': @@ -250,27 +260,86 @@ if __name__ == "__main__": If url is given additional arguments are allowed, which will be interpreted as consumer_key, token_key, token_secret, consumer_secret """ - import sys + import argparse import pprint - seed = sys.argv[1] - if seed.startswith("http://") or seed.startswith("https://"): - def my_headers(url): - headers = oauth_headers(url, c_key, t_key, t_sec, c_sec) - print "%s\n %s\n" % (url, headers) - return headers - (c_key, t_key, t_sec, c_sec) = (sys.argv + ["", "", "", ""])[2:6] - if c_key: - print "oauth headers (%s)" % str((c_key, t_key, t_sec, c_sec,)) - (userdata, metadata) = read_maas_seed_url(seed, my_headers) - else: - (userdata, metadata) = read_maas_seed_url(seed) - else: - (userdata, metadata) = read_maas_seed_dir(seed) + parser = argparse.ArgumentParser(description='Interact with Maas DS') + parser.add_argument("--config", metavar="file", + help="specify DS config file", default=None) + parser.add_argument("--ckey", metavar="key", + help="the consumer key to auth with", default=None) + parser.add_argument("--tkey", metavar="key", + help="the token key to auth with", default=None) + parser.add_argument("--csec", metavar="secret", + help="the consumer secret (likely '')", default="") + parser.add_argument("--tsec", metavar="secret", + help="the token secret to auth with", default=None) + parser.add_argument("--apiver", metavar="version", + help="the apiver to use ("" can be used)", default=MD_VERSION) + + subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") + subcmds.add_parser('crawl', help="crawl the datasource") + subcmds.add_parser('get', help="do a single GET of provided url") + subcmds.add_parser('check-seed', help="read andn verify seed at url") + + parser.add_argument("url", help="the data source to query") + + args = parser.parse_args() + + creds = {'consumer_key': args.ckey, 'token_key': args.tkey, + 'token_secret': args.tsec, 'consumer_secret': args.csec} + + if args.config: + import yaml + with open(args.config) as fp: + cfg = yaml.load(fp) + print cfg + if 'datasource' in cfg: + cfg = cfg['datasource']['MaaS'] + for key in creds.keys(): + if key in cfg and creds[key] == None: + creds[key] = cfg[key] + + def geturl(url, headers_cb): + req = urllib2.Request(url, data=None, headers=headers_cb(url)) + return(urllib2.urlopen(req).read()) + + def printurl(url, headers_cb): + print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) + + def crawl(url, headers_cb=None): + if url.endswith("/"): + for line in geturl(url, headers_cb).splitlines(): + if line.endswith("/"): + crawl("%s%s" % (url, line), headers_cb) + else: + printurl("%s%s" % (url, line), headers_cb) + else: + printurl(url, headers_cb) - print "=== userdata ===" - print userdata - print "=== metadata ===" - pprint.pprint(metadata) + def my_headers(url): + headers = {} + if creds.get('consumer_key', None) != None: + headers = oauth_headers(url, **creds) + return headers + + if args.subcmd == "check-seed": + if args.url.startswith("http"): + (userdata, metadata) = read_maas_seed_url(args.url, + header_cb=my_headers, version=args.apiver) + else: + (userdata, metadata) = read_maas_seed_url(args.url) + print "=== userdata ===" + print userdata + print "=== metadata ===" + pprint.pprint(metadata) + + elif args.subcmd == "get": + printurl(args.url, my_headers) + + elif args.subcmd == "crawl": + if not args.url.endswith("/"): + args.url = "%s/" % args.url + crawl(args.url, my_headers) main() -- cgit v1.2.3 From b11da90d4a932516f66643346fc3b6492cee9478 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Mar 2012 12:04:51 -0500 Subject: remove debug statement --- cloudinit/DataSourceMaaS.py | 1 - 1 file changed, 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index c320cb34..29f78130 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -293,7 +293,6 @@ if __name__ == "__main__": import yaml with open(args.config) as fp: cfg = yaml.load(fp) - print cfg if 'datasource' in cfg: cfg = cfg['datasource']['MaaS'] for key in creds.keys(): -- cgit v1.2.3 From a0a5d2af8109dfc0cd0685b2d7a54e261a3b289e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Mar 2012 12:15:29 -0500 Subject: add MaaS datasource to default searched --- cloudinit/__init__.py | 2 +- config/cloud.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py index ccaa28c8..9f188766 100644 --- a/cloudinit/__init__.py +++ b/cloudinit/__init__.py @@ -29,7 +29,7 @@ cfg_env_name = "CLOUD_CFG" cfg_builtin = """ log_cfgs: [] -datasource_list: ["NoCloud", "ConfigDrive", "OVF", "Ec2"] +datasource_list: ["NoCloud", "ConfigDrive", "OVF", "MaaS", "Ec2" ] def_log_file: /var/log/cloud-init.log syslog_fix_perms: syslog:adm """ diff --git a/config/cloud.cfg b/config/cloud.cfg index fe197ca8..01e63ef8 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -1,7 +1,7 @@ user: ubuntu disable_root: 1 preserve_hostname: False -# datasource_list: [ "NoCloud", "ConfigDrive", "OVF", "Ec2" ] +# datasource_list: ["NoCloud", "ConfigDrive", "OVF", "MaaS", "Ec2" ] cloud_init_modules: - bootcmd -- cgit v1.2.3 From 0756ed017d02916fac240a07173a4bbb27eb74c0 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Mar 2012 13:20:10 -0500 Subject: remove debug statement --- cloudinit/DataSourceMaaS.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index 29f78130..6bad0dfe 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -172,7 +172,6 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, files = ('meta-data/local-hostname', 'meta-data/instance-id', 'user-data') base_url = "%s/%s" % (seed_url, version) - print "seed_url=%s version=%s" % (seed_url, version) md = {} for fname in files: url = "%s/%s" % (base_url, fname) @@ -182,7 +181,6 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, headers = {} try: - print url req = urllib2.Request(url, data=None, headers=headers) resp = urllib2.urlopen(req, timeout=timeout) md[os.path.basename(fname)] = resp.read() -- cgit v1.2.3