From 2303079f3ad6c98cee8f27a0488057297cb049fb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 12:53:40 -0500 Subject: move wait_for_metadata_service for util, rename to wait_for_url Also, add in the headers_cb which will be required for oauth. --- cloudinit/DataSourceEc2.py | 85 ++-------------------------------------------- 1 file changed, 2 insertions(+), 83 deletions(-) (limited to 'cloudinit/DataSourceEc2.py') diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py index 06635746..4e06803d 100644 --- a/cloudinit/DataSourceEc2.py +++ b/cloudinit/DataSourceEc2.py @@ -134,8 +134,8 @@ class DataSourceEc2(DataSource.DataSource): url2base[cur] = url starttime = time.time() - url = wait_for_metadata_service(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=log.warn) + url = util.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, status_cb=log.warn) if url: log.debug("Using metadata source: '%s'" % url2base[url]) @@ -208,87 +208,6 @@ class DataSourceEc2(DataSource.DataSource): return False -def wait_for_metadata_service(urls, max_wait=None, timeout=None, - status_cb=None): - """ - urls: a list of urls to try - max_wait: roughly the maximum time to wait before giving up - The max time is *actually* len(urls)*timeout as each url will - be tried once and given the timeout provided. - timeout: the timeout provided to urllib2.urlopen - status_cb: call method with string message when a url is not available - - the idea of this routine is to wait for the EC2 metdata service to - come up. On both Eucalyptus and EC2 we have seen the case where - the instance hit the MD before the MD service was up. EC2 seems - to have permenantely fixed this, though. - - In openstack, the metadata service might be painfully slow, and - unable to avoid hitting a timeout of even up to 10 seconds or more - (LP: #894279) for a simple GET. - - Offset those needs with the need to not hang forever (and block boot) - on a system where cloud-init is configured to look for EC2 Metadata - service but is not going to find one. It is possible that the instance - data host (169.254.169.254) may be firewalled off Entirely for a sytem, - meaning that the connection will block forever unless a timeout is set. - """ - starttime = time.time() - - sleeptime = 1 - - def nullstatus_cb(msg): - return - - if status_cb == None: - status_cb = nullstatus_cb - - def timeup(max_wait, starttime): - return((max_wait <= 0 or max_wait == None) or - (time.time() - starttime > max_wait)) - - loop_n = 0 - while True: - sleeptime = int(loop_n / 5) + 1 - for url in urls: - now = time.time() - if loop_n != 0: - if timeup(max_wait, starttime): - break - if timeout and (now + timeout > (starttime + max_wait)): - # shorten timeout to not run way over max_time - timeout = int((starttime + max_wait) - now) - - reason = "" - try: - req = urllib2.Request(url) - resp = urllib2.urlopen(req, timeout=timeout) - if resp.read() != "": - return url - reason = "empty data [%s]" % resp.getcode() - except urllib2.HTTPError as e: - reason = "http error [%s]" % e.code - except urllib2.URLError as e: - reason = "url error [%s]" % e.reason - except socket.timeout as e: - reason = "socket timeout [%s]" % e - except Exception as e: - reason = "unexpected error [%s]" % e - - if log: - status_cb("'%s' failed [%s/%ss]: %s" % - (url, int(time.time() - starttime), max_wait, - reason)) - - if timeup(max_wait, starttime): - break - - loop_n = loop_n + 1 - time.sleep(sleeptime) - - return False - - datasources = [ (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), ] -- cgit v1.2.3 From 3703ed74cea613e96f3d882e90af1cadae30a092 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 7 Mar 2012 09:54:04 -0500 Subject: fix pylint and pep8 warnings --- cloudinit/DataSourceEc2.py | 1 - cloudinit/DataSourceMaaS.py | 8 +++----- cloudinit/util.py | 10 ++++------ tests/unittests/test_datasource/test_maas.py | 2 +- 4 files changed, 8 insertions(+), 13 deletions(-) (limited to 'cloudinit/DataSourceEc2.py') diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py index 4e06803d..7051ecda 100644 --- a/cloudinit/DataSourceEc2.py +++ b/cloudinit/DataSourceEc2.py @@ -24,7 +24,6 @@ from cloudinit import seeddir as base_seeddir from cloudinit import log import cloudinit.util as util import socket -import urllib2 import time import boto.utils as boto_utils import os.path diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index fc4e890d..08a48443 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -26,7 +26,6 @@ import cloudinit.util as util import errno import oauth.oauth as oauth import os.path -import socket import urllib2 import time @@ -185,7 +184,7 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None): def check_seed_contents(content, seed): - """Validate if content is Is the content a dict that is valid as a + """Validate if content is Is the content a dict that is valid as a return for a datasource. Either return a (userdata, metadata) tuple or Raise MaasSeedDirMalformed or MaasSeedDirNone @@ -201,12 +200,12 @@ def check_seed_contents(content, seed): raise MaasSeedDirMalformed("%s: missing files %s" % (seed, missing)) userdata = content['user-data'] - md = { } + md = {} for (key, val) in content.iteritems(): if key == 'user-data': continue md[key] = val - + return(userdata, md) @@ -260,4 +259,3 @@ if __name__ == "__main__": pprint.pprint(metadata) main() - diff --git a/cloudinit/util.py b/cloudinit/util.py index 897f0a0d..35d194ba 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -752,6 +752,7 @@ def mount_callback_umount(device, callback, data=None): return(ret) + def wait_for_url(urls, max_wait=None, timeout=None, status_cb=None, headers_cb=None): """ @@ -826,10 +827,9 @@ def wait_for_url(urls, max_wait=None, timeout=None, except Exception as e: reason = "unexpected error [%s]" % e - if log: - status_cb("'%s' failed [%s/%ss]: %s" % - (url, int(time.time() - starttime), max_wait, - reason)) + status_cb("'%s' failed [%s/%ss]: %s" % + (url, int(time.time() - starttime), max_wait, + reason)) if timeup(max_wait, starttime): break @@ -838,5 +838,3 @@ def wait_for_url(urls, max_wait=None, timeout=None, time.sleep(sleeptime) return False - - diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index dc8964d3..4f896add 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -39,7 +39,7 @@ class TestMaasDataSource(TestCase): # verify that 'userdata' is not returned as part of the metadata self.assertFalse(('user-data' in metadata)) - + def test_seed_dir_valid_extra(self): """Verify extra files do not affect seed_dir validity """ -- cgit v1.2.3