From 54b593a166c4ac4043615dddc94a941ebc712300 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 5 Mar 2012 16:35:35 -0500 Subject: use builtin runparts rather than system run-parts utility Because Fedora's run-parts does not accept '--regex' and debian's run-parts skips files with a '.' in the *without* '--regex=.*', we're forced to include our own version of run-parts. LP: #933553 --- cloudinit/util.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/util.py b/cloudinit/util.py index c37f0316..0032c6e0 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -209,16 +209,18 @@ def runparts(dirp, skip_no_exist=True): if skip_no_exist and not os.path.isdir(dirp): return - # per bug 857926, Fedora's run-parts will exit failure on empty dir - if os.path.isdir(dirp) and os.listdir(dirp) == []: - return - - cmd = ['run-parts', '--regex', '.*', dirp] - sp = subprocess.Popen(cmd) - sp.communicate() - if sp.returncode is not 0: - raise subprocess.CalledProcessError(sp.returncode, cmd) - return + failed = 0 + for exe_name in sorted(os.listdir(dirp)): + exe_path = os.path.join(dirp, exe_name) + if os.path.isfile(exe_path) and os.access(exe_path, os.X_OK): + popen = subprocess.Popen([exe_path]) + popen.communicate() + if popen.returncode is not 0: + failed += 1 + sys.stderr.write("failed: %s [%i]\n" % + (exe_path, popen.returncode)) + if failed: + raise RuntimeError('runparts: %i failures' % failed) def subp(args, input_=None): -- cgit v1.2.3 From 2303079f3ad6c98cee8f27a0488057297cb049fb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 12:53:40 -0500 Subject: move wait_for_metadata_service for util, rename to wait_for_url Also, add in the headers_cb which will be required for oauth. --- cloudinit/DataSourceEc2.py | 85 ++----------------------------------------- cloudinit/DataSourceMaaS.py | 88 +-------------------------------------------- cloudinit/util.py | 87 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 170 deletions(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py index 06635746..4e06803d 100644 --- a/cloudinit/DataSourceEc2.py +++ b/cloudinit/DataSourceEc2.py @@ -134,8 +134,8 @@ class DataSourceEc2(DataSource.DataSource): url2base[cur] = url starttime = time.time() - url = wait_for_metadata_service(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=log.warn) + url = util.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, status_cb=log.warn) if url: log.debug("Using metadata source: '%s'" % url2base[url]) @@ -208,87 +208,6 @@ class DataSourceEc2(DataSource.DataSource): return False -def wait_for_metadata_service(urls, max_wait=None, timeout=None, - status_cb=None): - """ - urls: a list of urls to try - max_wait: roughly the maximum time to wait before giving up - The max time is *actually* len(urls)*timeout as each url will - be tried once and given the timeout provided. - timeout: the timeout provided to urllib2.urlopen - status_cb: call method with string message when a url is not available - - the idea of this routine is to wait for the EC2 metdata service to - come up. On both Eucalyptus and EC2 we have seen the case where - the instance hit the MD before the MD service was up. EC2 seems - to have permenantely fixed this, though. - - In openstack, the metadata service might be painfully slow, and - unable to avoid hitting a timeout of even up to 10 seconds or more - (LP: #894279) for a simple GET. - - Offset those needs with the need to not hang forever (and block boot) - on a system where cloud-init is configured to look for EC2 Metadata - service but is not going to find one. It is possible that the instance - data host (169.254.169.254) may be firewalled off Entirely for a sytem, - meaning that the connection will block forever unless a timeout is set. - """ - starttime = time.time() - - sleeptime = 1 - - def nullstatus_cb(msg): - return - - if status_cb == None: - status_cb = nullstatus_cb - - def timeup(max_wait, starttime): - return((max_wait <= 0 or max_wait == None) or - (time.time() - starttime > max_wait)) - - loop_n = 0 - while True: - sleeptime = int(loop_n / 5) + 1 - for url in urls: - now = time.time() - if loop_n != 0: - if timeup(max_wait, starttime): - break - if timeout and (now + timeout > (starttime + max_wait)): - # shorten timeout to not run way over max_time - timeout = int((starttime + max_wait) - now) - - reason = "" - try: - req = urllib2.Request(url) - resp = urllib2.urlopen(req, timeout=timeout) - if resp.read() != "": - return url - reason = "empty data [%s]" % resp.getcode() - except urllib2.HTTPError as e: - reason = "http error [%s]" % e.code - except urllib2.URLError as e: - reason = "url error [%s]" % e.reason - except socket.timeout as e: - reason = "socket timeout [%s]" % e - except Exception as e: - reason = "unexpected error [%s]" % e - - if log: - status_cb("'%s' failed [%s/%ss]: %s" % - (url, int(time.time() - starttime), max_wait, - reason)) - - if timeup(max_wait, starttime): - break - - loop_n = loop_n + 1 - time.sleep(sleeptime) - - return False - - datasources = [ (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), ] diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index 2bc1f71f..d902ccb4 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -116,7 +116,7 @@ class DataSourceMaaS(DataSource.DataSource): starttime = time.time() check_url = "%s/instance-id" % url - url = wait_for_metadata_service(urls=[check_url], max_wait=max_wait, + url = util.wait_for_url(urls=[check_url], max_wait=max_wait, timeout=timeout, status_cb=log.warn, headers_cb=self.md_headers) @@ -129,92 +129,6 @@ class DataSourceMaaS(DataSource.DataSource): return (bool(url)) -def wait_for_metadata_service(urls, max_wait=None, timeout=None, - status_cb=None, headers_cb=None): - """ - urls: a list of urls to try - max_wait: roughly the maximum time to wait before giving up - The max time is *actually* len(urls)*timeout as each url will - be tried once and given the timeout provided. - timeout: the timeout provided to urllib2.urlopen - status_cb: call method with string message when a url is not available - - the idea of this routine is to wait for the EC2 metdata service to - come up. On both Eucalyptus and EC2 we have seen the case where - the instance hit the MD before the MD service was up. EC2 seems - to have permenantely fixed this, though. - - In openstack, the metadata service might be painfully slow, and - unable to avoid hitting a timeout of even up to 10 seconds or more - (LP: #894279) for a simple GET. - - Offset those needs with the need to not hang forever (and block boot) - on a system where cloud-init is configured to look for EC2 Metadata - service but is not going to find one. It is possible that the instance - data host (169.254.169.254) may be firewalled off Entirely for a sytem, - meaning that the connection will block forever unless a timeout is set. - """ - starttime = time.time() - - sleeptime = 1 - - def nullstatus_cb(msg): - return - - if status_cb == None: - status_cb = nullstatus_cb - - def timeup(max_wait, starttime): - return((max_wait <= 0 or max_wait == None) or - (time.time() - starttime > max_wait)) - - loop_n = 0 - while True: - sleeptime = int(loop_n / 5) + 1 - for url in urls: - now = time.time() - if loop_n != 0: - if timeup(max_wait, starttime): - break - if timeout and (now + timeout > (starttime + max_wait)): - # shorten timeout to not run way over max_time - timeout = int((starttime + max_wait) - now) - - reason = "" - try: - if headers_cb != None: - headers = headers_cb(url) - else: - headers = {} - - req = urllib2.Request(url, data=None, headers=headers) - resp = urllib2.urlopen(req, timeout=timeout) - if resp.read() != "": - return url - reason = "empty data [%s]" % resp.getcode() - except urllib2.HTTPError as e: - reason = "http error [%s]" % e.code - except urllib2.URLError as e: - reason = "url error [%s]" % e.reason - except socket.timeout as e: - reason = "socket timeout [%s]" % e - except Exception as e: - reason = "unexpected error [%s]" % e - - if log: - status_cb("'%s' failed [%s/%ss]: %s" % - (url, int(time.time() - starttime), max_wait, - reason)) - - if timeup(max_wait, starttime): - break - - loop_n = loop_n + 1 - time.sleep(sleeptime) - - return False - - def read_maas_seed_dir(seed_d): """ Return user-data and metadata for a maas seed dir in seed_d. diff --git a/cloudinit/util.py b/cloudinit/util.py index c37f0316..882fd9fb 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -751,3 +751,90 @@ def mount_callback_umount(device, callback, data=None): _cleanup(umount, tmpd) return(ret) + +def wait_for_url(urls, max_wait=None, timeout=None, + status_cb=None, headers_cb=None): + """ + urls: a list of urls to try + max_wait: roughly the maximum time to wait before giving up + The max time is *actually* len(urls)*timeout as each url will + be tried once and given the timeout provided. + timeout: the timeout provided to urllib2.urlopen + status_cb: call method with string message when a url is not available + + the idea of this routine is to wait for the EC2 metdata service to + come up. On both Eucalyptus and EC2 we have seen the case where + the instance hit the MD before the MD service was up. EC2 seems + to have permenantely fixed this, though. + + In openstack, the metadata service might be painfully slow, and + unable to avoid hitting a timeout of even up to 10 seconds or more + (LP: #894279) for a simple GET. + + Offset those needs with the need to not hang forever (and block boot) + on a system where cloud-init is configured to look for EC2 Metadata + service but is not going to find one. It is possible that the instance + data host (169.254.169.254) may be firewalled off Entirely for a sytem, + meaning that the connection will block forever unless a timeout is set. + """ + starttime = time.time() + + sleeptime = 1 + + def nullstatus_cb(msg): + return + + if status_cb == None: + status_cb = nullstatus_cb + + def timeup(max_wait, starttime): + return((max_wait <= 0 or max_wait == None) or + (time.time() - starttime > max_wait)) + + loop_n = 0 + while True: + sleeptime = int(loop_n / 5) + 1 + for url in urls: + now = time.time() + if loop_n != 0: + if timeup(max_wait, starttime): + break + if timeout and (now + timeout > (starttime + max_wait)): + # shorten timeout to not run way over max_time + timeout = int((starttime + max_wait) - now) + + reason = "" + try: + if headers_cb != None: + headers = headers_cb(url) + else: + headers = {} + + req = urllib2.Request(url, data=None, headers=headers) + resp = urllib2.urlopen(req, timeout=timeout) + if resp.read() != "": + return url + reason = "empty data [%s]" % resp.getcode() + except urllib2.HTTPError as e: + reason = "http error [%s]" % e.code + except urllib2.URLError as e: + reason = "url error [%s]" % e.reason + except socket.timeout as e: + reason = "socket timeout [%s]" % e + except Exception as e: + reason = "unexpected error [%s]" % e + + if log: + status_cb("'%s' failed [%s/%ss]: %s" % + (url, int(time.time() - starttime), max_wait, + reason)) + + if timeup(max_wait, starttime): + break + + loop_n = loop_n + 1 + time.sleep(sleeptime) + + return False + + -- cgit v1.2.3 From bb072c5661ffcc8b1d69f9d8d15e8ff5839f5305 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 17:06:09 -0500 Subject: add headers_cb to doc for wait_for_url --- cloudinit/util.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'cloudinit/util.py') diff --git a/cloudinit/util.py b/cloudinit/util.py index 882fd9fb..897f0a0d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -761,6 +761,8 @@ def wait_for_url(urls, max_wait=None, timeout=None, be tried once and given the timeout provided. timeout: the timeout provided to urllib2.urlopen status_cb: call method with string message when a url is not available + headers_cb: call method with single argument of url to get headers + for request. the idea of this routine is to wait for the EC2 metdata service to come up. On both Eucalyptus and EC2 we have seen the case where -- cgit v1.2.3 From 3703ed74cea613e96f3d882e90af1cadae30a092 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 7 Mar 2012 09:54:04 -0500 Subject: fix pylint and pep8 warnings --- cloudinit/DataSourceEc2.py | 1 - cloudinit/DataSourceMaaS.py | 8 +++----- cloudinit/util.py | 10 ++++------ tests/unittests/test_datasource/test_maas.py | 2 +- 4 files changed, 8 insertions(+), 13 deletions(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py index 4e06803d..7051ecda 100644 --- a/cloudinit/DataSourceEc2.py +++ b/cloudinit/DataSourceEc2.py @@ -24,7 +24,6 @@ from cloudinit import seeddir as base_seeddir from cloudinit import log import cloudinit.util as util import socket -import urllib2 import time import boto.utils as boto_utils import os.path diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index fc4e890d..08a48443 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -26,7 +26,6 @@ import cloudinit.util as util import errno import oauth.oauth as oauth import os.path -import socket import urllib2 import time @@ -185,7 +184,7 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None): def check_seed_contents(content, seed): - """Validate if content is Is the content a dict that is valid as a + """Validate if content is Is the content a dict that is valid as a return for a datasource. Either return a (userdata, metadata) tuple or Raise MaasSeedDirMalformed or MaasSeedDirNone @@ -201,12 +200,12 @@ def check_seed_contents(content, seed): raise MaasSeedDirMalformed("%s: missing files %s" % (seed, missing)) userdata = content['user-data'] - md = { } + md = {} for (key, val) in content.iteritems(): if key == 'user-data': continue md[key] = val - + return(userdata, md) @@ -260,4 +259,3 @@ if __name__ == "__main__": pprint.pprint(metadata) main() - diff --git a/cloudinit/util.py b/cloudinit/util.py index 897f0a0d..35d194ba 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -752,6 +752,7 @@ def mount_callback_umount(device, callback, data=None): return(ret) + def wait_for_url(urls, max_wait=None, timeout=None, status_cb=None, headers_cb=None): """ @@ -826,10 +827,9 @@ def wait_for_url(urls, max_wait=None, timeout=None, except Exception as e: reason = "unexpected error [%s]" % e - if log: - status_cb("'%s' failed [%s/%ss]: %s" % - (url, int(time.time() - starttime), max_wait, - reason)) + status_cb("'%s' failed [%s/%ss]: %s" % + (url, int(time.time() - starttime), max_wait, + reason)) if timeup(max_wait, starttime): break @@ -838,5 +838,3 @@ def wait_for_url(urls, max_wait=None, timeout=None, time.sleep(sleeptime) return False - - diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index dc8964d3..4f896add 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -39,7 +39,7 @@ class TestMaasDataSource(TestCase): # verify that 'userdata' is not returned as part of the metadata self.assertFalse(('user-data' in metadata)) - + def test_seed_dir_valid_extra(self): """Verify extra files do not affect seed_dir validity """ -- cgit v1.2.3