From 9f719a8c427f639e1f0ea6725073be3081dd008e Mon Sep 17 00:00:00 2001 From: Mike Milner Date: Fri, 24 Feb 2012 15:16:56 -0400 Subject: If we don't trust the default certs, don't add new certs from ca-certificates package upgrades. --- cloudinit/CloudConfig/cc_ca_certs.py | 3 +++ tests/unittests/test_handler_ca_certs.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/cloudinit/CloudConfig/cc_ca_certs.py b/cloudinit/CloudConfig/cc_ca_certs.py index c18821f9..c7bacb78 100644 --- a/cloudinit/CloudConfig/cc_ca_certs.py +++ b/cloudinit/CloudConfig/cc_ca_certs.py @@ -54,6 +54,9 @@ def remove_default_ca_certs(): delete_dir_contents(CA_CERT_PATH) delete_dir_contents(CA_CERT_SYSTEM_PATH) write_file(CA_CERT_CONFIG, "", mode=0644) + check_call([ + "echo 'ca-certificates ca-certificates/trust_new_crts select no' | " + "debconf-set-selections"], shell=True) def handle(_name, cfg, _cloud, log, _args): diff --git a/tests/unittests/test_handler_ca_certs.py b/tests/unittests/test_handler_ca_certs.py index d6513b5b..37bd7a08 100644 --- a/tests/unittests/test_handler_ca_certs.py +++ b/tests/unittests/test_handler_ca_certs.py @@ -169,10 +169,15 @@ class TestRemoveDefaultCaCerts(MockerTestCase): mock_delete_dir_contents = self.mocker.replace(delete_dir_contents, passthrough=False) mock_write = self.mocker.replace(write_file, passthrough=False) + mock_check_call = self.mocker.replace("subprocess.check_call", + passthrough=False) mock_delete_dir_contents("/usr/share/ca-certificates/") mock_delete_dir_contents("/etc/ssl/certs/") mock_write("/etc/ca-certificates.conf", "", mode=0644) + mock_check_call([ + "echo 'ca-certificates ca-certificates/trust_new_crts select no'" + " | debconf-set-selections"], shell=True) self.mocker.replay() remove_default_ca_certs() -- cgit v1.2.3 From 5e17bf1066b53359681f5164c662f779b268561b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 5 Mar 2012 16:08:15 -0500 Subject: commit initial config information for maas datasource --- doc/examples/cloud-config-datasources.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index b86c5ba6..f147aaf6 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -13,3 +13,18 @@ datasource: metadata_urls: - http://169.254.169.254:80 - http://instance-data:8773 + + MaaS: + timeout : 50 + max_wait : 120 + + # there are no default values for metadata_url or credentials + # credentials are for oath. + + # If no credentials are present, + # then non-authed attempts will be made. + # will be made + metadata_url: http://mass-host.localdomain/source + consumer_key: Xh234sdkljf + token_key: kjfhgb3n + token_secret: 24uysdfx1w4 -- cgit v1.2.3 From 54b593a166c4ac4043615dddc94a941ebc712300 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 5 Mar 2012 16:35:35 -0500 Subject: use builtin runparts rather than system run-parts utility Because Fedora's run-parts does not accept '--regex' and debian's run-parts skips files with a '.' in the *without* '--regex=.*', we're forced to include our own version of run-parts. LP: #933553 --- cloudinit/util.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index c37f0316..0032c6e0 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -209,16 +209,18 @@ def runparts(dirp, skip_no_exist=True): if skip_no_exist and not os.path.isdir(dirp): return - # per bug 857926, Fedora's run-parts will exit failure on empty dir - if os.path.isdir(dirp) and os.listdir(dirp) == []: - return - - cmd = ['run-parts', '--regex', '.*', dirp] - sp = subprocess.Popen(cmd) - sp.communicate() - if sp.returncode is not 0: - raise subprocess.CalledProcessError(sp.returncode, cmd) - return + failed = 0 + for exe_name in sorted(os.listdir(dirp)): + exe_path = os.path.join(dirp, exe_name) + if os.path.isfile(exe_path) and os.access(exe_path, os.X_OK): + popen = subprocess.Popen([exe_path]) + popen.communicate() + if popen.returncode is not 0: + failed += 1 + sys.stderr.write("failed: %s [%i]\n" % + (exe_path, popen.returncode)) + if failed: + raise RuntimeError('runparts: %i failures' % failed) def subp(args, input_=None): -- cgit v1.2.3 From 9fadb0784540ffd054c07a0edbe9b130d0efc902 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 11:58:19 -0500 Subject: tests/unittests/test_util.py: fix pylint error --- tests/unittests/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index d8da8bc9..ca96bc60 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -28,7 +28,7 @@ class TestMergeDict(TestCase): def test_merge_does_not_override(self): """Test that candidate doesn't override source.""" source = {"key1": "value1", "key2": "value2"} - candidate = {"key2": "value2", "key2": "NEW VALUE"} + candidate = {"key1": "value2", "key2": "NEW VALUE"} result = mergedict(source, candidate) self.assertEqual(source, result) -- cgit v1.2.3 From d53984dbd4fd8662f9cfda86cbab3d0d7c656511 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 11:58:33 -0500 Subject: add tests to run-pylint files --- tools/run-pylint | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/run-pylint b/tools/run-pylint index e271c3d5..46748ffb 100755 --- a/tools/run-pylint +++ b/tools/run-pylint @@ -1,6 +1,8 @@ #!/bin/bash -def_files='cloud*.py cloudinit/*.py cloudinit/CloudConfig/*.py' +ci_files='cloud*.py cloudinit/*.py cloudinit/CloudConfig/*.py' +test_files=$(find tests -name "*.py") +def_files="$ci_files $test_files" if [ $# -eq 0 ]; then files=( ) -- cgit v1.2.3 From f0a6ec70d13ea771efee86b2544346731cd79991 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 11:58:52 -0500 Subject: Add initial DataSourceMaaS. Tests at this point seem to indicate that seed-dir would work. --- cloudinit/DataSourceMaaS.py | 308 +++++++++++++++++++++++++++ tests/unittests/test_datasource/test_maas.py | 112 ++++++++++ 2 files changed, 420 insertions(+) create mode 100644 cloudinit/DataSourceMaaS.py create mode 100644 tests/unittests/test_datasource/test_maas.py diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py new file mode 100644 index 00000000..2bc1f71f --- /dev/null +++ b/cloudinit/DataSourceMaaS.py @@ -0,0 +1,308 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import errno +import oauth.oauth as oauth +import os.path +import socket +import urllib2 +import time + + +class DataSourceMaaS(DataSource.DataSource): + """ + DataSourceMaaS reads instance information from MaaS. + Given a config metadata_url, and oauth tokens, it expects to find + files under the root named: + instance-id + user-data + hostname + """ + seeddir = base_seeddir + '/maas' + baseurl = None + + def __str__(self): + return("DataSourceMaaS[%s]" % self.baseurl) + + def get_data(self): + mcfg = self.ds_cfg + + try: + (userdata, metadata) = read_maas_seed_dir(self.seeddir) + self.userdata_raw = userdata + self.metadata = metadata + self.baseurl = self.seeddir + return True + except MaasSeedDirNone: + pass + except MaasSeedDirMalformed as exc: + log.warn("%s was malformed: %s\n" % (self.seeddir, exc)) + raise + + try: + # if there is no metadata_url, then we're not configured + url = mcfg.get('metadata_url', None) + if url == None: + return False + + if not self.wait_for_metadata_service(url): + return False + + self.baseurl = url + + (userdata, metadata) = read_maas_seed_url(self.baseurl, + self.md_headers) + return True + except Exception: + util.logexc(log) + return False + + def md_headers(self, url): + mcfg = self.ds_cfg + + # if we are missing token_key, token_secret or consumer_key + # then just do non-authed requests + for required in ('token_key', 'token_secret', 'consumer_key'): + if required not in mcfg: + return({}) + + consumer_secret = mcfg.get('consumer_secret', "") + + return(oauth_headers(url=url, consumer_key=mcfg['consumer_key'], + token_key=mcfg['token_key'], token_secret=mcfg['token_secret'], + consumer_secret=consumer_secret)) + + def wait_for_metadata_service(self, url): + mcfg = self.ds_cfg + + max_wait = 120 + try: + max_wait = int(mcfg.get("max_wait", max_wait)) + except Exception: + util.logexc(log) + log.warn("Failed to get max wait. using %s" % max_wait) + + if max_wait == 0: + return False + + timeout = 50 + try: + timeout = int(mcfg.get("timeout", timeout)) + except Exception: + util.logexc(log) + log.warn("Failed to get timeout, using %s" % timeout) + + starttime = time.time() + check_url = "%s/instance-id" % url + url = wait_for_metadata_service(urls=[check_url], max_wait=max_wait, + timeout=timeout, status_cb=log.warn, + headers_cb=self.md_headers) + + if url: + log.debug("Using metadata source: '%s'" % url) + else: + log.critical("giving up on md after %i seconds\n" % + int(time.time() - starttime)) + + return (bool(url)) + + +def wait_for_metadata_service(urls, max_wait=None, timeout=None, + status_cb=None, headers_cb=None): + """ + urls: a list of urls to try + max_wait: roughly the maximum time to wait before giving up + The max time is *actually* len(urls)*timeout as each url will + be tried once and given the timeout provided. + timeout: the timeout provided to urllib2.urlopen + status_cb: call method with string message when a url is not available + + the idea of this routine is to wait for the EC2 metdata service to + come up. On both Eucalyptus and EC2 we have seen the case where + the instance hit the MD before the MD service was up. EC2 seems + to have permenantely fixed this, though. + + In openstack, the metadata service might be painfully slow, and + unable to avoid hitting a timeout of even up to 10 seconds or more + (LP: #894279) for a simple GET. + + Offset those needs with the need to not hang forever (and block boot) + on a system where cloud-init is configured to look for EC2 Metadata + service but is not going to find one. It is possible that the instance + data host (169.254.169.254) may be firewalled off Entirely for a sytem, + meaning that the connection will block forever unless a timeout is set. + """ + starttime = time.time() + + sleeptime = 1 + + def nullstatus_cb(msg): + return + + if status_cb == None: + status_cb = nullstatus_cb + + def timeup(max_wait, starttime): + return((max_wait <= 0 or max_wait == None) or + (time.time() - starttime > max_wait)) + + loop_n = 0 + while True: + sleeptime = int(loop_n / 5) + 1 + for url in urls: + now = time.time() + if loop_n != 0: + if timeup(max_wait, starttime): + break + if timeout and (now + timeout > (starttime + max_wait)): + # shorten timeout to not run way over max_time + timeout = int((starttime + max_wait) - now) + + reason = "" + try: + if headers_cb != None: + headers = headers_cb(url) + else: + headers = {} + + req = urllib2.Request(url, data=None, headers=headers) + resp = urllib2.urlopen(req, timeout=timeout) + if resp.read() != "": + return url + reason = "empty data [%s]" % resp.getcode() + except urllib2.HTTPError as e: + reason = "http error [%s]" % e.code + except urllib2.URLError as e: + reason = "url error [%s]" % e.reason + except socket.timeout as e: + reason = "socket timeout [%s]" % e + except Exception as e: + reason = "unexpected error [%s]" % e + + if log: + status_cb("'%s' failed [%s/%ss]: %s" % + (url, int(time.time() - starttime), max_wait, + reason)) + + if timeup(max_wait, starttime): + break + + loop_n = loop_n + 1 + time.sleep(sleeptime) + + return False + + +def read_maas_seed_dir(seed_d): + """ + Return user-data and metadata for a maas seed dir in seed_d. + Expected format of seed_d are the following files: + * instance-id + * hostname + * user-data + """ + md_required = set(('hostname', 'instance-id')) + files = md_required.union(set(('userdata',))) + userdata = None + md = {} + + if not os.path.isdir(seed_d): + raise MaasSeedDirNone("%s: not a directory") + + for fname in files: + try: + with open(os.path.join(seed_d, fname)) as fp: + if fname == 'userdata': + userdata = fp.read() + else: + md[fname] = fp.read() + fp.close() + except IOError as e: + if e.errno != errno.ENOENT: + raise + + if userdata == None and len(md) == 0: + raise MaasSeedDirNone("%s: no data files found" % seed_d) + + if userdata == None: + raise MaasSeedDirMalformed("%s: missing userdata" % seed_d) + + missing = md_required - set(md.keys()) + if len(missing): + raise MaasSeedDirMalformed("%s: missing files %s" % + (seed_d, str(missing))) + + return(userdata, md) + + +def read_maas_seed_url(seed_url, header_cb=None): + """ + Read the maas datasource at seed_url. + header_cb is a method that should return a headers dictionary that will + be given to urllib2.Request() + + Expected format of seed_url is are the following files: + * /instance-id + * /hostname + * /user-data + """ + userdata = "" + metadata = {'instance-id': 'i-maas-url', 'hostname': 'maas-url-hostname'} + + return(userdata, metadata) + + +def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): + consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) + token = oauth.OAuthToken(token_key, token_secret) + params = { + 'oauth_version': "1.0", + 'oauth_nonce': oauth.generate_nonce(), + 'oauth_timestamp': int(time.time()), + 'oauth_token': token.key, + 'oauth_consumer_key': consumer.key, + } + req = oauth.OAuthRequest(http_url=url, parameters=params) + req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(), + consumer, token) + return(req.to_header()) + + +class MaasSeedDirNone(Exception): + pass + + +class MaasSeedDirMalformed(Exception): + pass + + +datasources = [ + (DataSourceMaaS, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +] + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py new file mode 100644 index 00000000..ad169b97 --- /dev/null +++ b/tests/unittests/test_datasource/test_maas.py @@ -0,0 +1,112 @@ +from unittest import TestCase +from tempfile import mkdtemp +from shutil import rmtree +import os +from copy import copy +from cloudinit.DataSourceMaaS import ( + MaasSeedDirNone, + MaasSeedDirMalformed, + read_maas_seed_dir, +) + + +class TestMaasDataSource(TestCase): + + def setUp(self): + super(TestMaasDataSource, self).setUp() + # Make a temp directoy for tests to use. + self.tmp = mkdtemp(prefix="unittest_") + + def tearDown(self): + super(TestMaasDataSource, self).tearDown() + # Clean up temp directory + rmtree(self.tmp) + + def test_seed_dir_valid(self): + """Verify a valid seeddir is read as such""" + + data = {'instance-id': 'i-valid01', 'hostname': 'valid01-hostname', + 'userdata': 'valid01-userdata'} + + my_d = os.path.join(self.tmp, "valid") + populate_dir(my_d, data) + + (userdata, metadata) = read_maas_seed_dir(my_d) + + self.assertEqual(userdata, data['userdata']) + for key in ('instance-id', 'hostname'): + self.assertEqual(data[key], metadata[key]) + + def test_seed_dir_valid_extra(self): + """Verify extra files do not affect seed_dir validity """ + + data = {'instance-id': 'i-valid-extra', + 'hostname': 'valid-extra-hostname', + 'userdata': 'valid-extra-userdata', 'foo': 'bar'} + + my_d = os.path.join(self.tmp, "valid_extra") + populate_dir(my_d, data) + + (userdata, metadata) = read_maas_seed_dir(my_d) + + self.assertEqual(userdata, data['userdata']) + for key in ('instance-id', 'hostname'): + self.assertEqual(data[key], metadata[key]) + + # additional files should not just appear as keys in metadata atm + self.assertFalse(('foo' in metadata)) + + def test_seed_dir_invalid(self): + """Verify that invalid seed_dir raises MaasSeedDirMalformed""" + + valid = {'instance-id': 'i-instanceid', + 'hostname': 'test-hostname', 'userdata': ''} + + my_based = os.path.join(self.tmp, "valid_extra") + + # missing 'userdata' file + my_d = "%s-01" % my_based + invalid_data = copy(valid) + del invalid_data['userdata'] + populate_dir(my_d, invalid_data) + self.assertRaises(MaasSeedDirMalformed, read_maas_seed_dir, my_d) + + # missing 'instance-id' + my_d = "%s-02" % my_based + invalid_data = copy(valid) + del invalid_data['instance-id'] + populate_dir(my_d, invalid_data) + self.assertRaises(MaasSeedDirMalformed, read_maas_seed_dir, my_d) + + def test_seed_dir_none(self): + """Verify that empty seed_dir raises MaasSeedDirNone""" + + my_d = os.path.join(self.tmp, "valid_empty") + self.assertRaises(MaasSeedDirNone, read_maas_seed_dir, my_d) + + def test_seed_dir_missing(self): + """Verify that missing seed_dir raises MaasSeedDirNone""" + self.assertRaises(MaasSeedDirNone, read_maas_seed_dir, + os.path.join(self.tmp, "nonexistantdirectory")) + + def test_seed_url_valid(self): + """Verify that valid seed_url is read as such""" + pass + + def test_seed_url_invalid(self): + """Verify that invalid seed_url raises MaasSeedDirMalformed""" + pass + + def test_seed_url_missing(self): + """Verify seed_url with no found entries raises MaasSeedDirNone""" + pass + + +def populate_dir(seed_dir, files): + os.mkdir(seed_dir) + for (name, content) in files.iteritems(): + with open(os.path.join(seed_dir, name), "w") as fp: + fp.write(content) + fp.close() + +# vi: ts=4 expandtab -- cgit v1.2.3 From 2303079f3ad6c98cee8f27a0488057297cb049fb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 12:53:40 -0500 Subject: move wait_for_metadata_service for util, rename to wait_for_url Also, add in the headers_cb which will be required for oauth. --- cloudinit/DataSourceEc2.py | 85 ++----------------------------------------- cloudinit/DataSourceMaaS.py | 88 +-------------------------------------------- cloudinit/util.py | 87 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 170 deletions(-) diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py index 06635746..4e06803d 100644 --- a/cloudinit/DataSourceEc2.py +++ b/cloudinit/DataSourceEc2.py @@ -134,8 +134,8 @@ class DataSourceEc2(DataSource.DataSource): url2base[cur] = url starttime = time.time() - url = wait_for_metadata_service(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=log.warn) + url = util.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, status_cb=log.warn) if url: log.debug("Using metadata source: '%s'" % url2base[url]) @@ -208,87 +208,6 @@ class DataSourceEc2(DataSource.DataSource): return False -def wait_for_metadata_service(urls, max_wait=None, timeout=None, - status_cb=None): - """ - urls: a list of urls to try - max_wait: roughly the maximum time to wait before giving up - The max time is *actually* len(urls)*timeout as each url will - be tried once and given the timeout provided. - timeout: the timeout provided to urllib2.urlopen - status_cb: call method with string message when a url is not available - - the idea of this routine is to wait for the EC2 metdata service to - come up. On both Eucalyptus and EC2 we have seen the case where - the instance hit the MD before the MD service was up. EC2 seems - to have permenantely fixed this, though. - - In openstack, the metadata service might be painfully slow, and - unable to avoid hitting a timeout of even up to 10 seconds or more - (LP: #894279) for a simple GET. - - Offset those needs with the need to not hang forever (and block boot) - on a system where cloud-init is configured to look for EC2 Metadata - service but is not going to find one. It is possible that the instance - data host (169.254.169.254) may be firewalled off Entirely for a sytem, - meaning that the connection will block forever unless a timeout is set. - """ - starttime = time.time() - - sleeptime = 1 - - def nullstatus_cb(msg): - return - - if status_cb == None: - status_cb = nullstatus_cb - - def timeup(max_wait, starttime): - return((max_wait <= 0 or max_wait == None) or - (time.time() - starttime > max_wait)) - - loop_n = 0 - while True: - sleeptime = int(loop_n / 5) + 1 - for url in urls: - now = time.time() - if loop_n != 0: - if timeup(max_wait, starttime): - break - if timeout and (now + timeout > (starttime + max_wait)): - # shorten timeout to not run way over max_time - timeout = int((starttime + max_wait) - now) - - reason = "" - try: - req = urllib2.Request(url) - resp = urllib2.urlopen(req, timeout=timeout) - if resp.read() != "": - return url - reason = "empty data [%s]" % resp.getcode() - except urllib2.HTTPError as e: - reason = "http error [%s]" % e.code - except urllib2.URLError as e: - reason = "url error [%s]" % e.reason - except socket.timeout as e: - reason = "socket timeout [%s]" % e - except Exception as e: - reason = "unexpected error [%s]" % e - - if log: - status_cb("'%s' failed [%s/%ss]: %s" % - (url, int(time.time() - starttime), max_wait, - reason)) - - if timeup(max_wait, starttime): - break - - loop_n = loop_n + 1 - time.sleep(sleeptime) - - return False - - datasources = [ (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), ] diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index 2bc1f71f..d902ccb4 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -116,7 +116,7 @@ class DataSourceMaaS(DataSource.DataSource): starttime = time.time() check_url = "%s/instance-id" % url - url = wait_for_metadata_service(urls=[check_url], max_wait=max_wait, + url = util.wait_for_url(urls=[check_url], max_wait=max_wait, timeout=timeout, status_cb=log.warn, headers_cb=self.md_headers) @@ -129,92 +129,6 @@ class DataSourceMaaS(DataSource.DataSource): return (bool(url)) -def wait_for_metadata_service(urls, max_wait=None, timeout=None, - status_cb=None, headers_cb=None): - """ - urls: a list of urls to try - max_wait: roughly the maximum time to wait before giving up - The max time is *actually* len(urls)*timeout as each url will - be tried once and given the timeout provided. - timeout: the timeout provided to urllib2.urlopen - status_cb: call method with string message when a url is not available - - the idea of this routine is to wait for the EC2 metdata service to - come up. On both Eucalyptus and EC2 we have seen the case where - the instance hit the MD before the MD service was up. EC2 seems - to have permenantely fixed this, though. - - In openstack, the metadata service might be painfully slow, and - unable to avoid hitting a timeout of even up to 10 seconds or more - (LP: #894279) for a simple GET. - - Offset those needs with the need to not hang forever (and block boot) - on a system where cloud-init is configured to look for EC2 Metadata - service but is not going to find one. It is possible that the instance - data host (169.254.169.254) may be firewalled off Entirely for a sytem, - meaning that the connection will block forever unless a timeout is set. - """ - starttime = time.time() - - sleeptime = 1 - - def nullstatus_cb(msg): - return - - if status_cb == None: - status_cb = nullstatus_cb - - def timeup(max_wait, starttime): - return((max_wait <= 0 or max_wait == None) or - (time.time() - starttime > max_wait)) - - loop_n = 0 - while True: - sleeptime = int(loop_n / 5) + 1 - for url in urls: - now = time.time() - if loop_n != 0: - if timeup(max_wait, starttime): - break - if timeout and (now + timeout > (starttime + max_wait)): - # shorten timeout to not run way over max_time - timeout = int((starttime + max_wait) - now) - - reason = "" - try: - if headers_cb != None: - headers = headers_cb(url) - else: - headers = {} - - req = urllib2.Request(url, data=None, headers=headers) - resp = urllib2.urlopen(req, timeout=timeout) - if resp.read() != "": - return url - reason = "empty data [%s]" % resp.getcode() - except urllib2.HTTPError as e: - reason = "http error [%s]" % e.code - except urllib2.URLError as e: - reason = "url error [%s]" % e.reason - except socket.timeout as e: - reason = "socket timeout [%s]" % e - except Exception as e: - reason = "unexpected error [%s]" % e - - if log: - status_cb("'%s' failed [%s/%ss]: %s" % - (url, int(time.time() - starttime), max_wait, - reason)) - - if timeup(max_wait, starttime): - break - - loop_n = loop_n + 1 - time.sleep(sleeptime) - - return False - - def read_maas_seed_dir(seed_d): """ Return user-data and metadata for a maas seed dir in seed_d. diff --git a/cloudinit/util.py b/cloudinit/util.py index c37f0316..882fd9fb 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -751,3 +751,90 @@ def mount_callback_umount(device, callback, data=None): _cleanup(umount, tmpd) return(ret) + +def wait_for_url(urls, max_wait=None, timeout=None, + status_cb=None, headers_cb=None): + """ + urls: a list of urls to try + max_wait: roughly the maximum time to wait before giving up + The max time is *actually* len(urls)*timeout as each url will + be tried once and given the timeout provided. + timeout: the timeout provided to urllib2.urlopen + status_cb: call method with string message when a url is not available + + the idea of this routine is to wait for the EC2 metdata service to + come up. On both Eucalyptus and EC2 we have seen the case where + the instance hit the MD before the MD service was up. EC2 seems + to have permenantely fixed this, though. + + In openstack, the metadata service might be painfully slow, and + unable to avoid hitting a timeout of even up to 10 seconds or more + (LP: #894279) for a simple GET. + + Offset those needs with the need to not hang forever (and block boot) + on a system where cloud-init is configured to look for EC2 Metadata + service but is not going to find one. It is possible that the instance + data host (169.254.169.254) may be firewalled off Entirely for a sytem, + meaning that the connection will block forever unless a timeout is set. + """ + starttime = time.time() + + sleeptime = 1 + + def nullstatus_cb(msg): + return + + if status_cb == None: + status_cb = nullstatus_cb + + def timeup(max_wait, starttime): + return((max_wait <= 0 or max_wait == None) or + (time.time() - starttime > max_wait)) + + loop_n = 0 + while True: + sleeptime = int(loop_n / 5) + 1 + for url in urls: + now = time.time() + if loop_n != 0: + if timeup(max_wait, starttime): + break + if timeout and (now + timeout > (starttime + max_wait)): + # shorten timeout to not run way over max_time + timeout = int((starttime + max_wait) - now) + + reason = "" + try: + if headers_cb != None: + headers = headers_cb(url) + else: + headers = {} + + req = urllib2.Request(url, data=None, headers=headers) + resp = urllib2.urlopen(req, timeout=timeout) + if resp.read() != "": + return url + reason = "empty data [%s]" % resp.getcode() + except urllib2.HTTPError as e: + reason = "http error [%s]" % e.code + except urllib2.URLError as e: + reason = "url error [%s]" % e.reason + except socket.timeout as e: + reason = "socket timeout [%s]" % e + except Exception as e: + reason = "unexpected error [%s]" % e + + if log: + status_cb("'%s' failed [%s/%ss]: %s" % + (url, int(time.time() - starttime), max_wait, + reason)) + + if timeup(max_wait, starttime): + break + + loop_n = loop_n + 1 + time.sleep(sleeptime) + + return False + + -- cgit v1.2.3 From aacbeeb6dad8d987dfbc1a70f79214b72d85e67a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 17:05:43 -0500 Subject: assert that userdata is not returned as part of metadata --- tests/unittests/test_datasource/test_maas.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index ad169b97..22374415 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -36,6 +36,9 @@ class TestMaasDataSource(TestCase): self.assertEqual(userdata, data['userdata']) for key in ('instance-id', 'hostname'): self.assertEqual(data[key], metadata[key]) + + # verify that 'userdata' is not returned as part of the metadata + self.assertFalse(('userdata' in metadata)) def test_seed_dir_valid_extra(self): """Verify extra files do not affect seed_dir validity """ -- cgit v1.2.3 From bb072c5661ffcc8b1d69f9d8d15e8ff5839f5305 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 17:06:09 -0500 Subject: add headers_cb to doc for wait_for_url --- cloudinit/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudinit/util.py b/cloudinit/util.py index 882fd9fb..897f0a0d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -761,6 +761,8 @@ def wait_for_url(urls, max_wait=None, timeout=None, be tried once and given the timeout provided. timeout: the timeout provided to urllib2.urlopen status_cb: call method with string message when a url is not available + headers_cb: call method with single argument of url to get headers + for request. the idea of this routine is to wait for the EC2 metdata service to come up. On both Eucalyptus and EC2 we have seen the case where -- cgit v1.2.3 From 0e2e852bb448553479e68e91896419f3e06a6a19 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 17:06:28 -0500 Subject: functional read_maas_seed_url This commits a generally functional read_maas_seed_url, and re-works how the content is checked, to share between read_maas_seed{url,dir}. --- cloudinit/DataSourceMaaS.py | 90 +++++++++++++++++++++++++++++++++------------ 1 file changed, 67 insertions(+), 23 deletions(-) diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index d902ccb4..c25652b1 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -137,9 +137,7 @@ def read_maas_seed_dir(seed_d): * hostname * user-data """ - md_required = set(('hostname', 'instance-id')) - files = md_required.union(set(('userdata',))) - userdata = None + files = ('hostname', 'instance-id', 'userdata') md = {} if not os.path.isdir(seed_d): @@ -148,30 +146,16 @@ def read_maas_seed_dir(seed_d): for fname in files: try: with open(os.path.join(seed_d, fname)) as fp: - if fname == 'userdata': - userdata = fp.read() - else: - md[fname] = fp.read() + md[fname] = fp.read() fp.close() except IOError as e: if e.errno != errno.ENOENT: raise - if userdata == None and len(md) == 0: - raise MaasSeedDirNone("%s: no data files found" % seed_d) + return(check_seed_contents(md, seed_d)) - if userdata == None: - raise MaasSeedDirMalformed("%s: missing userdata" % seed_d) - missing = md_required - set(md.keys()) - if len(missing): - raise MaasSeedDirMalformed("%s: missing files %s" % - (seed_d, str(missing))) - - return(userdata, md) - - -def read_maas_seed_url(seed_url, header_cb=None): +def read_maas_seed_url(seed_url, header_cb=None, timeout=None): """ Read the maas datasource at seed_url. header_cb is a method that should return a headers dictionary that will @@ -182,10 +166,51 @@ def read_maas_seed_url(seed_url, header_cb=None): * /hostname * /user-data """ - userdata = "" - metadata = {'instance-id': 'i-maas-url', 'hostname': 'maas-url-hostname'} + files = ('hostname', 'instance-id', 'userdata') - return(userdata, metadata) + md = {} + for fname in files: + url = "%s/%s" % (seed_url, fname) + if header_cb: + headers = header_cb(url) + else: + headers = {} + + req = urllib2.Request(url, data=None, headers=headers) + resp = urllib2.urlopen(req, timeout=timeout) + + md[fname] = resp.read() + + return(check_seed_contents(md, seed_url)) + + +def check_seed_contents(content, seed): + """Validate if content is Is the content a dict that is valid as a + return for a datasource. + Either return a (userdata, metadata) tuple or + Raise MaasSeedDirMalformed or MaasSeedDirNone + """ + md_required = ('userdata', 'instance-id', 'hostname') + found = content.keys() + + if len(content) == 0: + raise MaasSeedDirNone("%s: no data files found" % seed) + + if 'userdata' not in content: + raise MaasSeedDirMalformed("%s: missing userdata" % seed) + + missing = [k for k in md_required if k not in found] + if len(missing): + raise MaasSeedDirMalformed("%s: missing files %s" % (seed, missing)) + + userdata = content['userdata'] + md = { } + for (key, val) in content.iteritems(): + if key == 'userdata': + continue + md[key] = val + + return(userdata, md) def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): @@ -220,3 +245,22 @@ datasources = [ # return a list of data sources that match this set of dependencies def get_datasource_list(depends): return(DataSource.list_from_depends(depends, datasources)) + + +if __name__ == "__main__": + def main(): + import sys + import pprint + seed = sys.argv[1] + if seed.startswith("http://") or seed.startswith("https://"): + (userdata, metadata) = read_maas_seed_url(seed) + else: + (userdata, metadata) = read_maas_seed_dir(seed) + + print "=== userdata ===" + print userdata + print "=== metadata ===" + pprint.pprint(metadata) + + main() + -- cgit v1.2.3 From 27ad683638a30e202a0def71485fade430976856 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 17:10:52 -0500 Subject: file for user-data should be 'user-data' (including the '-') --- cloudinit/DataSourceMaaS.py | 14 +++++++------- tests/unittests/test_datasource/test_maas.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index c25652b1..88686b13 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -137,7 +137,7 @@ def read_maas_seed_dir(seed_d): * hostname * user-data """ - files = ('hostname', 'instance-id', 'userdata') + files = ('hostname', 'instance-id', 'user-data') md = {} if not os.path.isdir(seed_d): @@ -166,7 +166,7 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None): * /hostname * /user-data """ - files = ('hostname', 'instance-id', 'userdata') + files = ('hostname', 'instance-id', 'user-data') md = {} for fname in files: @@ -190,23 +190,23 @@ def check_seed_contents(content, seed): Either return a (userdata, metadata) tuple or Raise MaasSeedDirMalformed or MaasSeedDirNone """ - md_required = ('userdata', 'instance-id', 'hostname') + md_required = ('user-data', 'instance-id', 'hostname') found = content.keys() if len(content) == 0: raise MaasSeedDirNone("%s: no data files found" % seed) - if 'userdata' not in content: - raise MaasSeedDirMalformed("%s: missing userdata" % seed) + if 'user-data' not in content: + raise MaasSeedDirMalformed("%s: missing user-data" % seed) missing = [k for k in md_required if k not in found] if len(missing): raise MaasSeedDirMalformed("%s: missing files %s" % (seed, missing)) - userdata = content['userdata'] + userdata = content['user-data'] md = { } for (key, val) in content.iteritems(): - if key == 'userdata': + if key == 'user-data': continue md[key] = val diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 22374415..dc8964d3 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -26,33 +26,33 @@ class TestMaasDataSource(TestCase): """Verify a valid seeddir is read as such""" data = {'instance-id': 'i-valid01', 'hostname': 'valid01-hostname', - 'userdata': 'valid01-userdata'} + 'user-data': 'valid01-userdata'} my_d = os.path.join(self.tmp, "valid") populate_dir(my_d, data) (userdata, metadata) = read_maas_seed_dir(my_d) - self.assertEqual(userdata, data['userdata']) + self.assertEqual(userdata, data['user-data']) for key in ('instance-id', 'hostname'): self.assertEqual(data[key], metadata[key]) # verify that 'userdata' is not returned as part of the metadata - self.assertFalse(('userdata' in metadata)) + self.assertFalse(('user-data' in metadata)) def test_seed_dir_valid_extra(self): """Verify extra files do not affect seed_dir validity """ data = {'instance-id': 'i-valid-extra', 'hostname': 'valid-extra-hostname', - 'userdata': 'valid-extra-userdata', 'foo': 'bar'} + 'user-data': 'valid-extra-userdata', 'foo': 'bar'} my_d = os.path.join(self.tmp, "valid_extra") populate_dir(my_d, data) (userdata, metadata) = read_maas_seed_dir(my_d) - self.assertEqual(userdata, data['userdata']) + self.assertEqual(userdata, data['user-data']) for key in ('instance-id', 'hostname'): self.assertEqual(data[key], metadata[key]) @@ -63,14 +63,14 @@ class TestMaasDataSource(TestCase): """Verify that invalid seed_dir raises MaasSeedDirMalformed""" valid = {'instance-id': 'i-instanceid', - 'hostname': 'test-hostname', 'userdata': ''} + 'hostname': 'test-hostname', 'user-data': ''} my_based = os.path.join(self.tmp, "valid_extra") # missing 'userdata' file my_d = "%s-01" % my_based invalid_data = copy(valid) - del invalid_data['userdata'] + del invalid_data['user-data'] populate_dir(my_d, invalid_data) self.assertRaises(MaasSeedDirMalformed, read_maas_seed_dir, my_d) -- cgit v1.2.3 From ea7ccdb7c118661b8e2487e5b6c7c00fda4e0c85 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 6 Mar 2012 17:11:53 -0500 Subject: no need to check explicitly for user-data --- cloudinit/DataSourceMaaS.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index 88686b13..fc4e890d 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -196,9 +196,6 @@ def check_seed_contents(content, seed): if len(content) == 0: raise MaasSeedDirNone("%s: no data files found" % seed) - if 'user-data' not in content: - raise MaasSeedDirMalformed("%s: missing user-data" % seed) - missing = [k for k in md_required if k not in found] if len(missing): raise MaasSeedDirMalformed("%s: missing files %s" % (seed, missing)) -- cgit v1.2.3 From 3703ed74cea613e96f3d882e90af1cadae30a092 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 7 Mar 2012 09:54:04 -0500 Subject: fix pylint and pep8 warnings --- cloudinit/DataSourceEc2.py | 1 - cloudinit/DataSourceMaaS.py | 8 +++----- cloudinit/util.py | 10 ++++------ tests/unittests/test_datasource/test_maas.py | 2 +- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py index 4e06803d..7051ecda 100644 --- a/cloudinit/DataSourceEc2.py +++ b/cloudinit/DataSourceEc2.py @@ -24,7 +24,6 @@ from cloudinit import seeddir as base_seeddir from cloudinit import log import cloudinit.util as util import socket -import urllib2 import time import boto.utils as boto_utils import os.path diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index fc4e890d..08a48443 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -26,7 +26,6 @@ import cloudinit.util as util import errno import oauth.oauth as oauth import os.path -import socket import urllib2 import time @@ -185,7 +184,7 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None): def check_seed_contents(content, seed): - """Validate if content is Is the content a dict that is valid as a + """Validate if content is Is the content a dict that is valid as a return for a datasource. Either return a (userdata, metadata) tuple or Raise MaasSeedDirMalformed or MaasSeedDirNone @@ -201,12 +200,12 @@ def check_seed_contents(content, seed): raise MaasSeedDirMalformed("%s: missing files %s" % (seed, missing)) userdata = content['user-data'] - md = { } + md = {} for (key, val) in content.iteritems(): if key == 'user-data': continue md[key] = val - + return(userdata, md) @@ -260,4 +259,3 @@ if __name__ == "__main__": pprint.pprint(metadata) main() - diff --git a/cloudinit/util.py b/cloudinit/util.py index 897f0a0d..35d194ba 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -752,6 +752,7 @@ def mount_callback_umount(device, callback, data=None): return(ret) + def wait_for_url(urls, max_wait=None, timeout=None, status_cb=None, headers_cb=None): """ @@ -826,10 +827,9 @@ def wait_for_url(urls, max_wait=None, timeout=None, except Exception as e: reason = "unexpected error [%s]" % e - if log: - status_cb("'%s' failed [%s/%ss]: %s" % - (url, int(time.time() - starttime), max_wait, - reason)) + status_cb("'%s' failed [%s/%ss]: %s" % + (url, int(time.time() - starttime), max_wait, + reason)) if timeup(max_wait, starttime): break @@ -838,5 +838,3 @@ def wait_for_url(urls, max_wait=None, timeout=None, time.sleep(sleeptime) return False - - diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index dc8964d3..4f896add 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -39,7 +39,7 @@ class TestMaasDataSource(TestCase): # verify that 'userdata' is not returned as part of the metadata self.assertFalse(('user-data' in metadata)) - + def test_seed_dir_valid_extra(self): """Verify extra files do not affect seed_dir validity """ -- cgit v1.2.3 From 8e705c8542719cf4d90083c9432497ff525247ce Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 7 Mar 2012 09:54:41 -0500 Subject: DataSourceMaaS: add test code for the oauth path This adds to the 'main' in cloudinit/DataSourceMaaS.py a method for testing oauth_headers. --- cloudinit/DataSourceMaaS.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index 08a48443..4c89c2eb 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -245,11 +245,27 @@ def get_datasource_list(depends): if __name__ == "__main__": def main(): + """ + Call with single argument of directory or http or https url. + If url is given additional arguments are allowed, which will be + interpreted as consumer_key, token_key, token_secret, consumer_secret + """ import sys import pprint seed = sys.argv[1] if seed.startswith("http://") or seed.startswith("https://"): - (userdata, metadata) = read_maas_seed_url(seed) + def my_headers(url): + headers = oauth_headers(url, c_key, t_key, t_sec, c_sec) + print "%s\n %s\n" % (url, headers) + return headers + cb = None + (c_key, t_key, t_sec, c_sec) = (sys.argv + ["", "", "", ""])[2:6] + if c_key: + print "oauth headers (%s)" % str((c_key, t_key, t_sec, c_sec,)) + (userdata, metadata) = read_maas_seed_url(seed, my_headers) + else: + (userdata, metadata) = read_maas_seed_url(seed) + else: (userdata, metadata) = read_maas_seed_dir(seed) -- cgit v1.2.3 From 78d33bcbabba8a38d659f54f92d703d3cc562a1f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 7 Mar 2012 09:58:54 -0500 Subject: pylint/pep8 cleanup --- cloudinit/DataSourceMaaS.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index 4c89c2eb..c168013b 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -258,14 +258,13 @@ if __name__ == "__main__": headers = oauth_headers(url, c_key, t_key, t_sec, c_sec) print "%s\n %s\n" % (url, headers) return headers - cb = None (c_key, t_key, t_sec, c_sec) = (sys.argv + ["", "", "", ""])[2:6] if c_key: print "oauth headers (%s)" % str((c_key, t_key, t_sec, c_sec,)) (userdata, metadata) = read_maas_seed_url(seed, my_headers) else: (userdata, metadata) = read_maas_seed_url(seed) - + else: (userdata, metadata) = read_maas_seed_dir(seed) -- cgit v1.2.3 From acf41ea8717646dedc9ddebed85d360996edfddc Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Wed, 7 Mar 2012 16:05:17 -0700 Subject: Added ability of cloud-init to manage apt http pipelining - cloud-config option of "apt-pipelining" - Address LP: 948461 --- cloudinit/CloudConfig/cc_apt_pipelining.py | 112 +++++++++++++++++++++++++++++ config/cloud.cfg | 1 + doc/examples/cloud-config.txt | 9 +++ 3 files changed, 122 insertions(+) create mode 100644 cloudinit/CloudConfig/cc_apt_pipelining.py diff --git a/cloudinit/CloudConfig/cc_apt_pipelining.py b/cloudinit/CloudConfig/cc_apt_pipelining.py new file mode 100644 index 00000000..8282bb0a --- /dev/null +++ b/cloudinit/CloudConfig/cc_apt_pipelining.py @@ -0,0 +1,112 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Ben Howard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import cloudinit.SshUtil as sshutil +import re +import os +from cloudinit.CloudConfig import per_always + +frequency = per_always +default_file = "/etc/apt/apt.conf.d/90cloud-init-tweaks" + +def handle(_name, cfg, cloud, log, _args): + + apt_pipelining_enabled = util.get_cfg_option_str(cfg, "apt-pipelining", False) + + if apt_pipelining_enabled in ("False", "false", False): + write_apt_snippet(0, log) + + elif apt_pipelining_enabled in ("Default", "default", "True", "true", True): + revert_os_default(log) + + else: + write_apt_snippet(apt_pipelining_enabled, log) + +def revert_os_default(log, f_name=default_file): + try: + + if os.path.exists(f_name): + os.unlink(f_name) + + except OSError: + log.debug("Unable to remove %s" % f_name) + + +def write_apt_snippet(setting, log, f_name=default_file): + """ + Reads f_name and determines if the setting matches or not. Sets to + desired value + """ + + acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' + try: + if os.path.exists(f_name): + update_file = False + skip_re = re.compile('^//CLOUD-INIT-IGNORE.*') + enabled_re = re.compile('Acquire::http::Pipeline-Depth.*') + + local_override = False + tweak = open(f_name, 'r') + new_file = [] + + for line in tweak.readlines(): + if skip_re.match(line): + local_override = True + continue + + if enabled_re.match(line): + + try: + value = line.replace('"','') + value = value.replace(';','') + enabled = value.split()[1] + + if enabled != setting: + update_file = True + line = acquire_pipeline_depth % setting + + except IndexError: + log.debug("Unable to determine current setting of 'Acquire::http::Pipeline-Depth'\n%s" % e) + return + + new_file.append(line) + + tweak.close() + + if local_override: + log.debug("Not updating apt pipelining settings due to local override in %s" % f_name) + return + + if update_file: + tweak = open(f_name, 'w') + for line in new_file: + tweak.write(line) + tweak.close() + + return + + tweak = open(f_name, 'w') + tweak.write("""//Cloud-init Tweaks\n//Disables APT HTTP pipelining\n""") + tweak.write(acquire_pipeline_depth % setting) + tweak.close() + + log.debug("Wrote %s with APT pipeline setting" % f_name ) + + except IOError as e: + log.debug("Unable to update pipeline settings in %s\n%s" % (f_name, e)) diff --git a/config/cloud.cfg b/config/cloud.cfg index fe197ca8..72a2d8e4 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -19,6 +19,7 @@ cloud_config_modules: - locale - set-passwords - grub-dpkg + - apt-pipelining - apt-update-upgrade - landscape - timezone diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 4f621274..ce188756 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -45,6 +45,15 @@ apt_mirror_search: # apt_proxy (configure Acquire::HTTP::Proxy) apt_proxy: http://my.apt.proxy:3128 +# apt_pipelining (confiugure Acquire::http::Pipeline-Depth) +# Default: disables HTTP pipelining. Certain web servers, such +# as S3 do not pipeline properly. +# Valid options: +# True/Default: Enables OS default +# False: Disables pipelining all-together +# Number: Set pipelining to some number (not recommended) +apt_pipelining: False + # Preserve existing /etc/apt/sources.list # Default: overwrite sources_list with mirror. If this is true # then apt_mirror above will have no effect -- cgit v1.2.3 From 0334e553a80f48362e5f8fd3fd5bb2f43b2ca3ea Mon Sep 17 00:00:00 2001 From: Mike Milner Date: Thu, 8 Mar 2012 08:45:43 -0400 Subject: Switch to using util.subp. --- cloudinit/CloudConfig/cc_ca_certs.py | 7 +++---- tests/unittests/test_handler_ca_certs.py | 9 ++++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/cloudinit/CloudConfig/cc_ca_certs.py b/cloudinit/CloudConfig/cc_ca_certs.py index c7bacb78..3af6238a 100644 --- a/cloudinit/CloudConfig/cc_ca_certs.py +++ b/cloudinit/CloudConfig/cc_ca_certs.py @@ -16,7 +16,7 @@ import os from subprocess import check_call from cloudinit.util import (write_file, get_cfg_option_list_or_str, - delete_dir_contents) + delete_dir_contents, subp) CA_CERT_PATH = "/usr/share/ca-certificates/" CA_CERT_FILENAME = "cloud-init-ca-certs.crt" @@ -54,9 +54,8 @@ def remove_default_ca_certs(): delete_dir_contents(CA_CERT_PATH) delete_dir_contents(CA_CERT_SYSTEM_PATH) write_file(CA_CERT_CONFIG, "", mode=0644) - check_call([ - "echo 'ca-certificates ca-certificates/trust_new_crts select no' | " - "debconf-set-selections"], shell=True) + debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" + subp(('debconf-set-selections', '-'), debconf_sel) def handle(_name, cfg, _cloud, log, _args): diff --git a/tests/unittests/test_handler_ca_certs.py b/tests/unittests/test_handler_ca_certs.py index 37bd7a08..21d2442f 100644 --- a/tests/unittests/test_handler_ca_certs.py +++ b/tests/unittests/test_handler_ca_certs.py @@ -169,15 +169,14 @@ class TestRemoveDefaultCaCerts(MockerTestCase): mock_delete_dir_contents = self.mocker.replace(delete_dir_contents, passthrough=False) mock_write = self.mocker.replace(write_file, passthrough=False) - mock_check_call = self.mocker.replace("subprocess.check_call", - passthrough=False) + mock_subp = self.mocker.replace("cloudinit.util.subp", + passthrough=False) mock_delete_dir_contents("/usr/share/ca-certificates/") mock_delete_dir_contents("/etc/ssl/certs/") mock_write("/etc/ca-certificates.conf", "", mode=0644) - mock_check_call([ - "echo 'ca-certificates ca-certificates/trust_new_crts select no'" - " | debconf-set-selections"], shell=True) + mock_subp(('debconf-set-selections', '-'), + "ca-certificates ca-certificates/trust_new_crts select no") self.mocker.replay() remove_default_ca_certs() -- cgit v1.2.3 From a352455707e192a4ea537578f3e11f221d378800 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Mar 2012 12:02:08 -0500 Subject: better 'main()', add and use version in api, do not require user-data main now is more useful for debugging. now it does: * get: just dump contents of a url provided after oauth * crawl: walk through using indexes, dumping content * check-seed: validate the seed is good uses MD_VERSION in the url, and appends that to the metadata url in the config file. (previously it assumed the url in the config was the full url) does not require user-data in the http seed. if the user did not specify user-data, it wont be there, so do not fail on that case. --- cloudinit/DataSourceMaaS.py | 135 +++++++++++++++++++++++++++++++++----------- 1 file changed, 102 insertions(+), 33 deletions(-) diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index c168013b..c320cb34 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -30,6 +30,9 @@ import urllib2 import time +MD_VERSION = "2012-03-01" + + class DataSourceMaaS(DataSource.DataSource): """ DataSourceMaaS reads instance information from MaaS. @@ -133,10 +136,10 @@ def read_maas_seed_dir(seed_d): Return user-data and metadata for a maas seed dir in seed_d. Expected format of seed_d are the following files: * instance-id - * hostname + * local-hostname * user-data """ - files = ('hostname', 'instance-id', 'user-data') + files = ('local-hostname', 'instance-id', 'user-data') md = {} if not os.path.isdir(seed_d): @@ -154,31 +157,38 @@ def read_maas_seed_dir(seed_d): return(check_seed_contents(md, seed_d)) -def read_maas_seed_url(seed_url, header_cb=None, timeout=None): +def read_maas_seed_url(seed_url, header_cb=None, timeout=None, + version=MD_VERSION): """ Read the maas datasource at seed_url. header_cb is a method that should return a headers dictionary that will be given to urllib2.Request() Expected format of seed_url is are the following files: - * /instance-id - * /hostname - * /user-data + * //instance-id + * //local-hostname + * //user-data """ - files = ('hostname', 'instance-id', 'user-data') + files = ('meta-data/local-hostname', 'meta-data/instance-id', 'user-data') + base_url = "%s/%s" % (seed_url, version) + print "seed_url=%s version=%s" % (seed_url, version) md = {} for fname in files: - url = "%s/%s" % (seed_url, fname) + url = "%s/%s" % (base_url, fname) if header_cb: headers = header_cb(url) else: headers = {} - req = urllib2.Request(url, data=None, headers=headers) - resp = urllib2.urlopen(req, timeout=timeout) - - md[fname] = resp.read() + try: + print url + req = urllib2.Request(url, data=None, headers=headers) + resp = urllib2.urlopen(req, timeout=timeout) + md[os.path.basename(fname)] = resp.read() + except urllib2.HTTPError as e: + if e.code != 404: + raise return(check_seed_contents(md, seed_url)) @@ -189,7 +199,7 @@ def check_seed_contents(content, seed): Either return a (userdata, metadata) tuple or Raise MaasSeedDirMalformed or MaasSeedDirNone """ - md_required = ('user-data', 'instance-id', 'hostname') + md_required = ('instance-id', 'local-hostname') found = content.keys() if len(content) == 0: @@ -199,7 +209,7 @@ def check_seed_contents(content, seed): if len(missing): raise MaasSeedDirMalformed("%s: missing files %s" % (seed, missing)) - userdata = content['user-data'] + userdata = content.get('user-data', "") md = {} for (key, val) in content.iteritems(): if key == 'user-data': @@ -250,27 +260,86 @@ if __name__ == "__main__": If url is given additional arguments are allowed, which will be interpreted as consumer_key, token_key, token_secret, consumer_secret """ - import sys + import argparse import pprint - seed = sys.argv[1] - if seed.startswith("http://") or seed.startswith("https://"): - def my_headers(url): - headers = oauth_headers(url, c_key, t_key, t_sec, c_sec) - print "%s\n %s\n" % (url, headers) - return headers - (c_key, t_key, t_sec, c_sec) = (sys.argv + ["", "", "", ""])[2:6] - if c_key: - print "oauth headers (%s)" % str((c_key, t_key, t_sec, c_sec,)) - (userdata, metadata) = read_maas_seed_url(seed, my_headers) - else: - (userdata, metadata) = read_maas_seed_url(seed) - else: - (userdata, metadata) = read_maas_seed_dir(seed) + parser = argparse.ArgumentParser(description='Interact with Maas DS') + parser.add_argument("--config", metavar="file", + help="specify DS config file", default=None) + parser.add_argument("--ckey", metavar="key", + help="the consumer key to auth with", default=None) + parser.add_argument("--tkey", metavar="key", + help="the token key to auth with", default=None) + parser.add_argument("--csec", metavar="secret", + help="the consumer secret (likely '')", default="") + parser.add_argument("--tsec", metavar="secret", + help="the token secret to auth with", default=None) + parser.add_argument("--apiver", metavar="version", + help="the apiver to use ("" can be used)", default=MD_VERSION) + + subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") + subcmds.add_parser('crawl', help="crawl the datasource") + subcmds.add_parser('get', help="do a single GET of provided url") + subcmds.add_parser('check-seed', help="read andn verify seed at url") + + parser.add_argument("url", help="the data source to query") + + args = parser.parse_args() + + creds = {'consumer_key': args.ckey, 'token_key': args.tkey, + 'token_secret': args.tsec, 'consumer_secret': args.csec} + + if args.config: + import yaml + with open(args.config) as fp: + cfg = yaml.load(fp) + print cfg + if 'datasource' in cfg: + cfg = cfg['datasource']['MaaS'] + for key in creds.keys(): + if key in cfg and creds[key] == None: + creds[key] = cfg[key] + + def geturl(url, headers_cb): + req = urllib2.Request(url, data=None, headers=headers_cb(url)) + return(urllib2.urlopen(req).read()) + + def printurl(url, headers_cb): + print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) + + def crawl(url, headers_cb=None): + if url.endswith("/"): + for line in geturl(url, headers_cb).splitlines(): + if line.endswith("/"): + crawl("%s%s" % (url, line), headers_cb) + else: + printurl("%s%s" % (url, line), headers_cb) + else: + printurl(url, headers_cb) - print "=== userdata ===" - print userdata - print "=== metadata ===" - pprint.pprint(metadata) + def my_headers(url): + headers = {} + if creds.get('consumer_key', None) != None: + headers = oauth_headers(url, **creds) + return headers + + if args.subcmd == "check-seed": + if args.url.startswith("http"): + (userdata, metadata) = read_maas_seed_url(args.url, + header_cb=my_headers, version=args.apiver) + else: + (userdata, metadata) = read_maas_seed_url(args.url) + print "=== userdata ===" + print userdata + print "=== metadata ===" + pprint.pprint(metadata) + + elif args.subcmd == "get": + printurl(args.url, my_headers) + + elif args.subcmd == "crawl": + if not args.url.endswith("/"): + args.url = "%s/" % args.url + crawl(args.url, my_headers) main() -- cgit v1.2.3 From b11da90d4a932516f66643346fc3b6492cee9478 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Mar 2012 12:04:51 -0500 Subject: remove debug statement --- cloudinit/DataSourceMaaS.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index c320cb34..29f78130 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -293,7 +293,6 @@ if __name__ == "__main__": import yaml with open(args.config) as fp: cfg = yaml.load(fp) - print cfg if 'datasource' in cfg: cfg = cfg['datasource']['MaaS'] for key in creds.keys(): -- cgit v1.2.3 From 996182c615548e56787551b294279158011b6bf2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Mar 2012 12:10:02 -0500 Subject: fix tests for 'hostname' to 'local-hostname' and user-data not required --- tests/unittests/test_datasource/test_maas.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 4f896add..a090f873 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -25,7 +25,7 @@ class TestMaasDataSource(TestCase): def test_seed_dir_valid(self): """Verify a valid seeddir is read as such""" - data = {'instance-id': 'i-valid01', 'hostname': 'valid01-hostname', + data = {'instance-id': 'i-valid01', 'local-hostname': 'valid01-hostname', 'user-data': 'valid01-userdata'} my_d = os.path.join(self.tmp, "valid") @@ -34,7 +34,7 @@ class TestMaasDataSource(TestCase): (userdata, metadata) = read_maas_seed_dir(my_d) self.assertEqual(userdata, data['user-data']) - for key in ('instance-id', 'hostname'): + for key in ('instance-id', 'local-hostname'): self.assertEqual(data[key], metadata[key]) # verify that 'userdata' is not returned as part of the metadata @@ -44,7 +44,7 @@ class TestMaasDataSource(TestCase): """Verify extra files do not affect seed_dir validity """ data = {'instance-id': 'i-valid-extra', - 'hostname': 'valid-extra-hostname', + 'local-hostname': 'valid-extra-hostname', 'user-data': 'valid-extra-userdata', 'foo': 'bar'} my_d = os.path.join(self.tmp, "valid_extra") @@ -53,7 +53,7 @@ class TestMaasDataSource(TestCase): (userdata, metadata) = read_maas_seed_dir(my_d) self.assertEqual(userdata, data['user-data']) - for key in ('instance-id', 'hostname'): + for key in ('instance-id', 'local-hostname'): self.assertEqual(data[key], metadata[key]) # additional files should not just appear as keys in metadata atm @@ -63,14 +63,14 @@ class TestMaasDataSource(TestCase): """Verify that invalid seed_dir raises MaasSeedDirMalformed""" valid = {'instance-id': 'i-instanceid', - 'hostname': 'test-hostname', 'user-data': ''} + 'local-hostname': 'test-hostname', 'user-data': ''} my_based = os.path.join(self.tmp, "valid_extra") # missing 'userdata' file my_d = "%s-01" % my_based invalid_data = copy(valid) - del invalid_data['user-data'] + del invalid_data['local-hostname'] populate_dir(my_d, invalid_data) self.assertRaises(MaasSeedDirMalformed, read_maas_seed_dir, my_d) -- cgit v1.2.3 From a0a5d2af8109dfc0cd0685b2d7a54e261a3b289e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Mar 2012 12:15:29 -0500 Subject: add MaaS datasource to default searched --- cloudinit/__init__.py | 2 +- config/cloud.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py index ccaa28c8..9f188766 100644 --- a/cloudinit/__init__.py +++ b/cloudinit/__init__.py @@ -29,7 +29,7 @@ cfg_env_name = "CLOUD_CFG" cfg_builtin = """ log_cfgs: [] -datasource_list: ["NoCloud", "ConfigDrive", "OVF", "Ec2"] +datasource_list: ["NoCloud", "ConfigDrive", "OVF", "MaaS", "Ec2" ] def_log_file: /var/log/cloud-init.log syslog_fix_perms: syslog:adm """ diff --git a/config/cloud.cfg b/config/cloud.cfg index fe197ca8..01e63ef8 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -1,7 +1,7 @@ user: ubuntu disable_root: 1 preserve_hostname: False -# datasource_list: [ "NoCloud", "ConfigDrive", "OVF", "Ec2" ] +# datasource_list: ["NoCloud", "ConfigDrive", "OVF", "MaaS", "Ec2" ] cloud_init_modules: - bootcmd -- cgit v1.2.3 From 0756ed017d02916fac240a07173a4bbb27eb74c0 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Mar 2012 13:20:10 -0500 Subject: remove debug statement --- cloudinit/DataSourceMaaS.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index 29f78130..6bad0dfe 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -172,7 +172,6 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, files = ('meta-data/local-hostname', 'meta-data/instance-id', 'user-data') base_url = "%s/%s" % (seed_url, version) - print "seed_url=%s version=%s" % (seed_url, version) md = {} for fname in files: url = "%s/%s" % (base_url, fname) @@ -182,7 +181,6 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, headers = {} try: - print url req = urllib2.Request(url, data=None, headers=headers) resp = urllib2.urlopen(req, timeout=timeout) md[os.path.basename(fname)] = resp.read() -- cgit v1.2.3 From 61011d088ca011c2bc3a9f519519a833064cc922 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Mar 2012 13:35:55 -0500 Subject: add a test for read_maas_seed_url --- tests/unittests/test_datasource/test_maas.py | 38 ++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index a090f873..3da2b0b3 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -2,15 +2,19 @@ from unittest import TestCase from tempfile import mkdtemp from shutil import rmtree import os +import urllib2 +from StringIO import StringIO from copy import copy from cloudinit.DataSourceMaaS import ( MaasSeedDirNone, MaasSeedDirMalformed, read_maas_seed_dir, + read_maas_seed_url, ) +from mocker import MockerTestCase -class TestMaasDataSource(TestCase): +class TestMaasDataSource(MockerTestCase): def setUp(self): super(TestMaasDataSource, self).setUp() @@ -94,7 +98,37 @@ class TestMaasDataSource(TestCase): def test_seed_url_valid(self): """Verify that valid seed_url is read as such""" - pass + valid = {'meta-data/instance-id': 'i-instanceid', + 'meta-data/local-hostname': 'test-hostname', 'user-data': 'foodata'} + + my_seed = "http://example.com/xmeta" + my_ver = "1999-99-99" + my_headers = {'header1': 'value1', 'header2': 'value2'} + + def my_headers_cb(url): + return(my_headers) + + mock_request = self.mocker.replace("urllib2.Request", passthrough=False) + mock_urlopen = self.mocker.replace("urllib2.urlopen", passthrough=False) + + for (key, val) in valid.iteritems(): + mock_request("%s/%s/%s" % (my_seed, my_ver, key), + data=None, headers=my_headers) + self.mocker.nospec() + self.mocker.result("fake-request-%s" % key) + mock_urlopen("fake-request-%s" % key, timeout=None) + self.mocker.result(StringIO(val)) + + self.mocker.replay() + + (userdata, metadata) = read_maas_seed_url(my_seed, + header_cb=my_headers_cb, version=my_ver) + + self.assertEqual("foodata", userdata) + self.assertEqual(metadata['instance-id'], + valid['meta-data/instance-id']) + self.assertEqual(metadata['local-hostname'], + valid['meta-data/local-hostname']) def test_seed_url_invalid(self): """Verify that invalid seed_url raises MaasSeedDirMalformed""" -- cgit v1.2.3 From 82a92493a08cf311ec6df6170ed7e31f2b1ce1ea Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Mar 2012 14:21:48 -0500 Subject: pep8 and pylint --- tests/unittests/test_datasource/test_maas.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 3da2b0b3..d0e121d6 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -1,8 +1,6 @@ -from unittest import TestCase from tempfile import mkdtemp from shutil import rmtree import os -import urllib2 from StringIO import StringIO from copy import copy from cloudinit.DataSourceMaaS import ( @@ -29,7 +27,8 @@ class TestMaasDataSource(MockerTestCase): def test_seed_dir_valid(self): """Verify a valid seeddir is read as such""" - data = {'instance-id': 'i-valid01', 'local-hostname': 'valid01-hostname', + data = {'instance-id': 'i-valid01', + 'local-hostname': 'valid01-hostname', 'user-data': 'valid01-userdata'} my_d = os.path.join(self.tmp, "valid") @@ -99,7 +98,8 @@ class TestMaasDataSource(MockerTestCase): def test_seed_url_valid(self): """Verify that valid seed_url is read as such""" valid = {'meta-data/instance-id': 'i-instanceid', - 'meta-data/local-hostname': 'test-hostname', 'user-data': 'foodata'} + 'meta-data/local-hostname': 'test-hostname', + 'user-data': 'foodata'} my_seed = "http://example.com/xmeta" my_ver = "1999-99-99" @@ -108,8 +108,10 @@ class TestMaasDataSource(MockerTestCase): def my_headers_cb(url): return(my_headers) - mock_request = self.mocker.replace("urllib2.Request", passthrough=False) - mock_urlopen = self.mocker.replace("urllib2.urlopen", passthrough=False) + mock_request = self.mocker.replace("urllib2.Request", + passthrough=False) + mock_urlopen = self.mocker.replace("urllib2.urlopen", + passthrough=False) for (key, val) in valid.iteritems(): mock_request("%s/%s/%s" % (my_seed, my_ver, key), -- cgit v1.2.3 From 62623b8c5ac10b2bff9f6205f696b4277bce9451 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Mar 2012 14:59:17 -0500 Subject: doc fixes --- doc/examples/cloud-config-datasources.txt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index f147aaf6..b3a26114 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -18,12 +18,8 @@ datasource: timeout : 50 max_wait : 120 - # there are no default values for metadata_url or credentials - # credentials are for oath. - - # If no credentials are present, - # then non-authed attempts will be made. - # will be made + # there are no default values for metadata_url or oauth credentials + # If no credentials are present, non-authed attempts will be made. metadata_url: http://mass-host.localdomain/source consumer_key: Xh234sdkljf token_key: kjfhgb3n -- cgit v1.2.3 From e40c7c0a1ae8119e38a877154070e5fa63677a66 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Mar 2012 15:38:27 -0500 Subject: DataSourceMaaS: some fixes found in testing --- cloudinit/DataSourceMaaS.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py index 6bad0dfe..fd9d6316 100644 --- a/cloudinit/DataSourceMaaS.py +++ b/cloudinit/DataSourceMaaS.py @@ -76,6 +76,8 @@ class DataSourceMaaS(DataSource.DataSource): (userdata, metadata) = read_maas_seed_url(self.baseurl, self.md_headers) + self.userdata_raw = userdata + self.metadata = metadata return True except Exception: util.logexc(log) @@ -117,7 +119,7 @@ class DataSourceMaaS(DataSource.DataSource): log.warn("Failed to get timeout, using %s" % timeout) starttime = time.time() - check_url = "%s/instance-id" % url + check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) url = util.wait_for_url(urls=[check_url], max_wait=max_wait, timeout=timeout, status_cb=log.warn, headers_cb=self.md_headers) -- cgit v1.2.3 From 162762d916c18ad83a6775cc1bd9a86cb78a5dd7 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 8 Mar 2012 16:22:16 -0700 Subject: Simplified proposed patch - Changed values to be more simplistic and intuitive - Only allow pipelining values up to 5 - Changed to per_instance over per_always to remove need for tracking the values - Fixed Python style --- cloudinit/CloudConfig/cc_apt_pipelining.py | 76 +++++++----------------------- doc/examples/cloud-config.txt | 6 +-- 2 files changed, 20 insertions(+), 62 deletions(-) diff --git a/cloudinit/CloudConfig/cc_apt_pipelining.py b/cloudinit/CloudConfig/cc_apt_pipelining.py index 8282bb0a..c1e65847 100644 --- a/cloudinit/CloudConfig/cc_apt_pipelining.py +++ b/cloudinit/CloudConfig/cc_apt_pipelining.py @@ -17,36 +17,29 @@ # along with this program. If not, see . import cloudinit.util as util -import cloudinit.SshUtil as sshutil import re import os -from cloudinit.CloudConfig import per_always +from cloudinit.CloudConfig import per_instance -frequency = per_always -default_file = "/etc/apt/apt.conf.d/90cloud-init-tweaks" +frequency = per_instance +default_file = "/etc/apt/apt.conf.d/90cloud-init-pipeling" def handle(_name, cfg, cloud, log, _args): - apt_pipelining_enabled = util.get_cfg_option_str(cfg, "apt-pipelining", False) + apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) + apt_pipe_value = str(apt_pipe_value).lower() - if apt_pipelining_enabled in ("False", "false", False): + if apt_pipe_value in ("false", "default", False): write_apt_snippet(0, log) - elif apt_pipelining_enabled in ("Default", "default", "True", "true", True): - revert_os_default(log) + elif apt_pipe_value in ("none", "unchanged", "os"): + return - else: - write_apt_snippet(apt_pipelining_enabled, log) - -def revert_os_default(log, f_name=default_file): - try: - - if os.path.exists(f_name): - os.unlink(f_name) - - except OSError: - log.debug("Unable to remove %s" % f_name) + elif apt_pipe_value in str(range(1, 5)): + write_apt_snippet(apt_pipe_value, log) + else: + log.warn("Invalid option for apt_pipeling") def write_apt_snippet(setting, log, f_name=default_file): """ @@ -57,54 +50,19 @@ def write_apt_snippet(setting, log, f_name=default_file): acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' try: if os.path.exists(f_name): - update_file = False skip_re = re.compile('^//CLOUD-INIT-IGNORE.*') - enabled_re = re.compile('Acquire::http::Pipeline-Depth.*') - - local_override = False - tweak = open(f_name, 'r') - new_file = [] for line in tweak.readlines(): if skip_re.match(line): - local_override = True - continue - - if enabled_re.match(line): - - try: - value = line.replace('"','') - value = value.replace(';','') - enabled = value.split()[1] - - if enabled != setting: - update_file = True - line = acquire_pipeline_depth % setting - - except IndexError: - log.debug("Unable to determine current setting of 'Acquire::http::Pipeline-Depth'\n%s" % e) - return - - new_file.append(line) + tweak.close() + return tweak.close() - if local_override: - log.debug("Not updating apt pipelining settings due to local override in %s" % f_name) - return - - if update_file: - tweak = open(f_name, 'w') - for line in new_file: - tweak.write(line) - tweak.close() - - return + file_contents = ("//Cloud-init Tweaks\n//Disables APT HTTP pipelining"\ + "\n" + (acquire_pipeline_depth % setting)) - tweak = open(f_name, 'w') - tweak.write("""//Cloud-init Tweaks\n//Disables APT HTTP pipelining\n""") - tweak.write(acquire_pipeline_depth % setting) - tweak.close() + util.write_file(f_name, file_contents) log.debug("Wrote %s with APT pipeline setting" % f_name ) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index ce188756..542a49c7 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -45,12 +45,12 @@ apt_mirror_search: # apt_proxy (configure Acquire::HTTP::Proxy) apt_proxy: http://my.apt.proxy:3128 -# apt_pipelining (confiugure Acquire::http::Pipeline-Depth) +# apt_pipelining (configure Acquire::http::Pipeline-Depth) # Default: disables HTTP pipelining. Certain web servers, such # as S3 do not pipeline properly. # Valid options: -# True/Default: Enables OS default -# False: Disables pipelining all-together +# False/default: Disables pipelining for APT +# None/Unchanged: Use OS default # Number: Set pipelining to some number (not recommended) apt_pipelining: False -- cgit v1.2.3 From 210dfb8840baded8a2282e8b0b49c3a034222bd8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 9 Mar 2012 09:50:04 -0500 Subject: Some cleanups before merge. * removed the 'CLOUD-INIT-IGNORE' section, as we're just blindly writing the file now. removed the now-unnecessary import of 're' and 'os' * removed try/except block around write_apt_snippet. This will bubble up and cloud-init will let it through even to the console. Catching it and turning it into a debug would just hide it. * removed 'default' as a synonym for 'whatever cloud-init thinks is best' If people are going to change this, I'd rather they be specific. * supported value of "0" * fixed some complaints from ./tools/run-pylint cloudinit/CloudConfig/cc_apt_pipelining.py --- cloudinit/CloudConfig/cc_apt_pipelining.py | 41 +++++++++--------------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/cloudinit/CloudConfig/cc_apt_pipelining.py b/cloudinit/CloudConfig/cc_apt_pipelining.py index c1e65847..7ca93e66 100644 --- a/cloudinit/CloudConfig/cc_apt_pipelining.py +++ b/cloudinit/CloudConfig/cc_apt_pipelining.py @@ -17,54 +17,37 @@ # along with this program. If not, see . import cloudinit.util as util -import re -import os from cloudinit.CloudConfig import per_instance frequency = per_instance default_file = "/etc/apt/apt.conf.d/90cloud-init-pipeling" -def handle(_name, cfg, cloud, log, _args): + +def handle(_name, cfg, _cloud, log, _args): apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) apt_pipe_value = str(apt_pipe_value).lower() - if apt_pipe_value in ("false", "default", False): - write_apt_snippet(0, log) + if apt_pipe_value == "false": + write_apt_snippet("0", log) elif apt_pipe_value in ("none", "unchanged", "os"): return - elif apt_pipe_value in str(range(1, 5)): + elif apt_pipe_value in str(range(0, 6)): write_apt_snippet(apt_pipe_value, log) else: - log.warn("Invalid option for apt_pipeling") + log.warn("Invalid option for apt_pipeling: %s" % apt_pipe_value) + def write_apt_snippet(setting, log, f_name=default_file): - """ - Reads f_name and determines if the setting matches or not. Sets to - desired value - """ + """ Writes f_name with apt pipeline depth 'setting' """ acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' - try: - if os.path.exists(f_name): - skip_re = re.compile('^//CLOUD-INIT-IGNORE.*') - - for line in tweak.readlines(): - if skip_re.match(line): - tweak.close() - return - - tweak.close() - - file_contents = ("//Cloud-init Tweaks\n//Disables APT HTTP pipelining"\ - "\n" + (acquire_pipeline_depth % setting)) - - util.write_file(f_name, file_contents) + file_contents = ("//Written by cloud-init per 'apt_pipelining'\n" + + (acquire_pipeline_depth % setting)) - log.debug("Wrote %s with APT pipeline setting" % f_name ) + util.write_file(f_name, file_contents) - except IOError as e: - log.debug("Unable to update pipeline settings in %s\n%s" % (f_name, e)) + log.debug("Wrote %s with APT pipeline setting" % f_name) -- cgit v1.2.3 From 91c1ef651077dbdbe7e8a85cb7b48926fbeb0aab Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 9 Mar 2012 09:51:26 -0500 Subject: mention bug number in cloud-config.txt --- doc/examples/cloud-config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 542a49c7..171802cc 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -47,7 +47,7 @@ apt_proxy: http://my.apt.proxy:3128 # apt_pipelining (configure Acquire::http::Pipeline-Depth) # Default: disables HTTP pipelining. Certain web servers, such -# as S3 do not pipeline properly. +# as S3 do not pipeline properly (LP: #948461). # Valid options: # False/default: Disables pipelining for APT # None/Unchanged: Use OS default -- cgit v1.2.3 From bff1590def00c3f7653ce34267bbe88e645bd9c6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 9 Mar 2012 10:26:09 -0500 Subject: fix spelling error in apt pipeline filename --- cloudinit/CloudConfig/cc_apt_pipelining.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/CloudConfig/cc_apt_pipelining.py b/cloudinit/CloudConfig/cc_apt_pipelining.py index 7ca93e66..0286a9ae 100644 --- a/cloudinit/CloudConfig/cc_apt_pipelining.py +++ b/cloudinit/CloudConfig/cc_apt_pipelining.py @@ -20,7 +20,7 @@ import cloudinit.util as util from cloudinit.CloudConfig import per_instance frequency = per_instance -default_file = "/etc/apt/apt.conf.d/90cloud-init-pipeling" +default_file = "/etc/apt/apt.conf.d/90cloud-init-pipelining" def handle(_name, cfg, _cloud, log, _args): -- cgit v1.2.3