From 508670bdaf9545b6bcc8e2009e8bd3f08d6f8796 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 22 Jan 2015 18:38:30 -0500 Subject: More test ports from mocker to mock. --- tests/unittests/test_datasource/test_maas.py | 75 +++++++++++++++++----------- 1 file changed, 47 insertions(+), 28 deletions(-) (limited to 'tests/unittests/test_datasource/test_maas.py') diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index c157beb8..6af0cd82 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -1,19 +1,26 @@ from copy import copy import os +import shutil +import tempfile +import unittest from cloudinit.sources import DataSourceMAAS from cloudinit import url_helper from ..helpers import populate_dir -import mocker +try: + from unittest import mock +except ImportError: + import mock -class TestMAASDataSource(mocker.MockerTestCase): +class TestMAASDataSource(unittest.TestCase): def setUp(self): super(TestMAASDataSource, self).setUp() # Make a temp directoy for tests to use. - self.tmp = self.makeDir() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) def test_seed_dir_valid(self): """Verify a valid seeddir is read as such.""" @@ -93,16 +100,18 @@ class TestMAASDataSource(mocker.MockerTestCase): def test_seed_url_valid(self): """Verify that valid seed_url is read as such.""" - valid = {'meta-data/instance-id': 'i-instanceid', + valid = { + 'meta-data/instance-id': 'i-instanceid', 'meta-data/local-hostname': 'test-hostname', 'meta-data/public-keys': 'test-hostname', - 'user-data': 'foodata'} + 'user-data': 'foodata', + } valid_order = [ 'meta-data/local-hostname', 'meta-data/instance-id', 'meta-data/public-keys', 'user-data', - ] + ] my_seed = "http://example.com/xmeta" my_ver = "1999-99-99" my_headers = {'header1': 'value1', 'header2': 'value2'} @@ -110,28 +119,38 @@ class TestMAASDataSource(mocker.MockerTestCase): def my_headers_cb(url): return my_headers - mock_request = self.mocker.replace(url_helper.readurl, - passthrough=False) - - for key in valid_order: - url = "%s/%s/%s" % (my_seed, my_ver, key) - mock_request(url, headers=None, timeout=mocker.ANY, - data=mocker.ANY, sec_between=mocker.ANY, - ssl_details=mocker.ANY, retries=mocker.ANY, - headers_cb=my_headers_cb, - exception_cb=mocker.ANY) - resp = valid.get(key) - self.mocker.result(url_helper.StringResponse(resp)) - self.mocker.replay() - - (userdata, metadata) = DataSourceMAAS.read_maas_seed_url(my_seed, - header_cb=my_headers_cb, version=my_ver) - - self.assertEqual("foodata", userdata) - self.assertEqual(metadata['instance-id'], - valid['meta-data/instance-id']) - self.assertEqual(metadata['local-hostname'], - valid['meta-data/local-hostname']) + # Each time url_helper.readurl() is called, something different is + # returned based on the canned data above. We need to build up a list + # of side effect return values, which the mock will return. At the + # same time, we'll build up a list of expected call arguments for + # asserting after the code under test is run. + calls = [] + + def side_effect(): + for key in valid_order: + resp = valid.get(key) + url = "%s/%s/%s" % (my_seed, my_ver, key) + calls.append( + mock.call(url, headers=None, timeout=mock.ANY, + data=mock.ANY, sec_between=mock.ANY, + ssl_details=mock.ANY, retries=mock.ANY, + headers_cb=my_headers_cb, + exception_cb=mock.ANY)) + yield url_helper.StringResponse(resp) + + # Now do the actual call of the code under test. + with mock.patch.object(url_helper, 'readurl', + side_effect=side_effect()) as mockobj: + userdata, metadata = DataSourceMAAS.read_maas_seed_url( + my_seed, header_cb=my_headers_cb, version=my_ver) + + self.assertEqual("foodata", userdata) + self.assertEqual(metadata['instance-id'], + valid['meta-data/instance-id']) + self.assertEqual(metadata['local-hostname'], + valid['meta-data/local-hostname']) + + mockobj.has_calls(calls) def test_seed_url_invalid(self): """Verify that invalid seed_url raises MAASSeedDirMalformed.""" -- cgit v1.2.3 From de5974fe93dd717e0c7ba6de17db3192cc258cff Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 26 Jan 2015 14:31:09 -0500 Subject: * More str/bytes fixes. * Temporarily skip the MAAS tests in py3 since they need to be ported to oauthlib. --- cloudinit/sources/DataSourceOpenNebula.py | 12 +++++++++--- cloudinit/sources/DataSourceSmartOS.py | 15 +++++++++++++-- tests/unittests/test_datasource/test_maas.py | 7 ++++++- tests/unittests/test_datasource/test_opennebula.py | 4 ++-- 4 files changed, 30 insertions(+), 8 deletions(-) (limited to 'tests/unittests/test_datasource/test_maas.py') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index f9dac29e..691b39f8 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -379,7 +379,8 @@ def read_context_disk_dir(source_dir, asuser=None): raise BrokenContextDiskDir("configured user '%s' " "does not exist", asuser) try: - with open(os.path.join(source_dir, 'context.sh'), 'r') as f: + path = os.path.join(source_dir, 'context.sh') + with open(path, 'r', encoding='utf-8') as f: content = f.read().strip() context = parse_shell_config(content, asuser=asuser) @@ -426,14 +427,19 @@ def read_context_disk_dir(source_dir, asuser=None): context.get('USER_DATA_ENCODING')) if encoding == "base64": try: - results['userdata'] = base64.b64decode(results['userdata']) + userdata = base64.b64decode(results['userdata']) + # In Python 3 we still expect a str, but b64decode will return + # bytes. Convert to str. + if isinstance(userdata, bytes): + userdata = userdata.decode('utf-8') + results['userdata'] = userdata except TypeError: LOG.warn("Failed base64 decoding of userdata") # generate static /etc/network/interfaces # only if there are any required context variables # http://opennebula.org/documentation:rel3.8:cong#network_configuration - for k in context.keys(): + for k in context: if re.match(r'^ETH\d+_IP$', k): (out, _) = util.subp(['/sbin/ip', 'link']) net = OpenNebulaNetwork(out, context) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 7a975d78..d3ed40c5 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -30,6 +30,7 @@ # Comments with "@datadictionary" are snippets of the definition import base64 +import binascii import os import serial @@ -350,8 +351,18 @@ def query_data(noun, seed_device, seed_timeout, strip=False, default=None, if b64: try: - return base64.b64decode(resp) - except TypeError: + # Generally, we want native strings in the values. Python 3's + # b64decode will return bytes though, so decode them to utf-8 if + # possible. If that fails, return the bytes. + decoded = base64.b64decode(resp) + try: + if isinstance(decoded, bytes): + return decoded.decode('utf-8') + except UnicodeDecodeError: + pass + return decoded + # Bogus input produces different errors in Python 2 and 3; catch both. + except (TypeError, binascii.Error): LOG.warn("Failed base64 decoding key '%s'", noun) return resp diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 6af0cd82..66fe22ae 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -4,7 +4,11 @@ import shutil import tempfile import unittest -from cloudinit.sources import DataSourceMAAS +# XXX DataSourceMAAS must be ported to oauthlib for Python 3 +import six +if not six.PY3: + from cloudinit.sources import DataSourceMAAS + from cloudinit import url_helper from ..helpers import populate_dir @@ -14,6 +18,7 @@ except ImportError: import mock +@unittest.skipIf(six.PY3, 'DataSourceMAAS must be ported to oauthlib') class TestMAASDataSource(unittest.TestCase): def setUp(self): diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index 31c6232f..ef534bab 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -33,7 +33,7 @@ TEST_VARS = { } INVALID_CONTEXT = ';' -USER_DATA = b'#cloud-config\napt_upgrade: true' +USER_DATA = '#cloud-config\napt_upgrade: true' SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' HOSTNAME = 'foo.example.com' PUBLIC_IP = '10.0.0.3' @@ -300,7 +300,7 @@ class TestParseShellConfig(unittest.TestCase): def populate_context_dir(path, variables): data = "# Context variables generated by OpenNebula\n" - for (k, v) in variables.items(): + for k, v in variables.items(): data += ("%s='%s'\n" % (k.upper(), v.replace(r"'", r"'\''"))) populate_dir(path, {'context.sh': data}) -- cgit v1.2.3 From fabff4aec884467729fc372bb67f240752c15511 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 26 Jan 2015 16:37:29 -0500 Subject: Port the MAAS code to oauthlib. --- cloudinit/sources/DataSourceMAAS.py | 56 ++++++++++++++++------------ requirements.txt | 2 +- tests/unittests/test_datasource/test_maas.py | 7 +--- 3 files changed, 35 insertions(+), 30 deletions(-) (limited to 'tests/unittests/test_datasource/test_maas.py') diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 8f9c81de..39296f08 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -22,10 +22,11 @@ from __future__ import print_function from email.utils import parsedate import errno -import oauth.oauth as oauth +import oauthlib import os import time -import urllib2 + +from six.moves.urllib_request import Request, urlopen from cloudinit import log as logging from cloudinit import sources @@ -274,25 +275,34 @@ def check_seed_contents(content, seed): def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret, timestamp=None): - consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) - token = oauth.OAuthToken(token_key, token_secret) - - if timestamp is None: - ts = int(time.time()) - else: - ts = timestamp - - params = { - 'oauth_version': "1.0", - 'oauth_nonce': oauth.generate_nonce(), - 'oauth_timestamp': ts, - 'oauth_token': token.key, - 'oauth_consumer_key': consumer.key, - } - req = oauth.OAuthRequest(http_url=url, parameters=params) - req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(), - consumer, token) - return req.to_header() + client = oauthlib.oauth1.Client( + consumer_key, + client_secret=consumer_secret, + resource_owner_key=token_key, + resource_owner_secret=token_secret, + signature_method=oauthlib.SIGNATURE_PLAINTEXT) + uri, signed_headers, body = client.sign(url) + return signed_headers + + ## consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) + ## token = oauth.OAuthToken(token_key, token_secret) + + ## if timestamp is None: + ## ts = int(time.time()) + ## else: + ## ts = timestamp + + ## params = { + ## 'oauth_version': "1.0", + ## 'oauth_nonce': oauth.generate_nonce(), + ## 'oauth_timestamp': ts, + ## 'oauth_token': token.key, + ## 'oauth_consumer_key': consumer.key, + ## } + ## req = oauth.OAuthRequest(http_url=url, parameters=params) + ## req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(), + ## consumer, token) + ## return req.to_header() class MAASSeedDirNone(Exception): @@ -359,8 +369,8 @@ if __name__ == "__main__": creds[key] = cfg[key] def geturl(url, headers_cb): - req = urllib2.Request(url, data=None, headers=headers_cb(url)) - return (urllib2.urlopen(req).read()) + req = Request(url, data=None, headers=headers_cb(url)) + return urlopen(req).read() def printurl(url, headers_cb): print("== %s ==\n%s\n" % (url, geturl(url, headers_cb))) diff --git a/requirements.txt b/requirements.txt index 2a12ca3e..19c88857 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ PrettyTable # This one is currently only used by the MAAS datasource. If that # datasource is removed, this is no longer needed -oauth +oauthlib # This one is currently used only by the CloudSigma and SmartOS datasources. # If these datasources are removed, this is no longer needed diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 66fe22ae..6af0cd82 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -4,11 +4,7 @@ import shutil import tempfile import unittest -# XXX DataSourceMAAS must be ported to oauthlib for Python 3 -import six -if not six.PY3: - from cloudinit.sources import DataSourceMAAS - +from cloudinit.sources import DataSourceMAAS from cloudinit import url_helper from ..helpers import populate_dir @@ -18,7 +14,6 @@ except ImportError: import mock -@unittest.skipIf(six.PY3, 'DataSourceMAAS must be ported to oauthlib') class TestMAASDataSource(unittest.TestCase): def setUp(self): -- cgit v1.2.3 From 5e2b8ef0703eb4582a5a8ba50ae7c83a8294d65a Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 26 Jan 2015 20:02:31 -0500 Subject: Repair the Python 2.6 tests. --- cloudinit/util.py | 18 ++++++++-------- tests/unittests/helpers.py | 25 +++++++++++++++++++--- tests/unittests/test__init__.py | 6 ++++-- tests/unittests/test_cs_util.py | 16 +++++++++++++- tests/unittests/test_datasource/test_azure.py | 7 +++--- tests/unittests/test_datasource/test_cloudsigma.py | 1 + .../unittests/test_datasource/test_configdrive.py | 6 ++++-- tests/unittests/test_datasource/test_maas.py | 5 ++--- tests/unittests/test_datasource/test_nocloud.py | 7 +++--- tests/unittests/test_datasource/test_opennebula.py | 4 ++-- tests/unittests/test_distros/test_netconfig.py | 4 ++-- tests/unittests/test_distros/test_resolv.py | 4 ++-- tests/unittests/test_distros/test_sysconfig.py | 4 ++-- .../test_distros/test_user_data_normalize.py | 7 +++--- .../test_handler/test_handler_apt_configure.py | 3 ++- .../test_handler/test_handler_ca_certs.py | 7 +++--- .../test_handler/test_handler_growpart.py | 3 ++- tests/unittests/test_pathprefix2dict.py | 6 +++--- tests/unittests/test_templating.py | 19 +++++++++++++++- tests/unittests/test_util.py | 15 ++++++------- tox.ini | 10 +++++++++ 21 files changed, 122 insertions(+), 55 deletions(-) (limited to 'tests/unittests/test_datasource/test_maas.py') diff --git a/cloudinit/util.py b/cloudinit/util.py index 32c19ba2..766f8e32 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2059,23 +2059,23 @@ def _read_dmi_syspath(key): Reads dmi data with from /sys/class/dmi/id """ - dmi_key = "{}/{}".format(DMI_SYS_PATH, key) - LOG.debug("querying dmi data {}".format(dmi_key)) + dmi_key = "{0}/{1}".format(DMI_SYS_PATH, key) + LOG.debug("querying dmi data {0}".format(dmi_key)) try: if not os.path.exists(dmi_key): - LOG.debug("did not find {}".format(dmi_key)) + LOG.debug("did not find {0}".format(dmi_key)) return None key_data = load_file(dmi_key) if not key_data: - LOG.debug("{} did not return any data".format(key)) + LOG.debug("{0} did not return any data".format(key)) return None - LOG.debug("dmi data {} returned {}".format(dmi_key, key_data)) + LOG.debug("dmi data {0} returned {0}".format(dmi_key, key_data)) return key_data.strip() except Exception as e: - logexc(LOG, "failed read of {}".format(dmi_key), e) + logexc(LOG, "failed read of {0}".format(dmi_key), e) return None @@ -2087,10 +2087,10 @@ def _call_dmidecode(key, dmidecode_path): try: cmd = [dmidecode_path, "--string", key] (result, _err) = subp(cmd) - LOG.debug("dmidecode returned '{}' for '{}'".format(result, key)) + LOG.debug("dmidecode returned '{0}' for '{0}'".format(result, key)) return result except OSError as _err: - LOG.debug('failed dmidecode cmd: {}\n{}'.format(cmd, _err.message)) + LOG.debug('failed dmidecode cmd: {0}\n{0}'.format(cmd, _err.message)) return None @@ -2106,7 +2106,7 @@ def read_dmi_data(key): if dmidecode_path: return _call_dmidecode(key, dmidecode_path) - LOG.warn("did not find either path {} or dmidecode command".format( + LOG.warn("did not find either path {0} or dmidecode command".format( DMI_SYS_PATH)) return None diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 828579e8..424d0626 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -39,8 +39,20 @@ else: PY3 = True if PY26: - # For now add these on, taken from python 2.7 + slightly adjusted + # For now add these on, taken from python 2.7 + slightly adjusted. Drop + # all this once Python 2.6 is dropped as a minimum requirement. class TestCase(unittest.TestCase): + def setUp(self): + unittest.TestCase.setUp(self) + self.__all_cleanups = ExitStack() + + def tearDown(self): + self.__all_cleanups.close() + unittest.TestCase.tearDown(self) + + def addCleanup(self, function, *args, **kws): + self.__all_cleanups.callback(function, *args, **kws) + def assertIs(self, expr1, expr2, msg=None): if expr1 is not expr2: standardMsg = '%r is not %r' % (expr1, expr2) @@ -63,6 +75,13 @@ if PY26: standardMsg = standardMsg % (value) self.fail(self._formatMessage(msg, standardMsg)) + def assertIsInstance(self, obj, cls, msg=None): + """Same as self.assertTrue(isinstance(obj, cls)), with a nicer + default message.""" + if not isinstance(obj, cls): + standardMsg = '%s is not an instance of %r' % (repr(obj), cls) + self.fail(self._formatMessage(msg, standardMsg)) + def assertDictContainsSubset(self, expected, actual, msg=None): missing = [] mismatched = [] @@ -126,9 +145,9 @@ def retarget_many_wrapper(new_base, am, old_func): return wrapper -class ResourceUsingTestCase(unittest.TestCase): +class ResourceUsingTestCase(TestCase): def setUp(self): - unittest.TestCase.setUp(self) + TestCase.setUp(self) self.resource_path = None def resourceLocation(self, subname=None): diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index f5dc3435..1a307e56 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -18,6 +18,8 @@ from cloudinit import settings from cloudinit import url_helper from cloudinit import util +from .helpers import TestCase + class FakeModule(handlers.Handler): def __init__(self): @@ -31,10 +33,10 @@ class FakeModule(handlers.Handler): pass -class TestWalkerHandleHandler(unittest.TestCase): +class TestWalkerHandleHandler(TestCase): def setUp(self): - unittest.TestCase.setUp(self) + super(TestWalkerHandleHandler, self).setUp() tmpdir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmpdir) diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/test_cs_util.py index 99fac84d..337ac9a0 100644 --- a/tests/unittests/test_cs_util.py +++ b/tests/unittests/test_cs_util.py @@ -1,7 +1,21 @@ +from __future__ import print_function + +import sys import unittest from cloudinit.cs_utils import Cepko +try: + skip = unittest.skip +except AttributeError: + # Python 2.6. Doesn't have to be high fidelity. + def skip(reason): + def decorator(func): + def wrapper(*args, **kws): + print(reason, file=sys.stderr) + return wrapper + return decorator + SERVER_CONTEXT = { "cpu": 1000, @@ -29,7 +43,7 @@ class CepkoMock(Cepko): # 2015-01-22 BAW: This test is completely useless because it only ever tests # the CepkoMock object. Even in its original form, I don't think it ever # touched the underlying Cepko class methods. -@unittest.skip('This test is completely useless') +@skip('This test is completely useless') class CepkoResultTests(unittest.TestCase): def setUp(self): pass diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 1f0330b3..97a53bee 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -1,7 +1,7 @@ from cloudinit import helpers from cloudinit.util import load_file from cloudinit.sources import DataSourceAzure -from ..helpers import populate_dir +from ..helpers import TestCase, populate_dir try: from unittest import mock @@ -84,9 +84,10 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): return content -class TestAzureDataSource(unittest.TestCase): +class TestAzureDataSource(TestCase): def setUp(self): + super(TestAzureDataSource, self).setUp() self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) @@ -416,7 +417,7 @@ class TestAzureDataSource(unittest.TestCase): load_file(os.path.join(self.waagent_d, 'ovf-env.xml'))) -class TestReadAzureOvf(unittest.TestCase): +class TestReadAzureOvf(TestCase): def test_invalid_xml_raises_non_azure_ds(self): invalid_xml = "" + construct_valid_ovf_env(data={}) self.assertRaises(DataSourceAzure.BrokenAzureDataSource, diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index 306ac7d8..772d189a 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -39,6 +39,7 @@ class CepkoMock(Cepko): class DataSourceCloudSigmaTest(test_helpers.TestCase): def setUp(self): + super(DataSourceCloudSigmaTest, self).setUp() self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "") self.datasource.is_running_in_cloudsigma = lambda: True self.datasource.cepko = CepkoMock(SERVER_CONTEXT) diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 258c68e2..fd930877 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -3,7 +3,6 @@ import json import os import shutil import tempfile -import unittest try: from unittest import mock @@ -20,6 +19,9 @@ from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit.sources.helpers import openstack from cloudinit import util +from ..helpers import TestCase + + PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' EC2_META = { 'ami-id': 'ami-00000001', @@ -70,7 +72,7 @@ CFG_DRIVE_FILES_V2 = { 'openstack/latest/user_data': USER_DATA} -class TestConfigDriveDataSource(unittest.TestCase): +class TestConfigDriveDataSource(TestCase): def setUp(self): super(TestConfigDriveDataSource, self).setUp() diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 6af0cd82..d25e1adc 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -2,11 +2,10 @@ from copy import copy import os import shutil import tempfile -import unittest from cloudinit.sources import DataSourceMAAS from cloudinit import url_helper -from ..helpers import populate_dir +from ..helpers import TestCase, populate_dir try: from unittest import mock @@ -14,7 +13,7 @@ except ImportError: import mock -class TestMAASDataSource(unittest.TestCase): +class TestMAASDataSource(TestCase): def setUp(self): super(TestMAASDataSource, self).setUp() diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 480a4012..4f967f58 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -1,7 +1,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNoCloud from cloudinit import util -from ..helpers import populate_dir +from ..helpers import TestCase, populate_dir import os import yaml @@ -19,9 +19,10 @@ except ImportError: from contextlib2 import ExitStack -class TestNoCloudDataSource(unittest.TestCase): +class TestNoCloudDataSource(TestCase): def setUp(self): + super(TestNoCloudDataSource, self).setUp() self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) self.paths = helpers.Paths({'cloud_dir': self.tmp}) @@ -34,8 +35,6 @@ class TestNoCloudDataSource(unittest.TestCase): self.mocks.enter_context( mock.patch.object(util, 'get_cmdline', return_value=self.cmdline)) - super(TestNoCloudDataSource, self).setUp() - def test_nocloud_seed_dir(self): md = {'instance-id': 'IID', 'dsmode': 'local'} ud = "USER_DATA_HERE" diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index ef534bab..e5a4bd18 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -1,7 +1,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceOpenNebula as ds from cloudinit import util -from ..helpers import populate_dir +from ..helpers import TestCase, populate_dir from base64 import b64encode import os @@ -46,7 +46,7 @@ CMD_IP_OUT = '''\ ''' -class TestOpenNebulaDataSource(unittest.TestCase): +class TestOpenNebulaDataSource(TestCase): parsed_user = None def setUp(self): diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 91e630ae..6d30c5b8 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -1,5 +1,4 @@ import os -import unittest try: from unittest import mock @@ -11,6 +10,7 @@ except ImportError: from contextlib2 import ExitStack from six import StringIO +from ..helpers import TestCase from cloudinit import distros from cloudinit import helpers @@ -80,7 +80,7 @@ class WriteBuffer(object): return self.buffer.getvalue() -class TestNetCfgDistro(unittest.TestCase): +class TestNetCfgDistro(TestCase): def _get_distro(self, dname): cls = distros.fetch(dname) diff --git a/tests/unittests/test_distros/test_resolv.py b/tests/unittests/test_distros/test_resolv.py index 779b83e3..faaf5b7f 100644 --- a/tests/unittests/test_distros/test_resolv.py +++ b/tests/unittests/test_distros/test_resolv.py @@ -1,7 +1,7 @@ from cloudinit.distros.parsers import resolv_conf import re -import unittest +from ..helpers import TestCase BASE_RESOLVE = ''' @@ -13,7 +13,7 @@ nameserver 10.15.30.92 BASE_RESOLVE = BASE_RESOLVE.strip() -class TestResolvHelper(unittest.TestCase): +class TestResolvHelper(TestCase): def test_parse_same(self): rp = resolv_conf.ResolvConf(BASE_RESOLVE) rp_r = str(rp).strip() diff --git a/tests/unittests/test_distros/test_sysconfig.py b/tests/unittests/test_distros/test_sysconfig.py index f66201b3..03d89a10 100644 --- a/tests/unittests/test_distros/test_sysconfig.py +++ b/tests/unittests/test_distros/test_sysconfig.py @@ -1,13 +1,13 @@ import re -import unittest from cloudinit.distros.parsers.sys_conf import SysConf +from ..helpers import TestCase # Lots of good examples @ # http://content.hccfl.edu/pollock/AUnix1/SysconfigFilesDesc.txt -class TestSysConfHelper(unittest.TestCase): +class TestSysConfHelper(TestCase): # This function was added in 2.7, make it work for 2.6 def assertRegMatches(self, text, regexp): regexp = re.compile(regexp) diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index b90d6185..e4488e2a 100644 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -1,9 +1,10 @@ -import unittest - from cloudinit import distros from cloudinit import helpers from cloudinit import settings +from ..helpers import TestCase + + bcfg = { 'name': 'bob', 'plain_text_passwd': 'ubuntu', @@ -15,7 +16,7 @@ bcfg = { } -class TestUGNormalize(unittest.TestCase): +class TestUGNormalize(TestCase): def _make_distro(self, dtype, def_user=None): cfg = dict(settings.CFG_BUILTIN) diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py index d72fa8c7..6bccff11 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure.py +++ b/tests/unittests/test_handler/test_handler_apt_configure.py @@ -1,6 +1,7 @@ from cloudinit import util from cloudinit.config import cc_apt_configure +from ..helpers import TestCase import os import re @@ -9,7 +10,7 @@ import tempfile import unittest -class TestAptProxyConfig(unittest.TestCase): +class TestAptProxyConfig(TestCase): def setUp(self): super(TestAptProxyConfig, self).setUp() self.tmp = tempfile.mkdtemp() diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index 97213a0c..a6b9c0fd 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -3,6 +3,7 @@ from cloudinit import helpers from cloudinit import util from cloudinit.config import cc_ca_certs +from ..helpers import TestCase import logging import shutil @@ -45,7 +46,7 @@ class TestNoConfig(unittest.TestCase): self.assertEqual(certs_mock.call_count, 0) -class TestConfig(unittest.TestCase): +class TestConfig(TestCase): def setUp(self): super(TestConfig, self).setUp() self.name = "ca-certs" @@ -139,7 +140,7 @@ class TestConfig(unittest.TestCase): self.assertEqual(self.mock_remove.call_count, 1) -class TestAddCaCerts(unittest.TestCase): +class TestAddCaCerts(TestCase): def setUp(self): super(TestAddCaCerts, self).setUp() @@ -241,7 +242,7 @@ class TestUpdateCaCerts(unittest.TestCase): ["update-ca-certificates"], capture=False) -class TestRemoveDefaultCaCerts(unittest.TestCase): +class TestRemoveDefaultCaCerts(TestCase): def setUp(self): super(TestRemoveDefaultCaCerts, self).setUp() diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index 89727863..bef0d80d 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -2,6 +2,7 @@ from cloudinit import cloud from cloudinit import util from cloudinit.config import cc_growpart +from ..helpers import TestCase import errno import logging @@ -72,7 +73,7 @@ class TestDisabled(unittest.TestCase): self.assertEqual(mockobj.call_count, 0) -class TestConfig(unittest.TestCase): +class TestConfig(TestCase): def setUp(self): super(TestConfig, self).setUp() self.name = "growpart" diff --git a/tests/unittests/test_pathprefix2dict.py b/tests/unittests/test_pathprefix2dict.py index 38a56dc2..d38260e6 100644 --- a/tests/unittests/test_pathprefix2dict.py +++ b/tests/unittests/test_pathprefix2dict.py @@ -1,15 +1,15 @@ from cloudinit import util -from .helpers import populate_dir +from .helpers import TestCase, populate_dir import shutil import tempfile -import unittest -class TestPathPrefix2Dict(unittest.TestCase): +class TestPathPrefix2Dict(TestCase): def setUp(self): + TestCase.setUp(self) self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index 957467f6..fbad405f 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -16,6 +16,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function + +import sys import six import unittest @@ -24,6 +27,20 @@ import textwrap from cloudinit import templater +try: + skipIf = unittest.skipIf +except AttributeError: + # Python 2.6. Doesn't have to be high fidelity. + def skipIf(condition, reason): + def decorator(func): + def wrapper(*args, **kws): + if condition: + return func(*args, **kws) + else: + print(reason, file=sys.stderr) + return wrapper + return decorator + class TestTemplates(test_helpers.TestCase): def test_render_basic(self): @@ -41,7 +58,7 @@ class TestTemplates(test_helpers.TestCase): out_data = templater.basic_render(in_data, {'b': 2}) self.assertEqual(expected_data.strip(), out_data) - @unittest.skipIf(six.PY3, 'Cheetah is not compatible with Python 3') + @skipIf(six.PY3, 'Cheetah is not compatible with Python 3') def test_detection(self): blob = "## template:cheetah" diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 7a224230..a1bd2c46 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -7,7 +7,6 @@ import shutil import tempfile from . import helpers -import unittest import six try: @@ -38,7 +37,7 @@ class FakeSelinux(object): self.restored.append(path) -class TestGetCfgOptionListOrStr(unittest.TestCase): +class TestGetCfgOptionListOrStr(helpers.TestCase): def test_not_found_no_default(self): """None is returned if key is not found and no default given.""" config = {} @@ -70,7 +69,7 @@ class TestGetCfgOptionListOrStr(unittest.TestCase): self.assertEqual([], result) -class TestWriteFile(unittest.TestCase): +class TestWriteFile(helpers.TestCase): def setUp(self): super(TestWriteFile, self).setUp() self.tmp = tempfile.mkdtemp() @@ -149,7 +148,7 @@ class TestWriteFile(unittest.TestCase): mockobj.assert_called_once_with('selinux') -class TestDeleteDirContents(unittest.TestCase): +class TestDeleteDirContents(helpers.TestCase): def setUp(self): super(TestDeleteDirContents, self).setUp() self.tmp = tempfile.mkdtemp() @@ -215,20 +214,20 @@ class TestDeleteDirContents(unittest.TestCase): self.assertDirEmpty(self.tmp) -class TestKeyValStrings(unittest.TestCase): +class TestKeyValStrings(helpers.TestCase): def test_keyval_str_to_dict(self): expected = {'1': 'one', '2': 'one+one', 'ro': True} cmdline = "1=one ro 2=one+one" self.assertEqual(expected, util.keyval_str_to_dict(cmdline)) -class TestGetCmdline(unittest.TestCase): +class TestGetCmdline(helpers.TestCase): def test_cmdline_reads_debug_env(self): os.environ['DEBUG_PROC_CMDLINE'] = 'abcd 123' self.assertEqual(os.environ['DEBUG_PROC_CMDLINE'], util.get_cmdline()) -class TestLoadYaml(unittest.TestCase): +class TestLoadYaml(helpers.TestCase): mydefault = "7b03a8ebace993d806255121073fed52" def test_simple(self): @@ -335,7 +334,7 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): self._patchIn(new_root) util.ensure_dir(os.path.join('sys', 'class', 'dmi', 'id')) - dmi_key = "/sys/class/dmi/id/{}".format(key) + dmi_key = "/sys/class/dmi/id/{0}".format(key) util.write_file(dmi_key, content) def _no_syspath(self, key, content): diff --git a/tox.ini b/tox.ini index e547c693..d04cd47c 100644 --- a/tox.ini +++ b/tox.ini @@ -11,3 +11,13 @@ deps = nose pep8==1.5.7 pyflakes + +[testenv:py26] +commands = nosetests tests +deps = + contextlib2 + httpretty>=0.7.1 + mock + nose + pep8==1.5.7 + pyflakes -- cgit v1.2.3 From 8cd5d7b143f882d80d45b1c04bdde1949846d4f1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 25 Feb 2015 19:40:33 -0500 Subject: move towards user-data being binary UrlResponse: biggest change... make readurl return bytes, making user know what to do with it. util: add load_tfile_or_url for loading text file or url as read_file_or_url now returns bytes ec2_utils: all meta-data is text, remove non-obvious string translations DigitalOcean: adjust for ec2_utils DataSourceGCE, DataSourceMAAS: user-data is binary other fields are text. openstack.py: read paths without decoding to text. This is ok as paths other than user-data are json, and load_json will handle load_file still returns text, and that is what most things use. --- cloudinit/ec2_utils.py | 14 +++++++++++--- cloudinit/sources/DataSourceDigitalOcean.py | 8 ++++++-- cloudinit/sources/DataSourceGCE.py | 21 ++++++++++++--------- cloudinit/sources/DataSourceMAAS.py | 14 +++++++++++--- cloudinit/sources/helpers/openstack.py | 2 +- cloudinit/url_helper.py | 2 +- cloudinit/util.py | 11 ++++++++--- tests/unittests/helpers.py | 5 ++++- tests/unittests/test_datasource/test_configdrive.py | 15 ++++++++++----- tests/unittests/test_datasource/test_gce.py | 2 +- tests/unittests/test_datasource/test_maas.py | 8 ++++---- tests/unittests/test_datasource/test_nocloud.py | 14 +++++++------- tests/unittests/test_datasource/test_openstack.py | 6 +++--- tests/unittests/test_ec2_util.py | 2 +- .../test_handler/test_handler_apt_configure.py | 12 ++++++------ tests/unittests/test_pathprefix2dict.py | 10 +++++----- 16 files changed, 91 insertions(+), 55 deletions(-) (limited to 'tests/unittests/test_datasource/test_maas.py') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index e1ed4091..7cf99186 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -41,6 +41,10 @@ class MetadataLeafDecoder(object): def __call__(self, field, blob): if not blob: return blob + try: + blob = util.decode_binary(blob) + except UnicodeDecodeError: + return blob if self._maybe_json_object(blob): try: # Assume it's json, unless it fails parsing... @@ -69,6 +73,8 @@ class MetadataMaterializer(object): def _parse(self, blob): leaves = {} children = [] + blob = util.decode_binary(blob) + if not blob: return (leaves, children) @@ -117,12 +123,12 @@ class MetadataMaterializer(object): child_url = url_helper.combine_url(base_url, c) if not child_url.endswith("/"): child_url += "/" - child_blob = str(self._caller(child_url)) + child_blob = self._caller(child_url) child_contents[c] = self._materialize(child_blob, child_url) leaf_contents = {} for (field, resource) in leaves.items(): leaf_url = url_helper.combine_url(base_url, resource) - leaf_blob = self._caller(leaf_url).contents + leaf_blob = self._caller(leaf_url) leaf_contents[field] = self._leaf_decoder(field, leaf_blob) joined = {} joined.update(child_contents) @@ -179,11 +185,13 @@ def get_instance_metadata(api_version='latest', caller = functools.partial(util.read_file_or_url, ssl_details=ssl_details, timeout=timeout, retries=retries) + def mcaller(url): + return caller(url).contents try: response = caller(md_url) materializer = MetadataMaterializer(response.contents, - md_url, caller, + md_url, mcaller, leaf_decoder=leaf_decoder) md = materializer.materialize() if not isinstance(md, (dict)): diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index 76ddaa9d..5d47564d 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -54,9 +54,13 @@ class DataSourceDigitalOcean(sources.DataSource): def get_data(self): caller = functools.partial(util.read_file_or_url, timeout=self.timeout, retries=self.retries) - md = ec2_utils.MetadataMaterializer(str(caller(self.metadata_address)), + + def mcaller(url): + return caller(url).contents + + md = ec2_utils.MetadataMaterializer(mcaller(self.metadata_address), base_url=self.metadata_address, - caller=caller) + caller=mcaller) self.metadata = md.materialize() diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 6936c74e..608c07f1 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -53,15 +53,15 @@ class DataSourceGCE(sources.DataSource): # GCE metadata server requires a custom header since v1 headers = {'X-Google-Metadata-Request': True} - # url_map: (our-key, path, required) + # url_map: (our-key, path, required, is_text) url_map = [ - ('instance-id', 'instance/id', True), - ('availability-zone', 'instance/zone', True), - ('local-hostname', 'instance/hostname', True), - ('public-keys', 'project/attributes/sshKeys', False), - ('user-data', 'instance/attributes/user-data', False), + ('instance-id', 'instance/id', True, True), + ('availability-zone', 'instance/zone', True, True), + ('local-hostname', 'instance/hostname', True, True), + ('public-keys', 'project/attributes/sshKeys', False, True), + ('user-data', 'instance/attributes/user-data', False, False), ('user-data-encoding', 'instance/attributes/user-data-encoding', - False), + False, True), ] # if we cannot resolve the metadata server, then no point in trying @@ -71,13 +71,16 @@ class DataSourceGCE(sources.DataSource): # iterate over url_map keys to get metadata items found = False - for (mkey, path, required) in url_map: + for (mkey, path, required, is_text) in url_map: try: resp = url_helper.readurl(url=self.metadata_address + path, headers=headers) if resp.code == 200: found = True - self.metadata[mkey] = resp.contents + if is_text: + self.metadata[mkey] = util.decode_binary(resp.contents) + else: + self.metadata[mkey] = resp.contents else: if required: msg = "required url %s returned code %s. not GCE" diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 082cc58f..35c5b5e1 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -36,6 +36,8 @@ from cloudinit import util LOG = logging.getLogger(__name__) MD_VERSION = "2012-03-01" +BINARY_FIELDS = ('user-data',) + class DataSourceMAAS(sources.DataSource): """ @@ -185,7 +187,9 @@ def read_maas_seed_dir(seed_d): md = {} for fname in files: try: - md[fname] = util.load_file(os.path.join(seed_d, fname)) + print("fname: %s / %s" % (fname, fname not in BINARY_FIELDS)) + md[fname] = util.load_file(os.path.join(seed_d, fname), + decode=fname not in BINARY_FIELDS) except IOError as e: if e.errno != errno.ENOENT: raise @@ -218,6 +222,7 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, 'public-keys': "%s/%s" % (base_url, 'meta-data/public-keys'), 'user-data': "%s/%s" % (base_url, 'user-data'), } + md = {} for name in file_order: url = files.get(name) @@ -238,7 +243,10 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, timeout=timeout, ssl_details=ssl_details) if resp.ok(): - md[name] = str(resp) + if name in BINARY_FIELDS: + md[name] = resp.contents + else: + md[name] = util.decode_binary(resp.contents) else: LOG.warn(("Fetching from %s resulted in" " an invalid http code %s"), url, resp.code) @@ -263,7 +271,7 @@ def check_seed_contents(content, seed): if len(missing): raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing)) - userdata = content.get('user-data', "") + userdata = content.get('user-data', b"") md = {} for (key, val) in content.items(): if key == 'user-data': diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 88c7a198..bd93d22f 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -327,7 +327,7 @@ class ConfigDriveReader(BaseReader): return os.path.join(*components) def _path_read(self, path): - return util.load_file(path) + return util.load_file(path, decode=False) def _fetch_available_versions(self): if self._versions is None: diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 62001dff..2d81a062 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -119,7 +119,7 @@ class UrlResponse(object): @property def contents(self): - return self._response.text + return self._response.content @property def url(self): diff --git a/cloudinit/util.py b/cloudinit/util.py index 4fbdf0a9..efbc3c8d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -739,6 +739,10 @@ def fetch_ssl_details(paths=None): return ssl_details +def load_tfile_or_url(*args, **kwargs): + return(decode_binary(read_file_or_url(*args, **kwargs).contents)) + + def read_file_or_url(url, timeout=5, retries=10, headers=None, data=None, sec_between=1, ssl_details=None, headers_cb=None, exception_cb=None): @@ -750,7 +754,7 @@ def read_file_or_url(url, timeout=5, retries=10, LOG.warn("Unable to post data to file resource %s", url) file_path = url[len("file://"):] try: - contents = load_file(file_path) + contents = load_file(file_path, decode=False) except IOError as e: code = e.errno if e.errno == errno.ENOENT: @@ -806,7 +810,7 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): ud_url = "%s%s%s" % (base, "user-data", ext) md_url = "%s%s%s" % (base, "meta-data", ext) - md_resp = read_file_or_url(md_url, timeout, retries, file_retries) + md_resp = load_tfile_or_url(md_url, timeout, retries, file_retries) md = None if md_resp.ok(): md = load_yaml(md_resp.contents, default={}) @@ -815,6 +819,7 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): ud = None if ud_resp.ok(): ud = ud_resp.contents + print("returning %s (%s)" % (ud_resp.contents.__class__, ud_resp.contents)) return (md, ud) @@ -2030,7 +2035,7 @@ def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep): ret = {} for f in required + optional: try: - ret[f] = load_file(base + delim + f, quiet=False) + ret[f] = load_file(base + delim + f, quiet=False, decode=False) except IOError as e: if e.errno != errno.ENOENT: raise diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 7516bd02..24e1e881 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -288,7 +288,10 @@ def populate_dir(path, files): os.makedirs(path) for (name, content) in files.items(): with open(os.path.join(path, name), "wb") as fp: - fp.write(content.encode('utf-8')) + if isinstance(content, six.binary_type): + fp.write(content) + else: + fp.write(content.encode('utf-8')) fp.close() diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index e28bdd84..83aca505 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -2,6 +2,7 @@ from copy import copy import json import os import shutil +import six import tempfile try: @@ -45,7 +46,7 @@ EC2_META = { 'reservation-id': 'r-iru5qm4m', 'security-groups': ['default'] } -USER_DATA = '#!/bin/sh\necho This is user data\n' +USER_DATA = b'#!/bin/sh\necho This is user data\n' OSTACK_META = { 'availability_zone': 'nova', 'files': [{'content_path': '/content/0000', 'path': '/etc/foo.cfg'}, @@ -56,8 +57,8 @@ OSTACK_META = { 'public_keys': {'mykey': PUBKEY}, 'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c'} -CONTENT_0 = 'This is contents of /etc/foo.cfg\n' -CONTENT_1 = '# this is /etc/bar/bar.cfg\n' +CONTENT_0 = b'This is contents of /etc/foo.cfg\n' +CONTENT_1 = b'# this is /etc/bar/bar.cfg\n' CFG_DRIVE_FILES_V2 = { 'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META), @@ -346,8 +347,12 @@ def populate_dir(seed_dir, files): dirname = os.path.dirname(path) if not os.path.isdir(dirname): os.makedirs(dirname) - with open(path, "w") as fp: + if isinstance(content, six.text_type): + mode = "w" + else: + mode = "wb" + + with open(path, mode) as fp: fp.write(content) - fp.close() # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 6dd4b5ed..d28f3b08 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -32,7 +32,7 @@ GCE_META = { 'instance/zone': 'foo/bar', 'project/attributes/sshKeys': 'user:ssh-rsa AA2..+aRD0fyVw== root@server', 'instance/hostname': 'server.project-foo.local', - 'instance/attributes/user-data': '/bin/echo foo\n', + 'instance/attributes/user-data': b'/bin/echo foo\n', } GCE_META_PARTIAL = { diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index d25e1adc..f109bb04 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -26,7 +26,7 @@ class TestMAASDataSource(TestCase): data = {'instance-id': 'i-valid01', 'local-hostname': 'valid01-hostname', - 'user-data': 'valid01-userdata', + 'user-data': b'valid01-userdata', 'public-keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname'} my_d = os.path.join(self.tmp, "valid") @@ -46,7 +46,7 @@ class TestMAASDataSource(TestCase): data = {'instance-id': 'i-valid-extra', 'local-hostname': 'valid-extra-hostname', - 'user-data': 'valid-extra-userdata', 'foo': 'bar'} + 'user-data': b'valid-extra-userdata', 'foo': 'bar'} my_d = os.path.join(self.tmp, "valid_extra") populate_dir(my_d, data) @@ -103,7 +103,7 @@ class TestMAASDataSource(TestCase): 'meta-data/instance-id': 'i-instanceid', 'meta-data/local-hostname': 'test-hostname', 'meta-data/public-keys': 'test-hostname', - 'user-data': 'foodata', + 'user-data': b'foodata', } valid_order = [ 'meta-data/local-hostname', @@ -143,7 +143,7 @@ class TestMAASDataSource(TestCase): userdata, metadata = DataSourceMAAS.read_maas_seed_url( my_seed, header_cb=my_headers_cb, version=my_ver) - self.assertEqual("foodata", userdata) + self.assertEqual(b"foodata", userdata) self.assertEqual(metadata['instance-id'], valid['meta-data/instance-id']) self.assertEqual(metadata['local-hostname'], diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 4f967f58..85b4c25a 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -37,7 +37,7 @@ class TestNoCloudDataSource(TestCase): def test_nocloud_seed_dir(self): md = {'instance-id': 'IID', 'dsmode': 'local'} - ud = "USER_DATA_HERE" + ud = b"USER_DATA_HERE" populate_dir(os.path.join(self.paths.seed_dir, "nocloud"), {'user-data': ud, 'meta-data': yaml.safe_dump(md)}) @@ -92,20 +92,20 @@ class TestNoCloudDataSource(TestCase): data = { 'fs_label': None, 'meta-data': yaml.safe_dump({'instance-id': 'IID'}), - 'user-data': "USER_DATA_RAW", + 'user-data': b"USER_DATA_RAW", } sys_cfg = {'datasource': {'NoCloud': data}} dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) ret = dsrc.get_data() - self.assertEqual(dsrc.userdata_raw, "USER_DATA_RAW") + self.assertEqual(dsrc.userdata_raw, b"USER_DATA_RAW") self.assertEqual(dsrc.metadata.get('instance-id'), 'IID') self.assertTrue(ret) def test_nocloud_seed_with_vendordata(self): md = {'instance-id': 'IID', 'dsmode': 'local'} - ud = "USER_DATA_HERE" - vd = "THIS IS MY VENDOR_DATA" + ud = b"USER_DATA_HERE" + vd = b"THIS IS MY VENDOR_DATA" populate_dir(os.path.join(self.paths.seed_dir, "nocloud"), {'user-data': ud, 'meta-data': yaml.safe_dump(md), @@ -126,7 +126,7 @@ class TestNoCloudDataSource(TestCase): def test_nocloud_no_vendordata(self): populate_dir(os.path.join(self.paths.seed_dir, "nocloud"), - {'user-data': "ud", 'meta-data': "instance-id: IID\n"}) + {'user-data': b"ud", 'meta-data': "instance-id: IID\n"}) sys_cfg = {'datasource': {'NoCloud': {'fs_label': None}}} @@ -134,7 +134,7 @@ class TestNoCloudDataSource(TestCase): dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) ret = dsrc.get_data() - self.assertEqual(dsrc.userdata_raw, "ud") + self.assertEqual(dsrc.userdata_raw, b"ud") self.assertFalse(dsrc.vendordata) self.assertTrue(ret) diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 81ef1546..81411ced 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -49,7 +49,7 @@ EC2_META = { 'public-ipv4': '0.0.0.1', 'reservation-id': 'r-iru5qm4m', } -USER_DATA = '#!/bin/sh\necho This is user data\n' +USER_DATA = b'#!/bin/sh\necho This is user data\n' VENDOR_DATA = { 'magic': '', } @@ -63,8 +63,8 @@ OSTACK_META = { 'public_keys': {'mykey': PUBKEY}, 'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c', } -CONTENT_0 = 'This is contents of /etc/foo.cfg\n' -CONTENT_1 = '# this is /etc/bar/bar.cfg\n' +CONTENT_0 = b'This is contents of /etc/foo.cfg\n' +CONTENT_1 = b'# this is /etc/bar/bar.cfg\n' OS_FILES = { 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), 'openstack/latest/user_data': USER_DATA, diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index 84aa002e..bd43accf 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -16,7 +16,7 @@ class TestEc2Util(helpers.HttprettyTestCase): body='stuff', status=200) userdata = eu.get_instance_userdata(self.VERSION) - self.assertEquals('stuff', userdata) + self.assertEquals('stuff', userdata.decode('utf-8')) @hp.activate def test_userdata_fetch_fail_not_found(self): diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py index d8fe9a4f..02cad8b2 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure.py +++ b/tests/unittests/test_handler/test_handler_apt_configure.py @@ -30,7 +30,7 @@ class TestAptProxyConfig(TestCase): self.assertTrue(os.path.isfile(self.pfile)) self.assertFalse(os.path.isfile(self.cfile)) - contents = str(util.read_file_or_url(self.pfile)) + contents = util.load_tfile_or_url(self.pfile) self.assertTrue(self._search_apt_config(contents, "http", "myproxy")) def test_apt_http_proxy_written(self): @@ -40,7 +40,7 @@ class TestAptProxyConfig(TestCase): self.assertTrue(os.path.isfile(self.pfile)) self.assertFalse(os.path.isfile(self.cfile)) - contents = str(util.read_file_or_url(self.pfile)) + contents = util.load_tfile_or_url(self.pfile) self.assertTrue(self._search_apt_config(contents, "http", "myproxy")) def test_apt_all_proxy_written(self): @@ -58,7 +58,7 @@ class TestAptProxyConfig(TestCase): self.assertTrue(os.path.isfile(self.pfile)) self.assertFalse(os.path.isfile(self.cfile)) - contents = str(util.read_file_or_url(self.pfile)) + contents = util.load_tfile_or_url(self.pfile) for ptype, pval in values.items(): self.assertTrue(self._search_apt_config(contents, ptype, pval)) @@ -74,7 +74,7 @@ class TestAptProxyConfig(TestCase): cc_apt_configure.apply_apt_config({'apt_proxy': "foo"}, self.pfile, self.cfile) self.assertTrue(os.path.isfile(self.pfile)) - contents = str(util.read_file_or_url(self.pfile)) + contents = util.load_tfile_or_url(self.pfile) self.assertTrue(self._search_apt_config(contents, "http", "foo")) def test_config_written(self): @@ -86,14 +86,14 @@ class TestAptProxyConfig(TestCase): self.assertTrue(os.path.isfile(self.cfile)) self.assertFalse(os.path.isfile(self.pfile)) - self.assertEqual(str(util.read_file_or_url(self.cfile)), payload) + self.assertEqual(util.load_tfile_or_url(self.cfile), payload) def test_config_replaced(self): util.write_file(self.pfile, "content doesnt matter") cc_apt_configure.apply_apt_config({'apt_config': "foo"}, self.pfile, self.cfile) self.assertTrue(os.path.isfile(self.cfile)) - self.assertEqual(str(util.read_file_or_url(self.cfile)), "foo") + self.assertEqual(util.load_tfile_or_url(self.cfile), "foo") def test_config_deleted(self): # if no 'apt_config' is provided, delete any previously written file diff --git a/tests/unittests/test_pathprefix2dict.py b/tests/unittests/test_pathprefix2dict.py index 7089bde6..38fd75b6 100644 --- a/tests/unittests/test_pathprefix2dict.py +++ b/tests/unittests/test_pathprefix2dict.py @@ -14,28 +14,28 @@ class TestPathPrefix2Dict(TestCase): self.addCleanup(shutil.rmtree, self.tmp) def test_required_only(self): - dirdata = {'f1': 'f1content', 'f2': 'f2content'} + dirdata = {'f1': b'f1content', 'f2': b'f2content'} populate_dir(self.tmp, dirdata) ret = util.pathprefix2dict(self.tmp, required=['f1', 'f2']) self.assertEqual(dirdata, ret) def test_required_missing(self): - dirdata = {'f1': 'f1content'} + dirdata = {'f1': b'f1content'} populate_dir(self.tmp, dirdata) kwargs = {'required': ['f1', 'f2']} self.assertRaises(ValueError, util.pathprefix2dict, self.tmp, **kwargs) def test_no_required_and_optional(self): - dirdata = {'f1': 'f1c', 'f2': 'f2c'} + dirdata = {'f1': b'f1c', 'f2': b'f2c'} populate_dir(self.tmp, dirdata) ret = util.pathprefix2dict(self.tmp, required=None, - optional=['f1', 'f2']) + optional=['f1', 'f2']) self.assertEqual(dirdata, ret) def test_required_and_optional(self): - dirdata = {'f1': 'f1c', 'f2': 'f2c'} + dirdata = {'f1': b'f1c', 'f2': b'f2c'} populate_dir(self.tmp, dirdata) ret = util.pathprefix2dict(self.tmp, required=['f1'], optional=['f2']) -- cgit v1.2.3 From 48cb8699efb5c6116dfa7b4d76d0a5fb6b3fbbbf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 7 Aug 2015 00:22:49 -0500 Subject: hopefully fix DataSourceMAAS --- cloudinit/sources/DataSourceMAAS.py | 58 +++++++++++----------------- tests/unittests/test_datasource/test_maas.py | 2 +- 2 files changed, 24 insertions(+), 36 deletions(-) (limited to 'tests/unittests/test_datasource/test_maas.py') diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 279da238..2f36bbe2 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -164,12 +164,12 @@ def read_maas_seed_dir(seed_d): return check_seed_contents(md, seed_d) -def read_maas_seed_url(seed_url, header_cb=None, timeout=None, +def read_maas_seed_url(seed_url, read_file_or_url=None, timeout=None, version=MD_VERSION, paths=None): """ Read the maas datasource at seed_url. - - header_cb is a method that should return a headers dictionary for - a given url + read_file_or_url is a method that should provide an interface + like util.read_file_or_url Expected format of seed_url is are the following files: * //meta-data/instance-id @@ -190,14 +190,12 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, 'user-data': "%s/%s" % (base_url, 'user-data'), } + if read_file_or_url is None: + read_file_or_url = util.read_file_or_url + md = {} for name in file_order: url = files.get(name) - if not header_cb: - def _cb(url): - return {} - header_cb = _cb - if name == 'user-data': retries = 0 else: @@ -205,10 +203,8 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, try: ssl_details = util.fetch_ssl_details(paths) - resp = util.read_file_or_url(url, retries=retries, - headers_cb=header_cb, - timeout=timeout, - ssl_details=ssl_details) + resp = read_file_or_url(url, retries=retries, + timeout=timeout, ssl_details=ssl_details) if resp.ok(): if name in BINARY_FIELDS: md[name] = resp.contents @@ -311,47 +307,39 @@ if __name__ == "__main__": if key in cfg and creds[key] is None: creds[key] = cfg[key] - def geturl(url, headers_cb): - req = Request(url, data=None, headers=headers_cb(url)) - return urlopen(req).read() + oauth_helper = url_helper.OauthUrlHelper(**creds) + + def geturl(url): + return oauth_helper.readurl(url).contents def printurl(url, headers_cb): - print("== %s ==\n%s\n" % (url, geturl(url, headers_cb))) + print("== %s ==\n%s\n" % (url, geturl(url))) - def crawl(url, headers_cb=None): + def crawl(url): if url.endswith("/"): - for line in geturl(url, headers_cb).splitlines(): + for line in geturl(url).splitlines(): if line.endswith("/"): - crawl("%s%s" % (url, line), headers_cb) + crawl("%s%s" % (url, line)) else: - printurl("%s%s" % (url, line), headers_cb) + printurl("%s%s" % (url, line)) else: - printurl(url, headers_cb) - - def my_headers(url): - headers = {} - if creds.get('consumer_key', None) is not None: - headers = oauth_headers(url, **creds) - return headers + printurl(url) if args.subcmd == "check-seed": - if args.url.startswith("http"): - (userdata, metadata) = read_maas_seed_url(args.url, - header_cb=my_headers, - version=args.apiver) - else: - (userdata, metadata) = read_maas_seed_url(args.url) + (userdata, metadata) = read_maas_seed_url( + args.url, read_file_or_url=oauth_helper.read_file_or_url, + version=args.apiver) print("=== userdata ===") print(userdata) print("=== metadata ===") pprint.pprint(metadata) elif args.subcmd == "get": - printurl(args.url, my_headers) + printurl(args.url) elif args.subcmd == "crawl": if not args.url.endswith("/"): args.url = "%s/" % args.url - crawl(args.url, my_headers) + crawl(args.url) main() diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index f109bb04..eb97b692 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -141,7 +141,7 @@ class TestMAASDataSource(TestCase): with mock.patch.object(url_helper, 'readurl', side_effect=side_effect()) as mockobj: userdata, metadata = DataSourceMAAS.read_maas_seed_url( - my_seed, header_cb=my_headers_cb, version=my_ver) + my_seed, version=my_ver) self.assertEqual(b"foodata", userdata) self.assertEqual(metadata['instance-id'], -- cgit v1.2.3 From 8cb7c3f7b5339e686bfbf95996b51afafeaf9c9e Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 3 Mar 2016 16:20:10 -0600 Subject: Update pep8 runner and fix pep8 issues --- Makefile | 9 ++-- bin/cloud-init | 43 +++++++++--------- cloudinit/config/cc_apt_configure.py | 6 ++- cloudinit/config/cc_disk_setup.py | 31 +++++++------ cloudinit/config/cc_grub_dpkg.py | 8 ++-- cloudinit/config/cc_keys_to_console.py | 2 +- cloudinit/config/cc_lxd.py | 2 +- cloudinit/config/cc_mounts.py | 12 ++--- cloudinit/config/cc_power_state_change.py | 2 +- cloudinit/config/cc_puppet.py | 6 +-- cloudinit/config/cc_resizefs.py | 2 +- cloudinit/config/cc_rh_subscription.py | 4 +- cloudinit/config/cc_set_hostname.py | 2 +- cloudinit/config/cc_ssh.py | 7 +-- cloudinit/config/cc_update_etc_hosts.py | 6 +-- cloudinit/config/cc_update_hostname.py | 2 +- cloudinit/config/cc_yum_add_repo.py | 2 +- cloudinit/distros/__init__.py | 12 ++--- cloudinit/distros/arch.py | 6 +-- cloudinit/distros/debian.py | 5 ++- cloudinit/distros/freebsd.py | 4 +- cloudinit/distros/gentoo.py | 4 +- cloudinit/distros/parsers/hostname.py | 2 +- cloudinit/distros/parsers/resolv_conf.py | 2 +- cloudinit/distros/parsers/sys_conf.py | 7 ++- cloudinit/filters/launch_index.py | 2 +- cloudinit/helpers.py | 7 +-- cloudinit/sources/DataSourceAzure.py | 21 +++++---- cloudinit/sources/DataSourceConfigDrive.py | 2 +- cloudinit/sources/DataSourceEc2.py | 10 ++--- cloudinit/sources/DataSourceMAAS.py | 15 ++++--- cloudinit/sources/DataSourceOVF.py | 4 +- cloudinit/sources/DataSourceOpenNebula.py | 3 +- cloudinit/sources/DataSourceSmartOS.py | 7 ++- cloudinit/ssh_util.py | 3 +- cloudinit/stages.py | 18 ++++---- cloudinit/url_helper.py | 6 +-- cloudinit/util.py | 15 ++++--- tests/unittests/test_data.py | 5 ++- tests/unittests/test_datasource/test_altcloud.py | 23 +++++----- tests/unittests/test_datasource/test_azure.py | 15 ++++--- .../unittests/test_datasource/test_configdrive.py | 12 ++--- tests/unittests/test_datasource/test_maas.py | 16 +++---- tests/unittests/test_datasource/test_smartos.py | 6 +-- .../test_handler/test_handler_power_state.py | 3 +- .../test_handler/test_handler_seed_random.py | 3 +- .../unittests/test_handler/test_handler_snappy.py | 3 +- tests/unittests/test_sshutil.py | 3 +- tests/unittests/test_templating.py | 3 +- tools/hacking.py | 16 +++---- tools/mock-meta.py | 27 +++++++----- tools/run-pep8 | 51 ++++++++-------------- 52 files changed, 244 insertions(+), 243 deletions(-) (limited to 'tests/unittests/test_datasource/test_maas.py') diff --git a/Makefile b/Makefile index 058ac199..fb65b70b 100644 --- a/Makefile +++ b/Makefile @@ -20,13 +20,14 @@ all: test check_version check: pep8 pyflakes pyflakes3 unittest pep8: - @$(CWD)/tools/run-pep8 $(PY_FILES) + @$(CWD)/tools/run-pep8 pyflakes: - @$(CWD)/tools/tox-venv py27 pyflakes $(PY_FILES) + @$(CWD)/tools/run-pyflakes -pyflakes: - @$(CWD)/tools/tox-venv py34 pyflakes $(PY_FILES) +pyflakes3: + @$(CWD)/tools/run-pyflakes3 + unittest: nosetests $(noseopts) tests/unittests diff --git a/bin/cloud-init b/bin/cloud-init index 9b90c45e..7f665e7e 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -194,7 +194,7 @@ def main_init(name, args): if args.debug: # Reset so that all the debug handlers are closed out LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) + " longer be active shortly")) logging.resetLogging() logging.setupLogging(init.cfg) apply_reporting_cfg(init.cfg) @@ -276,9 +276,9 @@ def main_init(name, args): # This may run user-data handlers and/or perform # url downloads and such as needed. (ran, _results) = init.cloudify().run('consume_data', - init.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE) + init.consume_data, + args=[PER_INSTANCE], + freq=PER_INSTANCE) if not ran: # Just consume anything that is set to run per-always # if nothing ran in the per-instance code @@ -349,7 +349,7 @@ def main_modules(action_name, args): if args.debug: # Reset so that all the debug handlers are closed out LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) + " longer be active shortly")) logging.resetLogging() logging.setupLogging(mods.cfg) apply_reporting_cfg(init.cfg) @@ -534,7 +534,8 @@ def status_wrapper(name, args, data_d=None, link_d=None): errors.extend(v1[m].get('errors', [])) atomic_write_json(result_path, - {'v1': {'datasource': v1['datasource'], 'errors': errors}}) + {'v1': {'datasource': v1['datasource'], + 'errors': errors}}) util.sym_link(os.path.relpath(result_path, link_d), result_link, force=True) @@ -578,13 +579,13 @@ def main(): # These settings are used for the 'config' and 'final' stages parser_mod = subparsers.add_parser('modules', - help=('activates modules ' - 'using a given configuration key')) + help=('activates modules using ' + 'a given configuration key')) parser_mod.add_argument("--mode", '-m', action='store', - help=("module configuration name " - "to use (default: %(default)s)"), - default='config', - choices=('init', 'config', 'final')) + help=("module configuration name " + "to use (default: %(default)s)"), + default='config', + choices=('init', 'config', 'final')) parser_mod.set_defaults(action=('modules', main_modules)) # These settings are used when you want to query information @@ -600,22 +601,22 @@ def main(): # This subcommand allows you to run a single module parser_single = subparsers.add_parser('single', - help=('run a single module ')) + help=('run a single module ')) parser_single.set_defaults(action=('single', main_single)) parser_single.add_argument("--name", '-n', action="store", - help="module name to run", - required=True) + help="module name to run", + required=True) parser_single.add_argument("--frequency", action="store", - help=("frequency of the module"), - required=False, - choices=list(FREQ_SHORT_NAMES.keys())) + help=("frequency of the module"), + required=False, + choices=list(FREQ_SHORT_NAMES.keys())) parser_single.add_argument("--report", action="store_true", help="enable reporting", required=False) parser_single.add_argument("module_args", nargs="*", - metavar='argument', - help=('any additional arguments to' - ' pass to this module')) + metavar='argument', + help=('any additional arguments to' + ' pass to this module')) parser_single.set_defaults(action=('single', main_single)) args = parser.parse_args() diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 9e9e9e26..702977cb 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -91,7 +91,8 @@ def handle(name, cfg, cloud, log, _args): if matchcfg: matcher = re.compile(matchcfg).search else: - matcher = lambda f: False + def matcher(x): + return False errors = add_sources(cfg['apt_sources'], params, aa_repo_match=matcher) @@ -173,7 +174,8 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): template_params = {} if aa_repo_match is None: - aa_repo_match = lambda f: False + def aa_repo_match(x): + return False errorlist = [] for ent in srclist: diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index d5b0d1d7..0ecc2e4c 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -167,11 +167,12 @@ def enumerate_disk(device, nodeps=False): parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0] for part in parts: - d = {'name': None, - 'type': None, - 'fstype': None, - 'label': None, - } + d = { + 'name': None, + 'type': None, + 'fstype': None, + 'label': None, + } for key, value in value_splitter(part): d[key.lower()] = value @@ -701,11 +702,12 @@ def lookup_force_flag(fs): """ A force flag might be -F or -F, this look it up """ - flags = {'ext': '-F', - 'btrfs': '-f', - 'xfs': '-f', - 'reiserfs': '-f', - } + flags = { + 'ext': '-F', + 'btrfs': '-f', + 'xfs': '-f', + 'reiserfs': '-f', + } if 'ext' in fs.lower(): fs = 'ext' @@ -824,10 +826,11 @@ def mkfs(fs_cfg): # Create the commands if fs_cmd: - fs_cmd = fs_cfg['cmd'] % {'label': label, - 'filesystem': fs_type, - 'device': device, - } + fs_cmd = fs_cfg['cmd'] % { + 'label': label, + 'filesystem': fs_type, + 'device': device, + } else: # Find the mkfs command mkfs_cmd = util.which("mkfs.%s" % fs_type) diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index 456597af..acd3e60a 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -38,11 +38,11 @@ def handle(name, cfg, _cloud, log, _args): idevs = util.get_cfg_option_str(mycfg, "grub-pc/install_devices", None) idevs_empty = util.get_cfg_option_str(mycfg, - "grub-pc/install_devices_empty", None) + "grub-pc/install_devices_empty", + None) if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") - and not os.path.exists("/dev/xvda"))): + (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): if idevs is None: idevs = "" if idevs_empty is None: @@ -66,7 +66,7 @@ def handle(name, cfg, _cloud, log, _args): (idevs, idevs_empty)) log.debug("Setting grub debconf-set-selections with '%s','%s'" % - (idevs, idevs_empty)) + (idevs, idevs_empty)) try: util.subp(['debconf-set-selections'], dconf_sel) diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index f1c1adff..aa844ee9 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -48,7 +48,7 @@ def handle(name, cfg, cloud, log, _args): "ssh_fp_console_blacklist", []) key_blacklist = util.get_cfg_option_list(cfg, "ssh_key_console_blacklist", - ["ssh-dss"]) + ["ssh-dss"]) try: cmd = [helper_path] diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 7d8a0202..e2fdf68e 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -59,7 +59,7 @@ def handle(name, cfg, cloud, log, args): if init_cfg: if not isinstance(init_cfg, dict): log.warn("lxd/init config must be a dictionary. found a '%s'", - type(init_cfg)) + type(init_cfg)) return cmd = ['lxd', 'init', '--auto'] for k in init_keys: diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 11089d8d..4fe3ee21 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -204,12 +204,12 @@ def setup_swapfile(fname, size=None, maxsize=None): try: util.ensure_dir(tdir) util.log_time(LOG.debug, msg, func=util.subp, - args=[['sh', '-c', - ('rm -f "$1" && umask 0066 && ' - '{ fallocate -l "${2}M" "$1" || ' - ' dd if=/dev/zero "of=$1" bs=1M "count=$2"; } && ' - 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), - 'setup_swap', fname, mbsize]]) + args=[['sh', '-c', + ('rm -f "$1" && umask 0066 && ' + '{ fallocate -l "${2}M" "$1" || ' + ' dd if=/dev/zero "of=$1" bs=1M "count=$2"; } && ' + 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), + 'setup_swap', fname, mbsize]]) except Exception as e: raise IOError("Failed %s: %s" % (msg, e)) diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 7d9567e3..cc3f7f70 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -105,7 +105,7 @@ def handle(_name, cfg, _cloud, log, _args): log.debug("After pid %s ends, will execute: %s" % (mypid, ' '.join(args))) - util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, + util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, condition, execmd, [args, devnull_fp]) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 4501598e..774d3322 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -36,8 +36,8 @@ def _autostart_puppet(log): # Set puppet to automatically start if os.path.exists('/etc/default/puppet'): util.subp(['sed', '-i', - '-e', 's/^START=.*/START=yes/', - '/etc/default/puppet'], capture=False) + '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet'], capture=False) elif os.path.exists('/bin/systemctl'): util.subp(['/bin/systemctl', 'enable', 'puppet.service'], capture=False) @@ -65,7 +65,7 @@ def handle(name, cfg, cloud, log, _args): " doing nothing.")) elif install: log.debug(("Attempting to install puppet %s,"), - version if version else 'latest') + version if version else 'latest') cloud.distro.install_packages(('puppet', version)) # ... and then update the puppet configuration diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index cbc07853..2a2a9f59 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -166,7 +166,7 @@ def handle(name, cfg, _cloud, log, args): func=do_resize, args=(resize_cmd, log)) else: util.log_time(logfunc=log.debug, msg="Resizing", - func=do_resize, args=(resize_cmd, log)) + func=do_resize, args=(resize_cmd, log)) action = 'Resized' if resize_root == NOBLOCK: diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index 3b30c47e..6f474aed 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -127,8 +127,8 @@ class SubscriptionManager(object): return False, not_bool if (self.servicelevel is not None) and \ - ((not self.auto_attach) - or (util.is_false(str(self.auto_attach)))): + ((not self.auto_attach) or + (util.is_false(str(self.auto_attach)))): no_auto = ("The service-level key must be used in conjunction " "with the auto-attach key. Please re-run with " diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index 5d7f4331..f43d8d5a 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -24,7 +24,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not setting the hostname in module %s"), name) + " not setting the hostname in module %s"), name) return (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 5bd2dec6..d24e43c0 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -30,9 +30,10 @@ from cloudinit import distros as ds from cloudinit import ssh_util from cloudinit import util -DISABLE_ROOT_OPTS = ("no-port-forwarding,no-agent-forwarding," -"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " -"rather than the user \\\"root\\\".\';echo;sleep 10\"") +DISABLE_ROOT_OPTS = ( + "no-port-forwarding,no-agent-forwarding," + "no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\"" + " rather than the user \\\"root\\\".\';echo;sleep 10\"") GENERATE_KEY_NAMES = ['rsa', 'dsa', 'ecdsa', 'ed25519'] KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key' diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index d3dd1f32..15703efe 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -41,10 +41,10 @@ def handle(name, cfg, cloud, log, _args): if not tpl_fn_name: raise RuntimeError(("No hosts template could be" " found for distro %s") % - (cloud.distro.osfamily)) + (cloud.distro.osfamily)) templater.render_to_file(tpl_fn_name, '/etc/hosts', - {'hostname': hostname, 'fqdn': fqdn}) + {'hostname': hostname, 'fqdn': fqdn}) elif manage_hosts == "localhost": (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) @@ -57,4 +57,4 @@ def handle(name, cfg, cloud, log, _args): cloud.distro.update_etc_hosts(hostname, fqdn) else: log.debug(("Configuration option 'manage_etc_hosts' is not set," - " not managing /etc/hosts in module %s"), name) + " not managing /etc/hosts in module %s"), name) diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index e396ba13..5b78afe1 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -29,7 +29,7 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not updating the hostname in module %s"), name) + " not updating the hostname in module %s"), name) return (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 3b821af9..64fba869 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -92,7 +92,7 @@ def handle(name, cfg, _cloud, log, _args): for req_field in ['baseurl']: if req_field not in repo_config: log.warn(("Repository %s does not contain a %s" - " configuration 'required' entry"), + " configuration 'required' entry"), repo_id, req_field) missing_required += 1 if not missing_required: diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 71884b32..661a9fd2 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -211,8 +211,8 @@ class Distro(object): # If the system hostname is different than the previous # one or the desired one lets update it as well - if (not sys_hostname) or (sys_hostname == prev_hostname - and sys_hostname != hostname): + if ((not sys_hostname) or (sys_hostname == prev_hostname and + sys_hostname != hostname)): update_files.append(sys_fn) # If something else has changed the hostname after we set it @@ -221,7 +221,7 @@ class Distro(object): if (sys_hostname and prev_hostname and sys_hostname != prev_hostname): LOG.info("%s differs from %s, assuming user maintained hostname.", - prev_hostname_fn, sys_fn) + prev_hostname_fn, sys_fn) return # Remove duplicates (incase the previous config filename) @@ -289,7 +289,7 @@ class Distro(object): def _bring_up_interface(self, device_name): cmd = ['ifup', device_name] LOG.debug("Attempting to run bring up interface %s using command %s", - device_name, cmd) + device_name, cmd) try: (_out, err) = util.subp(cmd) if len(err): @@ -548,7 +548,7 @@ class Distro(object): for member in members: if not util.is_user(member): LOG.warn("Unable to add group member '%s' to group '%s'" - "; user does not exist.", member, name) + "; user does not exist.", member, name) continue util.subp(['usermod', '-a', '-G', name, member]) @@ -886,7 +886,7 @@ def fetch(name): locs, looked_locs = importer.find_module(name, ['', __name__], ['Distro']) if not locs: raise ImportError("No distribution found for distro %s (searched %s)" - % (name, looked_locs)) + % (name, looked_locs)) mod = importer.import_module(locs[0]) cls = getattr(mod, 'Distro') return cls diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 45fcf26f..93a2e008 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -74,7 +74,7 @@ class Distro(distros.Distro): 'Interface': dev, 'IP': info.get('bootproto'), 'Address': "('%s/%s')" % (info.get('address'), - info.get('netmask')), + info.get('netmask')), 'Gateway': info.get('gateway'), 'DNS': str(tuple(info.get('dns-nameservers'))).replace(',', '') } @@ -86,7 +86,7 @@ class Distro(distros.Distro): if nameservers: util.write_file(self.resolve_conf_fn, - convert_resolv_conf(nameservers)) + convert_resolv_conf(nameservers)) return dev_names @@ -102,7 +102,7 @@ class Distro(distros.Distro): def _bring_up_interface(self, device_name): cmd = ['netctl', 'restart', device_name] LOG.debug("Attempting to run bring up interface %s using command %s", - device_name, cmd) + device_name, cmd) try: (_out, err) = util.subp(cmd) if len(err): diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 6d3a82bf..db5890b1 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -159,8 +159,9 @@ class Distro(distros.Distro): # Allow the output of this to flow outwards (ie not be captured) util.log_time(logfunc=LOG.debug, - msg="apt-%s [%s]" % (command, ' '.join(cmd)), func=util.subp, - args=(cmd,), kwargs={'env': e, 'capture': False}) + msg="apt-%s [%s]" % (command, ' '.join(cmd)), + func=util.subp, + args=(cmd,), kwargs={'env': e, 'capture': False}) def update_package_sources(self): self._runner.run("update-sources", self.package_command, diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index 4c484639..72012056 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -205,8 +205,8 @@ class Distro(distros.Distro): redact_opts = ['passwd'] for key, val in kwargs.items(): - if (key in adduser_opts and val - and isinstance(val, six.string_types)): + if (key in adduser_opts and val and + isinstance(val, six.string_types)): adduser_cmd.extend([adduser_opts[key], val]) # Redact certain fields from the logs diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 9e80583c..6267dd6e 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -66,7 +66,7 @@ class Distro(distros.Distro): def _bring_up_interface(self, device_name): cmd = ['/etc/init.d/net.%s' % device_name, 'restart'] LOG.debug("Attempting to run bring up interface %s using command %s", - device_name, cmd) + device_name, cmd) try: (_out, err) = util.subp(cmd) if len(err): @@ -88,7 +88,7 @@ class Distro(distros.Distro): (_out, err) = util.subp(cmd) if len(err): LOG.warn("Running %s resulted in stderr output: %s", cmd, - err) + err) except util.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False diff --git a/cloudinit/distros/parsers/hostname.py b/cloudinit/distros/parsers/hostname.py index 84a1de42..efb185d4 100644 --- a/cloudinit/distros/parsers/hostname.py +++ b/cloudinit/distros/parsers/hostname.py @@ -84,5 +84,5 @@ class HostnameConf(object): hostnames_found.add(head) if len(hostnames_found) > 1: raise IOError("Multiple hostnames (%s) found!" - % (hostnames_found)) + % (hostnames_found)) return entries diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py index 8aee03a4..2ed13d9c 100644 --- a/cloudinit/distros/parsers/resolv_conf.py +++ b/cloudinit/distros/parsers/resolv_conf.py @@ -132,7 +132,7 @@ class ResolvConf(object): # Some hard limit on 256 chars total raise ValueError(("Adding %r would go beyond the " "256 maximum search list character limit") - % (search_domain)) + % (search_domain)) self._remove_option('search') self._contents.append(('option', ['search', s_list, ''])) return flat_sds diff --git a/cloudinit/distros/parsers/sys_conf.py b/cloudinit/distros/parsers/sys_conf.py index d795e12f..6157cf32 100644 --- a/cloudinit/distros/parsers/sys_conf.py +++ b/cloudinit/distros/parsers/sys_conf.py @@ -77,8 +77,7 @@ class SysConf(configobj.ConfigObj): quot_func = None if value[0] in ['"', "'"] and value[-1] in ['"', "'"]: if len(value) == 1: - quot_func = (lambda x: - self._get_single_quote(x) % x) + quot_func = (lambda x: self._get_single_quote(x) % x) else: # Quote whitespace if it isn't the start + end of a shell command if value.strip().startswith("$(") and value.strip().endswith(")"): @@ -91,10 +90,10 @@ class SysConf(configobj.ConfigObj): # to use single quotes which won't get expanded... if re.search(r"[\n\"']", value): quot_func = (lambda x: - self._get_triple_quote(x) % x) + self._get_triple_quote(x) % x) else: quot_func = (lambda x: - self._get_single_quote(x) % x) + self._get_single_quote(x) % x) else: quot_func = pipes.quote if not quot_func: diff --git a/cloudinit/filters/launch_index.py b/cloudinit/filters/launch_index.py index 5bebd318..baecdac9 100644 --- a/cloudinit/filters/launch_index.py +++ b/cloudinit/filters/launch_index.py @@ -61,7 +61,7 @@ class Filter(object): discarded += 1 LOG.debug(("Discarding %s multipart messages " "which do not match launch index %s"), - discarded, self.wanted_idx) + discarded, self.wanted_idx) new_message = copy.copy(message) new_message.set_payload(new_msgs) new_message[ud.ATTACHMENT_FIELD] = str(len(new_msgs)) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 5e99d185..a6eb20fe 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -139,9 +139,10 @@ class FileSemaphores(object): # but the item had run before we did canon_sem_name. if cname != name and os.path.exists(self._get_path(name, freq)): LOG.warn("%s has run without canonicalized name [%s].\n" - "likely the migrator has not yet run. It will run next boot.\n" - "run manually with: cloud-init single --name=migrator" - % (name, cname)) + "likely the migrator has not yet run. " + "It will run next boot.\n" + "run manually with: cloud-init single --name=migrator" + % (name, cname)) return True return False diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index bd80a8a6..b03ab895 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -38,7 +38,8 @@ LOG = logging.getLogger(__name__) DS_NAME = 'Azure' DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] -BOUNCE_COMMAND = ['sh', '-xc', +BOUNCE_COMMAND = [ + 'sh', '-xc', "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"] BUILTIN_DS_CONFIG = { @@ -91,9 +92,9 @@ def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'): """ policy = cfg['hostname_bounce']['policy'] previous_hostname = get_hostname(hostname_command) - if (not util.is_true(cfg.get('set_hostname')) - or util.is_false(policy) - or (previous_hostname == temp_hostname and policy != 'force')): + if (not util.is_true(cfg.get('set_hostname')) or + util.is_false(policy) or + (previous_hostname == temp_hostname and policy != 'force')): yield None return set_hostname(temp_hostname, hostname_command) @@ -123,8 +124,8 @@ class DataSourceAzureNet(sources.DataSource): with temporary_hostname(temp_hostname, self.ds_cfg, hostname_command=hostname_command) \ as previous_hostname: - if (previous_hostname is not None - and util.is_true(self.ds_cfg.get('set_hostname'))): + if (previous_hostname is not None and + util.is_true(self.ds_cfg.get('set_hostname'))): cfg = self.ds_cfg['hostname_bounce'] try: perform_hostname_bounce(hostname=temp_hostname, @@ -152,7 +153,8 @@ class DataSourceAzureNet(sources.DataSource): else: bname = str(pk['fingerprint'] + ".crt") fp_files += [os.path.join(ddir, bname)] - LOG.debug("ssh authentication: using fingerprint from fabirc") + LOG.debug("ssh authentication: " + "using fingerprint from fabirc") missing = util.log_time(logfunc=LOG.debug, msg="waiting for files", func=wait_for_files, @@ -506,7 +508,7 @@ def read_azure_ovf(contents): raise BrokenAzureDataSource("invalid xml: %s" % e) results = find_child(dom.documentElement, - lambda n: n.localName == "ProvisioningSection") + lambda n: n.localName == "ProvisioningSection") if len(results) == 0: raise NonAzureDataSource("No ProvisioningSection") @@ -516,7 +518,8 @@ def read_azure_ovf(contents): provSection = results[0] lpcs_nodes = find_child(provSection, - lambda n: n.localName == "LinuxProvisioningConfigurationSet") + lambda n: + n.localName == "LinuxProvisioningConfigurationSet") if len(results) == 0: raise NonAzureDataSource("No LinuxProvisioningConfigurationSet") diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index eb474079..e3916208 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -39,7 +39,7 @@ FS_TYPES = ('vfat', 'iso9660') LABEL_TYPES = ('config-2',) POSSIBLE_MOUNTS = ('sr', 'cd') OPTICAL_DEVICES = tuple(('/dev/%s%s' % (z, i) for z in POSSIBLE_MOUNTS - for i in range(0, 2))) + for i in range(0, 2))) class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 0032d06c..6a897f7d 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -61,12 +61,12 @@ class DataSourceEc2(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = ec2.get_instance_userdata(self.api_ver, - self.metadata_address) + self.userdata_raw = \ + ec2.get_instance_userdata(self.api_ver, self.metadata_address) self.metadata = ec2.get_instance_metadata(self.api_ver, self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", - int(time.time() - start_time)) + int(time.time() - start_time)) return True except Exception: util.logexc(LOG, "Failed reading from metadata address %s", @@ -132,13 +132,13 @@ class DataSourceEc2(sources.DataSource): start_time = time.time() url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn) + timeout=timeout, status_cb=LOG.warn) if url: LOG.debug("Using metadata source: '%s'", url2base[url]) else: LOG.critical("Giving up on md from %s after %s seconds", - urls, int(time.time() - start_time)) + urls, int(time.time() - start_time)) self.metadata_address = url2base.get(url) return bool(url) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index cfc59ca5..f18c4cee 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -275,17 +275,18 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description='Interact with MAAS DS') parser.add_argument("--config", metavar="file", - help="specify DS config file", default=None) + help="specify DS config file", default=None) parser.add_argument("--ckey", metavar="key", - help="the consumer key to auth with", default=None) + help="the consumer key to auth with", default=None) parser.add_argument("--tkey", metavar="key", - help="the token key to auth with", default=None) + help="the token key to auth with", default=None) parser.add_argument("--csec", metavar="secret", - help="the consumer secret (likely '')", default="") + help="the consumer secret (likely '')", default="") parser.add_argument("--tsec", metavar="secret", - help="the token secret to auth with", default=None) + help="the token secret to auth with", default=None) parser.add_argument("--apiver", metavar="version", - help="the apiver to use ("" can be used)", default=MD_VERSION) + help="the apiver to use ("" can be used)", + default=MD_VERSION) subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") subcmds.add_parser('crawl', help="crawl the datasource") @@ -297,7 +298,7 @@ if __name__ == "__main__": args = parser.parse_args() creds = {'consumer_key': args.ckey, 'token_key': args.tkey, - 'token_secret': args.tsec, 'consumer_secret': args.csec} + 'token_secret': args.tsec, 'consumer_secret': args.csec} if args.config: cfg = util.read_conf(args.config) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 58a4b2a2..adf9b12e 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -264,14 +264,14 @@ def get_properties(contents): # could also check here that elem.namespaceURI == # "http://schemas.dmtf.org/ovf/environment/1" propSections = find_child(dom.documentElement, - lambda n: n.localName == "PropertySection") + lambda n: n.localName == "PropertySection") if len(propSections) == 0: raise XmlError("No 'PropertySection's") props = {} propElems = find_child(propSections[0], - (lambda n: n.localName == "Property")) + (lambda n: n.localName == "Property")) for elem in propElems: key = elem.attributes.getNamedItemNS(envNsURI, "key").value diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index ac2c3b45..b26940d1 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -404,7 +404,8 @@ def read_context_disk_dir(source_dir, asuser=None): if ssh_key_var: lines = context.get(ssh_key_var).splitlines() results['metadata']['public-keys'] = [l for l in lines - if len(l) and not l.startswith("#")] + if len(l) and not + l.startswith("#")] # custom hostname -- try hostname or leave cloud-init # itself create hostname from IP address later diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 7453379a..139ee52c 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -90,8 +90,7 @@ BUILTIN_DS_CONFIG = { 'user-data', 'user-script', 'sdc:datacenter_name', - 'sdc:uuid', - ], + 'sdc:uuid'], 'base64_keys': [], 'base64_all': False, 'disk_aliases': {'ephemeral0': '/dev/vdb'}, @@ -450,7 +449,7 @@ class JoyentMetadataClient(object): response = bytearray() response.extend(self.metasource.read(1)) - while response[-1:] != b'\n': + while response[-1:] != b'\n': response.extend(self.metasource.read(1)) response = response.rstrip().decode('ascii') LOG.debug('Read "%s" from metadata transport.', response) @@ -513,7 +512,7 @@ def write_boot_content(content, content_f, link=None, shebang=False, except Exception as e: util.logexc(LOG, ("Failed to identify script type for %s" % - content_f, e)) + content_f, e)) if link: try: diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index 9b2f5ed5..c74a7ae2 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -31,7 +31,8 @@ LOG = logging.getLogger(__name__) DEF_SSHD_CFG = "/etc/ssh/sshd_config" # taken from openssh source key.c/key_type_from_name -VALID_KEY_TYPES = ("rsa", "dsa", "ssh-rsa", "ssh-dss", "ecdsa", +VALID_KEY_TYPES = ( + "rsa", "dsa", "ssh-rsa", "ssh-dss", "ecdsa", "ssh-rsa-cert-v00@openssh.com", "ssh-dss-cert-v00@openssh.com", "ssh-rsa-cert-v00@openssh.com", "ssh-dss-cert-v00@openssh.com", "ssh-rsa-cert-v01@openssh.com", "ssh-dss-cert-v01@openssh.com", diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 9f192c8d..dbcf3d55 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -509,13 +509,13 @@ class Init(object): def consume_data(self, frequency=PER_INSTANCE): # Consume the userdata first, because we need want to let the part # handlers run first (for merging stuff) - with events.ReportEventStack( - "consume-user-data", "reading and applying user-data", - parent=self.reporter): + with events.ReportEventStack("consume-user-data", + "reading and applying user-data", + parent=self.reporter): self._consume_userdata(frequency) - with events.ReportEventStack( - "consume-vendor-data", "reading and applying vendor-data", - parent=self.reporter): + with events.ReportEventStack("consume-vendor-data", + "reading and applying vendor-data", + parent=self.reporter): self._consume_vendordata(frequency) # Perform post-consumption adjustments so that @@ -655,7 +655,7 @@ class Modules(object): else: raise TypeError(("Failed to read '%s' item in config," " unknown type %s") % - (item, type_utils.obj_name(item))) + (item, type_utils.obj_name(item))) return module_list def _fixup_modules(self, raw_mods): @@ -762,8 +762,8 @@ class Modules(object): if skipped: LOG.info("Skipping modules %s because they are not verified " - "on distro '%s'. To run anyway, add them to " - "'unverified_modules' in config.", skipped, d_name) + "on distro '%s'. To run anyway, add them to " + "'unverified_modules' in config.", skipped, d_name) if forced: LOG.info("running unverified_modules: %s", forced) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index f2e1390e..936f7da5 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -252,9 +252,9 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # attrs return UrlResponse(r) except exceptions.RequestException as e: - if (isinstance(e, (exceptions.HTTPError)) - and hasattr(e, 'response') # This appeared in v 0.10.8 - and hasattr(e.response, 'status_code')): + if (isinstance(e, (exceptions.HTTPError)) and + hasattr(e, 'response') and # This appeared in v 0.10.8 + hasattr(e.response, 'status_code')): excps.append(UrlError(e, code=e.response.status_code, headers=e.response.headers, url=url)) diff --git a/cloudinit/util.py b/cloudinit/util.py index 45d49e66..de37b0f5 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -612,7 +612,7 @@ def redirect_output(outfmt, errfmt, o_out=None, o_err=None): def make_url(scheme, host, port=None, - path='', params='', query='', fragment=''): + path='', params='', query='', fragment=''): pieces = [] pieces.append(scheme or '') @@ -804,8 +804,8 @@ def load_yaml(blob, default=None, allowed=(dict,)): blob = decode_binary(blob) try: LOG.debug("Attempting to load yaml from string " - "of length %s with allowed root types %s", - len(blob), allowed) + "of length %s with allowed root types %s", + len(blob), allowed) converted = safeyaml.load(blob) if not isinstance(converted, allowed): # Yes this will just be caught, but thats ok for now... @@ -878,7 +878,7 @@ def read_conf_with_confd(cfgfile): if not isinstance(confd, six.string_types): raise TypeError(("Config file %s contains 'conf_d' " "with non-string type %s") % - (cfgfile, type_utils.obj_name(confd))) + (cfgfile, type_utils.obj_name(confd))) else: confd = str(confd).strip() elif os.path.isdir("%s.d" % cfgfile): @@ -1041,7 +1041,8 @@ def is_resolvable(name): for iname in badnames: try: result = socket.getaddrinfo(iname, None, 0, 0, - socket.SOCK_STREAM, socket.AI_CANONNAME) + socket.SOCK_STREAM, + socket.AI_CANONNAME) badresults[iname] = [] for (_fam, _stype, _proto, cname, sockaddr) in result: badresults[iname].append("%s: %s" % (cname, sockaddr[0])) @@ -1109,7 +1110,7 @@ def close_stdin(): def find_devs_with(criteria=None, oformat='device', - tag=None, no_cache=False, path=None): + tag=None, no_cache=False, path=None): """ find devices matching given criteria (via blkid) criteria can be *one* of: @@ -1628,7 +1629,7 @@ def write_file(filename, content, mode=0o644, omode="wb"): content = decode_binary(content) write_type = 'characters' LOG.debug("Writing to %s - %s: [%s] %s %s", - filename, omode, mode, len(content), write_type) + filename, omode, mode, len(content), write_type) with SeLinuxGuard(path=filename): with open(filename, omode) as fh: fh.write(content) diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index c603bfdb..9c1ec1d4 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -27,11 +27,12 @@ from cloudinit import stages from cloudinit import user_data as ud from cloudinit import util -INSTANCE_ID = "i-testing" - from . import helpers +INSTANCE_ID = "i-testing" + + class FakeDataSource(sources.DataSource): def __init__(self, userdata=None, vendordata=None): diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py index e9cd2fa5..85759c68 100644 --- a/tests/unittests/test_datasource/test_altcloud.py +++ b/tests/unittests/test_datasource/test_altcloud.py @@ -134,8 +134,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('RHEV') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('RHEV', \ - dsrc.get_cloud_type()) + self.assertEquals('RHEV', dsrc.get_cloud_type()) def test_vsphere(self): ''' @@ -144,8 +143,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('VMware Virtual Platform') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('VSPHERE', \ - dsrc.get_cloud_type()) + self.assertEquals('VSPHERE', dsrc.get_cloud_type()) def test_unknown(self): ''' @@ -154,8 +152,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('Unrecognized Platform') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('UNKNOWN', \ - dsrc.get_cloud_type()) + self.assertEquals('UNKNOWN', dsrc.get_cloud_type()) class TestGetDataCloudInfoFile(TestCase): @@ -412,27 +409,27 @@ class TestReadUserDataCallback(TestCase): '''Test read_user_data_callback() with both files.''' self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + read_user_data_callback(self.mount_dir)) def test_callback_dc(self): '''Test read_user_data_callback() with only DC file.''' _remove_user_data_files(self.mount_dir, - dc_file=False, - non_dc_file=True) + dc_file=False, + non_dc_file=True) self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + read_user_data_callback(self.mount_dir)) def test_callback_non_dc(self): '''Test read_user_data_callback() with only non-DC file.''' _remove_user_data_files(self.mount_dir, - dc_file=True, - non_dc_file=False) + dc_file=True, + non_dc_file=False) self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + read_user_data_callback(self.mount_dir)) def test_callback_none(self): '''Test read_user_data_callback() no files are found.''' diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 3933794f..4c9c7d8b 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -207,7 +207,7 @@ class TestAzureDataSource(TestCase): yaml_cfg = "{agent_command: my_command}\n" cfg = yaml.safe_load(yaml_cfg) odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}} + 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) @@ -219,8 +219,8 @@ class TestAzureDataSource(TestCase): # set dscfg in via base64 encoded yaml cfg = {'agent_command': "my_command"} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': b64e(yaml.dump(cfg)), - 'encoding': 'base64'}} + 'dscfg': {'text': b64e(yaml.dump(cfg)), + 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) @@ -267,7 +267,8 @@ class TestAzureDataSource(TestCase): # should equal that after the '$' pos = defuser['passwd'].rfind("$") + 1 self.assertEqual(defuser['passwd'], - crypt.crypt(odata['UserPassword'], defuser['passwd'][0:pos])) + crypt.crypt(odata['UserPassword'], + defuser['passwd'][0:pos])) def test_userdata_plain(self): mydata = "FOOBAR" @@ -364,8 +365,8 @@ class TestAzureDataSource(TestCase): # Make sure that user can affect disk aliases dscfg = {'disk_aliases': {'ephemeral0': '/dev/sdc'}} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': b64e(yaml.dump(dscfg)), - 'encoding': 'base64'}} + 'dscfg': {'text': b64e(yaml.dump(dscfg)), + 'encoding': 'base64'}} usercfg = {'disk_setup': {'/dev/sdc': {'something': '...'}, 'ephemeral0': False}} userdata = '#cloud-config' + yaml.dump(usercfg) + "\n" @@ -634,7 +635,7 @@ class TestReadAzureOvf(TestCase): def test_invalid_xml_raises_non_azure_ds(self): invalid_xml = "" + construct_valid_ovf_env(data={}) self.assertRaises(DataSourceAzure.BrokenAzureDataSource, - DataSourceAzure.read_azure_ovf, invalid_xml) + DataSourceAzure.read_azure_ovf, invalid_xml) def test_load_with_pubkeys(self): mypklist = [{'fingerprint': 'fp1', 'path': 'path1', 'value': ''}] diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 83aca505..3954ceb3 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -293,9 +293,8 @@ class TestConfigDriveDataSource(TestCase): util.is_partition = my_is_partition devs_with_answers = {"TYPE=vfat": [], - "TYPE=iso9660": ["/dev/vdb"], - "LABEL=config-2": ["/dev/vdb"], - } + "TYPE=iso9660": ["/dev/vdb"], + "LABEL=config-2": ["/dev/vdb"]} self.assertEqual(["/dev/vdb"], ds.find_candidate_devs()) # add a vfat item @@ -306,9 +305,10 @@ class TestConfigDriveDataSource(TestCase): # verify that partitions are considered, that have correct label. devs_with_answers = {"TYPE=vfat": ["/dev/sda1"], - "TYPE=iso9660": [], "LABEL=config-2": ["/dev/vdb3"]} + "TYPE=iso9660": [], + "LABEL=config-2": ["/dev/vdb3"]} self.assertEqual(["/dev/vdb3"], - ds.find_candidate_devs()) + ds.find_candidate_devs()) finally: util.find_devs_with = orig_find_devs_with @@ -319,7 +319,7 @@ class TestConfigDriveDataSource(TestCase): populate_dir(self.tmp, CFG_DRIVE_FILES_V2) myds = cfg_ds_from_dir(self.tmp) self.assertEqual(myds.get_public_ssh_keys(), - [OSTACK_META['public_keys']['mykey']]) + [OSTACK_META['public_keys']['mykey']]) def cfg_ds_from_dir(seed_d): diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index eb97b692..77d15cac 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -25,9 +25,9 @@ class TestMAASDataSource(TestCase): """Verify a valid seeddir is read as such.""" data = {'instance-id': 'i-valid01', - 'local-hostname': 'valid01-hostname', - 'user-data': b'valid01-userdata', - 'public-keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname'} + 'local-hostname': 'valid01-hostname', + 'user-data': b'valid01-userdata', + 'public-keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname'} my_d = os.path.join(self.tmp, "valid") populate_dir(my_d, data) @@ -45,8 +45,8 @@ class TestMAASDataSource(TestCase): """Verify extra files do not affect seed_dir validity.""" data = {'instance-id': 'i-valid-extra', - 'local-hostname': 'valid-extra-hostname', - 'user-data': b'valid-extra-userdata', 'foo': 'bar'} + 'local-hostname': 'valid-extra-hostname', + 'user-data': b'valid-extra-userdata', 'foo': 'bar'} my_d = os.path.join(self.tmp, "valid_extra") populate_dir(my_d, data) @@ -64,7 +64,7 @@ class TestMAASDataSource(TestCase): """Verify that invalid seed_dir raises MAASSeedDirMalformed.""" valid = {'instance-id': 'i-instanceid', - 'local-hostname': 'test-hostname', 'user-data': ''} + 'local-hostname': 'test-hostname', 'user-data': ''} my_based = os.path.join(self.tmp, "valid_extra") @@ -94,8 +94,8 @@ class TestMAASDataSource(TestCase): def test_seed_dir_missing(self): """Verify that missing seed_dir raises MAASSeedDirNone.""" self.assertRaises(DataSourceMAAS.MAASSeedDirNone, - DataSourceMAAS.read_maas_seed_dir, - os.path.join(self.tmp, "nonexistantdirectory")) + DataSourceMAAS.read_maas_seed_dir, + os.path.join(self.tmp, "nonexistantdirectory")) def test_seed_url_valid(self): """Verify that valid seed_url is read as such.""" diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 1235436d..5e617b83 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -463,8 +463,8 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): payloadstr = ' {0}'.format(self.response_parts['payload']) return ('V2 {length} {crc} {request_id} ' '{command}{payloadstr}\n'.format( - payloadstr=payloadstr, - **self.response_parts).encode('ascii')) + payloadstr=payloadstr, + **self.response_parts).encode('ascii')) self.metasource_data = None @@ -501,7 +501,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): written_line = self.serial.write.call_args[0][0] print(type(written_line)) self.assertEndsWith(written_line.decode('ascii'), - b'\n'.decode('ascii')) + b'\n'.decode('ascii')) self.assertEqual(1, written_line.count(b'\n')) def _get_written_line(self, key='some_key'): diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 5687b10d..f9660ff6 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -74,7 +74,7 @@ class TestLoadPowerState(t_help.TestCase): class TestCheckCondition(t_help.TestCase): def cmd_with_exit(self, rc): return([sys.executable, '-c', 'import sys; sys.exit(%s)' % rc]) - + def test_true_is_true(self): self.assertEqual(psc.check_condition(True), True) @@ -94,7 +94,6 @@ class TestCheckCondition(t_help.TestCase): self.assertEqual(mocklog.warn.call_count, 1) - def check_lps_ret(psc_return, mode=None): if len(psc_return) != 3: raise TypeError("length returned = %d" % len(psc_return)) diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index 0bcdcb31..34d11f21 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -190,7 +190,8 @@ class TestRandomSeed(t_help.TestCase): c = self._get_cloud('ubuntu', {}) self.whichdata = {} self.assertRaises(ValueError, cc_seed_random.handle, - 'test', {'random_seed': {'command_required': True}}, c, LOG, []) + 'test', {'random_seed': {'command_required': True}}, + c, LOG, []) def test_seed_command_and_required(self): c = self._get_cloud('ubuntu', {}) diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index eceb14d9..8aeff53c 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -125,8 +125,7 @@ class TestInstallPackages(t_help.TestCase): "pkg1.smoser.config": "pkg1.smoser.config-data", "pkg1.config": "pkg1.config-data", "pkg2.smoser_0.0_amd64.snap": "pkg2-snapdata", - "pkg2.smoser_0.0_amd64.config": "pkg2.config", - }) + "pkg2.smoser_0.0_amd64.config": "pkg2.config"}) ret = get_package_ops( packages=[], configs={}, installed=[], fspath=self.tmp) diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index 3b317121..9aeb1cde 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -32,7 +32,8 @@ VALID_CONTENT = { ), } -TEST_OPTIONS = ("no-port-forwarding,no-agent-forwarding,no-X11-forwarding," +TEST_OPTIONS = ( + "no-port-forwarding,no-agent-forwarding,no-X11-forwarding," 'command="echo \'Please login as the user \"ubuntu\" rather than the' 'user \"root\".\';echo;sleep 10"') diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index 0c19a2c2..b9863650 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -114,5 +114,6 @@ $a,$b''' codename) out_data = templater.basic_render(in_data, - {'mirror': mirror, 'codename': codename}) + {'mirror': mirror, + 'codename': codename}) self.assertEqual(ex_data, out_data) diff --git a/tools/hacking.py b/tools/hacking.py index 3175df38..1a0631c2 100755 --- a/tools/hacking.py +++ b/tools/hacking.py @@ -47,10 +47,10 @@ def import_normalize(line): # handle "from x import y as z" to "import x.y as z" split_line = line.split() if (line.startswith("from ") and "," not in line and - split_line[2] == "import" and split_line[3] != "*" and - split_line[1] != "__future__" and - (len(split_line) == 4 or - (len(split_line) == 6 and split_line[4] == "as"))): + split_line[2] == "import" and split_line[3] != "*" and + split_line[1] != "__future__" and + (len(split_line) == 4 or + (len(split_line) == 6 and split_line[4] == "as"))): return "import %s.%s" % (split_line[1], split_line[3]) else: return line @@ -74,7 +74,7 @@ def cloud_import_alphabetical(physical_line, line_number, lines): split_line[0] == "import" and split_previous[0] == "import"): if split_line[1] < split_previous[1]: return (0, "N306: imports not in alphabetical order (%s, %s)" - % (split_previous[1], split_line[1])) + % (split_previous[1], split_line[1])) def cloud_docstring_start_space(physical_line): @@ -87,8 +87,8 @@ def cloud_docstring_start_space(physical_line): pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE]) # start if (pos != -1 and len(physical_line) > pos + 1): if (physical_line[pos + 3] == ' '): - return (pos, "N401: one line docstring should not start with" - " a space") + return (pos, + "N401: one line docstring should not start with a space") def cloud_todo_format(physical_line): @@ -167,4 +167,4 @@ if __name__ == "__main__": finally: if len(_missingImport) > 0: print >> sys.stderr, ("%i imports missing in this test environment" - % len(_missingImport)) + % len(_missingImport)) diff --git a/tools/mock-meta.py b/tools/mock-meta.py index dfbc2a71..1c746f17 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -126,11 +126,11 @@ class WebException(Exception): def yamlify(data): formatted = yaml.dump(data, - line_break="\n", - indent=4, - explicit_start=True, - explicit_end=True, - default_flow_style=False) + line_break="\n", + indent=4, + explicit_start=True, + explicit_end=True, + default_flow_style=False) return formatted @@ -282,7 +282,7 @@ class MetaDataHandler(object): else: log.warn(("Did not implement action %s, " "returning empty response: %r"), - action, NOT_IMPL_RESPONSE) + action, NOT_IMPL_RESPONSE) return NOT_IMPL_RESPONSE @@ -404,14 +404,17 @@ def setup_logging(log_level, fmt='%(levelname)s: @%(name)s : %(message)s'): def extract_opts(): parser = OptionParser() parser.add_option("-p", "--port", dest="port", action="store", type=int, - default=80, metavar="PORT", - help="port from which to serve traffic (default: %default)") + default=80, metavar="PORT", + help=("port from which to serve traffic" + " (default: %default)")) parser.add_option("-a", "--addr", dest="address", action="store", type=str, - default='0.0.0.0', metavar="ADDRESS", - help="address from which to serve traffic (default: %default)") + default='0.0.0.0', metavar="ADDRESS", + help=("address from which to serve traffic" + " (default: %default)")) parser.add_option("-f", '--user-data-file', dest='user_data_file', - action='store', metavar='FILE', - help="user data filename to serve back to incoming requests") + action='store', metavar='FILE', + help=("user data filename to serve back to" + "incoming requests")) (options, args) = parser.parse_args() out = dict() out['extra'] = args diff --git a/tools/run-pep8 b/tools/run-pep8 index ccd6be5a..086400fc 100755 --- a/tools/run-pep8 +++ b/tools/run-pep8 @@ -1,39 +1,22 @@ #!/bin/bash -if [ $# -eq 0 ]; then - files=( bin/cloud-init $(find * -name "*.py" -type f) ) +pycheck_dirs=( "cloudinit/" "bin/" "tests/" "tools/" ) +# FIXME: cloud-init modifies sys module path, pep8 does not like +# bin_files=( "bin/cloud-init" ) +CR=" +" +[ "$1" = "-v" ] && { verbose="$1"; shift; } || verbose="" + +set -f +if [ $# -eq 0 ]; then unset IFS + IFS="$CR" + files=( "${bin_files[@]}" "${pycheck_dirs[@]}" ) + unset IFS else - files=( "$@" ); + files=( "$@" ) fi -if [ -f 'hacking.py' ] -then - base=`pwd` -else - base=`pwd`/tools/ -fi - -IGNORE="" - -# King Arthur: Be quiet! ... Be Quiet! I Order You to Be Quiet. -IGNORE="$IGNORE,E121" # Continuation line indentation is not a multiple of four -IGNORE="$IGNORE,E123" # Closing bracket does not match indentation of opening bracket's line -IGNORE="$IGNORE,E124" # Closing bracket missing visual indentation -IGNORE="$IGNORE,E125" # Continuation line does not distinguish itself from next logical line -IGNORE="$IGNORE,E126" # Continuation line over-indented for hanging indent -IGNORE="$IGNORE,E127" # Continuation line over-indented for visual indent -IGNORE="$IGNORE,E128" # Continuation line under-indented for visual indent -IGNORE="$IGNORE,E502" # The backslash is redundant between brackets -IGNORE="${IGNORE#,}" # remove the leading ',' added above - -cmd=( - ${base}/hacking.py - - --ignore="$IGNORE" - - "${files[@]}" -) - -echo -e "\nRunning 'cloudinit' pep8:" -echo "${cmd[@]}" -"${cmd[@]}" +myname=${0##*/} +cmd=( "${myname#run-}" $verbose "${files[@]}" ) +echo "Running: " "${cmd[@]}" 1>&2 +exec "${cmd[@]}" -- cgit v1.2.3