From 6363b546c8bacb24b9350d108165a09bb65828a1 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 22 Jan 2015 11:38:51 -0500 Subject: Port test__init__.py to unittest.mock. --- tests/unittests/helpers.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) (limited to 'tests/unittests/helpers.py') diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 52305397..4ca460f1 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -2,11 +2,6 @@ import os import sys import unittest -from contextlib import contextmanager - -from mocker import Mocker -from mocker import MockerTestCase - from cloudinit import helpers as ch from cloudinit import util @@ -86,17 +81,6 @@ else: pass -@contextmanager -def mocker(verify_calls=True): - m = Mocker() - try: - yield m - finally: - m.restore() - if verify_calls: - m.verify() - - # Makes the old path start # with new base instead of whatever # it previously had @@ -126,9 +110,8 @@ def retarget_many_wrapper(new_base, am, old_func): return wrapper -class ResourceUsingTestCase(MockerTestCase): +class ResourceUsingTestCase(unittest.TestCase): def __init__(self, methodName="runTest"): - MockerTestCase.__init__(self, methodName) self.resource_path = None def resourceLocation(self, subname=None): -- cgit v1.2.3 From 6f3fadcfeec94711de32574f7738f98aa0a697ac Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 22 Jan 2015 15:41:52 -0500 Subject: Convert helpers.py and test_data.py from mocker to mock. --- tests/unittests/helpers.py | 56 +++++++++++++--------- tests/unittests/test_data.py | 110 +++++++++++++++++++++++++------------------ tox.ini | 2 +- 3 files changed, 97 insertions(+), 71 deletions(-) (limited to 'tests/unittests/helpers.py') diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 4ca460f1..43dd5aa0 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -1,7 +1,17 @@ import os import sys +import tempfile import unittest +try: + from unittest import mock +except ImportError: + import mock +try: + from contextlib import ExitStack +except ImportError: + from contextlib2 import ExitStack + from cloudinit import helpers as ch from cloudinit import util @@ -111,7 +121,11 @@ def retarget_many_wrapper(new_base, am, old_func): class ResourceUsingTestCase(unittest.TestCase): - def __init__(self, methodName="runTest"): + ## def __init__(self, methodName="runTest"): + ## self.resource_path = None + + def setUp(self): + unittest.TestCase.setUp(self) self.resource_path = None def resourceLocation(self, subname=None): @@ -139,17 +153,23 @@ class ResourceUsingTestCase(unittest.TestCase): return fh.read() def getCloudPaths(self): + tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmpdir) cp = ch.Paths({ - 'cloud_dir': self.makeDir(), + 'cloud_dir': tmpdir, 'templates_dir': self.resourceLocation(), }) return cp class FilesystemMockingTestCase(ResourceUsingTestCase): - def __init__(self, methodName="runTest"): - ResourceUsingTestCase.__init__(self, methodName) - self.patched_funcs = [] + ## def __init__(self, methodName="runTest"): + ## ResourceUsingTestCase.__init__(self, methodName) + + def setUp(self): + ResourceUsingTestCase.setUp(self) + self.patched_funcs = ExitStack() + self.addCleanup(self.patched_funcs.close) def replicateTestRoot(self, example_root, target_root): real_root = self.resourceLocation() @@ -163,15 +183,6 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): make_path = util.abs_join(make_path, f) shutil.copy(real_path, make_path) - def tearDown(self): - self.restore() - ResourceUsingTestCase.tearDown(self) - - def restore(self): - for (mod, f, func) in self.patched_funcs: - setattr(mod, f, func) - self.patched_funcs = [] - def patchUtils(self, new_root): patch_funcs = { util: [('write_file', 1), @@ -188,8 +199,8 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): for (f, am) in funcs: func = getattr(mod, f) trap_func = retarget_many_wrapper(new_root, am, func) - setattr(mod, f, trap_func) - self.patched_funcs.append((mod, f, func)) + self.patched_funcs.enter_context( + mock.patch.object(mod, f, trap_func)) # Handle subprocess calls func = getattr(util, 'subp') @@ -197,16 +208,15 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): def nsubp(*_args, **_kwargs): return ('', '') - setattr(util, 'subp', nsubp) - self.patched_funcs.append((util, 'subp', func)) + self.patched_funcs.enter_context( + mock.patch.object(util, 'subp', nsubp)) def null_func(*_args, **_kwargs): return None for f in ['chownbyid', 'chownbyname']: - func = getattr(util, f) - setattr(util, f, null_func) - self.patched_funcs.append((util, f, func)) + self.patched_funcs.enter_context( + mock.patch.object(util, f, null_func)) def patchOS(self, new_root): patch_funcs = { @@ -217,8 +227,8 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): for f in funcs: func = getattr(mod, f) trap_func = retarget_many_wrapper(new_root, 1, func) - setattr(mod, f, trap_func) - self.patched_funcs.append((mod, f, func)) + self.patched_funcs.enter_context( + mock.patch.object(mod, f, trap_func)) class HttprettyTestCase(TestCase): diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index a35afc27..71e02e5d 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -3,6 +3,13 @@ import gzip import logging import os +import shutil +import tempfile + +try: + from unittest import mock +except ImportError: + import mock from six import BytesIO, StringIO @@ -43,9 +50,9 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): self._log_handler = None def tearDown(self): - helpers.FilesystemMockingTestCase.tearDown(self) if self._log_handler and self._log: self._log.removeHandler(self._log_handler) + helpers.FilesystemMockingTestCase.tearDown(self) def _patchIn(self, root): self.restore() @@ -71,7 +78,8 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): ci = stages.Init() ci.datasource = FakeDataSource(blob) - new_root = self.makeDir() + new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, new_root) self.patchUtils(new_root) self.patchOS(new_root) ci.fetch() @@ -99,7 +107,8 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): { "op": "add", "path": "/foo", "value": "quxC" } ] ''' - new_root = self.makeDir() + new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, new_root) self._patchIn(new_root) initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) @@ -138,7 +147,8 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): { "op": "add", "path": "/foo", "value": "quxC" } ] ''' - new_root = self.makeDir() + new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, new_root) self._patchIn(new_root) initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) @@ -184,7 +194,8 @@ c: d ci = stages.Init() ci.datasource = FakeDataSource(str(message)) - new_root = self.makeDir() + new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, new_root) self.patchUtils(new_root) self.patchOS(new_root) ci.fetch() @@ -214,7 +225,8 @@ name: user run: - z ''' - new_root = self.makeDir() + new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, new_root) self._patchIn(new_root) initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) @@ -249,7 +261,8 @@ vendor_data: enabled: True prefix: /bin/true ''' - new_root = self.makeDir() + new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, new_root) self._patchIn(new_root) initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) @@ -309,7 +322,8 @@ p: 1 paths = c_helpers.Paths({}, ds=FakeDataSource('')) cloud_cfg = handlers.cloud_config.CloudConfigPartHandler(paths) - new_root = self.makeDir() + new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, new_root) self.patchUtils(new_root) self.patchOS(new_root) cloud_cfg.handle_part(None, handlers.CONTENT_START, None, None, None, @@ -374,7 +388,8 @@ c: 4 message.attach(gzip_part(base_content2)) ci = stages.Init() ci.datasource = FakeDataSource(str(message)) - new_root = self.makeDir() + new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, new_root) self.patchUtils(new_root) self.patchOS(new_root) ci.fetch() @@ -394,17 +409,15 @@ c: 4 message.set_payload("Just text") ci.datasource = FakeDataSource(message.as_string()) - mock_write = self.mocker.replace("cloudinit.util.write_file", - passthrough=False) - mock_write(ci.paths.get_ipath("cloud_config"), "", 0o600) - self.mocker.replay() - - log_file = self.capture_log(logging.WARNING) - ci.fetch() - ci.consume_data() - self.assertIn( - "Unhandled unknown content-type (text/plain)", - log_file.getvalue()) + with mock.patch('cloudinit.util.write_file') as mockobj: + log_file = self.capture_log(logging.WARNING) + ci.fetch() + ci.consume_data() + self.assertIn( + "Unhandled unknown content-type (text/plain)", + log_file.getvalue()) + mockobj.assert_called_once_with( + ci.paths.get_ipath("cloud_config"), "", 0o600) def test_shellscript(self): """Raw text starting #!/bin/sh is treated as script.""" @@ -413,16 +426,17 @@ c: 4 ci.datasource = FakeDataSource(script) outpath = os.path.join(ci.paths.get_ipath_cur("scripts"), "part-001") - mock_write = self.mocker.replace("cloudinit.util.write_file", - passthrough=False) - mock_write(ci.paths.get_ipath("cloud_config"), "", 0o600) - mock_write(outpath, script, 0o700) - self.mocker.replay() - log_file = self.capture_log(logging.WARNING) - ci.fetch() - ci.consume_data() - self.assertEqual("", log_file.getvalue()) + with mock.patch('cloudinit.util.write_file') as mockobj: + log_file = self.capture_log(logging.WARNING) + ci.fetch() + ci.consume_data() + self.assertEqual("", log_file.getvalue()) + + mockobj.assert_has_calls([ + mock.call(outpath, script, 0o700), + mock.call(ci.paths.get_ipath("cloud_config"), "", 0o600), + ]) def test_mime_text_x_shellscript(self): """Mime message of type text/x-shellscript is treated as script.""" @@ -433,16 +447,17 @@ c: 4 ci.datasource = FakeDataSource(message.as_string()) outpath = os.path.join(ci.paths.get_ipath_cur("scripts"), "part-001") - mock_write = self.mocker.replace("cloudinit.util.write_file", - passthrough=False) - mock_write(ci.paths.get_ipath("cloud_config"), "", 0o600) - mock_write(outpath, script, 0o700) - self.mocker.replay() - log_file = self.capture_log(logging.WARNING) - ci.fetch() - ci.consume_data() - self.assertEqual("", log_file.getvalue()) + with mock.patch('cloudinit.util.write_file') as mockobj: + log_file = self.capture_log(logging.WARNING) + ci.fetch() + ci.consume_data() + self.assertEqual("", log_file.getvalue()) + + mockobj.assert_has_calls([ + mock.call(outpath, script, 0o700), + mock.call(ci.paths.get_ipath("cloud_config"), "", 0o600), + ]) def test_mime_text_plain_shell(self): """Mime type text/plain starting #!/bin/sh is treated as script.""" @@ -453,13 +468,14 @@ c: 4 ci.datasource = FakeDataSource(message.as_string()) outpath = os.path.join(ci.paths.get_ipath_cur("scripts"), "part-001") - mock_write = self.mocker.replace("cloudinit.util.write_file", - passthrough=False) - mock_write(outpath, script, 0o700) - mock_write(ci.paths.get_ipath("cloud_config"), "", 0o600) - self.mocker.replay() - log_file = self.capture_log(logging.WARNING) - ci.fetch() - ci.consume_data() - self.assertEqual("", log_file.getvalue()) + with mock.patch('cloudinit.util.write_file') as mockobj: + log_file = self.capture_log(logging.WARNING) + ci.fetch() + ci.consume_data() + self.assertEqual("", log_file.getvalue()) + + mockobj.assert_has_calls([ + mock.call(outpath, script, 0o700), + mock.call(ci.paths.get_ipath("cloud_config"), "", 0o600), + ]) diff --git a/tox.ini b/tox.ini index d050d7c8..e547c693 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = py26,py27,py34 recreate = True [testenv] -commands = python -m nose -v tests +commands = python -m nose tests deps = contextlib2 httpretty>=0.7.1 -- cgit v1.2.3 From b3bbd3985a75d00115e1623dc1ebd4248a7024a6 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 22 Jan 2015 15:42:30 -0500 Subject: Clean up. --- tests/unittests/helpers.py | 6 ------ 1 file changed, 6 deletions(-) (limited to 'tests/unittests/helpers.py') diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 43dd5aa0..38a2176d 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -121,9 +121,6 @@ def retarget_many_wrapper(new_base, am, old_func): class ResourceUsingTestCase(unittest.TestCase): - ## def __init__(self, methodName="runTest"): - ## self.resource_path = None - def setUp(self): unittest.TestCase.setUp(self) self.resource_path = None @@ -163,9 +160,6 @@ class ResourceUsingTestCase(unittest.TestCase): class FilesystemMockingTestCase(ResourceUsingTestCase): - ## def __init__(self, methodName="runTest"): - ## ResourceUsingTestCase.__init__(self, methodName) - def setUp(self): ResourceUsingTestCase.setUp(self) self.patched_funcs = ExitStack() -- cgit v1.2.3 From 3b798b5d5c3caa5d0e8e534855e29010ca932aaa Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 22 Jan 2015 21:21:04 -0500 Subject: Low hanging Python 3 fruit. --- cloudinit/config/cc_ca_certs.py | 4 ++-- cloudinit/config/cc_chef.py | 6 ++++-- cloudinit/distros/__init__.py | 12 ++++++++++-- cloudinit/distros/debian.py | 2 +- cloudinit/distros/rhel_util.py | 4 ++-- cloudinit/distros/sles.py | 2 +- cloudinit/sources/DataSourceAltCloud.py | 12 ++++++------ cloudinit/sources/DataSourceAzure.py | 4 ++-- cloudinit/sources/DataSourceMAAS.py | 10 ++++++---- cloudinit/sources/DataSourceOpenNebula.py | 2 +- cloudinit/templater.py | 2 +- cloudinit/util.py | 7 +++++-- templates/resolv.conf.tmpl | 2 +- tests/unittests/helpers.py | 4 ++-- tests/unittests/test_datasource/test_configdrive.py | 2 +- tests/unittests/test_datasource/test_digitalocean.py | 7 +++---- tests/unittests/test_datasource/test_gce.py | 2 +- tests/unittests/test_datasource/test_opennebula.py | 2 +- tests/unittests/test_datasource/test_smartos.py | 4 +++- .../unittests/test_handler/test_handler_apt_configure.py | 2 +- tests/unittests/test_merging.py | 16 +++++++++------- tools/ccfg-merge-debug | 4 ++-- 22 files changed, 65 insertions(+), 47 deletions(-) (limited to 'tests/unittests/helpers.py') diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 4f2a46a1..8248b020 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -44,7 +44,7 @@ def add_ca_certs(certs): if certs: # First ensure they are strings... cert_file_contents = "\n".join([str(c) for c in certs]) - util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0644) + util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0o644) # Append cert filename to CA_CERT_CONFIG file. # We have to strip the content because blank lines in the file @@ -63,7 +63,7 @@ def remove_default_ca_certs(): """ util.delete_dir_contents(CA_CERT_PATH) util.delete_dir_contents(CA_CERT_SYSTEM_PATH) - util.write_file(CA_CERT_CONFIG, "", mode=0644) + util.write_file(CA_CERT_CONFIG, "", mode=0o644) debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" util.subp(('debconf-set-selections', '-'), debconf_sel) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index fc837363..584199e5 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -76,6 +76,8 @@ from cloudinit import templater from cloudinit import url_helper from cloudinit import util +import six + RUBY_VERSION_DEFAULT = "1.8" CHEF_DIRS = tuple([ @@ -261,7 +263,7 @@ def run_chef(chef_cfg, log): cmd_args = chef_cfg['exec_arguments'] if isinstance(cmd_args, (list, tuple)): cmd.extend(cmd_args) - elif isinstance(cmd_args, (str, basestring)): + elif isinstance(cmd_args, six.string_types): cmd.append(cmd_args) else: log.warn("Unknown type %s provided for chef" @@ -300,7 +302,7 @@ def install_chef(cloud, chef_cfg, log): with util.tempdir() as tmpd: # Use tmpdir over tmpfile to avoid 'text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd - util.write_file(tmpf, str(content), mode=0700) + util.write_file(tmpf, str(content), mode=0o700) util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type '%s'", install_type) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 4ebccdda..6b96d58c 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -25,7 +25,6 @@ import six from six import StringIO import abc -import itertools import os import re @@ -37,6 +36,15 @@ from cloudinit import util from cloudinit.distros.parsers import hosts +try: + # Python 3 + from six import filter +except ImportError: + # Python 2 + from itertools import ifilter as filter + + + OSFAMILIES = { 'debian': ['debian', 'ubuntu'], 'redhat': ['fedora', 'rhel'], @@ -853,7 +861,7 @@ def extract_default(users, default_name=None, default_config=None): return config['default'] tmp_users = users.items() - tmp_users = dict(itertools.ifilter(safe_find, tmp_users)) + tmp_users = dict(filter(safe_find, tmp_users)) if not tmp_users: return (default_name, default_config) else: diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index b09eb094..6d3a82bf 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -97,7 +97,7 @@ class Distro(distros.Distro): if not conf: conf = HostnameConf('') conf.set_hostname(your_hostname) - util.write_file(out_fn, str(conf), 0644) + util.write_file(out_fn, str(conf), 0o644) def _read_system_hostname(self): sys_hostname = self._read_hostname(self.hostname_conf_fn) diff --git a/cloudinit/distros/rhel_util.py b/cloudinit/distros/rhel_util.py index 063d536e..903d7793 100644 --- a/cloudinit/distros/rhel_util.py +++ b/cloudinit/distros/rhel_util.py @@ -50,7 +50,7 @@ def update_sysconfig_file(fn, adjustments, allow_empty=False): ] if not exists: lines.insert(0, util.make_header()) - util.write_file(fn, "\n".join(lines) + "\n", 0644) + util.write_file(fn, "\n".join(lines) + "\n", 0o644) # Helper function to read a RHEL/SUSE /etc/sysconfig/* file @@ -86,4 +86,4 @@ def update_resolve_conf_file(fn, dns_servers, search_servers): r_conf.add_search_domain(s) except ValueError: util.logexc(LOG, "Failed at adding search domain %s", s) - util.write_file(fn, str(r_conf), 0644) + util.write_file(fn, str(r_conf), 0o644) diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py index 0c6d1203..620c974c 100644 --- a/cloudinit/distros/sles.py +++ b/cloudinit/distros/sles.py @@ -113,7 +113,7 @@ class Distro(distros.Distro): if not conf: conf = HostnameConf('') conf.set_hostname(hostname) - util.write_file(out_fn, str(conf), 0644) + util.write_file(out_fn, str(conf), 0o644) def _read_system_hostname(self): host_fn = self.hostname_conf_fn diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index 1e913a6e..69053d0b 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -124,11 +124,11 @@ class DataSourceAltCloud(sources.DataSource): cmd = CMD_DMI_SYSTEM try: (cmd_out, _err) = util.subp(cmd) - except ProcessExecutionError, _err: + except ProcessExecutionError as _err: LOG.debug(('Failed command: %s\n%s') % \ (' '.join(cmd), _err.message)) return 'UNKNOWN' - except OSError, _err: + except OSError as _err: LOG.debug(('Failed command: %s\n%s') % \ (' '.join(cmd), _err.message)) return 'UNKNOWN' @@ -211,11 +211,11 @@ class DataSourceAltCloud(sources.DataSource): cmd = CMD_PROBE_FLOPPY (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) - except ProcessExecutionError, _err: + except ProcessExecutionError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err.message) return False - except OSError, _err: + except OSError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err.message) return False @@ -228,11 +228,11 @@ class DataSourceAltCloud(sources.DataSource): cmd.append('--exit-if-exists=' + floppy_dev) (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) - except ProcessExecutionError, _err: + except ProcessExecutionError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err.message) return False - except OSError, _err: + except OSError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err.message) return False diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 09bc196d..29ae2c22 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -151,7 +151,7 @@ class DataSourceAzureNet(sources.DataSource): # walinux agent writes files world readable, but expects # the directory to be protected. - write_files(ddir, files, dirmode=0700) + write_files(ddir, files, dirmode=0o700) # handle the hostname 'publishing' try: @@ -390,7 +390,7 @@ def write_files(datadir, files, dirmode=None): util.ensure_dir(datadir, dirmode) for (name, content) in files.items(): util.write_file(filename=os.path.join(datadir, name), - content=content, mode=0600) + content=content, mode=0o600) def invoke_agent(cmd): diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 9a3e30c5..8f9c81de 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -18,6 +18,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function + from email.utils import parsedate import errno import oauth.oauth as oauth @@ -361,7 +363,7 @@ if __name__ == "__main__": return (urllib2.urlopen(req).read()) def printurl(url, headers_cb): - print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) + print("== %s ==\n%s\n" % (url, geturl(url, headers_cb))) def crawl(url, headers_cb=None): if url.endswith("/"): @@ -386,9 +388,9 @@ if __name__ == "__main__": version=args.apiver) else: (userdata, metadata) = read_maas_seed_url(args.url) - print "=== userdata ===" - print userdata - print "=== metadata ===" + print("=== userdata ===") + print(userdata) + print("=== metadata ===") pprint.pprint(metadata) elif args.subcmd == "get": diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index e2469f6e..f9dac29e 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -280,7 +280,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None, # allvars expands to all existing variables by using '${!x*}' notation # where x is lower or upper case letters or '_' - allvars = ["${!%s*}" % x for x in string.letters + "_"] + allvars = ["${!%s*}" % x for x in string.ascii_letters + "_"] keylist_in = keylist if keylist is None: diff --git a/cloudinit/templater.py b/cloudinit/templater.py index 4cd3f13d..a9231482 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -137,7 +137,7 @@ def render_from_file(fn, params): return renderer(content, params) -def render_to_file(fn, outfn, params, mode=0644): +def render_to_file(fn, outfn, params, mode=0o644): contents = render_from_file(fn, params) util.write_file(outfn, contents, mode=mode) diff --git a/cloudinit/util.py b/cloudinit/util.py index 434ba7fb..94fd5c70 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -142,6 +142,9 @@ class ProcessExecutionError(IOError): 'reason': self.reason, } IOError.__init__(self, message) + # For backward compatibility with Python 2. + if not hasattr(self, 'message'): + self.message = message class SeLinuxGuard(object): @@ -260,7 +263,7 @@ def translate_bool(val, addons=None): def rand_str(strlen=32, select_from=None): if not select_from: - select_from = string.letters + string.digits + select_from = string.ascii_letters + string.digits return "".join([random.choice(select_from) for _x in range(0, strlen)]) @@ -1127,7 +1130,7 @@ def pipe_in_out(in_fh, out_fh, chunk_size=1024, chunk_cb=None): bytes_piped = 0 while True: data = in_fh.read(chunk_size) - if data == '': + if len(data) == 0: break else: out_fh.write(data) diff --git a/templates/resolv.conf.tmpl b/templates/resolv.conf.tmpl index 1300156c..bfae80db 100644 --- a/templates/resolv.conf.tmpl +++ b/templates/resolv.conf.tmpl @@ -24,7 +24,7 @@ sortlist {% for sort in sortlist %}{{sort}} {% endfor %} {% if options or flags %} options {% for flag in flags %}{{flag}} {% endfor %} -{% for key, value in options.iteritems() -%} +{% for key, value in options.items() -%} {{key}}:{{value}} {% endfor %} {% endif %} diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 38a2176d..70b8116f 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -65,7 +65,7 @@ if PY26: def assertDictContainsSubset(self, expected, actual, msg=None): missing = [] mismatched = [] - for k, v in expected.iteritems(): + for k, v in expected.items(): if k not in actual: missing.append(k) elif actual[k] != v: @@ -243,7 +243,7 @@ class HttprettyTestCase(TestCase): def populate_dir(path, files): if not os.path.exists(path): os.makedirs(path) - for (name, content) in files.iteritems(): + for (name, content) in files.items(): with open(os.path.join(path, name), "w") as fp: fp.write(content) fp.close() diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 800c5fd8..258c68e2 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -338,7 +338,7 @@ def populate_ds_from_read_config(cfg_ds, source, results): def populate_dir(seed_dir, files): - for (name, content) in files.iteritems(): + for (name, content) in files.items(): path = os.path.join(seed_dir, name) dirname = os.path.dirname(path) if not os.path.isdir(dirname): diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index d1270fc2..98f9cfac 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -18,8 +18,7 @@ import httpretty import re -from types import ListType -from urlparse import urlparse +from six.moves.urllib_parse import urlparse from cloudinit import settings from cloudinit import helpers @@ -110,7 +109,7 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase): self.assertEqual([DO_META.get('public-keys')], self.ds.get_public_ssh_keys()) - self.assertIs(type(self.ds.get_public_ssh_keys()), ListType) + self.assertIsInstance(self.ds.get_public_ssh_keys(), list) @httpretty.activate def test_multiple_ssh_keys(self): @@ -124,4 +123,4 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase): self.assertEqual(DO_META.get('public-keys').splitlines(), self.ds.get_public_ssh_keys()) - self.assertIs(type(self.ds.get_public_ssh_keys()), ListType) + self.assertIsInstance(self.ds.get_public_ssh_keys(), list) diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 06050bb1..aa60eb33 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -19,7 +19,7 @@ import httpretty import re from base64 import b64encode, b64decode -from urlparse import urlparse +from six.moves.urllib_parse import urlparse from cloudinit import settings from cloudinit import helpers diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index ddf77265..b79237f0 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -294,7 +294,7 @@ class TestParseShellConfig(unittest.TestCase): def populate_context_dir(path, variables): data = "# Context variables generated by OpenNebula\n" - for (k, v) in variables.iteritems(): + for (k, v) in variables.items(): data += ("%s='%s'\n" % (k.upper(), v.replace(r"'", r"'\''"))) populate_dir(path, {'context.sh': data}) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 35d7ef5e..01b9b73e 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -22,6 +22,8 @@ # return responses. # +from __future__ import print_function + import base64 from cloudinit import helpers as c_helpers from cloudinit.sources import DataSourceSmartOS @@ -369,7 +371,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): permissions = oct(os.stat(name_f)[stat.ST_MODE])[-3:] if re.match(r'.*\/mdata-user-data$', name_f): found_new = True - print name_f + print(name_f) self.assertEquals(permissions, '400') self.assertFalse(found_new) diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py index 2c3dad72..d72fa8c7 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure.py +++ b/tests/unittests/test_handler/test_handler_apt_configure.py @@ -62,7 +62,7 @@ class TestAptProxyConfig(unittest.TestCase): contents = str(util.read_file_or_url(self.pfile)) - for ptype, pval in values.iteritems(): + for ptype, pval in values.items(): self.assertTrue(self._search_apt_config(contents, ptype, pval)) def test_proxy_deleted(self): diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 07b610f7..976d8283 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -11,11 +11,13 @@ import glob import os import random import re +import six import string SOURCE_PAT = "source*.*yaml" EXPECTED_PAT = "expected%s.yaml" -TYPES = [long, int, dict, str, list, tuple, None] +TYPES = [dict, str, list, tuple, None] +TYPES.extend(six.integer_types) def _old_mergedict(src, cand): @@ -25,7 +27,7 @@ def _old_mergedict(src, cand): Nested dictionaries are merged recursively. """ if isinstance(src, dict) and isinstance(cand, dict): - for (k, v) in cand.iteritems(): + for (k, v) in cand.items(): if k not in src: src[k] = v else: @@ -42,8 +44,8 @@ def _old_mergemanydict(*args): def _random_str(rand): base = '' - for _i in xrange(rand.randint(1, 2 ** 8)): - base += rand.choice(string.letters + string.digits) + for _i in range(rand.randint(1, 2 ** 8)): + base += rand.choice(string.ascii_letters + string.digits) return base @@ -64,7 +66,7 @@ def _make_dict(current_depth, max_depth, rand): if t in [dict, list, tuple]: if t in [dict]: amount = rand.randint(0, 5) - keys = [_random_str(rand) for _i in xrange(0, amount)] + keys = [_random_str(rand) for _i in range(0, amount)] base = {} for k in keys: try: @@ -74,14 +76,14 @@ def _make_dict(current_depth, max_depth, rand): elif t in [list, tuple]: base = [] amount = rand.randint(0, 5) - for _i in xrange(0, amount): + for _i in range(0, amount): try: base.append(_make_dict(current_depth + 1, max_depth, rand)) except _NoMoreException: pass if t in [tuple]: base = tuple(base) - elif t in [long, int]: + elif t in six.integer_types: base = rand.randint(0, 2 ** 8) elif t in [str]: base = _random_str(rand) diff --git a/tools/ccfg-merge-debug b/tools/ccfg-merge-debug index 85227da7..1f08e0cb 100755 --- a/tools/ccfg-merge-debug +++ b/tools/ccfg-merge-debug @@ -51,7 +51,7 @@ def main(): c_handlers.register(ccph) called = [] - for (_ctype, mod) in c_handlers.iteritems(): + for (_ctype, mod) in c_handlers.items(): if mod in called: continue handlers.call_begin(mod, data, frequency) @@ -76,7 +76,7 @@ def main(): # Give callbacks opportunity to finalize called = [] - for (_ctype, mod) in c_handlers.iteritems(): + for (_ctype, mod) in c_handlers.items(): if mod in called: continue handlers.call_end(mod, data, frequency) -- cgit v1.2.3 From 841db73600e3f203243c773109d71ab88d3334bc Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 26 Jan 2015 11:14:06 -0500 Subject: More test repairs. --- cloudinit/distros/__init__.py | 2 +- cloudinit/user_data.py | 9 +++++++ tests/unittests/helpers.py | 12 ++++++--- tests/unittests/test_builtin_handlers.py | 1 - tests/unittests/test_datasource/test_azure.py | 31 +++++++++++++--------- tests/unittests/test_datasource/test_gce.py | 2 +- tests/unittests/test_datasource/test_opennebula.py | 10 +++++-- tests/unittests/test_datasource/test_smartos.py | 12 ++++++--- tests/unittests/test_filters/test_launch_index.py | 8 +++--- tests/unittests/test_handler/test_handler_chef.py | 3 ++- .../test_handler/test_handler_seed_random.py | 11 ++++++-- tests/unittests/test_util.py | 6 ++--- 12 files changed, 73 insertions(+), 34 deletions(-) (limited to 'tests/unittests/helpers.py') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 00fb95fb..ab874b45 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -857,7 +857,7 @@ def extract_default(users, default_name=None, default_config=None): if not tmp_users: return (default_name, default_config) else: - name = tmp_users.keys()[0] + name = list(tmp_users)[0] config = tmp_users[name] config.pop('default', None) return (name, config) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 9111bd39..ff21259c 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -109,6 +109,15 @@ class UserDataProcessor(object): ctype = None ctype_orig = part.get_content_type() payload = part.get_payload(decode=True) + # In Python 3, decoding the payload will ironically hand us a + # bytes object. 'decode' means to decode according to + # Content-Transfer-Encoding, not according to any charset in the + # Content-Type. So, if we end up with bytes, first try to decode + # to str via CT charset, and failing that, try utf-8 using + # surrogate escapes. + if six.PY3 and isinstance(payload, bytes): + charset = part.get_charset() or 'utf-8' + payload = payload.decode(charset, errors='surrogateescape') was_compressed = False # When the message states it is of a gzipped content type ensure diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 70b8116f..4b8dcc5c 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -1,8 +1,11 @@ import os import sys +import shutil import tempfile import unittest +import six + try: from unittest import mock except ImportError: @@ -15,8 +18,6 @@ except ImportError: from cloudinit import helpers as ch from cloudinit import util -import shutil - # Used for detecting different python versions PY2 = False PY26 = False @@ -115,7 +116,12 @@ def retarget_many_wrapper(new_base, am, old_func): nam = len(n_args) for i in range(0, nam): path = args[i] - n_args[i] = rebase_path(path, new_base) + # patchOS() wraps various os and os.path functions, however in + # Python 3 some of these now accept file-descriptors (integers). + # That breaks rebase_path() so in lieu of a better solution, just + # don't rebase if we get a fd. + if isinstance(path, six.string_types): + n_args[i] = rebase_path(path, new_base) return old_func(*n_args, **kwds) return wrapper diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 47ff6318..ad32d0b2 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -21,7 +21,6 @@ from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) class TestBuiltins(test_helpers.FilesystemMockingTestCase): - def test_upstart_frequency_no_out(self): c_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, c_root) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 2dbcd389..1f0330b3 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -22,6 +22,13 @@ import tempfile import unittest +def b64(source): + # In Python 3, b64encode only accepts bytes and returns bytes. + if not isinstance(source, bytes): + source = source.encode('utf-8') + return base64.b64encode(source).decode('us-ascii') + + def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): if data is None: data = {'HostName': 'FOOHOST'} @@ -51,7 +58,7 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): content += "<%s%s>%s\n" % (key, attrs, val, key) if userdata: - content += "%s\n" % (base64.b64encode(userdata)) + content += "%s\n" % (b64(userdata)) if pubkeys: content += "\n" @@ -181,7 +188,7 @@ class TestAzureDataSource(unittest.TestCase): # set dscfg in via base64 encoded yaml cfg = {'agent_command': "my_command"} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': base64.b64encode(yaml.dump(cfg)), + 'dscfg': {'text': b64(yaml.dump(cfg)), 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} @@ -233,13 +240,13 @@ class TestAzureDataSource(unittest.TestCase): def test_userdata_found(self): mydata = "FOOBAR" - odata = {'UserData': base64.b64encode(mydata)} + odata = {'UserData': b64(mydata)} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEqual(dsrc.userdata_raw, mydata) + self.assertEqual(dsrc.userdata_raw, mydata.encode('utf-8')) def test_no_datasource_expected(self): # no source should be found if no seed_dir and no devs @@ -281,7 +288,7 @@ class TestAzureDataSource(unittest.TestCase): 'command': 'my-bounce-command', 'hostname_command': 'my-hostname-command'}} odata = {'HostName': "xhost", - 'dscfg': {'text': base64.b64encode(yaml.dump(cfg)), + 'dscfg': {'text': b64(yaml.dump(cfg)), 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} self._get_ds(data).get_data() @@ -296,7 +303,7 @@ class TestAzureDataSource(unittest.TestCase): # config specifying set_hostname off should not bounce cfg = {'set_hostname': False} odata = {'HostName': "xhost", - 'dscfg': {'text': base64.b64encode(yaml.dump(cfg)), + 'dscfg': {'text': b64(yaml.dump(cfg)), 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} self._get_ds(data).get_data() @@ -325,7 +332,7 @@ class TestAzureDataSource(unittest.TestCase): # Make sure that user can affect disk aliases dscfg = {'disk_aliases': {'ephemeral0': '/dev/sdc'}} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': base64.b64encode(yaml.dump(dscfg)), + 'dscfg': {'text': b64(yaml.dump(dscfg)), 'encoding': 'base64'}} usercfg = {'disk_setup': {'/dev/sdc': {'something': '...'}, 'ephemeral0': False}} @@ -347,7 +354,7 @@ class TestAzureDataSource(unittest.TestCase): dsrc = self._get_ds(data) dsrc.get_data() - self.assertEqual(userdata, dsrc.userdata_raw) + self.assertEqual(userdata.encode('us-ascii'), dsrc.userdata_raw) def test_ovf_env_arrives_in_waagent_dir(self): xml = construct_valid_ovf_env(data={}, userdata="FOODATA") @@ -362,7 +369,7 @@ class TestAzureDataSource(unittest.TestCase): def test_existing_ovf_same(self): # waagent/SharedConfig left alone if found ovf-env.xml same as cached - odata = {'UserData': base64.b64encode("SOMEUSERDATA")} + odata = {'UserData': b64("SOMEUSERDATA")} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} populate_dir(self.waagent_d, @@ -386,9 +393,9 @@ class TestAzureDataSource(unittest.TestCase): # 'get_data' should remove SharedConfig.xml in /var/lib/waagent # if ovf-env.xml differs. cached_ovfenv = construct_valid_ovf_env( - {'userdata': base64.b64encode("FOO_USERDATA")}) + {'userdata': b64("FOO_USERDATA")}) new_ovfenv = construct_valid_ovf_env( - {'userdata': base64.b64encode("NEW_USERDATA")}) + {'userdata': b64("NEW_USERDATA")}) populate_dir(self.waagent_d, {'ovf-env.xml': cached_ovfenv, @@ -398,7 +405,7 @@ class TestAzureDataSource(unittest.TestCase): dsrc = self._get_ds({'ovfcontent': new_ovfenv}) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEqual(dsrc.userdata_raw, "NEW_USERDATA") + self.assertEqual(dsrc.userdata_raw, b"NEW_USERDATA") self.assertTrue(os.path.exists( os.path.join(self.waagent_d, 'otherfile'))) self.assertFalse( diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index aa60eb33..6dd4b5ed 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -45,7 +45,7 @@ GCE_META_ENCODING = { 'instance/id': '12345', 'instance/hostname': 'server.project-baz.local', 'instance/zone': 'baz/bang', - 'instance/attributes/user-data': b64encode('/bin/echo baz\n'), + 'instance/attributes/user-data': b64encode(b'/bin/echo baz\n'), 'instance/attributes/user-data-encoding': 'base64', } diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index b79237f0..1a8d2122 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -10,6 +10,12 @@ import shutil import tempfile import unittest +def b64(source): + # In Python 3, b64encode only accepts bytes and returns bytes. + if not isinstance(source, bytes): + source = source.encode('utf-8') + return b64encode(source).decode('us-ascii') + TEST_VARS = { 'VAR1': 'single', @@ -180,7 +186,7 @@ class TestOpenNebulaDataSource(unittest.TestCase): self.assertEqual(USER_DATA, results['userdata']) def test_user_data_encoding_required_for_decode(self): - b64userdata = b64encode(USER_DATA) + b64userdata = b64(USER_DATA) for k in ('USER_DATA', 'USERDATA'): my_d = os.path.join(self.tmp, k) populate_context_dir(my_d, {k: b64userdata}) @@ -192,7 +198,7 @@ class TestOpenNebulaDataSource(unittest.TestCase): def test_user_data_base64_encoding(self): for k in ('USER_DATA', 'USERDATA'): my_d = os.path.join(self.tmp, k) - populate_context_dir(my_d, {k: b64encode(USER_DATA), + populate_context_dir(my_d, {k: b64(USER_DATA), 'USERDATA_ENCODING': 'base64'}) results = ds.read_context_disk_dir(my_d) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 01b9b73e..2fb9e1b6 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -36,6 +36,12 @@ import tempfile import stat import uuid +def b64(source): + # In Python 3, b64encode only accepts bytes and returns bytes. + if not isinstance(source, bytes): + source = source.encode('utf-8') + return base64.b64encode(source).decode('us-ascii') + MOCK_RETURNS = { 'hostname': 'test-host', @@ -233,7 +239,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): my_returns = MOCK_RETURNS.copy() my_returns['base64_all'] = "true" for k in ('hostname', 'cloud-init:user-data'): - my_returns[k] = base64.b64encode(my_returns[k]) + my_returns[k] = b64(my_returns[k]) dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() @@ -254,7 +260,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): my_returns['b64-cloud-init:user-data'] = "true" my_returns['b64-hostname'] = "true" for k in ('hostname', 'cloud-init:user-data'): - my_returns[k] = base64.b64encode(my_returns[k]) + my_returns[k] = b64(my_returns[k]) dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() @@ -270,7 +276,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): my_returns = MOCK_RETURNS.copy() my_returns['base64_keys'] = 'hostname,ignored' for k in ('hostname',): - my_returns[k] = base64.b64encode(my_returns[k]) + my_returns[k] = b64(my_returns[k]) dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py index 2f4c2fda..95d24b9b 100644 --- a/tests/unittests/test_filters/test_launch_index.py +++ b/tests/unittests/test_filters/test_launch_index.py @@ -2,7 +2,7 @@ import copy from .. import helpers -import itertools +from six.moves import filterfalse from cloudinit.filters import launch_index from cloudinit import user_data as ud @@ -36,11 +36,9 @@ class TestLaunchFilter(helpers.ResourceUsingTestCase): return False # Do some basic payload checking msg1_msgs = [m for m in msg1.walk()] - msg1_msgs = [m for m in - itertools.ifilterfalse(ud.is_skippable, msg1_msgs)] + msg1_msgs = [m for m in filterfalse(ud.is_skippable, msg1_msgs)] msg2_msgs = [m for m in msg2.walk()] - msg2_msgs = [m for m in - itertools.ifilterfalse(ud.is_skippable, msg2_msgs)] + msg2_msgs = [m for m in filterfalse(ud.is_skippable, msg2_msgs)] for i in range(0, len(msg2_msgs)): m1_msg = msg1_msgs[i] m2_msg = msg2_msgs[i] diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index b06a160c..8ab27911 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -11,6 +11,7 @@ from cloudinit.sources import DataSourceNone from .. import helpers as t_help +import six import logging import shutil import tempfile @@ -77,7 +78,7 @@ class TestChef(t_help.FilesystemMockingTestCase): for k, v in cfg['chef'].items(): self.assertIn(v, c) for k, v in cc_chef.CHEF_RB_TPL_DEFAULTS.items(): - if isinstance(v, basestring): + if isinstance(v, six.string_types): self.assertIn(v, c) c = util.load_file(cc_chef.CHEF_FB_PATH) self.assertEqual({}, json.loads(c)) diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index 579377fb..c2da5ced 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -38,6 +38,13 @@ import logging LOG = logging.getLogger(__name__) +def b64(source): + # In Python 3, b64encode only accepts bytes and returns bytes. + if not isinstance(source, bytes): + source = source.encode('utf-8') + return base64.b64encode(source).decode('us-ascii') + + class TestRandomSeed(t_help.TestCase): def setUp(self): super(TestRandomSeed, self).setUp() @@ -134,7 +141,7 @@ class TestRandomSeed(t_help.TestCase): self.assertEquals("big-toe", contents) def test_append_random_base64(self): - data = base64.b64encode('bubbles') + data = b64('bubbles') cfg = { 'random_seed': { 'file': self._seed_file, @@ -147,7 +154,7 @@ class TestRandomSeed(t_help.TestCase): self.assertEquals("bubbles", contents) def test_append_random_b64(self): - data = base64.b64encode('kit-kat') + data = b64('kit-kat') cfg = { 'random_seed': { 'file': self._seed_file, diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index b1f5d62c..b0207ace 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -119,7 +119,7 @@ class TestWriteFile(unittest.TestCase): # Create file first with basic content with open(path, "wb") as f: - f.write("LINE1\n") + f.write(b"LINE1\n") util.write_file(path, contents, omode="a") self.assertTrue(os.path.exists(path)) @@ -194,7 +194,7 @@ class TestDeleteDirContents(unittest.TestCase): os.mkdir(os.path.join(self.tmp, "new_dir")) f_name = os.path.join(self.tmp, "new_dir", "new_file.txt") with open(f_name, "wb") as f: - f.write("DELETE ME") + f.write(b"DELETE ME") util.delete_dir_contents(self.tmp) @@ -205,7 +205,7 @@ class TestDeleteDirContents(unittest.TestCase): file_name = os.path.join(self.tmp, "new_file.txt") link_name = os.path.join(self.tmp, "new_file_link.txt") with open(file_name, "wb") as f: - f.write("DELETE ME") + f.write(b"DELETE ME") os.symlink(file_name, link_name) util.delete_dir_contents(self.tmp) -- cgit v1.2.3 From 5e2b8ef0703eb4582a5a8ba50ae7c83a8294d65a Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 26 Jan 2015 20:02:31 -0500 Subject: Repair the Python 2.6 tests. --- cloudinit/util.py | 18 ++++++++-------- tests/unittests/helpers.py | 25 +++++++++++++++++++--- tests/unittests/test__init__.py | 6 ++++-- tests/unittests/test_cs_util.py | 16 +++++++++++++- tests/unittests/test_datasource/test_azure.py | 7 +++--- tests/unittests/test_datasource/test_cloudsigma.py | 1 + .../unittests/test_datasource/test_configdrive.py | 6 ++++-- tests/unittests/test_datasource/test_maas.py | 5 ++--- tests/unittests/test_datasource/test_nocloud.py | 7 +++--- tests/unittests/test_datasource/test_opennebula.py | 4 ++-- tests/unittests/test_distros/test_netconfig.py | 4 ++-- tests/unittests/test_distros/test_resolv.py | 4 ++-- tests/unittests/test_distros/test_sysconfig.py | 4 ++-- .../test_distros/test_user_data_normalize.py | 7 +++--- .../test_handler/test_handler_apt_configure.py | 3 ++- .../test_handler/test_handler_ca_certs.py | 7 +++--- .../test_handler/test_handler_growpart.py | 3 ++- tests/unittests/test_pathprefix2dict.py | 6 +++--- tests/unittests/test_templating.py | 19 +++++++++++++++- tests/unittests/test_util.py | 15 ++++++------- tox.ini | 10 +++++++++ 21 files changed, 122 insertions(+), 55 deletions(-) (limited to 'tests/unittests/helpers.py') diff --git a/cloudinit/util.py b/cloudinit/util.py index 32c19ba2..766f8e32 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2059,23 +2059,23 @@ def _read_dmi_syspath(key): Reads dmi data with from /sys/class/dmi/id """ - dmi_key = "{}/{}".format(DMI_SYS_PATH, key) - LOG.debug("querying dmi data {}".format(dmi_key)) + dmi_key = "{0}/{1}".format(DMI_SYS_PATH, key) + LOG.debug("querying dmi data {0}".format(dmi_key)) try: if not os.path.exists(dmi_key): - LOG.debug("did not find {}".format(dmi_key)) + LOG.debug("did not find {0}".format(dmi_key)) return None key_data = load_file(dmi_key) if not key_data: - LOG.debug("{} did not return any data".format(key)) + LOG.debug("{0} did not return any data".format(key)) return None - LOG.debug("dmi data {} returned {}".format(dmi_key, key_data)) + LOG.debug("dmi data {0} returned {0}".format(dmi_key, key_data)) return key_data.strip() except Exception as e: - logexc(LOG, "failed read of {}".format(dmi_key), e) + logexc(LOG, "failed read of {0}".format(dmi_key), e) return None @@ -2087,10 +2087,10 @@ def _call_dmidecode(key, dmidecode_path): try: cmd = [dmidecode_path, "--string", key] (result, _err) = subp(cmd) - LOG.debug("dmidecode returned '{}' for '{}'".format(result, key)) + LOG.debug("dmidecode returned '{0}' for '{0}'".format(result, key)) return result except OSError as _err: - LOG.debug('failed dmidecode cmd: {}\n{}'.format(cmd, _err.message)) + LOG.debug('failed dmidecode cmd: {0}\n{0}'.format(cmd, _err.message)) return None @@ -2106,7 +2106,7 @@ def read_dmi_data(key): if dmidecode_path: return _call_dmidecode(key, dmidecode_path) - LOG.warn("did not find either path {} or dmidecode command".format( + LOG.warn("did not find either path {0} or dmidecode command".format( DMI_SYS_PATH)) return None diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 828579e8..424d0626 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -39,8 +39,20 @@ else: PY3 = True if PY26: - # For now add these on, taken from python 2.7 + slightly adjusted + # For now add these on, taken from python 2.7 + slightly adjusted. Drop + # all this once Python 2.6 is dropped as a minimum requirement. class TestCase(unittest.TestCase): + def setUp(self): + unittest.TestCase.setUp(self) + self.__all_cleanups = ExitStack() + + def tearDown(self): + self.__all_cleanups.close() + unittest.TestCase.tearDown(self) + + def addCleanup(self, function, *args, **kws): + self.__all_cleanups.callback(function, *args, **kws) + def assertIs(self, expr1, expr2, msg=None): if expr1 is not expr2: standardMsg = '%r is not %r' % (expr1, expr2) @@ -63,6 +75,13 @@ if PY26: standardMsg = standardMsg % (value) self.fail(self._formatMessage(msg, standardMsg)) + def assertIsInstance(self, obj, cls, msg=None): + """Same as self.assertTrue(isinstance(obj, cls)), with a nicer + default message.""" + if not isinstance(obj, cls): + standardMsg = '%s is not an instance of %r' % (repr(obj), cls) + self.fail(self._formatMessage(msg, standardMsg)) + def assertDictContainsSubset(self, expected, actual, msg=None): missing = [] mismatched = [] @@ -126,9 +145,9 @@ def retarget_many_wrapper(new_base, am, old_func): return wrapper -class ResourceUsingTestCase(unittest.TestCase): +class ResourceUsingTestCase(TestCase): def setUp(self): - unittest.TestCase.setUp(self) + TestCase.setUp(self) self.resource_path = None def resourceLocation(self, subname=None): diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index f5dc3435..1a307e56 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -18,6 +18,8 @@ from cloudinit import settings from cloudinit import url_helper from cloudinit import util +from .helpers import TestCase + class FakeModule(handlers.Handler): def __init__(self): @@ -31,10 +33,10 @@ class FakeModule(handlers.Handler): pass -class TestWalkerHandleHandler(unittest.TestCase): +class TestWalkerHandleHandler(TestCase): def setUp(self): - unittest.TestCase.setUp(self) + super(TestWalkerHandleHandler, self).setUp() tmpdir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmpdir) diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/test_cs_util.py index 99fac84d..337ac9a0 100644 --- a/tests/unittests/test_cs_util.py +++ b/tests/unittests/test_cs_util.py @@ -1,7 +1,21 @@ +from __future__ import print_function + +import sys import unittest from cloudinit.cs_utils import Cepko +try: + skip = unittest.skip +except AttributeError: + # Python 2.6. Doesn't have to be high fidelity. + def skip(reason): + def decorator(func): + def wrapper(*args, **kws): + print(reason, file=sys.stderr) + return wrapper + return decorator + SERVER_CONTEXT = { "cpu": 1000, @@ -29,7 +43,7 @@ class CepkoMock(Cepko): # 2015-01-22 BAW: This test is completely useless because it only ever tests # the CepkoMock object. Even in its original form, I don't think it ever # touched the underlying Cepko class methods. -@unittest.skip('This test is completely useless') +@skip('This test is completely useless') class CepkoResultTests(unittest.TestCase): def setUp(self): pass diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 1f0330b3..97a53bee 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -1,7 +1,7 @@ from cloudinit import helpers from cloudinit.util import load_file from cloudinit.sources import DataSourceAzure -from ..helpers import populate_dir +from ..helpers import TestCase, populate_dir try: from unittest import mock @@ -84,9 +84,10 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): return content -class TestAzureDataSource(unittest.TestCase): +class TestAzureDataSource(TestCase): def setUp(self): + super(TestAzureDataSource, self).setUp() self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) @@ -416,7 +417,7 @@ class TestAzureDataSource(unittest.TestCase): load_file(os.path.join(self.waagent_d, 'ovf-env.xml'))) -class TestReadAzureOvf(unittest.TestCase): +class TestReadAzureOvf(TestCase): def test_invalid_xml_raises_non_azure_ds(self): invalid_xml = "" + construct_valid_ovf_env(data={}) self.assertRaises(DataSourceAzure.BrokenAzureDataSource, diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index 306ac7d8..772d189a 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -39,6 +39,7 @@ class CepkoMock(Cepko): class DataSourceCloudSigmaTest(test_helpers.TestCase): def setUp(self): + super(DataSourceCloudSigmaTest, self).setUp() self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "") self.datasource.is_running_in_cloudsigma = lambda: True self.datasource.cepko = CepkoMock(SERVER_CONTEXT) diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 258c68e2..fd930877 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -3,7 +3,6 @@ import json import os import shutil import tempfile -import unittest try: from unittest import mock @@ -20,6 +19,9 @@ from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit.sources.helpers import openstack from cloudinit import util +from ..helpers import TestCase + + PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' EC2_META = { 'ami-id': 'ami-00000001', @@ -70,7 +72,7 @@ CFG_DRIVE_FILES_V2 = { 'openstack/latest/user_data': USER_DATA} -class TestConfigDriveDataSource(unittest.TestCase): +class TestConfigDriveDataSource(TestCase): def setUp(self): super(TestConfigDriveDataSource, self).setUp() diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 6af0cd82..d25e1adc 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -2,11 +2,10 @@ from copy import copy import os import shutil import tempfile -import unittest from cloudinit.sources import DataSourceMAAS from cloudinit import url_helper -from ..helpers import populate_dir +from ..helpers import TestCase, populate_dir try: from unittest import mock @@ -14,7 +13,7 @@ except ImportError: import mock -class TestMAASDataSource(unittest.TestCase): +class TestMAASDataSource(TestCase): def setUp(self): super(TestMAASDataSource, self).setUp() diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 480a4012..4f967f58 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -1,7 +1,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNoCloud from cloudinit import util -from ..helpers import populate_dir +from ..helpers import TestCase, populate_dir import os import yaml @@ -19,9 +19,10 @@ except ImportError: from contextlib2 import ExitStack -class TestNoCloudDataSource(unittest.TestCase): +class TestNoCloudDataSource(TestCase): def setUp(self): + super(TestNoCloudDataSource, self).setUp() self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) self.paths = helpers.Paths({'cloud_dir': self.tmp}) @@ -34,8 +35,6 @@ class TestNoCloudDataSource(unittest.TestCase): self.mocks.enter_context( mock.patch.object(util, 'get_cmdline', return_value=self.cmdline)) - super(TestNoCloudDataSource, self).setUp() - def test_nocloud_seed_dir(self): md = {'instance-id': 'IID', 'dsmode': 'local'} ud = "USER_DATA_HERE" diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index ef534bab..e5a4bd18 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -1,7 +1,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceOpenNebula as ds from cloudinit import util -from ..helpers import populate_dir +from ..helpers import TestCase, populate_dir from base64 import b64encode import os @@ -46,7 +46,7 @@ CMD_IP_OUT = '''\ ''' -class TestOpenNebulaDataSource(unittest.TestCase): +class TestOpenNebulaDataSource(TestCase): parsed_user = None def setUp(self): diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 91e630ae..6d30c5b8 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -1,5 +1,4 @@ import os -import unittest try: from unittest import mock @@ -11,6 +10,7 @@ except ImportError: from contextlib2 import ExitStack from six import StringIO +from ..helpers import TestCase from cloudinit import distros from cloudinit import helpers @@ -80,7 +80,7 @@ class WriteBuffer(object): return self.buffer.getvalue() -class TestNetCfgDistro(unittest.TestCase): +class TestNetCfgDistro(TestCase): def _get_distro(self, dname): cls = distros.fetch(dname) diff --git a/tests/unittests/test_distros/test_resolv.py b/tests/unittests/test_distros/test_resolv.py index 779b83e3..faaf5b7f 100644 --- a/tests/unittests/test_distros/test_resolv.py +++ b/tests/unittests/test_distros/test_resolv.py @@ -1,7 +1,7 @@ from cloudinit.distros.parsers import resolv_conf import re -import unittest +from ..helpers import TestCase BASE_RESOLVE = ''' @@ -13,7 +13,7 @@ nameserver 10.15.30.92 BASE_RESOLVE = BASE_RESOLVE.strip() -class TestResolvHelper(unittest.TestCase): +class TestResolvHelper(TestCase): def test_parse_same(self): rp = resolv_conf.ResolvConf(BASE_RESOLVE) rp_r = str(rp).strip() diff --git a/tests/unittests/test_distros/test_sysconfig.py b/tests/unittests/test_distros/test_sysconfig.py index f66201b3..03d89a10 100644 --- a/tests/unittests/test_distros/test_sysconfig.py +++ b/tests/unittests/test_distros/test_sysconfig.py @@ -1,13 +1,13 @@ import re -import unittest from cloudinit.distros.parsers.sys_conf import SysConf +from ..helpers import TestCase # Lots of good examples @ # http://content.hccfl.edu/pollock/AUnix1/SysconfigFilesDesc.txt -class TestSysConfHelper(unittest.TestCase): +class TestSysConfHelper(TestCase): # This function was added in 2.7, make it work for 2.6 def assertRegMatches(self, text, regexp): regexp = re.compile(regexp) diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index b90d6185..e4488e2a 100644 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -1,9 +1,10 @@ -import unittest - from cloudinit import distros from cloudinit import helpers from cloudinit import settings +from ..helpers import TestCase + + bcfg = { 'name': 'bob', 'plain_text_passwd': 'ubuntu', @@ -15,7 +16,7 @@ bcfg = { } -class TestUGNormalize(unittest.TestCase): +class TestUGNormalize(TestCase): def _make_distro(self, dtype, def_user=None): cfg = dict(settings.CFG_BUILTIN) diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py index d72fa8c7..6bccff11 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure.py +++ b/tests/unittests/test_handler/test_handler_apt_configure.py @@ -1,6 +1,7 @@ from cloudinit import util from cloudinit.config import cc_apt_configure +from ..helpers import TestCase import os import re @@ -9,7 +10,7 @@ import tempfile import unittest -class TestAptProxyConfig(unittest.TestCase): +class TestAptProxyConfig(TestCase): def setUp(self): super(TestAptProxyConfig, self).setUp() self.tmp = tempfile.mkdtemp() diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index 97213a0c..a6b9c0fd 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -3,6 +3,7 @@ from cloudinit import helpers from cloudinit import util from cloudinit.config import cc_ca_certs +from ..helpers import TestCase import logging import shutil @@ -45,7 +46,7 @@ class TestNoConfig(unittest.TestCase): self.assertEqual(certs_mock.call_count, 0) -class TestConfig(unittest.TestCase): +class TestConfig(TestCase): def setUp(self): super(TestConfig, self).setUp() self.name = "ca-certs" @@ -139,7 +140,7 @@ class TestConfig(unittest.TestCase): self.assertEqual(self.mock_remove.call_count, 1) -class TestAddCaCerts(unittest.TestCase): +class TestAddCaCerts(TestCase): def setUp(self): super(TestAddCaCerts, self).setUp() @@ -241,7 +242,7 @@ class TestUpdateCaCerts(unittest.TestCase): ["update-ca-certificates"], capture=False) -class TestRemoveDefaultCaCerts(unittest.TestCase): +class TestRemoveDefaultCaCerts(TestCase): def setUp(self): super(TestRemoveDefaultCaCerts, self).setUp() diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index 89727863..bef0d80d 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -2,6 +2,7 @@ from cloudinit import cloud from cloudinit import util from cloudinit.config import cc_growpart +from ..helpers import TestCase import errno import logging @@ -72,7 +73,7 @@ class TestDisabled(unittest.TestCase): self.assertEqual(mockobj.call_count, 0) -class TestConfig(unittest.TestCase): +class TestConfig(TestCase): def setUp(self): super(TestConfig, self).setUp() self.name = "growpart" diff --git a/tests/unittests/test_pathprefix2dict.py b/tests/unittests/test_pathprefix2dict.py index 38a56dc2..d38260e6 100644 --- a/tests/unittests/test_pathprefix2dict.py +++ b/tests/unittests/test_pathprefix2dict.py @@ -1,15 +1,15 @@ from cloudinit import util -from .helpers import populate_dir +from .helpers import TestCase, populate_dir import shutil import tempfile -import unittest -class TestPathPrefix2Dict(unittest.TestCase): +class TestPathPrefix2Dict(TestCase): def setUp(self): + TestCase.setUp(self) self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index 957467f6..fbad405f 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -16,6 +16,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function + +import sys import six import unittest @@ -24,6 +27,20 @@ import textwrap from cloudinit import templater +try: + skipIf = unittest.skipIf +except AttributeError: + # Python 2.6. Doesn't have to be high fidelity. + def skipIf(condition, reason): + def decorator(func): + def wrapper(*args, **kws): + if condition: + return func(*args, **kws) + else: + print(reason, file=sys.stderr) + return wrapper + return decorator + class TestTemplates(test_helpers.TestCase): def test_render_basic(self): @@ -41,7 +58,7 @@ class TestTemplates(test_helpers.TestCase): out_data = templater.basic_render(in_data, {'b': 2}) self.assertEqual(expected_data.strip(), out_data) - @unittest.skipIf(six.PY3, 'Cheetah is not compatible with Python 3') + @skipIf(six.PY3, 'Cheetah is not compatible with Python 3') def test_detection(self): blob = "## template:cheetah" diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 7a224230..a1bd2c46 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -7,7 +7,6 @@ import shutil import tempfile from . import helpers -import unittest import six try: @@ -38,7 +37,7 @@ class FakeSelinux(object): self.restored.append(path) -class TestGetCfgOptionListOrStr(unittest.TestCase): +class TestGetCfgOptionListOrStr(helpers.TestCase): def test_not_found_no_default(self): """None is returned if key is not found and no default given.""" config = {} @@ -70,7 +69,7 @@ class TestGetCfgOptionListOrStr(unittest.TestCase): self.assertEqual([], result) -class TestWriteFile(unittest.TestCase): +class TestWriteFile(helpers.TestCase): def setUp(self): super(TestWriteFile, self).setUp() self.tmp = tempfile.mkdtemp() @@ -149,7 +148,7 @@ class TestWriteFile(unittest.TestCase): mockobj.assert_called_once_with('selinux') -class TestDeleteDirContents(unittest.TestCase): +class TestDeleteDirContents(helpers.TestCase): def setUp(self): super(TestDeleteDirContents, self).setUp() self.tmp = tempfile.mkdtemp() @@ -215,20 +214,20 @@ class TestDeleteDirContents(unittest.TestCase): self.assertDirEmpty(self.tmp) -class TestKeyValStrings(unittest.TestCase): +class TestKeyValStrings(helpers.TestCase): def test_keyval_str_to_dict(self): expected = {'1': 'one', '2': 'one+one', 'ro': True} cmdline = "1=one ro 2=one+one" self.assertEqual(expected, util.keyval_str_to_dict(cmdline)) -class TestGetCmdline(unittest.TestCase): +class TestGetCmdline(helpers.TestCase): def test_cmdline_reads_debug_env(self): os.environ['DEBUG_PROC_CMDLINE'] = 'abcd 123' self.assertEqual(os.environ['DEBUG_PROC_CMDLINE'], util.get_cmdline()) -class TestLoadYaml(unittest.TestCase): +class TestLoadYaml(helpers.TestCase): mydefault = "7b03a8ebace993d806255121073fed52" def test_simple(self): @@ -335,7 +334,7 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): self._patchIn(new_root) util.ensure_dir(os.path.join('sys', 'class', 'dmi', 'id')) - dmi_key = "/sys/class/dmi/id/{}".format(key) + dmi_key = "/sys/class/dmi/id/{0}".format(key) util.write_file(dmi_key, content) def _no_syspath(self, key, content): diff --git a/tox.ini b/tox.ini index e547c693..d04cd47c 100644 --- a/tox.ini +++ b/tox.ini @@ -11,3 +11,13 @@ deps = nose pep8==1.5.7 pyflakes + +[testenv:py26] +commands = nosetests tests +deps = + contextlib2 + httpretty>=0.7.1 + mock + nose + pep8==1.5.7 + pyflakes -- cgit v1.2.3 From e2cfd12dc79a6350d9f1f3111a15bdb6666ccad7 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 26 Jan 2015 20:05:48 -0500 Subject: super() works in all of Python 2.6, 2.7, and 3.4. --- tests/unittests/helpers.py | 6 +++--- tests/unittests/test_data.py | 2 +- tests/unittests/test_datasource/test_smartos.py | 2 +- tests/unittests/test_pathprefix2dict.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) (limited to 'tests/unittests/helpers.py') diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 424d0626..f92e7ac2 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -43,7 +43,7 @@ if PY26: # all this once Python 2.6 is dropped as a minimum requirement. class TestCase(unittest.TestCase): def setUp(self): - unittest.TestCase.setUp(self) + super(TestCase, self).setUp() self.__all_cleanups = ExitStack() def tearDown(self): @@ -147,7 +147,7 @@ def retarget_many_wrapper(new_base, am, old_func): class ResourceUsingTestCase(TestCase): def setUp(self): - TestCase.setUp(self) + super(ResourceUsingTestCase, self).setUp() self.resource_path = None def resourceLocation(self, subname=None): @@ -186,7 +186,7 @@ class ResourceUsingTestCase(TestCase): class FilesystemMockingTestCase(ResourceUsingTestCase): def setUp(self): - ResourceUsingTestCase.setUp(self) + super(FilesystemMockingTestCase, self).setUp() self.patched_funcs = ExitStack() def tearDown(self): diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 7598837e..e900faa8 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -44,7 +44,7 @@ class FakeDataSource(sources.DataSource): class TestConsumeUserData(helpers.FilesystemMockingTestCase): def setUp(self): - helpers.FilesystemMockingTestCase.setUp(self) + super(TestConsumeUserData, self).setUp() self._log = None self._log_file = None self._log_handler = None diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 2fb9e1b6..b5ebf94d 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -118,7 +118,7 @@ class MockSerial(object): class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): def setUp(self): - helpers.FilesystemMockingTestCase.setUp(self) + super(TestSmartOSDataSource, self).setUp() self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) diff --git a/tests/unittests/test_pathprefix2dict.py b/tests/unittests/test_pathprefix2dict.py index d38260e6..7089bde6 100644 --- a/tests/unittests/test_pathprefix2dict.py +++ b/tests/unittests/test_pathprefix2dict.py @@ -9,7 +9,7 @@ import tempfile class TestPathPrefix2Dict(TestCase): def setUp(self): - TestCase.setUp(self) + super(TestPathPrefix2Dict, self).setUp() self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) -- cgit v1.2.3 From a4a6702758cf60ecb8742d78e576733dbbdbb9a0 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 10 Feb 2015 20:32:32 +0000 Subject: make bddeb work with python3 or python2 painful, and not perfect, but at this point the output builds on a vivid system python2 (bddeb --python2) or python3. * remove use of cheetah by bddeb in favor of builtin renderer * add '--python2' flag to bddeb and knowledge of python 2 and python3 package names. * read-dependencies can now read test-requirements also. * differenciate from build-requirements and runtime requirements. --- packages/bddeb | 102 ++++++++++++++++------ packages/debian/changelog.in | 2 +- packages/debian/control.in | 29 +++--- packages/debian/rules | 17 ---- packages/debian/rules.in | 19 ++++ test-requirements.txt | 3 +- tests/unittests/helpers.py | 16 ++++ tests/unittests/test_handler/test_handler_chef.py | 10 ++- tests/unittests/test_templating.py | 16 +--- tools/read-dependencies | 10 ++- 10 files changed, 140 insertions(+), 84 deletions(-) delete mode 100755 packages/debian/rules create mode 100755 packages/debian/rules.in (limited to 'tests/unittests/helpers.py') diff --git a/packages/bddeb b/packages/bddeb index 83ca68bb..c4efe264 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import os import shutil @@ -27,23 +27,35 @@ import argparse # Package names that will showup in requires to what we can actually # use in our debian 'control' file, this is a translation of the 'requires' # file pypi package name to a debian/ubuntu package name. -PKG_MP = { - 'argparse': 'python-argparse', - 'cheetah': 'python-cheetah', - 'configobj': 'python-configobj', - 'jinja2': 'python-jinja2', - 'jsonpatch': 'python-jsonpatch | python-json-patch', - 'oauth': 'python-oauth', - 'prettytable': 'python-prettytable', - 'pyserial': 'python-serial', - 'pyyaml': 'python-yaml', - 'requests': 'python-requests', - 'six': 'python-six', +STD_NAMED_PACKAGES = [ + 'configobj', + 'jinja2', + 'jsonpatch', + 'oauthlib', + 'prettytable', + 'requests', + 'six', + 'httpretty', + 'mock', + 'nose', + 'setuptools', +] +NONSTD_NAMED_PACKAGES = { + 'argparse': ('python-argparse', None), + 'contextlib2': ('python-contextlib2', None), + 'cheetah': ('python-cheetah', None), + 'pyserial': ('python-serial', 'python3-serial'), + 'pyyaml': ('python-yaml', 'python3-yaml'), + 'six': ('python-six', 'python3-six'), + 'pep8': ('pep8', 'python3-pep8'), + 'pyflakes': ('pyflakes', 'pyflakes'), } + DEBUILD_ARGS = ["-S", "-d"] -def write_debian_folder(root, version, revno, append_requires=[]): +def write_debian_folder(root, version, revno, pkgmap, + pyver="3", append_requires=[]): deb_dir = util.abs_join(root, 'debian') os.makedirs(deb_dir) @@ -59,25 +71,42 @@ def write_debian_folder(root, version, revno, append_requires=[]): # Write out the control file template cmd = [util.abs_join(find_root(), 'tools', 'read-dependencies')] (stdout, _stderr) = util.subp(cmd) - pkgs = [p.lower().strip() for p in stdout.splitlines()] + pypi_pkgs = [p.lower().strip() for p in stdout.splitlines()] + + (stdout, _stderr) = util.subp(cmd + ['test-requirements.txt']) + pypi_test_pkgs = [p.lower().strip() for p in stdout.splitlines()] # Map to known packages requires = append_requires - for p in pkgs: - tgt_pkg = PKG_MP.get(p) - if not tgt_pkg: - raise RuntimeError(("Do not know how to translate pypi dependency" - " %r to a known package") % (p)) - else: - requires.append(tgt_pkg) + test_requires = [] + lists = ((pypi_pkgs, requires), (pypi_test_pkgs, test_requires)) + for pypilist, target in lists: + for p in pypilist: + if p not in pkgmap: + raise RuntimeError(("Do not know how to translate pypi " + "dependency %r to a known package") % (p)) + elif pkgmap[p]: + target.append(pkgmap[p]) + + if pyver == "3": + python = "python3" + else: + python = "python" templater.render_to_file(util.abs_join(find_root(), 'packages', 'debian', 'control.in'), util.abs_join(deb_dir, 'control'), - params={'requires': requires}) + params={'requires': ','.join(requires), + 'test_requires': ','.join(test_requires), + 'python': python}) + + templater.render_to_file(util.abs_join(find_root(), + 'packages', 'debian', 'rules.in'), + util.abs_join(deb_dir, 'rules'), + params={'python': python, 'pyver': pyver}) # Just copy the following directly - for base_fn in ['dirs', 'copyright', 'compat', 'rules']: + for base_fn in ['dirs', 'copyright', 'compat']: shutil.copy(util.abs_join(find_root(), 'packages', 'debian', base_fn), util.abs_join(deb_dir, base_fn)) @@ -91,12 +120,16 @@ def main(): " (default: %(default)s)"), default=False, action='store_true') - parser.add_argument("--no-cloud-utils", dest="no_cloud_utils", - help=("don't depend on cloud-utils package" + parser.add_argument("--cloud-utils", dest="cloud_utils", + help=("depend on cloud-utils package" " (default: %(default)s)"), default=False, action='store_true') + parser.add_argument("--python2", dest="python2", + help=("build debs for python2 rather than python3"), + default=False, action='store_true') + parser.add_argument("--init-system", dest="init_system", help=("build deb with INIT_SYSTEM=xxx" " (default: %(default)s"), @@ -123,6 +156,18 @@ def main(): if args.verbose: capture = False + pkgmap = {} + for p in NONSTD_NAMED_PACKAGES: + pkgmap[p] = NONSTD_NAMED_PACKAGES[p][int(not args.python2)] + + for p in STD_NAMED_PACKAGES: + if args.python2: + pkgmap[p] = "python-" + p + pyver = "2" + else: + pkgmap[p] = "python3-" + p + pyver = "3" + with util.tempdir() as tdir: cmd = [util.abs_join(find_root(), 'tools', 'read-version')] @@ -153,11 +198,12 @@ def main(): shutil.move(extracted_name, xdir) print("Creating a debian/ folder in %r" % (xdir)) - if not args.no_cloud_utils: + if args.cloud_utils: append_requires=['cloud-utils | cloud-guest-utils'] else: append_requires=[] - write_debian_folder(xdir, version, revno, append_requires) + write_debian_folder(xdir, version, revno, pkgmap, + pyver=pyver, append_requires=append_requires) # The naming here seems to follow some debian standard # so it will whine if it is changed... diff --git a/packages/debian/changelog.in b/packages/debian/changelog.in index e3e94f54..c9affe47 100644 --- a/packages/debian/changelog.in +++ b/packages/debian/changelog.in @@ -1,4 +1,4 @@ -## This is a cheetah template +## template:basic cloud-init (${version}~bzr${revision}-1) UNRELEASED; urgency=low * build diff --git a/packages/debian/control.in b/packages/debian/control.in index 9207e5f4..bd6e3867 100644 --- a/packages/debian/control.in +++ b/packages/debian/control.in @@ -1,4 +1,4 @@ -## This is a cheetah template +## template:basic Source: cloud-init Section: admin Priority: optional @@ -6,31 +6,22 @@ Maintainer: Scott Moser Build-Depends: debhelper (>= 9), dh-python, dh-systemd, - python (>= 2.6.6-3~), - python-nose, pyflakes, - python-setuptools, - python-selinux, - python-cheetah, - python-mocker, - python-httpretty, -#for $r in $requires - ${r}, -#end for + ${python}, + ${test_requires}, + ${requires} XS-Python-Version: all -Standards-Version: 3.9.3 +Standards-Version: 3.9.6 Package: cloud-init Architecture: all Depends: procps, - python, -#for $r in $requires - ${r}, -#end for - python-software-properties | software-properties-common, - \${misc:Depends}, + ${python}, + ${requires}, + software-properties-common, + ${misc:Depends}, Recommends: sudo -XB-Python-Version: \${python:Versions} +XB-Python-Version: ${python:Versions} Description: Init scripts for cloud instances Cloud instances need special scripts to run during initialisation to retrieve and install ssh keys and to let the user run various scripts. diff --git a/packages/debian/rules b/packages/debian/rules deleted file mode 100755 index 9e0c5ddb..00000000 --- a/packages/debian/rules +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/make -f - -INIT_SYSTEM ?= upstart,systemd -export PYBUILD_INSTALL_ARGS=--init-system=$(INIT_SYSTEM) - -%: - dh $@ --with python2,systemd --buildsystem pybuild - -override_dh_install: - dh_install - install -d debian/cloud-init/etc/rsyslog.d - cp tools/21-cloudinit.conf debian/cloud-init/etc/rsyslog.d/21-cloudinit.conf - -override_dh_auto_test: - # Becuase setup tools didn't copy data... - cp -r tests/data .pybuild/pythonX.Y_2.7/build/tests - http_proxy= dh_auto_test -- --test-nose diff --git a/packages/debian/rules.in b/packages/debian/rules.in new file mode 100755 index 00000000..bb2e1d5c --- /dev/null +++ b/packages/debian/rules.in @@ -0,0 +1,19 @@ +## template:basic +#!/usr/bin/make -f + +INIT_SYSTEM ?= upstart,systemd +PYVER ?= python${pyver} +export PYBUILD_INSTALL_ARGS=--init-system=$(INIT_SYSTEM) + +%: + dh $@ --with $(PYVER),systemd --buildsystem pybuild + +override_dh_install: + dh_install + install -d debian/cloud-init/etc/rsyslog.d + cp tools/21-cloudinit.conf debian/cloud-init/etc/rsyslog.d/21-cloudinit.conf + +override_dh_auto_test: + # Because setup tools didn't copy data... + [ ! -d .pybuild/pythonX.Y_?.?/build/tests ] || cp -r tests/data .pybuild/pythonX.Y_?.?/build/tests + http_proxy= dh_auto_test -- --test-nose diff --git a/test-requirements.txt b/test-requirements.txt index 230f0404..9b3d07c5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,7 @@ httpretty>=0.7.1 mock -mocker nose pep8==1.5.7 pyflakes +contextlib2 +setuptools diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index f92e7ac2..6b9394b3 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import os import sys import shutil @@ -275,3 +277,17 @@ def populate_dir(path, files): with open(os.path.join(path, name), "w") as fp: fp.write(content) fp.close() + +try: + skipIf = unittest.skipIf +except AttributeError: + # Python 2.6. Doesn't have to be high fidelity. + def skipIf(condition, reason): + def decorator(func): + def wrapper(*args, **kws): + if condition: + return func(*args, **kws) + else: + print(reason, file=sys.stderr) + return wrapper + return decorator diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 8ab27911..edad88cb 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -18,6 +18,8 @@ import tempfile LOG = logging.getLogger(__name__) +CLIENT_TEMPL = os.path.sep.join(["templates", "chef_client.rb.tmpl"]) + class TestChef(t_help.FilesystemMockingTestCase): def setUp(self): @@ -41,9 +43,13 @@ class TestChef(t_help.FilesystemMockingTestCase): for d in cc_chef.CHEF_DIRS: self.assertFalse(os.path.isdir(d)) + @t_help.skipIf(not os.path.isfile(CLIENT_TEMPL), + CLIENT_TEMPL + " is not available") def test_basic_config(self): - # This should create a file of the format... """ + test basic config looks sane + + # This should create a file of the format... # Created by cloud-init v. 0.7.6 on Sat, 11 Oct 2014 23:57:21 +0000 log_level :info ssl_verify_mode :verify_none @@ -105,6 +111,8 @@ class TestChef(t_help.FilesystemMockingTestCase): 'c': 'd', }, json.loads(c)) + @t_help.skipIf(not os.path.isfile(CLIENT_TEMPL), + CLIENT_TEMPL + " is not available") def test_template_deletes(self): tpl_file = util.load_file('templates/chef_client.rb.tmpl') self.patchUtils(self.tmp) diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index fbad405f..2b821150 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -27,20 +27,6 @@ import textwrap from cloudinit import templater -try: - skipIf = unittest.skipIf -except AttributeError: - # Python 2.6. Doesn't have to be high fidelity. - def skipIf(condition, reason): - def decorator(func): - def wrapper(*args, **kws): - if condition: - return func(*args, **kws) - else: - print(reason, file=sys.stderr) - return wrapper - return decorator - class TestTemplates(test_helpers.TestCase): def test_render_basic(self): @@ -58,7 +44,7 @@ class TestTemplates(test_helpers.TestCase): out_data = templater.basic_render(in_data, {'b': 2}) self.assertEqual(expected_data.strip(), out_data) - @skipIf(six.PY3, 'Cheetah is not compatible with Python 3') + @test_helpers.skipIf(six.PY3, 'Cheetah is not compatible with Python 3') def test_detection(self): blob = "## template:cheetah" diff --git a/tools/read-dependencies b/tools/read-dependencies index fee3efcf..6a6f3e12 100755 --- a/tools/read-dependencies +++ b/tools/read-dependencies @@ -1,6 +1,7 @@ #!/usr/bin/env python import os +import re import sys if 'CLOUD_INIT_TOP_D' in os.environ: @@ -14,10 +15,15 @@ for fname in ("setup.py", "requirements.txt"): "exist in cloud-init root directory." % fname) sys.exit(1) -with open(os.path.join(topd, "requirements.txt"), "r") as fp: +if len(sys.argv) > 1: + reqfile = sys.argv[1] +else: + reqfile = "requirements.txt" + +with open(os.path.join(topd, reqfile), "r") as fp: for line in fp: if not line.strip() or line.startswith("#"): continue - sys.stdout.write(line) + sys.stdout.write(re.split("[>=.<]*", line)[0].strip() + "\n") sys.exit(0) -- cgit v1.2.3 From abccec1150d6fada29eae9819968e3d4419440ab Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 11 Feb 2015 11:54:48 +0000 Subject: Add helpers for patching open and stdout/stderr. --- tests/unittests/helpers.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'tests/unittests/helpers.py') diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 6b9394b3..ce77af93 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -254,6 +254,19 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): self.patched_funcs.enter_context( mock.patch.object(mod, f, trap_func)) + def patchOpen(self, new_root): + trap_func = retarget_many_wrapper(new_root, 1, open) + name = 'builtins.open' if PY3 else '__builtin__.open' + self.patched_funcs.enter_context(mock.patch(name, trap_func)) + + def patchStdoutAndStderr(self, stdout=None, stderr=None): + if stdout is not None: + self.patched_funcs.enter_context( + mock.patch.object(sys, 'stdout', stdout)) + if stderr is not None: + self.patched_funcs.enter_context( + mock.patch.object(sys, 'stderr', stderr)) + class HttprettyTestCase(TestCase): # necessary as http_proxy gets in the way of httpretty -- cgit v1.2.3 From 589ced475c9e200d4645f0b06f7846dae412b194 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 18 Feb 2015 13:30:51 +0000 Subject: Read ovf-env.xml as bytes. This should fix the Azure data source on Python 3, and is appropriate as XML shouldn't really be read as a string. --- cloudinit/sources/DataSourceAzure.py | 4 ++-- tests/unittests/helpers.py | 5 +++-- tests/unittests/test_datasource/test_azure.py | 6 ++++++ 3 files changed, 11 insertions(+), 4 deletions(-) (limited to 'tests/unittests/helpers.py') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 444070bb..6e030217 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -137,7 +137,7 @@ class DataSourceAzureNet(sources.DataSource): if found != ddir: cached_ovfenv = util.load_file( - os.path.join(ddir, 'ovf-env.xml'), quiet=True) + os.path.join(ddir, 'ovf-env.xml'), quiet=True, decode=False) if cached_ovfenv != files['ovf-env.xml']: # source was not walinux-agent's datadir, so we have to clean # up so 'wait_for_files' doesn't return early due to stale data @@ -593,7 +593,7 @@ def load_azure_ds_dir(source_dir): if not os.path.isfile(ovf_file): raise NonAzureDataSource("No ovf-env file found") - with open(ovf_file, "r") as fp: + with open(ovf_file, "rb") as fp: contents = fp.read() md, ud, cfg = read_azure_ovf(contents) diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index ce77af93..7516bd02 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -287,10 +287,11 @@ def populate_dir(path, files): if not os.path.exists(path): os.makedirs(path) for (name, content) in files.items(): - with open(os.path.join(path, name), "w") as fp: - fp.write(content) + with open(os.path.join(path, name), "wb") as fp: + fp.write(content.encode('utf-8')) fp.close() + try: skipIf = unittest.skipIf except AttributeError: diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 965bce4b..38d70fcd 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -360,6 +360,12 @@ class TestAzureDataSource(TestCase): self.assertTrue(os.path.exists(ovf_env_path)) self.assertEqual(xml, load_file(ovf_env_path)) + def test_ovf_can_include_unicode(self): + xml = construct_valid_ovf_env(data={}) + xml = u'\ufeff{0}'.format(xml) + dsrc = self._get_ds({'ovfcontent': xml}) + dsrc.get_data() + def test_existing_ovf_same(self): # waagent/SharedConfig left alone if found ovf-env.xml same as cached odata = {'UserData': b64e("SOMEUSERDATA")} -- cgit v1.2.3 From 8cd5d7b143f882d80d45b1c04bdde1949846d4f1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 25 Feb 2015 19:40:33 -0500 Subject: move towards user-data being binary UrlResponse: biggest change... make readurl return bytes, making user know what to do with it. util: add load_tfile_or_url for loading text file or url as read_file_or_url now returns bytes ec2_utils: all meta-data is text, remove non-obvious string translations DigitalOcean: adjust for ec2_utils DataSourceGCE, DataSourceMAAS: user-data is binary other fields are text. openstack.py: read paths without decoding to text. This is ok as paths other than user-data are json, and load_json will handle load_file still returns text, and that is what most things use. --- cloudinit/ec2_utils.py | 14 +++++++++++--- cloudinit/sources/DataSourceDigitalOcean.py | 8 ++++++-- cloudinit/sources/DataSourceGCE.py | 21 ++++++++++++--------- cloudinit/sources/DataSourceMAAS.py | 14 +++++++++++--- cloudinit/sources/helpers/openstack.py | 2 +- cloudinit/url_helper.py | 2 +- cloudinit/util.py | 11 ++++++++--- tests/unittests/helpers.py | 5 ++++- tests/unittests/test_datasource/test_configdrive.py | 15 ++++++++++----- tests/unittests/test_datasource/test_gce.py | 2 +- tests/unittests/test_datasource/test_maas.py | 8 ++++---- tests/unittests/test_datasource/test_nocloud.py | 14 +++++++------- tests/unittests/test_datasource/test_openstack.py | 6 +++--- tests/unittests/test_ec2_util.py | 2 +- .../test_handler/test_handler_apt_configure.py | 12 ++++++------ tests/unittests/test_pathprefix2dict.py | 10 +++++----- 16 files changed, 91 insertions(+), 55 deletions(-) (limited to 'tests/unittests/helpers.py') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index e1ed4091..7cf99186 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -41,6 +41,10 @@ class MetadataLeafDecoder(object): def __call__(self, field, blob): if not blob: return blob + try: + blob = util.decode_binary(blob) + except UnicodeDecodeError: + return blob if self._maybe_json_object(blob): try: # Assume it's json, unless it fails parsing... @@ -69,6 +73,8 @@ class MetadataMaterializer(object): def _parse(self, blob): leaves = {} children = [] + blob = util.decode_binary(blob) + if not blob: return (leaves, children) @@ -117,12 +123,12 @@ class MetadataMaterializer(object): child_url = url_helper.combine_url(base_url, c) if not child_url.endswith("/"): child_url += "/" - child_blob = str(self._caller(child_url)) + child_blob = self._caller(child_url) child_contents[c] = self._materialize(child_blob, child_url) leaf_contents = {} for (field, resource) in leaves.items(): leaf_url = url_helper.combine_url(base_url, resource) - leaf_blob = self._caller(leaf_url).contents + leaf_blob = self._caller(leaf_url) leaf_contents[field] = self._leaf_decoder(field, leaf_blob) joined = {} joined.update(child_contents) @@ -179,11 +185,13 @@ def get_instance_metadata(api_version='latest', caller = functools.partial(util.read_file_or_url, ssl_details=ssl_details, timeout=timeout, retries=retries) + def mcaller(url): + return caller(url).contents try: response = caller(md_url) materializer = MetadataMaterializer(response.contents, - md_url, caller, + md_url, mcaller, leaf_decoder=leaf_decoder) md = materializer.materialize() if not isinstance(md, (dict)): diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index 76ddaa9d..5d47564d 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -54,9 +54,13 @@ class DataSourceDigitalOcean(sources.DataSource): def get_data(self): caller = functools.partial(util.read_file_or_url, timeout=self.timeout, retries=self.retries) - md = ec2_utils.MetadataMaterializer(str(caller(self.metadata_address)), + + def mcaller(url): + return caller(url).contents + + md = ec2_utils.MetadataMaterializer(mcaller(self.metadata_address), base_url=self.metadata_address, - caller=caller) + caller=mcaller) self.metadata = md.materialize() diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 6936c74e..608c07f1 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -53,15 +53,15 @@ class DataSourceGCE(sources.DataSource): # GCE metadata server requires a custom header since v1 headers = {'X-Google-Metadata-Request': True} - # url_map: (our-key, path, required) + # url_map: (our-key, path, required, is_text) url_map = [ - ('instance-id', 'instance/id', True), - ('availability-zone', 'instance/zone', True), - ('local-hostname', 'instance/hostname', True), - ('public-keys', 'project/attributes/sshKeys', False), - ('user-data', 'instance/attributes/user-data', False), + ('instance-id', 'instance/id', True, True), + ('availability-zone', 'instance/zone', True, True), + ('local-hostname', 'instance/hostname', True, True), + ('public-keys', 'project/attributes/sshKeys', False, True), + ('user-data', 'instance/attributes/user-data', False, False), ('user-data-encoding', 'instance/attributes/user-data-encoding', - False), + False, True), ] # if we cannot resolve the metadata server, then no point in trying @@ -71,13 +71,16 @@ class DataSourceGCE(sources.DataSource): # iterate over url_map keys to get metadata items found = False - for (mkey, path, required) in url_map: + for (mkey, path, required, is_text) in url_map: try: resp = url_helper.readurl(url=self.metadata_address + path, headers=headers) if resp.code == 200: found = True - self.metadata[mkey] = resp.contents + if is_text: + self.metadata[mkey] = util.decode_binary(resp.contents) + else: + self.metadata[mkey] = resp.contents else: if required: msg = "required url %s returned code %s. not GCE" diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 082cc58f..35c5b5e1 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -36,6 +36,8 @@ from cloudinit import util LOG = logging.getLogger(__name__) MD_VERSION = "2012-03-01" +BINARY_FIELDS = ('user-data',) + class DataSourceMAAS(sources.DataSource): """ @@ -185,7 +187,9 @@ def read_maas_seed_dir(seed_d): md = {} for fname in files: try: - md[fname] = util.load_file(os.path.join(seed_d, fname)) + print("fname: %s / %s" % (fname, fname not in BINARY_FIELDS)) + md[fname] = util.load_file(os.path.join(seed_d, fname), + decode=fname not in BINARY_FIELDS) except IOError as e: if e.errno != errno.ENOENT: raise @@ -218,6 +222,7 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, 'public-keys': "%s/%s" % (base_url, 'meta-data/public-keys'), 'user-data': "%s/%s" % (base_url, 'user-data'), } + md = {} for name in file_order: url = files.get(name) @@ -238,7 +243,10 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, timeout=timeout, ssl_details=ssl_details) if resp.ok(): - md[name] = str(resp) + if name in BINARY_FIELDS: + md[name] = resp.contents + else: + md[name] = util.decode_binary(resp.contents) else: LOG.warn(("Fetching from %s resulted in" " an invalid http code %s"), url, resp.code) @@ -263,7 +271,7 @@ def check_seed_contents(content, seed): if len(missing): raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing)) - userdata = content.get('user-data', "") + userdata = content.get('user-data', b"") md = {} for (key, val) in content.items(): if key == 'user-data': diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 88c7a198..bd93d22f 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -327,7 +327,7 @@ class ConfigDriveReader(BaseReader): return os.path.join(*components) def _path_read(self, path): - return util.load_file(path) + return util.load_file(path, decode=False) def _fetch_available_versions(self): if self._versions is None: diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 62001dff..2d81a062 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -119,7 +119,7 @@ class UrlResponse(object): @property def contents(self): - return self._response.text + return self._response.content @property def url(self): diff --git a/cloudinit/util.py b/cloudinit/util.py index 4fbdf0a9..efbc3c8d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -739,6 +739,10 @@ def fetch_ssl_details(paths=None): return ssl_details +def load_tfile_or_url(*args, **kwargs): + return(decode_binary(read_file_or_url(*args, **kwargs).contents)) + + def read_file_or_url(url, timeout=5, retries=10, headers=None, data=None, sec_between=1, ssl_details=None, headers_cb=None, exception_cb=None): @@ -750,7 +754,7 @@ def read_file_or_url(url, timeout=5, retries=10, LOG.warn("Unable to post data to file resource %s", url) file_path = url[len("file://"):] try: - contents = load_file(file_path) + contents = load_file(file_path, decode=False) except IOError as e: code = e.errno if e.errno == errno.ENOENT: @@ -806,7 +810,7 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): ud_url = "%s%s%s" % (base, "user-data", ext) md_url = "%s%s%s" % (base, "meta-data", ext) - md_resp = read_file_or_url(md_url, timeout, retries, file_retries) + md_resp = load_tfile_or_url(md_url, timeout, retries, file_retries) md = None if md_resp.ok(): md = load_yaml(md_resp.contents, default={}) @@ -815,6 +819,7 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): ud = None if ud_resp.ok(): ud = ud_resp.contents + print("returning %s (%s)" % (ud_resp.contents.__class__, ud_resp.contents)) return (md, ud) @@ -2030,7 +2035,7 @@ def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep): ret = {} for f in required + optional: try: - ret[f] = load_file(base + delim + f, quiet=False) + ret[f] = load_file(base + delim + f, quiet=False, decode=False) except IOError as e: if e.errno != errno.ENOENT: raise diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 7516bd02..24e1e881 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -288,7 +288,10 @@ def populate_dir(path, files): os.makedirs(path) for (name, content) in files.items(): with open(os.path.join(path, name), "wb") as fp: - fp.write(content.encode('utf-8')) + if isinstance(content, six.binary_type): + fp.write(content) + else: + fp.write(content.encode('utf-8')) fp.close() diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index e28bdd84..83aca505 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -2,6 +2,7 @@ from copy import copy import json import os import shutil +import six import tempfile try: @@ -45,7 +46,7 @@ EC2_META = { 'reservation-id': 'r-iru5qm4m', 'security-groups': ['default'] } -USER_DATA = '#!/bin/sh\necho This is user data\n' +USER_DATA = b'#!/bin/sh\necho This is user data\n' OSTACK_META = { 'availability_zone': 'nova', 'files': [{'content_path': '/content/0000', 'path': '/etc/foo.cfg'}, @@ -56,8 +57,8 @@ OSTACK_META = { 'public_keys': {'mykey': PUBKEY}, 'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c'} -CONTENT_0 = 'This is contents of /etc/foo.cfg\n' -CONTENT_1 = '# this is /etc/bar/bar.cfg\n' +CONTENT_0 = b'This is contents of /etc/foo.cfg\n' +CONTENT_1 = b'# this is /etc/bar/bar.cfg\n' CFG_DRIVE_FILES_V2 = { 'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META), @@ -346,8 +347,12 @@ def populate_dir(seed_dir, files): dirname = os.path.dirname(path) if not os.path.isdir(dirname): os.makedirs(dirname) - with open(path, "w") as fp: + if isinstance(content, six.text_type): + mode = "w" + else: + mode = "wb" + + with open(path, mode) as fp: fp.write(content) - fp.close() # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 6dd4b5ed..d28f3b08 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -32,7 +32,7 @@ GCE_META = { 'instance/zone': 'foo/bar', 'project/attributes/sshKeys': 'user:ssh-rsa AA2..+aRD0fyVw== root@server', 'instance/hostname': 'server.project-foo.local', - 'instance/attributes/user-data': '/bin/echo foo\n', + 'instance/attributes/user-data': b'/bin/echo foo\n', } GCE_META_PARTIAL = { diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index d25e1adc..f109bb04 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -26,7 +26,7 @@ class TestMAASDataSource(TestCase): data = {'instance-id': 'i-valid01', 'local-hostname': 'valid01-hostname', - 'user-data': 'valid01-userdata', + 'user-data': b'valid01-userdata', 'public-keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname'} my_d = os.path.join(self.tmp, "valid") @@ -46,7 +46,7 @@ class TestMAASDataSource(TestCase): data = {'instance-id': 'i-valid-extra', 'local-hostname': 'valid-extra-hostname', - 'user-data': 'valid-extra-userdata', 'foo': 'bar'} + 'user-data': b'valid-extra-userdata', 'foo': 'bar'} my_d = os.path.join(self.tmp, "valid_extra") populate_dir(my_d, data) @@ -103,7 +103,7 @@ class TestMAASDataSource(TestCase): 'meta-data/instance-id': 'i-instanceid', 'meta-data/local-hostname': 'test-hostname', 'meta-data/public-keys': 'test-hostname', - 'user-data': 'foodata', + 'user-data': b'foodata', } valid_order = [ 'meta-data/local-hostname', @@ -143,7 +143,7 @@ class TestMAASDataSource(TestCase): userdata, metadata = DataSourceMAAS.read_maas_seed_url( my_seed, header_cb=my_headers_cb, version=my_ver) - self.assertEqual("foodata", userdata) + self.assertEqual(b"foodata", userdata) self.assertEqual(metadata['instance-id'], valid['meta-data/instance-id']) self.assertEqual(metadata['local-hostname'], diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 4f967f58..85b4c25a 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -37,7 +37,7 @@ class TestNoCloudDataSource(TestCase): def test_nocloud_seed_dir(self): md = {'instance-id': 'IID', 'dsmode': 'local'} - ud = "USER_DATA_HERE" + ud = b"USER_DATA_HERE" populate_dir(os.path.join(self.paths.seed_dir, "nocloud"), {'user-data': ud, 'meta-data': yaml.safe_dump(md)}) @@ -92,20 +92,20 @@ class TestNoCloudDataSource(TestCase): data = { 'fs_label': None, 'meta-data': yaml.safe_dump({'instance-id': 'IID'}), - 'user-data': "USER_DATA_RAW", + 'user-data': b"USER_DATA_RAW", } sys_cfg = {'datasource': {'NoCloud': data}} dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) ret = dsrc.get_data() - self.assertEqual(dsrc.userdata_raw, "USER_DATA_RAW") + self.assertEqual(dsrc.userdata_raw, b"USER_DATA_RAW") self.assertEqual(dsrc.metadata.get('instance-id'), 'IID') self.assertTrue(ret) def test_nocloud_seed_with_vendordata(self): md = {'instance-id': 'IID', 'dsmode': 'local'} - ud = "USER_DATA_HERE" - vd = "THIS IS MY VENDOR_DATA" + ud = b"USER_DATA_HERE" + vd = b"THIS IS MY VENDOR_DATA" populate_dir(os.path.join(self.paths.seed_dir, "nocloud"), {'user-data': ud, 'meta-data': yaml.safe_dump(md), @@ -126,7 +126,7 @@ class TestNoCloudDataSource(TestCase): def test_nocloud_no_vendordata(self): populate_dir(os.path.join(self.paths.seed_dir, "nocloud"), - {'user-data': "ud", 'meta-data': "instance-id: IID\n"}) + {'user-data': b"ud", 'meta-data': "instance-id: IID\n"}) sys_cfg = {'datasource': {'NoCloud': {'fs_label': None}}} @@ -134,7 +134,7 @@ class TestNoCloudDataSource(TestCase): dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) ret = dsrc.get_data() - self.assertEqual(dsrc.userdata_raw, "ud") + self.assertEqual(dsrc.userdata_raw, b"ud") self.assertFalse(dsrc.vendordata) self.assertTrue(ret) diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 81ef1546..81411ced 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -49,7 +49,7 @@ EC2_META = { 'public-ipv4': '0.0.0.1', 'reservation-id': 'r-iru5qm4m', } -USER_DATA = '#!/bin/sh\necho This is user data\n' +USER_DATA = b'#!/bin/sh\necho This is user data\n' VENDOR_DATA = { 'magic': '', } @@ -63,8 +63,8 @@ OSTACK_META = { 'public_keys': {'mykey': PUBKEY}, 'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c', } -CONTENT_0 = 'This is contents of /etc/foo.cfg\n' -CONTENT_1 = '# this is /etc/bar/bar.cfg\n' +CONTENT_0 = b'This is contents of /etc/foo.cfg\n' +CONTENT_1 = b'# this is /etc/bar/bar.cfg\n' OS_FILES = { 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), 'openstack/latest/user_data': USER_DATA, diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index 84aa002e..bd43accf 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -16,7 +16,7 @@ class TestEc2Util(helpers.HttprettyTestCase): body='stuff', status=200) userdata = eu.get_instance_userdata(self.VERSION) - self.assertEquals('stuff', userdata) + self.assertEquals('stuff', userdata.decode('utf-8')) @hp.activate def test_userdata_fetch_fail_not_found(self): diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py index d8fe9a4f..02cad8b2 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure.py +++ b/tests/unittests/test_handler/test_handler_apt_configure.py @@ -30,7 +30,7 @@ class TestAptProxyConfig(TestCase): self.assertTrue(os.path.isfile(self.pfile)) self.assertFalse(os.path.isfile(self.cfile)) - contents = str(util.read_file_or_url(self.pfile)) + contents = util.load_tfile_or_url(self.pfile) self.assertTrue(self._search_apt_config(contents, "http", "myproxy")) def test_apt_http_proxy_written(self): @@ -40,7 +40,7 @@ class TestAptProxyConfig(TestCase): self.assertTrue(os.path.isfile(self.pfile)) self.assertFalse(os.path.isfile(self.cfile)) - contents = str(util.read_file_or_url(self.pfile)) + contents = util.load_tfile_or_url(self.pfile) self.assertTrue(self._search_apt_config(contents, "http", "myproxy")) def test_apt_all_proxy_written(self): @@ -58,7 +58,7 @@ class TestAptProxyConfig(TestCase): self.assertTrue(os.path.isfile(self.pfile)) self.assertFalse(os.path.isfile(self.cfile)) - contents = str(util.read_file_or_url(self.pfile)) + contents = util.load_tfile_or_url(self.pfile) for ptype, pval in values.items(): self.assertTrue(self._search_apt_config(contents, ptype, pval)) @@ -74,7 +74,7 @@ class TestAptProxyConfig(TestCase): cc_apt_configure.apply_apt_config({'apt_proxy': "foo"}, self.pfile, self.cfile) self.assertTrue(os.path.isfile(self.pfile)) - contents = str(util.read_file_or_url(self.pfile)) + contents = util.load_tfile_or_url(self.pfile) self.assertTrue(self._search_apt_config(contents, "http", "foo")) def test_config_written(self): @@ -86,14 +86,14 @@ class TestAptProxyConfig(TestCase): self.assertTrue(os.path.isfile(self.cfile)) self.assertFalse(os.path.isfile(self.pfile)) - self.assertEqual(str(util.read_file_or_url(self.cfile)), payload) + self.assertEqual(util.load_tfile_or_url(self.cfile), payload) def test_config_replaced(self): util.write_file(self.pfile, "content doesnt matter") cc_apt_configure.apply_apt_config({'apt_config': "foo"}, self.pfile, self.cfile) self.assertTrue(os.path.isfile(self.cfile)) - self.assertEqual(str(util.read_file_or_url(self.cfile)), "foo") + self.assertEqual(util.load_tfile_or_url(self.cfile), "foo") def test_config_deleted(self): # if no 'apt_config' is provided, delete any previously written file diff --git a/tests/unittests/test_pathprefix2dict.py b/tests/unittests/test_pathprefix2dict.py index 7089bde6..38fd75b6 100644 --- a/tests/unittests/test_pathprefix2dict.py +++ b/tests/unittests/test_pathprefix2dict.py @@ -14,28 +14,28 @@ class TestPathPrefix2Dict(TestCase): self.addCleanup(shutil.rmtree, self.tmp) def test_required_only(self): - dirdata = {'f1': 'f1content', 'f2': 'f2content'} + dirdata = {'f1': b'f1content', 'f2': b'f2content'} populate_dir(self.tmp, dirdata) ret = util.pathprefix2dict(self.tmp, required=['f1', 'f2']) self.assertEqual(dirdata, ret) def test_required_missing(self): - dirdata = {'f1': 'f1content'} + dirdata = {'f1': b'f1content'} populate_dir(self.tmp, dirdata) kwargs = {'required': ['f1', 'f2']} self.assertRaises(ValueError, util.pathprefix2dict, self.tmp, **kwargs) def test_no_required_and_optional(self): - dirdata = {'f1': 'f1c', 'f2': 'f2c'} + dirdata = {'f1': b'f1c', 'f2': b'f2c'} populate_dir(self.tmp, dirdata) ret = util.pathprefix2dict(self.tmp, required=None, - optional=['f1', 'f2']) + optional=['f1', 'f2']) self.assertEqual(dirdata, ret) def test_required_and_optional(self): - dirdata = {'f1': 'f1c', 'f2': 'f2c'} + dirdata = {'f1': b'f1c', 'f2': b'f2c'} populate_dir(self.tmp, dirdata) ret = util.pathprefix2dict(self.tmp, required=['f1'], optional=['f2']) -- cgit v1.2.3 From 04a60cf949e085f06fa1f93b39a74b8d288f0e2e Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 4 Mar 2015 12:29:29 +0000 Subject: Fix hang caused by HTTPretty on Python 3.4.2. HTTPretty can causes hangs on Python 3.4.2 (and maybe Python 3.4.1), due to a Python bug (fixed in Python 3.4.3). This works around the problem in the appropriate Python versions. See https://github.com/gabrielfalcao/HTTPretty/pull/193 and https://github.com/gabrielfalcao/HTTPretty/issues/221 for details. --- tests/unittests/helpers.py | 37 +++++++++++++++++++++- .../unittests/test_datasource/test_digitalocean.py | 3 +- tests/unittests/test_datasource/test_gce.py | 3 +- tests/unittests/test_datasource/test_openstack.py | 2 +- tests/unittests/test_ec2_util.py | 2 +- 5 files changed, 42 insertions(+), 5 deletions(-) (limited to 'tests/unittests/helpers.py') diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 24e1e881..61a1f6ff 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -1,5 +1,6 @@ from __future__ import print_function +import functools import os import sys import shutil @@ -25,9 +26,10 @@ PY2 = False PY26 = False PY27 = False PY3 = False +FIX_HTTPRETTY = False _PY_VER = sys.version_info -_PY_MAJOR, _PY_MINOR = _PY_VER[0:2] +_PY_MAJOR, _PY_MINOR, _PY_MICRO = _PY_VER[0:3] if (_PY_MAJOR, _PY_MINOR) <= (2, 6): if (_PY_MAJOR, _PY_MINOR) == (2, 6): PY26 = True @@ -39,6 +41,8 @@ else: PY2 = True if (_PY_MAJOR, _PY_MINOR) >= (3, 0): PY3 = True + if _PY_MINOR == 4 and _PY_MICRO < 3: + FIX_HTTPRETTY = True if PY26: # For now add these on, taken from python 2.7 + slightly adjusted. Drop @@ -268,6 +272,37 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): mock.patch.object(sys, 'stderr', stderr)) +def import_httpretty(): + """Import HTTPretty and monkey patch Python 3.4 issue. + See https://github.com/gabrielfalcao/HTTPretty/pull/193 and + as well as https://github.com/gabrielfalcao/HTTPretty/issues/221. + + Lifted from + https://github.com/inveniosoftware/datacite/blob/master/tests/helpers.py + """ + if not FIX_HTTPRETTY: + import httpretty + else: + import socket + old_SocketType = socket.SocketType + + import httpretty + from httpretty import core + + def sockettype_patch(f): + @functools.wraps(f) + def inner(*args, **kwargs): + f(*args, **kwargs) + socket.SocketType = old_SocketType + socket.__dict__['SocketType'] = old_SocketType + return inner + + core.httpretty.disable = sockettype_patch( + httpretty.httpretty.disable + ) + return httpretty + + class HttprettyTestCase(TestCase): # necessary as http_proxy gets in the way of httpretty # https://github.com/gabrielfalcao/HTTPretty/issues/122 diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index 98f9cfac..679d1b82 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import httpretty import re from six.moves.urllib_parse import urlparse @@ -26,6 +25,8 @@ from cloudinit.sources import DataSourceDigitalOcean from .. import helpers as test_helpers +httpretty = test_helpers.import_httpretty() + # Abbreviated for the test DO_INDEX = """id hostname diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index d28f3b08..4280abc4 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import httpretty import re from base64 import b64encode, b64decode @@ -27,6 +26,8 @@ from cloudinit.sources import DataSourceGCE from .. import helpers as test_helpers +httpretty = test_helpers.import_httpretty() + GCE_META = { 'instance/id': '123', 'instance/zone': 'foo/bar', diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 81411ced..0aa1ba84 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -31,7 +31,7 @@ from cloudinit.sources import DataSourceOpenStack as ds from cloudinit.sources.helpers import openstack from cloudinit import util -import httpretty as hp +hp = test_helpers.import_httpretty() BASE_URL = "http://169.254.169.254" PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index bd43accf..99fc54be 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -3,7 +3,7 @@ from . import helpers from cloudinit import ec2_utils as eu from cloudinit import url_helper as uh -import httpretty as hp +hp = helpers.import_httpretty() class TestEc2Util(helpers.HttprettyTestCase): -- cgit v1.2.3 From 2a955375dbf0d546d8999793923c4e8f23e042c6 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Wed, 3 Jun 2015 13:16:23 -0400 Subject: transform paths in functions taking more than a single argument Patch FilesystemMockingTestcase.patchOS to support methods taking more than a single path argument. This is required in order to properly mock `os.symlink`, which takes two path arguments. --- tests/unittests/helpers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'tests/unittests/helpers.py') diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 61a1f6ff..7f4b8784 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -248,13 +248,15 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): def patchOS(self, new_root): patch_funcs = { - os.path: ['isfile', 'exists', 'islink', 'isdir'], - os: ['listdir'], + os.path: [('isfile', 1), ('exists', 1), + ('islink', 1), ('isdir', 1)], + os: [('listdir', 1), ('mkdir', 1), + ('lstat', 1), ('symlink', 2)], } for (mod, funcs) in patch_funcs.items(): - for f in funcs: + for f, nargs in funcs: func = getattr(mod, f) - trap_func = retarget_many_wrapper(new_root, 1, func) + trap_func = retarget_many_wrapper(new_root, nargs, func) self.patched_funcs.enter_context( mock.patch.object(mod, f, trap_func)) -- cgit v1.2.3