From a2249942522d140a063acdc007aced991d4f0588 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 5 May 2016 14:57:48 -0700 Subject: Work on refactoring (and adding) network conversion tests --- .../unittests/test_datasource/test_configdrive.py | 104 +++++++++++++++++---- tests/unittests/test_net.py | 1 + 2 files changed, 88 insertions(+), 17 deletions(-) (limited to 'tests/unittests') diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 89b15f54..06fba202 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -355,6 +355,14 @@ class TestConfigDriveDataSource(TestCase): self.assertEqual(myds.get_public_ssh_keys(), [OSTACK_META['public_keys']['mykey']]) + +class TestNetJson(TestCase): + def setUp(self): + super(TestNetJson, self).setUp() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.maxDiff = None + def test_network_data_is_found(self): """Verify that network_data is present in ds in config-drive-v2.""" populate_dir(self.tmp, CFG_DRIVE_FILES_V2) @@ -365,31 +373,94 @@ class TestConfigDriveDataSource(TestCase): """Verify that network_data is converted and present on ds object.""" populate_dir(self.tmp, CFG_DRIVE_FILES_V2) myds = cfg_ds_from_dir(self.tmp) - network_config = ds.convert_network_data(NETWORK_DATA) + network_config = openstack.convert_net_json(NETWORK_DATA) self.assertEqual(myds.network_config, network_config) + def test_network_config_conversions(self): + """Tests a bunch of input network json and checks the expected conversions.""" + in_datas = [ + NETWORK_DATA, + { + 'services': [{'type': 'dns', 'address': '172.19.0.12'}], + 'networks': [ + {'network_id': 'dacd568d-5be6-4786-91fe-750c374b78b4', + 'type': 'ipv4', 'netmask': '255.255.252.0', + 'link': 'tap1a81968a-79', + 'routes': [ + { + 'netmask': '0.0.0.0', + 'network': '0.0.0.0', + 'gateway': '172.19.3.254' + }, + ], + 'ip_address': '172.19.1.34', + 'id': 'network0', + }], + 'links': [ + {'type': 'bridge', + 'vif_id': '1a81968a-797a-400f-8a80-567f997eb93f', + 'ethernet_mac_address': 'fa:16:3e:ed:9a:59', + 'id': 'tap1a81968a-79', 'mtu': None}] + }, + ] + out_datas = [ + { + 'version': 1, + 'config': [ + { + 'subnets': [{'type': 'dhcp4'}], + 'type': 'physical', + 'mac_address': 'fa:16:3e:69:b0:58', + 'name': 'tap2ecc7709-b3', + }, + { + 'subnets': [{'type': 'dhcp4'}], + 'type': 'physical', + 'mac_address': 'fa:16:3e:d4:57:ad', + 'name': 'tap2f88d109-5b', + }, + { + 'subnets': [{'type': 'dhcp4'}], + 'type': 'physical', + 'mac_address': 'fa:16:3e:05:30:fe', + 'name': 'tap1a5382f8-04', + }, + { + 'type': 'nameserver', + 'address': '199.204.44.24', + }, + { + 'type': 'nameserver', + 'address': '199.204.47.54', + } + ], + + }, + { + 'version': 1, + 'config': [ + { + 'name': 'tap1a81968a-79', + + } + ], + }, + ] + for in_data, out_data in zip(in_datas, out_datas): + self.assertEqual(openstack.convert_net_json(in_data), + out_data) + def cfg_ds_from_dir(seed_d): - found = ds.read_config_drive(seed_d) cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None, helpers.Paths({})) - populate_ds_from_read_config(cfg_ds, seed_d, found) + cfg_ds.seed_dir = seed_d + if not cfg_ds.get_data(skip_first_boot=True): + raise RuntimeError("Data source did not extract itself from" + " seed directory %s" % seed_d) return cfg_ds -def populate_ds_from_read_config(cfg_ds, source, results): - """Patch the DataSourceConfigDrive from the results of - read_config_drive_dir hopefully in line with what it would have - if cfg_ds.get_data had been successfully called""" - cfg_ds.source = source - cfg_ds.metadata = results.get('metadata') - cfg_ds.ec2_metadata = results.get('ec2-metadata') - cfg_ds.userdata_raw = results.get('userdata') - cfg_ds.version = results.get('version') - cfg_ds.network_json = results.get('networkdata') - cfg_ds._network_config = ds.convert_network_data(cfg_ds.network_json) - - def populate_dir(seed_dir, files): for (name, content) in files.items(): path = os.path.join(seed_dir, name) @@ -400,7 +471,6 @@ def populate_dir(seed_dir, files): mode = "w" else: mode = "wb" - with open(path, mode) as fp: fp.write(content) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 09235c4d..a0cdc493 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -121,6 +121,7 @@ class TestNetConfigParsing(TestCase): self.assertEqual(found, self.simple_cfg) + def _gzip_data(data): with io.BytesIO() as iobuf: gzfp = gzip.GzipFile(mode="wb", fileobj=iobuf) -- cgit v1.2.3 From f5a1662e4a2cbbdee0766276fd516a25306545d6 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 5 May 2016 16:38:36 -0700 Subject: Get this test of net json -> net config working --- tests/unittests/test_datasource/test_configdrive.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'tests/unittests') diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 06fba202..8beaf95e 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -441,7 +441,25 @@ class TestNetJson(TestCase): 'config': [ { 'name': 'tap1a81968a-79', - + 'mac_address': 'fa:16:3e:ed:9a:59', + 'mtu': None, + 'type': 'bridge', + 'subnets': [ + { + 'address': '172.19.1.34', + 'netmask': '255.255.252.0', + 'type': 'static', + 'routes': [{ + 'gateway': '172.19.3.254', + 'netmask': '0.0.0.0', + 'network': '0.0.0.0', + }], + } + ] + }, + { + 'type': 'nameserver', + 'address': '172.19.0.12', } ], }, -- cgit v1.2.3 From cc56ef479a4cfa4520dfcc7cc27c35bb6ac86bd2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 10 May 2016 13:18:53 -0700 Subject: Fix up tests and debian distro --- cloudinit/distros/debian.py | 4 ++-- tests/unittests/test_net.py | 23 ++++++++++++----------- 2 files changed, 14 insertions(+), 13 deletions(-) (limited to 'tests/unittests') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 21a4f805..465413a8 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -27,7 +27,7 @@ from cloudinit import helpers from cloudinit import log as logging from cloudinit import util from cloudinit import net -from cloudinit.net.distros import debian +from cloudinit.net.renderers import eni from cloudinit.distros.parsers.hostname import HostnameConf @@ -57,7 +57,7 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'debian' - self._net_renderer = debian.Renderer() + self._net_renderer = eni.Renderer() def apply_locale(self, locale, out_fn=None): if not out_fn: diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index a0cdc493..6daf9601 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1,5 +1,6 @@ from cloudinit import util from cloudinit import net +from cloudinit.net import cmdline from .helpers import TestCase import base64 @@ -74,15 +75,15 @@ class TestNetConfigParsing(TestCase): "mac_address": "c0:d6:9f:2c:e8:80", "subnets": [{"type": "dhcp"}]}]} - def test_klibc_convert_dhcp(self): - found = net._klibc_to_config_entry(DHCP_CONTENT_1) + def test_cmdline_convert_dhcp(self): + found = cmdline._klibc_to_config_entry(DHCP_CONTENT_1) self.assertEqual(found, ('eth0', DHCP_EXPECTED_1)) - def test_klibc_convert_static(self): - found = net._klibc_to_config_entry(STATIC_CONTENT_1) + def test_cmdline_convert_static(self): + found = cmdline._klibc_to_config_entry(STATIC_CONTENT_1) self.assertEqual(found, ('eth1', STATIC_EXPECTED_1)) - def test_config_from_klibc_net_cfg(self): + def test_config_from_cmdline_net_cfg(self): files = [] pairs = (('net-eth0.cfg', DHCP_CONTENT_1), ('net-eth1.cfg', STATIC_CONTENT_1)) @@ -103,25 +104,25 @@ class TestNetConfigParsing(TestCase): files.append(fp) util.write_file(fp, content) - found = net.config_from_klibc_net_cfg(files=files, mac_addrs=macs) + found = cmdline.config_from_klibc_net_cfg(files=files, + mac_addrs=macs) self.assertEqual(found, expected) def test_cmdline_with_b64(self): data = base64.b64encode(json.dumps(self.simple_cfg).encode()) encoded_text = data.decode() - cmdline = 'ro network-config=' + encoded_text + ' root=foo' - found = net.read_kernel_cmdline_config(cmdline=cmdline) + raw_cmdline = 'ro network-config=' + encoded_text + ' root=foo' + found = cmdline.read_kernel_cmdline_config(cmdline=raw_cmdline) self.assertEqual(found, self.simple_cfg) def test_cmdline_with_b64_gz(self): data = _gzip_data(json.dumps(self.simple_cfg).encode()) encoded_text = base64.b64encode(data).decode() - cmdline = 'ro network-config=' + encoded_text + ' root=foo' - found = net.read_kernel_cmdline_config(cmdline=cmdline) + raw_cmdline = 'ro network-config=' + encoded_text + ' root=foo' + found = cmdline.read_kernel_cmdline_config(cmdline=raw_cmdline) self.assertEqual(found, self.simple_cfg) - def _gzip_data(data): with io.BytesIO() as iobuf: gzfp = gzip.GzipFile(mode="wb", fileobj=iobuf) -- cgit v1.2.3 From c5eb65ed705475640fce1025c74a54052c6e9731 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 10 May 2016 15:12:44 -0700 Subject: Add some basic eni rendering tests --- cloudinit/net/__init__.py | 11 ++++--- cloudinit/net/network_state.py | 2 +- tests/unittests/test_net.py | 68 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 7 deletions(-) (limited to 'tests/unittests') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index e911ed0c..0202cbd8 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -107,7 +107,7 @@ class ParserError(Exception): """Raised when parser has issue parsing the interfaces file.""" -def parse_net_config_data(net_config): +def parse_net_config_data(net_config, skip_broken=True): """Parses the config, returns NetworkState object :param net_config: curtin network config dict @@ -116,20 +116,19 @@ def parse_net_config_data(net_config): if 'version' in net_config and 'config' in net_config: ns = network_state.NetworkState(version=net_config.get('version'), config=net_config.get('config')) - ns.parse_config() + ns.parse_config(skip_broken=skip_broken) state = ns.network_state - return state -def parse_net_config(path): +def parse_net_config(path, skip_broken=True): """Parses a curtin network configuration file and return network state""" ns = None net_config = util.read_conf(path) if 'network' in net_config: - ns = parse_net_config_data(net_config.get('network')) - + ns = parse_net_config_data(net_config.get('network'), + skip_broken=skip_broken) return ns diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 2530a601..2feffa71 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -142,7 +142,7 @@ class NetworkState(object): raise RuntimeError("No handler found for" " command '%s'" % command_type) try: - handler(command) + handler(self, command) except InvalidCommand: if not skip_broken: raise diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 6daf9601..37b48efb 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1,7 +1,9 @@ from cloudinit import util from cloudinit import net from cloudinit.net import cmdline +from cloudinit.net import eni from .helpers import TestCase +from .helpers import mock import base64 import copy @@ -9,6 +11,8 @@ import io import gzip import json import os +import shutil +import tempfile DHCP_CONTENT_1 = """ DEVICE='eth0' @@ -69,6 +73,70 @@ STATIC_EXPECTED_1 = { } +class TestEniNetRendering(TestCase): + + @mock.patch("cloudinit.net.sys_dev_path") + @mock.patch("cloudinit.net.sys_netdev_info") + @mock.patch("cloudinit.net.get_devicelist") + def test_generation(self, mock_get_devicelist, mock_sys_netdev_info, + mock_sys_dev_path): + mock_get_devicelist.return_value = ['eth1000', 'lo'] + + dev_characteristics = { + 'eth1000': { + "bridge": False, + "carrier": False, + "dormant": False, + "operstate": "down", + "address": "07-1C-C6-75-A4-BE", + } + } + + def netdev_info(name, field): + return dev_characteristics[name][field] + + mock_sys_netdev_info.side_effect = netdev_info + + tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp_dir) + + def sys_dev_path(devname, path=""): + return tmp_dir + devname + "/" + path + + for dev in dev_characteristics: + os.makedirs(os.path.join(tmp_dir, dev)) + with open(os.path.join(tmp_dir, dev, 'operstate'), 'w') as fh: + fh.write("down") + + mock_sys_dev_path.side_effect = sys_dev_path + + network_cfg = net.generate_fallback_config() + network_state = net.parse_net_config_data(network_cfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + renderer = eni.Renderer() + renderer.render_network_state(render_dir, network_state, + eni="interfaces", + links_prefix=None, + netrules=None) + + self.assertTrue(os.path.exists(os.path.join(render_dir, + 'interfaces'))) + with open(os.path.join(render_dir, 'interfaces')) as fh: + contents = fh.read() + + expected = """auto lo +iface lo inet loopback + +auto eth1000 +iface eth1000 inet dhcp +""" + self.assertEqual(expected, contents) + + class TestNetConfigParsing(TestCase): simple_cfg = { 'config': [{"type": "physical", "name": "eth0", -- cgit v1.2.3 From 67e506a50dae2b0c1a806f482670b864e84809ae Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 10 May 2016 15:16:05 -0700 Subject: Use lstrip for easier viewing of test expected result --- tests/unittests/test_net.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'tests/unittests') diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 37b48efb..005957de 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -128,13 +128,14 @@ class TestEniNetRendering(TestCase): with open(os.path.join(render_dir, 'interfaces')) as fh: contents = fh.read() - expected = """auto lo + expected = """ +auto lo iface lo inet loopback auto eth1000 iface eth1000 inet dhcp """ - self.assertEqual(expected, contents) + self.assertEqual(expected.lstrip(), contents.lstrip()) class TestNetConfigParsing(TestCase): -- cgit v1.2.3 From 26ea813d293467921ab6b1e32abd2ab8fcefa3bd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 11 May 2016 14:18:02 -0700 Subject: Fix py26 for rhel (and older versions of python) --- cloudinit/net/cmdline.py | 26 ++++--- cloudinit/sources/DataSourceAzure.py | 2 +- cloudinit/sources/helpers/openstack.py | 8 +- cloudinit/util.py | 25 ++++++- requirements.txt | 8 +- setup.py | 1 - test-requirements.txt | 1 + tests/unittests/helpers.py | 86 +++------------------- tests/unittests/test__init__.py | 18 ++--- tests/unittests/test_cli.py | 7 +- tests/unittests/test_cs_util.py | 42 ++++------- tests/unittests/test_datasource/test_azure.py | 12 +-- .../unittests/test_datasource/test_azure_helper.py | 12 +-- tests/unittests/test_datasource/test_cloudsigma.py | 26 +++++-- tests/unittests/test_datasource/test_cloudstack.py | 11 +-- .../unittests/test_datasource/test_configdrive.py | 11 +-- tests/unittests/test_datasource/test_nocloud.py | 14 +--- tests/unittests/test_datasource/test_smartos.py | 21 ++++-- tests/unittests/test_net.py | 7 +- tests/unittests/test_reporting.py | 4 +- tests/unittests/test_rh_subscription.py | 23 ++++-- tox.ini | 36 ++++----- 22 files changed, 163 insertions(+), 238 deletions(-) (limited to 'tests/unittests') diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 958c264b..21bc35d9 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -20,7 +20,6 @@ import base64 import glob import gzip import io -import shlex from cloudinit.net import get_devicelist from cloudinit.net import sys_netdev_info @@ -34,13 +33,17 @@ def _load_shell_content(content, add_empty=False, empty_val=None): then add entries in to the returned dictionary for 'VAR=' variables. Set their value to empty_val.""" data = {} - for line in shlex.split(content): - key, value = line.split("=", 1) - if not value: - value = empty_val - if add_empty or value: - data[key] = value - + for line in util.shlex_split(content): + try: + key, value = line.split("=", 1) + except ValueError: + # Unsplittable line, skip it... + pass + else: + if not value: + value = empty_val + if add_empty or value: + data[key] = value return data @@ -60,6 +63,9 @@ def _klibc_to_config_entry(content, mac_addrs=None): if mac_addrs is None: mac_addrs = {} + print("Reading content") + print(content) + data = _load_shell_content(content) try: name = data['DEVICE'] @@ -185,7 +191,7 @@ def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): return None if mac_addrs is None: - mac_addrs = {k: sys_netdev_info(k, 'address') - for k in get_devicelist()} + mac_addrs = dict((k, sys_netdev_info(k, 'address')) + for k in get_devicelist()) return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 698f4cac..66c8ced8 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -421,7 +421,7 @@ def write_files(datadir, files, dirmode=None): elem.text = DEF_PASSWD_REDACTION return ET.tostring(root) except Exception: - LOG.critical("failed to redact userpassword in {}".format(fname)) + LOG.critical("failed to redact userpassword in %s", fname) return cnt if not datadir: diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 845ea971..b2acc648 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -534,13 +534,13 @@ def convert_net_json(network_json): config = [] for link in links: subnets = [] - cfg = {k: v for k, v in link.items() - if k in valid_keys['physical']} + cfg = dict((k, v) for k, v in link.items() + if k in valid_keys['physical']) cfg.update({'name': link['id']}) for network in [net for net in networks if net['link'] == link['id']]: - subnet = {k: v for k, v in network.items() - if k in valid_keys['subnet']} + subnet = dict((k, v) for k, v in network.items() + if k in valid_keys['subnet']) if 'dhcp' in network['type']: t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4' subnet.update({ diff --git a/cloudinit/util.py b/cloudinit/util.py index 0d21e11b..7562b97a 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -37,6 +37,7 @@ import pwd import random import re import shutil +import shlex import socket import stat import string @@ -81,6 +82,7 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], ['lxc-is-container']) PROC_CMDLINE = None +PY26 = sys.version_info[0:2] == (2, 6) def decode_binary(blob, encoding='utf-8'): @@ -171,7 +173,8 @@ class ProcessExecutionError(IOError): def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, - description=None, reason=None): + description=None, reason=None, + errno=None): if not cmd: self.cmd = '-' else: @@ -202,6 +205,7 @@ class ProcessExecutionError(IOError): else: self.reason = '-' + self.errno = errno message = self.MESSAGE_TMPL % { 'description': self.description, 'cmd': self.cmd, @@ -1147,7 +1151,14 @@ def find_devs_with(criteria=None, oformat='device', options.append(path) cmd = blk_id_cmd + options # See man blkid for why 2 is added - (out, _err) = subp(cmd, rcs=[0, 2]) + try: + (out, _err) = subp(cmd, rcs=[0, 2]) + except ProcessExecutionError as e: + if e.errno == errno.ENOENT: + # blkid not found... + out = "" + else: + raise entries = [] for line in out.splitlines(): line = line.strip() @@ -1191,6 +1202,13 @@ def load_file(fname, read_cb=None, quiet=False, decode=True): return contents +def shlex_split(blob): + if PY26 and isinstance(blob, six.text_type): + # Older versions don't support unicode input + blob = blob.encode("utf8") + return shlex.split(blob) + + def get_cmdline(): if 'DEBUG_PROC_CMDLINE' in os.environ: return os.environ["DEBUG_PROC_CMDLINE"] @@ -1696,7 +1714,8 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, sp = subprocess.Popen(args, **kws) (out, err) = sp.communicate(data) except OSError as e: - raise ProcessExecutionError(cmd=args, reason=e) + raise ProcessExecutionError(cmd=args, reason=e, + errno=e.errno) rc = sp.returncode if rc not in rcs: raise ProcessExecutionError(stdout=out, stderr=err, diff --git a/requirements.txt b/requirements.txt index 19c88857..cc1dc05f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,8 +11,12 @@ PrettyTable oauthlib # This one is currently used only by the CloudSigma and SmartOS datasources. -# If these datasources are removed, this is no longer needed -pyserial +# If these datasources are removed, this is no longer needed. +# +# This will not work in py2.6 so it is only optionally installed on +# python 2.7 and later. +# +# pyserial # This is only needed for places where we need to support configs in a manner # that the built-in config parser is not sufficent (ie diff --git a/setup.py b/setup.py index f86727b2..6b4fc031 100755 --- a/setup.py +++ b/setup.py @@ -197,7 +197,6 @@ requirements = read_requires() if sys.version_info < (3,): requirements.append('cheetah') - setuptools.setup( name='cloud-init', version=get_version(), diff --git a/test-requirements.txt b/test-requirements.txt index 9b3d07c5..d9c757e6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,3 +5,4 @@ pep8==1.5.7 pyflakes contextlib2 setuptools +unittest2 diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index fb9c83a7..33f89254 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -7,12 +7,10 @@ import shutil import tempfile import unittest +import mock import six +import unittest2 -try: - from unittest import mock -except ImportError: - import mock try: from contextlib import ExitStack except ImportError: @@ -21,6 +19,9 @@ except ImportError: from cloudinit import helpers as ch from cloudinit import util +# Used for skipping tests +SkipTest = unittest2.SkipTest + # Used for detecting different python versions PY2 = False PY26 = False @@ -44,79 +45,6 @@ else: if _PY_MINOR == 4 and _PY_MICRO < 3: FIX_HTTPRETTY = True -if PY26: - # For now add these on, taken from python 2.7 + slightly adjusted. Drop - # all this once Python 2.6 is dropped as a minimum requirement. - class TestCase(unittest.TestCase): - def setUp(self): - super(TestCase, self).setUp() - self.__all_cleanups = ExitStack() - - def tearDown(self): - self.__all_cleanups.close() - unittest.TestCase.tearDown(self) - - def addCleanup(self, function, *args, **kws): - self.__all_cleanups.callback(function, *args, **kws) - - def assertIs(self, expr1, expr2, msg=None): - if expr1 is not expr2: - standardMsg = '%r is not %r' % (expr1, expr2) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertIn(self, member, container, msg=None): - if member not in container: - standardMsg = '%r not found in %r' % (member, container) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertNotIn(self, member, container, msg=None): - if member in container: - standardMsg = '%r unexpectedly found in %r' - standardMsg = standardMsg % (member, container) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertIsNone(self, value, msg=None): - if value is not None: - standardMsg = '%r is not None' - standardMsg = standardMsg % (value) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertIsInstance(self, obj, cls, msg=None): - """Same as self.assertTrue(isinstance(obj, cls)), with a nicer - default message.""" - if not isinstance(obj, cls): - standardMsg = '%s is not an instance of %r' % (repr(obj), cls) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertDictContainsSubset(self, expected, actual, msg=None): - missing = [] - mismatched = [] - for k, v in expected.items(): - if k not in actual: - missing.append(k) - elif actual[k] != v: - mismatched.append('%r, expected: %r, actual: %r' - % (k, v, actual[k])) - - if len(missing) == 0 and len(mismatched) == 0: - return - - standardMsg = '' - if missing: - standardMsg = 'Missing: %r' % ','.join(m for m in missing) - if mismatched: - if standardMsg: - standardMsg += '; ' - standardMsg += 'Mismatched values: %s' % ','.join(mismatched) - - self.fail(self._formatMessage(msg, standardMsg)) - - -else: - class TestCase(unittest.TestCase): - pass - - # Makes the old path start # with new base instead of whatever # it previously had @@ -151,6 +79,10 @@ def retarget_many_wrapper(new_base, am, old_func): return wrapper +class TestCase(unittest2.TestCase): + pass + + class ResourceUsingTestCase(TestCase): def setUp(self): super(ResourceUsingTestCase, self).setUp() diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 153f1658..a9b35afe 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -1,16 +1,7 @@ import os import shutil import tempfile -import unittest - -try: - from unittest import mock -except ImportError: - import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack +import unittest2 from cloudinit import handlers from cloudinit import helpers @@ -18,7 +9,7 @@ from cloudinit import settings from cloudinit import url_helper from cloudinit import util -from .helpers import TestCase +from .helpers import TestCase, ExitStack, mock class FakeModule(handlers.Handler): @@ -99,9 +90,10 @@ class TestWalkerHandleHandler(TestCase): self.assertEqual(self.data['handlercount'], 0) -class TestHandlerHandlePart(unittest.TestCase): +class TestHandlerHandlePart(TestCase): def setUp(self): + super(TestHandlerHandlePart, self).setUp() self.data = "fake data" self.ctype = "fake ctype" self.filename = "fake filename" @@ -177,7 +169,7 @@ class TestHandlerHandlePart(unittest.TestCase): self.data, self.ctype, self.filename, self.payload) -class TestCmdlineUrl(unittest.TestCase): +class TestCmdlineUrl(TestCase): def test_invalid_content(self): url = "http://example.com/foo" key = "mykey" diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index ed863399..f8fe7c9b 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -4,12 +4,7 @@ import sys import six from . import helpers as test_helpers - -try: - from unittest import mock -except ImportError: - import mock - +mock = test_helpers.mock BIN_CLOUDINIT = "bin/cloud-init" diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/test_cs_util.py index d7273035..8c9ac0cd 100644 --- a/tests/unittests/test_cs_util.py +++ b/tests/unittests/test_cs_util.py @@ -1,20 +1,14 @@ from __future__ import print_function -import sys -import unittest +from . import helpers as test_helpers -from cloudinit.cs_utils import Cepko +import unittest2 try: - skip = unittest.skip -except AttributeError: - # Python 2.6. Doesn't have to be high fidelity. - def skip(reason): - def decorator(func): - def wrapper(*args, **kws): - print(reason, file=sys.stderr) - return wrapper - return decorator + from cloudinit.cs_utils import Cepko + WILL_WORK = True +except ImportError: + WILL_WORK = False SERVER_CONTEXT = { @@ -32,29 +26,21 @@ SERVER_CONTEXT = { } -class CepkoMock(Cepko): - def all(self): - return SERVER_CONTEXT +if WILL_WORK: + class CepkoMock(Cepko): + def all(self): + return SERVER_CONTEXT - def get(self, key="", request_pattern=None): - return SERVER_CONTEXT['tags'] + def get(self, key="", request_pattern=None): + return SERVER_CONTEXT['tags'] # 2015-01-22 BAW: This test is completely useless because it only ever tests # the CepkoMock object. Even in its original form, I don't think it ever # touched the underlying Cepko class methods. -@skip('This test is completely useless') -class CepkoResultTests(unittest.TestCase): +class CepkoResultTests(test_helpers.TestCase): def setUp(self): - pass - # self.mocked = self.mocker.replace("cloudinit.cs_utils.Cepko", - # spec=CepkoMock, - # count=False, - # passthrough=False) - # self.mocked() - # self.mocker.result(CepkoMock()) - # self.mocker.replay() - # self.c = Cepko() + raise unittest2.SkipTest('This test is completely useless') def test_getitem(self): result = self.c.all() diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 444e2799..aafe1bc2 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -1,16 +1,8 @@ from cloudinit import helpers from cloudinit.util import b64e, decode_binary, load_file from cloudinit.sources import DataSourceAzure -from ..helpers import TestCase, populate_dir -try: - from unittest import mock -except ImportError: - import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack +from ..helpers import TestCase, populate_dir, mock, ExitStack, PY26, SkipTest import crypt import os @@ -83,6 +75,8 @@ class TestAzureDataSource(TestCase): def setUp(self): super(TestAzureDataSource, self).setUp() + if PY26: + raise SkipTest("Does not work on python 2.6") self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 1134199b..736f4463 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -1,17 +1,11 @@ import os from cloudinit.sources.helpers import azure as azure_helper -from ..helpers import TestCase -try: - from unittest import mock -except ImportError: - import mock +from ..helpers import ExitStack +from ..helpers import TestCase -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack +from ..helpers import mock GOAL_STATE_TEMPLATE = """\ diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index 772d189a..11968796 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -1,11 +1,18 @@ # coding: utf-8 + import copy -from cloudinit.cs_utils import Cepko -from cloudinit.sources import DataSourceCloudSigma +try: + # Serial does not work on py2.6 (anymore) + import pyserial + from cloudinit.cs_utils import Cepko + from cloudinit.sources import DataSourceCloudSigma + WILL_WORK = True +except ImportError: + WILL_WORK = False from .. import helpers as test_helpers - +from ..helpers import SkipTest SERVER_CONTEXT = { "cpu": 1000, @@ -29,17 +36,20 @@ SERVER_CONTEXT = { } -class CepkoMock(Cepko): - def __init__(self, mocked_context): - self.result = mocked_context +if WILL_WORK: + class CepkoMock(Cepko): + def __init__(self, mocked_context): + self.result = mocked_context - def all(self): - return self + def all(self): + return self class DataSourceCloudSigmaTest(test_helpers.TestCase): def setUp(self): super(DataSourceCloudSigmaTest, self).setUp() + if not WILL_WORK: + raise SkipTest("Datasource testing not supported") self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "") self.datasource.is_running_in_cloudsigma = lambda: True self.datasource.cepko = CepkoMock(SERVER_CONTEXT) diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py index 656d80d1..4d6b47b4 100644 --- a/tests/unittests/test_datasource/test_cloudstack.py +++ b/tests/unittests/test_datasource/test_cloudstack.py @@ -1,15 +1,6 @@ from cloudinit import helpers from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack -from ..helpers import TestCase - -try: - from unittest import mock -except ImportError: - import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack +from ..helpers import TestCase, mock, ExitStack class TestCloudStackPasswordFetching(TestCase): diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 8beaf95e..14cc8266 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -5,22 +5,13 @@ import shutil import six import tempfile -try: - from unittest import mock -except ImportError: - import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack - from cloudinit import helpers from cloudinit import settings from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit.sources.helpers import openstack from cloudinit import util -from ..helpers import TestCase +from ..helpers import TestCase, ExitStack, mock PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 2d5fc37c..a92dd3b3 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -1,22 +1,12 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNoCloud from cloudinit import util -from ..helpers import TestCase, populate_dir +from ..helpers import TestCase, populate_dir, mock, ExitStack import os import yaml import shutil import tempfile -import unittest - -try: - from unittest import mock -except ImportError: - import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack class TestNoCloudDataSource(TestCase): @@ -139,7 +129,7 @@ class TestNoCloudDataSource(TestCase): self.assertTrue(ret) -class TestParseCommandLineData(unittest.TestCase): +class TestParseCommandLineData(TestCase): def test_parse_cmdline_data_valid(self): ds_id = "ds=nocloud" diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 5c49966a..6b628276 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -33,19 +33,21 @@ import tempfile import uuid from binascii import crc32 -import serial +try: + # Serial does not work on py2.6 (anymore) + import serial + from cloudinit.sources import DataSourceSmartOS + WILL_WORK = True +except ImportError: + WILL_WORK = False + import six from cloudinit import helpers as c_helpers -from cloudinit.sources import DataSourceSmartOS from cloudinit.util import b64e from .. import helpers - -try: - from unittest import mock -except ImportError: - import mock +from ..helpers import mock, SkipTest MOCK_RETURNS = { 'hostname': 'test-host', @@ -79,7 +81,8 @@ def get_mock_client(mockdata): class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): def setUp(self): super(TestSmartOSDataSource, self).setUp() - + if not WILL_WORK: + raise SkipTest("This test will not work") self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) self.legacy_user_d = tempfile.mkdtemp() @@ -445,6 +448,8 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): def setUp(self): super(TestJoyentMetadataClient, self).setUp() + if not WILL_WORK: + raise SkipTest("This test will not work") self.serial = mock.MagicMock(spec=serial.Serial) self.request_id = 0xabcdef12 self.metadata_value = 'value' diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 005957de..ed2c6d0f 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -78,8 +78,9 @@ class TestEniNetRendering(TestCase): @mock.patch("cloudinit.net.sys_dev_path") @mock.patch("cloudinit.net.sys_netdev_info") @mock.patch("cloudinit.net.get_devicelist") - def test_generation(self, mock_get_devicelist, mock_sys_netdev_info, - mock_sys_dev_path): + def test_default_generation(self, mock_get_devicelist, + mock_sys_netdev_info, + mock_sys_dev_path): mock_get_devicelist.return_value = ['eth1000', 'lo'] dev_characteristics = { @@ -138,7 +139,7 @@ iface eth1000 inet dhcp self.assertEqual(expected.lstrip(), contents.lstrip()) -class TestNetConfigParsing(TestCase): +class TestCmdlineConfigParsing(TestCase): simple_cfg = { 'config': [{"type": "physical", "name": "eth0", "mac_address": "c0:d6:9f:2c:e8:80", diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index 32356ef9..493bb261 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -7,7 +7,9 @@ from cloudinit import reporting from cloudinit.reporting import handlers from cloudinit.reporting import events -from .helpers import (mock, TestCase) +import mock + +from .helpers import TestCase def _fake_registry(): diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py index 8c586ad7..13045aaf 100644 --- a/tests/unittests/test_rh_subscription.py +++ b/tests/unittests/test_rh_subscription.py @@ -1,11 +1,24 @@ +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging + from cloudinit import util from cloudinit.config import cc_rh_subscription -import logging -import mock -import unittest + +from .helpers import TestCase, mock -class GoodTests(unittest.TestCase): +class GoodTests(TestCase): def setUp(self): super(GoodTests, self).setUp() self.name = "cc_rh_subscription" @@ -92,7 +105,7 @@ class GoodTests(unittest.TestCase): self.assertEqual(self.SM._sub_man_cli.call_count, 9) -class TestBadInput(unittest.TestCase): +class TestBadInput(TestCase): name = "cc_rh_subscription" cloud_init = None log = logging.getLogger("bad_tests") diff --git a/tox.ini b/tox.ini index bd7c27dd..3210b0ee 100644 --- a/tox.ini +++ b/tox.ini @@ -1,32 +1,32 @@ [tox] -envlist = py27,py3,pyflakes +envlist = py27,py26,py3,pyflakes recreate = True +usedevelop = True [testenv] commands = python -m nose {posargs:tests} deps = -r{toxinidir}/test-requirements.txt - -r{toxinidir}/requirements.txt - -[testenv:py3] -basepython = python3 + -r{toxinidir}/requirements.txt +setenv = + LC_ALL = en_US.utf-8 [testenv:pyflakes] basepython = python3 commands = {envpython} -m pyflakes {posargs:cloudinit/ tests/ tools/} - {envpython} -m pep8 {posargs:cloudinit/ tests/ tools/} + {envpython} -m pep8 {posargs:cloudinit/ tests/ tools/} -# https://github.com/gabrielfalcao/HTTPretty/issues/223 -setenv = - LC_ALL = en_US.utf-8 +[testenv:py3] +basepython = python3 +deps = -r{toxinidir}/test-requirements.txt + -r{toxinidir}/requirements.txt + pyserial + +[testenv:py27] +deps = -r{toxinidir}/test-requirements.txt + -r{toxinidir}/requirements.txt + pyserial [testenv:py26] commands = nosetests {posargs:tests} -deps = - contextlib2 - httpretty>=0.7.1 - mock - nose - pep8==1.5.7 - pyflakes -setenv = - LC_ALL = C +deps = -r{toxinidir}/test-requirements.txt + -r{toxinidir}/requirements.txt -- cgit v1.2.3 From 12d7ee2cb6589b866ab26b508b15c65326481d6c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 11 May 2016 16:47:50 -0700 Subject: Use a fake serial module that will allow tests to contine Instead of aborting all serial using tests instead just create a serial module in cloudinit that will create a fake and broken serial class when pyserial is not actually installed. This allows for using the datasource and tests that exist in a more functional and tested manner (even when pyserial is not found). --- cloudinit/cs_utils.py | 3 +- cloudinit/serial.py | 50 ++++++++++++++++++++++ cloudinit/sources/DataSourceSmartOS.py | 4 +- tests/unittests/test_cs_util.py | 21 +++------ tests/unittests/test_datasource/test_cloudsigma.py | 23 +++------- tests/unittests/test_datasource/test_smartos.py | 15 ++----- 6 files changed, 71 insertions(+), 45 deletions(-) create mode 100644 cloudinit/serial.py (limited to 'tests/unittests') diff --git a/cloudinit/cs_utils.py b/cloudinit/cs_utils.py index 83ac1a0e..412431f2 100644 --- a/cloudinit/cs_utils.py +++ b/cloudinit/cs_utils.py @@ -33,7 +33,8 @@ API Docs: http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html import json import platform -import serial +from cloudinit import serial + # these high timeouts are necessary as read may read a lot of data. READ_TIMEOUT = 60 diff --git a/cloudinit/serial.py b/cloudinit/serial.py new file mode 100644 index 00000000..af45c13e --- /dev/null +++ b/cloudinit/serial.py @@ -0,0 +1,50 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from __future__ import absolute_import + +try: + from serial import Serial +except ImportError: + # For older versions of python (ie 2.6) pyserial may not exist and/or + # work and/or be installed, so make a dummy/fake serial that blows up + # when used... + class Serial(object): + def __init__(self, *args, **kwargs): + pass + + @staticmethod + def isOpen(): + return False + + @staticmethod + def write(data): + raise IOError("Unable to perform serial `write` operation," + " pyserial not installed.") + + @staticmethod + def readline(): + raise IOError("Unable to perform serial `readline` operation," + " pyserial not installed.") + + @staticmethod + def flush(): + raise IOError("Unable to perform serial `flush` operation," + " pyserial not installed.") + + @staticmethod + def read(size=1): + raise IOError("Unable to perform serial `read` operation," + " pyserial not installed.") diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 6cbd8dfa..c7641eb3 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -40,13 +40,11 @@ import re import socket import stat -import serial - from cloudinit import log as logging +from cloudinit import serial from cloudinit import sources from cloudinit import util - LOG = logging.getLogger(__name__) SMARTOS_ATTRIB_MAP = { diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/test_cs_util.py index 8c9ac0cd..56c9ce9e 100644 --- a/tests/unittests/test_cs_util.py +++ b/tests/unittests/test_cs_util.py @@ -2,13 +2,7 @@ from __future__ import print_function from . import helpers as test_helpers -import unittest2 - -try: - from cloudinit.cs_utils import Cepko - WILL_WORK = True -except ImportError: - WILL_WORK = False +from cloudinit.cs_utils import Cepko SERVER_CONTEXT = { @@ -26,13 +20,12 @@ SERVER_CONTEXT = { } -if WILL_WORK: - class CepkoMock(Cepko): - def all(self): - return SERVER_CONTEXT +class CepkoMock(Cepko): + def all(self): + return SERVER_CONTEXT - def get(self, key="", request_pattern=None): - return SERVER_CONTEXT['tags'] + def get(self, key="", request_pattern=None): + return SERVER_CONTEXT['tags'] # 2015-01-22 BAW: This test is completely useless because it only ever tests @@ -40,7 +33,7 @@ if WILL_WORK: # touched the underlying Cepko class methods. class CepkoResultTests(test_helpers.TestCase): def setUp(self): - raise unittest2.SkipTest('This test is completely useless') + raise test_helpers.SkipTest('This test is completely useless') def test_getitem(self): result = self.c.all() diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index 11968796..7950fc52 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -2,14 +2,8 @@ import copy -try: - # Serial does not work on py2.6 (anymore) - import pyserial - from cloudinit.cs_utils import Cepko - from cloudinit.sources import DataSourceCloudSigma - WILL_WORK = True -except ImportError: - WILL_WORK = False +from cloudinit.cs_utils import Cepko +from cloudinit.sources import DataSourceCloudSigma from .. import helpers as test_helpers from ..helpers import SkipTest @@ -36,20 +30,17 @@ SERVER_CONTEXT = { } -if WILL_WORK: - class CepkoMock(Cepko): - def __init__(self, mocked_context): - self.result = mocked_context +class CepkoMock(Cepko): + def __init__(self, mocked_context): + self.result = mocked_context - def all(self): - return self + def all(self): + return self class DataSourceCloudSigmaTest(test_helpers.TestCase): def setUp(self): super(DataSourceCloudSigmaTest, self).setUp() - if not WILL_WORK: - raise SkipTest("Datasource testing not supported") self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "") self.datasource.is_running_in_cloudsigma = lambda: True self.datasource.cepko = CepkoMock(SERVER_CONTEXT) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 6b628276..f536ef4f 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -33,13 +33,8 @@ import tempfile import uuid from binascii import crc32 -try: - # Serial does not work on py2.6 (anymore) - import serial - from cloudinit.sources import DataSourceSmartOS - WILL_WORK = True -except ImportError: - WILL_WORK = False +from cloudinit import serial +from cloudinit.sources import DataSourceSmartOS import six @@ -81,8 +76,7 @@ def get_mock_client(mockdata): class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): def setUp(self): super(TestSmartOSDataSource, self).setUp() - if not WILL_WORK: - raise SkipTest("This test will not work") + self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) self.legacy_user_d = tempfile.mkdtemp() @@ -448,8 +442,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): def setUp(self): super(TestJoyentMetadataClient, self).setUp() - if not WILL_WORK: - raise SkipTest("This test will not work") + self.serial = mock.MagicMock(spec=serial.Serial) self.request_id = 0xabcdef12 self.metadata_value = 'value' -- cgit v1.2.3 From 880d9fc2f9c62abf19b1506595aa81e5417dea45 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 19 May 2016 14:13:07 -0700 Subject: Adjust net module to be more isolated This allows it to be used outside of cloudinit more easily in the future. --- cloudinit/net/__init__.py | 108 ++++++++++++++++++--------------- cloudinit/net/cmdline.py | 15 ++++- cloudinit/net/eni.py | 35 +++++------ cloudinit/net/network_state.py | 38 +++++++++--- cloudinit/sources/helpers/openstack.py | 4 ++ cloudinit/util.py | 9 --- tests/unittests/test_net.py | 7 ++- 7 files changed, 129 insertions(+), 87 deletions(-) (limited to 'tests/unittests') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 0202cbd8..07e7307e 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -17,44 +17,81 @@ # along with Curtin. If not, see . import errno +import logging import os - -from cloudinit import log as logging -from cloudinit.net import network_state -from cloudinit import util - +import six +import yaml LOG = logging.getLogger(__name__) SYS_CLASS_NET = "/sys/class/net/" LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-" DEFAULT_PRIMARY_INTERFACE = 'eth0' +# NOTE(harlowja): some of these are similar to what is in cloudinit main +# source or utils tree/module but the reason that is done is so that this +# whole module can be easily extracted and placed into other +# code-bases (curtin for example). -def sys_dev_path(devname, path=""): - return SYS_CLASS_NET + devname + "/" + path +def write_file(path, content): + """Simple writing a file helper.""" + base_path = os.path.dirname(path) + if not os.path.isdir(base_path): + os.makedirs(base_path) + with open(path, "wb+") as fh: + if isinstance(content, six.text_type): + content = content.encode("utf8") + fh.write(content) -def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None): +def read_file(path, decode='utf8', enoent=None): try: - contents = "" - with open(sys_dev_path(devname, path), "r") as fp: - contents = fp.read().strip() - if translate is None: - return contents - - try: - return translate.get(contents) - except KeyError: - LOG.debug("found unexpected value '%s' in '%s/%s'", contents, - devname, path) - if keyerror is not None: - return keyerror - raise + with open(path, "rb") as fh: + contents = fh.load() except OSError as e: if e.errno == errno.ENOENT and enoent is not None: return enoent raise + if decode: + return contents.decode(decode) + return contents + + +def dump_yaml(obj): + return yaml.safe_dump(obj, + line_break="\n", + indent=4, + explicit_start=True, + explicit_end=True, + default_flow_style=False) + + +def read_yaml_file(path): + val = yaml.safe_load(read_file(path)) + if not isinstance(val, dict): + gotten_type_name = type(val).__name__ + raise TypeError("Expected dict to be loaded from %s, got" + " '%s' instead" % (path, gotten_type_name)) + return val + + +def sys_dev_path(devname, path=""): + return SYS_CLASS_NET + devname + "/" + path + + +def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None): + contents = read_file(sys_dev_path(devname, path), enoent=enoent) + contents = contents.strip() + if translate is None: + return contents + try: + return translate.get(contents) + except KeyError: + LOG.debug("found unexpected value '%s' in '%s/%s'", contents, + devname, path) + if keyerror is not None: + return keyerror + raise def is_up(devname): @@ -107,31 +144,6 @@ class ParserError(Exception): """Raised when parser has issue parsing the interfaces file.""" -def parse_net_config_data(net_config, skip_broken=True): - """Parses the config, returns NetworkState object - - :param net_config: curtin network config dict - """ - state = None - if 'version' in net_config and 'config' in net_config: - ns = network_state.NetworkState(version=net_config.get('version'), - config=net_config.get('config')) - ns.parse_config(skip_broken=skip_broken) - state = ns.network_state - return state - - -def parse_net_config(path, skip_broken=True): - """Parses a curtin network configuration file and - return network state""" - ns = None - net_config = util.read_conf(path) - if 'network' in net_config: - ns = parse_net_config_data(net_config.get('network'), - skip_broken=skip_broken) - return ns - - def is_disabled_cfg(cfg): if not cfg or not isinstance(cfg, dict): return False @@ -146,7 +158,7 @@ def sys_netdev_info(name, field): fname = os.path.join(SYS_CLASS_NET, name, field) if not os.path.exists(fname): raise OSError("%s: could not find sysfs entry: %s" % (name, fname)) - data = util.load_file(fname) + data = read_file(fname) if data[-1] == '\n': data = data[:-1] return data diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index f5712533..b85d4b0a 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -20,12 +20,25 @@ import base64 import glob import gzip import io +import shlex +import sys + +import six from cloudinit.net import get_devicelist from cloudinit.net import sys_netdev_info from cloudinit import util +PY26 = sys.version_info[0:2] == (2, 6) + + +def _shlex_split(blob): + if PY26 and isinstance(blob, six.text_type): + # Older versions don't support unicode input + blob = blob.encode("utf8") + return shlex.split(blob) + def _load_shell_content(content, add_empty=False, empty_val=None): """Given shell like syntax (key=value\nkey2=value2\n) in content @@ -33,7 +46,7 @@ def _load_shell_content(content, add_empty=False, empty_val=None): then add entries in to the returned dictionary for 'VAR=' variables. Set their value to empty_val.""" data = {} - for line in util.shlex_split(content): + for line in _shlex_split(content): try: key, value = line.split("=", 1) except ValueError: diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index b1bdac24..adb31c22 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -16,10 +16,11 @@ import glob import os import re +from cloudinit import net + from cloudinit.net import LINKS_FNAME_PREFIX from cloudinit.net import ParserError from cloudinit.net.udev import generate_udev_rule -from cloudinit import util NET_CONFIG_COMMANDS = [ @@ -363,16 +364,13 @@ class Renderer(object): links_prefix=LINKS_FNAME_PREFIX, netrules='etc/udev/rules.d/70-persistent-net.rules'): - fpeni = os.path.sep.join((target, eni,)) - util.ensure_dir(os.path.dirname(fpeni)) - with open(fpeni, 'w+') as f: - f.write(self._render_interfaces(network_state)) + fpeni = os.path.join(target, eni) + net.write_file(fpeni, self._render_interfaces(network_state)) if netrules: - netrules = os.path.sep.join((target, netrules,)) - util.ensure_dir(os.path.dirname(netrules)) - with open(netrules, 'w+') as f: - f.write(self._render_persistent_net(network_state)) + netrules = os.path.join(target, netrules) + net.write_file(netrules, + self._render_persistent_net(network_state)) if links_prefix: self._render_systemd_links(target, network_state, links_prefix) @@ -382,18 +380,17 @@ class Renderer(object): fp_prefix = os.path.sep.join((target, links_prefix)) for f in glob.glob(fp_prefix + "*"): os.unlink(f) - interfaces = network_state.get('interfaces') for iface in interfaces.values(): if (iface['type'] == 'physical' and 'name' in iface and iface.get('mac_address')): fname = fp_prefix + iface['name'] + ".link" - with open(fname, "w") as fp: - fp.write("\n".join([ - "[Match]", - "MACAddress=" + iface['mac_address'], - "", - "[Link]", - "Name=" + iface['name'], - "" - ])) + content = "\n".join([ + "[Match]", + "MACAddress=" + iface['mac_address'], + "", + "[Link]", + "Name=" + iface['name'], + "" + ]) + net.write_file(fname, content) diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 2feffa71..c5aeadb5 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -16,12 +16,11 @@ # along with Curtin. If not, see . import copy +import logging import six -from cloudinit import log as logging -from cloudinit import util -from cloudinit.util import yaml_dumps as dump_config +from cloudinit import net LOG = logging.getLogger(__name__) @@ -31,9 +30,34 @@ NETWORK_STATE_REQUIRED_KEYS = { } +def parse_net_config_data(net_config, skip_broken=True): + """Parses the config, returns NetworkState object + + :param net_config: curtin network config dict + """ + state = None + if 'version' in net_config and 'config' in net_config: + ns = NetworkState(version=net_config.get('version'), + config=net_config.get('config')) + ns.parse_config(skip_broken=skip_broken) + state = ns.network_state + return state + + +def parse_net_config(path, skip_broken=True): + """Parses a curtin network configuration file and + return network state""" + ns = None + net_config = net.read_yaml_file(path) + if 'network' in net_config: + ns = parse_net_config_data(net_config.get('network'), + skip_broken=skip_broken) + return ns + + def from_state_file(state_file): network_state = None - state = util.read_conf(state_file) + state = net.read_yaml_file(state_file) network_state = NetworkState() network_state.load(state) return network_state @@ -111,7 +135,7 @@ class NetworkState(object): 'config': self.config, 'network_state': self.network_state, } - return dump_config(state) + return net.dump_yaml(state) def load(self, state): if 'version' not in state: @@ -121,7 +145,7 @@ class NetworkState(object): required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']] missing_keys = diff_keys(required_keys, state) if missing_keys: - msg = 'Invalid state, missing keys: %s'.format(missing_keys) + msg = 'Invalid state, missing keys: %s' % (missing_keys) LOG.error(msg) raise ValueError(msg) @@ -130,7 +154,7 @@ class NetworkState(object): setattr(self, key, state[key]) def dump_network_state(self): - return dump_config(self.network_state) + return net.dump_yaml(self.network_state) def parse_config(self, skip_broken=True): # rebuild network state diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index b2acc648..f85fd864 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -551,6 +551,10 @@ def convert_net_json(network_json): 'type': 'static', 'address': network.get('ip_address'), }) + if network['type'] == 'ipv6': + subnet['ipv6'] = True + else: + subnet['ipv4'] = True subnets.append(subnet) cfg.update({'subnets': subnets}) if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']: diff --git a/cloudinit/util.py b/cloudinit/util.py index 7562b97a..2bec476e 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -37,7 +37,6 @@ import pwd import random import re import shutil -import shlex import socket import stat import string @@ -82,7 +81,6 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], ['lxc-is-container']) PROC_CMDLINE = None -PY26 = sys.version_info[0:2] == (2, 6) def decode_binary(blob, encoding='utf-8'): @@ -1202,13 +1200,6 @@ def load_file(fname, read_cb=None, quiet=False, decode=True): return contents -def shlex_split(blob): - if PY26 and isinstance(blob, six.text_type): - # Older versions don't support unicode input - blob = blob.encode("utf8") - return shlex.split(blob) - - def get_cmdline(): if 'DEBUG_PROC_CMDLINE' in os.environ: return os.environ["DEBUG_PROC_CMDLINE"] diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index ed2c6d0f..75c433f6 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -2,6 +2,7 @@ from cloudinit import util from cloudinit import net from cloudinit.net import cmdline from cloudinit.net import eni +from cloudinit.net import network_state from .helpers import TestCase from .helpers import mock @@ -112,14 +113,14 @@ class TestEniNetRendering(TestCase): mock_sys_dev_path.side_effect = sys_dev_path network_cfg = net.generate_fallback_config() - network_state = net.parse_net_config_data(network_cfg, - skip_broken=False) + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) render_dir = os.path.join(tmp_dir, "render") os.makedirs(render_dir) renderer = eni.Renderer() - renderer.render_network_state(render_dir, network_state, + renderer.render_network_state(render_dir, ns, eni="interfaces", links_prefix=None, netrules=None) -- cgit v1.2.3