From 56103567fbf486625cdf5bfe40eea5ddcb7e8e04 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Mon, 31 Jul 2017 13:35:07 -0500 Subject: sysconfig: Dont repeat header when rendering resolv.conf The sysconfig renderer duplicates the cloud-init header string when rendering resolv.conf file. This leads to resolv.conf file growing with every reboot of a system. Fix this by checking for the header when loading content from existing file. Update one of the sysconfig unittests with multiple render calls to simulate the reboot to check that we don't repeat the header. LP: #1701420 --- tests/unittests/test_net.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'tests/unittests/test_net.py') diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index e49abcc4..4653be1a 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1683,6 +1683,9 @@ USERCTL=no ns = network_state.parse_net_config_data(network_cfg, skip_broken=False) renderer = sysconfig.Renderer() + # render a multiple times to simulate reboots + renderer.render_network_state(ns, render_dir) + renderer.render_network_state(ns, render_dir) renderer.render_network_state(ns, render_dir) for fn, expected_content in os_sample.get('out_sysconfig', []): with open(os.path.join(render_dir, fn)) as fh: -- cgit v1.2.3 From dc2bd79949492bccdc1d7df0132f98c354d51943 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 9 Aug 2017 14:44:20 -0500 Subject: network: add v2 passthrough and fix parsing v2 config with bonds/bridge params If the network-config sent to cloud-init is in version: 2 format then when rendering netplan, we can pass the content through and avoid consuming network_state elements. This removes the need for trying to map many v2 features onto network state where other renderers won't be able to use anyhow (for example match parameters for multi-interface configuration and wifi configuration support). Additionally ensure we retain bond/bridge v2 configuration in network state so when rendering to eni or sysconfig we don't lose the configuration - Drop the NotImplemented wifi exception, log a warning that it works for netplan only - Adjust unittests to new code path and output - Fix issue with v2 macaddress values getting dropped - Add unittests for consuming/validating v2 configurations LP: #1709180 --- cloudinit/net/netplan.py | 35 ++------ cloudinit/net/network_state.py | 85 ++++++++++++++---- tests/unittests/test_distros/test_netconfig.py | 4 +- tests/unittests/test_net.py | 115 +++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 44 deletions(-) (limited to 'tests/unittests/test_net.py') diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 9f35b72b..3b06fbf0 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -4,7 +4,7 @@ import copy import os from . import renderer -from .network_state import subnet_is_ipv6 +from .network_state import subnet_is_ipv6, NET_CONFIG_TO_V2 from cloudinit import log as logging from cloudinit import util @@ -27,31 +27,6 @@ network: """ LOG = logging.getLogger(__name__) -NET_CONFIG_TO_V2 = { - 'bond': {'bond-ad-select': 'ad-select', - 'bond-arp-interval': 'arp-interval', - 'bond-arp-ip-target': 'arp-ip-target', - 'bond-arp-validate': 'arp-validate', - 'bond-downdelay': 'down-delay', - 'bond-fail-over-mac': 'fail-over-mac-policy', - 'bond-lacp-rate': 'lacp-rate', - 'bond-miimon': 'mii-monitor-interval', - 'bond-min-links': 'min-links', - 'bond-mode': 'mode', - 'bond-num-grat-arp': 'gratuitious-arp', - 'bond-primary-reselect': 'primary-reselect-policy', - 'bond-updelay': 'up-delay', - 'bond-xmit-hash-policy': 'transmit-hash-policy'}, - 'bridge': {'bridge_ageing': 'ageing-time', - 'bridge_bridgeprio': 'priority', - 'bridge_fd': 'forward-delay', - 'bridge_gcint': None, - 'bridge_hello': 'hello-time', - 'bridge_maxage': 'max-age', - 'bridge_maxwait': None, - 'bridge_pathcost': 'path-cost', - 'bridge_portprio': None, - 'bridge_waitport': None}} def _get_params_dict_by_match(config, match): @@ -247,6 +222,14 @@ class Renderer(renderer.Renderer): util.subp(cmd, capture=True) def _render_content(self, network_state): + + # if content already in netplan format, pass it back + if network_state.version == 2: + LOG.debug('V2 to V2 passthrough') + return util.yaml_dumps({'network': network_state.config}, + explicit_start=False, + explicit_end=False) + ethernets = {} wifis = {} bridges = {} diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 87a7222d..6faf01b7 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -23,6 +23,33 @@ NETWORK_V2_KEY_FILTER = [ 'match', 'mtu', 'nameservers', 'renderer', 'set-name', 'wakeonlan' ] +NET_CONFIG_TO_V2 = { + 'bond': {'bond-ad-select': 'ad-select', + 'bond-arp-interval': 'arp-interval', + 'bond-arp-ip-target': 'arp-ip-target', + 'bond-arp-validate': 'arp-validate', + 'bond-downdelay': 'down-delay', + 'bond-fail-over-mac': 'fail-over-mac-policy', + 'bond-lacp-rate': 'lacp-rate', + 'bond-miimon': 'mii-monitor-interval', + 'bond-min-links': 'min-links', + 'bond-mode': 'mode', + 'bond-num-grat-arp': 'gratuitious-arp', + 'bond-primary': 'primary', + 'bond-primary-reselect': 'primary-reselect-policy', + 'bond-updelay': 'up-delay', + 'bond-xmit-hash-policy': 'transmit-hash-policy'}, + 'bridge': {'bridge_ageing': 'ageing-time', + 'bridge_bridgeprio': 'priority', + 'bridge_fd': 'forward-delay', + 'bridge_gcint': None, + 'bridge_hello': 'hello-time', + 'bridge_maxage': 'max-age', + 'bridge_maxwait': None, + 'bridge_pathcost': 'path-cost', + 'bridge_portprio': None, + 'bridge_waitport': None}} + def parse_net_config_data(net_config, skip_broken=True): """Parses the config, returns NetworkState object @@ -119,6 +146,10 @@ class NetworkState(object): self._version = version self.use_ipv6 = network_state.get('use_ipv6', False) + @property + def config(self): + return self._network_state['config'] + @property def version(self): return self._version @@ -166,12 +197,14 @@ class NetworkStateInterpreter(object): 'search': [], }, 'use_ipv6': False, + 'config': None, } def __init__(self, version=NETWORK_STATE_VERSION, config=None): self._version = version self._config = config self._network_state = copy.deepcopy(self.initial_network_state) + self._network_state['config'] = config self._parsed = False @property @@ -460,12 +493,15 @@ class NetworkStateInterpreter(object): v2_command = { bond0: { 'interfaces': ['interface0', 'interface1'], - 'miimon': 100, - 'mode': '802.3ad', - 'xmit_hash_policy': 'layer3+4'}, + 'parameters': { + 'mii-monitor-interval': 100, + 'mode': '802.3ad', + 'xmit_hash_policy': 'layer3+4'}}, bond1: { 'bond-slaves': ['interface2', 'interface7'], - 'mode': 1 + 'parameters': { + 'mode': 1, + } } } @@ -554,6 +590,7 @@ class NetworkStateInterpreter(object): if not mac_address: LOG.debug('NetworkState Version2: missing "macaddress" info ' 'in config entry: %s: %s', eth, str(cfg)) + phy_cmd.update({'mac_address': mac_address}) for key in ['mtu', 'match', 'wakeonlan']: if key in cfg: @@ -598,8 +635,8 @@ class NetworkStateInterpreter(object): self.handle_vlan(vlan_cmd) def handle_wifis(self, command): - raise NotImplementedError("NetworkState V2: " - "Skipping wifi configuration") + LOG.warning('Wifi configuration is only available to distros with' + 'netplan rendering support.') def _v2_common(self, cfg): LOG.debug('v2_common: handling config:\n%s', cfg) @@ -616,6 +653,11 @@ class NetworkStateInterpreter(object): def _handle_bond_bridge(self, command, cmd_type=None): """Common handler for bond and bridge types""" + + # inverse mapping for v2 keynames to v1 keynames + v2key_to_v1 = dict((v, k) for k, v in + NET_CONFIG_TO_V2.get(cmd_type).items()) + for item_name, item_cfg in command.items(): item_params = dict((key, value) for (key, value) in item_cfg.items() if key not in @@ -624,14 +666,20 @@ class NetworkStateInterpreter(object): 'type': cmd_type, 'name': item_name, cmd_type + '_interfaces': item_cfg.get('interfaces'), - 'params': item_params, + 'params': dict((v2key_to_v1[k], v) for k, v in + item_params.get('parameters', {}).items()) } subnets = self._v2_to_v1_ipcfg(item_cfg) if len(subnets) > 0: v1_cmd.update({'subnets': subnets}) - LOG.debug('v2(%ss) -> v1(%s):\n%s', cmd_type, cmd_type, v1_cmd) - self.handle_bridge(v1_cmd) + LOG.debug('v2(%s) -> v1(%s):\n%s', cmd_type, cmd_type, v1_cmd) + if cmd_type == "bridge": + self.handle_bridge(v1_cmd) + elif cmd_type == "bond": + self.handle_bond(v1_cmd) + else: + raise ValueError('Unknown command type: %s', cmd_type) def _v2_to_v1_ipcfg(self, cfg): """Common ipconfig extraction from v2 to v1 subnets array.""" @@ -651,12 +699,6 @@ class NetworkStateInterpreter(object): 'address': address, } - routes = [] - for route in cfg.get('routes', []): - routes.append(_normalize_route( - {'address': route.get('to'), 'gateway': route.get('via')})) - subnet['routes'] = routes - if ":" in address: if 'gateway6' in cfg and gateway6 is None: gateway6 = cfg.get('gateway6') @@ -667,6 +709,17 @@ class NetworkStateInterpreter(object): subnet.update({'gateway': gateway4}) subnets.append(subnet) + + routes = [] + for route in cfg.get('routes', []): + routes.append(_normalize_route( + {'destination': route.get('to'), 'gateway': route.get('via')})) + + # v2 routes are bound to the interface, in v1 we add them under + # the first subnet since there isn't an equivalent interface level. + if len(subnets) and len(routes): + subnets[0]['routes'] = routes + return subnets @@ -721,7 +774,7 @@ def _normalize_net_keys(network, address_keys=()): elif netmask: prefix = mask_to_net_prefix(netmask) elif 'prefix' in net: - prefix = int(prefix) + prefix = int(net['prefix']) else: prefix = 64 if ipv6 else 24 diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 2f505d93..6d89dba8 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -135,7 +135,7 @@ network: V2_NET_CFG = { 'ethernets': { 'eth7': { - 'addresses': ['192.168.1.5/255.255.255.0'], + 'addresses': ['192.168.1.5/24'], 'gateway4': '192.168.1.254'}, 'eth9': { 'dhcp4': True} @@ -151,7 +151,6 @@ V2_TO_V2_NET_CFG_OUTPUT = """ # /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: # network: {config: disabled} network: - version: 2 ethernets: eth7: addresses: @@ -159,6 +158,7 @@ network: gateway4: 192.168.1.254 eth9: dhcp4: true + version: 2 """ diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 4653be1a..f251024b 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1059,6 +1059,100 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true - type: static address: 2001:1::1/92 """), + 'expected_netplan': textwrap.dedent(""" + network: + version: 2 + ethernets: + bond0s0: + match: + macaddress: aa:bb:cc:dd:e8:00 + set-name: bond0s0 + bond0s1: + match: + macaddress: aa:bb:cc:dd:e8:01 + set-name: bond0s1 + bonds: + bond0: + addresses: + - 192.168.0.2/24 + - 192.168.1.2/24 + - 2001:1::1/92 + gateway4: 192.168.0.1 + interfaces: + - bond0s0 + - bond0s1 + parameters: + mii-monitor-interval: 100 + mode: active-backup + transmit-hash-policy: layer3+4 + routes: + - to: 10.1.3.0/24 + via: 192.168.0.3 + """), + 'yaml-v2': textwrap.dedent(""" + version: 2 + ethernets: + eth0: + match: + driver: "virtio_net" + macaddress: "aa:bb:cc:dd:e8:00" + vf0: + set-name: vf0 + match: + driver: "e1000" + macaddress: "aa:bb:cc:dd:e8:01" + bonds: + bond0: + addresses: + - 192.168.0.2/24 + - 192.168.1.2/24 + - 2001:1::1/92 + gateway4: 192.168.0.1 + interfaces: + - eth0 + - vf0 + parameters: + mii-monitor-interval: 100 + mode: active-backup + primary: vf0 + transmit-hash-policy: "layer3+4" + routes: + - to: 10.1.3.0/24 + via: 192.168.0.3 + """), + 'expected_netplan-v2': textwrap.dedent(""" + network: + bonds: + bond0: + addresses: + - 192.168.0.2/24 + - 192.168.1.2/24 + - 2001:1::1/92 + gateway4: 192.168.0.1 + interfaces: + - eth0 + - vf0 + parameters: + mii-monitor-interval: 100 + mode: active-backup + primary: vf0 + transmit-hash-policy: layer3+4 + routes: + - to: 10.1.3.0/24 + via: 192.168.0.3 + ethernets: + eth0: + match: + driver: virtio_net + macaddress: aa:bb:cc:dd:e8:00 + vf0: + match: + driver: e1000 + macaddress: aa:bb:cc:dd:e8:01 + set-name: vf0 + version: 2 + """), + 'expected_sysconfig': { 'ifcfg-bond0': textwrap.dedent("""\ BONDING_MASTER=yes @@ -2159,6 +2253,27 @@ class TestNetplanRoundTrip(CiTestCase): renderer.render_network_state(ns, target) return dir2dict(target) + def testsimple_render_bond_netplan(self): + entry = NETWORK_CONFIGS['bond'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + print(entry['expected_netplan']) + print('-- expected ^ | v rendered --') + print(files['/etc/netplan/50-cloud-init.yaml']) + self.assertEqual( + entry['expected_netplan'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + + def testsimple_render_bond_v2_input_netplan(self): + entry = NETWORK_CONFIGS['bond'] + files = self._render_and_read( + network_config=yaml.load(entry['yaml-v2'])) + print(entry['expected_netplan-v2']) + print('-- expected ^ | v rendered --') + print(files['/etc/netplan/50-cloud-init.yaml']) + self.assertEqual( + entry['expected_netplan-v2'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + def testsimple_render_small_netplan(self): entry = NETWORK_CONFIGS['small'] files = self._render_and_read(network_config=yaml.load(entry['yaml'])) -- cgit v1.2.3 From a3649e03206a3596131413956ea7ecc18790ec73 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Tue, 5 Sep 2017 11:03:59 -0600 Subject: relocate tests/unittests/helpers.py to cloudinit/tests This moves the base test case classes into into cloudinit/tests and updates all the corresponding imports. --- cloudinit/analyze/tests/test_dump.py | 2 +- cloudinit/net/tests/test_dhcp.py | 2 +- cloudinit/net/tests/test_init.py | 2 +- cloudinit/tests/helpers.py | 395 +++++++++++++++++++++ cloudinit/tests/test_url_helper.py | 2 +- tests/unittests/helpers.py | 391 -------------------- tests/unittests/test__init__.py | 2 +- tests/unittests/test_atomic_helper.py | 2 +- tests/unittests/test_builtin_handlers.py | 2 +- tests/unittests/test_cli.py | 2 +- tests/unittests/test_cs_util.py | 2 +- tests/unittests/test_data.py | 2 +- tests/unittests/test_datasource/test_aliyun.py | 2 +- tests/unittests/test_datasource/test_altcloud.py | 2 +- tests/unittests/test_datasource/test_azure.py | 6 +- .../unittests/test_datasource/test_azure_helper.py | 2 +- tests/unittests/test_datasource/test_cloudsigma.py | 2 +- tests/unittests/test_datasource/test_cloudstack.py | 2 +- tests/unittests/test_datasource/test_common.py | 2 +- .../unittests/test_datasource/test_configdrive.py | 2 +- .../unittests/test_datasource/test_digitalocean.py | 2 +- tests/unittests/test_datasource/test_ec2.py | 2 +- tests/unittests/test_datasource/test_gce.py | 2 +- tests/unittests/test_datasource/test_maas.py | 2 +- tests/unittests/test_datasource/test_nocloud.py | 2 +- tests/unittests/test_datasource/test_opennebula.py | 2 +- tests/unittests/test_datasource/test_openstack.py | 2 +- tests/unittests/test_datasource/test_ovf.py | 2 +- tests/unittests/test_datasource/test_scaleway.py | 2 +- tests/unittests/test_datasource/test_smartos.py | 2 +- tests/unittests/test_distros/test_arch.py | 2 +- tests/unittests/test_distros/test_create_users.py | 2 +- tests/unittests/test_distros/test_debian.py | 2 +- tests/unittests/test_distros/test_generic.py | 2 +- tests/unittests/test_distros/test_netconfig.py | 2 +- tests/unittests/test_distros/test_opensuse.py | 2 +- tests/unittests/test_distros/test_resolv.py | 2 +- tests/unittests/test_distros/test_sles.py | 2 +- tests/unittests/test_distros/test_sysconfig.py | 2 +- .../test_distros/test_user_data_normalize.py | 2 +- tests/unittests/test_ds_identify.py | 3 +- tests/unittests/test_ec2_util.py | 2 +- tests/unittests/test_filters/test_launch_index.py | 2 +- .../test_handler/test_handler_apt_conf_v1.py | 2 +- .../test_handler_apt_configure_sources_list_v1.py | 2 +- .../test_handler_apt_configure_sources_list_v3.py | 2 +- .../test_handler/test_handler_apt_source_v1.py | 2 +- .../test_handler/test_handler_apt_source_v3.py | 2 +- .../test_handler/test_handler_ca_certs.py | 2 +- tests/unittests/test_handler/test_handler_chef.py | 2 +- tests/unittests/test_handler/test_handler_debug.py | 2 +- .../test_handler/test_handler_disk_setup.py | 2 +- .../test_handler/test_handler_growpart.py | 2 +- .../test_handler/test_handler_landscape.py | 5 +- .../unittests/test_handler/test_handler_locale.py | 2 +- tests/unittests/test_handler/test_handler_lxd.py | 2 +- .../test_handler/test_handler_mcollective.py | 2 +- .../unittests/test_handler/test_handler_mounts.py | 2 +- tests/unittests/test_handler/test_handler_ntp.py | 2 +- .../test_handler/test_handler_power_state.py | 4 +- .../unittests/test_handler/test_handler_puppet.py | 2 +- .../unittests/test_handler/test_handler_rsyslog.py | 2 +- .../unittests/test_handler/test_handler_runcmd.py | 2 +- .../test_handler/test_handler_seed_random.py | 2 +- .../test_handler/test_handler_set_hostname.py | 2 +- .../unittests/test_handler/test_handler_snappy.py | 4 +- .../test_handler/test_handler_spacewalk.py | 2 +- .../test_handler/test_handler_timezone.py | 2 +- .../test_handler/test_handler_write_files.py | 2 +- .../test_handler/test_handler_yum_add_repo.py | 2 +- tests/unittests/test_handler/test_schema.py | 2 +- tests/unittests/test_helpers.py | 2 +- tests/unittests/test_log.py | 2 +- tests/unittests/test_merging.py | 2 +- tests/unittests/test_net.py | 8 +- tests/unittests/test_pathprefix2dict.py | 2 +- tests/unittests/test_registry.py | 2 +- tests/unittests/test_reporting.py | 2 +- tests/unittests/test_rh_subscription.py | 2 +- tests/unittests/test_runs/test_merge_run.py | 2 +- tests/unittests/test_runs/test_simple_run.py | 2 +- tests/unittests/test_sshutil.py | 3 +- tests/unittests/test_templating.py | 2 +- tests/unittests/test_util.py | 2 +- tests/unittests/test_version.py | 2 +- tests/unittests/test_vmware_config_file.py | 2 +- 86 files changed, 491 insertions(+), 482 deletions(-) create mode 100644 cloudinit/tests/helpers.py delete mode 100644 tests/unittests/helpers.py (limited to 'tests/unittests/test_net.py') diff --git a/cloudinit/analyze/tests/test_dump.py b/cloudinit/analyze/tests/test_dump.py index 2c0885d0..f4c42841 100644 --- a/cloudinit/analyze/tests/test_dump.py +++ b/cloudinit/analyze/tests/test_dump.py @@ -6,7 +6,7 @@ from textwrap import dedent from cloudinit.analyze.dump import ( dump_events, parse_ci_logline, parse_timestamp) from cloudinit.util import subp, write_file -from tests.unittests.helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase class TestParseTimestamp(CiTestCase): diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index 47d8d461..4a37e98a 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -8,7 +8,7 @@ from cloudinit.net.dhcp import ( InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery, parse_dhcp_lease_file, dhcp_discovery) from cloudinit.util import ensure_file, write_file -from tests.unittests.helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase class TestParseDHCPLeasesFile(CiTestCase): diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index cc052a7d..8cb4114e 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -7,7 +7,7 @@ import os import cloudinit.net as net from cloudinit.util import ensure_file, write_file, ProcessExecutionError -from tests.unittests.helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase class TestSysDevPath(CiTestCase): diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py new file mode 100644 index 00000000..28e26622 --- /dev/null +++ b/cloudinit/tests/helpers.py @@ -0,0 +1,395 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from __future__ import print_function + +import functools +import json +import logging +import os +import shutil +import sys +import tempfile +import unittest + +import mock +import six +import unittest2 + +try: + from contextlib import ExitStack +except ImportError: + from contextlib2 import ExitStack + +from cloudinit import helpers as ch +from cloudinit import util + +# Used for skipping tests +SkipTest = unittest2.SkipTest + +# Used for detecting different python versions +PY2 = False +PY26 = False +PY27 = False +PY3 = False + +_PY_VER = sys.version_info +_PY_MAJOR, _PY_MINOR, _PY_MICRO = _PY_VER[0:3] +if (_PY_MAJOR, _PY_MINOR) <= (2, 6): + if (_PY_MAJOR, _PY_MINOR) == (2, 6): + PY26 = True + if (_PY_MAJOR, _PY_MINOR) >= (2, 0): + PY2 = True +else: + if (_PY_MAJOR, _PY_MINOR) == (2, 7): + PY27 = True + PY2 = True + if (_PY_MAJOR, _PY_MINOR) >= (3, 0): + PY3 = True + + +# Makes the old path start +# with new base instead of whatever +# it previously had +def rebase_path(old_path, new_base): + if old_path.startswith(new_base): + # Already handled... + return old_path + # Retarget the base of that path + # to the new base instead of the + # old one... + path = os.path.join(new_base, old_path.lstrip("/")) + path = os.path.abspath(path) + return path + + +# Can work on anything that takes a path as arguments +def retarget_many_wrapper(new_base, am, old_func): + def wrapper(*args, **kwds): + n_args = list(args) + nam = am + if am == -1: + nam = len(n_args) + for i in range(0, nam): + path = args[i] + # patchOS() wraps various os and os.path functions, however in + # Python 3 some of these now accept file-descriptors (integers). + # That breaks rebase_path() so in lieu of a better solution, just + # don't rebase if we get a fd. + if isinstance(path, six.string_types): + n_args[i] = rebase_path(path, new_base) + return old_func(*n_args, **kwds) + return wrapper + + +class TestCase(unittest2.TestCase): + + def reset_global_state(self): + """Reset any global state to its original settings. + + cloudinit caches some values in cloudinit.util. Unit tests that + involved those cached paths were then subject to failure if the order + of invocation changed (LP: #1703697). + + This function resets any of these global state variables to their + initial state. + + In the future this should really be done with some registry that + can then be cleaned in a more obvious way. + """ + util.PROC_CMDLINE = None + util._DNS_REDIRECT_IP = None + util._LSB_RELEASE = {} + + def setUp(self): + super(TestCase, self).setUp() + self.reset_global_state() + + +class CiTestCase(TestCase): + """This is the preferred test case base class unless user + needs other test case classes below.""" + + # Subclass overrides for specific test behavior + # Whether or not a unit test needs logfile setup + with_logs = False + + def setUp(self): + super(CiTestCase, self).setUp() + if self.with_logs: + # Create a log handler so unit tests can search expected logs. + self.logger = logging.getLogger() + self.logs = six.StringIO() + formatter = logging.Formatter('%(levelname)s: %(message)s') + handler = logging.StreamHandler(self.logs) + handler.setFormatter(formatter) + self.old_handlers = self.logger.handlers + self.logger.handlers = [handler] + + def tearDown(self): + if self.with_logs: + # Remove the handler we setup + logging.getLogger().handlers = self.old_handlers + super(CiTestCase, self).tearDown() + + def tmp_dir(self, dir=None, cleanup=True): + # return a full path to a temporary directory that will be cleaned up. + if dir is None: + tmpd = tempfile.mkdtemp( + prefix="ci-%s." % self.__class__.__name__) + else: + tmpd = tempfile.mkdtemp(dir=dir) + self.addCleanup(functools.partial(shutil.rmtree, tmpd)) + return tmpd + + def tmp_path(self, path, dir=None): + # return an absolute path to 'path' under dir. + # if dir is None, one will be created with tmp_dir() + # the file is not created or modified. + if dir is None: + dir = self.tmp_dir() + return os.path.normpath(os.path.abspath(os.path.join(dir, path))) + + +class ResourceUsingTestCase(CiTestCase): + + def setUp(self): + super(ResourceUsingTestCase, self).setUp() + self.resource_path = None + + def resourceLocation(self, subname=None): + if self.resource_path is None: + paths = [ + os.path.join('tests', 'data'), + os.path.join('data'), + os.path.join(os.pardir, 'tests', 'data'), + os.path.join(os.pardir, 'data'), + ] + for p in paths: + if os.path.isdir(p): + self.resource_path = p + break + self.assertTrue((self.resource_path and + os.path.isdir(self.resource_path)), + msg="Unable to locate test resource data path!") + if not subname: + return self.resource_path + return os.path.join(self.resource_path, subname) + + def readResource(self, name): + where = self.resourceLocation(name) + with open(where, 'r') as fh: + return fh.read() + + def getCloudPaths(self, ds=None): + tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmpdir) + cp = ch.Paths({'cloud_dir': tmpdir, + 'templates_dir': self.resourceLocation()}, + ds=ds) + return cp + + +class FilesystemMockingTestCase(ResourceUsingTestCase): + + def setUp(self): + super(FilesystemMockingTestCase, self).setUp() + self.patched_funcs = ExitStack() + + def tearDown(self): + self.patched_funcs.close() + ResourceUsingTestCase.tearDown(self) + + def replicateTestRoot(self, example_root, target_root): + real_root = self.resourceLocation() + real_root = os.path.join(real_root, 'roots', example_root) + for (dir_path, _dirnames, filenames) in os.walk(real_root): + real_path = dir_path + make_path = rebase_path(real_path[len(real_root):], target_root) + util.ensure_dir(make_path) + for f in filenames: + real_path = util.abs_join(real_path, f) + make_path = util.abs_join(make_path, f) + shutil.copy(real_path, make_path) + + def patchUtils(self, new_root): + patch_funcs = { + util: [('write_file', 1), + ('append_file', 1), + ('load_file', 1), + ('ensure_dir', 1), + ('chmod', 1), + ('delete_dir_contents', 1), + ('del_file', 1), + ('sym_link', -1), + ('copy', -1)], + } + for (mod, funcs) in patch_funcs.items(): + for (f, am) in funcs: + func = getattr(mod, f) + trap_func = retarget_many_wrapper(new_root, am, func) + self.patched_funcs.enter_context( + mock.patch.object(mod, f, trap_func)) + + # Handle subprocess calls + func = getattr(util, 'subp') + + def nsubp(*_args, **_kwargs): + return ('', '') + + self.patched_funcs.enter_context( + mock.patch.object(util, 'subp', nsubp)) + + def null_func(*_args, **_kwargs): + return None + + for f in ['chownbyid', 'chownbyname']: + self.patched_funcs.enter_context( + mock.patch.object(util, f, null_func)) + + def patchOS(self, new_root): + patch_funcs = { + os.path: [('isfile', 1), ('exists', 1), + ('islink', 1), ('isdir', 1)], + os: [('listdir', 1), ('mkdir', 1), + ('lstat', 1), ('symlink', 2)], + } + for (mod, funcs) in patch_funcs.items(): + for f, nargs in funcs: + func = getattr(mod, f) + trap_func = retarget_many_wrapper(new_root, nargs, func) + self.patched_funcs.enter_context( + mock.patch.object(mod, f, trap_func)) + + def patchOpen(self, new_root): + trap_func = retarget_many_wrapper(new_root, 1, open) + name = 'builtins.open' if PY3 else '__builtin__.open' + self.patched_funcs.enter_context(mock.patch(name, trap_func)) + + def patchStdoutAndStderr(self, stdout=None, stderr=None): + if stdout is not None: + self.patched_funcs.enter_context( + mock.patch.object(sys, 'stdout', stdout)) + if stderr is not None: + self.patched_funcs.enter_context( + mock.patch.object(sys, 'stderr', stderr)) + + def reRoot(self, root=None): + if root is None: + root = self.tmp_dir() + self.patchUtils(root) + self.patchOS(root) + return root + + +class HttprettyTestCase(CiTestCase): + # necessary as http_proxy gets in the way of httpretty + # https://github.com/gabrielfalcao/HTTPretty/issues/122 + + def setUp(self): + self.restore_proxy = os.environ.get('http_proxy') + if self.restore_proxy is not None: + del os.environ['http_proxy'] + super(HttprettyTestCase, self).setUp() + + def tearDown(self): + if self.restore_proxy: + os.environ['http_proxy'] = self.restore_proxy + super(HttprettyTestCase, self).tearDown() + + +def populate_dir(path, files): + if not os.path.exists(path): + os.makedirs(path) + ret = [] + for (name, content) in files.items(): + p = os.path.sep.join([path, name]) + util.ensure_dir(os.path.dirname(p)) + with open(p, "wb") as fp: + if isinstance(content, six.binary_type): + fp.write(content) + else: + fp.write(content.encode('utf-8')) + fp.close() + ret.append(p) + + return ret + + +def dir2dict(startdir, prefix=None): + flist = {} + if prefix is None: + prefix = startdir + for root, dirs, files in os.walk(startdir): + for fname in files: + fpath = os.path.join(root, fname) + key = fpath[len(prefix):] + flist[key] = util.load_file(fpath) + return flist + + +def json_dumps(data): + # print data in nicely formatted json. + return json.dumps(data, indent=1, sort_keys=True, + separators=(',', ': ')) + + +def wrap_and_call(prefix, mocks, func, *args, **kwargs): + """ + call func(args, **kwargs) with mocks applied, then unapplies mocks + nicer to read than repeating dectorators on each function + + prefix: prefix for mock names (e.g. 'cloudinit.stages.util') or None + mocks: dictionary of names (under 'prefix') to mock and either + a return value or a dictionary to pass to the mock.patch call + func: function to call with mocks applied + *args,**kwargs: arguments for 'func' + + return_value: return from 'func' + """ + delim = '.' + if prefix is None: + prefix = '' + prefix = prefix.rstrip(delim) + unwraps = [] + for fname, kw in mocks.items(): + if prefix: + fname = delim.join((prefix, fname)) + if not isinstance(kw, dict): + kw = {'return_value': kw} + p = mock.patch(fname, **kw) + p.start() + unwraps.append(p) + try: + return func(*args, **kwargs) + finally: + for p in unwraps: + p.stop() + + +try: + skipIf = unittest.skipIf +except AttributeError: + # Python 2.6. Doesn't have to be high fidelity. + def skipIf(condition, reason): + def decorator(func): + def wrapper(*args, **kws): + if condition: + return func(*args, **kws) + else: + print(reason, file=sys.stderr) + return wrapper + return decorator + + +# older versions of mock do not have the useful 'assert_not_called' +if not hasattr(mock.Mock, 'assert_not_called'): + def __mock_assert_not_called(mmock): + if mmock.call_count != 0: + msg = ("[citest] Expected '%s' to not have been called. " + "Called %s times." % + (mmock._mock_name or 'mock', mmock.call_count)) + raise AssertionError(msg) + mock.Mock.assert_not_called = __mock_assert_not_called + + +# vi: ts=4 expandtab diff --git a/cloudinit/tests/test_url_helper.py b/cloudinit/tests/test_url_helper.py index 349110d9..b778a3a7 100644 --- a/cloudinit/tests/test_url_helper.py +++ b/cloudinit/tests/test_url_helper.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit.url_helper import oauth_headers -from tests.unittests.helpers import CiTestCase, mock, skipIf +from cloudinit.tests.helpers import CiTestCase, mock, skipIf try: diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py deleted file mode 100644 index bf1dc5df..00000000 --- a/tests/unittests/helpers.py +++ /dev/null @@ -1,391 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -from __future__ import print_function - -import functools -import json -import logging -import os -import shutil -import sys -import tempfile -import unittest - -import mock -import six -import unittest2 - -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack - -from cloudinit import helpers as ch -from cloudinit import util - -# Used for skipping tests -SkipTest = unittest2.SkipTest - -# Used for detecting different python versions -PY2 = False -PY26 = False -PY27 = False -PY3 = False - -_PY_VER = sys.version_info -_PY_MAJOR, _PY_MINOR, _PY_MICRO = _PY_VER[0:3] -if (_PY_MAJOR, _PY_MINOR) <= (2, 6): - if (_PY_MAJOR, _PY_MINOR) == (2, 6): - PY26 = True - if (_PY_MAJOR, _PY_MINOR) >= (2, 0): - PY2 = True -else: - if (_PY_MAJOR, _PY_MINOR) == (2, 7): - PY27 = True - PY2 = True - if (_PY_MAJOR, _PY_MINOR) >= (3, 0): - PY3 = True - - -# Makes the old path start -# with new base instead of whatever -# it previously had -def rebase_path(old_path, new_base): - if old_path.startswith(new_base): - # Already handled... - return old_path - # Retarget the base of that path - # to the new base instead of the - # old one... - path = os.path.join(new_base, old_path.lstrip("/")) - path = os.path.abspath(path) - return path - - -# Can work on anything that takes a path as arguments -def retarget_many_wrapper(new_base, am, old_func): - def wrapper(*args, **kwds): - n_args = list(args) - nam = am - if am == -1: - nam = len(n_args) - for i in range(0, nam): - path = args[i] - # patchOS() wraps various os and os.path functions, however in - # Python 3 some of these now accept file-descriptors (integers). - # That breaks rebase_path() so in lieu of a better solution, just - # don't rebase if we get a fd. - if isinstance(path, six.string_types): - n_args[i] = rebase_path(path, new_base) - return old_func(*n_args, **kwds) - return wrapper - - -class TestCase(unittest2.TestCase): - def reset_global_state(self): - """Reset any global state to its original settings. - - cloudinit caches some values in cloudinit.util. Unit tests that - involved those cached paths were then subject to failure if the order - of invocation changed (LP: #1703697). - - This function resets any of these global state variables to their - initial state. - - In the future this should really be done with some registry that - can then be cleaned in a more obvious way. - """ - util.PROC_CMDLINE = None - util._DNS_REDIRECT_IP = None - util._LSB_RELEASE = {} - - def setUp(self): - super(unittest2.TestCase, self).setUp() - self.reset_global_state() - - -class CiTestCase(TestCase): - """This is the preferred test case base class unless user - needs other test case classes below.""" - - # Subclass overrides for specific test behavior - # Whether or not a unit test needs logfile setup - with_logs = False - - def setUp(self): - super(CiTestCase, self).setUp() - if self.with_logs: - # Create a log handler so unit tests can search expected logs. - self.logger = logging.getLogger() - self.logs = six.StringIO() - formatter = logging.Formatter('%(levelname)s: %(message)s') - handler = logging.StreamHandler(self.logs) - handler.setFormatter(formatter) - self.old_handlers = self.logger.handlers - self.logger.handlers = [handler] - - def tearDown(self): - if self.with_logs: - # Remove the handler we setup - logging.getLogger().handlers = self.old_handlers - super(CiTestCase, self).tearDown() - - def tmp_dir(self, dir=None, cleanup=True): - # return a full path to a temporary directory that will be cleaned up. - if dir is None: - tmpd = tempfile.mkdtemp( - prefix="ci-%s." % self.__class__.__name__) - else: - tmpd = tempfile.mkdtemp(dir=dir) - self.addCleanup(functools.partial(shutil.rmtree, tmpd)) - return tmpd - - def tmp_path(self, path, dir=None): - # return an absolute path to 'path' under dir. - # if dir is None, one will be created with tmp_dir() - # the file is not created or modified. - if dir is None: - dir = self.tmp_dir() - return os.path.normpath(os.path.abspath(os.path.join(dir, path))) - - -class ResourceUsingTestCase(CiTestCase): - def setUp(self): - super(ResourceUsingTestCase, self).setUp() - self.resource_path = None - - def resourceLocation(self, subname=None): - if self.resource_path is None: - paths = [ - os.path.join('tests', 'data'), - os.path.join('data'), - os.path.join(os.pardir, 'tests', 'data'), - os.path.join(os.pardir, 'data'), - ] - for p in paths: - if os.path.isdir(p): - self.resource_path = p - break - self.assertTrue((self.resource_path and - os.path.isdir(self.resource_path)), - msg="Unable to locate test resource data path!") - if not subname: - return self.resource_path - return os.path.join(self.resource_path, subname) - - def readResource(self, name): - where = self.resourceLocation(name) - with open(where, 'r') as fh: - return fh.read() - - def getCloudPaths(self, ds=None): - tmpdir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, tmpdir) - cp = ch.Paths({'cloud_dir': tmpdir, - 'templates_dir': self.resourceLocation()}, - ds=ds) - return cp - - -class FilesystemMockingTestCase(ResourceUsingTestCase): - def setUp(self): - super(FilesystemMockingTestCase, self).setUp() - self.patched_funcs = ExitStack() - - def tearDown(self): - self.patched_funcs.close() - ResourceUsingTestCase.tearDown(self) - - def replicateTestRoot(self, example_root, target_root): - real_root = self.resourceLocation() - real_root = os.path.join(real_root, 'roots', example_root) - for (dir_path, _dirnames, filenames) in os.walk(real_root): - real_path = dir_path - make_path = rebase_path(real_path[len(real_root):], target_root) - util.ensure_dir(make_path) - for f in filenames: - real_path = util.abs_join(real_path, f) - make_path = util.abs_join(make_path, f) - shutil.copy(real_path, make_path) - - def patchUtils(self, new_root): - patch_funcs = { - util: [('write_file', 1), - ('append_file', 1), - ('load_file', 1), - ('ensure_dir', 1), - ('chmod', 1), - ('delete_dir_contents', 1), - ('del_file', 1), - ('sym_link', -1), - ('copy', -1)], - } - for (mod, funcs) in patch_funcs.items(): - for (f, am) in funcs: - func = getattr(mod, f) - trap_func = retarget_many_wrapper(new_root, am, func) - self.patched_funcs.enter_context( - mock.patch.object(mod, f, trap_func)) - - # Handle subprocess calls - func = getattr(util, 'subp') - - def nsubp(*_args, **_kwargs): - return ('', '') - - self.patched_funcs.enter_context( - mock.patch.object(util, 'subp', nsubp)) - - def null_func(*_args, **_kwargs): - return None - - for f in ['chownbyid', 'chownbyname']: - self.patched_funcs.enter_context( - mock.patch.object(util, f, null_func)) - - def patchOS(self, new_root): - patch_funcs = { - os.path: [('isfile', 1), ('exists', 1), - ('islink', 1), ('isdir', 1)], - os: [('listdir', 1), ('mkdir', 1), - ('lstat', 1), ('symlink', 2)], - } - for (mod, funcs) in patch_funcs.items(): - for f, nargs in funcs: - func = getattr(mod, f) - trap_func = retarget_many_wrapper(new_root, nargs, func) - self.patched_funcs.enter_context( - mock.patch.object(mod, f, trap_func)) - - def patchOpen(self, new_root): - trap_func = retarget_many_wrapper(new_root, 1, open) - name = 'builtins.open' if PY3 else '__builtin__.open' - self.patched_funcs.enter_context(mock.patch(name, trap_func)) - - def patchStdoutAndStderr(self, stdout=None, stderr=None): - if stdout is not None: - self.patched_funcs.enter_context( - mock.patch.object(sys, 'stdout', stdout)) - if stderr is not None: - self.patched_funcs.enter_context( - mock.patch.object(sys, 'stderr', stderr)) - - def reRoot(self, root=None): - if root is None: - root = self.tmp_dir() - self.patchUtils(root) - self.patchOS(root) - return root - - -class HttprettyTestCase(CiTestCase): - # necessary as http_proxy gets in the way of httpretty - # https://github.com/gabrielfalcao/HTTPretty/issues/122 - def setUp(self): - self.restore_proxy = os.environ.get('http_proxy') - if self.restore_proxy is not None: - del os.environ['http_proxy'] - super(HttprettyTestCase, self).setUp() - - def tearDown(self): - if self.restore_proxy: - os.environ['http_proxy'] = self.restore_proxy - super(HttprettyTestCase, self).tearDown() - - -def populate_dir(path, files): - if not os.path.exists(path): - os.makedirs(path) - ret = [] - for (name, content) in files.items(): - p = os.path.sep.join([path, name]) - util.ensure_dir(os.path.dirname(p)) - with open(p, "wb") as fp: - if isinstance(content, six.binary_type): - fp.write(content) - else: - fp.write(content.encode('utf-8')) - fp.close() - ret.append(p) - - return ret - - -def dir2dict(startdir, prefix=None): - flist = {} - if prefix is None: - prefix = startdir - for root, dirs, files in os.walk(startdir): - for fname in files: - fpath = os.path.join(root, fname) - key = fpath[len(prefix):] - flist[key] = util.load_file(fpath) - return flist - - -def json_dumps(data): - # print data in nicely formatted json. - return json.dumps(data, indent=1, sort_keys=True, - separators=(',', ': ')) - - -def wrap_and_call(prefix, mocks, func, *args, **kwargs): - """ - call func(args, **kwargs) with mocks applied, then unapplies mocks - nicer to read than repeating dectorators on each function - - prefix: prefix for mock names (e.g. 'cloudinit.stages.util') or None - mocks: dictionary of names (under 'prefix') to mock and either - a return value or a dictionary to pass to the mock.patch call - func: function to call with mocks applied - *args,**kwargs: arguments for 'func' - - return_value: return from 'func' - """ - delim = '.' - if prefix is None: - prefix = '' - prefix = prefix.rstrip(delim) - unwraps = [] - for fname, kw in mocks.items(): - if prefix: - fname = delim.join((prefix, fname)) - if not isinstance(kw, dict): - kw = {'return_value': kw} - p = mock.patch(fname, **kw) - p.start() - unwraps.append(p) - try: - return func(*args, **kwargs) - finally: - for p in unwraps: - p.stop() - - -try: - skipIf = unittest.skipIf -except AttributeError: - # Python 2.6. Doesn't have to be high fidelity. - def skipIf(condition, reason): - def decorator(func): - def wrapper(*args, **kws): - if condition: - return func(*args, **kws) - else: - print(reason, file=sys.stderr) - return wrapper - return decorator - - -# older versions of mock do not have the useful 'assert_not_called' -if not hasattr(mock.Mock, 'assert_not_called'): - def __mock_assert_not_called(mmock): - if mmock.call_count != 0: - msg = ("[citest] Expected '%s' to not have been called. " - "Called %s times." % - (mmock._mock_name or 'mock', mmock.call_count)) - raise AssertionError(msg) - mock.Mock.assert_not_called = __mock_assert_not_called - - -# vi: ts=4 expandtab diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 781f6d54..25878d7a 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -12,7 +12,7 @@ from cloudinit import settings from cloudinit import url_helper from cloudinit import util -from .helpers import TestCase, CiTestCase, ExitStack, mock +from cloudinit.tests.helpers import TestCase, CiTestCase, ExitStack, mock class FakeModule(handlers.Handler): diff --git a/tests/unittests/test_atomic_helper.py b/tests/unittests/test_atomic_helper.py index 515919d8..0101b0e3 100644 --- a/tests/unittests/test_atomic_helper.py +++ b/tests/unittests/test_atomic_helper.py @@ -6,7 +6,7 @@ import stat from cloudinit import atomic_helper -from .helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase class TestAtomicHelper(CiTestCase): diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index dd9d0357..9751ed95 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -11,7 +11,7 @@ try: except ImportError: import mock -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit import handlers from cloudinit import helpers diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 12f01852..495bdc9f 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -2,7 +2,7 @@ import six -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit.cmd import main as cli diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/test_cs_util.py index b8f5031c..ee88520d 100644 --- a/tests/unittests/test_cs_util.py +++ b/tests/unittests/test_cs_util.py @@ -2,7 +2,7 @@ from __future__ import print_function -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit.cs_utils import Cepko diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 4ad86bb6..6d621d26 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -27,7 +27,7 @@ from cloudinit import stages from cloudinit import user_data as ud from cloudinit import util -from . import helpers +from cloudinit.tests import helpers INSTANCE_ID = "i-testing" diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py index 996560e4..82ee9714 100644 --- a/tests/unittests/test_datasource/test_aliyun.py +++ b/tests/unittests/test_datasource/test_aliyun.py @@ -5,9 +5,9 @@ import httpretty import mock import os -from .. import helpers as test_helpers from cloudinit import helpers from cloudinit.sources import DataSourceAliYun as ay +from cloudinit.tests import helpers as test_helpers DEFAULT_METADATA = { 'instance-id': 'aliyun-test-vm-00', diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py index 9c46abc1..3b274d90 100644 --- a/tests/unittests/test_datasource/test_altcloud.py +++ b/tests/unittests/test_datasource/test_altcloud.py @@ -18,7 +18,7 @@ import tempfile from cloudinit import helpers from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import cloudinit.sources.DataSourceAltCloud as dsac diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 20e70fb7..0a117771 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -6,8 +6,8 @@ from cloudinit.sources import DataSourceAzure as dsaz from cloudinit.util import find_freebsd_part from cloudinit.util import get_path_dev_freebsd -from ..helpers import (CiTestCase, TestCase, populate_dir, mock, - ExitStack, PY26, SkipTest) +from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock, + ExitStack, PY26, SkipTest) import crypt import os @@ -871,6 +871,7 @@ class TestLoadAzureDsDir(CiTestCase): class TestReadAzureOvf(TestCase): + def test_invalid_xml_raises_non_azure_ds(self): invalid_xml = "" + construct_valid_ovf_env(data={}) self.assertRaises(dsaz.BrokenAzureDataSource, @@ -1079,6 +1080,7 @@ class TestCanDevBeReformatted(CiTestCase): class TestAzureNetExists(CiTestCase): + def test_azure_net_must_exist_for_legacy_objpkl(self): """DataSourceAzureNet must exist for old obj.pkl files that reference it.""" diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index b2d2971b..80ce003d 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -3,7 +3,7 @@ import os from cloudinit.sources.helpers import azure as azure_helper -from ..helpers import ExitStack, mock, TestCase +from cloudinit.tests.helpers import ExitStack, mock, TestCase GOAL_STATE_TEMPLATE = """\ diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index 5997102c..e4c59907 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -6,7 +6,7 @@ from cloudinit.cs_utils import Cepko from cloudinit import sources from cloudinit.sources import DataSourceCloudSigma -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers SERVER_CONTEXT = { "cpu": 1000, diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py index e94aad61..2dc90305 100644 --- a/tests/unittests/test_datasource/test_cloudstack.py +++ b/tests/unittests/test_datasource/test_cloudstack.py @@ -3,7 +3,7 @@ from cloudinit import helpers from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack -from ..helpers import TestCase, mock, ExitStack +from cloudinit.tests.helpers import TestCase, mock, ExitStack class TestCloudStackPasswordFetching(TestCase): diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index 4802f105..80b9c650 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -24,7 +24,7 @@ from cloudinit.sources import ( ) from cloudinit.sources import DataSourceNone as DSNone -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers DEFAULT_LOCAL = [ Azure.DataSourceAzure, diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 337be667..237c189b 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -15,7 +15,7 @@ from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit.sources.helpers import openstack from cloudinit import util -from ..helpers import TestCase, ExitStack, mock +from cloudinit.tests.helpers import TestCase, ExitStack, mock PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index e97a679a..f264f361 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -13,7 +13,7 @@ from cloudinit import settings from cloudinit.sources import DataSourceDigitalOcean from cloudinit.sources.helpers import digitalocean -from ..helpers import mock, TestCase +from cloudinit.tests.helpers import mock, TestCase DO_MULTIPLE_KEYS = ["ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@do.co", "ssh-rsa AAAAB3NzaC1yc2EAAAA... test2@do.co"] diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index b7a84e21..9fb90483 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -4,9 +4,9 @@ import copy import httpretty import mock -from .. import helpers as test_helpers from cloudinit import helpers from cloudinit.sources import DataSourceEc2 as ec2 +from cloudinit.tests import helpers as test_helpers # collected from api version 2016-09-02/ with diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index ad608bec..50e49a10 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -15,7 +15,7 @@ from cloudinit import helpers from cloudinit import settings from cloudinit.sources import DataSourceGCE -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers GCE_META = { diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index c1911bf4..289c6a40 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -8,7 +8,7 @@ import yaml from cloudinit.sources import DataSourceMAAS from cloudinit import url_helper -from ..helpers import TestCase, populate_dir +from cloudinit.tests.helpers import TestCase, populate_dir try: from unittest import mock diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index ff294395..fea9156b 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -3,7 +3,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNoCloud from cloudinit import util -from ..helpers import TestCase, populate_dir, mock, ExitStack +from cloudinit.tests.helpers import TestCase, populate_dir, mock, ExitStack import os import shutil diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index b0f8e435..e7d55692 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -3,7 +3,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceOpenNebula as ds from cloudinit import util -from ..helpers import mock, populate_dir, TestCase +from cloudinit.tests.helpers import mock, populate_dir, TestCase import os import pwd diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index c2905d1a..177e9808 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -9,7 +9,7 @@ import httpretty as hp import json import re -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from six.moves.urllib.parse import urlparse from six import StringIO diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index 477cf8ed..9dbf4dd9 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -6,7 +6,7 @@ import base64 -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit.sources import DataSourceOVF as dsovf diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py index 65d83ad7..436df9ee 100644 --- a/tests/unittests/test_datasource/test_scaleway.py +++ b/tests/unittests/test_datasource/test_scaleway.py @@ -9,7 +9,7 @@ from cloudinit import helpers from cloudinit import settings from cloudinit.sources import DataSourceScaleway -from ..helpers import mock, HttprettyTestCase, TestCase +from cloudinit.tests.helpers import mock, HttprettyTestCase, TestCase class DataResponses(object): diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index e3c99bbb..933d5b63 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -33,7 +33,7 @@ import six from cloudinit import helpers as c_helpers from cloudinit.util import b64e -from ..helpers import mock, FilesystemMockingTestCase, TestCase +from cloudinit.tests.helpers import mock, FilesystemMockingTestCase, TestCase SDC_NICS = json.loads(""" [ diff --git a/tests/unittests/test_distros/test_arch.py b/tests/unittests/test_distros/test_arch.py index 3d4c9a70..a95ba3b5 100644 --- a/tests/unittests/test_distros/test_arch.py +++ b/tests/unittests/test_distros/test_arch.py @@ -3,7 +3,7 @@ from cloudinit.distros.arch import _render_network from cloudinit import util -from ..helpers import (CiTestCase, dir2dict) +from cloudinit.tests.helpers import (CiTestCase, dir2dict) from . import _get_distro diff --git a/tests/unittests/test_distros/test_create_users.py b/tests/unittests/test_distros/test_create_users.py index 1d02f7bd..aa13670a 100644 --- a/tests/unittests/test_distros/test_create_users.py +++ b/tests/unittests/test_distros/test_create_users.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import distros -from ..helpers import (TestCase, mock) +from cloudinit.tests.helpers import (TestCase, mock) class MyBaseDistro(distros.Distro): diff --git a/tests/unittests/test_distros/test_debian.py b/tests/unittests/test_distros/test_debian.py index 72d3aad6..da16a797 100644 --- a/tests/unittests/test_distros/test_debian.py +++ b/tests/unittests/test_distros/test_debian.py @@ -2,7 +2,7 @@ from cloudinit import distros from cloudinit import util -from ..helpers import (FilesystemMockingTestCase, mock) +from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock) @mock.patch("cloudinit.distros.debian.util.subp") diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index b355a19e..791fe612 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -3,7 +3,7 @@ from cloudinit import distros from cloudinit import util -from .. import helpers +from cloudinit.tests import helpers import os import shutil diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 6d89dba8..c4bd11bc 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -12,7 +12,7 @@ try: except ImportError: from contextlib2 import ExitStack -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase from cloudinit import distros from cloudinit.distros.parsers.sys_conf import SysConf diff --git a/tests/unittests/test_distros/test_opensuse.py b/tests/unittests/test_distros/test_opensuse.py index bdb1d633..b9bb9b3e 100644 --- a/tests/unittests/test_distros/test_opensuse.py +++ b/tests/unittests/test_distros/test_opensuse.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from ..helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase from . import _get_distro diff --git a/tests/unittests/test_distros/test_resolv.py b/tests/unittests/test_distros/test_resolv.py index 97168cf9..68ea0083 100644 --- a/tests/unittests/test_distros/test_resolv.py +++ b/tests/unittests/test_distros/test_resolv.py @@ -3,7 +3,7 @@ from cloudinit.distros.parsers import resolv_conf from cloudinit.distros import rhel_util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import re import tempfile diff --git a/tests/unittests/test_distros/test_sles.py b/tests/unittests/test_distros/test_sles.py index c656aacc..33e3c457 100644 --- a/tests/unittests/test_distros/test_sles.py +++ b/tests/unittests/test_distros/test_sles.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from ..helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase from . import _get_distro diff --git a/tests/unittests/test_distros/test_sysconfig.py b/tests/unittests/test_distros/test_sysconfig.py index 235ecebb..c1d5b693 100644 --- a/tests/unittests/test_distros/test_sysconfig.py +++ b/tests/unittests/test_distros/test_sysconfig.py @@ -4,7 +4,7 @@ import re from cloudinit.distros.parsers.sys_conf import SysConf -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase # Lots of good examples @ diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index 88746e0a..0fa9cdb5 100644 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -5,7 +5,7 @@ from cloudinit.distros import ug_util from cloudinit import helpers from cloudinit import settings -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import mock diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 8ccfe55c..1a81a89e 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -6,7 +6,8 @@ from uuid import uuid4 from cloudinit import safeyaml from cloudinit import util -from .helpers import CiTestCase, dir2dict, json_dumps, populate_dir +from cloudinit.tests.helpers import ( + CiTestCase, dir2dict, json_dumps, populate_dir) UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu " "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux") diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index 65fdb519..af78997f 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -2,7 +2,7 @@ import httpretty as hp -from . import helpers +from cloudinit.tests import helpers from cloudinit import ec2_utils as eu from cloudinit import url_helper as uh diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py index 13137f6d..6364d38e 100644 --- a/tests/unittests/test_filters/test_launch_index.py +++ b/tests/unittests/test_filters/test_launch_index.py @@ -2,7 +2,7 @@ import copy -from .. import helpers +from cloudinit.tests import helpers from six.moves import filterfalse diff --git a/tests/unittests/test_handler/test_handler_apt_conf_v1.py b/tests/unittests/test_handler/test_handler_apt_conf_v1.py index 554277ff..83f962a9 100644 --- a/tests/unittests/test_handler/test_handler_apt_conf_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_conf_v1.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_apt_configure from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import copy import os diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py index f53ddbb2..d2b96f0b 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py @@ -24,7 +24,7 @@ from cloudinit.sources import DataSourceNone from cloudinit.distros.debian import Distro -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help LOG = logging.getLogger(__name__) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py index 1ca915b4..f7608c28 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py @@ -24,7 +24,7 @@ from cloudinit.sources import DataSourceNone from cloudinit.distros.debian import Distro -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help LOG = logging.getLogger(__name__) diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py index 12502d05..3a3f95ca 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py @@ -20,7 +20,7 @@ from cloudinit.config import cc_apt_configure from cloudinit import gpg from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK----- Version: GnuPG v1 diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py index 292d3f59..7bb1b7c4 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py @@ -28,7 +28,7 @@ from cloudinit import util from cloudinit.config import cc_apt_configure from cloudinit.sources import DataSourceNone -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help EXPECTEDKEY = u"""-----BEGIN PGP PUBLIC KEY BLOCK----- Version: GnuPG v1 diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index 7cee2c3f..06e14db0 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -5,7 +5,7 @@ from cloudinit.config import cc_ca_certs from cloudinit import helpers from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import logging import shutil diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 6a152ea2..e5785cfd 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -14,7 +14,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNone from cloudinit import util -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help LOG = logging.getLogger(__name__) diff --git a/tests/unittests/test_handler/test_handler_debug.py b/tests/unittests/test_handler/test_handler_debug.py index 1873c3e1..787ba350 100644 --- a/tests/unittests/test_handler/test_handler_debug.py +++ b/tests/unittests/test_handler/test_handler_debug.py @@ -11,7 +11,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNone -from ..helpers import (FilesystemMockingTestCase, mock) +from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock) import logging import shutil diff --git a/tests/unittests/test_handler/test_handler_disk_setup.py b/tests/unittests/test_handler/test_handler_disk_setup.py index 8a6d49ed..5afcacaf 100644 --- a/tests/unittests/test_handler/test_handler_disk_setup.py +++ b/tests/unittests/test_handler/test_handler_disk_setup.py @@ -3,7 +3,7 @@ import random from cloudinit.config import cc_disk_setup -from ..helpers import CiTestCase, ExitStack, mock, TestCase +from cloudinit.tests.helpers import CiTestCase, ExitStack, mock, TestCase class TestIsDiskUsed(TestCase): diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index c5fc8c9b..a3e46351 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -4,7 +4,7 @@ from cloudinit import cloud from cloudinit.config import cc_growpart from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import errno import logging diff --git a/tests/unittests/test_handler/test_handler_landscape.py b/tests/unittests/test_handler/test_handler_landscape.py index 7c247fa9..db92a7e2 100644 --- a/tests/unittests/test_handler/test_handler_landscape.py +++ b/tests/unittests/test_handler/test_handler_landscape.py @@ -1,9 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit.config import cc_landscape -from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from ..helpers import FilesystemMockingTestCase, mock, wrap_and_call +from cloudinit.sources import DataSourceNone +from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock, + wrap_and_call) from configobj import ConfigObj import logging diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index a789db32..e29a06f9 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -13,7 +13,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNoCloud -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help from configobj import ConfigObj diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 351226bf..f132a778 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_lxd from cloudinit.sources import DataSourceNoCloud from cloudinit import (distros, helpers, cloud) -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help import logging diff --git a/tests/unittests/test_handler/test_handler_mcollective.py b/tests/unittests/test_handler/test_handler_mcollective.py index 2a9f3823..7eec7352 100644 --- a/tests/unittests/test_handler/test_handler_mcollective.py +++ b/tests/unittests/test_handler/test_handler_mcollective.py @@ -4,7 +4,7 @@ from cloudinit import (cloud, distros, helpers, util) from cloudinit.config import cc_mcollective from cloudinit.sources import DataSourceNoCloud -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help import configobj import logging diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index 650ca0ec..fe492d4b 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -6,7 +6,7 @@ import tempfile from cloudinit.config import cc_mounts -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers try: from unittest import mock diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 83d5faa2..4f291248 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_ntp from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from ..helpers import FilesystemMockingTestCase, mock, skipIf +from cloudinit.tests.helpers import FilesystemMockingTestCase, mock, skipIf import os diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index e382210d..85a0fe0a 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -4,8 +4,8 @@ import sys from cloudinit.config import cc_power_state_change as psc -from .. import helpers as t_help -from ..helpers import mock +from cloudinit.tests import helpers as t_help +from cloudinit.tests.helpers import mock class TestLoadPowerState(t_help.TestCase): diff --git a/tests/unittests/test_handler/test_handler_puppet.py b/tests/unittests/test_handler/test_handler_puppet.py index 805c76ba..0b6e3b58 100644 --- a/tests/unittests/test_handler/test_handler_puppet.py +++ b/tests/unittests/test_handler/test_handler_puppet.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_puppet from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from ..helpers import CiTestCase, mock +from cloudinit.tests.helpers import CiTestCase, mock import logging diff --git a/tests/unittests/test_handler/test_handler_rsyslog.py b/tests/unittests/test_handler/test_handler_rsyslog.py index cca06678..8c8e2838 100644 --- a/tests/unittests/test_handler/test_handler_rsyslog.py +++ b/tests/unittests/test_handler/test_handler_rsyslog.py @@ -9,7 +9,7 @@ from cloudinit.config.cc_rsyslog import ( parse_remotes_line, remotes_to_rsyslog_cfg) from cloudinit import util -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help class TestLoadConfig(t_help.TestCase): diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py index 7880ee72..374c1d31 100644 --- a/tests/unittests/test_handler/test_handler_runcmd.py +++ b/tests/unittests/test_handler/test_handler_runcmd.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_runcmd from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from ..helpers import FilesystemMockingTestCase, skipIf +from cloudinit.tests.helpers import FilesystemMockingTestCase, skipIf import logging import os diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index e5e607fb..f60dedc2 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -22,7 +22,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNone -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help import logging diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py index 8165bf9a..abdc17e7 100644 --- a/tests/unittests/test_handler/test_handler_set_hostname.py +++ b/tests/unittests/test_handler/test_handler_set_hostname.py @@ -7,7 +7,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import util -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help from configobj import ConfigObj import logging diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index e4d07622..76b79c29 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -7,9 +7,9 @@ from cloudinit.config.cc_snap_config import ( from cloudinit import (distros, helpers, cloud, util) from cloudinit.config.cc_snap_config import handle as snap_handle from cloudinit.sources import DataSourceNone -from ..helpers import FilesystemMockingTestCase, mock +from cloudinit.tests.helpers import FilesystemMockingTestCase, mock -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help import logging import os diff --git a/tests/unittests/test_handler/test_handler_spacewalk.py b/tests/unittests/test_handler/test_handler_spacewalk.py index 28b5892a..ddbf4a79 100644 --- a/tests/unittests/test_handler/test_handler_spacewalk.py +++ b/tests/unittests/test_handler/test_handler_spacewalk.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_spacewalk from cloudinit import util -from .. import helpers +from cloudinit.tests import helpers import logging diff --git a/tests/unittests/test_handler/test_handler_timezone.py b/tests/unittests/test_handler/test_handler_timezone.py index c30fbdfe..27eedded 100644 --- a/tests/unittests/test_handler/test_handler_timezone.py +++ b/tests/unittests/test_handler/test_handler_timezone.py @@ -13,7 +13,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNoCloud -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help from configobj import ConfigObj import logging diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py index 1129e77d..7fa8fd21 100644 --- a/tests/unittests/test_handler/test_handler_write_files.py +++ b/tests/unittests/test_handler/test_handler_write_files.py @@ -4,7 +4,7 @@ from cloudinit.config.cc_write_files import write_files, decode_perms from cloudinit import log as logging from cloudinit import util -from ..helpers import CiTestCase, FilesystemMockingTestCase +from cloudinit.tests.helpers import CiTestCase, FilesystemMockingTestCase import base64 import gzip diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py index c4396df5..b7adbe50 100644 --- a/tests/unittests/test_handler/test_handler_yum_add_repo.py +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_yum_add_repo from cloudinit import util -from .. import helpers +from cloudinit.tests import helpers try: from configparser import ConfigParser diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 640f11d4..6137e3cf 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -6,7 +6,7 @@ from cloudinit.config.schema import ( validate_cloudconfig_schema, main) from cloudinit.util import write_file -from ..helpers import CiTestCase, mock, skipIf +from cloudinit.tests.helpers import CiTestCase, mock, skipIf from copy import copy from six import StringIO diff --git a/tests/unittests/test_helpers.py b/tests/unittests/test_helpers.py index f1979e89..2e4582a0 100644 --- a/tests/unittests/test_helpers.py +++ b/tests/unittests/test_helpers.py @@ -4,7 +4,7 @@ import os -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit import sources diff --git a/tests/unittests/test_log.py b/tests/unittests/test_log.py index 68fb4b8d..cd6296d6 100644 --- a/tests/unittests/test_log.py +++ b/tests/unittests/test_log.py @@ -2,9 +2,9 @@ """Tests for cloudinit.log """ -from .helpers import CiTestCase from cloudinit.analyze.dump import CLOUD_INIT_ASCTIME_FMT from cloudinit import log as ci_logging +from cloudinit.tests.helpers import CiTestCase import datetime import logging import six diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 0658b6b4..f51358da 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from . import helpers +from cloudinit.tests import helpers from cloudinit.handlers import cloud_config from cloudinit.handlers import (CONTENT_START, CONTENT_END) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index f251024b..c10ef905 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -11,10 +11,10 @@ from cloudinit.net import sysconfig from cloudinit.sources.helpers import openstack from cloudinit import util -from .helpers import CiTestCase -from .helpers import dir2dict -from .helpers import mock -from .helpers import populate_dir +from cloudinit.tests.helpers import CiTestCase +from cloudinit.tests.helpers import dir2dict +from cloudinit.tests.helpers import mock +from cloudinit.tests.helpers import populate_dir import base64 import copy diff --git a/tests/unittests/test_pathprefix2dict.py b/tests/unittests/test_pathprefix2dict.py index a4ae284f..abbb29b8 100644 --- a/tests/unittests/test_pathprefix2dict.py +++ b/tests/unittests/test_pathprefix2dict.py @@ -2,7 +2,7 @@ from cloudinit import util -from .helpers import TestCase, populate_dir +from cloudinit.tests.helpers import TestCase, populate_dir import shutil import tempfile diff --git a/tests/unittests/test_registry.py b/tests/unittests/test_registry.py index acf0bf4f..2b625026 100644 --- a/tests/unittests/test_registry.py +++ b/tests/unittests/test_registry.py @@ -2,7 +2,7 @@ from cloudinit.registry import DictRegistry -from .helpers import (mock, TestCase) +from cloudinit.tests.helpers import (mock, TestCase) class TestDictRegistry(TestCase): diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index f3b8f992..571420ed 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -8,7 +8,7 @@ from cloudinit.reporting import handlers import mock -from .helpers import TestCase +from cloudinit.tests.helpers import TestCase def _fake_registry(): diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py index ca14cd46..e9d5702a 100644 --- a/tests/unittests/test_rh_subscription.py +++ b/tests/unittests/test_rh_subscription.py @@ -7,7 +7,7 @@ import logging from cloudinit.config import cc_rh_subscription from cloudinit import util -from .helpers import TestCase, mock +from cloudinit.tests.helpers import TestCase, mock class GoodTests(TestCase): diff --git a/tests/unittests/test_runs/test_merge_run.py b/tests/unittests/test_runs/test_merge_run.py index 65895273..add93653 100644 --- a/tests/unittests/test_runs/test_merge_run.py +++ b/tests/unittests/test_runs/test_merge_run.py @@ -4,7 +4,7 @@ import os import shutil import tempfile -from .. import helpers +from cloudinit.tests import helpers from cloudinit.settings import PER_INSTANCE from cloudinit import stages diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py index 55f15b55..5cf666fe 100644 --- a/tests/unittests/test_runs/test_simple_run.py +++ b/tests/unittests/test_runs/test_simple_run.py @@ -4,7 +4,7 @@ import os import shutil import tempfile -from .. import helpers +from cloudinit.tests import helpers from cloudinit.settings import PER_INSTANCE from cloudinit import stages diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index 991f45a6..2a8e6abe 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -2,8 +2,8 @@ from mock import patch -from . import helpers as test_helpers from cloudinit import ssh_util +from cloudinit.tests import helpers as test_helpers VALID_CONTENT = { @@ -57,6 +57,7 @@ TEST_OPTIONS = ( class TestAuthKeyLineParser(test_helpers.TestCase): + def test_simple_parse(self): # test key line with common 3 fields (keytype, base64, comment) parser = ssh_util.AuthKeyLineParser() diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index 4e627826..b911d929 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -6,7 +6,7 @@ from __future__ import print_function -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers import textwrap from cloudinit import templater diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 5f11c88f..3e4154ca 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -12,7 +12,7 @@ import six import yaml from cloudinit import importer, util -from . import helpers +from cloudinit.tests import helpers try: from unittest import mock diff --git a/tests/unittests/test_version.py b/tests/unittests/test_version.py index 1662ce09..d012f69d 100644 --- a/tests/unittests/test_version.py +++ b/tests/unittests/test_version.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from .helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase from cloudinit import version diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py index 03b36d31..d8651077 100644 --- a/tests/unittests/test_vmware_config_file.py +++ b/tests/unittests/test_vmware_config_file.py @@ -8,10 +8,10 @@ import logging import sys -from .helpers import CiTestCase from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProtoEnum from cloudinit.sources.helpers.vmware.imc.config import Config from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile +from cloudinit.tests.helpers import CiTestCase logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) logger = logging.getLogger(__name__) -- cgit v1.2.3 From 409918f9ba83e45e9bc5cc0b6c589e2fc8ae9b60 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 29 Aug 2017 09:59:20 -0400 Subject: Use /run/cloud-init for tempfile operations. During boot, the usage of /tmp is not safe. In systemd systems, systemd-tmpfiles-clean may run at any point and clear out a temp file while cloud-init is using it. The solution here is to use /run/cloud-init/tmp. LP: #1707222 --- cloudinit/config/cc_bootcmd.py | 3 +- cloudinit/config/cc_chef.py | 3 +- cloudinit/config/cc_snappy.py | 4 +- cloudinit/net/dhcp.py | 3 +- cloudinit/sources/helpers/azure.py | 4 +- cloudinit/temp_utils.py | 93 ++++++++++++++++++++++ cloudinit/util.py | 36 +-------- packages/bddeb | 5 +- .../unittests/test_datasource/test_azure_helper.py | 4 +- tests/unittests/test_net.py | 3 +- 10 files changed, 112 insertions(+), 46 deletions(-) create mode 100644 cloudinit/temp_utils.py (limited to 'tests/unittests/test_net.py') diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 604f93b0..9c0476af 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -37,6 +37,7 @@ specified either as lists or strings. For invocation details, see ``runcmd``. import os from cloudinit.settings import PER_ALWAYS +from cloudinit import temp_utils from cloudinit import util frequency = PER_ALWAYS @@ -49,7 +50,7 @@ def handle(name, cfg, cloud, log, _args): " no 'bootcmd' key in configuration"), name) return - with util.ExtendedTemporaryFile(suffix=".sh") as tmpf: + with temp_utils.ExtendedTemporaryFile(suffix=".sh") as tmpf: try: content = util.shellify(cfg["bootcmd"]) tmpf.write(util.encode_text(content)) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 02c70b10..c192dd32 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -71,6 +71,7 @@ import itertools import json import os +from cloudinit import temp_utils from cloudinit import templater from cloudinit import url_helper from cloudinit import util @@ -303,7 +304,7 @@ def install_chef(cloud, chef_cfg, log): "omnibus_url_retries", default=OMNIBUS_URL_RETRIES)) content = url_helper.readurl(url=url, retries=retries).contents - with util.tempdir() as tmpd: + with temp_utils.tempdir() as tmpd: # Use tmpdir over tmpfile to avoid 'text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd util.write_file(tmpf, content, mode=0o700) diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index a9682f19..eecb8178 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -63,11 +63,11 @@ is ``auto``. Options are: from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import temp_utils from cloudinit import util import glob import os -import tempfile LOG = logging.getLogger(__name__) @@ -183,7 +183,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): # config # Note, however, we do not touch config files on disk. nested_cfg = {'config': {shortname: config}} - (fd, cfg_tmpf) = tempfile.mkstemp() + (fd, cfg_tmpf) = temp_utils.mkstemp() os.write(fd, util.yaml_dumps(nested_cfg).encode()) os.close(fd) cfgfile = cfg_tmpf diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index c7febc57..c842c839 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -9,6 +9,7 @@ import os import re from cloudinit.net import find_fallback_nic, get_devicelist +from cloudinit import temp_utils from cloudinit import util LOG = logging.getLogger(__name__) @@ -47,7 +48,7 @@ def maybe_perform_dhcp_discovery(nic=None): if not dhclient_path: LOG.debug('Skip dhclient configuration: No dhclient command found.') return {} - with util.tempdir(prefix='cloud-init-dhcp-') as tmpdir: + with temp_utils.tempdir(prefix='cloud-init-dhcp-') as tmpdir: return dhcp_discovery(dhclient_path, nic, tmpdir) diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index e22409d1..28ed0ae2 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -6,10 +6,10 @@ import os import re import socket import struct -import tempfile import time from cloudinit import stages +from cloudinit import temp_utils from contextlib import contextmanager from xml.etree import ElementTree @@ -111,7 +111,7 @@ class OpenSSLManager(object): } def __init__(self): - self.tmpdir = tempfile.mkdtemp() + self.tmpdir = temp_utils.mkdtemp() self.certificate = None self.generate_certificate() diff --git a/cloudinit/temp_utils.py b/cloudinit/temp_utils.py new file mode 100644 index 00000000..0355f19d --- /dev/null +++ b/cloudinit/temp_utils.py @@ -0,0 +1,93 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import contextlib +import errno +import os +import shutil +import tempfile + +_TMPDIR = None +_ROOT_TMPDIR = "/run/cloud-init/tmp" + + +def _tempfile_dir_arg(odir=None): + """Return the proper 'dir' argument for tempfile functions. + + When root, cloud-init will use /run/cloud-init/tmp to avoid + any cleaning that a distro boot might do on /tmp (such as + systemd-tmpfiles-clean). + + If the caller of this function (mkdtemp or mkstemp) was provided + with a 'dir' argument, then that is respected. + + @param odir: original 'dir' arg to 'mkdtemp' or other.""" + + if odir is not None: + return odir + + global _TMPDIR + if _TMPDIR: + return _TMPDIR + + if os.getuid() == 0: + tdir = _ROOT_TMPDIR + else: + tdir = os.environ.get('TMPDIR', '/tmp') + if not os.path.isdir(tdir): + os.makedirs(tdir) + os.chmod(tdir, 0o1777) + + _TMPDIR = tdir + return tdir + + +def ExtendedTemporaryFile(**kwargs): + kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + fh = tempfile.NamedTemporaryFile(**kwargs) + # Replace its unlink with a quiet version + # that does not raise errors when the + # file to unlink has been unlinked elsewhere.. + + def _unlink_if_exists(path): + try: + os.unlink(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise e + + fh.unlink = _unlink_if_exists + + # Add a new method that will unlink + # right 'now' but still lets the exit + # method attempt to remove it (which will + # not throw due to our del file being quiet + # about files that are not there) + def unlink_now(): + fh.unlink(fh.name) + + setattr(fh, 'unlink_now', unlink_now) + return fh + + +@contextlib.contextmanager +def tempdir(**kwargs): + # This seems like it was only added in python 3.2 + # Make it since its useful... + # See: http://bugs.python.org/file12970/tempdir.patch + tdir = mkdtemp(**kwargs) + try: + yield tdir + finally: + shutil.rmtree(tdir) + + +def mkdtemp(**kwargs): + kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + return tempfile.mkdtemp(**kwargs) + + +def mkstemp(**kwargs): + kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + return tempfile.mkstemp(**kwargs) + +# vi: ts=4 expandtab diff --git a/cloudinit/util.py b/cloudinit/util.py index 609e94c8..ae5cda8d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -30,7 +30,6 @@ import stat import string import subprocess import sys -import tempfile import time from errno import ENOENT, ENOEXEC @@ -45,6 +44,7 @@ from cloudinit import importer from cloudinit import log as logging from cloudinit import mergers from cloudinit import safeyaml +from cloudinit import temp_utils from cloudinit import type_utils from cloudinit import url_helper from cloudinit import version @@ -349,26 +349,6 @@ class DecompressionError(Exception): pass -def ExtendedTemporaryFile(**kwargs): - fh = tempfile.NamedTemporaryFile(**kwargs) - # Replace its unlink with a quiet version - # that does not raise errors when the - # file to unlink has been unlinked elsewhere.. - LOG.debug("Created temporary file %s", fh.name) - fh.unlink = del_file - - # Add a new method that will unlink - # right 'now' but still lets the exit - # method attempt to remove it (which will - # not throw due to our del file being quiet - # about files that are not there) - def unlink_now(): - fh.unlink(fh.name) - - setattr(fh, 'unlink_now', unlink_now) - return fh - - def fork_cb(child_cb, *args, **kwargs): fid = os.fork() if fid == 0: @@ -790,18 +770,6 @@ def umask(n_msk): os.umask(old) -@contextlib.contextmanager -def tempdir(**kwargs): - # This seems like it was only added in python 3.2 - # Make it since its useful... - # See: http://bugs.python.org/file12970/tempdir.patch - tdir = tempfile.mkdtemp(**kwargs) - try: - yield tdir - finally: - del_dir(tdir) - - def center(text, fill, max_len): return '{0:{fill}{align}{size}}'.format(text, fill=fill, align="^", size=max_len) @@ -1587,7 +1555,7 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True): mtypes = [''] mounted = mounts() - with tempdir() as tmpd: + with temp_utils.tempdir() as tmpd: umount = False if os.path.realpath(device) in mounted: mountpoint = mounted[os.path.realpath(device)]['mountpoint'] diff --git a/packages/bddeb b/packages/bddeb index 7c123548..4f2e2ddf 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -21,8 +21,9 @@ def find_root(): if "avoid-pep8-E402-import-not-top-of-file": # Use the util functions from cloudinit sys.path.insert(0, find_root()) - from cloudinit import templater from cloudinit import util + from cloudinit import temp_utils + from cloudinit import templater DEBUILD_ARGS = ["-S", "-d"] @@ -148,7 +149,7 @@ def main(): capture = False templ_data = {'debian_release': args.release} - with util.tempdir() as tdir: + with temp_utils.tempdir() as tdir: # output like 0.7.6-1022-g36e92d3 ver_data = read_version() diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 80ce003d..44b99eca 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -275,7 +275,7 @@ class TestOpenSSLManager(TestCase): mock.patch('builtins.open')) @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) - @mock.patch.object(azure_helper.tempfile, 'mkdtemp') + @mock.patch.object(azure_helper.temp_utils, 'mkdtemp') def test_openssl_manager_creates_a_tmpdir(self, mkdtemp): manager = azure_helper.OpenSSLManager() self.assertEqual(mkdtemp.return_value, manager.tmpdir) @@ -292,7 +292,7 @@ class TestOpenSSLManager(TestCase): manager.clean_up() @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) - @mock.patch.object(azure_helper.tempfile, 'mkdtemp', mock.MagicMock()) + @mock.patch.object(azure_helper.temp_utils, 'mkdtemp', mock.MagicMock()) @mock.patch.object(azure_helper.util, 'del_dir') def test_clean_up(self, del_dir): manager = azure_helper.OpenSSLManager() diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index c10ef905..f2496151 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -9,6 +9,7 @@ from cloudinit.net import network_state from cloudinit.net import renderers from cloudinit.net import sysconfig from cloudinit.sources.helpers import openstack +from cloudinit import temp_utils from cloudinit import util from cloudinit.tests.helpers import CiTestCase @@ -2150,7 +2151,7 @@ class TestCmdlineConfigParsing(CiTestCase): static['mac_address'] = macs['eth1'] expected = {'version': 1, 'config': [dhcp, static]} - with util.tempdir() as tmpd: + with temp_utils.tempdir() as tmpd: for fname, content in pairs: fp = os.path.join(tmpd, fname) files.append(fp) -- cgit v1.2.3 From 57e2e01c703cdd1818d4f4ab8a67f37037d78582 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Tue, 3 Oct 2017 18:56:52 -0500 Subject: network: bridge_stp value not always correct Update network_state to store the bridge_stp value as a boolean. The various renderers then can map the boolean value to the correct output as needed; eni uses 'on/off', sysconfig uses 'yes/no' and netplan will use the boolean directly. Update unittest values for sysconfig and netplan. Both contained the network_state string value which resulted in not correctly enable/disable STP in the target system. Update network_state comment (fd -> forward-delay, add stp as boolean) on bridge commands to match the expected format of a netplan bridge command. LP: #1721157 --- cloudinit/net/eni.py | 3 +++ cloudinit/net/netplan.py | 5 +++-- cloudinit/net/network_state.py | 17 +++++++++++++++-- tests/unittests/test_net.py | 5 +++-- 4 files changed, 24 insertions(+), 6 deletions(-) (limited to 'tests/unittests/test_net.py') diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index bb80ec02..c6a71d16 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -95,6 +95,9 @@ def _iface_add_attrs(iface, index): ignore_map.append('mac_address') for key, value in iface.items(): + # convert bool to string for eni + if type(value) == bool: + value = 'on' if iface[key] else 'off' if not value or key in ignore_map: continue if key in multiline_keys: diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 3b06fbf0..d3788af8 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -244,9 +244,9 @@ class Renderer(renderer.Renderer): for config in network_state.iter_interfaces(): ifname = config.get('name') - # filter None entries up front so we can do simple if key in dict + # filter None (but not False) entries up front ifcfg = dict((key, value) for (key, value) in config.items() - if value) + if value is not None) if_type = ifcfg.get('type') if if_type == 'physical': @@ -318,6 +318,7 @@ class Renderer(renderer.Renderer): (port, cost) = costval.split() newvalue[port] = int(cost) br_config.update({newname: newvalue}) + if len(br_config) > 0: bridge.update({'parameters': br_config}) _extract_addresses(ifcfg, bridge) diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 6faf01b7..890dbf8d 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -48,6 +48,7 @@ NET_CONFIG_TO_V2 = { 'bridge_maxwait': None, 'bridge_pathcost': 'path-cost', 'bridge_portprio': None, + 'bridge_stp': 'stp', 'bridge_waitport': None}} @@ -465,6 +466,18 @@ class NetworkStateInterpreter(object): for param, val in command.get('params', {}).items(): iface.update({param: val}) + # convert value to boolean + bridge_stp = iface.get('bridge_stp') + if bridge_stp and type(bridge_stp) != bool: + if bridge_stp in ['on', '1', 1]: + bridge_stp = True + elif bridge_stp in ['off', '0', 0]: + bridge_stp = False + else: + raise ValueError("Cannot convert bridge_stp value" + "(%s) to boolean", bridge_stp) + iface.update({'bridge_stp': bridge_stp}) + interfaces.update({iface['name']: iface}) @ensure_command_keys(['address']) @@ -525,8 +538,8 @@ class NetworkStateInterpreter(object): v2_command = { br0: { 'interfaces': ['interface0', 'interface1'], - 'fd': 0, - 'stp': 'off', + 'forward-delay': 0, + 'stp': False, 'maxwait': 0, } } diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index f2496151..17c9342b 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -756,6 +756,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true eth3: 50 eth4: 75 priority: 22 + stp: false routes: - to: ::/0 via: 2001:4800:78ff:1b::1 @@ -820,7 +821,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true NM_CONTROLLED=no ONBOOT=yes PRIO=22 - STP=off + STP=no TYPE=Bridge USERCTL=no"""), 'ifcfg-eth0': textwrap.dedent("""\ @@ -1296,7 +1297,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true NM_CONTROLLED=no ONBOOT=yes PRIO=22 - STP=off + STP=no TYPE=Bridge USERCTL=no """), -- cgit v1.2.3 From 45d361cb0b7f5e4e7d79522bd285871898358623 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 5 Oct 2017 14:26:34 -0600 Subject: net: Handle bridge stp values of 0 and convert to boolean type Update unit tests to pass a 0 instead of 'off' to validate that network state is properly written. --- cloudinit/net/network_state.py | 2 +- tests/unittests/test_net.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'tests/unittests/test_net.py') diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 890dbf8d..0e830ee8 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -468,7 +468,7 @@ class NetworkStateInterpreter(object): # convert value to boolean bridge_stp = iface.get('bridge_stp') - if bridge_stp and type(bridge_stp) != bool: + if bridge_stp is not None and type(bridge_stp) != bool: if bridge_stp in ['on', '1', 1]: bridge_stp = True elif bridge_stp in ['off', '0', 0]: diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 17c9342b..bbb63cb3 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1283,7 +1283,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true - eth0 - eth1 params: - bridge_stp: 'off' + bridge_stp: 0 bridge_bridgeprio: 22 subnets: - type: static -- cgit v1.2.3