From d5f855dd96ccbea77f61b0515b574ad2c43d116d Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 9 Aug 2017 21:55:52 -0600 Subject: ec2: Allow Ec2 to run in init-local using dhclient in a sandbox. This branch is a prerequisite for IPv6 support in AWS by allowing Ec2 datasource to query the metadata source version 2016-09-02 about whether or not it needs to configure IPv6 on interfaces. If version 2016-09-02 is not present, fallback to the min_metadata_version of 2009-04-04. The DataSourceEc2Local not run on FreeBSD because dhclient in doesn't support the -sf flag allowing us to run dhclient without filesystem side-effects. To query AWS' metadata address @ 169.254.169.254, the instance must have a dhcp-allocated address configured. Configuring IPv4 link-local addresses result in timeouts from the metadata service. We introduced a DataSourceEc2Local subclass which will perform a sandboxed dhclient discovery which obtains an authorized IP address on eth0 and crawl metadata about full instance network configuration. Since ec2 IPv6 metadata is not sufficient in itself to tell us all the ipv6 knownledge we need, it only be used as a boolean to tell us which nics need IPv6. Cloud-init will then configure desired interfaces to DHCPv6 versus DHCPv4. Performance side note: Shifting the dhcp work into init-local for Ec2 actually gets us 1 second faster deployments by skipping init-network phase of alternate datasource checks because Ec2Local is configured in an ealier boot stage. In 3 test runs prior to this change: cloud-init runs were 5.5 seconds, with the change we now average 4.6 seconds. This efficiency could be even further improved if we avoiding dhcp discovery in order to talk to the metadata service from an AWS authorized dhcp address if there were some way to advertize the dhcp configuration via DMI/SMBIOS or system environment variables. Inspecting time costs of the dhclient setup/teardown in 3 live runs the time cost for the dhcp setup round trip on AWS is: test 1: 76 milliseconds dhcp discovery + metadata: 0.347 seconds metadata alone: 0.271 seconds test 2: 88 milliseconds dhcp discovery + metadata: 0.388 seconds metadata alone: 0.300 seconds test 3: 75 milliseconds dhcp discovery + metadata: 0.366 seconds metadata alone: 0.291 seconds LP: #1709772 --- tests/unittests/test_datasource/test_ec2.py | 136 +++++++++++++++++++++++----- 1 file changed, 112 insertions(+), 24 deletions(-) (limited to 'tests/unittests/test_datasource/test_ec2.py') diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index 12230ae2..33d02619 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -8,35 +8,67 @@ from cloudinit import helpers from cloudinit.sources import DataSourceEc2 as ec2 -# collected from api version 2009-04-04/ with +# collected from api version 2016-09-02/ with # python3 -c 'import json # from cloudinit.ec2_utils import get_instance_metadata as gm -# print(json.dumps(gm("2009-04-04"), indent=1, sort_keys=True))' +# print(json.dumps(gm("2016-09-02"), indent=1, sort_keys=True))' DEFAULT_METADATA = { - "ami-id": "ami-80861296", + "ami-id": "ami-8b92b4ee", "ami-launch-index": "0", "ami-manifest-path": "(unknown)", "block-device-mapping": {"ami": "/dev/sda1", "root": "/dev/sda1"}, - "hostname": "ip-10-0-0-149", + "hostname": "ip-172-31-31-158.us-east-2.compute.internal", "instance-action": "none", - "instance-id": "i-0052913950685138c", - "instance-type": "t2.micro", - "local-hostname": "ip-10-0-0-149", - "local-ipv4": "10.0.0.149", - "placement": {"availability-zone": "us-east-1b"}, + "instance-id": "i-0a33f80f09c96477f", + "instance-type": "t2.small", + "local-hostname": "ip-172-3-3-15.us-east-2.compute.internal", + "local-ipv4": "172.3.3.15", + "mac": "06:17:04:d7:26:09", + "metrics": {"vhostmd": ""}, + "network": { + "interfaces": { + "macs": { + "06:17:04:d7:26:09": { + "device-number": "0", + "interface-id": "eni-e44ef49e", + "ipv4-associations": {"13.59.77.202": "172.3.3.15"}, + "ipv6s": "2600:1f16:aeb:b20b:9d87:a4af:5cc9:73dc", + "local-hostname": ("ip-172-3-3-15.us-east-2." + "compute.internal"), + "local-ipv4s": "172.3.3.15", + "mac": "06:17:04:d7:26:09", + "owner-id": "950047163771", + "public-hostname": ("ec2-13-59-77-202.us-east-2." + "compute.amazonaws.com"), + "public-ipv4s": "13.59.77.202", + "security-group-ids": "sg-5a61d333", + "security-groups": "wide-open", + "subnet-id": "subnet-20b8565b", + "subnet-ipv4-cidr-block": "172.31.16.0/20", + "subnet-ipv6-cidr-blocks": "2600:1f16:aeb:b20b::/64", + "vpc-id": "vpc-87e72bee", + "vpc-ipv4-cidr-block": "172.31.0.0/16", + "vpc-ipv4-cidr-blocks": "172.31.0.0/16", + "vpc-ipv6-cidr-blocks": "2600:1f16:aeb:b200::/56" + } + } + } + }, + "placement": {"availability-zone": "us-east-2b"}, "profile": "default-hvm", - "public-hostname": "", - "public-ipv4": "107.23.188.247", + "public-hostname": "ec2-13-59-77-202.us-east-2.compute.amazonaws.com", + "public-ipv4": "13.59.77.202", "public-keys": {"brickies": ["ssh-rsa AAAAB3Nz....w== brickies"]}, - "reservation-id": "r-00a2c173fb5782a08", - "security-groups": "wide-open" + "reservation-id": "r-01efbc9996bac1bd6", + "security-groups": "my-wide-open", + "services": {"domain": "amazonaws.com", "partition": "aws"} } def _register_ssh_keys(rfunc, base_url, keys_data): """handle ssh key inconsistencies. - public-keys in the ec2 metadata is inconsistently formatted compared + public-keys in the ec2 metadata is inconsistently formated compared to other entries. Given keys_data of {name1: pubkey1, name2: pubkey2} @@ -115,6 +147,8 @@ def register_mock_metaserver(base_url, data): class TestEc2(test_helpers.HttprettyTestCase): + with_logs = True + valid_platform_data = { 'uuid': 'ec212f79-87d1-2f1d-588f-d86dc0fd5412', 'uuid_source': 'dmi', @@ -123,16 +157,20 @@ class TestEc2(test_helpers.HttprettyTestCase): def setUp(self): super(TestEc2, self).setUp() - self.metadata_addr = ec2.DataSourceEc2.metadata_urls[0] - self.api_ver = '2009-04-04' + self.datasource = ec2.DataSourceEc2 + self.metadata_addr = self.datasource.metadata_urls[0] @property def metadata_url(self): - return '/'.join([self.metadata_addr, self.api_ver, 'meta-data', '']) + return '/'.join([ + self.metadata_addr, + self.datasource.min_metadata_version, 'meta-data', '']) @property def userdata_url(self): - return '/'.join([self.metadata_addr, self.api_ver, 'user-data']) + return '/'.join([ + self.metadata_addr, + self.datasource.min_metadata_version, 'user-data']) def _patch_add_cleanup(self, mpath, *args, **kwargs): p = mock.patch(mpath, *args, **kwargs) @@ -144,7 +182,7 @@ class TestEc2(test_helpers.HttprettyTestCase): paths = helpers.Paths({}) if sys_cfg is None: sys_cfg = {} - ds = ec2.DataSourceEc2(sys_cfg=sys_cfg, distro=distro, paths=paths) + ds = self.datasource(sys_cfg=sys_cfg, distro=distro, paths=paths) if platform_data is not None: self._patch_add_cleanup( "cloudinit.sources.DataSourceEc2._collect_platform_data", @@ -157,14 +195,16 @@ class TestEc2(test_helpers.HttprettyTestCase): return ds @httpretty.activate - def test_valid_platform_with_strict_true(self): + @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') + def test_valid_platform_with_strict_true(self, m_dhcp): """Valid platform data should return true with strict_id true.""" ds = self._setup_ds( platform_data=self.valid_platform_data, sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, md=DEFAULT_METADATA) ret = ds.get_data() - self.assertEqual(True, ret) + self.assertTrue(ret) + self.assertEqual(0, m_dhcp.call_count) @httpretty.activate def test_valid_platform_with_strict_false(self): @@ -174,7 +214,7 @@ class TestEc2(test_helpers.HttprettyTestCase): sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, md=DEFAULT_METADATA) ret = ds.get_data() - self.assertEqual(True, ret) + self.assertTrue(ret) @httpretty.activate def test_unknown_platform_with_strict_true(self): @@ -185,7 +225,7 @@ class TestEc2(test_helpers.HttprettyTestCase): sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, md=DEFAULT_METADATA) ret = ds.get_data() - self.assertEqual(False, ret) + self.assertFalse(ret) @httpretty.activate def test_unknown_platform_with_strict_false(self): @@ -196,7 +236,55 @@ class TestEc2(test_helpers.HttprettyTestCase): sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, md=DEFAULT_METADATA) ret = ds.get_data() - self.assertEqual(True, ret) + self.assertTrue(ret) + + @httpretty.activate + @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD') + def test_ec2_local_returns_false_on_bsd(self, m_is_freebsd): + """DataSourceEc2Local returns False on BSD. + + FreeBSD dhclient doesn't support dhclient -sf to run in a sandbox. + """ + m_is_freebsd.return_value = True + self.datasource = ec2.DataSourceEc2Local + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, + md=DEFAULT_METADATA) + ret = ds.get_data() + self.assertFalse(ret) + self.assertIn( + "FreeBSD doesn't support running dhclient with -sf", + self.logs.getvalue()) + + @httpretty.activate + @mock.patch('cloudinit.net.EphemeralIPv4Network') + @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') + @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD') + def test_ec2_local_performs_dhcp_on_non_bsd(self, m_is_bsd, m_dhcp, m_net): + """Ec2Local returns True for valid platform data on non-BSD with dhcp. + + DataSourceEc2Local will setup initial IPv4 network via dhcp discovery. + Then the metadata services is crawled for more network config info. + When the platform data is valid, return True. + """ + m_is_bsd.return_value = False + m_dhcp.return_value = [{ + 'interface': 'eth9', 'fixed-address': '192.168.2.9', + 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0', + 'broadcast-address': '192.168.2.255'}] + self.datasource = ec2.DataSourceEc2Local + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, + md=DEFAULT_METADATA) + ret = ds.get_data() + self.assertTrue(ret) + m_dhcp.assert_called_once_with() + m_net.assert_called_once_with( + broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9', + prefix_or_mask='255.255.255.0', router='192.168.2.1') + self.assertIn('Crawl of metadata service took', self.logs.getvalue()) # vi: ts=4 expandtab -- cgit v1.2.3 From 3c45330af2a301f2bf219da556844d01cef6778e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 29 Aug 2017 10:34:19 -0600 Subject: ec2: Add IPv6 dhcp support to Ec2DataSource. DataSourceEc2 now parses the metadata for each nic to determine if configured for ipv6 and/or ipv4 addresses. In AWS for metadata version 2016-09-02, nics configured for ipv4 or ipv6 addresses will have non-zero values stored in metadata at network/interfaces/macs//public-ipv4 or ipv6s respectively. Those metadata files are only non-zero when an ipv4 or ipv6 ip is associated to the specific nic. A new DataSourceEc2.network_config property is added which parses the metadata and renders a network version 1 dictionary representing both dhcp4 and dhcp6 configuration for associated nics. The network configuration returned from the datasource will also 'pin' the nic name to the name presented on the instance for each nic. LP: #1639030 --- cloudinit/sources/DataSourceEc2.py | 38 +++++++++++ tests/unittests/test_datasource/test_ec2.py | 97 +++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) (limited to 'tests/unittests/test_datasource/test_ec2.py') diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 8e5f8ee4..07c12bb4 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -57,6 +57,8 @@ class DataSourceEc2(sources.DataSource): _cloud_platform = None + _network_config = None # Used for caching calculated network config v1 + # Whether we want to get network configuration from the metadata service. get_network_metadata = False @@ -279,6 +281,15 @@ class DataSourceEc2(sources.DataSource): util.get_cfg_by_path(cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT), cfg) + @property + def network_config(self): + """Return a network config dict for rendering ENI or netplan files.""" + if self._network_config is None: + if self.metadata is not None: + self._network_config = convert_ec2_metadata_network_config( + self.metadata) + return self._network_config + def _crawl_metadata(self): """Crawl metadata service when available. @@ -423,6 +434,33 @@ def _collect_platform_data(): return data +def convert_ec2_metadata_network_config(metadata=None, macs_to_nics=None): + """Convert ec2 metadata to network config version 1 data dict. + + @param: metadata: Dictionary of metadata crawled from EC2 metadata url. + @param: macs_to_name: Optional dict mac addresses and the nic name. If + not provided, get_interfaces_by_mac is called to get it from the OS. + + @return A dict of network config version 1 based on the metadata and macs. + """ + netcfg = {'version': 1, 'config': []} + if not macs_to_nics: + macs_to_nics = net.get_interfaces_by_mac() + macs_metadata = metadata['network']['interfaces']['macs'] + for mac, nic_name in macs_to_nics.items(): + nic_metadata = macs_metadata.get(mac) + if not nic_metadata: + continue # Not a physical nic represented in metadata + nic_cfg = {'type': 'physical', 'name': nic_name, 'subnets': []} + nic_cfg['mac_address'] = mac + if nic_metadata.get('public-ipv4s'): + nic_cfg['subnets'].append({'type': 'dhcp4'}) + if nic_metadata.get('ipv6s'): + nic_cfg['subnets'].append({'type': 'dhcp6'}) + netcfg['config'].append(nic_cfg) + return netcfg + + # Used to match classes to dependencies datasources = [ (DataSourceEc2Local, (sources.DEP_FILESYSTEM,)), # Run at init-local diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index 33d02619..e1ce6446 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. +import copy import httpretty import mock @@ -194,6 +195,34 @@ class TestEc2(test_helpers.HttprettyTestCase): return ds + @httpretty.activate + def test_network_config_property_returns_version_1_network_data(self): + """network_config property returns network version 1 for metadata.""" + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, + md=DEFAULT_METADATA) + ds.get_data() + mac1 = '06:17:04:d7:26:09' # Defined in DEFAULT_METADATA + expected = {'version': 1, 'config': [ + {'mac_address': '06:17:04:d7:26:09', 'name': 'eth9', + 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}], + 'type': 'physical'}]} + patch_path = ( + 'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac') + with mock.patch(patch_path) as m_get_interfaces_by_mac: + m_get_interfaces_by_mac.return_value = {mac1: 'eth9'} + self.assertEqual(expected, ds.network_config) + + def test_network_config_property_is_cached_in_datasource(self): + """network_config property is cached in DataSourceEc2.""" + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, + md=DEFAULT_METADATA) + ds._network_config = {'cached': 'data'} + self.assertEqual({'cached': 'data'}, ds.network_config) + @httpretty.activate @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') def test_valid_platform_with_strict_true(self, m_dhcp): @@ -287,4 +316,72 @@ class TestEc2(test_helpers.HttprettyTestCase): self.assertIn('Crawl of metadata service took', self.logs.getvalue()) +class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): + + def setUp(self): + super(TestConvertEc2MetadataNetworkConfig, self).setUp() + self.mac1 = '06:17:04:d7:26:09' + self.network_metadata = { + 'network': {'interfaces': {'macs': { + self.mac1: {'public-ipv4s': '172.31.2.16'}}}}} + + def test_convert_ec2_metadata_network_config_skips_absent_macs(self): + """Any mac absent from metadata is skipped by network config.""" + macs_to_nics = {self.mac1: 'eth9', 'DE:AD:BE:EF:FF:FF': 'vitualnic2'} + + # DE:AD:BE:EF:FF:FF represented by OS but not in metadata + expected = {'version': 1, 'config': [ + {'mac_address': self.mac1, 'type': 'physical', + 'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]} + self.assertEqual( + expected, + ec2.convert_ec2_metadata_network_config( + self.network_metadata, macs_to_nics)) + + def test_convert_ec2_metadata_network_config_handles_only_dhcp6(self): + """Config dhcp6 when ipv6s is in metadata for a mac.""" + macs_to_nics = {self.mac1: 'eth9'} + network_metadata_ipv6 = copy.deepcopy(self.network_metadata) + nic1_metadata = ( + network_metadata_ipv6['network']['interfaces']['macs'][self.mac1]) + nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64' + nic1_metadata.pop('public-ipv4s') + expected = {'version': 1, 'config': [ + {'mac_address': self.mac1, 'type': 'physical', + 'name': 'eth9', 'subnets': [{'type': 'dhcp6'}]}]} + self.assertEqual( + expected, + ec2.convert_ec2_metadata_network_config( + network_metadata_ipv6, macs_to_nics)) + + def test_convert_ec2_metadata_network_config_handles_dhcp4_and_dhcp6(self): + """Config both dhcp4 and dhcp6 when both vpc-ipv6 and ipv4 exists.""" + macs_to_nics = {self.mac1: 'eth9'} + network_metadata_both = copy.deepcopy(self.network_metadata) + nic1_metadata = ( + network_metadata_both['network']['interfaces']['macs'][self.mac1]) + nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64' + expected = {'version': 1, 'config': [ + {'mac_address': self.mac1, 'type': 'physical', + 'name': 'eth9', + 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}]}]} + self.assertEqual( + expected, + ec2.convert_ec2_metadata_network_config( + network_metadata_both, macs_to_nics)) + + def test_convert_ec2_metadata_gets_macs_from_get_interfaces_by_mac(self): + """Convert Ec2 Metadata calls get_interfaces_by_mac by default.""" + expected = {'version': 1, 'config': [ + {'mac_address': self.mac1, 'type': 'physical', + 'name': 'eth9', + 'subnets': [{'type': 'dhcp4'}]}]} + patch_path = ( + 'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac') + with mock.patch(patch_path) as m_get_interfaces_by_mac: + m_get_interfaces_by_mac.return_value = {self.mac1: 'eth9'} + self.assertEqual( + expected, + ec2.convert_ec2_metadata_network_config(self.network_metadata)) + # vi: ts=4 expandtab -- cgit v1.2.3 From 1770a1eb647d24e14732194e72210ea494986ad2 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 31 Aug 2017 16:24:35 -0600 Subject: tests: Stop leaking calls through unmocked metadata addresses DataSourceEc2 behavior changed to first check a minimum acceptable metadata version uri http://169.154.169.254//instance-id, retrying on 404, until the metadata service is available. After the metadata service is up, the datasource inspects preferred extended_metadata_versions for availability. Unit tests only mocked the preferred extended_metadata_version so all Ec2 tests were retrying attempts against http://169.254.169.254/meta-data//instance-id adding a lot of time cost to the unit test runs. This branch uses httpretty to properly mock the following: - 404s from metadata on undesired extended_metadata_version test routes - https://169.254.169.254/meta-data/2016-09-02/instance-id - full metadata dictionary represented on min_metadata_version - https://169.254.169.254/meta-data/2016-09-02/* The branch also tightens httpretty to raise a MockError for any URL which isn't mocked via httpretty.HTTPretty.allow_net_connect=False. LP: #1714117 --- tests/unittests/test_datasource/test_ec2.py | 46 +++++++++++++++++++---------- 1 file changed, 30 insertions(+), 16 deletions(-) (limited to 'tests/unittests/test_datasource/test_ec2.py') diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index e1ce6446..b7a84e21 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -116,6 +116,9 @@ def register_mock_metaserver(base_url, data): In the index, references to lists or dictionaries have a trailing /. """ def register_helper(register, base_url, body): + if not isinstance(base_url, str): + register(base_url, body) + return base_url = base_url.rstrip("/") if isinstance(body, str): register(base_url, body) @@ -138,7 +141,7 @@ def register_mock_metaserver(base_url, data): register(base_url, '\n'.join(vals) + '\n') register(base_url + '/', '\n'.join(vals) + '\n') elif body is None: - register(base_url, 'not found', status_code=404) + register(base_url, 'not found', status=404) def myreg(*argc, **kwargs): # print("register_url(%s, %s)" % (argc, kwargs)) @@ -161,38 +164,47 @@ class TestEc2(test_helpers.HttprettyTestCase): self.datasource = ec2.DataSourceEc2 self.metadata_addr = self.datasource.metadata_urls[0] - @property - def metadata_url(self): - return '/'.join([ - self.metadata_addr, - self.datasource.min_metadata_version, 'meta-data', '']) - - @property - def userdata_url(self): - return '/'.join([ - self.metadata_addr, - self.datasource.min_metadata_version, 'user-data']) + def data_url(self, version): + """Return a metadata url based on the version provided.""" + return '/'.join([self.metadata_addr, version, 'meta-data', '']) def _patch_add_cleanup(self, mpath, *args, **kwargs): p = mock.patch(mpath, *args, **kwargs) p.start() self.addCleanup(p.stop) - def _setup_ds(self, sys_cfg, platform_data, md, ud=None): + def _setup_ds(self, sys_cfg, platform_data, md, md_version=None): + self.uris = [] distro = {} paths = helpers.Paths({}) if sys_cfg is None: sys_cfg = {} ds = self.datasource(sys_cfg=sys_cfg, distro=distro, paths=paths) + if not md_version: + md_version = ds.min_metadata_version if platform_data is not None: self._patch_add_cleanup( "cloudinit.sources.DataSourceEc2._collect_platform_data", return_value=platform_data) if md: - register_mock_metaserver(self.metadata_url, md) - register_mock_metaserver(self.userdata_url, ud) - + httpretty.HTTPretty.allow_net_connect = False + all_versions = ( + [ds.min_metadata_version] + ds.extended_metadata_versions) + for version in all_versions: + metadata_url = self.data_url(version) + if version == md_version: + # Register all metadata for desired version + register_mock_metaserver(metadata_url, md) + else: + instance_id_url = metadata_url + 'instance-id' + if version == ds.min_metadata_version: + # Add min_metadata_version service availability check + register_mock_metaserver( + instance_id_url, DEFAULT_METADATA['instance-id']) + else: + # Register 404s for all unrequested extended versions + register_mock_metaserver(instance_id_url, None) return ds @httpretty.activate @@ -297,6 +309,7 @@ class TestEc2(test_helpers.HttprettyTestCase): Then the metadata services is crawled for more network config info. When the platform data is valid, return True. """ + m_is_bsd.return_value = False m_dhcp.return_value = [{ 'interface': 'eth9', 'fixed-address': '192.168.2.9', @@ -307,6 +320,7 @@ class TestEc2(test_helpers.HttprettyTestCase): platform_data=self.valid_platform_data, sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, md=DEFAULT_METADATA) + ret = ds.get_data() self.assertTrue(ret) m_dhcp.assert_called_once_with() -- cgit v1.2.3 From a3649e03206a3596131413956ea7ecc18790ec73 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Tue, 5 Sep 2017 11:03:59 -0600 Subject: relocate tests/unittests/helpers.py to cloudinit/tests This moves the base test case classes into into cloudinit/tests and updates all the corresponding imports. --- cloudinit/analyze/tests/test_dump.py | 2 +- cloudinit/net/tests/test_dhcp.py | 2 +- cloudinit/net/tests/test_init.py | 2 +- cloudinit/tests/helpers.py | 395 +++++++++++++++++++++ cloudinit/tests/test_url_helper.py | 2 +- tests/unittests/helpers.py | 391 -------------------- tests/unittests/test__init__.py | 2 +- tests/unittests/test_atomic_helper.py | 2 +- tests/unittests/test_builtin_handlers.py | 2 +- tests/unittests/test_cli.py | 2 +- tests/unittests/test_cs_util.py | 2 +- tests/unittests/test_data.py | 2 +- tests/unittests/test_datasource/test_aliyun.py | 2 +- tests/unittests/test_datasource/test_altcloud.py | 2 +- tests/unittests/test_datasource/test_azure.py | 6 +- .../unittests/test_datasource/test_azure_helper.py | 2 +- tests/unittests/test_datasource/test_cloudsigma.py | 2 +- tests/unittests/test_datasource/test_cloudstack.py | 2 +- tests/unittests/test_datasource/test_common.py | 2 +- .../unittests/test_datasource/test_configdrive.py | 2 +- .../unittests/test_datasource/test_digitalocean.py | 2 +- tests/unittests/test_datasource/test_ec2.py | 2 +- tests/unittests/test_datasource/test_gce.py | 2 +- tests/unittests/test_datasource/test_maas.py | 2 +- tests/unittests/test_datasource/test_nocloud.py | 2 +- tests/unittests/test_datasource/test_opennebula.py | 2 +- tests/unittests/test_datasource/test_openstack.py | 2 +- tests/unittests/test_datasource/test_ovf.py | 2 +- tests/unittests/test_datasource/test_scaleway.py | 2 +- tests/unittests/test_datasource/test_smartos.py | 2 +- tests/unittests/test_distros/test_arch.py | 2 +- tests/unittests/test_distros/test_create_users.py | 2 +- tests/unittests/test_distros/test_debian.py | 2 +- tests/unittests/test_distros/test_generic.py | 2 +- tests/unittests/test_distros/test_netconfig.py | 2 +- tests/unittests/test_distros/test_opensuse.py | 2 +- tests/unittests/test_distros/test_resolv.py | 2 +- tests/unittests/test_distros/test_sles.py | 2 +- tests/unittests/test_distros/test_sysconfig.py | 2 +- .../test_distros/test_user_data_normalize.py | 2 +- tests/unittests/test_ds_identify.py | 3 +- tests/unittests/test_ec2_util.py | 2 +- tests/unittests/test_filters/test_launch_index.py | 2 +- .../test_handler/test_handler_apt_conf_v1.py | 2 +- .../test_handler_apt_configure_sources_list_v1.py | 2 +- .../test_handler_apt_configure_sources_list_v3.py | 2 +- .../test_handler/test_handler_apt_source_v1.py | 2 +- .../test_handler/test_handler_apt_source_v3.py | 2 +- .../test_handler/test_handler_ca_certs.py | 2 +- tests/unittests/test_handler/test_handler_chef.py | 2 +- tests/unittests/test_handler/test_handler_debug.py | 2 +- .../test_handler/test_handler_disk_setup.py | 2 +- .../test_handler/test_handler_growpart.py | 2 +- .../test_handler/test_handler_landscape.py | 5 +- .../unittests/test_handler/test_handler_locale.py | 2 +- tests/unittests/test_handler/test_handler_lxd.py | 2 +- .../test_handler/test_handler_mcollective.py | 2 +- .../unittests/test_handler/test_handler_mounts.py | 2 +- tests/unittests/test_handler/test_handler_ntp.py | 2 +- .../test_handler/test_handler_power_state.py | 4 +- .../unittests/test_handler/test_handler_puppet.py | 2 +- .../unittests/test_handler/test_handler_rsyslog.py | 2 +- .../unittests/test_handler/test_handler_runcmd.py | 2 +- .../test_handler/test_handler_seed_random.py | 2 +- .../test_handler/test_handler_set_hostname.py | 2 +- .../unittests/test_handler/test_handler_snappy.py | 4 +- .../test_handler/test_handler_spacewalk.py | 2 +- .../test_handler/test_handler_timezone.py | 2 +- .../test_handler/test_handler_write_files.py | 2 +- .../test_handler/test_handler_yum_add_repo.py | 2 +- tests/unittests/test_handler/test_schema.py | 2 +- tests/unittests/test_helpers.py | 2 +- tests/unittests/test_log.py | 2 +- tests/unittests/test_merging.py | 2 +- tests/unittests/test_net.py | 8 +- tests/unittests/test_pathprefix2dict.py | 2 +- tests/unittests/test_registry.py | 2 +- tests/unittests/test_reporting.py | 2 +- tests/unittests/test_rh_subscription.py | 2 +- tests/unittests/test_runs/test_merge_run.py | 2 +- tests/unittests/test_runs/test_simple_run.py | 2 +- tests/unittests/test_sshutil.py | 3 +- tests/unittests/test_templating.py | 2 +- tests/unittests/test_util.py | 2 +- tests/unittests/test_version.py | 2 +- tests/unittests/test_vmware_config_file.py | 2 +- 86 files changed, 491 insertions(+), 482 deletions(-) create mode 100644 cloudinit/tests/helpers.py delete mode 100644 tests/unittests/helpers.py (limited to 'tests/unittests/test_datasource/test_ec2.py') diff --git a/cloudinit/analyze/tests/test_dump.py b/cloudinit/analyze/tests/test_dump.py index 2c0885d0..f4c42841 100644 --- a/cloudinit/analyze/tests/test_dump.py +++ b/cloudinit/analyze/tests/test_dump.py @@ -6,7 +6,7 @@ from textwrap import dedent from cloudinit.analyze.dump import ( dump_events, parse_ci_logline, parse_timestamp) from cloudinit.util import subp, write_file -from tests.unittests.helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase class TestParseTimestamp(CiTestCase): diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index 47d8d461..4a37e98a 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -8,7 +8,7 @@ from cloudinit.net.dhcp import ( InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery, parse_dhcp_lease_file, dhcp_discovery) from cloudinit.util import ensure_file, write_file -from tests.unittests.helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase class TestParseDHCPLeasesFile(CiTestCase): diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index cc052a7d..8cb4114e 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -7,7 +7,7 @@ import os import cloudinit.net as net from cloudinit.util import ensure_file, write_file, ProcessExecutionError -from tests.unittests.helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase class TestSysDevPath(CiTestCase): diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py new file mode 100644 index 00000000..28e26622 --- /dev/null +++ b/cloudinit/tests/helpers.py @@ -0,0 +1,395 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from __future__ import print_function + +import functools +import json +import logging +import os +import shutil +import sys +import tempfile +import unittest + +import mock +import six +import unittest2 + +try: + from contextlib import ExitStack +except ImportError: + from contextlib2 import ExitStack + +from cloudinit import helpers as ch +from cloudinit import util + +# Used for skipping tests +SkipTest = unittest2.SkipTest + +# Used for detecting different python versions +PY2 = False +PY26 = False +PY27 = False +PY3 = False + +_PY_VER = sys.version_info +_PY_MAJOR, _PY_MINOR, _PY_MICRO = _PY_VER[0:3] +if (_PY_MAJOR, _PY_MINOR) <= (2, 6): + if (_PY_MAJOR, _PY_MINOR) == (2, 6): + PY26 = True + if (_PY_MAJOR, _PY_MINOR) >= (2, 0): + PY2 = True +else: + if (_PY_MAJOR, _PY_MINOR) == (2, 7): + PY27 = True + PY2 = True + if (_PY_MAJOR, _PY_MINOR) >= (3, 0): + PY3 = True + + +# Makes the old path start +# with new base instead of whatever +# it previously had +def rebase_path(old_path, new_base): + if old_path.startswith(new_base): + # Already handled... + return old_path + # Retarget the base of that path + # to the new base instead of the + # old one... + path = os.path.join(new_base, old_path.lstrip("/")) + path = os.path.abspath(path) + return path + + +# Can work on anything that takes a path as arguments +def retarget_many_wrapper(new_base, am, old_func): + def wrapper(*args, **kwds): + n_args = list(args) + nam = am + if am == -1: + nam = len(n_args) + for i in range(0, nam): + path = args[i] + # patchOS() wraps various os and os.path functions, however in + # Python 3 some of these now accept file-descriptors (integers). + # That breaks rebase_path() so in lieu of a better solution, just + # don't rebase if we get a fd. + if isinstance(path, six.string_types): + n_args[i] = rebase_path(path, new_base) + return old_func(*n_args, **kwds) + return wrapper + + +class TestCase(unittest2.TestCase): + + def reset_global_state(self): + """Reset any global state to its original settings. + + cloudinit caches some values in cloudinit.util. Unit tests that + involved those cached paths were then subject to failure if the order + of invocation changed (LP: #1703697). + + This function resets any of these global state variables to their + initial state. + + In the future this should really be done with some registry that + can then be cleaned in a more obvious way. + """ + util.PROC_CMDLINE = None + util._DNS_REDIRECT_IP = None + util._LSB_RELEASE = {} + + def setUp(self): + super(TestCase, self).setUp() + self.reset_global_state() + + +class CiTestCase(TestCase): + """This is the preferred test case base class unless user + needs other test case classes below.""" + + # Subclass overrides for specific test behavior + # Whether or not a unit test needs logfile setup + with_logs = False + + def setUp(self): + super(CiTestCase, self).setUp() + if self.with_logs: + # Create a log handler so unit tests can search expected logs. + self.logger = logging.getLogger() + self.logs = six.StringIO() + formatter = logging.Formatter('%(levelname)s: %(message)s') + handler = logging.StreamHandler(self.logs) + handler.setFormatter(formatter) + self.old_handlers = self.logger.handlers + self.logger.handlers = [handler] + + def tearDown(self): + if self.with_logs: + # Remove the handler we setup + logging.getLogger().handlers = self.old_handlers + super(CiTestCase, self).tearDown() + + def tmp_dir(self, dir=None, cleanup=True): + # return a full path to a temporary directory that will be cleaned up. + if dir is None: + tmpd = tempfile.mkdtemp( + prefix="ci-%s." % self.__class__.__name__) + else: + tmpd = tempfile.mkdtemp(dir=dir) + self.addCleanup(functools.partial(shutil.rmtree, tmpd)) + return tmpd + + def tmp_path(self, path, dir=None): + # return an absolute path to 'path' under dir. + # if dir is None, one will be created with tmp_dir() + # the file is not created or modified. + if dir is None: + dir = self.tmp_dir() + return os.path.normpath(os.path.abspath(os.path.join(dir, path))) + + +class ResourceUsingTestCase(CiTestCase): + + def setUp(self): + super(ResourceUsingTestCase, self).setUp() + self.resource_path = None + + def resourceLocation(self, subname=None): + if self.resource_path is None: + paths = [ + os.path.join('tests', 'data'), + os.path.join('data'), + os.path.join(os.pardir, 'tests', 'data'), + os.path.join(os.pardir, 'data'), + ] + for p in paths: + if os.path.isdir(p): + self.resource_path = p + break + self.assertTrue((self.resource_path and + os.path.isdir(self.resource_path)), + msg="Unable to locate test resource data path!") + if not subname: + return self.resource_path + return os.path.join(self.resource_path, subname) + + def readResource(self, name): + where = self.resourceLocation(name) + with open(where, 'r') as fh: + return fh.read() + + def getCloudPaths(self, ds=None): + tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmpdir) + cp = ch.Paths({'cloud_dir': tmpdir, + 'templates_dir': self.resourceLocation()}, + ds=ds) + return cp + + +class FilesystemMockingTestCase(ResourceUsingTestCase): + + def setUp(self): + super(FilesystemMockingTestCase, self).setUp() + self.patched_funcs = ExitStack() + + def tearDown(self): + self.patched_funcs.close() + ResourceUsingTestCase.tearDown(self) + + def replicateTestRoot(self, example_root, target_root): + real_root = self.resourceLocation() + real_root = os.path.join(real_root, 'roots', example_root) + for (dir_path, _dirnames, filenames) in os.walk(real_root): + real_path = dir_path + make_path = rebase_path(real_path[len(real_root):], target_root) + util.ensure_dir(make_path) + for f in filenames: + real_path = util.abs_join(real_path, f) + make_path = util.abs_join(make_path, f) + shutil.copy(real_path, make_path) + + def patchUtils(self, new_root): + patch_funcs = { + util: [('write_file', 1), + ('append_file', 1), + ('load_file', 1), + ('ensure_dir', 1), + ('chmod', 1), + ('delete_dir_contents', 1), + ('del_file', 1), + ('sym_link', -1), + ('copy', -1)], + } + for (mod, funcs) in patch_funcs.items(): + for (f, am) in funcs: + func = getattr(mod, f) + trap_func = retarget_many_wrapper(new_root, am, func) + self.patched_funcs.enter_context( + mock.patch.object(mod, f, trap_func)) + + # Handle subprocess calls + func = getattr(util, 'subp') + + def nsubp(*_args, **_kwargs): + return ('', '') + + self.patched_funcs.enter_context( + mock.patch.object(util, 'subp', nsubp)) + + def null_func(*_args, **_kwargs): + return None + + for f in ['chownbyid', 'chownbyname']: + self.patched_funcs.enter_context( + mock.patch.object(util, f, null_func)) + + def patchOS(self, new_root): + patch_funcs = { + os.path: [('isfile', 1), ('exists', 1), + ('islink', 1), ('isdir', 1)], + os: [('listdir', 1), ('mkdir', 1), + ('lstat', 1), ('symlink', 2)], + } + for (mod, funcs) in patch_funcs.items(): + for f, nargs in funcs: + func = getattr(mod, f) + trap_func = retarget_many_wrapper(new_root, nargs, func) + self.patched_funcs.enter_context( + mock.patch.object(mod, f, trap_func)) + + def patchOpen(self, new_root): + trap_func = retarget_many_wrapper(new_root, 1, open) + name = 'builtins.open' if PY3 else '__builtin__.open' + self.patched_funcs.enter_context(mock.patch(name, trap_func)) + + def patchStdoutAndStderr(self, stdout=None, stderr=None): + if stdout is not None: + self.patched_funcs.enter_context( + mock.patch.object(sys, 'stdout', stdout)) + if stderr is not None: + self.patched_funcs.enter_context( + mock.patch.object(sys, 'stderr', stderr)) + + def reRoot(self, root=None): + if root is None: + root = self.tmp_dir() + self.patchUtils(root) + self.patchOS(root) + return root + + +class HttprettyTestCase(CiTestCase): + # necessary as http_proxy gets in the way of httpretty + # https://github.com/gabrielfalcao/HTTPretty/issues/122 + + def setUp(self): + self.restore_proxy = os.environ.get('http_proxy') + if self.restore_proxy is not None: + del os.environ['http_proxy'] + super(HttprettyTestCase, self).setUp() + + def tearDown(self): + if self.restore_proxy: + os.environ['http_proxy'] = self.restore_proxy + super(HttprettyTestCase, self).tearDown() + + +def populate_dir(path, files): + if not os.path.exists(path): + os.makedirs(path) + ret = [] + for (name, content) in files.items(): + p = os.path.sep.join([path, name]) + util.ensure_dir(os.path.dirname(p)) + with open(p, "wb") as fp: + if isinstance(content, six.binary_type): + fp.write(content) + else: + fp.write(content.encode('utf-8')) + fp.close() + ret.append(p) + + return ret + + +def dir2dict(startdir, prefix=None): + flist = {} + if prefix is None: + prefix = startdir + for root, dirs, files in os.walk(startdir): + for fname in files: + fpath = os.path.join(root, fname) + key = fpath[len(prefix):] + flist[key] = util.load_file(fpath) + return flist + + +def json_dumps(data): + # print data in nicely formatted json. + return json.dumps(data, indent=1, sort_keys=True, + separators=(',', ': ')) + + +def wrap_and_call(prefix, mocks, func, *args, **kwargs): + """ + call func(args, **kwargs) with mocks applied, then unapplies mocks + nicer to read than repeating dectorators on each function + + prefix: prefix for mock names (e.g. 'cloudinit.stages.util') or None + mocks: dictionary of names (under 'prefix') to mock and either + a return value or a dictionary to pass to the mock.patch call + func: function to call with mocks applied + *args,**kwargs: arguments for 'func' + + return_value: return from 'func' + """ + delim = '.' + if prefix is None: + prefix = '' + prefix = prefix.rstrip(delim) + unwraps = [] + for fname, kw in mocks.items(): + if prefix: + fname = delim.join((prefix, fname)) + if not isinstance(kw, dict): + kw = {'return_value': kw} + p = mock.patch(fname, **kw) + p.start() + unwraps.append(p) + try: + return func(*args, **kwargs) + finally: + for p in unwraps: + p.stop() + + +try: + skipIf = unittest.skipIf +except AttributeError: + # Python 2.6. Doesn't have to be high fidelity. + def skipIf(condition, reason): + def decorator(func): + def wrapper(*args, **kws): + if condition: + return func(*args, **kws) + else: + print(reason, file=sys.stderr) + return wrapper + return decorator + + +# older versions of mock do not have the useful 'assert_not_called' +if not hasattr(mock.Mock, 'assert_not_called'): + def __mock_assert_not_called(mmock): + if mmock.call_count != 0: + msg = ("[citest] Expected '%s' to not have been called. " + "Called %s times." % + (mmock._mock_name or 'mock', mmock.call_count)) + raise AssertionError(msg) + mock.Mock.assert_not_called = __mock_assert_not_called + + +# vi: ts=4 expandtab diff --git a/cloudinit/tests/test_url_helper.py b/cloudinit/tests/test_url_helper.py index 349110d9..b778a3a7 100644 --- a/cloudinit/tests/test_url_helper.py +++ b/cloudinit/tests/test_url_helper.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit.url_helper import oauth_headers -from tests.unittests.helpers import CiTestCase, mock, skipIf +from cloudinit.tests.helpers import CiTestCase, mock, skipIf try: diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py deleted file mode 100644 index bf1dc5df..00000000 --- a/tests/unittests/helpers.py +++ /dev/null @@ -1,391 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -from __future__ import print_function - -import functools -import json -import logging -import os -import shutil -import sys -import tempfile -import unittest - -import mock -import six -import unittest2 - -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack - -from cloudinit import helpers as ch -from cloudinit import util - -# Used for skipping tests -SkipTest = unittest2.SkipTest - -# Used for detecting different python versions -PY2 = False -PY26 = False -PY27 = False -PY3 = False - -_PY_VER = sys.version_info -_PY_MAJOR, _PY_MINOR, _PY_MICRO = _PY_VER[0:3] -if (_PY_MAJOR, _PY_MINOR) <= (2, 6): - if (_PY_MAJOR, _PY_MINOR) == (2, 6): - PY26 = True - if (_PY_MAJOR, _PY_MINOR) >= (2, 0): - PY2 = True -else: - if (_PY_MAJOR, _PY_MINOR) == (2, 7): - PY27 = True - PY2 = True - if (_PY_MAJOR, _PY_MINOR) >= (3, 0): - PY3 = True - - -# Makes the old path start -# with new base instead of whatever -# it previously had -def rebase_path(old_path, new_base): - if old_path.startswith(new_base): - # Already handled... - return old_path - # Retarget the base of that path - # to the new base instead of the - # old one... - path = os.path.join(new_base, old_path.lstrip("/")) - path = os.path.abspath(path) - return path - - -# Can work on anything that takes a path as arguments -def retarget_many_wrapper(new_base, am, old_func): - def wrapper(*args, **kwds): - n_args = list(args) - nam = am - if am == -1: - nam = len(n_args) - for i in range(0, nam): - path = args[i] - # patchOS() wraps various os and os.path functions, however in - # Python 3 some of these now accept file-descriptors (integers). - # That breaks rebase_path() so in lieu of a better solution, just - # don't rebase if we get a fd. - if isinstance(path, six.string_types): - n_args[i] = rebase_path(path, new_base) - return old_func(*n_args, **kwds) - return wrapper - - -class TestCase(unittest2.TestCase): - def reset_global_state(self): - """Reset any global state to its original settings. - - cloudinit caches some values in cloudinit.util. Unit tests that - involved those cached paths were then subject to failure if the order - of invocation changed (LP: #1703697). - - This function resets any of these global state variables to their - initial state. - - In the future this should really be done with some registry that - can then be cleaned in a more obvious way. - """ - util.PROC_CMDLINE = None - util._DNS_REDIRECT_IP = None - util._LSB_RELEASE = {} - - def setUp(self): - super(unittest2.TestCase, self).setUp() - self.reset_global_state() - - -class CiTestCase(TestCase): - """This is the preferred test case base class unless user - needs other test case classes below.""" - - # Subclass overrides for specific test behavior - # Whether or not a unit test needs logfile setup - with_logs = False - - def setUp(self): - super(CiTestCase, self).setUp() - if self.with_logs: - # Create a log handler so unit tests can search expected logs. - self.logger = logging.getLogger() - self.logs = six.StringIO() - formatter = logging.Formatter('%(levelname)s: %(message)s') - handler = logging.StreamHandler(self.logs) - handler.setFormatter(formatter) - self.old_handlers = self.logger.handlers - self.logger.handlers = [handler] - - def tearDown(self): - if self.with_logs: - # Remove the handler we setup - logging.getLogger().handlers = self.old_handlers - super(CiTestCase, self).tearDown() - - def tmp_dir(self, dir=None, cleanup=True): - # return a full path to a temporary directory that will be cleaned up. - if dir is None: - tmpd = tempfile.mkdtemp( - prefix="ci-%s." % self.__class__.__name__) - else: - tmpd = tempfile.mkdtemp(dir=dir) - self.addCleanup(functools.partial(shutil.rmtree, tmpd)) - return tmpd - - def tmp_path(self, path, dir=None): - # return an absolute path to 'path' under dir. - # if dir is None, one will be created with tmp_dir() - # the file is not created or modified. - if dir is None: - dir = self.tmp_dir() - return os.path.normpath(os.path.abspath(os.path.join(dir, path))) - - -class ResourceUsingTestCase(CiTestCase): - def setUp(self): - super(ResourceUsingTestCase, self).setUp() - self.resource_path = None - - def resourceLocation(self, subname=None): - if self.resource_path is None: - paths = [ - os.path.join('tests', 'data'), - os.path.join('data'), - os.path.join(os.pardir, 'tests', 'data'), - os.path.join(os.pardir, 'data'), - ] - for p in paths: - if os.path.isdir(p): - self.resource_path = p - break - self.assertTrue((self.resource_path and - os.path.isdir(self.resource_path)), - msg="Unable to locate test resource data path!") - if not subname: - return self.resource_path - return os.path.join(self.resource_path, subname) - - def readResource(self, name): - where = self.resourceLocation(name) - with open(where, 'r') as fh: - return fh.read() - - def getCloudPaths(self, ds=None): - tmpdir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, tmpdir) - cp = ch.Paths({'cloud_dir': tmpdir, - 'templates_dir': self.resourceLocation()}, - ds=ds) - return cp - - -class FilesystemMockingTestCase(ResourceUsingTestCase): - def setUp(self): - super(FilesystemMockingTestCase, self).setUp() - self.patched_funcs = ExitStack() - - def tearDown(self): - self.patched_funcs.close() - ResourceUsingTestCase.tearDown(self) - - def replicateTestRoot(self, example_root, target_root): - real_root = self.resourceLocation() - real_root = os.path.join(real_root, 'roots', example_root) - for (dir_path, _dirnames, filenames) in os.walk(real_root): - real_path = dir_path - make_path = rebase_path(real_path[len(real_root):], target_root) - util.ensure_dir(make_path) - for f in filenames: - real_path = util.abs_join(real_path, f) - make_path = util.abs_join(make_path, f) - shutil.copy(real_path, make_path) - - def patchUtils(self, new_root): - patch_funcs = { - util: [('write_file', 1), - ('append_file', 1), - ('load_file', 1), - ('ensure_dir', 1), - ('chmod', 1), - ('delete_dir_contents', 1), - ('del_file', 1), - ('sym_link', -1), - ('copy', -1)], - } - for (mod, funcs) in patch_funcs.items(): - for (f, am) in funcs: - func = getattr(mod, f) - trap_func = retarget_many_wrapper(new_root, am, func) - self.patched_funcs.enter_context( - mock.patch.object(mod, f, trap_func)) - - # Handle subprocess calls - func = getattr(util, 'subp') - - def nsubp(*_args, **_kwargs): - return ('', '') - - self.patched_funcs.enter_context( - mock.patch.object(util, 'subp', nsubp)) - - def null_func(*_args, **_kwargs): - return None - - for f in ['chownbyid', 'chownbyname']: - self.patched_funcs.enter_context( - mock.patch.object(util, f, null_func)) - - def patchOS(self, new_root): - patch_funcs = { - os.path: [('isfile', 1), ('exists', 1), - ('islink', 1), ('isdir', 1)], - os: [('listdir', 1), ('mkdir', 1), - ('lstat', 1), ('symlink', 2)], - } - for (mod, funcs) in patch_funcs.items(): - for f, nargs in funcs: - func = getattr(mod, f) - trap_func = retarget_many_wrapper(new_root, nargs, func) - self.patched_funcs.enter_context( - mock.patch.object(mod, f, trap_func)) - - def patchOpen(self, new_root): - trap_func = retarget_many_wrapper(new_root, 1, open) - name = 'builtins.open' if PY3 else '__builtin__.open' - self.patched_funcs.enter_context(mock.patch(name, trap_func)) - - def patchStdoutAndStderr(self, stdout=None, stderr=None): - if stdout is not None: - self.patched_funcs.enter_context( - mock.patch.object(sys, 'stdout', stdout)) - if stderr is not None: - self.patched_funcs.enter_context( - mock.patch.object(sys, 'stderr', stderr)) - - def reRoot(self, root=None): - if root is None: - root = self.tmp_dir() - self.patchUtils(root) - self.patchOS(root) - return root - - -class HttprettyTestCase(CiTestCase): - # necessary as http_proxy gets in the way of httpretty - # https://github.com/gabrielfalcao/HTTPretty/issues/122 - def setUp(self): - self.restore_proxy = os.environ.get('http_proxy') - if self.restore_proxy is not None: - del os.environ['http_proxy'] - super(HttprettyTestCase, self).setUp() - - def tearDown(self): - if self.restore_proxy: - os.environ['http_proxy'] = self.restore_proxy - super(HttprettyTestCase, self).tearDown() - - -def populate_dir(path, files): - if not os.path.exists(path): - os.makedirs(path) - ret = [] - for (name, content) in files.items(): - p = os.path.sep.join([path, name]) - util.ensure_dir(os.path.dirname(p)) - with open(p, "wb") as fp: - if isinstance(content, six.binary_type): - fp.write(content) - else: - fp.write(content.encode('utf-8')) - fp.close() - ret.append(p) - - return ret - - -def dir2dict(startdir, prefix=None): - flist = {} - if prefix is None: - prefix = startdir - for root, dirs, files in os.walk(startdir): - for fname in files: - fpath = os.path.join(root, fname) - key = fpath[len(prefix):] - flist[key] = util.load_file(fpath) - return flist - - -def json_dumps(data): - # print data in nicely formatted json. - return json.dumps(data, indent=1, sort_keys=True, - separators=(',', ': ')) - - -def wrap_and_call(prefix, mocks, func, *args, **kwargs): - """ - call func(args, **kwargs) with mocks applied, then unapplies mocks - nicer to read than repeating dectorators on each function - - prefix: prefix for mock names (e.g. 'cloudinit.stages.util') or None - mocks: dictionary of names (under 'prefix') to mock and either - a return value or a dictionary to pass to the mock.patch call - func: function to call with mocks applied - *args,**kwargs: arguments for 'func' - - return_value: return from 'func' - """ - delim = '.' - if prefix is None: - prefix = '' - prefix = prefix.rstrip(delim) - unwraps = [] - for fname, kw in mocks.items(): - if prefix: - fname = delim.join((prefix, fname)) - if not isinstance(kw, dict): - kw = {'return_value': kw} - p = mock.patch(fname, **kw) - p.start() - unwraps.append(p) - try: - return func(*args, **kwargs) - finally: - for p in unwraps: - p.stop() - - -try: - skipIf = unittest.skipIf -except AttributeError: - # Python 2.6. Doesn't have to be high fidelity. - def skipIf(condition, reason): - def decorator(func): - def wrapper(*args, **kws): - if condition: - return func(*args, **kws) - else: - print(reason, file=sys.stderr) - return wrapper - return decorator - - -# older versions of mock do not have the useful 'assert_not_called' -if not hasattr(mock.Mock, 'assert_not_called'): - def __mock_assert_not_called(mmock): - if mmock.call_count != 0: - msg = ("[citest] Expected '%s' to not have been called. " - "Called %s times." % - (mmock._mock_name or 'mock', mmock.call_count)) - raise AssertionError(msg) - mock.Mock.assert_not_called = __mock_assert_not_called - - -# vi: ts=4 expandtab diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 781f6d54..25878d7a 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -12,7 +12,7 @@ from cloudinit import settings from cloudinit import url_helper from cloudinit import util -from .helpers import TestCase, CiTestCase, ExitStack, mock +from cloudinit.tests.helpers import TestCase, CiTestCase, ExitStack, mock class FakeModule(handlers.Handler): diff --git a/tests/unittests/test_atomic_helper.py b/tests/unittests/test_atomic_helper.py index 515919d8..0101b0e3 100644 --- a/tests/unittests/test_atomic_helper.py +++ b/tests/unittests/test_atomic_helper.py @@ -6,7 +6,7 @@ import stat from cloudinit import atomic_helper -from .helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase class TestAtomicHelper(CiTestCase): diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index dd9d0357..9751ed95 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -11,7 +11,7 @@ try: except ImportError: import mock -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit import handlers from cloudinit import helpers diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 12f01852..495bdc9f 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -2,7 +2,7 @@ import six -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit.cmd import main as cli diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/test_cs_util.py index b8f5031c..ee88520d 100644 --- a/tests/unittests/test_cs_util.py +++ b/tests/unittests/test_cs_util.py @@ -2,7 +2,7 @@ from __future__ import print_function -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit.cs_utils import Cepko diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 4ad86bb6..6d621d26 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -27,7 +27,7 @@ from cloudinit import stages from cloudinit import user_data as ud from cloudinit import util -from . import helpers +from cloudinit.tests import helpers INSTANCE_ID = "i-testing" diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py index 996560e4..82ee9714 100644 --- a/tests/unittests/test_datasource/test_aliyun.py +++ b/tests/unittests/test_datasource/test_aliyun.py @@ -5,9 +5,9 @@ import httpretty import mock import os -from .. import helpers as test_helpers from cloudinit import helpers from cloudinit.sources import DataSourceAliYun as ay +from cloudinit.tests import helpers as test_helpers DEFAULT_METADATA = { 'instance-id': 'aliyun-test-vm-00', diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py index 9c46abc1..3b274d90 100644 --- a/tests/unittests/test_datasource/test_altcloud.py +++ b/tests/unittests/test_datasource/test_altcloud.py @@ -18,7 +18,7 @@ import tempfile from cloudinit import helpers from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import cloudinit.sources.DataSourceAltCloud as dsac diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 20e70fb7..0a117771 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -6,8 +6,8 @@ from cloudinit.sources import DataSourceAzure as dsaz from cloudinit.util import find_freebsd_part from cloudinit.util import get_path_dev_freebsd -from ..helpers import (CiTestCase, TestCase, populate_dir, mock, - ExitStack, PY26, SkipTest) +from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock, + ExitStack, PY26, SkipTest) import crypt import os @@ -871,6 +871,7 @@ class TestLoadAzureDsDir(CiTestCase): class TestReadAzureOvf(TestCase): + def test_invalid_xml_raises_non_azure_ds(self): invalid_xml = "" + construct_valid_ovf_env(data={}) self.assertRaises(dsaz.BrokenAzureDataSource, @@ -1079,6 +1080,7 @@ class TestCanDevBeReformatted(CiTestCase): class TestAzureNetExists(CiTestCase): + def test_azure_net_must_exist_for_legacy_objpkl(self): """DataSourceAzureNet must exist for old obj.pkl files that reference it.""" diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index b2d2971b..80ce003d 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -3,7 +3,7 @@ import os from cloudinit.sources.helpers import azure as azure_helper -from ..helpers import ExitStack, mock, TestCase +from cloudinit.tests.helpers import ExitStack, mock, TestCase GOAL_STATE_TEMPLATE = """\ diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index 5997102c..e4c59907 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -6,7 +6,7 @@ from cloudinit.cs_utils import Cepko from cloudinit import sources from cloudinit.sources import DataSourceCloudSigma -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers SERVER_CONTEXT = { "cpu": 1000, diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py index e94aad61..2dc90305 100644 --- a/tests/unittests/test_datasource/test_cloudstack.py +++ b/tests/unittests/test_datasource/test_cloudstack.py @@ -3,7 +3,7 @@ from cloudinit import helpers from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack -from ..helpers import TestCase, mock, ExitStack +from cloudinit.tests.helpers import TestCase, mock, ExitStack class TestCloudStackPasswordFetching(TestCase): diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index 4802f105..80b9c650 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -24,7 +24,7 @@ from cloudinit.sources import ( ) from cloudinit.sources import DataSourceNone as DSNone -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers DEFAULT_LOCAL = [ Azure.DataSourceAzure, diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 337be667..237c189b 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -15,7 +15,7 @@ from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit.sources.helpers import openstack from cloudinit import util -from ..helpers import TestCase, ExitStack, mock +from cloudinit.tests.helpers import TestCase, ExitStack, mock PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index e97a679a..f264f361 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -13,7 +13,7 @@ from cloudinit import settings from cloudinit.sources import DataSourceDigitalOcean from cloudinit.sources.helpers import digitalocean -from ..helpers import mock, TestCase +from cloudinit.tests.helpers import mock, TestCase DO_MULTIPLE_KEYS = ["ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@do.co", "ssh-rsa AAAAB3NzaC1yc2EAAAA... test2@do.co"] diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index b7a84e21..9fb90483 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -4,9 +4,9 @@ import copy import httpretty import mock -from .. import helpers as test_helpers from cloudinit import helpers from cloudinit.sources import DataSourceEc2 as ec2 +from cloudinit.tests import helpers as test_helpers # collected from api version 2016-09-02/ with diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index ad608bec..50e49a10 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -15,7 +15,7 @@ from cloudinit import helpers from cloudinit import settings from cloudinit.sources import DataSourceGCE -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers GCE_META = { diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index c1911bf4..289c6a40 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -8,7 +8,7 @@ import yaml from cloudinit.sources import DataSourceMAAS from cloudinit import url_helper -from ..helpers import TestCase, populate_dir +from cloudinit.tests.helpers import TestCase, populate_dir try: from unittest import mock diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index ff294395..fea9156b 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -3,7 +3,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNoCloud from cloudinit import util -from ..helpers import TestCase, populate_dir, mock, ExitStack +from cloudinit.tests.helpers import TestCase, populate_dir, mock, ExitStack import os import shutil diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index b0f8e435..e7d55692 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -3,7 +3,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceOpenNebula as ds from cloudinit import util -from ..helpers import mock, populate_dir, TestCase +from cloudinit.tests.helpers import mock, populate_dir, TestCase import os import pwd diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index c2905d1a..177e9808 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -9,7 +9,7 @@ import httpretty as hp import json import re -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from six.moves.urllib.parse import urlparse from six import StringIO diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index 477cf8ed..9dbf4dd9 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -6,7 +6,7 @@ import base64 -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit.sources import DataSourceOVF as dsovf diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py index 65d83ad7..436df9ee 100644 --- a/tests/unittests/test_datasource/test_scaleway.py +++ b/tests/unittests/test_datasource/test_scaleway.py @@ -9,7 +9,7 @@ from cloudinit import helpers from cloudinit import settings from cloudinit.sources import DataSourceScaleway -from ..helpers import mock, HttprettyTestCase, TestCase +from cloudinit.tests.helpers import mock, HttprettyTestCase, TestCase class DataResponses(object): diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index e3c99bbb..933d5b63 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -33,7 +33,7 @@ import six from cloudinit import helpers as c_helpers from cloudinit.util import b64e -from ..helpers import mock, FilesystemMockingTestCase, TestCase +from cloudinit.tests.helpers import mock, FilesystemMockingTestCase, TestCase SDC_NICS = json.loads(""" [ diff --git a/tests/unittests/test_distros/test_arch.py b/tests/unittests/test_distros/test_arch.py index 3d4c9a70..a95ba3b5 100644 --- a/tests/unittests/test_distros/test_arch.py +++ b/tests/unittests/test_distros/test_arch.py @@ -3,7 +3,7 @@ from cloudinit.distros.arch import _render_network from cloudinit import util -from ..helpers import (CiTestCase, dir2dict) +from cloudinit.tests.helpers import (CiTestCase, dir2dict) from . import _get_distro diff --git a/tests/unittests/test_distros/test_create_users.py b/tests/unittests/test_distros/test_create_users.py index 1d02f7bd..aa13670a 100644 --- a/tests/unittests/test_distros/test_create_users.py +++ b/tests/unittests/test_distros/test_create_users.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import distros -from ..helpers import (TestCase, mock) +from cloudinit.tests.helpers import (TestCase, mock) class MyBaseDistro(distros.Distro): diff --git a/tests/unittests/test_distros/test_debian.py b/tests/unittests/test_distros/test_debian.py index 72d3aad6..da16a797 100644 --- a/tests/unittests/test_distros/test_debian.py +++ b/tests/unittests/test_distros/test_debian.py @@ -2,7 +2,7 @@ from cloudinit import distros from cloudinit import util -from ..helpers import (FilesystemMockingTestCase, mock) +from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock) @mock.patch("cloudinit.distros.debian.util.subp") diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index b355a19e..791fe612 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -3,7 +3,7 @@ from cloudinit import distros from cloudinit import util -from .. import helpers +from cloudinit.tests import helpers import os import shutil diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 6d89dba8..c4bd11bc 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -12,7 +12,7 @@ try: except ImportError: from contextlib2 import ExitStack -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase from cloudinit import distros from cloudinit.distros.parsers.sys_conf import SysConf diff --git a/tests/unittests/test_distros/test_opensuse.py b/tests/unittests/test_distros/test_opensuse.py index bdb1d633..b9bb9b3e 100644 --- a/tests/unittests/test_distros/test_opensuse.py +++ b/tests/unittests/test_distros/test_opensuse.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from ..helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase from . import _get_distro diff --git a/tests/unittests/test_distros/test_resolv.py b/tests/unittests/test_distros/test_resolv.py index 97168cf9..68ea0083 100644 --- a/tests/unittests/test_distros/test_resolv.py +++ b/tests/unittests/test_distros/test_resolv.py @@ -3,7 +3,7 @@ from cloudinit.distros.parsers import resolv_conf from cloudinit.distros import rhel_util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import re import tempfile diff --git a/tests/unittests/test_distros/test_sles.py b/tests/unittests/test_distros/test_sles.py index c656aacc..33e3c457 100644 --- a/tests/unittests/test_distros/test_sles.py +++ b/tests/unittests/test_distros/test_sles.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from ..helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase from . import _get_distro diff --git a/tests/unittests/test_distros/test_sysconfig.py b/tests/unittests/test_distros/test_sysconfig.py index 235ecebb..c1d5b693 100644 --- a/tests/unittests/test_distros/test_sysconfig.py +++ b/tests/unittests/test_distros/test_sysconfig.py @@ -4,7 +4,7 @@ import re from cloudinit.distros.parsers.sys_conf import SysConf -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase # Lots of good examples @ diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index 88746e0a..0fa9cdb5 100644 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -5,7 +5,7 @@ from cloudinit.distros import ug_util from cloudinit import helpers from cloudinit import settings -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import mock diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 8ccfe55c..1a81a89e 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -6,7 +6,8 @@ from uuid import uuid4 from cloudinit import safeyaml from cloudinit import util -from .helpers import CiTestCase, dir2dict, json_dumps, populate_dir +from cloudinit.tests.helpers import ( + CiTestCase, dir2dict, json_dumps, populate_dir) UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu " "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux") diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index 65fdb519..af78997f 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -2,7 +2,7 @@ import httpretty as hp -from . import helpers +from cloudinit.tests import helpers from cloudinit import ec2_utils as eu from cloudinit import url_helper as uh diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py index 13137f6d..6364d38e 100644 --- a/tests/unittests/test_filters/test_launch_index.py +++ b/tests/unittests/test_filters/test_launch_index.py @@ -2,7 +2,7 @@ import copy -from .. import helpers +from cloudinit.tests import helpers from six.moves import filterfalse diff --git a/tests/unittests/test_handler/test_handler_apt_conf_v1.py b/tests/unittests/test_handler/test_handler_apt_conf_v1.py index 554277ff..83f962a9 100644 --- a/tests/unittests/test_handler/test_handler_apt_conf_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_conf_v1.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_apt_configure from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import copy import os diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py index f53ddbb2..d2b96f0b 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py @@ -24,7 +24,7 @@ from cloudinit.sources import DataSourceNone from cloudinit.distros.debian import Distro -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help LOG = logging.getLogger(__name__) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py index 1ca915b4..f7608c28 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py @@ -24,7 +24,7 @@ from cloudinit.sources import DataSourceNone from cloudinit.distros.debian import Distro -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help LOG = logging.getLogger(__name__) diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py index 12502d05..3a3f95ca 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py @@ -20,7 +20,7 @@ from cloudinit.config import cc_apt_configure from cloudinit import gpg from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK----- Version: GnuPG v1 diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py index 292d3f59..7bb1b7c4 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py @@ -28,7 +28,7 @@ from cloudinit import util from cloudinit.config import cc_apt_configure from cloudinit.sources import DataSourceNone -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help EXPECTEDKEY = u"""-----BEGIN PGP PUBLIC KEY BLOCK----- Version: GnuPG v1 diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index 7cee2c3f..06e14db0 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -5,7 +5,7 @@ from cloudinit.config import cc_ca_certs from cloudinit import helpers from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import logging import shutil diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 6a152ea2..e5785cfd 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -14,7 +14,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNone from cloudinit import util -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help LOG = logging.getLogger(__name__) diff --git a/tests/unittests/test_handler/test_handler_debug.py b/tests/unittests/test_handler/test_handler_debug.py index 1873c3e1..787ba350 100644 --- a/tests/unittests/test_handler/test_handler_debug.py +++ b/tests/unittests/test_handler/test_handler_debug.py @@ -11,7 +11,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNone -from ..helpers import (FilesystemMockingTestCase, mock) +from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock) import logging import shutil diff --git a/tests/unittests/test_handler/test_handler_disk_setup.py b/tests/unittests/test_handler/test_handler_disk_setup.py index 8a6d49ed..5afcacaf 100644 --- a/tests/unittests/test_handler/test_handler_disk_setup.py +++ b/tests/unittests/test_handler/test_handler_disk_setup.py @@ -3,7 +3,7 @@ import random from cloudinit.config import cc_disk_setup -from ..helpers import CiTestCase, ExitStack, mock, TestCase +from cloudinit.tests.helpers import CiTestCase, ExitStack, mock, TestCase class TestIsDiskUsed(TestCase): diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index c5fc8c9b..a3e46351 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -4,7 +4,7 @@ from cloudinit import cloud from cloudinit.config import cc_growpart from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import errno import logging diff --git a/tests/unittests/test_handler/test_handler_landscape.py b/tests/unittests/test_handler/test_handler_landscape.py index 7c247fa9..db92a7e2 100644 --- a/tests/unittests/test_handler/test_handler_landscape.py +++ b/tests/unittests/test_handler/test_handler_landscape.py @@ -1,9 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit.config import cc_landscape -from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from ..helpers import FilesystemMockingTestCase, mock, wrap_and_call +from cloudinit.sources import DataSourceNone +from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock, + wrap_and_call) from configobj import ConfigObj import logging diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index a789db32..e29a06f9 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -13,7 +13,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNoCloud -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help from configobj import ConfigObj diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 351226bf..f132a778 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_lxd from cloudinit.sources import DataSourceNoCloud from cloudinit import (distros, helpers, cloud) -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help import logging diff --git a/tests/unittests/test_handler/test_handler_mcollective.py b/tests/unittests/test_handler/test_handler_mcollective.py index 2a9f3823..7eec7352 100644 --- a/tests/unittests/test_handler/test_handler_mcollective.py +++ b/tests/unittests/test_handler/test_handler_mcollective.py @@ -4,7 +4,7 @@ from cloudinit import (cloud, distros, helpers, util) from cloudinit.config import cc_mcollective from cloudinit.sources import DataSourceNoCloud -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help import configobj import logging diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index 650ca0ec..fe492d4b 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -6,7 +6,7 @@ import tempfile from cloudinit.config import cc_mounts -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers try: from unittest import mock diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 83d5faa2..4f291248 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_ntp from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from ..helpers import FilesystemMockingTestCase, mock, skipIf +from cloudinit.tests.helpers import FilesystemMockingTestCase, mock, skipIf import os diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index e382210d..85a0fe0a 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -4,8 +4,8 @@ import sys from cloudinit.config import cc_power_state_change as psc -from .. import helpers as t_help -from ..helpers import mock +from cloudinit.tests import helpers as t_help +from cloudinit.tests.helpers import mock class TestLoadPowerState(t_help.TestCase): diff --git a/tests/unittests/test_handler/test_handler_puppet.py b/tests/unittests/test_handler/test_handler_puppet.py index 805c76ba..0b6e3b58 100644 --- a/tests/unittests/test_handler/test_handler_puppet.py +++ b/tests/unittests/test_handler/test_handler_puppet.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_puppet from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from ..helpers import CiTestCase, mock +from cloudinit.tests.helpers import CiTestCase, mock import logging diff --git a/tests/unittests/test_handler/test_handler_rsyslog.py b/tests/unittests/test_handler/test_handler_rsyslog.py index cca06678..8c8e2838 100644 --- a/tests/unittests/test_handler/test_handler_rsyslog.py +++ b/tests/unittests/test_handler/test_handler_rsyslog.py @@ -9,7 +9,7 @@ from cloudinit.config.cc_rsyslog import ( parse_remotes_line, remotes_to_rsyslog_cfg) from cloudinit import util -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help class TestLoadConfig(t_help.TestCase): diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py index 7880ee72..374c1d31 100644 --- a/tests/unittests/test_handler/test_handler_runcmd.py +++ b/tests/unittests/test_handler/test_handler_runcmd.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_runcmd from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from ..helpers import FilesystemMockingTestCase, skipIf +from cloudinit.tests.helpers import FilesystemMockingTestCase, skipIf import logging import os diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index e5e607fb..f60dedc2 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -22,7 +22,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNone -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help import logging diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py index 8165bf9a..abdc17e7 100644 --- a/tests/unittests/test_handler/test_handler_set_hostname.py +++ b/tests/unittests/test_handler/test_handler_set_hostname.py @@ -7,7 +7,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import util -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help from configobj import ConfigObj import logging diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index e4d07622..76b79c29 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -7,9 +7,9 @@ from cloudinit.config.cc_snap_config import ( from cloudinit import (distros, helpers, cloud, util) from cloudinit.config.cc_snap_config import handle as snap_handle from cloudinit.sources import DataSourceNone -from ..helpers import FilesystemMockingTestCase, mock +from cloudinit.tests.helpers import FilesystemMockingTestCase, mock -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help import logging import os diff --git a/tests/unittests/test_handler/test_handler_spacewalk.py b/tests/unittests/test_handler/test_handler_spacewalk.py index 28b5892a..ddbf4a79 100644 --- a/tests/unittests/test_handler/test_handler_spacewalk.py +++ b/tests/unittests/test_handler/test_handler_spacewalk.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_spacewalk from cloudinit import util -from .. import helpers +from cloudinit.tests import helpers import logging diff --git a/tests/unittests/test_handler/test_handler_timezone.py b/tests/unittests/test_handler/test_handler_timezone.py index c30fbdfe..27eedded 100644 --- a/tests/unittests/test_handler/test_handler_timezone.py +++ b/tests/unittests/test_handler/test_handler_timezone.py @@ -13,7 +13,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNoCloud -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help from configobj import ConfigObj import logging diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py index 1129e77d..7fa8fd21 100644 --- a/tests/unittests/test_handler/test_handler_write_files.py +++ b/tests/unittests/test_handler/test_handler_write_files.py @@ -4,7 +4,7 @@ from cloudinit.config.cc_write_files import write_files, decode_perms from cloudinit import log as logging from cloudinit import util -from ..helpers import CiTestCase, FilesystemMockingTestCase +from cloudinit.tests.helpers import CiTestCase, FilesystemMockingTestCase import base64 import gzip diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py index c4396df5..b7adbe50 100644 --- a/tests/unittests/test_handler/test_handler_yum_add_repo.py +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_yum_add_repo from cloudinit import util -from .. import helpers +from cloudinit.tests import helpers try: from configparser import ConfigParser diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 640f11d4..6137e3cf 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -6,7 +6,7 @@ from cloudinit.config.schema import ( validate_cloudconfig_schema, main) from cloudinit.util import write_file -from ..helpers import CiTestCase, mock, skipIf +from cloudinit.tests.helpers import CiTestCase, mock, skipIf from copy import copy from six import StringIO diff --git a/tests/unittests/test_helpers.py b/tests/unittests/test_helpers.py index f1979e89..2e4582a0 100644 --- a/tests/unittests/test_helpers.py +++ b/tests/unittests/test_helpers.py @@ -4,7 +4,7 @@ import os -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit import sources diff --git a/tests/unittests/test_log.py b/tests/unittests/test_log.py index 68fb4b8d..cd6296d6 100644 --- a/tests/unittests/test_log.py +++ b/tests/unittests/test_log.py @@ -2,9 +2,9 @@ """Tests for cloudinit.log """ -from .helpers import CiTestCase from cloudinit.analyze.dump import CLOUD_INIT_ASCTIME_FMT from cloudinit import log as ci_logging +from cloudinit.tests.helpers import CiTestCase import datetime import logging import six diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 0658b6b4..f51358da 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from . import helpers +from cloudinit.tests import helpers from cloudinit.handlers import cloud_config from cloudinit.handlers import (CONTENT_START, CONTENT_END) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index f251024b..c10ef905 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -11,10 +11,10 @@ from cloudinit.net import sysconfig from cloudinit.sources.helpers import openstack from cloudinit import util -from .helpers import CiTestCase -from .helpers import dir2dict -from .helpers import mock -from .helpers import populate_dir +from cloudinit.tests.helpers import CiTestCase +from cloudinit.tests.helpers import dir2dict +from cloudinit.tests.helpers import mock +from cloudinit.tests.helpers import populate_dir import base64 import copy diff --git a/tests/unittests/test_pathprefix2dict.py b/tests/unittests/test_pathprefix2dict.py index a4ae284f..abbb29b8 100644 --- a/tests/unittests/test_pathprefix2dict.py +++ b/tests/unittests/test_pathprefix2dict.py @@ -2,7 +2,7 @@ from cloudinit import util -from .helpers import TestCase, populate_dir +from cloudinit.tests.helpers import TestCase, populate_dir import shutil import tempfile diff --git a/tests/unittests/test_registry.py b/tests/unittests/test_registry.py index acf0bf4f..2b625026 100644 --- a/tests/unittests/test_registry.py +++ b/tests/unittests/test_registry.py @@ -2,7 +2,7 @@ from cloudinit.registry import DictRegistry -from .helpers import (mock, TestCase) +from cloudinit.tests.helpers import (mock, TestCase) class TestDictRegistry(TestCase): diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index f3b8f992..571420ed 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -8,7 +8,7 @@ from cloudinit.reporting import handlers import mock -from .helpers import TestCase +from cloudinit.tests.helpers import TestCase def _fake_registry(): diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py index ca14cd46..e9d5702a 100644 --- a/tests/unittests/test_rh_subscription.py +++ b/tests/unittests/test_rh_subscription.py @@ -7,7 +7,7 @@ import logging from cloudinit.config import cc_rh_subscription from cloudinit import util -from .helpers import TestCase, mock +from cloudinit.tests.helpers import TestCase, mock class GoodTests(TestCase): diff --git a/tests/unittests/test_runs/test_merge_run.py b/tests/unittests/test_runs/test_merge_run.py index 65895273..add93653 100644 --- a/tests/unittests/test_runs/test_merge_run.py +++ b/tests/unittests/test_runs/test_merge_run.py @@ -4,7 +4,7 @@ import os import shutil import tempfile -from .. import helpers +from cloudinit.tests import helpers from cloudinit.settings import PER_INSTANCE from cloudinit import stages diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py index 55f15b55..5cf666fe 100644 --- a/tests/unittests/test_runs/test_simple_run.py +++ b/tests/unittests/test_runs/test_simple_run.py @@ -4,7 +4,7 @@ import os import shutil import tempfile -from .. import helpers +from cloudinit.tests import helpers from cloudinit.settings import PER_INSTANCE from cloudinit import stages diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index 991f45a6..2a8e6abe 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -2,8 +2,8 @@ from mock import patch -from . import helpers as test_helpers from cloudinit import ssh_util +from cloudinit.tests import helpers as test_helpers VALID_CONTENT = { @@ -57,6 +57,7 @@ TEST_OPTIONS = ( class TestAuthKeyLineParser(test_helpers.TestCase): + def test_simple_parse(self): # test key line with common 3 fields (keytype, base64, comment) parser = ssh_util.AuthKeyLineParser() diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index 4e627826..b911d929 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -6,7 +6,7 @@ from __future__ import print_function -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers import textwrap from cloudinit import templater diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 5f11c88f..3e4154ca 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -12,7 +12,7 @@ import six import yaml from cloudinit import importer, util -from . import helpers +from cloudinit.tests import helpers try: from unittest import mock diff --git a/tests/unittests/test_version.py b/tests/unittests/test_version.py index 1662ce09..d012f69d 100644 --- a/tests/unittests/test_version.py +++ b/tests/unittests/test_version.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from .helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase from cloudinit import version diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py index 03b36d31..d8651077 100644 --- a/tests/unittests/test_vmware_config_file.py +++ b/tests/unittests/test_vmware_config_file.py @@ -8,10 +8,10 @@ import logging import sys -from .helpers import CiTestCase from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProtoEnum from cloudinit.sources.helpers.vmware.imc.config import Config from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile +from cloudinit.tests.helpers import CiTestCase logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) logger = logging.getLogger(__name__) -- cgit v1.2.3 From 922c3c5c1a86f2d58e95a328e72b49a3bb234ca8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 7 Sep 2017 09:59:47 -0400 Subject: Ec2: only attempt to operate at local mode on known platforms. This change makes the DataSourceEc2Local do nothing unless it is on actual AWS platform. The motivation is twofold: a.) It is generally safer to only make this function available to Ec2 clones that explicitly identify themselves to the guest. (It also gives them a reason to supply identification code to cloud-init.) b.) On non-intel OpenStack platforms ds-identify would enable both the Ec2 and OpenStack sources. That is because there is not good data (such as dmi) to positively identify the platform. Previously that would be fine as OpenStack would run first and be successful. The change to add Ec2Local meant that an Ec2 now runs first. The best case for 'b' would be a slow down as attempts at the Ec2 metadata service time out. The discovered case was worse. Additionally we add a simple check for datatype of 'network' in the metadata before attempting to read it. LP: #1715128 --- cloudinit/sources/DataSourceEc2.py | 43 +++++++++++++++++++++++------ tests/unittests/test_datasource/test_ec2.py | 29 ++++++++++++++++--- 2 files changed, 60 insertions(+), 12 deletions(-) (limited to 'tests/unittests/test_datasource/test_ec2.py') diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 07c12bb4..41367a8b 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -27,6 +27,8 @@ SKIP_METADATA_URL_CODES = frozenset([uhelp.NOT_FOUND]) STRICT_ID_PATH = ("datasource", "Ec2", "strict_id") STRICT_ID_DEFAULT = "warn" +_unset = "_unset" + class Platforms(object): ALIYUN = "AliYun" @@ -57,7 +59,7 @@ class DataSourceEc2(sources.DataSource): _cloud_platform = None - _network_config = None # Used for caching calculated network config v1 + _network_config = _unset # Used for caching calculated network config v1 # Whether we want to get network configuration from the metadata service. get_network_metadata = False @@ -284,10 +286,24 @@ class DataSourceEc2(sources.DataSource): @property def network_config(self): """Return a network config dict for rendering ENI or netplan files.""" - if self._network_config is None: - if self.metadata is not None: - self._network_config = convert_ec2_metadata_network_config( - self.metadata) + if self._network_config != _unset: + return self._network_config + + if self.metadata is None: + # this would happen if get_data hadn't been called. leave as _unset + LOG.warning( + "Unexpected call to network_config when metadata is None.") + return None + + result = None + net_md = self.metadata.get('network') + if isinstance(net_md, dict): + result = convert_ec2_metadata_network_config(net_md) + else: + LOG.warning("unexpected metadata 'network' key not valid: %s", + net_md) + self._network_config = result + return self._network_config def _crawl_metadata(self): @@ -321,6 +337,14 @@ class DataSourceEc2Local(DataSourceEc2): """ get_network_metadata = True # Get metadata network config if present + def get_data(self): + supported_platforms = (Platforms.AWS,) + if self.cloud_platform not in supported_platforms: + LOG.debug("Local Ec2 mode only supported on %s, not %s", + supported_platforms, self.cloud_platform) + return False + return super(DataSourceEc2Local, self).get_data() + def read_strict_mode(cfgval, default): try: @@ -434,10 +458,13 @@ def _collect_platform_data(): return data -def convert_ec2_metadata_network_config(metadata=None, macs_to_nics=None): +def convert_ec2_metadata_network_config(network_md, macs_to_nics=None): """Convert ec2 metadata to network config version 1 data dict. - @param: metadata: Dictionary of metadata crawled from EC2 metadata url. + @param: network_md: 'network' portion of EC2 metadata. + generally formed as {"interfaces": {"macs": {}} where + 'macs' is a dictionary with mac address as key and contents like: + {"device-number": "0", "interface-id": "...", "local-ipv4s": ...} @param: macs_to_name: Optional dict mac addresses and the nic name. If not provided, get_interfaces_by_mac is called to get it from the OS. @@ -446,7 +473,7 @@ def convert_ec2_metadata_network_config(metadata=None, macs_to_nics=None): netcfg = {'version': 1, 'config': []} if not macs_to_nics: macs_to_nics = net.get_interfaces_by_mac() - macs_metadata = metadata['network']['interfaces']['macs'] + macs_metadata = network_md['interfaces']['macs'] for mac, nic_name in macs_to_nics.items(): nic_metadata = macs_metadata.get(mac) if not nic_metadata: diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index 9fb90483..a7301dbf 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -279,6 +279,27 @@ class TestEc2(test_helpers.HttprettyTestCase): ret = ds.get_data() self.assertTrue(ret) + def test_ec2_local_returns_false_on_non_aws(self): + """DataSourceEc2Local returns False when platform is not AWS.""" + self.datasource = ec2.DataSourceEc2Local + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, + md=DEFAULT_METADATA) + platform_attrs = [ + attr for attr in ec2.Platforms.__dict__.keys() + if not attr.startswith('__')] + for attr_name in platform_attrs: + platform_name = getattr(ec2.Platforms, attr_name) + if platform_name != 'AWS': + ds._cloud_platform = platform_name + ret = ds.get_data() + self.assertFalse(ret) + message = ( + "Local Ec2 mode only supported on ('AWS',)," + ' not {0}'.format(platform_name)) + self.assertIn(message, self.logs.getvalue()) + @httpretty.activate @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD') def test_ec2_local_returns_false_on_bsd(self, m_is_freebsd): @@ -336,8 +357,8 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): super(TestConvertEc2MetadataNetworkConfig, self).setUp() self.mac1 = '06:17:04:d7:26:09' self.network_metadata = { - 'network': {'interfaces': {'macs': { - self.mac1: {'public-ipv4s': '172.31.2.16'}}}}} + 'interfaces': {'macs': { + self.mac1: {'public-ipv4s': '172.31.2.16'}}}} def test_convert_ec2_metadata_network_config_skips_absent_macs(self): """Any mac absent from metadata is skipped by network config.""" @@ -357,7 +378,7 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): macs_to_nics = {self.mac1: 'eth9'} network_metadata_ipv6 = copy.deepcopy(self.network_metadata) nic1_metadata = ( - network_metadata_ipv6['network']['interfaces']['macs'][self.mac1]) + network_metadata_ipv6['interfaces']['macs'][self.mac1]) nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64' nic1_metadata.pop('public-ipv4s') expected = {'version': 1, 'config': [ @@ -373,7 +394,7 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): macs_to_nics = {self.mac1: 'eth9'} network_metadata_both = copy.deepcopy(self.network_metadata) nic1_metadata = ( - network_metadata_both['network']['interfaces']['macs'][self.mac1]) + network_metadata_both['interfaces']['macs'][self.mac1]) nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64' expected = {'version': 1, 'config': [ {'mac_address': self.mac1, 'type': 'physical', -- cgit v1.2.3