From d5f855dd96ccbea77f61b0515b574ad2c43d116d Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Wed, 9 Aug 2017 21:55:52 -0600
Subject: ec2: Allow Ec2 to run in init-local using dhclient in a sandbox.

This branch is a prerequisite for IPv6 support in AWS by allowing Ec2
datasource to query the metadata source version 2016-09-02 about whether
or not it needs to configure IPv6 on interfaces. If version 2016-09-02
is not present, fallback to the min_metadata_version of 2009-04-04. The
DataSourceEc2Local not run on FreeBSD because dhclient in doesn't
support the -sf flag allowing us to run dhclient without filesystem
side-effects.

To query AWS' metadata address @ 169.254.169.254, the instance must have
a dhcp-allocated address configured. Configuring IPv4 link-local
addresses result in timeouts from the metadata service. We introduced a
DataSourceEc2Local subclass which will perform a sandboxed dhclient
discovery which obtains an authorized IP address on eth0 and crawl
metadata about full instance network configuration.

Since ec2 IPv6 metadata is not sufficient in itself to tell us all the
ipv6 knownledge we need, it only be used as a boolean to tell us which
nics need IPv6. Cloud-init will then configure desired interfaces to
DHCPv6 versus DHCPv4.

Performance side note: Shifting the dhcp work into init-local for Ec2
actually gets us 1 second faster deployments by skipping init-network
phase of alternate datasource checks because Ec2Local is configured in
an ealier boot stage. In 3 test runs prior to this change: cloud-init
runs were 5.5 seconds, with the change we now average 4.6 seconds.

This efficiency could be even further improved if we avoiding dhcp
discovery in order to talk to the metadata service from an AWS
authorized dhcp address if there were some way to advertize the dhcp
configuration via DMI/SMBIOS or system environment variables.

Inspecting time costs of the dhclient setup/teardown in 3 live runs the
time cost for the dhcp setup round trip on AWS is:
test 1: 76 milliseconds
         dhcp discovery + metadata: 0.347 seconds
         metadata alone: 0.271 seconds
test 2: 88 milliseconds
         dhcp discovery + metadata: 0.388 seconds
         metadata alone: 0.300 seconds
test 3: 75 milliseconds
         dhcp discovery + metadata: 0.366 seconds
         metadata alone: 0.291 seconds

LP: #1709772
---
 tests/unittests/test_datasource/test_aliyun.py |  11 +-
 tests/unittests/test_datasource/test_common.py |   1 +
 tests/unittests/test_datasource/test_ec2.py    | 136 ++++++++++++++++++++-----
 3 files changed, 119 insertions(+), 29 deletions(-)

(limited to 'tests/unittests/test_datasource')

diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py
index 990bff2c..996560e4 100644
--- a/tests/unittests/test_datasource/test_aliyun.py
+++ b/tests/unittests/test_datasource/test_aliyun.py
@@ -70,7 +70,6 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
         paths = helpers.Paths({})
         self.ds = ay.DataSourceAliYun(cfg, distro, paths)
         self.metadata_address = self.ds.metadata_urls[0]
-        self.api_ver = self.ds.api_ver
 
     @property
     def default_metadata(self):
@@ -82,13 +81,15 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
 
     @property
     def metadata_url(self):
-        return os.path.join(self.metadata_address,
-                            self.api_ver, 'meta-data') + '/'
+        return os.path.join(
+            self.metadata_address,
+            self.ds.min_metadata_version, 'meta-data') + '/'
 
     @property
     def userdata_url(self):
-        return os.path.join(self.metadata_address,
-                            self.api_ver, 'user-data')
+        return os.path.join(
+            self.metadata_address,
+            self.ds.min_metadata_version, 'user-data')
 
     def regist_default_server(self):
         register_mock_metaserver(self.metadata_url, self.default_metadata)
diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
index 413e87ac..4802f105 100644
--- a/tests/unittests/test_datasource/test_common.py
+++ b/tests/unittests/test_datasource/test_common.py
@@ -35,6 +35,7 @@ DEFAULT_LOCAL = [
     OpenNebula.DataSourceOpenNebula,
     OVF.DataSourceOVF,
     SmartOS.DataSourceSmartOS,
+    Ec2.DataSourceEc2Local,
 ]
 
 DEFAULT_NETWORK = [
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 12230ae2..33d02619 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -8,35 +8,67 @@ from cloudinit import helpers
 from cloudinit.sources import DataSourceEc2 as ec2
 
 
-# collected from api version 2009-04-04/ with
+# collected from api version 2016-09-02/ with
 # python3 -c 'import json
 # from cloudinit.ec2_utils import get_instance_metadata as gm
-# print(json.dumps(gm("2009-04-04"), indent=1, sort_keys=True))'
+# print(json.dumps(gm("2016-09-02"), indent=1, sort_keys=True))'
 DEFAULT_METADATA = {
-    "ami-id": "ami-80861296",
+    "ami-id": "ami-8b92b4ee",
     "ami-launch-index": "0",
     "ami-manifest-path": "(unknown)",
     "block-device-mapping": {"ami": "/dev/sda1", "root": "/dev/sda1"},
-    "hostname": "ip-10-0-0-149",
+    "hostname": "ip-172-31-31-158.us-east-2.compute.internal",
     "instance-action": "none",
-    "instance-id": "i-0052913950685138c",
-    "instance-type": "t2.micro",
-    "local-hostname": "ip-10-0-0-149",
-    "local-ipv4": "10.0.0.149",
-    "placement": {"availability-zone": "us-east-1b"},
+    "instance-id": "i-0a33f80f09c96477f",
+    "instance-type": "t2.small",
+    "local-hostname": "ip-172-3-3-15.us-east-2.compute.internal",
+    "local-ipv4": "172.3.3.15",
+    "mac": "06:17:04:d7:26:09",
+    "metrics": {"vhostmd": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"},
+    "network": {
+        "interfaces": {
+            "macs": {
+                "06:17:04:d7:26:09": {
+                    "device-number": "0",
+                    "interface-id": "eni-e44ef49e",
+                    "ipv4-associations": {"13.59.77.202": "172.3.3.15"},
+                    "ipv6s": "2600:1f16:aeb:b20b:9d87:a4af:5cc9:73dc",
+                    "local-hostname": ("ip-172-3-3-15.us-east-2."
+                                       "compute.internal"),
+                    "local-ipv4s": "172.3.3.15",
+                    "mac": "06:17:04:d7:26:09",
+                    "owner-id": "950047163771",
+                    "public-hostname": ("ec2-13-59-77-202.us-east-2."
+                                        "compute.amazonaws.com"),
+                    "public-ipv4s": "13.59.77.202",
+                    "security-group-ids": "sg-5a61d333",
+                    "security-groups": "wide-open",
+                    "subnet-id": "subnet-20b8565b",
+                    "subnet-ipv4-cidr-block": "172.31.16.0/20",
+                    "subnet-ipv6-cidr-blocks": "2600:1f16:aeb:b20b::/64",
+                    "vpc-id": "vpc-87e72bee",
+                    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+                    "vpc-ipv4-cidr-blocks": "172.31.0.0/16",
+                    "vpc-ipv6-cidr-blocks": "2600:1f16:aeb:b200::/56"
+                }
+            }
+        }
+    },
+    "placement": {"availability-zone": "us-east-2b"},
     "profile": "default-hvm",
-    "public-hostname": "",
-    "public-ipv4": "107.23.188.247",
+    "public-hostname": "ec2-13-59-77-202.us-east-2.compute.amazonaws.com",
+    "public-ipv4": "13.59.77.202",
     "public-keys": {"brickies": ["ssh-rsa AAAAB3Nz....w== brickies"]},
-    "reservation-id": "r-00a2c173fb5782a08",
-    "security-groups": "wide-open"
+    "reservation-id": "r-01efbc9996bac1bd6",
+    "security-groups": "my-wide-open",
+    "services": {"domain": "amazonaws.com", "partition": "aws"}
 }
 
 
 def _register_ssh_keys(rfunc, base_url, keys_data):
     """handle ssh key inconsistencies.
 
-    public-keys in the ec2 metadata is inconsistently formatted compared
+    public-keys in the ec2 metadata is inconsistently formated compared
     to other entries.
     Given keys_data of {name1: pubkey1, name2: pubkey2}
 
@@ -115,6 +147,8 @@ def register_mock_metaserver(base_url, data):
 
 
 class TestEc2(test_helpers.HttprettyTestCase):
+    with_logs = True
+
     valid_platform_data = {
         'uuid': 'ec212f79-87d1-2f1d-588f-d86dc0fd5412',
         'uuid_source': 'dmi',
@@ -123,16 +157,20 @@ class TestEc2(test_helpers.HttprettyTestCase):
 
     def setUp(self):
         super(TestEc2, self).setUp()
-        self.metadata_addr = ec2.DataSourceEc2.metadata_urls[0]
-        self.api_ver = '2009-04-04'
+        self.datasource = ec2.DataSourceEc2
+        self.metadata_addr = self.datasource.metadata_urls[0]
 
     @property
     def metadata_url(self):
-        return '/'.join([self.metadata_addr, self.api_ver, 'meta-data', ''])
+        return '/'.join([
+            self.metadata_addr,
+            self.datasource.min_metadata_version, 'meta-data', ''])
 
     @property
     def userdata_url(self):
-        return '/'.join([self.metadata_addr, self.api_ver, 'user-data'])
+        return '/'.join([
+            self.metadata_addr,
+            self.datasource.min_metadata_version, 'user-data'])
 
     def _patch_add_cleanup(self, mpath, *args, **kwargs):
         p = mock.patch(mpath, *args, **kwargs)
@@ -144,7 +182,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
         paths = helpers.Paths({})
         if sys_cfg is None:
             sys_cfg = {}
-        ds = ec2.DataSourceEc2(sys_cfg=sys_cfg, distro=distro, paths=paths)
+        ds = self.datasource(sys_cfg=sys_cfg, distro=distro, paths=paths)
         if platform_data is not None:
             self._patch_add_cleanup(
                 "cloudinit.sources.DataSourceEc2._collect_platform_data",
@@ -157,14 +195,16 @@ class TestEc2(test_helpers.HttprettyTestCase):
         return ds
 
     @httpretty.activate
-    def test_valid_platform_with_strict_true(self):
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    def test_valid_platform_with_strict_true(self, m_dhcp):
         """Valid platform data should return true with strict_id true."""
         ds = self._setup_ds(
             platform_data=self.valid_platform_data,
             sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(True, ret)
+        self.assertTrue(ret)
+        self.assertEqual(0, m_dhcp.call_count)
 
     @httpretty.activate
     def test_valid_platform_with_strict_false(self):
@@ -174,7 +214,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
             sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(True, ret)
+        self.assertTrue(ret)
 
     @httpretty.activate
     def test_unknown_platform_with_strict_true(self):
@@ -185,7 +225,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
             sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(False, ret)
+        self.assertFalse(ret)
 
     @httpretty.activate
     def test_unknown_platform_with_strict_false(self):
@@ -196,7 +236,55 @@ class TestEc2(test_helpers.HttprettyTestCase):
             sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(True, ret)
+        self.assertTrue(ret)
+
+    @httpretty.activate
+    @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
+    def test_ec2_local_returns_false_on_bsd(self, m_is_freebsd):
+        """DataSourceEc2Local returns False on BSD.
+
+        FreeBSD dhclient doesn't support dhclient -sf to run in a sandbox.
+        """
+        m_is_freebsd.return_value = True
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md=DEFAULT_METADATA)
+        ret = ds.get_data()
+        self.assertFalse(ret)
+        self.assertIn(
+            "FreeBSD doesn't support running dhclient with -sf",
+            self.logs.getvalue())
+
+    @httpretty.activate
+    @mock.patch('cloudinit.net.EphemeralIPv4Network')
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
+    def test_ec2_local_performs_dhcp_on_non_bsd(self, m_is_bsd, m_dhcp, m_net):
+        """Ec2Local returns True for valid platform data on non-BSD with dhcp.
+
+        DataSourceEc2Local will setup initial IPv4 network via dhcp discovery.
+        Then the metadata services is crawled for more network config info.
+        When the platform data is valid, return True.
+        """
+        m_is_bsd.return_value = False
+        m_dhcp.return_value = [{
+            'interface': 'eth9', 'fixed-address': '192.168.2.9',
+            'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0',
+            'broadcast-address': '192.168.2.255'}]
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md=DEFAULT_METADATA)
+        ret = ds.get_data()
+        self.assertTrue(ret)
+        m_dhcp.assert_called_once_with()
+        m_net.assert_called_once_with(
+            broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9',
+            prefix_or_mask='255.255.255.0', router='192.168.2.1')
+        self.assertIn('Crawl of metadata service took', self.logs.getvalue())
 
 
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From 3c45330af2a301f2bf219da556844d01cef6778e Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Tue, 29 Aug 2017 10:34:19 -0600
Subject: ec2: Add IPv6 dhcp support to Ec2DataSource.

DataSourceEc2 now parses the metadata for each nic to determine if
configured for ipv6 and/or ipv4 addresses. In AWS for metadata version
2016-09-02, nics configured for ipv4 or ipv6 addresses will have non-zero
values stored in metadata at network/interfaces/macs/<MAC>/public-ipv4 or
ipv6s respectively. Those metadata files are only non-zero when an ipv4 or
ipv6 ip is associated to the specific nic. A new
DataSourceEc2.network_config property is added which parses the metadata
and renders a network version 1 dictionary representing both dhcp4 and
dhcp6 configuration for associated nics.

The network configuration returned from the datasource will also 'pin' the
nic name to the name presented on the instance for each nic.

LP: #1639030
---
 cloudinit/sources/DataSourceEc2.py          | 38 +++++++++++
 tests/unittests/test_datasource/test_ec2.py | 97 +++++++++++++++++++++++++++++
 2 files changed, 135 insertions(+)

(limited to 'tests/unittests/test_datasource')

diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 8e5f8ee4..07c12bb4 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -57,6 +57,8 @@ class DataSourceEc2(sources.DataSource):
 
     _cloud_platform = None
 
+    _network_config = None  # Used for caching calculated network config v1
+
     # Whether we want to get network configuration from the metadata service.
     get_network_metadata = False
 
@@ -279,6 +281,15 @@ class DataSourceEc2(sources.DataSource):
                 util.get_cfg_by_path(cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT),
                 cfg)
 
+    @property
+    def network_config(self):
+        """Return a network config dict for rendering ENI or netplan files."""
+        if self._network_config is None:
+            if self.metadata is not None:
+                self._network_config = convert_ec2_metadata_network_config(
+                    self.metadata)
+        return self._network_config
+
     def _crawl_metadata(self):
         """Crawl metadata service when available.
 
@@ -423,6 +434,33 @@ def _collect_platform_data():
     return data
 
 
+def convert_ec2_metadata_network_config(metadata=None, macs_to_nics=None):
+    """Convert ec2 metadata to network config version 1 data dict.
+
+    @param: metadata: Dictionary of metadata crawled from EC2 metadata url.
+    @param: macs_to_name: Optional dict mac addresses and the nic name. If
+       not provided, get_interfaces_by_mac is called to get it from the OS.
+
+    @return A dict of network config version 1 based on the metadata and macs.
+    """
+    netcfg = {'version': 1, 'config': []}
+    if not macs_to_nics:
+        macs_to_nics = net.get_interfaces_by_mac()
+    macs_metadata = metadata['network']['interfaces']['macs']
+    for mac, nic_name in macs_to_nics.items():
+        nic_metadata = macs_metadata.get(mac)
+        if not nic_metadata:
+            continue  # Not a physical nic represented in metadata
+        nic_cfg = {'type': 'physical', 'name': nic_name, 'subnets': []}
+        nic_cfg['mac_address'] = mac
+        if nic_metadata.get('public-ipv4s'):
+            nic_cfg['subnets'].append({'type': 'dhcp4'})
+        if nic_metadata.get('ipv6s'):
+            nic_cfg['subnets'].append({'type': 'dhcp6'})
+        netcfg['config'].append(nic_cfg)
+    return netcfg
+
+
 # Used to match classes to dependencies
 datasources = [
     (DataSourceEc2Local, (sources.DEP_FILESYSTEM,)),  # Run at init-local
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 33d02619..e1ce6446 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -1,5 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
+import copy
 import httpretty
 import mock
 
@@ -194,6 +195,34 @@ class TestEc2(test_helpers.HttprettyTestCase):
 
         return ds
 
+    @httpretty.activate
+    def test_network_config_property_returns_version_1_network_data(self):
+        """network_config property returns network version 1 for metadata."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
+            md=DEFAULT_METADATA)
+        ds.get_data()
+        mac1 = '06:17:04:d7:26:09'  # Defined in DEFAULT_METADATA
+        expected = {'version': 1, 'config': [
+            {'mac_address': '06:17:04:d7:26:09', 'name': 'eth9',
+             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}],
+             'type': 'physical'}]}
+        patch_path = (
+            'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
+        with mock.patch(patch_path) as m_get_interfaces_by_mac:
+            m_get_interfaces_by_mac.return_value = {mac1: 'eth9'}
+            self.assertEqual(expected, ds.network_config)
+
+    def test_network_config_property_is_cached_in_datasource(self):
+        """network_config property is cached in DataSourceEc2."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
+            md=DEFAULT_METADATA)
+        ds._network_config = {'cached': 'data'}
+        self.assertEqual({'cached': 'data'}, ds.network_config)
+
     @httpretty.activate
     @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
     def test_valid_platform_with_strict_true(self, m_dhcp):
@@ -287,4 +316,72 @@ class TestEc2(test_helpers.HttprettyTestCase):
         self.assertIn('Crawl of metadata service took', self.logs.getvalue())
 
 
+class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
+
+    def setUp(self):
+        super(TestConvertEc2MetadataNetworkConfig, self).setUp()
+        self.mac1 = '06:17:04:d7:26:09'
+        self.network_metadata = {
+            'network': {'interfaces': {'macs': {
+                self.mac1: {'public-ipv4s': '172.31.2.16'}}}}}
+
+    def test_convert_ec2_metadata_network_config_skips_absent_macs(self):
+        """Any mac absent from metadata is skipped by network config."""
+        macs_to_nics = {self.mac1: 'eth9', 'DE:AD:BE:EF:FF:FF': 'vitualnic2'}
+
+        # DE:AD:BE:EF:FF:FF represented by OS but not in metadata
+        expected = {'version': 1, 'config': [
+            {'mac_address': self.mac1, 'type': 'physical',
+             'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]}
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                self.network_metadata, macs_to_nics))
+
+    def test_convert_ec2_metadata_network_config_handles_only_dhcp6(self):
+        """Config dhcp6 when ipv6s is in metadata for a mac."""
+        macs_to_nics = {self.mac1: 'eth9'}
+        network_metadata_ipv6 = copy.deepcopy(self.network_metadata)
+        nic1_metadata = (
+            network_metadata_ipv6['network']['interfaces']['macs'][self.mac1])
+        nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
+        nic1_metadata.pop('public-ipv4s')
+        expected = {'version': 1, 'config': [
+            {'mac_address': self.mac1, 'type': 'physical',
+             'name': 'eth9', 'subnets': [{'type': 'dhcp6'}]}]}
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_ipv6, macs_to_nics))
+
+    def test_convert_ec2_metadata_network_config_handles_dhcp4_and_dhcp6(self):
+        """Config both dhcp4 and dhcp6 when both vpc-ipv6 and ipv4 exists."""
+        macs_to_nics = {self.mac1: 'eth9'}
+        network_metadata_both = copy.deepcopy(self.network_metadata)
+        nic1_metadata = (
+            network_metadata_both['network']['interfaces']['macs'][self.mac1])
+        nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
+        expected = {'version': 1, 'config': [
+            {'mac_address': self.mac1, 'type': 'physical',
+             'name': 'eth9',
+             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}]}]}
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_both, macs_to_nics))
+
+    def test_convert_ec2_metadata_gets_macs_from_get_interfaces_by_mac(self):
+        """Convert Ec2 Metadata calls get_interfaces_by_mac by default."""
+        expected = {'version': 1, 'config': [
+            {'mac_address': self.mac1, 'type': 'physical',
+             'name': 'eth9',
+             'subnets': [{'type': 'dhcp4'}]}]}
+        patch_path = (
+            'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
+        with mock.patch(patch_path) as m_get_interfaces_by_mac:
+            m_get_interfaces_by_mac.return_value = {self.mac1: 'eth9'}
+            self.assertEqual(
+                expected,
+                ec2.convert_ec2_metadata_network_config(self.network_metadata))
+
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From 1770a1eb647d24e14732194e72210ea494986ad2 Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Thu, 31 Aug 2017 16:24:35 -0600
Subject: tests: Stop leaking calls through unmocked metadata addresses

DataSourceEc2 behavior changed to first check a minimum acceptable
metadata version uri http://169.154.169.254/<min_version>/instance-id,
retrying on 404, until the metadata service is available. After the
metadata service is up, the datasource inspects preferred
extended_metadata_versions for availability. Unit tests only mocked the
preferred extended_metadata_version so all Ec2 tests were retrying
attempts against
http://169.254.169.254/meta-data/<min-version>/instance-id adding a lot of
time cost to the unit test runs.

This branch uses httpretty to properly mock the following:
  - 404s from metadata on undesired extended_metadata_version test routes
    - https://169.254.169.254/meta-data/2016-09-02/instance-id
  - full metadata dictionary represented on min_metadata_version
    - https://169.254.169.254/meta-data/2016-09-02/*

The branch also tightens httpretty to raise a MockError for any URL which
isn't mocked via httpretty.HTTPretty.allow_net_connect=False.

LP: #1714117
---
 tests/unittests/test_datasource/test_ec2.py | 46 +++++++++++++++++++----------
 1 file changed, 30 insertions(+), 16 deletions(-)

(limited to 'tests/unittests/test_datasource')

diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index e1ce6446..b7a84e21 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -116,6 +116,9 @@ def register_mock_metaserver(base_url, data):
     In the index, references to lists or dictionaries have a trailing /.
     """
     def register_helper(register, base_url, body):
+        if not isinstance(base_url, str):
+            register(base_url, body)
+            return
         base_url = base_url.rstrip("/")
         if isinstance(body, str):
             register(base_url, body)
@@ -138,7 +141,7 @@ def register_mock_metaserver(base_url, data):
             register(base_url, '\n'.join(vals) + '\n')
             register(base_url + '/', '\n'.join(vals) + '\n')
         elif body is None:
-            register(base_url, 'not found', status_code=404)
+            register(base_url, 'not found', status=404)
 
     def myreg(*argc, **kwargs):
         # print("register_url(%s, %s)" % (argc, kwargs))
@@ -161,38 +164,47 @@ class TestEc2(test_helpers.HttprettyTestCase):
         self.datasource = ec2.DataSourceEc2
         self.metadata_addr = self.datasource.metadata_urls[0]
 
-    @property
-    def metadata_url(self):
-        return '/'.join([
-            self.metadata_addr,
-            self.datasource.min_metadata_version, 'meta-data', ''])
-
-    @property
-    def userdata_url(self):
-        return '/'.join([
-            self.metadata_addr,
-            self.datasource.min_metadata_version, 'user-data'])
+    def data_url(self, version):
+        """Return a metadata url based on the version provided."""
+        return '/'.join([self.metadata_addr, version, 'meta-data', ''])
 
     def _patch_add_cleanup(self, mpath, *args, **kwargs):
         p = mock.patch(mpath, *args, **kwargs)
         p.start()
         self.addCleanup(p.stop)
 
-    def _setup_ds(self, sys_cfg, platform_data, md, ud=None):
+    def _setup_ds(self, sys_cfg, platform_data, md, md_version=None):
+        self.uris = []
         distro = {}
         paths = helpers.Paths({})
         if sys_cfg is None:
             sys_cfg = {}
         ds = self.datasource(sys_cfg=sys_cfg, distro=distro, paths=paths)
+        if not md_version:
+            md_version = ds.min_metadata_version
         if platform_data is not None:
             self._patch_add_cleanup(
                 "cloudinit.sources.DataSourceEc2._collect_platform_data",
                 return_value=platform_data)
 
         if md:
-            register_mock_metaserver(self.metadata_url, md)
-            register_mock_metaserver(self.userdata_url, ud)
-
+            httpretty.HTTPretty.allow_net_connect = False
+            all_versions = (
+                [ds.min_metadata_version] + ds.extended_metadata_versions)
+            for version in all_versions:
+                metadata_url = self.data_url(version)
+                if version == md_version:
+                    # Register all metadata for desired version
+                    register_mock_metaserver(metadata_url, md)
+                else:
+                    instance_id_url = metadata_url + 'instance-id'
+                    if version == ds.min_metadata_version:
+                        # Add min_metadata_version service availability check
+                        register_mock_metaserver(
+                            instance_id_url, DEFAULT_METADATA['instance-id'])
+                    else:
+                        # Register 404s for all unrequested extended versions
+                        register_mock_metaserver(instance_id_url, None)
         return ds
 
     @httpretty.activate
@@ -297,6 +309,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
         Then the metadata services is crawled for more network config info.
         When the platform data is valid, return True.
         """
+
         m_is_bsd.return_value = False
         m_dhcp.return_value = [{
             'interface': 'eth9', 'fixed-address': '192.168.2.9',
@@ -307,6 +320,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
             platform_data=self.valid_platform_data,
             sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
             md=DEFAULT_METADATA)
+
         ret = ds.get_data()
         self.assertTrue(ret)
         m_dhcp.assert_called_once_with()
-- 
cgit v1.2.3


From a3649e03206a3596131413956ea7ecc18790ec73 Mon Sep 17 00:00:00 2001
From: Lars Kellogg-Stedman <lars@redhat.com>
Date: Tue, 5 Sep 2017 11:03:59 -0600
Subject: relocate tests/unittests/helpers.py to cloudinit/tests

This moves the base test case classes into into cloudinit/tests and
updates all the corresponding imports.
---
 cloudinit/analyze/tests/test_dump.py               |   2 +-
 cloudinit/net/tests/test_dhcp.py                   |   2 +-
 cloudinit/net/tests/test_init.py                   |   2 +-
 cloudinit/tests/helpers.py                         | 395 +++++++++++++++++++++
 cloudinit/tests/test_url_helper.py                 |   2 +-
 tests/unittests/helpers.py                         | 391 --------------------
 tests/unittests/test__init__.py                    |   2 +-
 tests/unittests/test_atomic_helper.py              |   2 +-
 tests/unittests/test_builtin_handlers.py           |   2 +-
 tests/unittests/test_cli.py                        |   2 +-
 tests/unittests/test_cs_util.py                    |   2 +-
 tests/unittests/test_data.py                       |   2 +-
 tests/unittests/test_datasource/test_aliyun.py     |   2 +-
 tests/unittests/test_datasource/test_altcloud.py   |   2 +-
 tests/unittests/test_datasource/test_azure.py      |   6 +-
 .../unittests/test_datasource/test_azure_helper.py |   2 +-
 tests/unittests/test_datasource/test_cloudsigma.py |   2 +-
 tests/unittests/test_datasource/test_cloudstack.py |   2 +-
 tests/unittests/test_datasource/test_common.py     |   2 +-
 .../unittests/test_datasource/test_configdrive.py  |   2 +-
 .../unittests/test_datasource/test_digitalocean.py |   2 +-
 tests/unittests/test_datasource/test_ec2.py        |   2 +-
 tests/unittests/test_datasource/test_gce.py        |   2 +-
 tests/unittests/test_datasource/test_maas.py       |   2 +-
 tests/unittests/test_datasource/test_nocloud.py    |   2 +-
 tests/unittests/test_datasource/test_opennebula.py |   2 +-
 tests/unittests/test_datasource/test_openstack.py  |   2 +-
 tests/unittests/test_datasource/test_ovf.py        |   2 +-
 tests/unittests/test_datasource/test_scaleway.py   |   2 +-
 tests/unittests/test_datasource/test_smartos.py    |   2 +-
 tests/unittests/test_distros/test_arch.py          |   2 +-
 tests/unittests/test_distros/test_create_users.py  |   2 +-
 tests/unittests/test_distros/test_debian.py        |   2 +-
 tests/unittests/test_distros/test_generic.py       |   2 +-
 tests/unittests/test_distros/test_netconfig.py     |   2 +-
 tests/unittests/test_distros/test_opensuse.py      |   2 +-
 tests/unittests/test_distros/test_resolv.py        |   2 +-
 tests/unittests/test_distros/test_sles.py          |   2 +-
 tests/unittests/test_distros/test_sysconfig.py     |   2 +-
 .../test_distros/test_user_data_normalize.py       |   2 +-
 tests/unittests/test_ds_identify.py                |   3 +-
 tests/unittests/test_ec2_util.py                   |   2 +-
 tests/unittests/test_filters/test_launch_index.py  |   2 +-
 .../test_handler/test_handler_apt_conf_v1.py       |   2 +-
 .../test_handler_apt_configure_sources_list_v1.py  |   2 +-
 .../test_handler_apt_configure_sources_list_v3.py  |   2 +-
 .../test_handler/test_handler_apt_source_v1.py     |   2 +-
 .../test_handler/test_handler_apt_source_v3.py     |   2 +-
 .../test_handler/test_handler_ca_certs.py          |   2 +-
 tests/unittests/test_handler/test_handler_chef.py  |   2 +-
 tests/unittests/test_handler/test_handler_debug.py |   2 +-
 .../test_handler/test_handler_disk_setup.py        |   2 +-
 .../test_handler/test_handler_growpart.py          |   2 +-
 .../test_handler/test_handler_landscape.py         |   5 +-
 .../unittests/test_handler/test_handler_locale.py  |   2 +-
 tests/unittests/test_handler/test_handler_lxd.py   |   2 +-
 .../test_handler/test_handler_mcollective.py       |   2 +-
 .../unittests/test_handler/test_handler_mounts.py  |   2 +-
 tests/unittests/test_handler/test_handler_ntp.py   |   2 +-
 .../test_handler/test_handler_power_state.py       |   4 +-
 .../unittests/test_handler/test_handler_puppet.py  |   2 +-
 .../unittests/test_handler/test_handler_rsyslog.py |   2 +-
 .../unittests/test_handler/test_handler_runcmd.py  |   2 +-
 .../test_handler/test_handler_seed_random.py       |   2 +-
 .../test_handler/test_handler_set_hostname.py      |   2 +-
 .../unittests/test_handler/test_handler_snappy.py  |   4 +-
 .../test_handler/test_handler_spacewalk.py         |   2 +-
 .../test_handler/test_handler_timezone.py          |   2 +-
 .../test_handler/test_handler_write_files.py       |   2 +-
 .../test_handler/test_handler_yum_add_repo.py      |   2 +-
 tests/unittests/test_handler/test_schema.py        |   2 +-
 tests/unittests/test_helpers.py                    |   2 +-
 tests/unittests/test_log.py                        |   2 +-
 tests/unittests/test_merging.py                    |   2 +-
 tests/unittests/test_net.py                        |   8 +-
 tests/unittests/test_pathprefix2dict.py            |   2 +-
 tests/unittests/test_registry.py                   |   2 +-
 tests/unittests/test_reporting.py                  |   2 +-
 tests/unittests/test_rh_subscription.py            |   2 +-
 tests/unittests/test_runs/test_merge_run.py        |   2 +-
 tests/unittests/test_runs/test_simple_run.py       |   2 +-
 tests/unittests/test_sshutil.py                    |   3 +-
 tests/unittests/test_templating.py                 |   2 +-
 tests/unittests/test_util.py                       |   2 +-
 tests/unittests/test_version.py                    |   2 +-
 tests/unittests/test_vmware_config_file.py         |   2 +-
 86 files changed, 491 insertions(+), 482 deletions(-)
 create mode 100644 cloudinit/tests/helpers.py
 delete mode 100644 tests/unittests/helpers.py

(limited to 'tests/unittests/test_datasource')

diff --git a/cloudinit/analyze/tests/test_dump.py b/cloudinit/analyze/tests/test_dump.py
index 2c0885d0..f4c42841 100644
--- a/cloudinit/analyze/tests/test_dump.py
+++ b/cloudinit/analyze/tests/test_dump.py
@@ -6,7 +6,7 @@ from textwrap import dedent
 from cloudinit.analyze.dump import (
     dump_events, parse_ci_logline, parse_timestamp)
 from cloudinit.util import subp, write_file
-from tests.unittests.helpers import CiTestCase
+from cloudinit.tests.helpers import CiTestCase
 
 
 class TestParseTimestamp(CiTestCase):
diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py
index 47d8d461..4a37e98a 100644
--- a/cloudinit/net/tests/test_dhcp.py
+++ b/cloudinit/net/tests/test_dhcp.py
@@ -8,7 +8,7 @@ from cloudinit.net.dhcp import (
     InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery,
     parse_dhcp_lease_file, dhcp_discovery)
 from cloudinit.util import ensure_file, write_file
-from tests.unittests.helpers import CiTestCase
+from cloudinit.tests.helpers import CiTestCase
 
 
 class TestParseDHCPLeasesFile(CiTestCase):
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
index cc052a7d..8cb4114e 100644
--- a/cloudinit/net/tests/test_init.py
+++ b/cloudinit/net/tests/test_init.py
@@ -7,7 +7,7 @@ import os
 
 import cloudinit.net as net
 from cloudinit.util import ensure_file, write_file, ProcessExecutionError
-from tests.unittests.helpers import CiTestCase
+from cloudinit.tests.helpers import CiTestCase
 
 
 class TestSysDevPath(CiTestCase):
diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py
new file mode 100644
index 00000000..28e26622
--- /dev/null
+++ b/cloudinit/tests/helpers.py
@@ -0,0 +1,395 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from __future__ import print_function
+
+import functools
+import json
+import logging
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+
+import mock
+import six
+import unittest2
+
+try:
+    from contextlib import ExitStack
+except ImportError:
+    from contextlib2 import ExitStack
+
+from cloudinit import helpers as ch
+from cloudinit import util
+
+# Used for skipping tests
+SkipTest = unittest2.SkipTest
+
+# Used for detecting different python versions
+PY2 = False
+PY26 = False
+PY27 = False
+PY3 = False
+
+_PY_VER = sys.version_info
+_PY_MAJOR, _PY_MINOR, _PY_MICRO = _PY_VER[0:3]
+if (_PY_MAJOR, _PY_MINOR) <= (2, 6):
+    if (_PY_MAJOR, _PY_MINOR) == (2, 6):
+        PY26 = True
+    if (_PY_MAJOR, _PY_MINOR) >= (2, 0):
+        PY2 = True
+else:
+    if (_PY_MAJOR, _PY_MINOR) == (2, 7):
+        PY27 = True
+        PY2 = True
+    if (_PY_MAJOR, _PY_MINOR) >= (3, 0):
+        PY3 = True
+
+
+# Makes the old path start
+# with new base instead of whatever
+# it previously had
+def rebase_path(old_path, new_base):
+    if old_path.startswith(new_base):
+        # Already handled...
+        return old_path
+    # Retarget the base of that path
+    # to the new base instead of the
+    # old one...
+    path = os.path.join(new_base, old_path.lstrip("/"))
+    path = os.path.abspath(path)
+    return path
+
+
+# Can work on anything that takes a path as arguments
+def retarget_many_wrapper(new_base, am, old_func):
+    def wrapper(*args, **kwds):
+        n_args = list(args)
+        nam = am
+        if am == -1:
+            nam = len(n_args)
+        for i in range(0, nam):
+            path = args[i]
+            # patchOS() wraps various os and os.path functions, however in
+            # Python 3 some of these now accept file-descriptors (integers).
+            # That breaks rebase_path() so in lieu of a better solution, just
+            # don't rebase if we get a fd.
+            if isinstance(path, six.string_types):
+                n_args[i] = rebase_path(path, new_base)
+        return old_func(*n_args, **kwds)
+    return wrapper
+
+
+class TestCase(unittest2.TestCase):
+
+    def reset_global_state(self):
+        """Reset any global state to its original settings.
+
+        cloudinit caches some values in cloudinit.util.  Unit tests that
+        involved those cached paths were then subject to failure if the order
+        of invocation changed (LP: #1703697).
+
+        This function resets any of these global state variables to their
+        initial state.
+
+        In the future this should really be done with some registry that
+        can then be cleaned in a more obvious way.
+        """
+        util.PROC_CMDLINE = None
+        util._DNS_REDIRECT_IP = None
+        util._LSB_RELEASE = {}
+
+    def setUp(self):
+        super(TestCase, self).setUp()
+        self.reset_global_state()
+
+
+class CiTestCase(TestCase):
+    """This is the preferred test case base class unless user
+       needs other test case classes below."""
+
+    # Subclass overrides for specific test behavior
+    # Whether or not a unit test needs logfile setup
+    with_logs = False
+
+    def setUp(self):
+        super(CiTestCase, self).setUp()
+        if self.with_logs:
+            # Create a log handler so unit tests can search expected logs.
+            self.logger = logging.getLogger()
+            self.logs = six.StringIO()
+            formatter = logging.Formatter('%(levelname)s: %(message)s')
+            handler = logging.StreamHandler(self.logs)
+            handler.setFormatter(formatter)
+            self.old_handlers = self.logger.handlers
+            self.logger.handlers = [handler]
+
+    def tearDown(self):
+        if self.with_logs:
+            # Remove the handler we setup
+            logging.getLogger().handlers = self.old_handlers
+        super(CiTestCase, self).tearDown()
+
+    def tmp_dir(self, dir=None, cleanup=True):
+        # return a full path to a temporary directory that will be cleaned up.
+        if dir is None:
+            tmpd = tempfile.mkdtemp(
+                prefix="ci-%s." % self.__class__.__name__)
+        else:
+            tmpd = tempfile.mkdtemp(dir=dir)
+        self.addCleanup(functools.partial(shutil.rmtree, tmpd))
+        return tmpd
+
+    def tmp_path(self, path, dir=None):
+        # return an absolute path to 'path' under dir.
+        # if dir is None, one will be created with tmp_dir()
+        # the file is not created or modified.
+        if dir is None:
+            dir = self.tmp_dir()
+        return os.path.normpath(os.path.abspath(os.path.join(dir, path)))
+
+
+class ResourceUsingTestCase(CiTestCase):
+
+    def setUp(self):
+        super(ResourceUsingTestCase, self).setUp()
+        self.resource_path = None
+
+    def resourceLocation(self, subname=None):
+        if self.resource_path is None:
+            paths = [
+                os.path.join('tests', 'data'),
+                os.path.join('data'),
+                os.path.join(os.pardir, 'tests', 'data'),
+                os.path.join(os.pardir, 'data'),
+            ]
+            for p in paths:
+                if os.path.isdir(p):
+                    self.resource_path = p
+                    break
+        self.assertTrue((self.resource_path and
+                         os.path.isdir(self.resource_path)),
+                        msg="Unable to locate test resource data path!")
+        if not subname:
+            return self.resource_path
+        return os.path.join(self.resource_path, subname)
+
+    def readResource(self, name):
+        where = self.resourceLocation(name)
+        with open(where, 'r') as fh:
+            return fh.read()
+
+    def getCloudPaths(self, ds=None):
+        tmpdir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, tmpdir)
+        cp = ch.Paths({'cloud_dir': tmpdir,
+                       'templates_dir': self.resourceLocation()},
+                      ds=ds)
+        return cp
+
+
+class FilesystemMockingTestCase(ResourceUsingTestCase):
+
+    def setUp(self):
+        super(FilesystemMockingTestCase, self).setUp()
+        self.patched_funcs = ExitStack()
+
+    def tearDown(self):
+        self.patched_funcs.close()
+        ResourceUsingTestCase.tearDown(self)
+
+    def replicateTestRoot(self, example_root, target_root):
+        real_root = self.resourceLocation()
+        real_root = os.path.join(real_root, 'roots', example_root)
+        for (dir_path, _dirnames, filenames) in os.walk(real_root):
+            real_path = dir_path
+            make_path = rebase_path(real_path[len(real_root):], target_root)
+            util.ensure_dir(make_path)
+            for f in filenames:
+                real_path = util.abs_join(real_path, f)
+                make_path = util.abs_join(make_path, f)
+                shutil.copy(real_path, make_path)
+
+    def patchUtils(self, new_root):
+        patch_funcs = {
+            util: [('write_file', 1),
+                   ('append_file', 1),
+                   ('load_file', 1),
+                   ('ensure_dir', 1),
+                   ('chmod', 1),
+                   ('delete_dir_contents', 1),
+                   ('del_file', 1),
+                   ('sym_link', -1),
+                   ('copy', -1)],
+        }
+        for (mod, funcs) in patch_funcs.items():
+            for (f, am) in funcs:
+                func = getattr(mod, f)
+                trap_func = retarget_many_wrapper(new_root, am, func)
+                self.patched_funcs.enter_context(
+                    mock.patch.object(mod, f, trap_func))
+
+        # Handle subprocess calls
+        func = getattr(util, 'subp')
+
+        def nsubp(*_args, **_kwargs):
+            return ('', '')
+
+        self.patched_funcs.enter_context(
+            mock.patch.object(util, 'subp', nsubp))
+
+        def null_func(*_args, **_kwargs):
+            return None
+
+        for f in ['chownbyid', 'chownbyname']:
+            self.patched_funcs.enter_context(
+                mock.patch.object(util, f, null_func))
+
+    def patchOS(self, new_root):
+        patch_funcs = {
+            os.path: [('isfile', 1), ('exists', 1),
+                      ('islink', 1), ('isdir', 1)],
+            os: [('listdir', 1), ('mkdir', 1),
+                 ('lstat', 1), ('symlink', 2)],
+        }
+        for (mod, funcs) in patch_funcs.items():
+            for f, nargs in funcs:
+                func = getattr(mod, f)
+                trap_func = retarget_many_wrapper(new_root, nargs, func)
+                self.patched_funcs.enter_context(
+                    mock.patch.object(mod, f, trap_func))
+
+    def patchOpen(self, new_root):
+        trap_func = retarget_many_wrapper(new_root, 1, open)
+        name = 'builtins.open' if PY3 else '__builtin__.open'
+        self.patched_funcs.enter_context(mock.patch(name, trap_func))
+
+    def patchStdoutAndStderr(self, stdout=None, stderr=None):
+        if stdout is not None:
+            self.patched_funcs.enter_context(
+                mock.patch.object(sys, 'stdout', stdout))
+        if stderr is not None:
+            self.patched_funcs.enter_context(
+                mock.patch.object(sys, 'stderr', stderr))
+
+    def reRoot(self, root=None):
+        if root is None:
+            root = self.tmp_dir()
+        self.patchUtils(root)
+        self.patchOS(root)
+        return root
+
+
+class HttprettyTestCase(CiTestCase):
+    # necessary as http_proxy gets in the way of httpretty
+    # https://github.com/gabrielfalcao/HTTPretty/issues/122
+
+    def setUp(self):
+        self.restore_proxy = os.environ.get('http_proxy')
+        if self.restore_proxy is not None:
+            del os.environ['http_proxy']
+        super(HttprettyTestCase, self).setUp()
+
+    def tearDown(self):
+        if self.restore_proxy:
+            os.environ['http_proxy'] = self.restore_proxy
+        super(HttprettyTestCase, self).tearDown()
+
+
+def populate_dir(path, files):
+    if not os.path.exists(path):
+        os.makedirs(path)
+    ret = []
+    for (name, content) in files.items():
+        p = os.path.sep.join([path, name])
+        util.ensure_dir(os.path.dirname(p))
+        with open(p, "wb") as fp:
+            if isinstance(content, six.binary_type):
+                fp.write(content)
+            else:
+                fp.write(content.encode('utf-8'))
+            fp.close()
+        ret.append(p)
+
+    return ret
+
+
+def dir2dict(startdir, prefix=None):
+    flist = {}
+    if prefix is None:
+        prefix = startdir
+    for root, dirs, files in os.walk(startdir):
+        for fname in files:
+            fpath = os.path.join(root, fname)
+            key = fpath[len(prefix):]
+            flist[key] = util.load_file(fpath)
+    return flist
+
+
+def json_dumps(data):
+    # print data in nicely formatted json.
+    return json.dumps(data, indent=1, sort_keys=True,
+                      separators=(',', ': '))
+
+
+def wrap_and_call(prefix, mocks, func, *args, **kwargs):
+    """
+    call func(args, **kwargs) with mocks applied, then unapplies mocks
+    nicer to read than repeating dectorators on each function
+
+    prefix: prefix for mock names (e.g. 'cloudinit.stages.util') or None
+    mocks: dictionary of names (under 'prefix') to mock and either
+        a return value or a dictionary to pass to the mock.patch call
+    func: function to call with mocks applied
+    *args,**kwargs: arguments for 'func'
+
+    return_value: return from 'func'
+    """
+    delim = '.'
+    if prefix is None:
+        prefix = ''
+    prefix = prefix.rstrip(delim)
+    unwraps = []
+    for fname, kw in mocks.items():
+        if prefix:
+            fname = delim.join((prefix, fname))
+        if not isinstance(kw, dict):
+            kw = {'return_value': kw}
+        p = mock.patch(fname, **kw)
+        p.start()
+        unwraps.append(p)
+    try:
+        return func(*args, **kwargs)
+    finally:
+        for p in unwraps:
+            p.stop()
+
+
+try:
+    skipIf = unittest.skipIf
+except AttributeError:
+    # Python 2.6.  Doesn't have to be high fidelity.
+    def skipIf(condition, reason):
+        def decorator(func):
+            def wrapper(*args, **kws):
+                if condition:
+                    return func(*args, **kws)
+                else:
+                    print(reason, file=sys.stderr)
+            return wrapper
+        return decorator
+
+
+# older versions of mock do not have the useful 'assert_not_called'
+if not hasattr(mock.Mock, 'assert_not_called'):
+    def __mock_assert_not_called(mmock):
+        if mmock.call_count != 0:
+            msg = ("[citest] Expected '%s' to not have been called. "
+                   "Called %s times." %
+                   (mmock._mock_name or 'mock', mmock.call_count))
+            raise AssertionError(msg)
+    mock.Mock.assert_not_called = __mock_assert_not_called
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/tests/test_url_helper.py b/cloudinit/tests/test_url_helper.py
index 349110d9..b778a3a7 100644
--- a/cloudinit/tests/test_url_helper.py
+++ b/cloudinit/tests/test_url_helper.py
@@ -1,7 +1,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit.url_helper import oauth_headers
-from tests.unittests.helpers import CiTestCase, mock, skipIf
+from cloudinit.tests.helpers import CiTestCase, mock, skipIf
 
 
 try:
diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
deleted file mode 100644
index bf1dc5df..00000000
--- a/tests/unittests/helpers.py
+++ /dev/null
@@ -1,391 +0,0 @@
-# This file is part of cloud-init. See LICENSE file for license information.
-
-from __future__ import print_function
-
-import functools
-import json
-import logging
-import os
-import shutil
-import sys
-import tempfile
-import unittest
-
-import mock
-import six
-import unittest2
-
-try:
-    from contextlib import ExitStack
-except ImportError:
-    from contextlib2 import ExitStack
-
-from cloudinit import helpers as ch
-from cloudinit import util
-
-# Used for skipping tests
-SkipTest = unittest2.SkipTest
-
-# Used for detecting different python versions
-PY2 = False
-PY26 = False
-PY27 = False
-PY3 = False
-
-_PY_VER = sys.version_info
-_PY_MAJOR, _PY_MINOR, _PY_MICRO = _PY_VER[0:3]
-if (_PY_MAJOR, _PY_MINOR) <= (2, 6):
-    if (_PY_MAJOR, _PY_MINOR) == (2, 6):
-        PY26 = True
-    if (_PY_MAJOR, _PY_MINOR) >= (2, 0):
-        PY2 = True
-else:
-    if (_PY_MAJOR, _PY_MINOR) == (2, 7):
-        PY27 = True
-        PY2 = True
-    if (_PY_MAJOR, _PY_MINOR) >= (3, 0):
-        PY3 = True
-
-
-# Makes the old path start
-# with new base instead of whatever
-# it previously had
-def rebase_path(old_path, new_base):
-    if old_path.startswith(new_base):
-        # Already handled...
-        return old_path
-    # Retarget the base of that path
-    # to the new base instead of the
-    # old one...
-    path = os.path.join(new_base, old_path.lstrip("/"))
-    path = os.path.abspath(path)
-    return path
-
-
-# Can work on anything that takes a path as arguments
-def retarget_many_wrapper(new_base, am, old_func):
-    def wrapper(*args, **kwds):
-        n_args = list(args)
-        nam = am
-        if am == -1:
-            nam = len(n_args)
-        for i in range(0, nam):
-            path = args[i]
-            # patchOS() wraps various os and os.path functions, however in
-            # Python 3 some of these now accept file-descriptors (integers).
-            # That breaks rebase_path() so in lieu of a better solution, just
-            # don't rebase if we get a fd.
-            if isinstance(path, six.string_types):
-                n_args[i] = rebase_path(path, new_base)
-        return old_func(*n_args, **kwds)
-    return wrapper
-
-
-class TestCase(unittest2.TestCase):
-    def reset_global_state(self):
-        """Reset any global state to its original settings.
-
-        cloudinit caches some values in cloudinit.util.  Unit tests that
-        involved those cached paths were then subject to failure if the order
-        of invocation changed (LP: #1703697).
-
-        This function resets any of these global state variables to their
-        initial state.
-
-        In the future this should really be done with some registry that
-        can then be cleaned in a more obvious way.
-        """
-        util.PROC_CMDLINE = None
-        util._DNS_REDIRECT_IP = None
-        util._LSB_RELEASE = {}
-
-    def setUp(self):
-        super(unittest2.TestCase, self).setUp()
-        self.reset_global_state()
-
-
-class CiTestCase(TestCase):
-    """This is the preferred test case base class unless user
-       needs other test case classes below."""
-
-    # Subclass overrides for specific test behavior
-    # Whether or not a unit test needs logfile setup
-    with_logs = False
-
-    def setUp(self):
-        super(CiTestCase, self).setUp()
-        if self.with_logs:
-            # Create a log handler so unit tests can search expected logs.
-            self.logger = logging.getLogger()
-            self.logs = six.StringIO()
-            formatter = logging.Formatter('%(levelname)s: %(message)s')
-            handler = logging.StreamHandler(self.logs)
-            handler.setFormatter(formatter)
-            self.old_handlers = self.logger.handlers
-            self.logger.handlers = [handler]
-
-    def tearDown(self):
-        if self.with_logs:
-            # Remove the handler we setup
-            logging.getLogger().handlers = self.old_handlers
-        super(CiTestCase, self).tearDown()
-
-    def tmp_dir(self, dir=None, cleanup=True):
-        # return a full path to a temporary directory that will be cleaned up.
-        if dir is None:
-            tmpd = tempfile.mkdtemp(
-                prefix="ci-%s." % self.__class__.__name__)
-        else:
-            tmpd = tempfile.mkdtemp(dir=dir)
-        self.addCleanup(functools.partial(shutil.rmtree, tmpd))
-        return tmpd
-
-    def tmp_path(self, path, dir=None):
-        # return an absolute path to 'path' under dir.
-        # if dir is None, one will be created with tmp_dir()
-        # the file is not created or modified.
-        if dir is None:
-            dir = self.tmp_dir()
-        return os.path.normpath(os.path.abspath(os.path.join(dir, path)))
-
-
-class ResourceUsingTestCase(CiTestCase):
-    def setUp(self):
-        super(ResourceUsingTestCase, self).setUp()
-        self.resource_path = None
-
-    def resourceLocation(self, subname=None):
-        if self.resource_path is None:
-            paths = [
-                os.path.join('tests', 'data'),
-                os.path.join('data'),
-                os.path.join(os.pardir, 'tests', 'data'),
-                os.path.join(os.pardir, 'data'),
-            ]
-            for p in paths:
-                if os.path.isdir(p):
-                    self.resource_path = p
-                    break
-        self.assertTrue((self.resource_path and
-                         os.path.isdir(self.resource_path)),
-                        msg="Unable to locate test resource data path!")
-        if not subname:
-            return self.resource_path
-        return os.path.join(self.resource_path, subname)
-
-    def readResource(self, name):
-        where = self.resourceLocation(name)
-        with open(where, 'r') as fh:
-            return fh.read()
-
-    def getCloudPaths(self, ds=None):
-        tmpdir = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, tmpdir)
-        cp = ch.Paths({'cloud_dir': tmpdir,
-                       'templates_dir': self.resourceLocation()},
-                      ds=ds)
-        return cp
-
-
-class FilesystemMockingTestCase(ResourceUsingTestCase):
-    def setUp(self):
-        super(FilesystemMockingTestCase, self).setUp()
-        self.patched_funcs = ExitStack()
-
-    def tearDown(self):
-        self.patched_funcs.close()
-        ResourceUsingTestCase.tearDown(self)
-
-    def replicateTestRoot(self, example_root, target_root):
-        real_root = self.resourceLocation()
-        real_root = os.path.join(real_root, 'roots', example_root)
-        for (dir_path, _dirnames, filenames) in os.walk(real_root):
-            real_path = dir_path
-            make_path = rebase_path(real_path[len(real_root):], target_root)
-            util.ensure_dir(make_path)
-            for f in filenames:
-                real_path = util.abs_join(real_path, f)
-                make_path = util.abs_join(make_path, f)
-                shutil.copy(real_path, make_path)
-
-    def patchUtils(self, new_root):
-        patch_funcs = {
-            util: [('write_file', 1),
-                   ('append_file', 1),
-                   ('load_file', 1),
-                   ('ensure_dir', 1),
-                   ('chmod', 1),
-                   ('delete_dir_contents', 1),
-                   ('del_file', 1),
-                   ('sym_link', -1),
-                   ('copy', -1)],
-        }
-        for (mod, funcs) in patch_funcs.items():
-            for (f, am) in funcs:
-                func = getattr(mod, f)
-                trap_func = retarget_many_wrapper(new_root, am, func)
-                self.patched_funcs.enter_context(
-                    mock.patch.object(mod, f, trap_func))
-
-        # Handle subprocess calls
-        func = getattr(util, 'subp')
-
-        def nsubp(*_args, **_kwargs):
-            return ('', '')
-
-        self.patched_funcs.enter_context(
-            mock.patch.object(util, 'subp', nsubp))
-
-        def null_func(*_args, **_kwargs):
-            return None
-
-        for f in ['chownbyid', 'chownbyname']:
-            self.patched_funcs.enter_context(
-                mock.patch.object(util, f, null_func))
-
-    def patchOS(self, new_root):
-        patch_funcs = {
-            os.path: [('isfile', 1), ('exists', 1),
-                      ('islink', 1), ('isdir', 1)],
-            os: [('listdir', 1), ('mkdir', 1),
-                 ('lstat', 1), ('symlink', 2)],
-        }
-        for (mod, funcs) in patch_funcs.items():
-            for f, nargs in funcs:
-                func = getattr(mod, f)
-                trap_func = retarget_many_wrapper(new_root, nargs, func)
-                self.patched_funcs.enter_context(
-                    mock.patch.object(mod, f, trap_func))
-
-    def patchOpen(self, new_root):
-        trap_func = retarget_many_wrapper(new_root, 1, open)
-        name = 'builtins.open' if PY3 else '__builtin__.open'
-        self.patched_funcs.enter_context(mock.patch(name, trap_func))
-
-    def patchStdoutAndStderr(self, stdout=None, stderr=None):
-        if stdout is not None:
-            self.patched_funcs.enter_context(
-                mock.patch.object(sys, 'stdout', stdout))
-        if stderr is not None:
-            self.patched_funcs.enter_context(
-                mock.patch.object(sys, 'stderr', stderr))
-
-    def reRoot(self, root=None):
-        if root is None:
-            root = self.tmp_dir()
-        self.patchUtils(root)
-        self.patchOS(root)
-        return root
-
-
-class HttprettyTestCase(CiTestCase):
-    # necessary as http_proxy gets in the way of httpretty
-    # https://github.com/gabrielfalcao/HTTPretty/issues/122
-    def setUp(self):
-        self.restore_proxy = os.environ.get('http_proxy')
-        if self.restore_proxy is not None:
-            del os.environ['http_proxy']
-        super(HttprettyTestCase, self).setUp()
-
-    def tearDown(self):
-        if self.restore_proxy:
-            os.environ['http_proxy'] = self.restore_proxy
-        super(HttprettyTestCase, self).tearDown()
-
-
-def populate_dir(path, files):
-    if not os.path.exists(path):
-        os.makedirs(path)
-    ret = []
-    for (name, content) in files.items():
-        p = os.path.sep.join([path, name])
-        util.ensure_dir(os.path.dirname(p))
-        with open(p, "wb") as fp:
-            if isinstance(content, six.binary_type):
-                fp.write(content)
-            else:
-                fp.write(content.encode('utf-8'))
-            fp.close()
-        ret.append(p)
-
-    return ret
-
-
-def dir2dict(startdir, prefix=None):
-    flist = {}
-    if prefix is None:
-        prefix = startdir
-    for root, dirs, files in os.walk(startdir):
-        for fname in files:
-            fpath = os.path.join(root, fname)
-            key = fpath[len(prefix):]
-            flist[key] = util.load_file(fpath)
-    return flist
-
-
-def json_dumps(data):
-    # print data in nicely formatted json.
-    return json.dumps(data, indent=1, sort_keys=True,
-                      separators=(',', ': '))
-
-
-def wrap_and_call(prefix, mocks, func, *args, **kwargs):
-    """
-    call func(args, **kwargs) with mocks applied, then unapplies mocks
-    nicer to read than repeating dectorators on each function
-
-    prefix: prefix for mock names (e.g. 'cloudinit.stages.util') or None
-    mocks: dictionary of names (under 'prefix') to mock and either
-        a return value or a dictionary to pass to the mock.patch call
-    func: function to call with mocks applied
-    *args,**kwargs: arguments for 'func'
-
-    return_value: return from 'func'
-    """
-    delim = '.'
-    if prefix is None:
-        prefix = ''
-    prefix = prefix.rstrip(delim)
-    unwraps = []
-    for fname, kw in mocks.items():
-        if prefix:
-            fname = delim.join((prefix, fname))
-        if not isinstance(kw, dict):
-            kw = {'return_value': kw}
-        p = mock.patch(fname, **kw)
-        p.start()
-        unwraps.append(p)
-    try:
-        return func(*args, **kwargs)
-    finally:
-        for p in unwraps:
-            p.stop()
-
-
-try:
-    skipIf = unittest.skipIf
-except AttributeError:
-    # Python 2.6.  Doesn't have to be high fidelity.
-    def skipIf(condition, reason):
-        def decorator(func):
-            def wrapper(*args, **kws):
-                if condition:
-                    return func(*args, **kws)
-                else:
-                    print(reason, file=sys.stderr)
-            return wrapper
-        return decorator
-
-
-# older versions of mock do not have the useful 'assert_not_called'
-if not hasattr(mock.Mock, 'assert_not_called'):
-    def __mock_assert_not_called(mmock):
-        if mmock.call_count != 0:
-            msg = ("[citest] Expected '%s' to not have been called. "
-                   "Called %s times." %
-                   (mmock._mock_name or 'mock', mmock.call_count))
-            raise AssertionError(msg)
-    mock.Mock.assert_not_called = __mock_assert_not_called
-
-
-# vi: ts=4 expandtab
diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py
index 781f6d54..25878d7a 100644
--- a/tests/unittests/test__init__.py
+++ b/tests/unittests/test__init__.py
@@ -12,7 +12,7 @@ from cloudinit import settings
 from cloudinit import url_helper
 from cloudinit import util
 
-from .helpers import TestCase, CiTestCase, ExitStack, mock
+from cloudinit.tests.helpers import TestCase, CiTestCase, ExitStack, mock
 
 
 class FakeModule(handlers.Handler):
diff --git a/tests/unittests/test_atomic_helper.py b/tests/unittests/test_atomic_helper.py
index 515919d8..0101b0e3 100644
--- a/tests/unittests/test_atomic_helper.py
+++ b/tests/unittests/test_atomic_helper.py
@@ -6,7 +6,7 @@ import stat
 
 from cloudinit import atomic_helper
 
-from .helpers import CiTestCase
+from cloudinit.tests.helpers import CiTestCase
 
 
 class TestAtomicHelper(CiTestCase):
diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py
index dd9d0357..9751ed95 100644
--- a/tests/unittests/test_builtin_handlers.py
+++ b/tests/unittests/test_builtin_handlers.py
@@ -11,7 +11,7 @@ try:
 except ImportError:
     import mock
 
-from . import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 from cloudinit import handlers
 from cloudinit import helpers
diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py
index 12f01852..495bdc9f 100644
--- a/tests/unittests/test_cli.py
+++ b/tests/unittests/test_cli.py
@@ -2,7 +2,7 @@
 
 import six
 
-from . import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 from cloudinit.cmd import main as cli
 
diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/test_cs_util.py
index b8f5031c..ee88520d 100644
--- a/tests/unittests/test_cs_util.py
+++ b/tests/unittests/test_cs_util.py
@@ -2,7 +2,7 @@
 
 from __future__ import print_function
 
-from . import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 from cloudinit.cs_utils import Cepko
 
diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py
index 4ad86bb6..6d621d26 100644
--- a/tests/unittests/test_data.py
+++ b/tests/unittests/test_data.py
@@ -27,7 +27,7 @@ from cloudinit import stages
 from cloudinit import user_data as ud
 from cloudinit import util
 
-from . import helpers
+from cloudinit.tests import helpers
 
 
 INSTANCE_ID = "i-testing"
diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py
index 996560e4..82ee9714 100644
--- a/tests/unittests/test_datasource/test_aliyun.py
+++ b/tests/unittests/test_datasource/test_aliyun.py
@@ -5,9 +5,9 @@ import httpretty
 import mock
 import os
 
-from .. import helpers as test_helpers
 from cloudinit import helpers
 from cloudinit.sources import DataSourceAliYun as ay
+from cloudinit.tests import helpers as test_helpers
 
 DEFAULT_METADATA = {
     'instance-id': 'aliyun-test-vm-00',
diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py
index 9c46abc1..3b274d90 100644
--- a/tests/unittests/test_datasource/test_altcloud.py
+++ b/tests/unittests/test_datasource/test_altcloud.py
@@ -18,7 +18,7 @@ import tempfile
 from cloudinit import helpers
 from cloudinit import util
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 import cloudinit.sources.DataSourceAltCloud as dsac
 
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 20e70fb7..0a117771 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -6,8 +6,8 @@ from cloudinit.sources import DataSourceAzure as dsaz
 from cloudinit.util import find_freebsd_part
 from cloudinit.util import get_path_dev_freebsd
 
-from ..helpers import (CiTestCase, TestCase, populate_dir, mock,
-                       ExitStack, PY26, SkipTest)
+from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock,
+                                     ExitStack, PY26, SkipTest)
 
 import crypt
 import os
@@ -871,6 +871,7 @@ class TestLoadAzureDsDir(CiTestCase):
 
 
 class TestReadAzureOvf(TestCase):
+
     def test_invalid_xml_raises_non_azure_ds(self):
         invalid_xml = "<foo>" + construct_valid_ovf_env(data={})
         self.assertRaises(dsaz.BrokenAzureDataSource,
@@ -1079,6 +1080,7 @@ class TestCanDevBeReformatted(CiTestCase):
 
 
 class TestAzureNetExists(CiTestCase):
+
     def test_azure_net_must_exist_for_legacy_objpkl(self):
         """DataSourceAzureNet must exist for old obj.pkl files
            that reference it."""
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
index b2d2971b..80ce003d 100644
--- a/tests/unittests/test_datasource/test_azure_helper.py
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -3,7 +3,7 @@
 import os
 
 from cloudinit.sources.helpers import azure as azure_helper
-from ..helpers import ExitStack, mock, TestCase
+from cloudinit.tests.helpers import ExitStack, mock, TestCase
 
 
 GOAL_STATE_TEMPLATE = """\
diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py
index 5997102c..e4c59907 100644
--- a/tests/unittests/test_datasource/test_cloudsigma.py
+++ b/tests/unittests/test_datasource/test_cloudsigma.py
@@ -6,7 +6,7 @@ from cloudinit.cs_utils import Cepko
 from cloudinit import sources
 from cloudinit.sources import DataSourceCloudSigma
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 SERVER_CONTEXT = {
     "cpu": 1000,
diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py
index e94aad61..2dc90305 100644
--- a/tests/unittests/test_datasource/test_cloudstack.py
+++ b/tests/unittests/test_datasource/test_cloudstack.py
@@ -3,7 +3,7 @@
 from cloudinit import helpers
 from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack
 
-from ..helpers import TestCase, mock, ExitStack
+from cloudinit.tests.helpers import TestCase, mock, ExitStack
 
 
 class TestCloudStackPasswordFetching(TestCase):
diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
index 4802f105..80b9c650 100644
--- a/tests/unittests/test_datasource/test_common.py
+++ b/tests/unittests/test_datasource/test_common.py
@@ -24,7 +24,7 @@ from cloudinit.sources import (
 )
 from cloudinit.sources import DataSourceNone as DSNone
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 DEFAULT_LOCAL = [
     Azure.DataSourceAzure,
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index 337be667..237c189b 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -15,7 +15,7 @@ from cloudinit.sources import DataSourceConfigDrive as ds
 from cloudinit.sources.helpers import openstack
 from cloudinit import util
 
-from ..helpers import TestCase, ExitStack, mock
+from cloudinit.tests.helpers import TestCase, ExitStack, mock
 
 
 PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n'
diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py
index e97a679a..f264f361 100644
--- a/tests/unittests/test_datasource/test_digitalocean.py
+++ b/tests/unittests/test_datasource/test_digitalocean.py
@@ -13,7 +13,7 @@ from cloudinit import settings
 from cloudinit.sources import DataSourceDigitalOcean
 from cloudinit.sources.helpers import digitalocean
 
-from ..helpers import mock, TestCase
+from cloudinit.tests.helpers import mock, TestCase
 
 DO_MULTIPLE_KEYS = ["ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@do.co",
                     "ssh-rsa AAAAB3NzaC1yc2EAAAA... test2@do.co"]
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index b7a84e21..9fb90483 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -4,9 +4,9 @@ import copy
 import httpretty
 import mock
 
-from .. import helpers as test_helpers
 from cloudinit import helpers
 from cloudinit.sources import DataSourceEc2 as ec2
+from cloudinit.tests import helpers as test_helpers
 
 
 # collected from api version 2016-09-02/ with
diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py
index ad608bec..50e49a10 100644
--- a/tests/unittests/test_datasource/test_gce.py
+++ b/tests/unittests/test_datasource/test_gce.py
@@ -15,7 +15,7 @@ from cloudinit import helpers
 from cloudinit import settings
 from cloudinit.sources import DataSourceGCE
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 
 GCE_META = {
diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py
index c1911bf4..289c6a40 100644
--- a/tests/unittests/test_datasource/test_maas.py
+++ b/tests/unittests/test_datasource/test_maas.py
@@ -8,7 +8,7 @@ import yaml
 
 from cloudinit.sources import DataSourceMAAS
 from cloudinit import url_helper
-from ..helpers import TestCase, populate_dir
+from cloudinit.tests.helpers import TestCase, populate_dir
 
 try:
     from unittest import mock
diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py
index ff294395..fea9156b 100644
--- a/tests/unittests/test_datasource/test_nocloud.py
+++ b/tests/unittests/test_datasource/test_nocloud.py
@@ -3,7 +3,7 @@
 from cloudinit import helpers
 from cloudinit.sources import DataSourceNoCloud
 from cloudinit import util
-from ..helpers import TestCase, populate_dir, mock, ExitStack
+from cloudinit.tests.helpers import TestCase, populate_dir, mock, ExitStack
 
 import os
 import shutil
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index b0f8e435..e7d55692 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -3,7 +3,7 @@
 from cloudinit import helpers
 from cloudinit.sources import DataSourceOpenNebula as ds
 from cloudinit import util
-from ..helpers import mock, populate_dir, TestCase
+from cloudinit.tests.helpers import mock, populate_dir, TestCase
 
 import os
 import pwd
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index c2905d1a..177e9808 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -9,7 +9,7 @@ import httpretty as hp
 import json
 import re
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 from six.moves.urllib.parse import urlparse
 from six import StringIO
diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py
index 477cf8ed..9dbf4dd9 100644
--- a/tests/unittests/test_datasource/test_ovf.py
+++ b/tests/unittests/test_datasource/test_ovf.py
@@ -6,7 +6,7 @@
 
 import base64
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 from cloudinit.sources import DataSourceOVF as dsovf
 
diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py
index 65d83ad7..436df9ee 100644
--- a/tests/unittests/test_datasource/test_scaleway.py
+++ b/tests/unittests/test_datasource/test_scaleway.py
@@ -9,7 +9,7 @@ from cloudinit import helpers
 from cloudinit import settings
 from cloudinit.sources import DataSourceScaleway
 
-from ..helpers import mock, HttprettyTestCase, TestCase
+from cloudinit.tests.helpers import mock, HttprettyTestCase, TestCase
 
 
 class DataResponses(object):
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index e3c99bbb..933d5b63 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -33,7 +33,7 @@ import six
 from cloudinit import helpers as c_helpers
 from cloudinit.util import b64e
 
-from ..helpers import mock, FilesystemMockingTestCase, TestCase
+from cloudinit.tests.helpers import mock, FilesystemMockingTestCase, TestCase
 
 SDC_NICS = json.loads("""
 [
diff --git a/tests/unittests/test_distros/test_arch.py b/tests/unittests/test_distros/test_arch.py
index 3d4c9a70..a95ba3b5 100644
--- a/tests/unittests/test_distros/test_arch.py
+++ b/tests/unittests/test_distros/test_arch.py
@@ -3,7 +3,7 @@
 from cloudinit.distros.arch import _render_network
 from cloudinit import util
 
-from ..helpers import (CiTestCase, dir2dict)
+from cloudinit.tests.helpers import (CiTestCase, dir2dict)
 
 from . import _get_distro
 
diff --git a/tests/unittests/test_distros/test_create_users.py b/tests/unittests/test_distros/test_create_users.py
index 1d02f7bd..aa13670a 100644
--- a/tests/unittests/test_distros/test_create_users.py
+++ b/tests/unittests/test_distros/test_create_users.py
@@ -1,7 +1,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit import distros
-from ..helpers import (TestCase, mock)
+from cloudinit.tests.helpers import (TestCase, mock)
 
 
 class MyBaseDistro(distros.Distro):
diff --git a/tests/unittests/test_distros/test_debian.py b/tests/unittests/test_distros/test_debian.py
index 72d3aad6..da16a797 100644
--- a/tests/unittests/test_distros/test_debian.py
+++ b/tests/unittests/test_distros/test_debian.py
@@ -2,7 +2,7 @@
 
 from cloudinit import distros
 from cloudinit import util
-from ..helpers import (FilesystemMockingTestCase, mock)
+from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock)
 
 
 @mock.patch("cloudinit.distros.debian.util.subp")
diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py
index b355a19e..791fe612 100644
--- a/tests/unittests/test_distros/test_generic.py
+++ b/tests/unittests/test_distros/test_generic.py
@@ -3,7 +3,7 @@
 from cloudinit import distros
 from cloudinit import util
 
-from .. import helpers
+from cloudinit.tests import helpers
 
 import os
 import shutil
diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py
index 6d89dba8..c4bd11bc 100644
--- a/tests/unittests/test_distros/test_netconfig.py
+++ b/tests/unittests/test_distros/test_netconfig.py
@@ -12,7 +12,7 @@ try:
 except ImportError:
     from contextlib2 import ExitStack
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 from cloudinit import distros
 from cloudinit.distros.parsers.sys_conf import SysConf
diff --git a/tests/unittests/test_distros/test_opensuse.py b/tests/unittests/test_distros/test_opensuse.py
index bdb1d633..b9bb9b3e 100644
--- a/tests/unittests/test_distros/test_opensuse.py
+++ b/tests/unittests/test_distros/test_opensuse.py
@@ -1,6 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-from ..helpers import CiTestCase
+from cloudinit.tests.helpers import CiTestCase
 
 from . import _get_distro
 
diff --git a/tests/unittests/test_distros/test_resolv.py b/tests/unittests/test_distros/test_resolv.py
index 97168cf9..68ea0083 100644
--- a/tests/unittests/test_distros/test_resolv.py
+++ b/tests/unittests/test_distros/test_resolv.py
@@ -3,7 +3,7 @@
 from cloudinit.distros.parsers import resolv_conf
 from cloudinit.distros import rhel_util
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 import re
 import tempfile
diff --git a/tests/unittests/test_distros/test_sles.py b/tests/unittests/test_distros/test_sles.py
index c656aacc..33e3c457 100644
--- a/tests/unittests/test_distros/test_sles.py
+++ b/tests/unittests/test_distros/test_sles.py
@@ -1,6 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-from ..helpers import CiTestCase
+from cloudinit.tests.helpers import CiTestCase
 
 from . import _get_distro
 
diff --git a/tests/unittests/test_distros/test_sysconfig.py b/tests/unittests/test_distros/test_sysconfig.py
index 235ecebb..c1d5b693 100644
--- a/tests/unittests/test_distros/test_sysconfig.py
+++ b/tests/unittests/test_distros/test_sysconfig.py
@@ -4,7 +4,7 @@ import re
 
 from cloudinit.distros.parsers.sys_conf import SysConf
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 
 # Lots of good examples @
diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py
index 88746e0a..0fa9cdb5 100644
--- a/tests/unittests/test_distros/test_user_data_normalize.py
+++ b/tests/unittests/test_distros/test_user_data_normalize.py
@@ -5,7 +5,7 @@ from cloudinit.distros import ug_util
 from cloudinit import helpers
 from cloudinit import settings
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 import mock
 
 
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 8ccfe55c..1a81a89e 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -6,7 +6,8 @@ from uuid import uuid4
 
 from cloudinit import safeyaml
 from cloudinit import util
-from .helpers import CiTestCase, dir2dict, json_dumps, populate_dir
+from cloudinit.tests.helpers import (
+    CiTestCase, dir2dict, json_dumps, populate_dir)
 
 UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu "
                "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux")
diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py
index 65fdb519..af78997f 100644
--- a/tests/unittests/test_ec2_util.py
+++ b/tests/unittests/test_ec2_util.py
@@ -2,7 +2,7 @@
 
 import httpretty as hp
 
-from . import helpers
+from cloudinit.tests import helpers
 
 from cloudinit import ec2_utils as eu
 from cloudinit import url_helper as uh
diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py
index 13137f6d..6364d38e 100644
--- a/tests/unittests/test_filters/test_launch_index.py
+++ b/tests/unittests/test_filters/test_launch_index.py
@@ -2,7 +2,7 @@
 
 import copy
 
-from .. import helpers
+from cloudinit.tests import helpers
 
 from six.moves import filterfalse
 
diff --git a/tests/unittests/test_handler/test_handler_apt_conf_v1.py b/tests/unittests/test_handler/test_handler_apt_conf_v1.py
index 554277ff..83f962a9 100644
--- a/tests/unittests/test_handler/test_handler_apt_conf_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_conf_v1.py
@@ -3,7 +3,7 @@
 from cloudinit.config import cc_apt_configure
 from cloudinit import util
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 import copy
 import os
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
index f53ddbb2..d2b96f0b 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
@@ -24,7 +24,7 @@ from cloudinit.sources import DataSourceNone
 
 from cloudinit.distros.debian import Distro
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 LOG = logging.getLogger(__name__)
 
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py
index 1ca915b4..f7608c28 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py
@@ -24,7 +24,7 @@ from cloudinit.sources import DataSourceNone
 
 from cloudinit.distros.debian import Distro
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 LOG = logging.getLogger(__name__)
 
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py
index 12502d05..3a3f95ca 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py
@@ -20,7 +20,7 @@ from cloudinit.config import cc_apt_configure
 from cloudinit import gpg
 from cloudinit import util
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK-----
 Version: GnuPG v1
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py
index 292d3f59..7bb1b7c4 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v3.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py
@@ -28,7 +28,7 @@ from cloudinit import util
 from cloudinit.config import cc_apt_configure
 from cloudinit.sources import DataSourceNone
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 EXPECTEDKEY = u"""-----BEGIN PGP PUBLIC KEY BLOCK-----
 Version: GnuPG v1
diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py
index 7cee2c3f..06e14db0 100644
--- a/tests/unittests/test_handler/test_handler_ca_certs.py
+++ b/tests/unittests/test_handler/test_handler_ca_certs.py
@@ -5,7 +5,7 @@ from cloudinit.config import cc_ca_certs
 from cloudinit import helpers
 from cloudinit import util
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 import logging
 import shutil
diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py
index 6a152ea2..e5785cfd 100644
--- a/tests/unittests/test_handler/test_handler_chef.py
+++ b/tests/unittests/test_handler/test_handler_chef.py
@@ -14,7 +14,7 @@ from cloudinit import helpers
 from cloudinit.sources import DataSourceNone
 from cloudinit import util
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 LOG = logging.getLogger(__name__)
 
diff --git a/tests/unittests/test_handler/test_handler_debug.py b/tests/unittests/test_handler/test_handler_debug.py
index 1873c3e1..787ba350 100644
--- a/tests/unittests/test_handler/test_handler_debug.py
+++ b/tests/unittests/test_handler/test_handler_debug.py
@@ -11,7 +11,7 @@ from cloudinit import util
 
 from cloudinit.sources import DataSourceNone
 
-from ..helpers import (FilesystemMockingTestCase, mock)
+from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock)
 
 import logging
 import shutil
diff --git a/tests/unittests/test_handler/test_handler_disk_setup.py b/tests/unittests/test_handler/test_handler_disk_setup.py
index 8a6d49ed..5afcacaf 100644
--- a/tests/unittests/test_handler/test_handler_disk_setup.py
+++ b/tests/unittests/test_handler/test_handler_disk_setup.py
@@ -3,7 +3,7 @@
 import random
 
 from cloudinit.config import cc_disk_setup
-from ..helpers import CiTestCase, ExitStack, mock, TestCase
+from cloudinit.tests.helpers import CiTestCase, ExitStack, mock, TestCase
 
 
 class TestIsDiskUsed(TestCase):
diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py
index c5fc8c9b..a3e46351 100644
--- a/tests/unittests/test_handler/test_handler_growpart.py
+++ b/tests/unittests/test_handler/test_handler_growpart.py
@@ -4,7 +4,7 @@ from cloudinit import cloud
 from cloudinit.config import cc_growpart
 from cloudinit import util
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 import errno
 import logging
diff --git a/tests/unittests/test_handler/test_handler_landscape.py b/tests/unittests/test_handler/test_handler_landscape.py
index 7c247fa9..db92a7e2 100644
--- a/tests/unittests/test_handler/test_handler_landscape.py
+++ b/tests/unittests/test_handler/test_handler_landscape.py
@@ -1,9 +1,10 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit.config import cc_landscape
-from cloudinit.sources import DataSourceNone
 from cloudinit import (distros, helpers, cloud, util)
-from ..helpers import FilesystemMockingTestCase, mock, wrap_and_call
+from cloudinit.sources import DataSourceNone
+from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock,
+                                     wrap_and_call)
 
 from configobj import ConfigObj
 import logging
diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py
index a789db32..e29a06f9 100644
--- a/tests/unittests/test_handler/test_handler_locale.py
+++ b/tests/unittests/test_handler/test_handler_locale.py
@@ -13,7 +13,7 @@ from cloudinit import util
 
 from cloudinit.sources import DataSourceNoCloud
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 from configobj import ConfigObj
 
diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py
index 351226bf..f132a778 100644
--- a/tests/unittests/test_handler/test_handler_lxd.py
+++ b/tests/unittests/test_handler/test_handler_lxd.py
@@ -3,7 +3,7 @@
 from cloudinit.config import cc_lxd
 from cloudinit.sources import DataSourceNoCloud
 from cloudinit import (distros, helpers, cloud)
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 import logging
 
diff --git a/tests/unittests/test_handler/test_handler_mcollective.py b/tests/unittests/test_handler/test_handler_mcollective.py
index 2a9f3823..7eec7352 100644
--- a/tests/unittests/test_handler/test_handler_mcollective.py
+++ b/tests/unittests/test_handler/test_handler_mcollective.py
@@ -4,7 +4,7 @@ from cloudinit import (cloud, distros, helpers, util)
 from cloudinit.config import cc_mcollective
 from cloudinit.sources import DataSourceNoCloud
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 import configobj
 import logging
diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py
index 650ca0ec..fe492d4b 100644
--- a/tests/unittests/test_handler/test_handler_mounts.py
+++ b/tests/unittests/test_handler/test_handler_mounts.py
@@ -6,7 +6,7 @@ import tempfile
 
 from cloudinit.config import cc_mounts
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 try:
     from unittest import mock
diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
index 83d5faa2..4f291248 100644
--- a/tests/unittests/test_handler/test_handler_ntp.py
+++ b/tests/unittests/test_handler/test_handler_ntp.py
@@ -3,7 +3,7 @@
 from cloudinit.config import cc_ntp
 from cloudinit.sources import DataSourceNone
 from cloudinit import (distros, helpers, cloud, util)
-from ..helpers import FilesystemMockingTestCase, mock, skipIf
+from cloudinit.tests.helpers import FilesystemMockingTestCase, mock, skipIf
 
 
 import os
diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py
index e382210d..85a0fe0a 100644
--- a/tests/unittests/test_handler/test_handler_power_state.py
+++ b/tests/unittests/test_handler/test_handler_power_state.py
@@ -4,8 +4,8 @@ import sys
 
 from cloudinit.config import cc_power_state_change as psc
 
-from .. import helpers as t_help
-from ..helpers import mock
+from cloudinit.tests import helpers as t_help
+from cloudinit.tests.helpers import mock
 
 
 class TestLoadPowerState(t_help.TestCase):
diff --git a/tests/unittests/test_handler/test_handler_puppet.py b/tests/unittests/test_handler/test_handler_puppet.py
index 805c76ba..0b6e3b58 100644
--- a/tests/unittests/test_handler/test_handler_puppet.py
+++ b/tests/unittests/test_handler/test_handler_puppet.py
@@ -3,7 +3,7 @@
 from cloudinit.config import cc_puppet
 from cloudinit.sources import DataSourceNone
 from cloudinit import (distros, helpers, cloud, util)
-from ..helpers import CiTestCase, mock
+from cloudinit.tests.helpers import CiTestCase, mock
 
 import logging
 
diff --git a/tests/unittests/test_handler/test_handler_rsyslog.py b/tests/unittests/test_handler/test_handler_rsyslog.py
index cca06678..8c8e2838 100644
--- a/tests/unittests/test_handler/test_handler_rsyslog.py
+++ b/tests/unittests/test_handler/test_handler_rsyslog.py
@@ -9,7 +9,7 @@ from cloudinit.config.cc_rsyslog import (
     parse_remotes_line, remotes_to_rsyslog_cfg)
 from cloudinit import util
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 
 class TestLoadConfig(t_help.TestCase):
diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py
index 7880ee72..374c1d31 100644
--- a/tests/unittests/test_handler/test_handler_runcmd.py
+++ b/tests/unittests/test_handler/test_handler_runcmd.py
@@ -3,7 +3,7 @@
 from cloudinit.config import cc_runcmd
 from cloudinit.sources import DataSourceNone
 from cloudinit import (distros, helpers, cloud, util)
-from ..helpers import FilesystemMockingTestCase, skipIf
+from cloudinit.tests.helpers import FilesystemMockingTestCase, skipIf
 
 import logging
 import os
diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py
index e5e607fb..f60dedc2 100644
--- a/tests/unittests/test_handler/test_handler_seed_random.py
+++ b/tests/unittests/test_handler/test_handler_seed_random.py
@@ -22,7 +22,7 @@ from cloudinit import util
 
 from cloudinit.sources import DataSourceNone
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 import logging
 
diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py
index 8165bf9a..abdc17e7 100644
--- a/tests/unittests/test_handler/test_handler_set_hostname.py
+++ b/tests/unittests/test_handler/test_handler_set_hostname.py
@@ -7,7 +7,7 @@ from cloudinit import distros
 from cloudinit import helpers
 from cloudinit import util
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 from configobj import ConfigObj
 import logging
diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py
index e4d07622..76b79c29 100644
--- a/tests/unittests/test_handler/test_handler_snappy.py
+++ b/tests/unittests/test_handler/test_handler_snappy.py
@@ -7,9 +7,9 @@ from cloudinit.config.cc_snap_config import (
 from cloudinit import (distros, helpers, cloud, util)
 from cloudinit.config.cc_snap_config import handle as snap_handle
 from cloudinit.sources import DataSourceNone
-from ..helpers import FilesystemMockingTestCase, mock
+from cloudinit.tests.helpers import FilesystemMockingTestCase, mock
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 import logging
 import os
diff --git a/tests/unittests/test_handler/test_handler_spacewalk.py b/tests/unittests/test_handler/test_handler_spacewalk.py
index 28b5892a..ddbf4a79 100644
--- a/tests/unittests/test_handler/test_handler_spacewalk.py
+++ b/tests/unittests/test_handler/test_handler_spacewalk.py
@@ -3,7 +3,7 @@
 from cloudinit.config import cc_spacewalk
 from cloudinit import util
 
-from .. import helpers
+from cloudinit.tests import helpers
 
 import logging
 
diff --git a/tests/unittests/test_handler/test_handler_timezone.py b/tests/unittests/test_handler/test_handler_timezone.py
index c30fbdfe..27eedded 100644
--- a/tests/unittests/test_handler/test_handler_timezone.py
+++ b/tests/unittests/test_handler/test_handler_timezone.py
@@ -13,7 +13,7 @@ from cloudinit import util
 
 from cloudinit.sources import DataSourceNoCloud
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 from configobj import ConfigObj
 import logging
diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py
index 1129e77d..7fa8fd21 100644
--- a/tests/unittests/test_handler/test_handler_write_files.py
+++ b/tests/unittests/test_handler/test_handler_write_files.py
@@ -4,7 +4,7 @@ from cloudinit.config.cc_write_files import write_files, decode_perms
 from cloudinit import log as logging
 from cloudinit import util
 
-from ..helpers import CiTestCase, FilesystemMockingTestCase
+from cloudinit.tests.helpers import CiTestCase, FilesystemMockingTestCase
 
 import base64
 import gzip
diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py
index c4396df5..b7adbe50 100644
--- a/tests/unittests/test_handler/test_handler_yum_add_repo.py
+++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py
@@ -3,7 +3,7 @@
 from cloudinit.config import cc_yum_add_repo
 from cloudinit import util
 
-from .. import helpers
+from cloudinit.tests import helpers
 
 try:
     from configparser import ConfigParser
diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
index 640f11d4..6137e3cf 100644
--- a/tests/unittests/test_handler/test_schema.py
+++ b/tests/unittests/test_handler/test_schema.py
@@ -6,7 +6,7 @@ from cloudinit.config.schema import (
     validate_cloudconfig_schema, main)
 from cloudinit.util import write_file
 
-from ..helpers import CiTestCase, mock, skipIf
+from cloudinit.tests.helpers import CiTestCase, mock, skipIf
 
 from copy import copy
 from six import StringIO
diff --git a/tests/unittests/test_helpers.py b/tests/unittests/test_helpers.py
index f1979e89..2e4582a0 100644
--- a/tests/unittests/test_helpers.py
+++ b/tests/unittests/test_helpers.py
@@ -4,7 +4,7 @@
 
 import os
 
-from . import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 from cloudinit import sources
 
diff --git a/tests/unittests/test_log.py b/tests/unittests/test_log.py
index 68fb4b8d..cd6296d6 100644
--- a/tests/unittests/test_log.py
+++ b/tests/unittests/test_log.py
@@ -2,9 +2,9 @@
 
 """Tests for cloudinit.log """
 
-from .helpers import CiTestCase
 from cloudinit.analyze.dump import CLOUD_INIT_ASCTIME_FMT
 from cloudinit import log as ci_logging
+from cloudinit.tests.helpers import CiTestCase
 import datetime
 import logging
 import six
diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py
index 0658b6b4..f51358da 100644
--- a/tests/unittests/test_merging.py
+++ b/tests/unittests/test_merging.py
@@ -1,6 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-from . import helpers
+from cloudinit.tests import helpers
 
 from cloudinit.handlers import cloud_config
 from cloudinit.handlers import (CONTENT_START, CONTENT_END)
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index f251024b..c10ef905 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -11,10 +11,10 @@ from cloudinit.net import sysconfig
 from cloudinit.sources.helpers import openstack
 from cloudinit import util
 
-from .helpers import CiTestCase
-from .helpers import dir2dict
-from .helpers import mock
-from .helpers import populate_dir
+from cloudinit.tests.helpers import CiTestCase
+from cloudinit.tests.helpers import dir2dict
+from cloudinit.tests.helpers import mock
+from cloudinit.tests.helpers import populate_dir
 
 import base64
 import copy
diff --git a/tests/unittests/test_pathprefix2dict.py b/tests/unittests/test_pathprefix2dict.py
index a4ae284f..abbb29b8 100644
--- a/tests/unittests/test_pathprefix2dict.py
+++ b/tests/unittests/test_pathprefix2dict.py
@@ -2,7 +2,7 @@
 
 from cloudinit import util
 
-from .helpers import TestCase, populate_dir
+from cloudinit.tests.helpers import TestCase, populate_dir
 
 import shutil
 import tempfile
diff --git a/tests/unittests/test_registry.py b/tests/unittests/test_registry.py
index acf0bf4f..2b625026 100644
--- a/tests/unittests/test_registry.py
+++ b/tests/unittests/test_registry.py
@@ -2,7 +2,7 @@
 
 from cloudinit.registry import DictRegistry
 
-from .helpers import (mock, TestCase)
+from cloudinit.tests.helpers import (mock, TestCase)
 
 
 class TestDictRegistry(TestCase):
diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py
index f3b8f992..571420ed 100644
--- a/tests/unittests/test_reporting.py
+++ b/tests/unittests/test_reporting.py
@@ -8,7 +8,7 @@ from cloudinit.reporting import handlers
 
 import mock
 
-from .helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 
 def _fake_registry():
diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py
index ca14cd46..e9d5702a 100644
--- a/tests/unittests/test_rh_subscription.py
+++ b/tests/unittests/test_rh_subscription.py
@@ -7,7 +7,7 @@ import logging
 from cloudinit.config import cc_rh_subscription
 from cloudinit import util
 
-from .helpers import TestCase, mock
+from cloudinit.tests.helpers import TestCase, mock
 
 
 class GoodTests(TestCase):
diff --git a/tests/unittests/test_runs/test_merge_run.py b/tests/unittests/test_runs/test_merge_run.py
index 65895273..add93653 100644
--- a/tests/unittests/test_runs/test_merge_run.py
+++ b/tests/unittests/test_runs/test_merge_run.py
@@ -4,7 +4,7 @@ import os
 import shutil
 import tempfile
 
-from .. import helpers
+from cloudinit.tests import helpers
 
 from cloudinit.settings import PER_INSTANCE
 from cloudinit import stages
diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py
index 55f15b55..5cf666fe 100644
--- a/tests/unittests/test_runs/test_simple_run.py
+++ b/tests/unittests/test_runs/test_simple_run.py
@@ -4,7 +4,7 @@ import os
 import shutil
 import tempfile
 
-from .. import helpers
+from cloudinit.tests import helpers
 
 from cloudinit.settings import PER_INSTANCE
 from cloudinit import stages
diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py
index 991f45a6..2a8e6abe 100644
--- a/tests/unittests/test_sshutil.py
+++ b/tests/unittests/test_sshutil.py
@@ -2,8 +2,8 @@
 
 from mock import patch
 
-from . import helpers as test_helpers
 from cloudinit import ssh_util
+from cloudinit.tests import helpers as test_helpers
 
 
 VALID_CONTENT = {
@@ -57,6 +57,7 @@ TEST_OPTIONS = (
 
 
 class TestAuthKeyLineParser(test_helpers.TestCase):
+
     def test_simple_parse(self):
         # test key line with common 3 fields (keytype, base64, comment)
         parser = ssh_util.AuthKeyLineParser()
diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py
index 4e627826..b911d929 100644
--- a/tests/unittests/test_templating.py
+++ b/tests/unittests/test_templating.py
@@ -6,7 +6,7 @@
 
 from __future__ import print_function
 
-from . import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 import textwrap
 
 from cloudinit import templater
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index 5f11c88f..3e4154ca 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -12,7 +12,7 @@ import six
 import yaml
 
 from cloudinit import importer, util
-from . import helpers
+from cloudinit.tests import helpers
 
 try:
     from unittest import mock
diff --git a/tests/unittests/test_version.py b/tests/unittests/test_version.py
index 1662ce09..d012f69d 100644
--- a/tests/unittests/test_version.py
+++ b/tests/unittests/test_version.py
@@ -1,6 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-from .helpers import CiTestCase
+from cloudinit.tests.helpers import CiTestCase
 from cloudinit import version
 
 
diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py
index 03b36d31..d8651077 100644
--- a/tests/unittests/test_vmware_config_file.py
+++ b/tests/unittests/test_vmware_config_file.py
@@ -8,10 +8,10 @@
 import logging
 import sys
 
-from .helpers import CiTestCase
 from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProtoEnum
 from cloudinit.sources.helpers.vmware.imc.config import Config
 from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile
+from cloudinit.tests.helpers import CiTestCase
 
 logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
 logger = logging.getLogger(__name__)
-- 
cgit v1.2.3


From 5582e4a266118b63ff86b6258b23d66df6d129d5 Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Wed, 6 Sep 2017 12:11:13 -0600
Subject: tests: mock missed openstack metadata uri network_data.json

This missed mock in test_openstack resulted in a costly unit test timeout.

LP: #1714376
---
 tests/unittests/test_datasource/test_openstack.py | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'tests/unittests/test_datasource')

diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index 177e9808..ed367e05 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -57,6 +57,8 @@ OS_FILES = {
     'openstack/content/0000': CONTENT_0,
     'openstack/content/0001': CONTENT_1,
     'openstack/latest/meta_data.json': json.dumps(OSTACK_META),
+    'openstack/latest/network_data.json': json.dumps(
+        {'links': [], 'networks': [], 'services': []}),
     'openstack/latest/user_data': USER_DATA,
     'openstack/latest/vendor_data.json': json.dumps(VENDOR_DATA),
 }
@@ -68,6 +70,7 @@ EC2_VERSIONS = [
 ]
 
 
+# TODO _register_uris should leverage test_ec2.register_mock_metaserver.
 def _register_uris(version, ec2_files, ec2_meta, os_files):
     """Registers a set of url patterns into httpretty that will mimic the
     same data returned by the openstack metadata service (and ec2 service)."""
-- 
cgit v1.2.3


From 409918f9ba83e45e9bc5cc0b6c589e2fc8ae9b60 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@brickies.net>
Date: Tue, 29 Aug 2017 09:59:20 -0400
Subject: Use /run/cloud-init for tempfile operations.

During boot, the usage of /tmp is not safe.  In systemd systems,
systemd-tmpfiles-clean may run at any point and clear out a temp file
while cloud-init is using it.  The solution here is to use
/run/cloud-init/tmp.

LP: #1707222
---
 cloudinit/config/cc_bootcmd.py                     |  3 +-
 cloudinit/config/cc_chef.py                        |  3 +-
 cloudinit/config/cc_snappy.py                      |  4 +-
 cloudinit/net/dhcp.py                              |  3 +-
 cloudinit/sources/helpers/azure.py                 |  4 +-
 cloudinit/temp_utils.py                            | 93 ++++++++++++++++++++++
 cloudinit/util.py                                  | 36 +--------
 packages/bddeb                                     |  5 +-
 .../unittests/test_datasource/test_azure_helper.py |  4 +-
 tests/unittests/test_net.py                        |  3 +-
 10 files changed, 112 insertions(+), 46 deletions(-)
 create mode 100644 cloudinit/temp_utils.py

(limited to 'tests/unittests/test_datasource')

diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py
index 604f93b0..9c0476af 100644
--- a/cloudinit/config/cc_bootcmd.py
+++ b/cloudinit/config/cc_bootcmd.py
@@ -37,6 +37,7 @@ specified either as lists or strings. For invocation details, see ``runcmd``.
 import os
 
 from cloudinit.settings import PER_ALWAYS
+from cloudinit import temp_utils
 from cloudinit import util
 
 frequency = PER_ALWAYS
@@ -49,7 +50,7 @@ def handle(name, cfg, cloud, log, _args):
                    " no 'bootcmd' key in configuration"), name)
         return
 
-    with util.ExtendedTemporaryFile(suffix=".sh") as tmpf:
+    with temp_utils.ExtendedTemporaryFile(suffix=".sh") as tmpf:
         try:
             content = util.shellify(cfg["bootcmd"])
             tmpf.write(util.encode_text(content))
diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py
index 02c70b10..c192dd32 100644
--- a/cloudinit/config/cc_chef.py
+++ b/cloudinit/config/cc_chef.py
@@ -71,6 +71,7 @@ import itertools
 import json
 import os
 
+from cloudinit import temp_utils
 from cloudinit import templater
 from cloudinit import url_helper
 from cloudinit import util
@@ -303,7 +304,7 @@ def install_chef(cloud, chef_cfg, log):
                                                  "omnibus_url_retries",
                                                  default=OMNIBUS_URL_RETRIES))
         content = url_helper.readurl(url=url, retries=retries).contents
-        with util.tempdir() as tmpd:
+        with temp_utils.tempdir() as tmpd:
             # Use tmpdir over tmpfile to avoid 'text file busy' on execute
             tmpf = "%s/chef-omnibus-install" % tmpd
             util.write_file(tmpf, content, mode=0o700)
diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py
index a9682f19..eecb8178 100644
--- a/cloudinit/config/cc_snappy.py
+++ b/cloudinit/config/cc_snappy.py
@@ -63,11 +63,11 @@ is ``auto``. Options are:
 
 from cloudinit import log as logging
 from cloudinit.settings import PER_INSTANCE
+from cloudinit import temp_utils
 from cloudinit import util
 
 import glob
 import os
-import tempfile
 
 LOG = logging.getLogger(__name__)
 
@@ -183,7 +183,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None):
             #      config
             # Note, however, we do not touch config files on disk.
             nested_cfg = {'config': {shortname: config}}
-            (fd, cfg_tmpf) = tempfile.mkstemp()
+            (fd, cfg_tmpf) = temp_utils.mkstemp()
             os.write(fd, util.yaml_dumps(nested_cfg).encode())
             os.close(fd)
             cfgfile = cfg_tmpf
diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py
index c7febc57..c842c839 100644
--- a/cloudinit/net/dhcp.py
+++ b/cloudinit/net/dhcp.py
@@ -9,6 +9,7 @@ import os
 import re
 
 from cloudinit.net import find_fallback_nic, get_devicelist
+from cloudinit import temp_utils
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -47,7 +48,7 @@ def maybe_perform_dhcp_discovery(nic=None):
     if not dhclient_path:
         LOG.debug('Skip dhclient configuration: No dhclient command found.')
         return {}
-    with util.tempdir(prefix='cloud-init-dhcp-') as tmpdir:
+    with temp_utils.tempdir(prefix='cloud-init-dhcp-') as tmpdir:
         return dhcp_discovery(dhclient_path, nic, tmpdir)
 
 
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index e22409d1..28ed0ae2 100644
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -6,10 +6,10 @@ import os
 import re
 import socket
 import struct
-import tempfile
 import time
 
 from cloudinit import stages
+from cloudinit import temp_utils
 from contextlib import contextmanager
 from xml.etree import ElementTree
 
@@ -111,7 +111,7 @@ class OpenSSLManager(object):
     }
 
     def __init__(self):
-        self.tmpdir = tempfile.mkdtemp()
+        self.tmpdir = temp_utils.mkdtemp()
         self.certificate = None
         self.generate_certificate()
 
diff --git a/cloudinit/temp_utils.py b/cloudinit/temp_utils.py
new file mode 100644
index 00000000..0355f19d
--- /dev/null
+++ b/cloudinit/temp_utils.py
@@ -0,0 +1,93 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import contextlib
+import errno
+import os
+import shutil
+import tempfile
+
+_TMPDIR = None
+_ROOT_TMPDIR = "/run/cloud-init/tmp"
+
+
+def _tempfile_dir_arg(odir=None):
+    """Return the proper 'dir' argument for tempfile functions.
+
+    When root, cloud-init will use /run/cloud-init/tmp to avoid
+    any cleaning that a distro boot might do on /tmp (such as
+    systemd-tmpfiles-clean).
+
+    If the caller of this function (mkdtemp or mkstemp) was provided
+    with a 'dir' argument, then that is respected.
+
+    @param odir: original 'dir' arg to 'mkdtemp' or other."""
+
+    if odir is not None:
+        return odir
+
+    global _TMPDIR
+    if _TMPDIR:
+        return _TMPDIR
+
+    if os.getuid() == 0:
+        tdir = _ROOT_TMPDIR
+    else:
+        tdir = os.environ.get('TMPDIR', '/tmp')
+    if not os.path.isdir(tdir):
+        os.makedirs(tdir)
+        os.chmod(tdir, 0o1777)
+
+    _TMPDIR = tdir
+    return tdir
+
+
+def ExtendedTemporaryFile(**kwargs):
+    kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None))
+    fh = tempfile.NamedTemporaryFile(**kwargs)
+    # Replace its unlink with a quiet version
+    # that does not raise errors when the
+    # file to unlink has been unlinked elsewhere..
+
+    def _unlink_if_exists(path):
+        try:
+            os.unlink(path)
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                raise e
+
+    fh.unlink = _unlink_if_exists
+
+    # Add a new method that will unlink
+    # right 'now' but still lets the exit
+    # method attempt to remove it (which will
+    # not throw due to our del file being quiet
+    # about files that are not there)
+    def unlink_now():
+        fh.unlink(fh.name)
+
+    setattr(fh, 'unlink_now', unlink_now)
+    return fh
+
+
+@contextlib.contextmanager
+def tempdir(**kwargs):
+    # This seems like it was only added in python 3.2
+    # Make it since its useful...
+    # See: http://bugs.python.org/file12970/tempdir.patch
+    tdir = mkdtemp(**kwargs)
+    try:
+        yield tdir
+    finally:
+        shutil.rmtree(tdir)
+
+
+def mkdtemp(**kwargs):
+    kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None))
+    return tempfile.mkdtemp(**kwargs)
+
+
+def mkstemp(**kwargs):
+    kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None))
+    return tempfile.mkstemp(**kwargs)
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 609e94c8..ae5cda8d 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -30,7 +30,6 @@ import stat
 import string
 import subprocess
 import sys
-import tempfile
 import time
 
 from errno import ENOENT, ENOEXEC
@@ -45,6 +44,7 @@ from cloudinit import importer
 from cloudinit import log as logging
 from cloudinit import mergers
 from cloudinit import safeyaml
+from cloudinit import temp_utils
 from cloudinit import type_utils
 from cloudinit import url_helper
 from cloudinit import version
@@ -349,26 +349,6 @@ class DecompressionError(Exception):
     pass
 
 
-def ExtendedTemporaryFile(**kwargs):
-    fh = tempfile.NamedTemporaryFile(**kwargs)
-    # Replace its unlink with a quiet version
-    # that does not raise errors when the
-    # file to unlink has been unlinked elsewhere..
-    LOG.debug("Created temporary file %s", fh.name)
-    fh.unlink = del_file
-
-    # Add a new method that will unlink
-    # right 'now' but still lets the exit
-    # method attempt to remove it (which will
-    # not throw due to our del file being quiet
-    # about files that are not there)
-    def unlink_now():
-        fh.unlink(fh.name)
-
-    setattr(fh, 'unlink_now', unlink_now)
-    return fh
-
-
 def fork_cb(child_cb, *args, **kwargs):
     fid = os.fork()
     if fid == 0:
@@ -790,18 +770,6 @@ def umask(n_msk):
         os.umask(old)
 
 
-@contextlib.contextmanager
-def tempdir(**kwargs):
-    # This seems like it was only added in python 3.2
-    # Make it since its useful...
-    # See: http://bugs.python.org/file12970/tempdir.patch
-    tdir = tempfile.mkdtemp(**kwargs)
-    try:
-        yield tdir
-    finally:
-        del_dir(tdir)
-
-
 def center(text, fill, max_len):
     return '{0:{fill}{align}{size}}'.format(text, fill=fill,
                                             align="^", size=max_len)
@@ -1587,7 +1555,7 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True):
         mtypes = ['']
 
     mounted = mounts()
-    with tempdir() as tmpd:
+    with temp_utils.tempdir() as tmpd:
         umount = False
         if os.path.realpath(device) in mounted:
             mountpoint = mounted[os.path.realpath(device)]['mountpoint']
diff --git a/packages/bddeb b/packages/bddeb
index 7c123548..4f2e2ddf 100755
--- a/packages/bddeb
+++ b/packages/bddeb
@@ -21,8 +21,9 @@ def find_root():
 if "avoid-pep8-E402-import-not-top-of-file":
     # Use the util functions from cloudinit
     sys.path.insert(0, find_root())
-    from cloudinit import templater
     from cloudinit import util
+    from cloudinit import temp_utils
+    from cloudinit import templater
 
 DEBUILD_ARGS = ["-S", "-d"]
 
@@ -148,7 +149,7 @@ def main():
         capture = False
 
     templ_data = {'debian_release': args.release}
-    with util.tempdir() as tdir:
+    with temp_utils.tempdir() as tdir:
 
         # output like 0.7.6-1022-g36e92d3
         ver_data = read_version()
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
index 80ce003d..44b99eca 100644
--- a/tests/unittests/test_datasource/test_azure_helper.py
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -275,7 +275,7 @@ class TestOpenSSLManager(TestCase):
                 mock.patch('builtins.open'))
 
     @mock.patch.object(azure_helper, 'cd', mock.MagicMock())
-    @mock.patch.object(azure_helper.tempfile, 'mkdtemp')
+    @mock.patch.object(azure_helper.temp_utils, 'mkdtemp')
     def test_openssl_manager_creates_a_tmpdir(self, mkdtemp):
         manager = azure_helper.OpenSSLManager()
         self.assertEqual(mkdtemp.return_value, manager.tmpdir)
@@ -292,7 +292,7 @@ class TestOpenSSLManager(TestCase):
         manager.clean_up()
 
     @mock.patch.object(azure_helper, 'cd', mock.MagicMock())
-    @mock.patch.object(azure_helper.tempfile, 'mkdtemp', mock.MagicMock())
+    @mock.patch.object(azure_helper.temp_utils, 'mkdtemp', mock.MagicMock())
     @mock.patch.object(azure_helper.util, 'del_dir')
     def test_clean_up(self, del_dir):
         manager = azure_helper.OpenSSLManager()
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index c10ef905..f2496151 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -9,6 +9,7 @@ from cloudinit.net import network_state
 from cloudinit.net import renderers
 from cloudinit.net import sysconfig
 from cloudinit.sources.helpers import openstack
+from cloudinit import temp_utils
 from cloudinit import util
 
 from cloudinit.tests.helpers import CiTestCase
@@ -2150,7 +2151,7 @@ class TestCmdlineConfigParsing(CiTestCase):
         static['mac_address'] = macs['eth1']
 
         expected = {'version': 1, 'config': [dhcp, static]}
-        with util.tempdir() as tmpd:
+        with temp_utils.tempdir() as tmpd:
             for fname, content in pairs:
                 fp = os.path.join(tmpd, fname)
                 files.append(fp)
-- 
cgit v1.2.3


From 922c3c5c1a86f2d58e95a328e72b49a3bb234ca8 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@brickies.net>
Date: Thu, 7 Sep 2017 09:59:47 -0400
Subject: Ec2: only attempt to operate at local mode on known platforms.

This change makes the DataSourceEc2Local do nothing unless it is on
actual AWS platform. The motivation is twofold:

a.) It is generally safer to only make this function available to Ec2
clones that explicitly identify themselves to the guest. (It also
gives them a reason to supply identification code to cloud-init.)

b.) On non-intel OpenStack platforms ds-identify would enable both the Ec2
and OpenStack sources. That is because there is not good data (such as
dmi) to positively identify the platform. Previously that would be fine
as OpenStack would run first and be successful. The change to add Ec2Local
meant that an Ec2 now runs first.

The best case for 'b' would be a slow down as attempts at the Ec2 metadata
service time out. The discovered case was worse.

Additionally we add a simple check for datatype of 'network' in the
metadata before attempting to read it.

LP: #1715128
---
 cloudinit/sources/DataSourceEc2.py          | 43 +++++++++++++++++++++++------
 tests/unittests/test_datasource/test_ec2.py | 29 ++++++++++++++++---
 2 files changed, 60 insertions(+), 12 deletions(-)

(limited to 'tests/unittests/test_datasource')

diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 07c12bb4..41367a8b 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -27,6 +27,8 @@ SKIP_METADATA_URL_CODES = frozenset([uhelp.NOT_FOUND])
 STRICT_ID_PATH = ("datasource", "Ec2", "strict_id")
 STRICT_ID_DEFAULT = "warn"
 
+_unset = "_unset"
+
 
 class Platforms(object):
     ALIYUN = "AliYun"
@@ -57,7 +59,7 @@ class DataSourceEc2(sources.DataSource):
 
     _cloud_platform = None
 
-    _network_config = None  # Used for caching calculated network config v1
+    _network_config = _unset  # Used for caching calculated network config v1
 
     # Whether we want to get network configuration from the metadata service.
     get_network_metadata = False
@@ -284,10 +286,24 @@ class DataSourceEc2(sources.DataSource):
     @property
     def network_config(self):
         """Return a network config dict for rendering ENI or netplan files."""
-        if self._network_config is None:
-            if self.metadata is not None:
-                self._network_config = convert_ec2_metadata_network_config(
-                    self.metadata)
+        if self._network_config != _unset:
+            return self._network_config
+
+        if self.metadata is None:
+            # this would happen if get_data hadn't been called. leave as _unset
+            LOG.warning(
+                "Unexpected call to network_config when metadata is None.")
+            return None
+
+        result = None
+        net_md = self.metadata.get('network')
+        if isinstance(net_md, dict):
+            result = convert_ec2_metadata_network_config(net_md)
+        else:
+            LOG.warning("unexpected metadata 'network' key not valid: %s",
+                        net_md)
+        self._network_config = result
+
         return self._network_config
 
     def _crawl_metadata(self):
@@ -321,6 +337,14 @@ class DataSourceEc2Local(DataSourceEc2):
     """
     get_network_metadata = True  # Get metadata network config if present
 
+    def get_data(self):
+        supported_platforms = (Platforms.AWS,)
+        if self.cloud_platform not in supported_platforms:
+            LOG.debug("Local Ec2 mode only supported on %s, not %s",
+                      supported_platforms, self.cloud_platform)
+            return False
+        return super(DataSourceEc2Local, self).get_data()
+
 
 def read_strict_mode(cfgval, default):
     try:
@@ -434,10 +458,13 @@ def _collect_platform_data():
     return data
 
 
-def convert_ec2_metadata_network_config(metadata=None, macs_to_nics=None):
+def convert_ec2_metadata_network_config(network_md, macs_to_nics=None):
     """Convert ec2 metadata to network config version 1 data dict.
 
-    @param: metadata: Dictionary of metadata crawled from EC2 metadata url.
+    @param: network_md: 'network' portion of EC2 metadata.
+       generally formed as {"interfaces": {"macs": {}} where
+       'macs' is a dictionary with mac address as key and contents like:
+       {"device-number": "0", "interface-id": "...", "local-ipv4s": ...}
     @param: macs_to_name: Optional dict mac addresses and the nic name. If
        not provided, get_interfaces_by_mac is called to get it from the OS.
 
@@ -446,7 +473,7 @@ def convert_ec2_metadata_network_config(metadata=None, macs_to_nics=None):
     netcfg = {'version': 1, 'config': []}
     if not macs_to_nics:
         macs_to_nics = net.get_interfaces_by_mac()
-    macs_metadata = metadata['network']['interfaces']['macs']
+    macs_metadata = network_md['interfaces']['macs']
     for mac, nic_name in macs_to_nics.items():
         nic_metadata = macs_metadata.get(mac)
         if not nic_metadata:
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 9fb90483..a7301dbf 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -279,6 +279,27 @@ class TestEc2(test_helpers.HttprettyTestCase):
         ret = ds.get_data()
         self.assertTrue(ret)
 
+    def test_ec2_local_returns_false_on_non_aws(self):
+        """DataSourceEc2Local returns False when platform is not AWS."""
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md=DEFAULT_METADATA)
+        platform_attrs = [
+            attr for attr in ec2.Platforms.__dict__.keys()
+            if not attr.startswith('__')]
+        for attr_name in platform_attrs:
+            platform_name = getattr(ec2.Platforms, attr_name)
+            if platform_name != 'AWS':
+                ds._cloud_platform = platform_name
+                ret = ds.get_data()
+                self.assertFalse(ret)
+                message = (
+                    "Local Ec2 mode only supported on ('AWS',),"
+                    ' not {0}'.format(platform_name))
+                self.assertIn(message, self.logs.getvalue())
+
     @httpretty.activate
     @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
     def test_ec2_local_returns_false_on_bsd(self, m_is_freebsd):
@@ -336,8 +357,8 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
         super(TestConvertEc2MetadataNetworkConfig, self).setUp()
         self.mac1 = '06:17:04:d7:26:09'
         self.network_metadata = {
-            'network': {'interfaces': {'macs': {
-                self.mac1: {'public-ipv4s': '172.31.2.16'}}}}}
+            'interfaces': {'macs': {
+                self.mac1: {'public-ipv4s': '172.31.2.16'}}}}
 
     def test_convert_ec2_metadata_network_config_skips_absent_macs(self):
         """Any mac absent from metadata is skipped by network config."""
@@ -357,7 +378,7 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
         macs_to_nics = {self.mac1: 'eth9'}
         network_metadata_ipv6 = copy.deepcopy(self.network_metadata)
         nic1_metadata = (
-            network_metadata_ipv6['network']['interfaces']['macs'][self.mac1])
+            network_metadata_ipv6['interfaces']['macs'][self.mac1])
         nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
         nic1_metadata.pop('public-ipv4s')
         expected = {'version': 1, 'config': [
@@ -373,7 +394,7 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
         macs_to_nics = {self.mac1: 'eth9'}
         network_metadata_both = copy.deepcopy(self.network_metadata)
         nic1_metadata = (
-            network_metadata_both['network']['interfaces']['macs'][self.mac1])
+            network_metadata_both['interfaces']['macs'][self.mac1])
         nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
         expected = {'version': 1, 'config': [
             {'mac_address': self.mac1, 'type': 'physical',
-- 
cgit v1.2.3


From da1db792b2721d94ef85df8c136e78012c49c6e5 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@brickies.net>
Date: Fri, 15 Sep 2017 13:37:55 -0600
Subject: CloudStack: consider dhclient lease files named with a hyphen.

A regression in 'get_latest_lease' made it ignore files starting with
'dhclient-' rather than just 'dhclient.'. The fix here is to allow those
files to be considered.

There is a lot more we could do here to better ensure that we pick the
most recent lease, but this change fixes the regression.

LP: #1717147
---
 cloudinit/sources/DataSourceCloudStack.py          | 34 +++++++---
 tests/unittests/test_datasource/test_cloudstack.py | 79 +++++++++++++++++++++-
 2 files changed, 100 insertions(+), 13 deletions(-)

(limited to 'tests/unittests/test_datasource')

diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index 0188d894..7e0f9bb8 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -187,22 +187,36 @@ def get_dhclient_d():
     return None
 
 
-def get_latest_lease():
+def get_latest_lease(lease_d=None):
     # find latest lease file
-    lease_d = get_dhclient_d()
+    if lease_d is None:
+        lease_d = get_dhclient_d()
     if not lease_d:
         return None
     lease_files = os.listdir(lease_d)
     latest_mtime = -1
     latest_file = None
-    for file_name in lease_files:
-        if file_name.startswith("dhclient.") and \
-           (file_name.endswith(".lease") or file_name.endswith(".leases")):
-            abs_path = os.path.join(lease_d, file_name)
-            mtime = os.path.getmtime(abs_path)
-            if mtime > latest_mtime:
-                latest_mtime = mtime
-                latest_file = abs_path
+
+    # lease files are named inconsistently across distros.
+    # We assume that 'dhclient6' indicates ipv6 and ignore it.
+    # ubuntu:
+    #   dhclient.<iface>.leases, dhclient.leases, dhclient6.leases
+    # centos6:
+    #   dhclient-<iface>.leases, dhclient6.leases
+    # centos7: ('--' is not a typo)
+    #   dhclient--<iface>.lease, dhclient6.leases
+    for fname in lease_files:
+        if fname.startswith("dhclient6"):
+            # avoid files that start with dhclient6 assuming dhcpv6.
+            continue
+        if not (fname.endswith(".lease") or fname.endswith(".leases")):
+            continue
+
+        abs_path = os.path.join(lease_d, fname)
+        mtime = os.path.getmtime(abs_path)
+        if mtime > latest_mtime:
+            latest_mtime = mtime
+            latest_file = abs_path
     return latest_file
 
 
diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py
index 2dc90305..8e98e1bb 100644
--- a/tests/unittests/test_datasource/test_cloudstack.py
+++ b/tests/unittests/test_datasource/test_cloudstack.py
@@ -1,12 +1,17 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit import helpers
-from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack
+from cloudinit import util
+from cloudinit.sources.DataSourceCloudStack import (
+    DataSourceCloudStack, get_latest_lease)
 
-from cloudinit.tests.helpers import TestCase, mock, ExitStack
+from cloudinit.tests.helpers import CiTestCase, ExitStack, mock
 
+import os
+import time
 
-class TestCloudStackPasswordFetching(TestCase):
+
+class TestCloudStackPasswordFetching(CiTestCase):
 
     def setUp(self):
         super(TestCloudStackPasswordFetching, self).setUp()
@@ -89,4 +94,72 @@ class TestCloudStackPasswordFetching(TestCase):
     def test_password_not_saved_if_bad_request(self):
         self._check_password_not_saved_for('bad_request')
 
+
+class TestGetLatestLease(CiTestCase):
+
+    def _populate_dir_list(self, bdir, files):
+        """populate_dir_list([(name, data), (name, data)])
+
+        writes files to bdir, and updates timestamps to ensure
+        that their mtime increases with each file."""
+
+        start = int(time.time())
+        for num, fname in enumerate(reversed(files)):
+            fpath = os.path.sep.join((bdir, fname))
+            util.write_file(fpath, fname.encode())
+            os.utime(fpath, (start - num, start - num))
+
+    def _pop_and_test(self, files, expected):
+        lease_d = self.tmp_dir()
+        self._populate_dir_list(lease_d, files)
+        self.assertEqual(self.tmp_path(expected, lease_d),
+                         get_latest_lease(lease_d))
+
+    def test_skips_dhcpv6_files(self):
+        """files started with dhclient6 should be skipped."""
+        expected = "dhclient.lease"
+        self._pop_and_test([expected, "dhclient6.lease"], expected)
+
+    def test_selects_dhclient_dot_files(self):
+        """files named dhclient.lease or dhclient.leases should be used.
+
+        Ubuntu names files dhclient.eth0.leases dhclient6.leases and
+        sometimes dhclient.leases."""
+        self._pop_and_test(["dhclient.lease"], "dhclient.lease")
+        self._pop_and_test(["dhclient.leases"], "dhclient.leases")
+
+    def test_selects_dhclient_dash_files(self):
+        """files named dhclient-lease or dhclient-leases should be used.
+
+        Redhat/Centos names files with dhclient--eth0.lease (centos 7) or
+        dhclient-eth0.leases (centos 6).
+        """
+        self._pop_and_test(["dhclient-eth0.lease"], "dhclient-eth0.lease")
+        self._pop_and_test(["dhclient--eth0.lease"], "dhclient--eth0.lease")
+
+    def test_ignores_by_extension(self):
+        """only .lease or .leases file should be considered."""
+
+        self._pop_and_test(["dhclient.lease", "dhclient.lease.bk",
+                            "dhclient.lease-old", "dhclient.leaselease"],
+                           "dhclient.lease")
+
+    def test_selects_newest_matching(self):
+        """If multiple files match, the newest written should be used."""
+        lease_d = self.tmp_dir()
+        valid_1 = "dhclient.leases"
+        valid_2 = "dhclient.lease"
+        valid_1_path = self.tmp_path(valid_1, lease_d)
+        valid_2_path = self.tmp_path(valid_2, lease_d)
+
+        self._populate_dir_list(lease_d, [valid_1, valid_2])
+        self.assertEqual(valid_2_path, get_latest_lease(lease_d))
+
+        # now update mtime on valid_2 to be older than valid_1 and re-check.
+        mtime = int(os.path.getmtime(valid_1_path)) - 1
+        os.utime(valid_2_path, (mtime, mtime))
+
+        self.assertEqual(valid_1_path, get_latest_lease(lease_d))
+
+
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From 10f067d87a2d48092c593862e686c517c57b987c Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@brickies.net>
Date: Fri, 15 Sep 2017 22:50:21 -0400
Subject: GCE: Fix usage of user-data.

This regressed in the rework of GCE datasource to have a main.
The fix really just stores the user-data that was read in
self.userdata_raw, rather than self.userdata.  That is consistent
with other datasources and ulitimately how it was before the refactor.

The main is updated to address the fact that user-data is binary data
and may not be able to be printed.

LP: #1717598
---
 cloudinit/sources/DataSourceGCE.py          | 26 +++++++++++++++++++-------
 tests/unittests/test_datasource/test_gce.py |  3 ++-
 2 files changed, 21 insertions(+), 8 deletions(-)

(limited to 'tests/unittests/test_datasource')

diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py
index 94484d60..ccae4200 100644
--- a/cloudinit/sources/DataSourceGCE.py
+++ b/cloudinit/sources/DataSourceGCE.py
@@ -62,7 +62,7 @@ class DataSourceGCE(sources.DataSource):
                 LOG.debug(ret['reason'])
             return False
         self.metadata = ret['meta-data']
-        self.userdata = ret['user-data']
+        self.userdata_raw = ret['user-data']
         return True
 
     @property
@@ -80,9 +80,6 @@ class DataSourceGCE(sources.DataSource):
         # GCE has long FDQN's and has asked for short hostnames
         return self.metadata['local-hostname'].split('.')[0]
 
-    def get_userdata_raw(self):
-        return self.userdata
-
     @property
     def availability_zone(self):
         return self.metadata['availability-zone']
@@ -202,6 +199,9 @@ def get_datasource_list(depends):
 if __name__ == "__main__":
     import argparse
     import json
+    import sys
+
+    from base64 import b64encode
 
     parser = argparse.ArgumentParser(description='Query GCE Metadata Service')
     parser.add_argument("--endpoint", metavar="URL",
@@ -211,8 +211,20 @@ if __name__ == "__main__":
                         help="Ignore smbios platform check",
                         action='store_false', default=True)
     args = parser.parse_args()
-    print(json.dumps(
-        read_md(address=args.endpoint, platform_check=args.platform_check),
-        indent=1, sort_keys=True, separators=(',', ': ')))
+    data = read_md(address=args.endpoint, platform_check=args.platform_check)
+    if 'user-data' in data:
+        # user-data is bytes not string like other things. Handle it specially.
+        # if it can be represented as utf-8 then do so.  Otherwise print base64
+        # encoded value in the key user-data-b64.
+        try:
+            data['user-data'] = data['user-data'].decode()
+        except UnicodeDecodeError:
+            sys.stderr.write("User-data cannot be decoded. "
+                             "Writing as base64\n")
+            del data['user-data']
+            # b64encode returns a bytes value. decode to get the string.
+            data['user-data-b64'] = b64encode(data['user-data']).decode()
+
+    print(json.dumps(data, indent=1, sort_keys=True, separators=(',', ': ')))
 
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py
index 50e49a10..d399ae7a 100644
--- a/tests/unittests/test_datasource/test_gce.py
+++ b/tests/unittests/test_datasource/test_gce.py
@@ -23,7 +23,8 @@ GCE_META = {
     'instance/zone': 'foo/bar',
     'project/attributes/sshKeys': 'user:ssh-rsa AA2..+aRD0fyVw== root@server',
     'instance/hostname': 'server.project-foo.local',
-    'instance/attributes/user-data': b'/bin/echo foo\n',
+    # UnicodeDecodeError below if set to ds.userdata instead of userdata_raw
+    'instance/attributes/user-data': b'/bin/echo \xff\n',
 }
 
 GCE_META_PARTIAL = {
-- 
cgit v1.2.3


From da6562e21d0b17a0957adc0c5a2c9da076e0d219 Mon Sep 17 00:00:00 2001
From: Ryan Harper <ryan.harper@canonical.com>
Date: Tue, 19 Sep 2017 11:10:09 -0500
Subject: DataSourceOVF: use util.find_devs_with(TYPE=iso9660)

DataSourceOVF attempts to find iso files via walking os.listdir('/dev/')
which is far too wide.  This approach is too invasive and can sometimes
race with systemd attempting to fsck and mount devices.

Instead, utilize cloudinit.util.find_devs_with to filter devices by
criteria (which uses blkid under the covers).  This results in fewer
attempts to mount block devices which do not contain iso filesystems.

Unittest changes include:
- cloudinit.tests.helpers; introduce add_patch() helper
- Add unittest coverage for DataSourceOVF use of transport_iso9660

LP: #1718287
---
 cloudinit/sources/DataSourceOVF.py          |  74 ++++++++-----
 cloudinit/tests/helpers.py                  |  10 ++
 tests/unittests/test_datasource/test_ovf.py | 164 ++++++++++++++++++++++++++++
 3 files changed, 221 insertions(+), 27 deletions(-)

(limited to 'tests/unittests/test_datasource')

diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index 24b45d55..ccebf11a 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -375,26 +375,56 @@ def get_ovf_env(dirname):
     return (None, False)
 
 
-# Transport functions take no input and return
-# a 3 tuple of content, path, filename
-def transport_iso9660(require_iso=True):
+def maybe_cdrom_device(devname):
+    """Test if devname matches known list of devices which may contain iso9660
+       filesystems.
 
-    # default_regex matches values in
-    # /lib/udev/rules.d/60-cdrom_id.rules
-    # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end"
-    envname = "CLOUD_INIT_CDROM_DEV_REGEX"
-    default_regex = "^(sr[0-9]+|hd[a-z]|xvd.*)"
+    Be helpful in accepting either knames (with no leading /dev/) or full path
+    names, but do not allow paths outside of /dev/, like /dev/foo/bar/xxx.
+    """
+    if not devname:
+        return False
+    elif not isinstance(devname, util.string_types):
+        raise ValueError("Unexpected input for devname: %s" % devname)
+
+    # resolve '..' and multi '/' elements
+    devname = os.path.normpath(devname)
 
-    devname_regex = os.environ.get(envname, default_regex)
+    # drop leading '/dev/'
+    if devname.startswith("/dev/"):
+        # partition returns tuple (before, partition, after)
+        devname = devname.partition("/dev/")[-1]
+
+    # ignore leading slash (/sr0), else fail on / in name (foo/bar/xvdc)
+    if devname.startswith("/"):
+        devname = devname.split("/")[-1]
+    elif devname.count("/") > 0:
+        return False
+
+    # if empty string
+    if not devname:
+        return False
+
+    # default_regex matches values in /lib/udev/rules.d/60-cdrom_id.rules
+    # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end"
+    default_regex = r"^(sr[0-9]+|hd[a-z]|xvd.*)"
+    devname_regex = os.environ.get("CLOUD_INIT_CDROM_DEV_REGEX", default_regex)
     cdmatch = re.compile(devname_regex)
 
+    return cdmatch.match(devname) is not None
+
+
+# Transport functions take no input and return
+# a 3 tuple of content, path, filename
+def transport_iso9660(require_iso=True):
+
     # Go through mounts to see if it was already mounted
     mounts = util.mounts()
     for (dev, info) in mounts.items():
         fstype = info['fstype']
         if fstype != "iso9660" and require_iso:
             continue
-        if cdmatch.match(dev[5:]) is None:  # take off '/dev/'
+        if not maybe_cdrom_device(dev):
             continue
         mp = info['mountpoint']
         (fname, contents) = get_ovf_env(mp)
@@ -406,29 +436,19 @@ def transport_iso9660(require_iso=True):
     else:
         mtype = None
 
-    devs = os.listdir("/dev/")
-    devs.sort()
+    # generate a list of devices with mtype filesystem, filter by regex
+    devs = [dev for dev in
+            util.find_devs_with("TYPE=%s" % mtype if mtype else None)
+            if maybe_cdrom_device(dev)]
     for dev in devs:
-        fullp = os.path.join("/dev/", dev)
-
-        if (fullp in mounts or
-                not cdmatch.match(dev) or os.path.isdir(fullp)):
-            continue
-
-        try:
-            # See if we can read anything at all...??
-            util.peek_file(fullp, 512)
-        except IOError:
-            continue
-
         try:
-            (fname, contents) = util.mount_cb(fullp, get_ovf_env, mtype=mtype)
+            (fname, contents) = util.mount_cb(dev, get_ovf_env, mtype=mtype)
         except util.MountFailedError:
-            LOG.debug("%s not mountable as iso9660", fullp)
+            LOG.debug("%s not mountable as iso9660", dev)
             continue
 
         if contents is not False:
-            return (contents, fullp, fname)
+            return (contents, dev, fname)
 
     return (False, None, None)
 
diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py
index 28e26622..6f88a5b7 100644
--- a/cloudinit/tests/helpers.py
+++ b/cloudinit/tests/helpers.py
@@ -104,6 +104,16 @@ class TestCase(unittest2.TestCase):
         super(TestCase, self).setUp()
         self.reset_global_state()
 
+    def add_patch(self, target, attr, **kwargs):
+        """Patches specified target object and sets it as attr on test
+        instance also schedules cleanup"""
+        if 'autospec' not in kwargs:
+            kwargs['autospec'] = True
+        m = mock.patch(target, **kwargs)
+        p = m.start()
+        self.addCleanup(m.stop)
+        setattr(self, attr, p)
+
 
 class CiTestCase(TestCase):
     """This is the preferred test case base class unless user
diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py
index 9dbf4dd9..700da86c 100644
--- a/tests/unittests/test_datasource/test_ovf.py
+++ b/tests/unittests/test_datasource/test_ovf.py
@@ -5,6 +5,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import base64
+from collections import OrderedDict
 
 from cloudinit.tests import helpers as test_helpers
 
@@ -70,4 +71,167 @@ class TestReadOvfEnv(test_helpers.TestCase):
         self.assertEqual({'password': "passw0rd"}, cfg)
         self.assertIsNone(ud)
 
+
+class TestTransportIso9660(test_helpers.CiTestCase):
+
+    def setUp(self):
+        super(TestTransportIso9660, self).setUp()
+        self.add_patch('cloudinit.util.find_devs_with',
+                       'm_find_devs_with')
+        self.add_patch('cloudinit.util.mounts', 'm_mounts')
+        self.add_patch('cloudinit.util.mount_cb', 'm_mount_cb')
+        self.add_patch('cloudinit.sources.DataSourceOVF.get_ovf_env',
+                       'm_get_ovf_env')
+        self.m_get_ovf_env.return_value = ('myfile', 'mycontent')
+
+    def test_find_already_mounted(self):
+        """Check we call get_ovf_env from on matching mounted devices"""
+        mounts = {
+            '/dev/sr9': {
+                'fstype': 'iso9660',
+                'mountpoint': 'wark/media/sr9',
+                'opts': 'ro',
+            }
+        }
+        self.m_mounts.return_value = mounts
+
+        (contents, fullp, fname) = dsovf.transport_iso9660()
+        self.assertEqual("mycontent", contents)
+        self.assertEqual("/dev/sr9", fullp)
+        self.assertEqual("myfile", fname)
+
+    def test_find_already_mounted_skips_non_iso9660(self):
+        """Check we call get_ovf_env ignoring non iso9660"""
+        mounts = {
+            '/dev/xvdb': {
+                'fstype': 'vfat',
+                'mountpoint': 'wark/foobar',
+                'opts': 'defaults,noatime',
+            },
+            '/dev/xvdc': {
+                'fstype': 'iso9660',
+                'mountpoint': 'wark/media/sr9',
+                'opts': 'ro',
+            }
+        }
+        # We use an OrderedDict here to ensure we check xvdb before xvdc
+        # as we're not mocking the regex matching, however, if we place
+        # an entry in the results then we can be reasonably sure that
+        # we're skipping an entry which fails to match.
+        self.m_mounts.return_value = (
+            OrderedDict(sorted(mounts.items(), key=lambda t: t[0])))
+
+        (contents, fullp, fname) = dsovf.transport_iso9660()
+        self.assertEqual("mycontent", contents)
+        self.assertEqual("/dev/xvdc", fullp)
+        self.assertEqual("myfile", fname)
+
+    def test_find_already_mounted_matches_kname(self):
+        """Check we dont regex match on basename of the device"""
+        mounts = {
+            '/dev/foo/bar/xvdc': {
+                'fstype': 'iso9660',
+                'mountpoint': 'wark/media/sr9',
+                'opts': 'ro',
+            }
+        }
+        # we're skipping an entry which fails to match.
+        self.m_mounts.return_value = mounts
+
+        (contents, fullp, fname) = dsovf.transport_iso9660()
+        self.assertEqual(False, contents)
+        self.assertIsNone(fullp)
+        self.assertIsNone(fname)
+
+    def test_mount_cb_called_on_blkdevs_with_iso9660(self):
+        """Check we call mount_cb on blockdevs with iso9660 only"""
+        self.m_mounts.return_value = {}
+        self.m_find_devs_with.return_value = ['/dev/sr0']
+        self.m_mount_cb.return_value = ("myfile", "mycontent")
+
+        (contents, fullp, fname) = dsovf.transport_iso9660()
+
+        self.m_mount_cb.assert_called_with(
+            "/dev/sr0", dsovf.get_ovf_env, mtype="iso9660")
+        self.assertEqual("mycontent", contents)
+        self.assertEqual("/dev/sr0", fullp)
+        self.assertEqual("myfile", fname)
+
+    def test_mount_cb_called_on_blkdevs_with_iso9660_check_regex(self):
+        """Check we call mount_cb on blockdevs with iso9660 and match regex"""
+        self.m_mounts.return_value = {}
+        self.m_find_devs_with.return_value = [
+            '/dev/abc', '/dev/my-cdrom', '/dev/sr0']
+        self.m_mount_cb.return_value = ("myfile", "mycontent")
+
+        (contents, fullp, fname) = dsovf.transport_iso9660()
+
+        self.m_mount_cb.assert_called_with(
+            "/dev/sr0", dsovf.get_ovf_env, mtype="iso9660")
+        self.assertEqual("mycontent", contents)
+        self.assertEqual("/dev/sr0", fullp)
+        self.assertEqual("myfile", fname)
+
+    def test_mount_cb_not_called_no_matches(self):
+        """Check we don't call mount_cb if nothing matches"""
+        self.m_mounts.return_value = {}
+        self.m_find_devs_with.return_value = ['/dev/vg/myovf']
+
+        (contents, fullp, fname) = dsovf.transport_iso9660()
+
+        self.assertEqual(0, self.m_mount_cb.call_count)
+        self.assertEqual(False, contents)
+        self.assertIsNone(fullp)
+        self.assertIsNone(fname)
+
+    def test_mount_cb_called_require_iso_false(self):
+        """Check we call mount_cb on blockdevs with require_iso=False"""
+        self.m_mounts.return_value = {}
+        self.m_find_devs_with.return_value = ['/dev/xvdz']
+        self.m_mount_cb.return_value = ("myfile", "mycontent")
+
+        (contents, fullp, fname) = dsovf.transport_iso9660(require_iso=False)
+
+        self.m_mount_cb.assert_called_with(
+            "/dev/xvdz", dsovf.get_ovf_env, mtype=None)
+        self.assertEqual("mycontent", contents)
+        self.assertEqual("/dev/xvdz", fullp)
+        self.assertEqual("myfile", fname)
+
+    def test_maybe_cdrom_device_none(self):
+        """Test maybe_cdrom_device returns False for none/empty input"""
+        self.assertFalse(dsovf.maybe_cdrom_device(None))
+        self.assertFalse(dsovf.maybe_cdrom_device(''))
+
+    def test_maybe_cdrom_device_non_string_exception(self):
+        """Test maybe_cdrom_device raises ValueError on non-string types"""
+        with self.assertRaises(ValueError):
+            dsovf.maybe_cdrom_device({'a': 'eleven'})
+
+    def test_maybe_cdrom_device_false_on_multi_dir_paths(self):
+        """Test maybe_cdrom_device is false on /dev[/.*]/* paths"""
+        self.assertFalse(dsovf.maybe_cdrom_device('/dev/foo/sr0'))
+        self.assertFalse(dsovf.maybe_cdrom_device('foo/sr0'))
+        self.assertFalse(dsovf.maybe_cdrom_device('../foo/sr0'))
+        self.assertFalse(dsovf.maybe_cdrom_device('../foo/sr0'))
+
+    def test_maybe_cdrom_device_true_on_hd_partitions(self):
+        """Test maybe_cdrom_device is false on /dev/hd[a-z][0-9]+ paths"""
+        self.assertTrue(dsovf.maybe_cdrom_device('/dev/hda1'))
+        self.assertTrue(dsovf.maybe_cdrom_device('hdz9'))
+
+    def test_maybe_cdrom_device_true_on_valid_relative_paths(self):
+        """Test maybe_cdrom_device normalizes paths"""
+        self.assertTrue(dsovf.maybe_cdrom_device('/dev/wark/../sr9'))
+        self.assertTrue(dsovf.maybe_cdrom_device('///sr0'))
+        self.assertTrue(dsovf.maybe_cdrom_device('/sr0'))
+        self.assertTrue(dsovf.maybe_cdrom_device('//dev//hda'))
+
+    def test_maybe_cdrom_device_true_on_xvd_partitions(self):
+        """Test maybe_cdrom_device returns true on xvd*"""
+        self.assertTrue(dsovf.maybe_cdrom_device('/dev/xvda'))
+        self.assertTrue(dsovf.maybe_cdrom_device('/dev/xvda1'))
+        self.assertTrue(dsovf.maybe_cdrom_device('xvdza1'))
+
+#
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From ad099a53d120e88719a5ad50f29d22e9f7a52bc7 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Mon, 25 Sep 2017 14:29:13 -0400
Subject: AltCloud: Trust PATH for udevadm and modprobe.

Previously we had hard coded paths in /sbin for the udevadm and modprobe
programs invoked by AltCloud.  Its more flexible to expect the PATH to
be set correctly.

Debian: #852564
---
 cloudinit/sources/DataSourceAltCloud.py          | 4 ++--
 tests/unittests/test_datasource/test_altcloud.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'tests/unittests/test_datasource')

diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index ed1d691a..c78ad9eb 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -28,8 +28,8 @@ LOG = logging.getLogger(__name__)
 CLOUD_INFO_FILE = '/etc/sysconfig/cloud-info'
 
 # Shell command lists
-CMD_PROBE_FLOPPY = ['/sbin/modprobe', 'floppy']
-CMD_UDEVADM_SETTLE = ['/sbin/udevadm', 'settle', '--timeout=5']
+CMD_PROBE_FLOPPY = ['modprobe', 'floppy']
+CMD_UDEVADM_SETTLE = ['udevadm', 'settle', '--timeout=5']
 
 META_DATA_NOT_SUPPORTED = {
     'block-device-mapping': {},
diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py
index 3b274d90..a4dfb540 100644
--- a/tests/unittests/test_datasource/test_altcloud.py
+++ b/tests/unittests/test_datasource/test_altcloud.py
@@ -280,8 +280,8 @@ class TestUserDataRhevm(TestCase):
             pass
 
         dsac.CLOUD_INFO_FILE = '/etc/sysconfig/cloud-info'
-        dsac.CMD_PROBE_FLOPPY = ['/sbin/modprobe', 'floppy']
-        dsac.CMD_UDEVADM_SETTLE = ['/sbin/udevadm', 'settle',
+        dsac.CMD_PROBE_FLOPPY = ['modprobe', 'floppy']
+        dsac.CMD_UDEVADM_SETTLE = ['udevadm', 'settle',
                                    '--quiet', '--timeout=5']
 
     def test_mount_cb_fails(self):
-- 
cgit v1.2.3


From 9d2a87dc386b7aed1a8243d599676e78ed358749 Mon Sep 17 00:00:00 2001
From: Dimitri John Ledkov <xnox@ubuntu.com>
Date: Sat, 30 Sep 2017 00:00:18 -0400
Subject: Azure, CloudStack: Support reading dhcp options from
 systemd-networkd.

Systems that used systemd-networkd's dhcp client would not be able to get
information on the Azure endpoint (placed in Option 245) or the CloudStack
server (in 'server_address').

The change here supports reading these files in /run/systemd/netif/leases.
The files declare that "This is private data. Do not parse.", but at this
point we do not have another option.

LP: #1718029
---
 cloudinit/net/dhcp.py                              |  42 ++++++
 cloudinit/net/tests/test_dhcp.py                   | 113 +++++++++++++++-
 cloudinit/sources/DataSourceCloudStack.py          |  17 ++-
 cloudinit/sources/helpers/azure.py                 |  20 ++-
 .../unittests/test_datasource/test_azure_helper.py | 143 ++++++++++++++-------
 tests/unittests/test_datasource/test_cloudstack.py |  11 +-
 6 files changed, 282 insertions(+), 64 deletions(-)

(limited to 'tests/unittests/test_datasource')

diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py
index 05350639..0cba7032 100644
--- a/cloudinit/net/dhcp.py
+++ b/cloudinit/net/dhcp.py
@@ -4,6 +4,7 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
+import configobj
 import logging
 import os
 import re
@@ -11,9 +12,12 @@ import re
 from cloudinit.net import find_fallback_nic, get_devicelist
 from cloudinit import temp_utils
 from cloudinit import util
+from six import StringIO
 
 LOG = logging.getLogger(__name__)
 
+NETWORKD_LEASES_DIR = '/run/systemd/netif/leases'
+
 
 class InvalidDHCPLeaseFileError(Exception):
     """Raised when parsing an empty or invalid dhcp.leases file.
@@ -118,4 +122,42 @@ def dhcp_discovery(dhclient_cmd_path, interface, cleandir):
     return parse_dhcp_lease_file(lease_file)
 
 
+def networkd_parse_lease(content):
+    """Parse a systemd lease file content as in /run/systemd/netif/leases/
+
+    Parse this (almost) ini style file even though it says:
+      # This is private data. Do not parse.
+
+    Simply return a dictionary of key/values."""
+
+    return dict(configobj.ConfigObj(StringIO(content), list_values=False))
+
+
+def networkd_load_leases(leases_d=None):
+    """Return a dictionary of dictionaries representing each lease
+    found in lease_d.i
+
+    The top level key will be the filename, which is typically the ifindex."""
+
+    if leases_d is None:
+        leases_d = NETWORKD_LEASES_DIR
+
+    ret = {}
+    if not os.path.isdir(leases_d):
+        return ret
+    for lfile in os.listdir(leases_d):
+        ret[lfile] = networkd_parse_lease(
+            util.load_file(os.path.join(leases_d, lfile)))
+    return ret
+
+
+def networkd_get_option_from_leases(keyname, leases_d=None):
+    if leases_d is None:
+        leases_d = NETWORKD_LEASES_DIR
+    leases = networkd_load_leases(leases_d=leases_d)
+    for ifindex, data in sorted(leases.items()):
+        if data.get(keyname):
+            return data[keyname]
+    return None
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py
index a38edaec..1c1f504a 100644
--- a/cloudinit/net/tests/test_dhcp.py
+++ b/cloudinit/net/tests/test_dhcp.py
@@ -6,9 +6,9 @@ from textwrap import dedent
 
 from cloudinit.net.dhcp import (
     InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery,
-    parse_dhcp_lease_file, dhcp_discovery)
+    parse_dhcp_lease_file, dhcp_discovery, networkd_load_leases)
 from cloudinit.util import ensure_file, write_file
-from cloudinit.tests.helpers import CiTestCase, wrap_and_call
+from cloudinit.tests.helpers import CiTestCase, wrap_and_call, populate_dir
 
 
 class TestParseDHCPLeasesFile(CiTestCase):
@@ -149,3 +149,112 @@ class TestDHCPDiscoveryClean(CiTestCase):
                 [os.path.join(tmpdir, 'dhclient'), '-1', '-v', '-lf',
                  lease_file, '-pf', os.path.join(tmpdir, 'dhclient.pid'),
                  'eth9', '-sf', '/bin/true'], capture=True)])
+
+
+class TestSystemdParseLeases(CiTestCase):
+
+    lxd_lease = dedent("""\
+    # This is private data. Do not parse.
+    ADDRESS=10.75.205.242
+    NETMASK=255.255.255.0
+    ROUTER=10.75.205.1
+    SERVER_ADDRESS=10.75.205.1
+    NEXT_SERVER=10.75.205.1
+    BROADCAST=10.75.205.255
+    T1=1580
+    T2=2930
+    LIFETIME=3600
+    DNS=10.75.205.1
+    DOMAINNAME=lxd
+    HOSTNAME=a1
+    CLIENTID=ffe617693400020000ab110c65a6a0866931c2
+    """)
+
+    lxd_parsed = {
+        'ADDRESS': '10.75.205.242',
+        'NETMASK': '255.255.255.0',
+        'ROUTER': '10.75.205.1',
+        'SERVER_ADDRESS': '10.75.205.1',
+        'NEXT_SERVER': '10.75.205.1',
+        'BROADCAST': '10.75.205.255',
+        'T1': '1580',
+        'T2': '2930',
+        'LIFETIME': '3600',
+        'DNS': '10.75.205.1',
+        'DOMAINNAME': 'lxd',
+        'HOSTNAME': 'a1',
+        'CLIENTID': 'ffe617693400020000ab110c65a6a0866931c2',
+    }
+
+    azure_lease = dedent("""\
+    # This is private data. Do not parse.
+    ADDRESS=10.132.0.5
+    NETMASK=255.255.255.255
+    ROUTER=10.132.0.1
+    SERVER_ADDRESS=169.254.169.254
+    NEXT_SERVER=10.132.0.1
+    MTU=1460
+    T1=43200
+    T2=75600
+    LIFETIME=86400
+    DNS=169.254.169.254
+    NTP=169.254.169.254
+    DOMAINNAME=c.ubuntu-foundations.internal
+    DOMAIN_SEARCH_LIST=c.ubuntu-foundations.internal google.internal
+    HOSTNAME=tribaal-test-171002-1349.c.ubuntu-foundations.internal
+    ROUTES=10.132.0.1/32,0.0.0.0 0.0.0.0/0,10.132.0.1
+    CLIENTID=ff405663a200020000ab11332859494d7a8b4c
+    OPTION_245=624c3620
+    """)
+
+    azure_parsed = {
+        'ADDRESS': '10.132.0.5',
+        'NETMASK': '255.255.255.255',
+        'ROUTER': '10.132.0.1',
+        'SERVER_ADDRESS': '169.254.169.254',
+        'NEXT_SERVER': '10.132.0.1',
+        'MTU': '1460',
+        'T1': '43200',
+        'T2': '75600',
+        'LIFETIME': '86400',
+        'DNS': '169.254.169.254',
+        'NTP': '169.254.169.254',
+        'DOMAINNAME': 'c.ubuntu-foundations.internal',
+        'DOMAIN_SEARCH_LIST': 'c.ubuntu-foundations.internal google.internal',
+        'HOSTNAME': 'tribaal-test-171002-1349.c.ubuntu-foundations.internal',
+        'ROUTES': '10.132.0.1/32,0.0.0.0 0.0.0.0/0,10.132.0.1',
+        'CLIENTID': 'ff405663a200020000ab11332859494d7a8b4c',
+        'OPTION_245': '624c3620'}
+
+    def setUp(self):
+        super(TestSystemdParseLeases, self).setUp()
+        self.lease_d = self.tmp_dir()
+
+    def test_no_leases_returns_empty_dict(self):
+        """A leases dir with no lease files should return empty dictionary."""
+        self.assertEqual({}, networkd_load_leases(self.lease_d))
+
+    def test_no_leases_dir_returns_empty_dict(self):
+        """A non-existing leases dir should return empty dict."""
+        enodir = os.path.join(self.lease_d, 'does-not-exist')
+        self.assertEqual({}, networkd_load_leases(enodir))
+
+    def test_single_leases_file(self):
+        """A leases dir with one leases file."""
+        populate_dir(self.lease_d, {'2': self.lxd_lease})
+        self.assertEqual(
+            {'2': self.lxd_parsed}, networkd_load_leases(self.lease_d))
+
+    def test_single_azure_leases_file(self):
+        """On Azure, option 245 should be present, verify it specifically."""
+        populate_dir(self.lease_d, {'1': self.azure_lease})
+        self.assertEqual(
+            {'1': self.azure_parsed}, networkd_load_leases(self.lease_d))
+
+    def test_multiple_files(self):
+        """Multiple leases files on azure with one found return that value."""
+        self.maxDiff = None
+        populate_dir(self.lease_d, {'1': self.azure_lease,
+                                    '9': self.lxd_lease})
+        self.assertEqual({'1': self.azure_parsed, '9': self.lxd_parsed},
+                         networkd_load_leases(self.lease_d))
diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index 7e0f9bb8..9dc473fc 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -19,6 +19,7 @@ import time
 
 from cloudinit import ec2_utils as ec2
 from cloudinit import log as logging
+from cloudinit.net import dhcp
 from cloudinit import sources
 from cloudinit import url_helper as uhelp
 from cloudinit import util
@@ -224,20 +225,28 @@ def get_vr_address():
     # Get the address of the virtual router via dhcp leases
     # If no virtual router is detected, fallback on default gateway.
     # See http://docs.cloudstack.apache.org/projects/cloudstack-administration/en/4.8/virtual_machines/user-data.html # noqa
+
+    # Try networkd first...
+    latest_address = dhcp.networkd_get_option_from_leases('SERVER_ADDRESS')
+    if latest_address:
+        LOG.debug("Found SERVER_ADDRESS '%s' via networkd_leases",
+                  latest_address)
+        return latest_address
+
+    # Try dhcp lease files next...
     lease_file = get_latest_lease()
     if not lease_file:
         LOG.debug("No lease file found, using default gateway")
         return get_default_gateway()
 
-    latest_address = None
     with open(lease_file, "r") as fd:
         for line in fd:
             if "dhcp-server-identifier" in line:
                 words = line.strip(" ;\r\n").split(" ")
                 if len(words) > 2:
-                    dhcp = words[2]
-                    LOG.debug("Found DHCP identifier %s", dhcp)
-                    latest_address = dhcp
+                    dhcptok = words[2]
+                    LOG.debug("Found DHCP identifier %s", dhcptok)
+                    latest_address = dhcptok
     if not latest_address:
         # No virtual router found, fallback on default gateway
         LOG.debug("No DHCP found, using default gateway")
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index 28ed0ae2..959b1bda 100644
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -8,6 +8,7 @@ import socket
 import struct
 import time
 
+from cloudinit.net import dhcp
 from cloudinit import stages
 from cloudinit import temp_utils
 from contextlib import contextmanager
@@ -15,7 +16,6 @@ from xml.etree import ElementTree
 
 from cloudinit import util
 
-
 LOG = logging.getLogger(__name__)
 
 
@@ -238,6 +238,11 @@ class WALinuxAgentShim(object):
             packed_bytes = unescaped_value.encode('utf-8')
         return socket.inet_ntoa(packed_bytes)
 
+    @staticmethod
+    def _networkd_get_value_from_leases(leases_d=None):
+        return dhcp.networkd_get_option_from_leases(
+            'OPTION_245', leases_d=leases_d)
+
     @staticmethod
     def _get_value_from_leases_file(fallback_lease_file):
         leases = []
@@ -287,12 +292,15 @@ class WALinuxAgentShim(object):
 
     @staticmethod
     def find_endpoint(fallback_lease_file=None):
-        LOG.debug('Finding Azure endpoint...')
         value = None
-        # Option-245 stored in /run/cloud-init/dhclient.hooks/<ifc>.json
-        # a dhclient exit hook that calls cloud-init-dhclient-hook
-        dhcp_options = WALinuxAgentShim._load_dhclient_json()
-        value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options)
+        LOG.debug('Finding Azure endpoint from networkd...')
+        value = WALinuxAgentShim._networkd_get_value_from_leases()
+        if value is None:
+            # Option-245 stored in /run/cloud-init/dhclient.hooks/<ifc>.json
+            # a dhclient exit hook that calls cloud-init-dhclient-hook
+            LOG.debug('Finding Azure endpoint from hook json...')
+            dhcp_options = WALinuxAgentShim._load_dhclient_json()
+            value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options)
         if value is None:
             # Fallback and check the leases file if unsuccessful
             LOG.debug("Unable to find endpoint in dhclient logs. "
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
index 44b99eca..b42b073f 100644
--- a/tests/unittests/test_datasource/test_azure_helper.py
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -1,10 +1,12 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import os
+from textwrap import dedent
 
 from cloudinit.sources.helpers import azure as azure_helper
-from cloudinit.tests.helpers import ExitStack, mock, TestCase
+from cloudinit.tests.helpers import CiTestCase, ExitStack, mock, populate_dir
 
+from cloudinit.sources.helpers.azure import WALinuxAgentShim as wa_shim
 
 GOAL_STATE_TEMPLATE = """\
 <?xml version="1.0" encoding="utf-8"?>
@@ -45,7 +47,7 @@ GOAL_STATE_TEMPLATE = """\
 """
 
 
-class TestFindEndpoint(TestCase):
+class TestFindEndpoint(CiTestCase):
 
     def setUp(self):
         super(TestFindEndpoint, self).setUp()
@@ -56,18 +58,19 @@ class TestFindEndpoint(TestCase):
             mock.patch.object(azure_helper.util, 'load_file'))
 
         self.dhcp_options = patches.enter_context(
-            mock.patch.object(azure_helper.WALinuxAgentShim,
-                              '_load_dhclient_json'))
+            mock.patch.object(wa_shim, '_load_dhclient_json'))
+
+        self.networkd_leases = patches.enter_context(
+            mock.patch.object(wa_shim, '_networkd_get_value_from_leases'))
+        self.networkd_leases.return_value = None
 
     def test_missing_file(self):
-        self.assertRaises(ValueError,
-                          azure_helper.WALinuxAgentShim.find_endpoint)
+        self.assertRaises(ValueError, wa_shim.find_endpoint)
 
     def test_missing_special_azure_line(self):
         self.load_file.return_value = ''
         self.dhcp_options.return_value = {'eth0': {'key': 'value'}}
-        self.assertRaises(ValueError,
-                          azure_helper.WALinuxAgentShim.find_endpoint)
+        self.assertRaises(ValueError, wa_shim.find_endpoint)
 
     @staticmethod
     def _build_lease_content(encoded_address):
@@ -80,8 +83,7 @@ class TestFindEndpoint(TestCase):
 
     def test_from_dhcp_client(self):
         self.dhcp_options.return_value = {"eth0": {"unknown_245": "5:4:3:2"}}
-        self.assertEqual('5.4.3.2',
-                         azure_helper.WALinuxAgentShim.find_endpoint(None))
+        self.assertEqual('5.4.3.2', wa_shim.find_endpoint(None))
 
     def test_latest_lease_used(self):
         encoded_addresses = ['5:4:3:2', '4:3:2:1']
@@ -89,53 +91,38 @@ class TestFindEndpoint(TestCase):
                                   for encoded_address in encoded_addresses])
         self.load_file.return_value = file_content
         self.assertEqual(encoded_addresses[-1].replace(':', '.'),
-                         azure_helper.WALinuxAgentShim.find_endpoint("foobar"))
+                         wa_shim.find_endpoint("foobar"))
 
 
-class TestExtractIpAddressFromLeaseValue(TestCase):
+class TestExtractIpAddressFromLeaseValue(CiTestCase):
 
     def test_hex_string(self):
         ip_address, encoded_address = '98.76.54.32', '62:4c:36:20'
         self.assertEqual(
-            ip_address,
-            azure_helper.WALinuxAgentShim.get_ip_from_lease_value(
-                encoded_address
-            ))
+            ip_address, wa_shim.get_ip_from_lease_value(encoded_address))
 
     def test_hex_string_with_single_character_part(self):
         ip_address, encoded_address = '4.3.2.1', '4:3:2:1'
         self.assertEqual(
-            ip_address,
-            azure_helper.WALinuxAgentShim.get_ip_from_lease_value(
-                encoded_address
-            ))
+            ip_address, wa_shim.get_ip_from_lease_value(encoded_address))
 
     def test_packed_string(self):
         ip_address, encoded_address = '98.76.54.32', 'bL6 '
         self.assertEqual(
-            ip_address,
-            azure_helper.WALinuxAgentShim.get_ip_from_lease_value(
-                encoded_address
-            ))
+            ip_address, wa_shim.get_ip_from_lease_value(encoded_address))
 
     def test_packed_string_with_escaped_quote(self):
         ip_address, encoded_address = '100.72.34.108', 'dH\\"l'
         self.assertEqual(
-            ip_address,
-            azure_helper.WALinuxAgentShim.get_ip_from_lease_value(
-                encoded_address
-            ))
+            ip_address, wa_shim.get_ip_from_lease_value(encoded_address))
 
     def test_packed_string_containing_a_colon(self):
         ip_address, encoded_address = '100.72.58.108', 'dH:l'
         self.assertEqual(
-            ip_address,
-            azure_helper.WALinuxAgentShim.get_ip_from_lease_value(
-                encoded_address
-            ))
+            ip_address, wa_shim.get_ip_from_lease_value(encoded_address))
 
 
-class TestGoalStateParsing(TestCase):
+class TestGoalStateParsing(CiTestCase):
 
     default_parameters = {
         'incarnation': 1,
@@ -195,7 +182,7 @@ class TestGoalStateParsing(TestCase):
         self.assertIsNone(certificates_xml)
 
 
-class TestAzureEndpointHttpClient(TestCase):
+class TestAzureEndpointHttpClient(CiTestCase):
 
     regular_headers = {
         'x-ms-agent-name': 'WALinuxAgent',
@@ -258,7 +245,7 @@ class TestAzureEndpointHttpClient(TestCase):
             self.read_file_or_url.call_args)
 
 
-class TestOpenSSLManager(TestCase):
+class TestOpenSSLManager(CiTestCase):
 
     def setUp(self):
         super(TestOpenSSLManager, self).setUp()
@@ -300,7 +287,7 @@ class TestOpenSSLManager(TestCase):
         self.assertEqual([mock.call(manager.tmpdir)], del_dir.call_args_list)
 
 
-class TestWALinuxAgentShim(TestCase):
+class TestWALinuxAgentShim(CiTestCase):
 
     def setUp(self):
         super(TestWALinuxAgentShim, self).setUp()
@@ -310,8 +297,7 @@ class TestWALinuxAgentShim(TestCase):
         self.AzureEndpointHttpClient = patches.enter_context(
             mock.patch.object(azure_helper, 'AzureEndpointHttpClient'))
         self.find_endpoint = patches.enter_context(
-            mock.patch.object(
-                azure_helper.WALinuxAgentShim, 'find_endpoint'))
+            mock.patch.object(wa_shim, 'find_endpoint'))
         self.GoalState = patches.enter_context(
             mock.patch.object(azure_helper, 'GoalState'))
         self.OpenSSLManager = patches.enter_context(
@@ -320,7 +306,7 @@ class TestWALinuxAgentShim(TestCase):
             mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock()))
 
     def test_http_client_uses_certificate(self):
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         shim.register_with_azure_and_fetch_data()
         self.assertEqual(
             [mock.call(self.OpenSSLManager.return_value.certificate)],
@@ -328,7 +314,7 @@ class TestWALinuxAgentShim(TestCase):
 
     def test_correct_url_used_for_goalstate(self):
         self.find_endpoint.return_value = 'test_endpoint'
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         shim.register_with_azure_and_fetch_data()
         get = self.AzureEndpointHttpClient.return_value.get
         self.assertEqual(
@@ -340,7 +326,7 @@ class TestWALinuxAgentShim(TestCase):
             self.GoalState.call_args_list)
 
     def test_certificates_used_to_determine_public_keys(self):
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         data = shim.register_with_azure_and_fetch_data()
         self.assertEqual(
             [mock.call(self.GoalState.return_value.certificates_xml)],
@@ -351,13 +337,13 @@ class TestWALinuxAgentShim(TestCase):
 
     def test_absent_certificates_produces_empty_public_keys(self):
         self.GoalState.return_value.certificates_xml = None
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         data = shim.register_with_azure_and_fetch_data()
         self.assertEqual([], data['public-keys'])
 
     def test_correct_url_used_for_report_ready(self):
         self.find_endpoint.return_value = 'test_endpoint'
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         shim.register_with_azure_and_fetch_data()
         expected_url = 'http://test_endpoint/machine?comp=health'
         self.assertEqual(
@@ -368,7 +354,7 @@ class TestWALinuxAgentShim(TestCase):
         self.GoalState.return_value.incarnation = 'TestIncarnation'
         self.GoalState.return_value.container_id = 'TestContainerId'
         self.GoalState.return_value.instance_id = 'TestInstanceId'
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         shim.register_with_azure_and_fetch_data()
         posted_document = (
             self.AzureEndpointHttpClient.return_value.post.call_args[1]['data']
@@ -378,11 +364,11 @@ class TestWALinuxAgentShim(TestCase):
         self.assertIn('TestInstanceId', posted_document)
 
     def test_clean_up_can_be_called_at_any_time(self):
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         shim.clean_up()
 
     def test_clean_up_will_clean_up_openssl_manager_if_instantiated(self):
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         shim.register_with_azure_and_fetch_data()
         shim.clean_up()
         self.assertEqual(
@@ -393,12 +379,12 @@ class TestWALinuxAgentShim(TestCase):
             pass
         self.AzureEndpointHttpClient.return_value.get.side_effect = (
             SentinelException)
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         self.assertRaises(SentinelException,
                           shim.register_with_azure_and_fetch_data)
 
 
-class TestGetMetadataFromFabric(TestCase):
+class TestGetMetadataFromFabric(CiTestCase):
 
     @mock.patch.object(azure_helper, 'WALinuxAgentShim')
     def test_data_from_shim_returned(self, shim):
@@ -422,4 +408,65 @@ class TestGetMetadataFromFabric(TestCase):
                           azure_helper.get_metadata_from_fabric)
         self.assertEqual(1, shim.return_value.clean_up.call_count)
 
+
+class TestExtractIpAddressFromNetworkd(CiTestCase):
+
+    azure_lease = dedent("""\
+    # This is private data. Do not parse.
+    ADDRESS=10.132.0.5
+    NETMASK=255.255.255.255
+    ROUTER=10.132.0.1
+    SERVER_ADDRESS=169.254.169.254
+    NEXT_SERVER=10.132.0.1
+    MTU=1460
+    T1=43200
+    T2=75600
+    LIFETIME=86400
+    DNS=169.254.169.254
+    NTP=169.254.169.254
+    DOMAINNAME=c.ubuntu-foundations.internal
+    DOMAIN_SEARCH_LIST=c.ubuntu-foundations.internal google.internal
+    HOSTNAME=tribaal-test-171002-1349.c.ubuntu-foundations.internal
+    ROUTES=10.132.0.1/32,0.0.0.0 0.0.0.0/0,10.132.0.1
+    CLIENTID=ff405663a200020000ab11332859494d7a8b4c
+    OPTION_245=624c3620
+    """)
+
+    def setUp(self):
+        super(TestExtractIpAddressFromNetworkd, self).setUp()
+        self.lease_d = self.tmp_dir()
+
+    def test_no_valid_leases_is_none(self):
+        """No valid leases should return None."""
+        self.assertIsNone(
+            wa_shim._networkd_get_value_from_leases(self.lease_d))
+
+    def test_option_245_is_found_in_single(self):
+        """A single valid lease with 245 option should return it."""
+        populate_dir(self.lease_d, {'9': self.azure_lease})
+        self.assertEqual(
+            '624c3620', wa_shim._networkd_get_value_from_leases(self.lease_d))
+
+    def test_option_245_not_found_returns_None(self):
+        """A valid lease, but no option 245 should return None."""
+        populate_dir(
+            self.lease_d,
+            {'9': self.azure_lease.replace("OPTION_245", "OPTION_999")})
+        self.assertIsNone(
+            wa_shim._networkd_get_value_from_leases(self.lease_d))
+
+    def test_multiple_returns_first(self):
+        """Somewhat arbitrarily return the first address when multiple.
+
+        Most important at the moment is that this is consistent behavior
+        rather than changing randomly as in order of a dictionary."""
+        myval = "624c3601"
+        populate_dir(
+            self.lease_d,
+            {'9': self.azure_lease,
+             '2': self.azure_lease.replace("624c3620", myval)})
+        self.assertEqual(
+            myval, wa_shim._networkd_get_value_from_leases(self.lease_d))
+
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py
index 8e98e1bb..96144b64 100644
--- a/tests/unittests/test_datasource/test_cloudstack.py
+++ b/tests/unittests/test_datasource/test_cloudstack.py
@@ -23,13 +23,16 @@ class TestCloudStackPasswordFetching(CiTestCase):
         default_gw = "192.201.20.0"
         get_latest_lease = mock.MagicMock(return_value=None)
         self.patches.enter_context(mock.patch(
-            'cloudinit.sources.DataSourceCloudStack.get_latest_lease',
-            get_latest_lease))
+            mod_name + '.get_latest_lease', get_latest_lease))
 
         get_default_gw = mock.MagicMock(return_value=default_gw)
         self.patches.enter_context(mock.patch(
-            'cloudinit.sources.DataSourceCloudStack.get_default_gateway',
-            get_default_gw))
+            mod_name + '.get_default_gateway', get_default_gw))
+
+        get_networkd_server_address = mock.MagicMock(return_value=None)
+        self.patches.enter_context(mock.patch(
+            mod_name + '.dhcp.networkd_get_option_from_leases',
+            get_networkd_server_address))
 
     def _set_password_server_response(self, response_string):
         subp = mock.MagicMock(return_value=(response_string, ''))
-- 
cgit v1.2.3