merge from master at 17.1-17-g45d361cb

18 files changed, 631 insertions, 116 deletions

diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py
index 990bff2c..82ee9714 100644
--- a/tests/unittests/test_datasource/test_aliyun.py
+++ b/tests/unittests/test_datasource/test_aliyun.py
@@ -5,9 +5,9 @@ import httpretty
 import mock
 import os
 
-from .. import helpers as test_helpers
 from cloudinit import helpers
 from cloudinit.sources import DataSourceAliYun as ay
+from cloudinit.tests import helpers as test_helpers
 
 DEFAULT_METADATA = {
     'instance-id': 'aliyun-test-vm-00',
@@ -70,7 +70,6 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
         paths = helpers.Paths({})
         self.ds = ay.DataSourceAliYun(cfg, distro, paths)
         self.metadata_address = self.ds.metadata_urls[0]
-        self.api_ver = self.ds.api_ver
 
     @property
     def default_metadata(self):
@@ -82,13 +81,15 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
 
     @property
     def metadata_url(self):
-        return os.path.join(self.metadata_address,
-                            self.api_ver, 'meta-data') + '/'
+        return os.path.join(
+            self.metadata_address,
+            self.ds.min_metadata_version, 'meta-data') + '/'
 
     @property
     def userdata_url(self):
-        return os.path.join(self.metadata_address,
-                            self.api_ver, 'user-data')
+        return os.path.join(
+            self.metadata_address,
+            self.ds.min_metadata_version, 'user-data')
 
     def regist_default_server(self):
         register_mock_metaserver(self.metadata_url, self.default_metadata)
diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py
index 9c46abc1..a4dfb540 100644
--- a/tests/unittests/test_datasource/test_altcloud.py
+++ b/tests/unittests/test_datasource/test_altcloud.py
@@ -18,7 +18,7 @@ import tempfile
 from cloudinit import helpers
 from cloudinit import util
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 import cloudinit.sources.DataSourceAltCloud as dsac
 
@@ -280,8 +280,8 @@ class TestUserDataRhevm(TestCase):
             pass
 
         dsac.CLOUD_INFO_FILE = '/etc/sysconfig/cloud-info'
-        dsac.CMD_PROBE_FLOPPY = ['/sbin/modprobe', 'floppy']
-        dsac.CMD_UDEVADM_SETTLE = ['/sbin/udevadm', 'settle',
+        dsac.CMD_PROBE_FLOPPY = ['modprobe', 'floppy']
+        dsac.CMD_UDEVADM_SETTLE = ['udevadm', 'settle',
                                    '--quiet', '--timeout=5']
 
     def test_mount_cb_fails(self):
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 20e70fb7..0a117771 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -6,8 +6,8 @@ from cloudinit.sources import DataSourceAzure as dsaz
 from cloudinit.util import find_freebsd_part
 from cloudinit.util import get_path_dev_freebsd
 
-from ..helpers import (CiTestCase, TestCase, populate_dir, mock,
-                       ExitStack, PY26, SkipTest)
+from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock,
+                                     ExitStack, PY26, SkipTest)
 
 import crypt
 import os
@@ -871,6 +871,7 @@ class TestLoadAzureDsDir(CiTestCase):
 
 
 class TestReadAzureOvf(TestCase):
+
     def test_invalid_xml_raises_non_azure_ds(self):
         invalid_xml = "<foo>" + construct_valid_ovf_env(data={})
         self.assertRaises(dsaz.BrokenAzureDataSource,
@@ -1079,6 +1080,7 @@ class TestCanDevBeReformatted(CiTestCase):
 
 
 class TestAzureNetExists(CiTestCase):
+
     def test_azure_net_must_exist_for_legacy_objpkl(self):
         """DataSourceAzureNet must exist for old obj.pkl files
            that reference it."""
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
index b2d2971b..b42b073f 100644
--- a/tests/unittests/test_datasource/test_azure_helper.py
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -1,10 +1,12 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import os
+from textwrap import dedent
 
 from cloudinit.sources.helpers import azure as azure_helper
-from ..helpers import ExitStack, mock, TestCase
+from cloudinit.tests.helpers import CiTestCase, ExitStack, mock, populate_dir
 
+from cloudinit.sources.helpers.azure import WALinuxAgentShim as wa_shim
 
 GOAL_STATE_TEMPLATE = """\
 <?xml version="1.0" encoding="utf-8"?>
@@ -45,7 +47,7 @@ GOAL_STATE_TEMPLATE = """\
 """
 
 
-class TestFindEndpoint(TestCase):
+class TestFindEndpoint(CiTestCase):
 
     def setUp(self):
         super(TestFindEndpoint, self).setUp()
@@ -56,18 +58,19 @@ class TestFindEndpoint(TestCase):
             mock.patch.object(azure_helper.util, 'load_file'))
 
         self.dhcp_options = patches.enter_context(
-            mock.patch.object(azure_helper.WALinuxAgentShim,
-                              '_load_dhclient_json'))
+            mock.patch.object(wa_shim, '_load_dhclient_json'))
+
+        self.networkd_leases = patches.enter_context(
+            mock.patch.object(wa_shim, '_networkd_get_value_from_leases'))
+        self.networkd_leases.return_value = None
 
     def test_missing_file(self):
-        self.assertRaises(ValueError,
-                          azure_helper.WALinuxAgentShim.find_endpoint)
+        self.assertRaises(ValueError, wa_shim.find_endpoint)
 
     def test_missing_special_azure_line(self):
         self.load_file.return_value = ''
         self.dhcp_options.return_value = {'eth0': {'key': 'value'}}
-        self.assertRaises(ValueError,
-                          azure_helper.WALinuxAgentShim.find_endpoint)
+        self.assertRaises(ValueError, wa_shim.find_endpoint)
 
     @staticmethod
     def _build_lease_content(encoded_address):
@@ -80,8 +83,7 @@ class TestFindEndpoint(TestCase):
 
     def test_from_dhcp_client(self):
         self.dhcp_options.return_value = {"eth0": {"unknown_245": "5:4:3:2"}}
-        self.assertEqual('5.4.3.2',
-                         azure_helper.WALinuxAgentShim.find_endpoint(None))
+        self.assertEqual('5.4.3.2', wa_shim.find_endpoint(None))
 
     def test_latest_lease_used(self):
         encoded_addresses = ['5:4:3:2', '4:3:2:1']
@@ -89,53 +91,38 @@ class TestFindEndpoint(TestCase):
                                   for encoded_address in encoded_addresses])
         self.load_file.return_value = file_content
         self.assertEqual(encoded_addresses[-1].replace(':', '.'),
-                         azure_helper.WALinuxAgentShim.find_endpoint("foobar"))
+                         wa_shim.find_endpoint("foobar"))
 
 
-class TestExtractIpAddressFromLeaseValue(TestCase):
+class TestExtractIpAddressFromLeaseValue(CiTestCase):
 
     def test_hex_string(self):
         ip_address, encoded_address = '98.76.54.32', '62:4c:36:20'
         self.assertEqual(
-            ip_address,
-            azure_helper.WALinuxAgentShim.get_ip_from_lease_value(
-                encoded_address
-            ))
+            ip_address, wa_shim.get_ip_from_lease_value(encoded_address))
 
     def test_hex_string_with_single_character_part(self):
         ip_address, encoded_address = '4.3.2.1', '4:3:2:1'
         self.assertEqual(
-            ip_address,
-            azure_helper.WALinuxAgentShim.get_ip_from_lease_value(
-                encoded_address
-            ))
+            ip_address, wa_shim.get_ip_from_lease_value(encoded_address))
 
     def test_packed_string(self):
         ip_address, encoded_address = '98.76.54.32', 'bL6 '
         self.assertEqual(
-            ip_address,
-            azure_helper.WALinuxAgentShim.get_ip_from_lease_value(
-                encoded_address
-            ))
+            ip_address, wa_shim.get_ip_from_lease_value(encoded_address))
 
     def test_packed_string_with_escaped_quote(self):
         ip_address, encoded_address = '100.72.34.108', 'dH\\"l'
         self.assertEqual(
-            ip_address,
-            azure_helper.WALinuxAgentShim.get_ip_from_lease_value(
-                encoded_address
-            ))
+            ip_address, wa_shim.get_ip_from_lease_value(encoded_address))
 
     def test_packed_string_containing_a_colon(self):
         ip_address, encoded_address = '100.72.58.108', 'dH:l'
         self.assertEqual(
-            ip_address,
-            azure_helper.WALinuxAgentShim.get_ip_from_lease_value(
-                encoded_address
-            ))
+            ip_address, wa_shim.get_ip_from_lease_value(encoded_address))
 
 
-class TestGoalStateParsing(TestCase):
+class TestGoalStateParsing(CiTestCase):
 
     default_parameters = {
         'incarnation': 1,
@@ -195,7 +182,7 @@ class TestGoalStateParsing(TestCase):
         self.assertIsNone(certificates_xml)
 
 
-class TestAzureEndpointHttpClient(TestCase):
+class TestAzureEndpointHttpClient(CiTestCase):
 
     regular_headers = {
         'x-ms-agent-name': 'WALinuxAgent',
@@ -258,7 +245,7 @@ class TestAzureEndpointHttpClient(TestCase):
             self.read_file_or_url.call_args)
 
 
-class TestOpenSSLManager(TestCase):
+class TestOpenSSLManager(CiTestCase):
 
     def setUp(self):
         super(TestOpenSSLManager, self).setUp()
@@ -275,7 +262,7 @@ class TestOpenSSLManager(TestCase):
                 mock.patch('builtins.open'))
 
     @mock.patch.object(azure_helper, 'cd', mock.MagicMock())
-    @mock.patch.object(azure_helper.tempfile, 'mkdtemp')
+    @mock.patch.object(azure_helper.temp_utils, 'mkdtemp')
     def test_openssl_manager_creates_a_tmpdir(self, mkdtemp):
         manager = azure_helper.OpenSSLManager()
         self.assertEqual(mkdtemp.return_value, manager.tmpdir)
@@ -292,7 +279,7 @@ class TestOpenSSLManager(TestCase):
         manager.clean_up()
 
     @mock.patch.object(azure_helper, 'cd', mock.MagicMock())
-    @mock.patch.object(azure_helper.tempfile, 'mkdtemp', mock.MagicMock())
+    @mock.patch.object(azure_helper.temp_utils, 'mkdtemp', mock.MagicMock())
     @mock.patch.object(azure_helper.util, 'del_dir')
     def test_clean_up(self, del_dir):
         manager = azure_helper.OpenSSLManager()
@@ -300,7 +287,7 @@ class TestOpenSSLManager(TestCase):
         self.assertEqual([mock.call(manager.tmpdir)], del_dir.call_args_list)
 
 
-class TestWALinuxAgentShim(TestCase):
+class TestWALinuxAgentShim(CiTestCase):
 
     def setUp(self):
         super(TestWALinuxAgentShim, self).setUp()
@@ -310,8 +297,7 @@ class TestWALinuxAgentShim(TestCase):
         self.AzureEndpointHttpClient = patches.enter_context(
             mock.patch.object(azure_helper, 'AzureEndpointHttpClient'))
         self.find_endpoint = patches.enter_context(
-            mock.patch.object(
-                azure_helper.WALinuxAgentShim, 'find_endpoint'))
+            mock.patch.object(wa_shim, 'find_endpoint'))
         self.GoalState = patches.enter_context(
             mock.patch.object(azure_helper, 'GoalState'))
         self.OpenSSLManager = patches.enter_context(
@@ -320,7 +306,7 @@ class TestWALinuxAgentShim(TestCase):
             mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock()))
 
     def test_http_client_uses_certificate(self):
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         shim.register_with_azure_and_fetch_data()
         self.assertEqual(
             [mock.call(self.OpenSSLManager.return_value.certificate)],
@@ -328,7 +314,7 @@ class TestWALinuxAgentShim(TestCase):
 
     def test_correct_url_used_for_goalstate(self):
         self.find_endpoint.return_value = 'test_endpoint'
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         shim.register_with_azure_and_fetch_data()
         get = self.AzureEndpointHttpClient.return_value.get
         self.assertEqual(
@@ -340,7 +326,7 @@ class TestWALinuxAgentShim(TestCase):
             self.GoalState.call_args_list)
 
     def test_certificates_used_to_determine_public_keys(self):
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         data = shim.register_with_azure_and_fetch_data()
         self.assertEqual(
             [mock.call(self.GoalState.return_value.certificates_xml)],
@@ -351,13 +337,13 @@ class TestWALinuxAgentShim(TestCase):
 
     def test_absent_certificates_produces_empty_public_keys(self):
         self.GoalState.return_value.certificates_xml = None
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         data = shim.register_with_azure_and_fetch_data()
         self.assertEqual([], data['public-keys'])
 
     def test_correct_url_used_for_report_ready(self):
         self.find_endpoint.return_value = 'test_endpoint'
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         shim.register_with_azure_and_fetch_data()
         expected_url = 'http://test_endpoint/machine?comp=health'
         self.assertEqual(
@@ -368,7 +354,7 @@ class TestWALinuxAgentShim(TestCase):
         self.GoalState.return_value.incarnation = 'TestIncarnation'
         self.GoalState.return_value.container_id = 'TestContainerId'
         self.GoalState.return_value.instance_id = 'TestInstanceId'
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         shim.register_with_azure_and_fetch_data()
         posted_document = (
             self.AzureEndpointHttpClient.return_value.post.call_args[1]['data']
@@ -378,11 +364,11 @@ class TestWALinuxAgentShim(TestCase):
         self.assertIn('TestInstanceId', posted_document)
 
     def test_clean_up_can_be_called_at_any_time(self):
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         shim.clean_up()
 
     def test_clean_up_will_clean_up_openssl_manager_if_instantiated(self):
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         shim.register_with_azure_and_fetch_data()
         shim.clean_up()
         self.assertEqual(
@@ -393,12 +379,12 @@ class TestWALinuxAgentShim(TestCase):
             pass
         self.AzureEndpointHttpClient.return_value.get.side_effect = (
             SentinelException)
-        shim = azure_helper.WALinuxAgentShim()
+        shim = wa_shim()
         self.assertRaises(SentinelException,
                           shim.register_with_azure_and_fetch_data)
 
 
-class TestGetMetadataFromFabric(TestCase):
+class TestGetMetadataFromFabric(CiTestCase):
 
     @mock.patch.object(azure_helper, 'WALinuxAgentShim')
     def test_data_from_shim_returned(self, shim):
@@ -422,4 +408,65 @@ class TestGetMetadataFromFabric(TestCase):
                           azure_helper.get_metadata_from_fabric)
         self.assertEqual(1, shim.return_value.clean_up.call_count)
 
+
+class TestExtractIpAddressFromNetworkd(CiTestCase):
+
+    azure_lease = dedent("""\
+    # This is private data. Do not parse.
+    ADDRESS=10.132.0.5
+    NETMASK=255.255.255.255
+    ROUTER=10.132.0.1
+    SERVER_ADDRESS=169.254.169.254
+    NEXT_SERVER=10.132.0.1
+    MTU=1460
+    T1=43200
+    T2=75600
+    LIFETIME=86400
+    DNS=169.254.169.254
+    NTP=169.254.169.254
+    DOMAINNAME=c.ubuntu-foundations.internal
+    DOMAIN_SEARCH_LIST=c.ubuntu-foundations.internal google.internal
+    HOSTNAME=tribaal-test-171002-1349.c.ubuntu-foundations.internal
+    ROUTES=10.132.0.1/32,0.0.0.0 0.0.0.0/0,10.132.0.1
+    CLIENTID=ff405663a200020000ab11332859494d7a8b4c
+    OPTION_245=624c3620
+    """)
+
+    def setUp(self):
+        super(TestExtractIpAddressFromNetworkd, self).setUp()
+        self.lease_d = self.tmp_dir()
+
+    def test_no_valid_leases_is_none(self):
+        """No valid leases should return None."""
+        self.assertIsNone(
+            wa_shim._networkd_get_value_from_leases(self.lease_d))
+
+    def test_option_245_is_found_in_single(self):
+        """A single valid lease with 245 option should return it."""
+        populate_dir(self.lease_d, {'9': self.azure_lease})
+        self.assertEqual(
+            '624c3620', wa_shim._networkd_get_value_from_leases(self.lease_d))
+
+    def test_option_245_not_found_returns_None(self):
+        """A valid lease, but no option 245 should return None."""
+        populate_dir(
+            self.lease_d,
+            {'9': self.azure_lease.replace("OPTION_245", "OPTION_999")})
+        self.assertIsNone(
+            wa_shim._networkd_get_value_from_leases(self.lease_d))
+
+    def test_multiple_returns_first(self):
+        """Somewhat arbitrarily return the first address when multiple.
+
+        Most important at the moment is that this is consistent behavior
+        rather than changing randomly as in order of a dictionary."""
+        myval = "624c3601"
+        populate_dir(
+            self.lease_d,
+            {'9': self.azure_lease,
+             '2': self.azure_lease.replace("624c3620", myval)})
+        self.assertEqual(
+            myval, wa_shim._networkd_get_value_from_leases(self.lease_d))
+
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py
index 5997102c..e4c59907 100644
--- a/tests/unittests/test_datasource/test_cloudsigma.py
+++ b/tests/unittests/test_datasource/test_cloudsigma.py
@@ -6,7 +6,7 @@ from cloudinit.cs_utils import Cepko
 from cloudinit import sources
 from cloudinit.sources import DataSourceCloudSigma
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 SERVER_CONTEXT = {
     "cpu": 1000,
diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py
index e94aad61..96144b64 100644
--- a/tests/unittests/test_datasource/test_cloudstack.py
+++ b/tests/unittests/test_datasource/test_cloudstack.py
@@ -1,12 +1,17 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit import helpers
-from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack
+from cloudinit import util
+from cloudinit.sources.DataSourceCloudStack import (
+    DataSourceCloudStack, get_latest_lease)
 
-from ..helpers import TestCase, mock, ExitStack
+from cloudinit.tests.helpers import CiTestCase, ExitStack, mock
 
+import os
+import time
 
-class TestCloudStackPasswordFetching(TestCase):
+
+class TestCloudStackPasswordFetching(CiTestCase):
 
     def setUp(self):
         super(TestCloudStackPasswordFetching, self).setUp()
@@ -18,13 +23,16 @@ class TestCloudStackPasswordFetching(TestCase):
         default_gw = "192.201.20.0"
         get_latest_lease = mock.MagicMock(return_value=None)
         self.patches.enter_context(mock.patch(
-            'cloudinit.sources.DataSourceCloudStack.get_latest_lease',
-            get_latest_lease))
+            mod_name + '.get_latest_lease', get_latest_lease))
 
         get_default_gw = mock.MagicMock(return_value=default_gw)
         self.patches.enter_context(mock.patch(
-            'cloudinit.sources.DataSourceCloudStack.get_default_gateway',
-            get_default_gw))
+            mod_name + '.get_default_gateway', get_default_gw))
+
+        get_networkd_server_address = mock.MagicMock(return_value=None)
+        self.patches.enter_context(mock.patch(
+            mod_name + '.dhcp.networkd_get_option_from_leases',
+            get_networkd_server_address))
 
     def _set_password_server_response(self, response_string):
         subp = mock.MagicMock(return_value=(response_string, ''))
@@ -89,4 +97,72 @@ class TestCloudStackPasswordFetching(TestCase):
     def test_password_not_saved_if_bad_request(self):
         self._check_password_not_saved_for('bad_request')
 
+
+class TestGetLatestLease(CiTestCase):
+
+    def _populate_dir_list(self, bdir, files):
+        """populate_dir_list([(name, data), (name, data)])
+
+        writes files to bdir, and updates timestamps to ensure
+        that their mtime increases with each file."""
+
+        start = int(time.time())
+        for num, fname in enumerate(reversed(files)):
+            fpath = os.path.sep.join((bdir, fname))
+            util.write_file(fpath, fname.encode())
+            os.utime(fpath, (start - num, start - num))
+
+    def _pop_and_test(self, files, expected):
+        lease_d = self.tmp_dir()
+        self._populate_dir_list(lease_d, files)
+        self.assertEqual(self.tmp_path(expected, lease_d),
+                         get_latest_lease(lease_d))
+
+    def test_skips_dhcpv6_files(self):
+        """files started with dhclient6 should be skipped."""
+        expected = "dhclient.lease"
+        self._pop_and_test([expected, "dhclient6.lease"], expected)
+
+    def test_selects_dhclient_dot_files(self):
+        """files named dhclient.lease or dhclient.leases should be used.
+
+        Ubuntu names files dhclient.eth0.leases dhclient6.leases and
+        sometimes dhclient.leases."""
+        self._pop_and_test(["dhclient.lease"], "dhclient.lease")
+        self._pop_and_test(["dhclient.leases"], "dhclient.leases")
+
+    def test_selects_dhclient_dash_files(self):
+        """files named dhclient-lease or dhclient-leases should be used.
+
+        Redhat/Centos names files with dhclient--eth0.lease (centos 7) or
+        dhclient-eth0.leases (centos 6).
+        """
+        self._pop_and_test(["dhclient-eth0.lease"], "dhclient-eth0.lease")
+        self._pop_and_test(["dhclient--eth0.lease"], "dhclient--eth0.lease")
+
+    def test_ignores_by_extension(self):
+        """only .lease or .leases file should be considered."""
+
+        self._pop_and_test(["dhclient.lease", "dhclient.lease.bk",
+                            "dhclient.lease-old", "dhclient.leaselease"],
+                           "dhclient.lease")
+
+    def test_selects_newest_matching(self):
+        """If multiple files match, the newest written should be used."""
+        lease_d = self.tmp_dir()
+        valid_1 = "dhclient.leases"
+        valid_2 = "dhclient.lease"
+        valid_1_path = self.tmp_path(valid_1, lease_d)
+        valid_2_path = self.tmp_path(valid_2, lease_d)
+
+        self._populate_dir_list(lease_d, [valid_1, valid_2])
+        self.assertEqual(valid_2_path, get_latest_lease(lease_d))
+
+        # now update mtime on valid_2 to be older than valid_1 and re-check.
+        mtime = int(os.path.getmtime(valid_1_path)) - 1
+        os.utime(valid_2_path, (mtime, mtime))
+
+        self.assertEqual(valid_1_path, get_latest_lease(lease_d))
+
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
index 413e87ac..80b9c650 100644
--- a/tests/unittests/test_datasource/test_common.py
+++ b/tests/unittests/test_datasource/test_common.py
@@ -24,7 +24,7 @@ from cloudinit.sources import (
 )
 from cloudinit.sources import DataSourceNone as DSNone
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 DEFAULT_LOCAL = [
     Azure.DataSourceAzure,
@@ -35,6 +35,7 @@ DEFAULT_LOCAL = [
     OpenNebula.DataSourceOpenNebula,
     OVF.DataSourceOVF,
     SmartOS.DataSourceSmartOS,
+    Ec2.DataSourceEc2Local,
 ]
 
 DEFAULT_NETWORK = [
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index 337be667..237c189b 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -15,7 +15,7 @@ from cloudinit.sources import DataSourceConfigDrive as ds
 from cloudinit.sources.helpers import openstack
 from cloudinit import util
 
-from ..helpers import TestCase, ExitStack, mock
+from cloudinit.tests.helpers import TestCase, ExitStack, mock
 
 
 PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n'
diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py
index e97a679a..f264f361 100644
--- a/tests/unittests/test_datasource/test_digitalocean.py
+++ b/tests/unittests/test_datasource/test_digitalocean.py
@@ -13,7 +13,7 @@ from cloudinit import settings
 from cloudinit.sources import DataSourceDigitalOcean
 from cloudinit.sources.helpers import digitalocean
 
-from ..helpers import mock, TestCase
+from cloudinit.tests.helpers import mock, TestCase
 
 DO_MULTIPLE_KEYS = ["ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@do.co",
                     "ssh-rsa AAAAB3NzaC1yc2EAAAA... test2@do.co"]
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 12230ae2..a7301dbf 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -1,42 +1,75 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
+import copy
 import httpretty
 import mock
 
-from .. import helpers as test_helpers
 from cloudinit import helpers
 from cloudinit.sources import DataSourceEc2 as ec2
+from cloudinit.tests import helpers as test_helpers
 
 
-# collected from api version 2009-04-04/ with
+# collected from api version 2016-09-02/ with
 # python3 -c 'import json
 # from cloudinit.ec2_utils import get_instance_metadata as gm
-# print(json.dumps(gm("2009-04-04"), indent=1, sort_keys=True))'
+# print(json.dumps(gm("2016-09-02"), indent=1, sort_keys=True))'
 DEFAULT_METADATA = {
-    "ami-id": "ami-80861296",
+    "ami-id": "ami-8b92b4ee",
     "ami-launch-index": "0",
     "ami-manifest-path": "(unknown)",
     "block-device-mapping": {"ami": "/dev/sda1", "root": "/dev/sda1"},
-    "hostname": "ip-10-0-0-149",
+    "hostname": "ip-172-31-31-158.us-east-2.compute.internal",
     "instance-action": "none",
-    "instance-id": "i-0052913950685138c",
-    "instance-type": "t2.micro",
-    "local-hostname": "ip-10-0-0-149",
-    "local-ipv4": "10.0.0.149",
-    "placement": {"availability-zone": "us-east-1b"},
+    "instance-id": "i-0a33f80f09c96477f",
+    "instance-type": "t2.small",
+    "local-hostname": "ip-172-3-3-15.us-east-2.compute.internal",
+    "local-ipv4": "172.3.3.15",
+    "mac": "06:17:04:d7:26:09",
+    "metrics": {"vhostmd": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"},
+    "network": {
+        "interfaces": {
+            "macs": {
+                "06:17:04:d7:26:09": {
+                    "device-number": "0",
+                    "interface-id": "eni-e44ef49e",
+                    "ipv4-associations": {"13.59.77.202": "172.3.3.15"},
+                    "ipv6s": "2600:1f16:aeb:b20b:9d87:a4af:5cc9:73dc",
+                    "local-hostname": ("ip-172-3-3-15.us-east-2."
+                                       "compute.internal"),
+                    "local-ipv4s": "172.3.3.15",
+                    "mac": "06:17:04:d7:26:09",
+                    "owner-id": "950047163771",
+                    "public-hostname": ("ec2-13-59-77-202.us-east-2."
+                                        "compute.amazonaws.com"),
+                    "public-ipv4s": "13.59.77.202",
+                    "security-group-ids": "sg-5a61d333",
+                    "security-groups": "wide-open",
+                    "subnet-id": "subnet-20b8565b",
+                    "subnet-ipv4-cidr-block": "172.31.16.0/20",
+                    "subnet-ipv6-cidr-blocks": "2600:1f16:aeb:b20b::/64",
+                    "vpc-id": "vpc-87e72bee",
+                    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+                    "vpc-ipv4-cidr-blocks": "172.31.0.0/16",
+                    "vpc-ipv6-cidr-blocks": "2600:1f16:aeb:b200::/56"
+                }
+            }
+        }
+    },
+    "placement": {"availability-zone": "us-east-2b"},
     "profile": "default-hvm",
-    "public-hostname": "",
-    "public-ipv4": "107.23.188.247",
+    "public-hostname": "ec2-13-59-77-202.us-east-2.compute.amazonaws.com",
+    "public-ipv4": "13.59.77.202",
     "public-keys": {"brickies": ["ssh-rsa AAAAB3Nz....w== brickies"]},
-    "reservation-id": "r-00a2c173fb5782a08",
-    "security-groups": "wide-open"
+    "reservation-id": "r-01efbc9996bac1bd6",
+    "security-groups": "my-wide-open",
+    "services": {"domain": "amazonaws.com", "partition": "aws"}
 }
 
 
 def _register_ssh_keys(rfunc, base_url, keys_data):
     """handle ssh key inconsistencies.
 
-    public-keys in the ec2 metadata is inconsistently formatted compared
+    public-keys in the ec2 metadata is inconsistently formated compared
     to other entries.
     Given keys_data of {name1: pubkey1, name2: pubkey2}
 
@@ -83,6 +116,9 @@ def register_mock_metaserver(base_url, data):
     In the index, references to lists or dictionaries have a trailing /.
     """
     def register_helper(register, base_url, body):
+        if not isinstance(base_url, str):
+            register(base_url, body)
+            return
         base_url = base_url.rstrip("/")
         if isinstance(body, str):
             register(base_url, body)
@@ -105,7 +141,7 @@ def register_mock_metaserver(base_url, data):
             register(base_url, '\n'.join(vals) + '\n')
             register(base_url + '/', '\n'.join(vals) + '\n')
         elif body is None:
-            register(base_url, 'not found', status_code=404)
+            register(base_url, 'not found', status=404)
 
     def myreg(*argc, **kwargs):
         # print("register_url(%s, %s)" % (argc, kwargs))
@@ -115,6 +151,8 @@ def register_mock_metaserver(base_url, data):
 
 
 class TestEc2(test_helpers.HttprettyTestCase):
+    with_logs = True
+
     valid_platform_data = {
         'uuid': 'ec212f79-87d1-2f1d-588f-d86dc0fd5412',
         'uuid_source': 'dmi',
@@ -123,48 +161,91 @@ class TestEc2(test_helpers.HttprettyTestCase):
 
     def setUp(self):
         super(TestEc2, self).setUp()
-        self.metadata_addr = ec2.DataSourceEc2.metadata_urls[0]
-        self.api_ver = '2009-04-04'
-
-    @property
-    def metadata_url(self):
-        return '/'.join([self.metadata_addr, self.api_ver, 'meta-data', ''])
+        self.datasource = ec2.DataSourceEc2
+        self.metadata_addr = self.datasource.metadata_urls[0]
 
-    @property
-    def userdata_url(self):
-        return '/'.join([self.metadata_addr, self.api_ver, 'user-data'])
+    def data_url(self, version):
+        """Return a metadata url based on the version provided."""
+        return '/'.join([self.metadata_addr, version, 'meta-data', ''])
 
     def _patch_add_cleanup(self, mpath, *args, **kwargs):
         p = mock.patch(mpath, *args, **kwargs)
         p.start()
         self.addCleanup(p.stop)
 
-    def _setup_ds(self, sys_cfg, platform_data, md, ud=None):
+    def _setup_ds(self, sys_cfg, platform_data, md, md_version=None):
+        self.uris = []
         distro = {}
         paths = helpers.Paths({})
         if sys_cfg is None:
             sys_cfg = {}
-        ds = ec2.DataSourceEc2(sys_cfg=sys_cfg, distro=distro, paths=paths)
+        ds = self.datasource(sys_cfg=sys_cfg, distro=distro, paths=paths)
+        if not md_version:
+            md_version = ds.min_metadata_version
         if platform_data is not None:
             self._patch_add_cleanup(
                 "cloudinit.sources.DataSourceEc2._collect_platform_data",
                 return_value=platform_data)
 
         if md:
-            register_mock_metaserver(self.metadata_url, md)
-            register_mock_metaserver(self.userdata_url, ud)
-
+            httpretty.HTTPretty.allow_net_connect = False
+            all_versions = (
+                [ds.min_metadata_version] + ds.extended_metadata_versions)
+            for version in all_versions:
+                metadata_url = self.data_url(version)
+                if version == md_version:
+                    # Register all metadata for desired version
+                    register_mock_metaserver(metadata_url, md)
+                else:
+                    instance_id_url = metadata_url + 'instance-id'
+                    if version == ds.min_metadata_version:
+                        # Add min_metadata_version service availability check
+                        register_mock_metaserver(
+                            instance_id_url, DEFAULT_METADATA['instance-id'])
+                    else:
+                        # Register 404s for all unrequested extended versions
+                        register_mock_metaserver(instance_id_url, None)
         return ds
 
     @httpretty.activate
-    def test_valid_platform_with_strict_true(self):
+    def test_network_config_property_returns_version_1_network_data(self):
+        """network_config property returns network version 1 for metadata."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
+            md=DEFAULT_METADATA)
+        ds.get_data()
+        mac1 = '06:17:04:d7:26:09'  # Defined in DEFAULT_METADATA
+        expected = {'version': 1, 'config': [
+            {'mac_address': '06:17:04:d7:26:09', 'name': 'eth9',
+             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}],
+             'type': 'physical'}]}
+        patch_path = (
+            'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
+        with mock.patch(patch_path) as m_get_interfaces_by_mac:
+            m_get_interfaces_by_mac.return_value = {mac1: 'eth9'}
+            self.assertEqual(expected, ds.network_config)
+
+    def test_network_config_property_is_cached_in_datasource(self):
+        """network_config property is cached in DataSourceEc2."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
+            md=DEFAULT_METADATA)
+        ds._network_config = {'cached': 'data'}
+        self.assertEqual({'cached': 'data'}, ds.network_config)
+
+    @httpretty.activate
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    def test_valid_platform_with_strict_true(self, m_dhcp):
         """Valid platform data should return true with strict_id true."""
         ds = self._setup_ds(
             platform_data=self.valid_platform_data,
             sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(True, ret)
+        self.assertTrue(ret)
+        self.assertEqual(0, m_dhcp.call_count)
 
     @httpretty.activate
     def test_valid_platform_with_strict_false(self):
@@ -174,7 +255,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
             sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(True, ret)
+        self.assertTrue(ret)
 
     @httpretty.activate
     def test_unknown_platform_with_strict_true(self):
@@ -185,7 +266,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
             sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(False, ret)
+        self.assertFalse(ret)
 
     @httpretty.activate
     def test_unknown_platform_with_strict_false(self):
@@ -196,7 +277,146 @@ class TestEc2(test_helpers.HttprettyTestCase):
             sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(True, ret)
+        self.assertTrue(ret)
+
+    def test_ec2_local_returns_false_on_non_aws(self):
+        """DataSourceEc2Local returns False when platform is not AWS."""
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md=DEFAULT_METADATA)
+        platform_attrs = [
+            attr for attr in ec2.Platforms.__dict__.keys()
+            if not attr.startswith('__')]
+        for attr_name in platform_attrs:
+            platform_name = getattr(ec2.Platforms, attr_name)
+            if platform_name != 'AWS':
+                ds._cloud_platform = platform_name
+                ret = ds.get_data()
+                self.assertFalse(ret)
+                message = (
+                    "Local Ec2 mode only supported on ('AWS',),"
+                    ' not {0}'.format(platform_name))
+                self.assertIn(message, self.logs.getvalue())
+
+    @httpretty.activate
+    @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
+    def test_ec2_local_returns_false_on_bsd(self, m_is_freebsd):
+        """DataSourceEc2Local returns False on BSD.
+
+        FreeBSD dhclient doesn't support dhclient -sf to run in a sandbox.
+        """
+        m_is_freebsd.return_value = True
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md=DEFAULT_METADATA)
+        ret = ds.get_data()
+        self.assertFalse(ret)
+        self.assertIn(
+            "FreeBSD doesn't support running dhclient with -sf",
+            self.logs.getvalue())
+
+    @httpretty.activate
+    @mock.patch('cloudinit.net.EphemeralIPv4Network')
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
+    def test_ec2_local_performs_dhcp_on_non_bsd(self, m_is_bsd, m_dhcp, m_net):
+        """Ec2Local returns True for valid platform data on non-BSD with dhcp.
+
+        DataSourceEc2Local will setup initial IPv4 network via dhcp discovery.
+        Then the metadata services is crawled for more network config info.
+        When the platform data is valid, return True.
+        """
+
+        m_is_bsd.return_value = False
+        m_dhcp.return_value = [{
+            'interface': 'eth9', 'fixed-address': '192.168.2.9',
+            'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0',
+            'broadcast-address': '192.168.2.255'}]
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md=DEFAULT_METADATA)
+
+        ret = ds.get_data()
+        self.assertTrue(ret)
+        m_dhcp.assert_called_once_with()
+        m_net.assert_called_once_with(
+            broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9',
+            prefix_or_mask='255.255.255.0', router='192.168.2.1')
+        self.assertIn('Crawl of metadata service took', self.logs.getvalue())
+
+
+class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
+
+    def setUp(self):
+        super(TestConvertEc2MetadataNetworkConfig, self).setUp()
+        self.mac1 = '06:17:04:d7:26:09'
+        self.network_metadata = {
+            'interfaces': {'macs': {
+                self.mac1: {'public-ipv4s': '172.31.2.16'}}}}
+
+    def test_convert_ec2_metadata_network_config_skips_absent_macs(self):
+        """Any mac absent from metadata is skipped by network config."""
+        macs_to_nics = {self.mac1: 'eth9', 'DE:AD:BE:EF:FF:FF': 'vitualnic2'}
+
+        # DE:AD:BE:EF:FF:FF represented by OS but not in metadata
+        expected = {'version': 1, 'config': [
+            {'mac_address': self.mac1, 'type': 'physical',
+             'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]}
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                self.network_metadata, macs_to_nics))
+
+    def test_convert_ec2_metadata_network_config_handles_only_dhcp6(self):
+        """Config dhcp6 when ipv6s is in metadata for a mac."""
+        macs_to_nics = {self.mac1: 'eth9'}
+        network_metadata_ipv6 = copy.deepcopy(self.network_metadata)
+        nic1_metadata = (
+            network_metadata_ipv6['interfaces']['macs'][self.mac1])
+        nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
+        nic1_metadata.pop('public-ipv4s')
+        expected = {'version': 1, 'config': [
+            {'mac_address': self.mac1, 'type': 'physical',
+             'name': 'eth9', 'subnets': [{'type': 'dhcp6'}]}]}
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_ipv6, macs_to_nics))
+
+    def test_convert_ec2_metadata_network_config_handles_dhcp4_and_dhcp6(self):
+        """Config both dhcp4 and dhcp6 when both vpc-ipv6 and ipv4 exists."""
+        macs_to_nics = {self.mac1: 'eth9'}
+        network_metadata_both = copy.deepcopy(self.network_metadata)
+        nic1_metadata = (
+            network_metadata_both['interfaces']['macs'][self.mac1])
+        nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
+        expected = {'version': 1, 'config': [
+            {'mac_address': self.mac1, 'type': 'physical',
+             'name': 'eth9',
+             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}]}]}
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_both, macs_to_nics))
 
+    def test_convert_ec2_metadata_gets_macs_from_get_interfaces_by_mac(self):
+        """Convert Ec2 Metadata calls get_interfaces_by_mac by default."""
+        expected = {'version': 1, 'config': [
+            {'mac_address': self.mac1, 'type': 'physical',
+             'name': 'eth9',
+             'subnets': [{'type': 'dhcp4'}]}]}
+        patch_path = (
+            'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
+        with mock.patch(patch_path) as m_get_interfaces_by_mac:
+            m_get_interfaces_by_mac.return_value = {self.mac1: 'eth9'}
+            self.assertEqual(
+                expected,
+                ec2.convert_ec2_metadata_network_config(self.network_metadata))
 
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py
index ad608bec..d399ae7a 100644
--- a/tests/unittests/test_datasource/test_gce.py
+++ b/tests/unittests/test_datasource/test_gce.py
@@ -15,7 +15,7 @@ from cloudinit import helpers
 from cloudinit import settings
 from cloudinit.sources import DataSourceGCE
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 
 GCE_META = {
@@ -23,7 +23,8 @@ GCE_META = {
     'instance/zone': 'foo/bar',
     'project/attributes/sshKeys': 'user:ssh-rsa AA2..+aRD0fyVw== root@server',
     'instance/hostname': 'server.project-foo.local',
-    'instance/attributes/user-data': b'/bin/echo foo\n',
+    # UnicodeDecodeError below if set to ds.userdata instead of userdata_raw
+    'instance/attributes/user-data': b'/bin/echo \xff\n',
 }
 
 GCE_META_PARTIAL = {
diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py
index c1911bf4..289c6a40 100644
--- a/tests/unittests/test_datasource/test_maas.py
+++ b/tests/unittests/test_datasource/test_maas.py
@@ -8,7 +8,7 @@ import yaml
 
 from cloudinit.sources import DataSourceMAAS
 from cloudinit import url_helper
-from ..helpers import TestCase, populate_dir
+from cloudinit.tests.helpers import TestCase, populate_dir
 
 try:
     from unittest import mock
diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py
index ff294395..fea9156b 100644
--- a/tests/unittests/test_datasource/test_nocloud.py
+++ b/tests/unittests/test_datasource/test_nocloud.py
@@ -3,7 +3,7 @@
 from cloudinit import helpers
 from cloudinit.sources import DataSourceNoCloud
 from cloudinit import util
-from ..helpers import TestCase, populate_dir, mock, ExitStack
+from cloudinit.tests.helpers import TestCase, populate_dir, mock, ExitStack
 
 import os
 import shutil
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index b0f8e435..e7d55692 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -3,7 +3,7 @@
 from cloudinit import helpers
 from cloudinit.sources import DataSourceOpenNebula as ds
 from cloudinit import util
-from ..helpers import mock, populate_dir, TestCase
+from cloudinit.tests.helpers import mock, populate_dir, TestCase
 
 import os
 import pwd
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index c2905d1a..ed367e05 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -9,7 +9,7 @@ import httpretty as hp
 import json
 import re
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 from six.moves.urllib.parse import urlparse
 from six import StringIO
@@ -57,6 +57,8 @@ OS_FILES = {
     'openstack/content/0000': CONTENT_0,
     'openstack/content/0001': CONTENT_1,
     'openstack/latest/meta_data.json': json.dumps(OSTACK_META),
+    'openstack/latest/network_data.json': json.dumps(
+        {'links': [], 'networks': [], 'services': []}),
     'openstack/latest/user_data': USER_DATA,
     'openstack/latest/vendor_data.json': json.dumps(VENDOR_DATA),
 }
@@ -68,6 +70,7 @@ EC2_VERSIONS = [
 ]
 
 
+# TODO _register_uris should leverage test_ec2.register_mock_metaserver.
 def _register_uris(version, ec2_files, ec2_meta, os_files):
     """Registers a set of url patterns into httpretty that will mimic the
     same data returned by the openstack metadata service (and ec2 service)."""
diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py
index 477cf8ed..700da86c 100644
--- a/tests/unittests/test_datasource/test_ovf.py
+++ b/tests/unittests/test_datasource/test_ovf.py
@@ -5,8 +5,9 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import base64
+from collections import OrderedDict
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 from cloudinit.sources import DataSourceOVF as dsovf
 
@@ -70,4 +71,167 @@ class TestReadOvfEnv(test_helpers.TestCase):
         self.assertEqual({'password': "passw0rd"}, cfg)
         self.assertIsNone(ud)
 
+
+class TestTransportIso9660(test_helpers.CiTestCase):
+
+    def setUp(self):
+        super(TestTransportIso9660, self).setUp()
+        self.add_patch('cloudinit.util.find_devs_with',
+                       'm_find_devs_with')
+        self.add_patch('cloudinit.util.mounts', 'm_mounts')
+        self.add_patch('cloudinit.util.mount_cb', 'm_mount_cb')
+        self.add_patch('cloudinit.sources.DataSourceOVF.get_ovf_env',
+                       'm_get_ovf_env')
+        self.m_get_ovf_env.return_value = ('myfile', 'mycontent')
+
+    def test_find_already_mounted(self):
+        """Check we call get_ovf_env from on matching mounted devices"""
+        mounts = {
+            '/dev/sr9': {
+                'fstype': 'iso9660',
+                'mountpoint': 'wark/media/sr9',
+                'opts': 'ro',
+            }
+        }
+        self.m_mounts.return_value = mounts
+
+        (contents, fullp, fname) = dsovf.transport_iso9660()
+        self.assertEqual("mycontent", contents)
+        self.assertEqual("/dev/sr9", fullp)
+        self.assertEqual("myfile", fname)
+
+    def test_find_already_mounted_skips_non_iso9660(self):
+        """Check we call get_ovf_env ignoring non iso9660"""
+        mounts = {
+            '/dev/xvdb': {
+                'fstype': 'vfat',
+                'mountpoint': 'wark/foobar',
+                'opts': 'defaults,noatime',
+            },
+            '/dev/xvdc': {
+                'fstype': 'iso9660',
+                'mountpoint': 'wark/media/sr9',
+                'opts': 'ro',
+            }
+        }
+        # We use an OrderedDict here to ensure we check xvdb before xvdc
+        # as we're not mocking the regex matching, however, if we place
+        # an entry in the results then we can be reasonably sure that
+        # we're skipping an entry which fails to match.
+        self.m_mounts.return_value = (
+            OrderedDict(sorted(mounts.items(), key=lambda t: t[0])))
+
+        (contents, fullp, fname) = dsovf.transport_iso9660()
+        self.assertEqual("mycontent", contents)
+        self.assertEqual("/dev/xvdc", fullp)
+        self.assertEqual("myfile", fname)
+
+    def test_find_already_mounted_matches_kname(self):
+        """Check we dont regex match on basename of the device"""
+        mounts = {
+            '/dev/foo/bar/xvdc': {
+                'fstype': 'iso9660',
+                'mountpoint': 'wark/media/sr9',
+                'opts': 'ro',
+            }
+        }
+        # we're skipping an entry which fails to match.
+        self.m_mounts.return_value = mounts
+
+        (contents, fullp, fname) = dsovf.transport_iso9660()
+        self.assertEqual(False, contents)
+        self.assertIsNone(fullp)
+        self.assertIsNone(fname)
+
+    def test_mount_cb_called_on_blkdevs_with_iso9660(self):
+        """Check we call mount_cb on blockdevs with iso9660 only"""
+        self.m_mounts.return_value = {}
+        self.m_find_devs_with.return_value = ['/dev/sr0']
+        self.m_mount_cb.return_value = ("myfile", "mycontent")
+
+        (contents, fullp, fname) = dsovf.transport_iso9660()
+
+        self.m_mount_cb.assert_called_with(
+            "/dev/sr0", dsovf.get_ovf_env, mtype="iso9660")
+        self.assertEqual("mycontent", contents)
+        self.assertEqual("/dev/sr0", fullp)
+        self.assertEqual("myfile", fname)
+
+    def test_mount_cb_called_on_blkdevs_with_iso9660_check_regex(self):
+        """Check we call mount_cb on blockdevs with iso9660 and match regex"""
+        self.m_mounts.return_value = {}
+        self.m_find_devs_with.return_value = [
+            '/dev/abc', '/dev/my-cdrom', '/dev/sr0']
+        self.m_mount_cb.return_value = ("myfile", "mycontent")
+
+        (contents, fullp, fname) = dsovf.transport_iso9660()
+
+        self.m_mount_cb.assert_called_with(
+            "/dev/sr0", dsovf.get_ovf_env, mtype="iso9660")
+        self.assertEqual("mycontent", contents)
+        self.assertEqual("/dev/sr0", fullp)
+        self.assertEqual("myfile", fname)
+
+    def test_mount_cb_not_called_no_matches(self):
+        """Check we don't call mount_cb if nothing matches"""
+        self.m_mounts.return_value = {}
+        self.m_find_devs_with.return_value = ['/dev/vg/myovf']
+
+        (contents, fullp, fname) = dsovf.transport_iso9660()
+
+        self.assertEqual(0, self.m_mount_cb.call_count)
+        self.assertEqual(False, contents)
+        self.assertIsNone(fullp)
+        self.assertIsNone(fname)
+
+    def test_mount_cb_called_require_iso_false(self):
+        """Check we call mount_cb on blockdevs with require_iso=False"""
+        self.m_mounts.return_value = {}
+        self.m_find_devs_with.return_value = ['/dev/xvdz']
+        self.m_mount_cb.return_value = ("myfile", "mycontent")
+
+        (contents, fullp, fname) = dsovf.transport_iso9660(require_iso=False)
+
+        self.m_mount_cb.assert_called_with(
+            "/dev/xvdz", dsovf.get_ovf_env, mtype=None)
+        self.assertEqual("mycontent", contents)
+        self.assertEqual("/dev/xvdz", fullp)
+        self.assertEqual("myfile", fname)
+
+    def test_maybe_cdrom_device_none(self):
+        """Test maybe_cdrom_device returns False for none/empty input"""
+        self.assertFalse(dsovf.maybe_cdrom_device(None))
+        self.assertFalse(dsovf.maybe_cdrom_device(''))
+
+    def test_maybe_cdrom_device_non_string_exception(self):
+        """Test maybe_cdrom_device raises ValueError on non-string types"""
+        with self.assertRaises(ValueError):
+            dsovf.maybe_cdrom_device({'a': 'eleven'})
+
+    def test_maybe_cdrom_device_false_on_multi_dir_paths(self):
+        """Test maybe_cdrom_device is false on /dev[/.*]/* paths"""
+        self.assertFalse(dsovf.maybe_cdrom_device('/dev/foo/sr0'))
+        self.assertFalse(dsovf.maybe_cdrom_device('foo/sr0'))
+        self.assertFalse(dsovf.maybe_cdrom_device('../foo/sr0'))
+        self.assertFalse(dsovf.maybe_cdrom_device('../foo/sr0'))
+
+    def test_maybe_cdrom_device_true_on_hd_partitions(self):
+        """Test maybe_cdrom_device is false on /dev/hd[a-z][0-9]+ paths"""
+        self.assertTrue(dsovf.maybe_cdrom_device('/dev/hda1'))
+        self.assertTrue(dsovf.maybe_cdrom_device('hdz9'))
+
+    def test_maybe_cdrom_device_true_on_valid_relative_paths(self):
+        """Test maybe_cdrom_device normalizes paths"""
+        self.assertTrue(dsovf.maybe_cdrom_device('/dev/wark/../sr9'))
+        self.assertTrue(dsovf.maybe_cdrom_device('///sr0'))
+        self.assertTrue(dsovf.maybe_cdrom_device('/sr0'))
+        self.assertTrue(dsovf.maybe_cdrom_device('//dev//hda'))
+
+    def test_maybe_cdrom_device_true_on_xvd_partitions(self):
+        """Test maybe_cdrom_device returns true on xvd*"""
+        self.assertTrue(dsovf.maybe_cdrom_device('/dev/xvda'))
+        self.assertTrue(dsovf.maybe_cdrom_device('/dev/xvda1'))
+        self.assertTrue(dsovf.maybe_cdrom_device('xvdza1'))
+
+#
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py
index 65d83ad7..436df9ee 100644
--- a/tests/unittests/test_datasource/test_scaleway.py
+++ b/tests/unittests/test_datasource/test_scaleway.py
@@ -9,7 +9,7 @@ from cloudinit import helpers
 from cloudinit import settings
 from cloudinit.sources import DataSourceScaleway
 
-from ..helpers import mock, HttprettyTestCase, TestCase
+from cloudinit.tests.helpers import mock, HttprettyTestCase, TestCase
 
 
 class DataResponses(object):
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index e3c99bbb..933d5b63 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -33,7 +33,7 @@ import six
 from cloudinit import helpers as c_helpers
 from cloudinit.util import b64e
 
-from ..helpers import mock, FilesystemMockingTestCase, TestCase
+from cloudinit.tests.helpers import mock, FilesystemMockingTestCase, TestCase
 
 SDC_NICS = json.loads("""
 [