Datasources: Formalize DataSource get_data and related properties.

Each DataSource subclass must define its own get_data method. This branch
formalizes our DataSource class to require that subclasses define an
explicit dsname for sourcing cloud-config datasource configuration.
Subclasses must also override the _get_data method or a
NotImplementedError is raised.

The branch also writes /run/cloud-init/instance-data.json. This file
contains all meta-data, user-data and vendor-data and a standardized set
of metadata keys in a json blob which other utilities with root-access
could make use of. Because some meta-data or user-data is potentially
sensitive the file is only readable by root.

Generally most metadata content types should be json serializable. If
specific keys or values are not serializable, those specific values will
be base64encoded and the key path will be listed under the top-level key
'base64-encoded-keys' in instance-data.json. If json writing fails due to
other TypeErrors or UnicodeDecodeErrors, a warning log will be emitted to
/var/log/cloud-init.log and no instance-data.json will be created.

17 files changed, 99 insertions, 67 deletions

diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py
index 82ee9714..714f5dac 100644
--- a/tests/unittests/test_datasource/test_aliyun.py
+++ b/tests/unittests/test_datasource/test_aliyun.py
@@ -67,7 +67,7 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
         super(TestAliYunDatasource, self).setUp()
         cfg = {'datasource': {'AliYun': {'timeout': '1', 'max_wait': '1'}}}
         distro = {}
-        paths = helpers.Paths({})
+        paths = helpers.Paths({'run_dir': self.tmp_dir()})
         self.ds = ay.DataSourceAliYun(cfg, distro, paths)
         self.metadata_address = self.ds.metadata_urls[0]
 
diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py
index a4dfb540..3253f3ad 100644
--- a/tests/unittests/test_datasource/test_altcloud.py
+++ b/tests/unittests/test_datasource/test_altcloud.py
@@ -18,7 +18,7 @@ import tempfile
 from cloudinit import helpers
 from cloudinit import util
 
-from cloudinit.tests.helpers import TestCase
+from cloudinit.tests.helpers import CiTestCase
 
 import cloudinit.sources.DataSourceAltCloud as dsac
 
@@ -97,7 +97,7 @@ def _dmi_data(expected):
     return _data
 
 
-class TestGetCloudType(TestCase):
+class TestGetCloudType(CiTestCase):
     '''
     Test to exercise method: DataSourceAltCloud.get_cloud_type()
     '''
@@ -143,14 +143,16 @@ class TestGetCloudType(TestCase):
         self.assertEqual('UNKNOWN', dsrc.get_cloud_type())
 
 
-class TestGetDataCloudInfoFile(TestCase):
+class TestGetDataCloudInfoFile(CiTestCase):
     '''
     Test to exercise method: DataSourceAltCloud.get_data()
     With a contrived CLOUD_INFO_FILE
     '''
     def setUp(self):
         '''Set up.'''
-        self.paths = helpers.Paths({'cloud_dir': '/tmp'})
+        self.tmp = self.tmp_dir()
+        self.paths = helpers.Paths(
+            {'cloud_dir': self.tmp, 'run_dir': self.tmp})
         self.cloud_info_file = tempfile.mkstemp()[1]
         self.dmi_data = util.read_dmi_data
         dsac.CLOUD_INFO_FILE = self.cloud_info_file
@@ -207,14 +209,16 @@ class TestGetDataCloudInfoFile(TestCase):
         self.assertEqual(False, dsrc.get_data())
 
 
-class TestGetDataNoCloudInfoFile(TestCase):
+class TestGetDataNoCloudInfoFile(CiTestCase):
     '''
     Test to exercise method: DataSourceAltCloud.get_data()
     Without a CLOUD_INFO_FILE
     '''
     def setUp(self):
         '''Set up.'''
-        self.paths = helpers.Paths({'cloud_dir': '/tmp'})
+        self.tmp = self.tmp_dir()
+        self.paths = helpers.Paths(
+            {'cloud_dir': self.tmp, 'run_dir': self.tmp})
         self.dmi_data = util.read_dmi_data
         dsac.CLOUD_INFO_FILE = \
             'no such file'
@@ -254,7 +258,7 @@ class TestGetDataNoCloudInfoFile(TestCase):
         self.assertEqual(False, dsrc.get_data())
 
 
-class TestUserDataRhevm(TestCase):
+class TestUserDataRhevm(CiTestCase):
     '''
     Test to exercise method: DataSourceAltCloud.user_data_rhevm()
     '''
@@ -320,7 +324,7 @@ class TestUserDataRhevm(TestCase):
         self.assertEqual(False, dsrc.user_data_rhevm())
 
 
-class TestUserDataVsphere(TestCase):
+class TestUserDataVsphere(CiTestCase):
     '''
     Test to exercise method: DataSourceAltCloud.user_data_vsphere()
     '''
@@ -368,7 +372,7 @@ class TestUserDataVsphere(TestCase):
         self.assertEqual(1, m_mount_cb.call_count)
 
 
-class TestReadUserDataCallback(TestCase):
+class TestReadUserDataCallback(CiTestCase):
     '''
     Test to exercise method: DataSourceAltCloud.read_user_data_callback()
     '''
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 7cb1812a..226c214a 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -11,9 +11,7 @@ from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock,
 
 import crypt
 import os
-import shutil
 import stat
-import tempfile
 import xml.etree.ElementTree as ET
 import yaml
 
@@ -84,11 +82,11 @@ class TestAzureDataSource(CiTestCase):
         super(TestAzureDataSource, self).setUp()
         if PY26:
             raise SkipTest("Does not work on python 2.6")
-        self.tmp = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.tmp)
+        self.tmp = self.tmp_dir()
 
         # patch cloud_dir, so our 'seed_dir' is guaranteed empty
-        self.paths = helpers.Paths({'cloud_dir': self.tmp})
+        self.paths = helpers.Paths(
+            {'cloud_dir': self.tmp, 'run_dir': self.tmp})
         self.waagent_d = os.path.join(self.tmp, 'var', 'lib', 'waagent')
 
         self.patches = ExitStack()
@@ -642,7 +640,7 @@ fdescfs            /dev/fd          fdescfs rw              0 0
         self.assertEqual(netconfig, expected_config)
 
 
-class TestAzureBounce(TestCase):
+class TestAzureBounce(CiTestCase):
 
     def mock_out_azure_moving_parts(self):
         self.patches.enter_context(
@@ -669,10 +667,10 @@ class TestAzureBounce(TestCase):
 
     def setUp(self):
         super(TestAzureBounce, self).setUp()
-        self.tmp = tempfile.mkdtemp()
+        self.tmp = self.tmp_dir()
         self.waagent_d = os.path.join(self.tmp, 'var', 'lib', 'waagent')
-        self.paths = helpers.Paths({'cloud_dir': self.tmp})
-        self.addCleanup(shutil.rmtree, self.tmp)
+        self.paths = helpers.Paths(
+            {'cloud_dir': self.tmp, 'run_dir': self.tmp})
         dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d
         self.patches = ExitStack()
         self.mock_out_azure_moving_parts()
@@ -714,21 +712,24 @@ class TestAzureBounce(TestCase):
 
     def test_disabled_bounce_does_not_change_hostname(self):
         cfg = {'hostname_bounce': {'policy': 'off'}}
-        self._get_ds(self.get_ovf_env_with_dscfg('test-host', cfg)).get_data()
+        ds = self._get_ds(self.get_ovf_env_with_dscfg('test-host', cfg))
+        ds.get_data()
         self.assertEqual(0, self.set_hostname.call_count)
 
     @mock.patch('cloudinit.sources.DataSourceAzure.perform_hostname_bounce')
     def test_disabled_bounce_does_not_perform_bounce(
             self, perform_hostname_bounce):
         cfg = {'hostname_bounce': {'policy': 'off'}}
-        self._get_ds(self.get_ovf_env_with_dscfg('test-host', cfg)).get_data()
+        ds = self._get_ds(self.get_ovf_env_with_dscfg('test-host', cfg))
+        ds.get_data()
         self.assertEqual(0, perform_hostname_bounce.call_count)
 
     def test_same_hostname_does_not_change_hostname(self):
         host_name = 'unchanged-host-name'
         self.get_hostname.return_value = host_name
         cfg = {'hostname_bounce': {'policy': 'yes'}}
-        self._get_ds(self.get_ovf_env_with_dscfg(host_name, cfg)).get_data()
+        ds = self._get_ds(self.get_ovf_env_with_dscfg(host_name, cfg))
+        ds.get_data()
         self.assertEqual(0, self.set_hostname.call_count)
 
     @mock.patch('cloudinit.sources.DataSourceAzure.perform_hostname_bounce')
@@ -737,7 +738,8 @@ class TestAzureBounce(TestCase):
         host_name = 'unchanged-host-name'
         self.get_hostname.return_value = host_name
         cfg = {'hostname_bounce': {'policy': 'yes'}}
-        self._get_ds(self.get_ovf_env_with_dscfg(host_name, cfg)).get_data()
+        ds = self._get_ds(self.get_ovf_env_with_dscfg(host_name, cfg))
+        ds.get_data()
         self.assertEqual(0, perform_hostname_bounce.call_count)
 
     @mock.patch('cloudinit.sources.DataSourceAzure.perform_hostname_bounce')
diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py
index e4c59907..f6a59b6b 100644
--- a/tests/unittests/test_datasource/test_cloudsigma.py
+++ b/tests/unittests/test_datasource/test_cloudsigma.py
@@ -3,6 +3,7 @@
 import copy
 
 from cloudinit.cs_utils import Cepko
+from cloudinit import helpers
 from cloudinit import sources
 from cloudinit.sources import DataSourceCloudSigma
 
@@ -38,10 +39,12 @@ class CepkoMock(Cepko):
         return self
 
 
-class DataSourceCloudSigmaTest(test_helpers.TestCase):
+class DataSourceCloudSigmaTest(test_helpers.CiTestCase):
     def setUp(self):
         super(DataSourceCloudSigmaTest, self).setUp()
-        self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "")
+        self.paths = helpers.Paths({'run_dir': self.tmp_dir()})
+        self.datasource = DataSourceCloudSigma.DataSourceCloudSigma(
+            "", "", paths=self.paths)
         self.datasource.is_running_in_cloudsigma = lambda: True
         self.datasource.cepko = CepkoMock(SERVER_CONTEXT)
         self.datasource.get_data()
@@ -85,7 +88,8 @@ class DataSourceCloudSigmaTest(test_helpers.TestCase):
     def test_lack_of_vendor_data(self):
         stripped_context = copy.deepcopy(SERVER_CONTEXT)
         del stripped_context["vendor_data"]
-        self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "")
+        self.datasource = DataSourceCloudSigma.DataSourceCloudSigma(
+            "", "", paths=self.paths)
         self.datasource.cepko = CepkoMock(stripped_context)
         self.datasource.get_data()
 
@@ -94,7 +98,8 @@ class DataSourceCloudSigmaTest(test_helpers.TestCase):
     def test_lack_of_cloudinit_key_in_vendor_data(self):
         stripped_context = copy.deepcopy(SERVER_CONTEXT)
         del stripped_context["vendor_data"]["cloudinit"]
-        self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "")
+        self.datasource = DataSourceCloudSigma.DataSourceCloudSigma(
+            "", "", paths=self.paths)
         self.datasource.cepko = CepkoMock(stripped_context)
         self.datasource.get_data()
 
diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py
index 96144b64..d6d2d6b2 100644
--- a/tests/unittests/test_datasource/test_cloudstack.py
+++ b/tests/unittests/test_datasource/test_cloudstack.py
@@ -33,6 +33,7 @@ class TestCloudStackPasswordFetching(CiTestCase):
         self.patches.enter_context(mock.patch(
             mod_name + '.dhcp.networkd_get_option_from_leases',
             get_networkd_server_address))
+        self.tmp = self.tmp_dir()
 
     def _set_password_server_response(self, response_string):
         subp = mock.MagicMock(return_value=(response_string, ''))
@@ -43,26 +44,30 @@ class TestCloudStackPasswordFetching(CiTestCase):
 
     def test_empty_password_doesnt_create_config(self):
         self._set_password_server_response('')
-        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds = DataSourceCloudStack(
+            {}, None, helpers.Paths({'run_dir': self.tmp}))
         ds.get_data()
         self.assertEqual({}, ds.get_config_obj())
 
     def test_saved_password_doesnt_create_config(self):
         self._set_password_server_response('saved_password')
-        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds = DataSourceCloudStack(
+            {}, None, helpers.Paths({'run_dir': self.tmp}))
         ds.get_data()
         self.assertEqual({}, ds.get_config_obj())
 
     def test_password_sets_password(self):
         password = 'SekritSquirrel'
         self._set_password_server_response(password)
-        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds = DataSourceCloudStack(
+            {}, None, helpers.Paths({'run_dir': self.tmp}))
         ds.get_data()
         self.assertEqual(password, ds.get_config_obj()['password'])
 
     def test_bad_request_doesnt_stop_ds_from_working(self):
         self._set_password_server_response('bad_request')
-        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds = DataSourceCloudStack(
+            {}, None, helpers.Paths({'run_dir': self.tmp}))
         self.assertTrue(ds.get_data())
 
     def assertRequestTypesSent(self, subp, expected_request_types):
@@ -77,14 +82,16 @@ class TestCloudStackPasswordFetching(CiTestCase):
     def test_valid_response_means_password_marked_as_saved(self):
         password = 'SekritSquirrel'
         subp = self._set_password_server_response(password)
-        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds = DataSourceCloudStack(
+            {}, None, helpers.Paths({'run_dir': self.tmp}))
         ds.get_data()
         self.assertRequestTypesSent(subp,
                                     ['send_my_password', 'saved_password'])
 
     def _check_password_not_saved_for(self, response_string):
         subp = self._set_password_server_response(response_string)
-        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds = DataSourceCloudStack(
+            {}, None, helpers.Paths({'run_dir': self.tmp}))
         ds.get_data()
         self.assertRequestTypesSent(subp, ['send_my_password'])
 
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index 237c189b..98497886 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -725,8 +725,9 @@ class TestConvertNetworkData(TestCase):
 
 
 def cfg_ds_from_dir(seed_d):
+    tmp = tempfile.mkdtemp()
     cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None,
-                                      helpers.Paths({}))
+                                      helpers.Paths({'run_dir': tmp}))
     cfg_ds.seed_dir = seed_d
     cfg_ds.known_macs = KNOWN_MACS.copy()
     if not cfg_ds.get_data():
diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py
index f264f361..ec321733 100644
--- a/tests/unittests/test_datasource/test_digitalocean.py
+++ b/tests/unittests/test_datasource/test_digitalocean.py
@@ -13,7 +13,7 @@ from cloudinit import settings
 from cloudinit.sources import DataSourceDigitalOcean
 from cloudinit.sources.helpers import digitalocean
 
-from cloudinit.tests.helpers import mock, TestCase
+from cloudinit.tests.helpers import mock, CiTestCase
 
 DO_MULTIPLE_KEYS = ["ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@do.co",
                     "ssh-rsa AAAAB3NzaC1yc2EAAAA... test2@do.co"]
@@ -135,14 +135,17 @@ def _mock_dmi():
     return (True, DO_META.get('id'))
 
 
-class TestDataSourceDigitalOcean(TestCase):
+class TestDataSourceDigitalOcean(CiTestCase):
     """
     Test reading the meta-data
     """
+    def setUp(self):
+        super(TestDataSourceDigitalOcean, self).setUp()
+        self.tmp = self.tmp_dir()
 
     def get_ds(self, get_sysinfo=_mock_dmi):
         ds = DataSourceDigitalOcean.DataSourceDigitalOcean(
-            settings.CFG_BUILTIN, None, helpers.Paths({}))
+            settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp}))
         ds.use_ip4LL = False
         if get_sysinfo is not None:
             ds._get_sysinfo = get_sysinfo
@@ -194,7 +197,7 @@ class TestDataSourceDigitalOcean(TestCase):
         self.assertIsInstance(ds.get_public_ssh_keys(), list)
 
 
-class TestNetworkConvert(TestCase):
+class TestNetworkConvert(CiTestCase):
 
     @mock.patch('cloudinit.net.get_interfaces_by_mac')
     def _get_networking(self, m_get_by_mac):
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index ba328ee9..ba042eac 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -186,6 +186,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
         super(TestEc2, self).setUp()
         self.datasource = ec2.DataSourceEc2
         self.metadata_addr = self.datasource.metadata_urls[0]
+        self.tmp = self.tmp_dir()
 
     def data_url(self, version):
         """Return a metadata url based on the version provided."""
@@ -199,7 +200,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
     def _setup_ds(self, sys_cfg, platform_data, md, md_version=None):
         self.uris = []
         distro = {}
-        paths = helpers.Paths({})
+        paths = helpers.Paths({'run_dir': self.tmp})
         if sys_cfg is None:
             sys_cfg = {}
         ds = self.datasource(sys_cfg=sys_cfg, distro=distro, paths=paths)
diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py
index d399ae7a..82c788dc 100644
--- a/tests/unittests/test_datasource/test_gce.py
+++ b/tests/unittests/test_datasource/test_gce.py
@@ -70,9 +70,10 @@ def _set_mock_metadata(gce_meta=None):
 class TestDataSourceGCE(test_helpers.HttprettyTestCase):
 
     def setUp(self):
+        tmp = self.tmp_dir()
         self.ds = DataSourceGCE.DataSourceGCE(
             settings.CFG_BUILTIN, None,
-            helpers.Paths({}))
+            helpers.Paths({'run_dir': tmp}))
         ppatch = self.m_platform_reports_gce = mock.patch(
             'cloudinit.sources.DataSourceGCE.platform_reports_gce')
         self.m_platform_reports_gce = ppatch.start()
diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py
index fea9156b..70d50de4 100644
--- a/tests/unittests/test_datasource/test_nocloud.py
+++ b/tests/unittests/test_datasource/test_nocloud.py
@@ -3,22 +3,20 @@
 from cloudinit import helpers
 from cloudinit.sources import DataSourceNoCloud
 from cloudinit import util
-from cloudinit.tests.helpers import TestCase, populate_dir, mock, ExitStack
+from cloudinit.tests.helpers import CiTestCase, populate_dir, mock, ExitStack
 
 import os
-import shutil
-import tempfile
 import textwrap
 import yaml
 
 
-class TestNoCloudDataSource(TestCase):
+class TestNoCloudDataSource(CiTestCase):
 
     def setUp(self):
         super(TestNoCloudDataSource, self).setUp()
-        self.tmp = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.tmp)
-        self.paths = helpers.Paths({'cloud_dir': self.tmp})
+        self.tmp = self.tmp_dir()
+        self.paths = helpers.Paths(
+            {'cloud_dir': self.tmp, 'run_dir': self.tmp})
 
         self.cmdline = "root=TESTCMDLINE"
 
@@ -215,7 +213,7 @@ class TestNoCloudDataSource(TestCase):
         self.assertNotIn(gateway, str(dsrc.network_config))
 
 
-class TestParseCommandLineData(TestCase):
+class TestParseCommandLineData(CiTestCase):
 
     def test_parse_cmdline_data_valid(self):
         ds_id = "ds=nocloud"
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index e7d55692..2326dd58 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -3,12 +3,10 @@
 from cloudinit import helpers
 from cloudinit.sources import DataSourceOpenNebula as ds
 from cloudinit import util
-from cloudinit.tests.helpers import mock, populate_dir, TestCase
+from cloudinit.tests.helpers import mock, populate_dir, CiTestCase
 
 import os
 import pwd
-import shutil
-import tempfile
 import unittest
 
 
@@ -36,14 +34,14 @@ PUBLIC_IP = '10.0.0.3'
 DS_PATH = "cloudinit.sources.DataSourceOpenNebula"
 
 
-class TestOpenNebulaDataSource(TestCase):
+class TestOpenNebulaDataSource(CiTestCase):
     parsed_user = None
 
     def setUp(self):
         super(TestOpenNebulaDataSource, self).setUp()
-        self.tmp = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.tmp)
-        self.paths = helpers.Paths({'cloud_dir': self.tmp})
+        self.tmp = self.tmp_dir()
+        self.paths = helpers.Paths(
+            {'cloud_dir': self.tmp, 'run_dir': self.tmp})
 
         # defaults for few tests
         self.ds = ds.DataSourceOpenNebula
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index ed367e05..42c31554 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -131,6 +131,10 @@ def _read_metadata_service():
 class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
     VERSION = 'latest'
 
+    def setUp(self):
+        super(TestOpenStackDataSource, self).setUp()
+        self.tmp = self.tmp_dir()
+
     @hp.activate
     def test_successful(self):
         _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES)
@@ -232,7 +236,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES)
         ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN,
                                        None,
-                                       helpers.Paths({}))
+                                       helpers.Paths({'run_dir': self.tmp}))
         self.assertIsNone(ds_os.version)
         found = ds_os.get_data()
         self.assertTrue(found)
@@ -256,7 +260,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, {}, {}, os_files)
         ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN,
                                        None,
-                                       helpers.Paths({}))
+                                       helpers.Paths({'run_dir': self.tmp}))
         self.assertIsNone(ds_os.version)
         found = ds_os.get_data()
         self.assertFalse(found)
@@ -271,7 +275,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, {}, {}, os_files)
         ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN,
                                        None,
-                                       helpers.Paths({}))
+                                       helpers.Paths({'run_dir': self.tmp}))
         ds_os.ds_cfg = {
             'max_wait': 0,
             'timeout': 0,
@@ -294,7 +298,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, {}, {}, os_files)
         ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN,
                                        None,
-                                       helpers.Paths({}))
+                                       helpers.Paths({'run_dir': self.tmp}))
         ds_os.ds_cfg = {
             'max_wait': 0,
             'timeout': 0,
diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py
index 436df9ee..8dec06b1 100644
--- a/tests/unittests/test_datasource/test_scaleway.py
+++ b/tests/unittests/test_datasource/test_scaleway.py
@@ -9,7 +9,7 @@ from cloudinit import helpers
 from cloudinit import settings
 from cloudinit.sources import DataSourceScaleway
 
-from cloudinit.tests.helpers import mock, HttprettyTestCase, TestCase
+from cloudinit.tests.helpers import mock, HttprettyTestCase, CiTestCase
 
 
 class DataResponses(object):
@@ -63,7 +63,11 @@ class MetadataResponses(object):
         return 200, headers, json.dumps(cls.FAKE_METADATA)
 
 
-class TestOnScaleway(TestCase):
+class TestOnScaleway(CiTestCase):
+
+    def setUp(self):
+        super(TestOnScaleway, self).setUp()
+        self.tmp = self.tmp_dir()
 
     def install_mocks(self, fake_dmi, fake_file_exists, fake_cmdline):
         mock, faked = fake_dmi
@@ -91,7 +95,7 @@ class TestOnScaleway(TestCase):
 
         # When not on Scaleway, get_data() returns False.
         datasource = DataSourceScaleway.DataSourceScaleway(
-            settings.CFG_BUILTIN, None, helpers.Paths({})
+            settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp})
         )
         self.assertFalse(datasource.get_data())
 
@@ -159,8 +163,9 @@ def get_source_address_adapter(*args, **kwargs):
 class TestDataSourceScaleway(HttprettyTestCase):
 
     def setUp(self):
+        tmp = self.tmp_dir()
         self.datasource = DataSourceScaleway.DataSourceScaleway(
-            settings.CFG_BUILTIN, None, helpers.Paths({})
+            settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': tmp})
         )
         super(TestDataSourceScaleway, self).setUp()
 
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 933d5b63..88bae5f9 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -359,7 +359,8 @@ class TestSmartOSDataSource(FilesystemMockingTestCase):
 
         self.tmp = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, self.tmp)
-        self.paths = c_helpers.Paths({'cloud_dir': self.tmp})
+        self.paths = c_helpers.Paths(
+            {'cloud_dir': self.tmp, 'run_dir': self.tmp})
 
         self.legacy_user_d = os.path.join(self.tmp, 'legacy_user_tmp')
         os.mkdir(self.legacy_user_d)
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 1284e755..7a920d42 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -7,7 +7,7 @@ from uuid import uuid4
 from cloudinit import safeyaml
 from cloudinit import util
 from cloudinit.tests.helpers import (
-    CiTestCase, dir2dict, json_dumps, populate_dir)
+    CiTestCase, dir2dict, populate_dir)
 
 UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu "
                "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux")
@@ -319,7 +319,7 @@ def _print_run_output(rc, out, err, cfg, files):
         '-- rc = %s --' % rc,
         '-- out --', str(out),
         '-- err --', str(err),
-        '-- cfg --', json_dumps(cfg)]))
+        '-- cfg --', util.json_dumps(cfg)]))
     print('-- files --')
     for k, v in files.items():
         if "/_shwrap" in k:
diff --git a/tests/unittests/test_runs/test_merge_run.py b/tests/unittests/test_runs/test_merge_run.py
index add93653..5d3f1ca3 100644
--- a/tests/unittests/test_runs/test_merge_run.py
+++ b/tests/unittests/test_runs/test_merge_run.py
@@ -23,6 +23,7 @@ class TestMergeRun(helpers.FilesystemMockingTestCase):
         cfg = {
             'datasource_list': ['None'],
             'cloud_init_modules': ['write-files'],
+            'system_info': {'paths': {'run_dir': new_root}}
         }
         ud = self.readResource('user_data.1.txt')
         cloud_cfg = util.yaml_dumps(cfg)
diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py
index b8fb4794..762974e9 100644
--- a/tests/unittests/test_runs/test_simple_run.py
+++ b/tests/unittests/test_runs/test_simple_run.py
@@ -2,10 +2,10 @@
 
 import os
 
-from cloudinit.tests import helpers
 
 from cloudinit.settings import PER_INSTANCE
 from cloudinit import stages
+from cloudinit.tests import helpers
 from cloudinit import util
 
 
@@ -23,6 +23,7 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase):
             'datasource_list': ['None'],
             'runcmd': ['ls /etc'],  # test ALL_DISTROS
             'spacewalk': {},  # test non-ubuntu distros module definition
+            'system_info': {'paths': {'run_dir': self.new_root}},
             'write_files': [
                 {
                     'path': '/etc/blah.ini',