From 025ddc0329d9314f131cea35075734916797b439 Mon Sep 17 00:00:00 2001
From: Mike Gerdts <mike.gerdts@joyent.com>
Date: Wed, 18 Apr 2018 13:14:31 -0400
Subject: DataSourceSmartOS: change default fs on ephemeral disk from ext3 to
 ext4.

ext3 is not able to support file system sizes that are needed in Joyent's
cloud.  For the default block size of 4k, the maximum filesystem size
for ext3 is 2^32 * 4096 = 16 TiB.

This changes the default file system type from ext3 to ext4.

LP: #1763511
---
 cloudinit/sources/DataSourceSmartOS.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 86bfa5d8..5dd8a125 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -108,7 +108,7 @@ BUILTIN_CLOUD_CONFIG = {
                        'overwrite': False}
     },
     'fs_setup': [{'label': 'ephemeral0',
-                  'filesystem': 'ext3',
+                  'filesystem': 'ext4',
                   'device': 'ephemeral0'}],
 }
 
-- 
cgit v1.2.3


From 4c573d0e0173d2b1e99a383c54a0a6c957aa1cbb Mon Sep 17 00:00:00 2001
From: Mike Gerdts <mike.gerdts@joyent.com>
Date: Wed, 18 Apr 2018 13:55:17 -0400
Subject: DataSourceSmartOS: fix hang when metadata service is down

If the metadata service in the host is down while a guest that uses
DataSourceSmartOS is booting, the request from the guest falls into the
bit bucket.  When the metadata service is eventually started, the guest
has no awareness of this and does not resend the request.  This results in
cloud-init hanging forever with a guest reboot as the only recovery
option.

This fix updates the metadata protocol to implement the initialization
phase, just as is implemented by mdata-get and related utilities.  The
initialization phase includes draining all pending data from the serial
port, writing an empty command and getting an expected error message in
reply.  If the initialization phase times out, it is retried every five
seconds.  Each timeout results in a warning message: "Timeout while
initializing metadata client. Is the host metadata service running?"  By
default, warning messages are logged to the console, thus the reason for a
hung boot is readily apparent.

LP: #1667735
---
 cloudinit/sources/DataSourceSmartOS.py          | 117 +++++++++++++++++++++---
 tests/unittests/test_datasource/test_smartos.py | 103 ++++++++++++++++++++-
 2 files changed, 204 insertions(+), 16 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 5dd8a125..c8998b40 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -1,4 +1,5 @@
 # Copyright (C) 2013 Canonical Ltd.
+# Copyright (c) 2018, Joyent, Inc.
 #
 # Author: Ben Howard <ben.howard@canonical.com>
 #
@@ -21,6 +22,7 @@
 
 import base64
 import binascii
+import errno
 import json
 import os
 import random
@@ -229,6 +231,9 @@ class DataSourceSmartOS(sources.DataSource):
                       self.md_client)
             return False
 
+        # Open once for many requests, rather than once for each request
+        self.md_client.open_transport()
+
         for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items():
             smartos_noun, strip = attribute
             md[ci_noun] = self.md_client.get(smartos_noun, strip=strip)
@@ -236,6 +241,8 @@ class DataSourceSmartOS(sources.DataSource):
         for ci_noun, smartos_noun in SMARTOS_ATTRIB_JSON.items():
             md[ci_noun] = self.md_client.get_json(smartos_noun)
 
+        self.md_client.close_transport()
+
         # @datadictionary: This key may contain a program that is written
         # to a file in the filesystem of the guest on each boot and then
         # executed. It may be of any format that would be considered
@@ -316,6 +323,10 @@ class JoyentMetadataFetchException(Exception):
     pass
 
 
+class JoyentMetadataTimeoutException(JoyentMetadataFetchException):
+    pass
+
+
 class JoyentMetadataClient(object):
     """
     A client implementing v2 of the Joyent Metadata Protocol Specification.
@@ -360,6 +371,47 @@ class JoyentMetadataClient(object):
         LOG.debug('Value "%s" found.', value)
         return value
 
+    def _readline(self):
+        """
+           Reads a line a byte at a time until \n is encountered.  Returns an
+           ascii string with the trailing newline removed.
+
+           If a timeout (per-byte) is set and it expires, a
+           JoyentMetadataFetchException will be thrown.
+        """
+        response = []
+
+        def as_ascii():
+            return b''.join(response).decode('ascii')
+
+        msg = "Partial response: '%s'"
+        while True:
+            try:
+                byte = self.fp.read(1)
+                if len(byte) == 0:
+                    raise JoyentMetadataTimeoutException(msg % as_ascii())
+                if byte == b'\n':
+                    return as_ascii()
+                response.append(byte)
+            except OSError as exc:
+                if exc.errno == errno.EAGAIN:
+                    raise JoyentMetadataTimeoutException(msg % as_ascii())
+                raise
+
+    def _write(self, msg):
+        self.fp.write(msg.encode('ascii'))
+        self.fp.flush()
+
+    def _negotiate(self):
+        LOG.debug('Negotiating protocol V2')
+        self._write('NEGOTIATE V2\n')
+        response = self._readline()
+        LOG.debug('read "%s"', response)
+        if response != 'V2_OK':
+            raise JoyentMetadataFetchException(
+                'Invalid response "%s" to "NEGOTIATE V2"' % response)
+        LOG.debug('Negotiation complete')
+
     def request(self, rtype, param=None):
         request_id = '{0:08x}'.format(random.randint(0, 0xffffffff))
         message_body = ' '.join((request_id, rtype,))
@@ -374,18 +426,11 @@ class JoyentMetadataClient(object):
             self.open_transport()
             need_close = True
 
-        self.fp.write(msg.encode('ascii'))
-        self.fp.flush()
-
-        response = bytearray()
-        response.extend(self.fp.read(1))
-        while response[-1:] != b'\n':
-            response.extend(self.fp.read(1))
-
+        self._write(msg)
+        response = self._readline()
         if need_close:
             self.close_transport()
 
-        response = response.rstrip().decode('ascii')
         LOG.debug('Read "%s" from metadata transport.', response)
 
         if 'SUCCESS' not in response:
@@ -450,6 +495,7 @@ class JoyentMetadataSocketClient(JoyentMetadataClient):
         sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
         sock.connect(self.socketpath)
         self.fp = sock.makefile('rwb')
+        self._negotiate()
 
     def exists(self):
         return os.path.exists(self.socketpath)
@@ -459,8 +505,9 @@ class JoyentMetadataSocketClient(JoyentMetadataClient):
 
 
 class JoyentMetadataSerialClient(JoyentMetadataClient):
-    def __init__(self, device, timeout=10, smartos_type=SMARTOS_ENV_KVM):
-        super(JoyentMetadataSerialClient, self).__init__(smartos_type)
+    def __init__(self, device, timeout=10, smartos_type=SMARTOS_ENV_KVM,
+                 fp=None):
+        super(JoyentMetadataSerialClient, self).__init__(smartos_type, fp)
         self.device = device
         self.timeout = timeout
 
@@ -468,10 +515,50 @@ class JoyentMetadataSerialClient(JoyentMetadataClient):
         return os.path.exists(self.device)
 
     def open_transport(self):
-        ser = serial.Serial(self.device, timeout=self.timeout)
-        if not ser.isOpen():
-            raise SystemError("Unable to open %s" % self.device)
-        self.fp = ser
+        if self.fp is None:
+            ser = serial.Serial(self.device, timeout=self.timeout)
+            if not ser.isOpen():
+                raise SystemError("Unable to open %s" % self.device)
+            self.fp = ser
+        self._flush()
+        self._negotiate()
+
+    def _flush(self):
+        LOG.debug('Flushing input')
+        # Read any pending data
+        timeout = self.fp.timeout
+        self.fp.timeout = 0.1
+        while True:
+            try:
+                self._readline()
+            except JoyentMetadataTimeoutException:
+                break
+        LOG.debug('Input empty')
+
+        # Send a newline and expect "invalid command".  Keep trying until
+        # successful.  Retry rather frequently so that the "Is the host
+        # metadata service running" appears on the console soon after someone
+        # attaches in an effort to debug.
+        if timeout > 5:
+            self.fp.timeout = 5
+        else:
+            self.fp.timeout = timeout
+        while True:
+            LOG.debug('Writing newline, expecting "invalid command"')
+            self._write('\n')
+            try:
+                response = self._readline()
+                if response == 'invalid command':
+                    break
+                if response == 'FAILURE':
+                    LOG.debug('Got "FAILURE".  Retrying.')
+                    continue
+                LOG.warning('Unexpected response "%s" during flush', response)
+            except JoyentMetadataTimeoutException:
+                LOG.warning('Timeout while initializing metadata client. ' +
+                            'Is the host metadata service running?')
+        LOG.debug('Got "invalid command".  Flush complete.')
+        self.fp.timeout = timeout
 
     def __repr__(self):
         return "%s(device=%s, timeout=%s)" % (
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 88bae5f9..2bea7a17 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -1,4 +1,5 @@
 # Copyright (C) 2013 Canonical Ltd.
+# Copyright (c) 2018, Joyent, Inc.
 #
 # Author: Ben Howard <ben.howard@canonical.com>
 #
@@ -324,6 +325,7 @@ class PsuedoJoyentClient(object):
         if data is None:
             data = MOCK_RETURNS.copy()
         self.data = data
+        self._is_open = False
         return
 
     def get(self, key, default=None, strip=False):
@@ -344,6 +346,14 @@ class PsuedoJoyentClient(object):
     def exists(self):
         return True
 
+    def open_transport(self):
+        assert(not self._is_open)
+        self._is_open = True
+
+    def close_transport(self):
+        assert(self._is_open)
+        self._is_open = False
+
 
 class TestSmartOSDataSource(FilesystemMockingTestCase):
     def setUp(self):
@@ -592,8 +602,46 @@ class TestSmartOSDataSource(FilesystemMockingTestCase):
                          mydscfg['disk_aliases']['FOO'])
 
 
+class ShortReader(object):
+    """Implements a 'read' interface for bytes provided.
+    much like io.BytesIO but the 'endbyte' acts as if EOF.
+    When it is reached a short will be returned."""
+    def __init__(self, initial_bytes, endbyte=b'\0'):
+        self.data = initial_bytes
+        self.index = 0
+        self.len = len(self.data)
+        self.endbyte = endbyte
+
+    @property
+    def emptied(self):
+        return self.index >= self.len
+
+    def read(self, size=-1):
+        """Read size bytes but not past a null."""
+        if size == 0 or self.index >= self.len:
+            return b''
+
+        rsize = size
+        if size < 0 or size + self.index > self.len:
+            rsize = self.len - self.index
+
+        next_null = self.data.find(self.endbyte, self.index, rsize)
+        if next_null >= 0:
+            rsize = next_null - self.index + 1
+        i = self.index
+        self.index += rsize
+        ret = self.data[i:i + rsize]
+        if len(ret) and ret[-1:] == self.endbyte:
+            ret = ret[:-1]
+        return ret
+
+
 class TestJoyentMetadataClient(FilesystemMockingTestCase):
 
+    invalid = b'invalid command\n'
+    failure = b'FAILURE\n'
+    v2_ok = b'V2_OK\n'
+
     def setUp(self):
         super(TestJoyentMetadataClient, self).setUp()
 
@@ -636,6 +684,11 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase):
         return DataSourceSmartOS.JoyentMetadataClient(
             fp=self.serial, smartos_type=DataSourceSmartOS.SMARTOS_ENV_KVM)
 
+    def _get_serial_client(self):
+        self.serial.timeout = 1
+        return DataSourceSmartOS.JoyentMetadataSerialClient(None,
+                                                            fp=self.serial)
+
     def assertEndsWith(self, haystack, prefix):
         self.assertTrue(haystack.endswith(prefix),
                         "{0} does not end with '{1}'".format(
@@ -646,12 +699,14 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase):
                         "{0} does not start with '{1}'".format(
                             repr(haystack), prefix))
 
+    def assertNoMoreSideEffects(self, obj):
+        self.assertRaises(StopIteration, obj)
+
     def test_get_metadata_writes_a_single_line(self):
         client = self._get_client()
         client.get('some_key')
         self.assertEqual(1, self.serial.write.call_count)
         written_line = self.serial.write.call_args[0][0]
-        print(type(written_line))
         self.assertEndsWith(written_line.decode('ascii'),
                             b'\n'.decode('ascii'))
         self.assertEqual(1, written_line.count(b'\n'))
@@ -737,6 +792,52 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase):
         client._checksum = lambda _: self.response_parts['crc']
         self.assertIsNone(client.get('some_key'))
 
+    def test_negotiate(self):
+        client = self._get_client()
+        reader = ShortReader(self.v2_ok)
+        client.fp.read.side_effect = reader.read
+        client._negotiate()
+        self.assertTrue(reader.emptied)
+
+    def test_negotiate_short_response(self):
+        client = self._get_client()
+        # chopped '\n' from v2_ok.
+        reader = ShortReader(self.v2_ok[:-1] + b'\0')
+        client.fp.read.side_effect = reader.read
+        self.assertRaises(DataSourceSmartOS.JoyentMetadataTimeoutException,
+                          client._negotiate)
+        self.assertTrue(reader.emptied)
+
+    def test_negotiate_bad_response(self):
+        client = self._get_client()
+        reader = ShortReader(b'garbage\n' + self.v2_ok)
+        client.fp.read.side_effect = reader.read
+        self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException,
+                          client._negotiate)
+        self.assertEqual(self.v2_ok, client.fp.read())
+
+    def test_serial_open_transport(self):
+        client = self._get_serial_client()
+        reader = ShortReader(b'garbage\0' + self.invalid + self.v2_ok)
+        client.fp.read.side_effect = reader.read
+        client.open_transport()
+        self.assertTrue(reader.emptied)
+
+    def test_flush_failure(self):
+        client = self._get_serial_client()
+        reader = ShortReader(b'garbage' + b'\0' + self.failure +
+                             self.invalid + self.v2_ok)
+        client.fp.read.side_effect = reader.read
+        client.open_transport()
+        self.assertTrue(reader.emptied)
+
+    def test_flush_many_timeouts(self):
+        client = self._get_serial_client()
+        reader = ShortReader(b'\0' * 100 + self.invalid + self.v2_ok)
+        client.fp.read.side_effect = reader.read
+        client.open_transport()
+        self.assertTrue(reader.emptied)
+
 
 class TestNetworkConversion(TestCase):
     def test_convert_simple(self):
-- 
cgit v1.2.3


From 1081962eacf2814fea6f4fa3255c530de14e4a24 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Thu, 19 Apr 2018 21:30:08 -0600
Subject: pylint: pay attention to unused variable warnings.

This enables warnings produced by pylint for unused variables (W0612),
and fixes the existing errors.
---
 .pylintrc                                          |  2 +-
 cloudinit/analyze/dump.py                          |  2 +-
 cloudinit/cmd/tests/test_main.py                   |  6 ++--
 cloudinit/config/cc_apt_configure.py               |  2 +-
 cloudinit/config/cc_emit_upstart.py                |  2 +-
 cloudinit/config/cc_resizefs.py                    |  8 ++----
 cloudinit/config/cc_rh_subscription.py             | 18 ++++++------
 cloudinit/config/cc_snap.py                        |  4 +--
 cloudinit/config/cc_snappy.py                      |  4 +--
 cloudinit/config/cc_ubuntu_advantage.py            |  4 +--
 cloudinit/config/schema.py                         |  4 +--
 cloudinit/distros/freebsd.py                       |  2 +-
 cloudinit/distros/ubuntu.py                        |  2 +-
 cloudinit/net/__init__.py                          |  2 +-
 cloudinit/net/cmdline.py                           |  2 +-
 cloudinit/net/dhcp.py                              |  2 +-
 cloudinit/net/sysconfig.py                         |  2 +-
 cloudinit/reporting/events.py                      |  2 +-
 cloudinit/sources/DataSourceAliYun.py              |  2 +-
 cloudinit/sources/DataSourceAzure.py               | 33 +++++++++-------------
 cloudinit/sources/DataSourceMAAS.py                |  2 +-
 cloudinit/sources/DataSourceOVF.py                 |  2 +-
 cloudinit/sources/DataSourceOpenStack.py           |  4 +--
 cloudinit/sources/helpers/digitalocean.py          |  7 ++---
 cloudinit/sources/helpers/openstack.py             |  2 +-
 cloudinit/sources/helpers/vmware/imc/config_nic.py |  2 +-
 .../sources/helpers/vmware/imc/config_passwd.py    |  4 +--
 .../sources/helpers/vmware/imc/guestcust_util.py   |  4 +--
 cloudinit/sources/tests/test_init.py               |  2 +-
 cloudinit/templater.py                             |  2 +-
 cloudinit/tests/helpers.py                         |  2 +-
 cloudinit/tests/test_util.py                       |  2 +-
 cloudinit/url_helper.py                            |  2 +-
 cloudinit/util.py                                  |  2 +-
 tests/cloud_tests/bddeb.py                         |  2 +-
 tests/cloud_tests/collect.py                       |  3 +-
 tests/cloud_tests/platforms/instances.py           |  2 +-
 tests/cloud_tests/platforms/lxd/instance.py        | 10 +++----
 tests/cloud_tests/setup_image.py                   | 11 ++++----
 tests/cloud_tests/testcases/base.py                |  2 +-
 .../testcases/examples/including_user_groups.py    |  2 +-
 tests/cloud_tests/testcases/modules/user_groups.py |  2 +-
 tests/cloud_tests/util.py                          |  2 +-
 tests/unittests/test__init__.py                    |  2 +-
 tests/unittests/test_datasource/test_azure.py      |  4 +--
 tests/unittests/test_datasource/test_maas.py       |  4 +--
 tests/unittests/test_datasource/test_nocloud.py    |  3 --
 .../test_handler/test_handler_apt_source_v3.py     |  2 +-
 tests/unittests/test_handler/test_handler_ntp.py   |  2 +-
 tests/unittests/test_templating.py                 |  4 +--
 tests/unittests/test_util.py                       |  6 ++--
 51 files changed, 95 insertions(+), 112 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/.pylintrc b/.pylintrc
index 0bdfa59d..3bfa0c81 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -28,7 +28,7 @@ jobs=4
 # W0703(broad-except)
 # W1401(anomalous-backslash-in-string)
 
-disable=C, F, I, R, W0105, W0107, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0612, W0613, W0621, W0622, W0631, W0703, W1401
+disable=C, F, I, R, W0105, W0107, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401
 
 
 [REPORTS]
diff --git a/cloudinit/analyze/dump.py b/cloudinit/analyze/dump.py
index b071aa19..1f3060d0 100644
--- a/cloudinit/analyze/dump.py
+++ b/cloudinit/analyze/dump.py
@@ -112,7 +112,7 @@ def parse_ci_logline(line):
             return None
         event_description = stage_to_description[event_name]
     else:
-        (pymodloglvl, event_type, event_name) = eventstr.split()[0:3]
+        (_pymodloglvl, event_type, event_name) = eventstr.split()[0:3]
         event_description = eventstr.split(event_name)[1].strip()
 
     event = {
diff --git a/cloudinit/cmd/tests/test_main.py b/cloudinit/cmd/tests/test_main.py
index dbe421c0..e2c54ae8 100644
--- a/cloudinit/cmd/tests/test_main.py
+++ b/cloudinit/cmd/tests/test_main.py
@@ -56,7 +56,7 @@ class TestMain(FilesystemMockingTestCase):
         cmdargs = myargs(
             debug=False, files=None, force=False, local=False, reporter=None,
             subcommand='init')
-        (item1, item2) = wrap_and_call(
+        (_item1, item2) = wrap_and_call(
             'cloudinit.cmd.main',
             {'util.close_stdin': True,
              'netinfo.debug_info': 'my net debug info',
@@ -85,7 +85,7 @@ class TestMain(FilesystemMockingTestCase):
         cmdargs = myargs(
             debug=False, files=None, force=False, local=False, reporter=None,
             subcommand='init')
-        (item1, item2) = wrap_and_call(
+        (_item1, item2) = wrap_and_call(
             'cloudinit.cmd.main',
             {'util.close_stdin': True,
              'netinfo.debug_info': 'my net debug info',
@@ -133,7 +133,7 @@ class TestMain(FilesystemMockingTestCase):
             self.assertEqual(main.LOG, log)
             self.assertIsNone(args)
 
-        (item1, item2) = wrap_and_call(
+        (_item1, item2) = wrap_and_call(
             'cloudinit.cmd.main',
             {'util.close_stdin': True,
              'netinfo.debug_info': 'my net debug info',
diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index afaca464..e18944ec 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -378,7 +378,7 @@ def apply_debconf_selections(cfg, target=None):
 
     # get a complete list of packages listed in input
     pkgs_cfgd = set()
-    for key, content in selsets.items():
+    for _key, content in selsets.items():
         for line in content.splitlines():
             if line.startswith("#"):
                 continue
diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py
index 69dc2d5e..eb9fbe66 100644
--- a/cloudinit/config/cc_emit_upstart.py
+++ b/cloudinit/config/cc_emit_upstart.py
@@ -43,7 +43,7 @@ def is_upstart_system():
         del myenv['UPSTART_SESSION']
     check_cmd = ['initctl', 'version']
     try:
-        (out, err) = util.subp(check_cmd, env=myenv)
+        (out, _err) = util.subp(check_cmd, env=myenv)
         return 'upstart' in out
     except util.ProcessExecutionError as e:
         LOG.debug("'%s' returned '%s', not using upstart",
diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py
index 013e69b5..82f29e10 100644
--- a/cloudinit/config/cc_resizefs.py
+++ b/cloudinit/config/cc_resizefs.py
@@ -89,13 +89,11 @@ def _resize_zfs(mount_point, devpth):
 
 
 def _get_dumpfs_output(mount_point):
-    dumpfs_res, err = util.subp(['dumpfs', '-m', mount_point])
-    return dumpfs_res
+    return util.subp(['dumpfs', '-m', mount_point])[0]
 
 
 def _get_gpart_output(part):
-    gpart_res, err = util.subp(['gpart', 'show', part])
-    return gpart_res
+    return util.subp(['gpart', 'show', part])[0]
 
 
 def _can_skip_resize_ufs(mount_point, devpth):
@@ -113,7 +111,7 @@ def _can_skip_resize_ufs(mount_point, devpth):
         if not line.startswith('#'):
             newfs_cmd = shlex.split(line)
             opt_value = 'O:Ua:s:b:d:e:f:g:h:i:jk:m:o:'
-            optlist, args = getopt.getopt(newfs_cmd[1:], opt_value)
+            optlist, _args = getopt.getopt(newfs_cmd[1:], opt_value)
             for o, a in optlist:
                 if o == "-s":
                     cur_fs_sz = int(a)
diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py
index 530808ce..1c679430 100644
--- a/cloudinit/config/cc_rh_subscription.py
+++ b/cloudinit/config/cc_rh_subscription.py
@@ -209,8 +209,7 @@ class SubscriptionManager(object):
                 cmd.append("--serverurl={0}".format(self.server_hostname))
 
             try:
-                return_out, return_err = self._sub_man_cli(cmd,
-                                                           logstring_val=True)
+                return_out = self._sub_man_cli(cmd, logstring_val=True)[0]
             except util.ProcessExecutionError as e:
                 if e.stdout == "":
                     self.log_warn("Registration failed due "
@@ -233,8 +232,7 @@ class SubscriptionManager(object):
 
             # Attempting to register the system only
             try:
-                return_out, return_err = self._sub_man_cli(cmd,
-                                                           logstring_val=True)
+                return_out = self._sub_man_cli(cmd, logstring_val=True)[0]
             except util.ProcessExecutionError as e:
                 if e.stdout == "":
                     self.log_warn("Registration failed due "
@@ -257,7 +255,7 @@ class SubscriptionManager(object):
                .format(self.servicelevel)]
 
         try:
-            return_out, return_err = self._sub_man_cli(cmd)
+            return_out = self._sub_man_cli(cmd)[0]
         except util.ProcessExecutionError as e:
             if e.stdout.rstrip() != '':
                 for line in e.stdout.split("\n"):
@@ -275,7 +273,7 @@ class SubscriptionManager(object):
     def _set_auto_attach(self):
         cmd = ['attach', '--auto']
         try:
-            return_out, return_err = self._sub_man_cli(cmd)
+            return_out = self._sub_man_cli(cmd)[0]
         except util.ProcessExecutionError as e:
             self.log_warn("Auto-attach failed with: {0}".format(e))
             return False
@@ -294,12 +292,12 @@ class SubscriptionManager(object):
 
         # Get all available pools
         cmd = ['list', '--available', '--pool-only']
-        results, errors = self._sub_man_cli(cmd)
+        results = self._sub_man_cli(cmd)[0]
         available = (results.rstrip()).split("\n")
 
         # Get all consumed pools
         cmd = ['list', '--consumed', '--pool-only']
-        results, errors = self._sub_man_cli(cmd)
+        results = self._sub_man_cli(cmd)[0]
         consumed = (results.rstrip()).split("\n")
 
         return available, consumed
@@ -311,14 +309,14 @@ class SubscriptionManager(object):
         '''
 
         cmd = ['repos', '--list-enabled']
-        return_out, return_err = self._sub_man_cli(cmd)
+        return_out = self._sub_man_cli(cmd)[0]
         active_repos = []
         for repo in return_out.split("\n"):
             if "Repo ID:" in repo:
                 active_repos.append((repo.split(':')[1]).strip())
 
         cmd = ['repos', '--list-disabled']
-        return_out, return_err = self._sub_man_cli(cmd)
+        return_out = self._sub_man_cli(cmd)[0]
 
         inactive_repos = []
         for repo in return_out.split("\n"):
diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py
index a7a03214..90724b81 100644
--- a/cloudinit/config/cc_snap.py
+++ b/cloudinit/config/cc_snap.py
@@ -203,12 +203,12 @@ def maybe_install_squashfuse(cloud):
         return
     try:
         cloud.distro.update_package_sources()
-    except Exception as e:
+    except Exception:
         util.logexc(LOG, "Package update failed")
         raise
     try:
         cloud.distro.install_packages(['squashfuse'])
-    except Exception as e:
+    except Exception:
         util.logexc(LOG, "Failed to install squashfuse")
         raise
 
diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py
index bab80bbe..15bee2d3 100644
--- a/cloudinit/config/cc_snappy.py
+++ b/cloudinit/config/cc_snappy.py
@@ -213,7 +213,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None):
 
 def read_installed_packages():
     ret = []
-    for (name, date, version, dev) in read_pkg_data():
+    for (name, _date, _version, dev) in read_pkg_data():
         if dev:
             ret.append(NAMESPACE_DELIM.join([name, dev]))
         else:
@@ -222,7 +222,7 @@ def read_installed_packages():
 
 
 def read_pkg_data():
-    out, err = util.subp([SNAPPY_CMD, "list"])
+    out, _err = util.subp([SNAPPY_CMD, "list"])
     pkg_data = []
     for line in out.splitlines()[1:]:
         toks = line.split(sep=None, maxsplit=3)
diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py
index 29d18c96..5e082bd6 100644
--- a/cloudinit/config/cc_ubuntu_advantage.py
+++ b/cloudinit/config/cc_ubuntu_advantage.py
@@ -148,12 +148,12 @@ def maybe_install_ua_tools(cloud):
         return
     try:
         cloud.distro.update_package_sources()
-    except Exception as e:
+    except Exception:
         util.logexc(LOG, "Package update failed")
         raise
     try:
         cloud.distro.install_packages(['ubuntu-advantage-tools'])
-    except Exception as e:
+    except Exception:
         util.logexc(LOG, "Failed to install ubuntu-advantage-tools")
         raise
 
diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py
index ca7d0d5b..76826e05 100644
--- a/cloudinit/config/schema.py
+++ b/cloudinit/config/schema.py
@@ -297,8 +297,8 @@ def get_schema():
 
     configs_dir = os.path.dirname(os.path.abspath(__file__))
     potential_handlers = find_modules(configs_dir)
-    for (fname, mod_name) in potential_handlers.items():
-        mod_locs, looked_locs = importer.find_module(
+    for (_fname, mod_name) in potential_handlers.items():
+        mod_locs, _looked_locs = importer.find_module(
             mod_name, ['cloudinit.config'], ['schema'])
         if mod_locs:
             mod = importer.import_module(mod_locs[0])
diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py
index 099fac5c..5b1718a4 100644
--- a/cloudinit/distros/freebsd.py
+++ b/cloudinit/distros/freebsd.py
@@ -113,7 +113,7 @@ class Distro(distros.Distro):
         n = re.search(r'\d+$', dev)
         index = n.group(0)
 
-        (out, err) = util.subp(['ifconfig', '-a'])
+        (out, _err) = util.subp(['ifconfig', '-a'])
         ifconfigoutput = [x for x in (out.strip()).splitlines()
                           if len(x.split()) > 0]
         bsddev = 'NOT_FOUND'
diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py
index fdc1f622..68154104 100644
--- a/cloudinit/distros/ubuntu.py
+++ b/cloudinit/distros/ubuntu.py
@@ -25,7 +25,7 @@ class Distro(debian.Distro):
     def preferred_ntp_clients(self):
         """The preferred ntp client is dependent on the version."""
         if not self._preferred_ntp_clients:
-            (name, version, codename) = util.system_info()['dist']
+            (_name, _version, codename) = util.system_info()['dist']
             # Xenial cloud-init only installed ntp, UbuntuCore has timesyncd.
             if codename == "xenial" and not util.system_is_snappy():
                 self._preferred_ntp_clients = ['ntp']
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index f69c0ef2..80054546 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -295,7 +295,7 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True):
 
     def _version_2(netcfg):
         renames = []
-        for key, ent in netcfg.get('ethernets', {}).items():
+        for ent in netcfg.get('ethernets', {}).values():
             # only rename if configured to do so
             name = ent.get('set-name')
             if not name:
diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py
index 9e9fe0fe..f89a0f73 100755
--- a/cloudinit/net/cmdline.py
+++ b/cloudinit/net/cmdline.py
@@ -65,7 +65,7 @@ def _klibc_to_config_entry(content, mac_addrs=None):
         iface['mac_address'] = mac_addrs[name]
 
     # Handle both IPv4 and IPv6 values
-    for v, pre in (('ipv4', 'IPV4'), ('ipv6', 'IPV6')):
+    for pre in ('IPV4', 'IPV6'):
         # if no IPV4ADDR or IPV6ADDR, then go on.
         if pre + "ADDR" not in data:
             continue
diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py
index 087c0c03..12cf5097 100644
--- a/cloudinit/net/dhcp.py
+++ b/cloudinit/net/dhcp.py
@@ -216,7 +216,7 @@ def networkd_get_option_from_leases(keyname, leases_d=None):
     if leases_d is None:
         leases_d = NETWORKD_LEASES_DIR
     leases = networkd_load_leases(leases_d=leases_d)
-    for ifindex, data in sorted(leases.items()):
+    for _ifindex, data in sorted(leases.items()):
         if data.get(keyname):
             return data[keyname]
     return None
diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py
index 39d89c46..7a7f5093 100644
--- a/cloudinit/net/sysconfig.py
+++ b/cloudinit/net/sysconfig.py
@@ -364,7 +364,7 @@ class Renderer(renderer.Renderer):
 
     @classmethod
     def _render_subnet_routes(cls, iface_cfg, route_cfg, subnets):
-        for i, subnet in enumerate(subnets, start=len(iface_cfg.children)):
+        for _, subnet in enumerate(subnets, start=len(iface_cfg.children)):
             for route in subnet.get('routes', []):
                 is_ipv6 = subnet.get('ipv6') or is_ipv6_addr(route['gateway'])
 
diff --git a/cloudinit/reporting/events.py b/cloudinit/reporting/events.py
index 4f62d2f9..e5dfab33 100644
--- a/cloudinit/reporting/events.py
+++ b/cloudinit/reporting/events.py
@@ -192,7 +192,7 @@ class ReportEventStack(object):
 
     def _childrens_finish_info(self):
         for cand_result in (status.FAIL, status.WARN):
-            for name, (value, msg) in self.children.items():
+            for _name, (value, _msg) in self.children.items():
                 if value == cand_result:
                     return (value, self.message)
         return (self.result, self.message)
diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py
index 22279d09..858e0827 100644
--- a/cloudinit/sources/DataSourceAliYun.py
+++ b/cloudinit/sources/DataSourceAliYun.py
@@ -45,7 +45,7 @@ def _is_aliyun():
 
 def parse_public_keys(public_keys):
     keys = []
-    for key_id, key_body in public_keys.items():
+    for _key_id, key_body in public_keys.items():
         if isinstance(key_body, str):
             keys.append(key_body.strip())
         elif isinstance(key_body, list):
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 0ee622e2..a71197a6 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -107,31 +107,24 @@ def find_dev_from_busdev(camcontrol_out, busdev):
     return None
 
 
-def get_dev_storvsc_sysctl():
+def execute_or_debug(cmd, fail_ret=None):
     try:
-        sysctl_out, err = util.subp(['sysctl', 'dev.storvsc'])
+        return util.subp(cmd)[0]
     except util.ProcessExecutionError:
-        LOG.debug("Fail to execute sysctl dev.storvsc")
-        sysctl_out = ""
-    return sysctl_out
+        LOG.debug("Failed to execute: %s", ' '.join(cmd))
+        return fail_ret
+
+
+def get_dev_storvsc_sysctl():
+    return execute_or_debug(["sysctl", "dev.storvsc"], fail_ret="")
 
 
 def get_camcontrol_dev_bus():
-    try:
-        camcontrol_b_out, err = util.subp(['camcontrol', 'devlist', '-b'])
-    except util.ProcessExecutionError:
-        LOG.debug("Fail to execute camcontrol devlist -b")
-        return None
-    return camcontrol_b_out
+    return execute_or_debug(['camcontrol', 'devlist', '-b'])
 
 
 def get_camcontrol_dev():
-    try:
-        camcontrol_out, err = util.subp(['camcontrol', 'devlist'])
-    except util.ProcessExecutionError:
-        LOG.debug("Fail to execute camcontrol devlist")
-        return None
-    return camcontrol_out
+    return execute_or_debug(['camcontrol', 'devlist'])
 
 
 def get_resource_disk_on_freebsd(port_id):
@@ -474,7 +467,7 @@ class DataSourceAzure(sources.DataSource):
            before we go into our polling loop."""
         try:
             get_metadata_from_fabric(None, lease['unknown-245'])
-        except Exception as exc:
+        except Exception:
             LOG.warning(
                 "Error communicating with Azure fabric; You may experience."
                 "connectivity issues.", exc_info=True)
@@ -492,7 +485,7 @@ class DataSourceAzure(sources.DataSource):
         jump back into the polling loop in order to retrieve the ovf_env."""
         if not ret:
             return False
-        (md, self.userdata_raw, cfg, files) = ret
+        (_md, self.userdata_raw, cfg, _files) = ret
         path = REPROVISION_MARKER_FILE
         if (cfg.get('PreprovisionedVm') is True or
                 os.path.isfile(path)):
@@ -528,7 +521,7 @@ class DataSourceAzure(sources.DataSource):
                   self.ds_cfg['agent_command'])
         try:
             fabric_data = metadata_func()
-        except Exception as exc:
+        except Exception:
             LOG.warning(
                 "Error communicating with Azure fabric; You may experience."
                 "connectivity issues.", exc_info=True)
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index 6ac88635..aa56addb 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -204,7 +204,7 @@ def read_maas_seed_url(seed_url, read_file_or_url=None, timeout=None,
         seed_url = seed_url[:-1]
 
     md = {}
-    for path, dictname, binary, optional in DS_FIELDS:
+    for path, _dictname, binary, optional in DS_FIELDS:
         if version is None:
             url = "%s/%s" % (seed_url, path)
         else:
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index dc914a72..178ccb0f 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -556,7 +556,7 @@ def search_file(dirpath, filename):
     if not dirpath or not filename:
         return None
 
-    for root, dirs, files in os.walk(dirpath):
+    for root, _dirs, files in os.walk(dirpath):
         if filename in files:
             return os.path.join(root, filename)
 
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index e55a7638..fb166ae1 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -86,7 +86,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
             md_urls.append(md_url)
             url2base[md_url] = url
 
-        (max_wait, timeout, retries) = self._get_url_settings()
+        (max_wait, timeout, _retries) = self._get_url_settings()
         start_time = time.time()
         avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait,
                                             timeout=timeout)
@@ -106,7 +106,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
         except IOError:
             return False
 
-        (max_wait, timeout, retries) = self._get_url_settings()
+        (_max_wait, timeout, retries) = self._get_url_settings()
 
         try:
             results = util.log_time(LOG.debug,
diff --git a/cloudinit/sources/helpers/digitalocean.py b/cloudinit/sources/helpers/digitalocean.py
index 693f8d5c..0e7cccac 100644
--- a/cloudinit/sources/helpers/digitalocean.py
+++ b/cloudinit/sources/helpers/digitalocean.py
@@ -41,10 +41,9 @@ def assign_ipv4_link_local(nic=None):
                            "address")
 
     try:
-        (result, _err) = util.subp(ip_addr_cmd)
+        util.subp(ip_addr_cmd)
         LOG.debug("assigned ip4LL address '%s' to '%s'", addr, nic)
-
-        (result, _err) = util.subp(ip_link_cmd)
+        util.subp(ip_link_cmd)
         LOG.debug("brought device '%s' up", nic)
     except Exception:
         util.logexc(LOG, "ip4LL address assignment of '%s' to '%s' failed."
@@ -75,7 +74,7 @@ def del_ipv4_link_local(nic=None):
     ip_addr_cmd = ['ip', 'addr', 'flush', 'dev', nic]
 
     try:
-        (result, _err) = util.subp(ip_addr_cmd)
+        util.subp(ip_addr_cmd)
         LOG.debug("removed ip4LL addresses from %s", nic)
 
     except Exception as e:
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 26f3168d..a4cf0667 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -638,7 +638,7 @@ def convert_net_json(network_json=None, known_macs=None):
             known_macs = net.get_interfaces_by_mac()
 
         # go through and fill out the link_id_info with names
-        for link_id, info in link_id_info.items():
+        for _link_id, info in link_id_info.items():
             if info.get('name'):
                 continue
             if info.get('mac') in known_macs:
diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py
index 2d8900e2..3ef8c624 100644
--- a/cloudinit/sources/helpers/vmware/imc/config_nic.py
+++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py
@@ -73,7 +73,7 @@ class NicConfigurator(object):
         The mac address(es) are in the lower case
         """
         cmd = ['ip', 'addr', 'show']
-        (output, err) = util.subp(cmd)
+        output, _err = util.subp(cmd)
         sections = re.split(r'\n\d+: ', '\n' + output)[1:]
 
         macPat = r'link/ether (([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2}))'
diff --git a/cloudinit/sources/helpers/vmware/imc/config_passwd.py b/cloudinit/sources/helpers/vmware/imc/config_passwd.py
index 75cfbaaf..8c91fa41 100644
--- a/cloudinit/sources/helpers/vmware/imc/config_passwd.py
+++ b/cloudinit/sources/helpers/vmware/imc/config_passwd.py
@@ -56,10 +56,10 @@ class PasswordConfigurator(object):
         LOG.info('Expiring password.')
         for user in uidUserList:
             try:
-                out, err = util.subp(['passwd', '--expire', user])
+                util.subp(['passwd', '--expire', user])
             except util.ProcessExecutionError as e:
                 if os.path.exists('/usr/bin/chage'):
-                    out, e = util.subp(['chage', '-d', '0', user])
+                    util.subp(['chage', '-d', '0', user])
                 else:
                     LOG.warning('Failed to expire password for %s with error: '
                                 '%s', user, e)
diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py
index 44075255..a590f323 100644
--- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py
+++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py
@@ -91,7 +91,7 @@ def enable_nics(nics):
 
     for attempt in range(0, enableNicsWaitRetries):
         logger.debug("Trying to connect interfaces, attempt %d", attempt)
-        (out, err) = set_customization_status(
+        (out, _err) = set_customization_status(
             GuestCustStateEnum.GUESTCUST_STATE_RUNNING,
             GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS,
             nics)
@@ -104,7 +104,7 @@ def enable_nics(nics):
             return
 
         for count in range(0, enableNicsWaitCount):
-            (out, err) = set_customization_status(
+            (out, _err) = set_customization_status(
                 GuestCustStateEnum.GUESTCUST_STATE_RUNNING,
                 GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS,
                 nics)
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index e7fda22a..452e9219 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -278,7 +278,7 @@ class TestDataSource(CiTestCase):
         base_args = get_args(DataSource.get_hostname)  # pylint: disable=W1505
         # Import all DataSource subclasses so we can inspect them.
         modules = util.find_modules(os.path.dirname(os.path.dirname(__file__)))
-        for loc, name in modules.items():
+        for _loc, name in modules.items():
             mod_locs, _ = importer.find_module(name, ['cloudinit.sources'], [])
             if mod_locs:
                 importer.import_module(mod_locs[0])
diff --git a/cloudinit/templater.py b/cloudinit/templater.py
index 9a087e1c..7e7acb86 100644
--- a/cloudinit/templater.py
+++ b/cloudinit/templater.py
@@ -147,7 +147,7 @@ def render_string(content, params):
     Warning: py2 str with non-ascii chars will cause UnicodeDecodeError."""
     if not params:
         params = {}
-    template_type, renderer, content = detect_template(content)
+    _template_type, renderer, content = detect_template(content)
     return renderer(content, params)
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py
index 82fd347b..5aada6e7 100644
--- a/cloudinit/tests/helpers.py
+++ b/cloudinit/tests/helpers.py
@@ -334,7 +334,7 @@ def dir2dict(startdir, prefix=None):
     flist = {}
     if prefix is None:
         prefix = startdir
-    for root, dirs, files in os.walk(startdir):
+    for root, _dirs, files in os.walk(startdir):
         for fname in files:
             fpath = os.path.join(root, fname)
             key = fpath[len(prefix):]
diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py
index 3f37dbb6..76eed076 100644
--- a/cloudinit/tests/test_util.py
+++ b/cloudinit/tests/test_util.py
@@ -135,7 +135,7 @@ class TestGetHostnameFqdn(CiTestCase):
     def test_get_hostname_fqdn_from_passes_metadata_only_to_cloud(self):
         """Calls to cloud.get_hostname pass the metadata_only parameter."""
         mycloud = FakeCloud('cloudhost', 'cloudhost.mycloud.com')
-        hostname, fqdn = util.get_hostname_fqdn(
+        _hn, _fqdn = util.get_hostname_fqdn(
             cfg={}, cloud=mycloud, metadata_only=True)
         self.assertEqual(
             [{'fqdn': True, 'metadata_only': True},
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 03a573af..1de07b1c 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -519,7 +519,7 @@ def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret,
         resource_owner_secret=token_secret,
         signature_method=oauth1.SIGNATURE_PLAINTEXT,
         timestamp=timestamp)
-    uri, signed_headers, body = client.sign(url)
+    _uri, signed_headers, _body = client.sign(url)
     return signed_headers
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 1717b529..310758dd 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -2214,7 +2214,7 @@ def parse_mtab(path):
 def find_freebsd_part(label_part):
     if label_part.startswith("/dev/label/"):
         target_label = label_part[5:]
-        (label_part, err) = subp(['glabel', 'status', '-s'])
+        (label_part, _err) = subp(['glabel', 'status', '-s'])
         for labels in label_part.split("\n"):
             items = labels.split()
             if len(items) > 0 and items[0].startswith(target_label):
diff --git a/tests/cloud_tests/bddeb.py b/tests/cloud_tests/bddeb.py
index b9cfcfa6..f04d0cd4 100644
--- a/tests/cloud_tests/bddeb.py
+++ b/tests/cloud_tests/bddeb.py
@@ -113,7 +113,7 @@ def bddeb(args):
     @return_value: fail count
     """
     LOG.info('preparing to build cloud-init deb')
-    (res, failed) = run_stage('build deb', [partial(setup_build, args)])
+    _res, failed = run_stage('build deb', [partial(setup_build, args)])
     return failed
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py
index d4f9135b..1ba72856 100644
--- a/tests/cloud_tests/collect.py
+++ b/tests/cloud_tests/collect.py
@@ -25,7 +25,8 @@ def collect_script(instance, base_dir, script, script_name):
         script.encode(), rcs=False,
         description='collect: {}'.format(script_name))
     if err:
-        LOG.debug("collect script %s had stderr: %s", script_name, err)
+        LOG.debug("collect script %s exited '%s' and had stderr: %s",
+                  script_name, err, exit)
     if not isinstance(out, bytes):
         raise util.PlatformError(
             "Collection of '%s' returned type %s, expected bytes: %s" %
diff --git a/tests/cloud_tests/platforms/instances.py b/tests/cloud_tests/platforms/instances.py
index 3bad021f..cc439d29 100644
--- a/tests/cloud_tests/platforms/instances.py
+++ b/tests/cloud_tests/platforms/instances.py
@@ -108,7 +108,7 @@ class Instance(TargetBase):
                 return client
             except (ConnectionRefusedError, AuthenticationException,
                     BadHostKeyException, ConnectionResetError, SSHException,
-                    OSError) as e:
+                    OSError):
                 retries -= 1
                 time.sleep(10)
 
diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py
index 0d957bca..1c17c781 100644
--- a/tests/cloud_tests/platforms/lxd/instance.py
+++ b/tests/cloud_tests/platforms/lxd/instance.py
@@ -152,9 +152,8 @@ class LXDInstance(Instance):
                 return fp.read()
 
         try:
-            stdout, stderr = subp(
-                ['lxc', 'console', '--show-log', self.name], decode=False)
-            return stdout
+            return subp(['lxc', 'console', '--show-log', self.name],
+                        decode=False)[0]
         except ProcessExecutionError as e:
             raise PlatformError(
                 "console log",
@@ -214,11 +213,10 @@ def _has_proper_console_support():
             reason = "LXD Driver version not 3.x+ (%s)" % dver
         else:
             try:
-                stdout, stderr = subp(['lxc', 'console', '--help'],
-                                      decode=False)
+                stdout = subp(['lxc', 'console', '--help'], decode=False)[0]
                 if not (b'console' in stdout and b'log' in stdout):
                     reason = "no '--log' in lxc console --help"
-            except ProcessExecutionError as e:
+            except ProcessExecutionError:
                 reason = "no 'console' command in lxc client"
 
     if reason:
diff --git a/tests/cloud_tests/setup_image.py b/tests/cloud_tests/setup_image.py
index 6d242115..4e195709 100644
--- a/tests/cloud_tests/setup_image.py
+++ b/tests/cloud_tests/setup_image.py
@@ -25,10 +25,9 @@ def installed_package_version(image, package, ensure_installed=True):
     else:
         raise NotImplementedError
 
-    msg = 'query version for package: {}'.format(package)
-    (out, err, exit) = image.execute(
-        cmd, description=msg, rcs=(0,) if ensure_installed else range(0, 256))
-    return out.strip()
+    return image.execute(
+        cmd, description='query version for package: {}'.format(package),
+        rcs=(0,) if ensure_installed else range(0, 256))[0].strip()
 
 
 def install_deb(args, image):
@@ -54,7 +53,7 @@ def install_deb(args, image):
          remote_path], description=msg)
     # check installed deb version matches package
     fmt = ['-W', "--showformat=${Version}"]
-    (out, err, exit) = image.execute(['dpkg-deb'] + fmt + [remote_path])
+    out = image.execute(['dpkg-deb'] + fmt + [remote_path])[0]
     expected_version = out.strip()
     found_version = installed_package_version(image, 'cloud-init')
     if expected_version != found_version:
@@ -85,7 +84,7 @@ def install_rpm(args, image):
     image.execute(['rpm', '-U', remote_path], description=msg)
 
     fmt = ['--queryformat', '"%{VERSION}"']
-    (out, err, exit) = image.execute(['rpm', '-q'] + fmt + [remote_path])
+    (out, _err, _exit) = image.execute(['rpm', '-q'] + fmt + [remote_path])
     expected_version = out.strip()
     found_version = installed_package_version(image, 'cloud-init')
     if expected_version != found_version:
diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py
index 4fda8f91..0d1916b4 100644
--- a/tests/cloud_tests/testcases/base.py
+++ b/tests/cloud_tests/testcases/base.py
@@ -159,7 +159,7 @@ class CloudTestCase(unittest.TestCase):
         expected_net_keys = [
             'public-ipv4s', 'ipv4-associations', 'local-hostname',
             'public-hostname']
-        for mac, mac_data in macs.items():
+        for mac_data in macs.values():
             for key in expected_net_keys:
                 self.assertIn(key, mac_data)
         self.assertIsNotNone(
diff --git a/tests/cloud_tests/testcases/examples/including_user_groups.py b/tests/cloud_tests/testcases/examples/including_user_groups.py
index 93b7a82d..4067348d 100644
--- a/tests/cloud_tests/testcases/examples/including_user_groups.py
+++ b/tests/cloud_tests/testcases/examples/including_user_groups.py
@@ -42,7 +42,7 @@ class TestUserGroups(base.CloudTestCase):
 
     def test_user_root_in_secret(self):
         """Test root user is in 'secret' group."""
-        user, _, groups = self.get_data_file('root_groups').partition(":")
+        _user, _, groups = self.get_data_file('root_groups').partition(":")
         self.assertIn("secret", groups.split(),
                       msg="User root is not in group 'secret'")
 
diff --git a/tests/cloud_tests/testcases/modules/user_groups.py b/tests/cloud_tests/testcases/modules/user_groups.py
index 93b7a82d..4067348d 100644
--- a/tests/cloud_tests/testcases/modules/user_groups.py
+++ b/tests/cloud_tests/testcases/modules/user_groups.py
@@ -42,7 +42,7 @@ class TestUserGroups(base.CloudTestCase):
 
     def test_user_root_in_secret(self):
         """Test root user is in 'secret' group."""
-        user, _, groups = self.get_data_file('root_groups').partition(":")
+        _user, _, groups = self.get_data_file('root_groups').partition(":")
         self.assertIn("secret", groups.split(),
                       msg="User root is not in group 'secret'")
 
diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py
index 3dd4996d..06f7d865 100644
--- a/tests/cloud_tests/util.py
+++ b/tests/cloud_tests/util.py
@@ -358,7 +358,7 @@ class TargetBase(object):
         # when sh is invoked with '-c', then the first argument is "$0"
         # which is commonly understood as the "program name".
         # 'read_data' is the program name, and 'remote_path' is '$1'
-        stdout, stderr, rc = self._execute(
+        stdout, _stderr, rc = self._execute(
             ["sh", "-c", 'exec cat "$1"', 'read_data', remote_path])
         if rc != 0:
             raise RuntimeError("Failed to read file '%s'" % remote_path)
diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py
index 25878d7a..f1ab02e9 100644
--- a/tests/unittests/test__init__.py
+++ b/tests/unittests/test__init__.py
@@ -214,7 +214,7 @@ class TestCmdlineUrl(CiTestCase):
     def test_no_key_found(self, m_read):
         cmdline = "ro mykey=http://example.com/foo root=foo"
         fpath = self.tmp_path("ccpath")
-        lvl, msg = main.attempt_cmdline_url(
+        lvl, _msg = main.attempt_cmdline_url(
             fpath, network=True, cmdline=cmdline)
 
         m_read.assert_not_called()
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 3e8b7913..88fe76c7 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -214,7 +214,7 @@ scbus-1 on xpt0 bus 0
                 self.assertIn(tag, x)
 
         def tags_equal(x, y):
-            for x_tag, x_val in x.items():
+            for x_val in x.values():
                 y_val = y.get(x_val.tag)
                 self.assertEqual(x_val.text, y_val.text)
 
@@ -1216,7 +1216,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase):
         fake_resp.return_value = mock.MagicMock(status_code=200, text=content,
                                                 content=content)
         dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
-        md, ud, cfg, d = dsa._reprovision()
+        md, _ud, cfg, _d = dsa._reprovision()
         self.assertEqual(md['local-hostname'], hostname)
         self.assertEqual(cfg['system_info']['default_user']['name'], username)
         self.assertEqual(fake_resp.call_args_list,
diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py
index 6e4031cf..c84d067e 100644
--- a/tests/unittests/test_datasource/test_maas.py
+++ b/tests/unittests/test_datasource/test_maas.py
@@ -53,7 +53,7 @@ class TestMAASDataSource(CiTestCase):
         my_d = os.path.join(self.tmp, "valid_extra")
         populate_dir(my_d, data)
 
-        ud, md, vd = DataSourceMAAS.read_maas_seed_dir(my_d)
+        ud, md, _vd = DataSourceMAAS.read_maas_seed_dir(my_d)
 
         self.assertEqual(userdata, ud)
         for key in ('instance-id', 'local-hostname'):
@@ -149,7 +149,7 @@ class TestMAASDataSource(CiTestCase):
             'meta-data/local-hostname': 'test-hostname',
             'meta-data/vendor-data': yaml.safe_dump(expected_vd).encode(),
         }
-        ud, md, vd = self.mock_read_maas_seed_url(
+        _ud, md, vd = self.mock_read_maas_seed_url(
             valid, "http://example.com/foo")
 
         self.assertEqual(valid['meta-data/instance-id'], md['instance-id'])
diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py
index 70d50de4..cdbd1e1a 100644
--- a/tests/unittests/test_datasource/test_nocloud.py
+++ b/tests/unittests/test_datasource/test_nocloud.py
@@ -51,9 +51,6 @@ class TestNoCloudDataSource(CiTestCase):
         class PsuedoException(Exception):
             pass
 
-        def my_find_devs_with(*args, **kwargs):
-            raise PsuedoException
-
         self.mocks.enter_context(
             mock.patch.object(util, 'find_devs_with',
                               side_effect=PsuedoException))
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py
index 7bb1b7c4..e486862d 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v3.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py
@@ -528,7 +528,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
 
         expected = sorted([npre + suff for opre, npre, suff in files])
         # create files
-        for (opre, npre, suff) in files:
+        for (opre, _npre, suff) in files:
             fpath = os.path.join(apt_lists_d, opre + suff)
             util.write_file(fpath, content=fpath)
 
diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
index 02676aa6..17c53559 100644
--- a/tests/unittests/test_handler/test_handler_ntp.py
+++ b/tests/unittests/test_handler/test_handler_ntp.py
@@ -76,7 +76,7 @@ class TestNtp(FilesystemMockingTestCase):
             template = TIMESYNCD_TEMPLATE
         else:
             template = NTP_TEMPLATE
-        (confpath, template_fn) = self._generate_template(template=template)
+        (confpath, _template_fn) = self._generate_template(template=template)
         ntpconfig = copy.deepcopy(dcfg[client])
         ntpconfig['confpath'] = confpath
         ntpconfig['template_name'] = os.path.basename(confpath)
diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py
index 1080e135..20c87efa 100644
--- a/tests/unittests/test_templating.py
+++ b/tests/unittests/test_templating.py
@@ -50,12 +50,12 @@ class TestTemplates(test_helpers.CiTestCase):
     def test_detection(self):
         blob = "## template:cheetah"
 
-        (template_type, renderer, contents) = templater.detect_template(blob)
+        (template_type, _renderer, contents) = templater.detect_template(blob)
         self.assertIn("cheetah", template_type)
         self.assertEqual("", contents.strip())
 
         blob = "blahblah $blah"
-        (template_type, renderer, contents) = templater.detect_template(blob)
+        (template_type, _renderer, _contents) = templater.detect_template(blob)
         self.assertIn("cheetah", template_type)
         self.assertEqual(blob, contents)
 
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index e04ea031..84941c7d 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -774,11 +774,11 @@ class TestSubp(helpers.CiTestCase):
 
     def test_subp_reads_env(self):
         with mock.patch.dict("os.environ", values={'FOO': 'BAR'}):
-            out, err = util.subp(self.printenv + ['FOO'], capture=True)
+            out, _err = util.subp(self.printenv + ['FOO'], capture=True)
         self.assertEqual('FOO=BAR', out.splitlines()[0])
 
     def test_subp_env_and_update_env(self):
-        out, err = util.subp(
+        out, _err = util.subp(
             self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True,
             env={'FOO': 'BAR'},
             update_env={'HOME': '/myhome', 'K2': 'V2'})
@@ -788,7 +788,7 @@ class TestSubp(helpers.CiTestCase):
     def test_subp_update_env(self):
         extra = {'FOO': 'BAR', 'HOME': '/root', 'K1': 'V1'}
         with mock.patch.dict("os.environ", values=extra):
-            out, err = util.subp(
+            out, _err = util.subp(
                 self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True,
                 update_env={'HOME': '/myhome', 'K2': 'V2'})
 
-- 
cgit v1.2.3


From 8e111502a0f9d437ff6045f1269c95e5756fc43b Mon Sep 17 00:00:00 2001
From: Mike Gerdts <mike.gerdts@joyent.com>
Date: Fri, 20 Apr 2018 20:30:44 -0400
Subject: DataSourceSmartOS: list() should always return a list

If customer_metadata has no keys, the KEYS request returns an empty
string. Callers of the list() method expect a list to be returned and
will give a stack trace if this expectation is not met.

LP: #1763480
---
 cloudinit/sources/DataSourceSmartOS.py          |  6 +++---
 tests/unittests/test_datasource/test_smartos.py | 26 +++++++++++++++++++++++--
 2 files changed, 27 insertions(+), 5 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index c8998b40..d4386fec 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -455,9 +455,9 @@ class JoyentMetadataClient(object):
 
     def list(self):
         result = self.request(rtype='KEYS')
-        if result:
-            result = result.split('\n')
-        return result
+        if not result:
+            return []
+        return result.split('\n')
 
     def put(self, key, val):
         param = b' '.join([base64.b64encode(i.encode())
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 2bea7a17..6fd431f9 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -319,6 +319,12 @@ MOCK_RETURNS = {
 
 DMI_DATA_RETURN = 'smartdc'
 
+# Useful for calculating the length of a frame body.  A SUCCESS body will be
+# followed by more characters or be one character less if SUCCESS with no
+# payload.  See Section 4.3 of https://eng.joyent.com/mdata/protocol.html.
+SUCCESS_LEN = len('0123abcd SUCCESS ')
+NOTFOUND_LEN = len('0123abcd NOTFOUND')
+
 
 class PsuedoJoyentClient(object):
     def __init__(self, data=None):
@@ -651,7 +657,7 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase):
         self.response_parts = {
             'command': 'SUCCESS',
             'crc': 'b5a9ff00',
-            'length': 17 + len(b64e(self.metadata_value)),
+            'length': SUCCESS_LEN + len(b64e(self.metadata_value)),
             'payload': b64e(self.metadata_value),
             'request_id': '{0:08x}'.format(self.request_id),
         }
@@ -787,7 +793,7 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase):
     def test_get_metadata_returns_None_if_value_not_found(self):
         self.response_parts['payload'] = ''
         self.response_parts['command'] = 'NOTFOUND'
-        self.response_parts['length'] = 17
+        self.response_parts['length'] = NOTFOUND_LEN
         client = self._get_client()
         client._checksum = lambda _: self.response_parts['crc']
         self.assertIsNone(client.get('some_key'))
@@ -838,6 +844,22 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase):
         client.open_transport()
         self.assertTrue(reader.emptied)
 
+    def test_list_metadata_returns_list(self):
+        parts = ['foo', 'bar']
+        value = b64e('\n'.join(parts))
+        self.response_parts['payload'] = value
+        self.response_parts['crc'] = '40873553'
+        self.response_parts['length'] = SUCCESS_LEN + len(value)
+        client = self._get_client()
+        self.assertEqual(client.list(), parts)
+
+    def test_list_metadata_returns_empty_list_if_no_customer_metadata(self):
+        del self.response_parts['payload']
+        self.response_parts['length'] = SUCCESS_LEN - 1
+        self.response_parts['crc'] = '14e563ba'
+        client = self._get_client()
+        self.assertEqual(client.list(), [])
+
 
 class TestNetworkConversion(TestCase):
     def test_convert_simple(self):
-- 
cgit v1.2.3


From 23479881f51bae7a3f5743ce677ed82317ea8b9f Mon Sep 17 00:00:00 2001
From: Mike Gerdts <mike.gerdts@joyent.com>
Date: Fri, 20 Apr 2018 20:56:36 -0400
Subject: DataSourceSmartOS: sdc:hostname is ignored

There are three potential sources of the hostname, one of which is
documented SmartOS's vmadm(1M) via the hostname property.  That
property's value is retrieved via the sdc:hostname key.  The other
two sources for the hostname are a hostname key in customer_metadata
and the VM's uuid (sdc:uuid).  Of these three, the sdc:hostname value
is not used in a meaningful way by DataSourceSmartOS.

This fix changes the fallback mechanism when hostname is not
specified in customer_metadata.  The order of precedence for setting
the hostname is now 1) hostname in customer_metadata,
2) sdc:hostname, then 3) sdc:uuid.

LP: #1765085
---
 cloudinit/sources/DataSourceSmartOS.py          | 10 +++++++--
 tests/unittests/test_datasource/test_smartos.py | 28 +++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 2 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index d4386fec..0ef10035 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -11,7 +11,7 @@
 #    SmartOS hosts use a serial console (/dev/ttyS1) on KVM Linux Guests
 #        The meta-data is transmitted via key/value pairs made by
 #        requests on the console. For example, to get the hostname, you
-#        would send "GET hostname" on /dev/ttyS1.
+#        would send "GET sdc:hostname" on /dev/ttyS1.
 #        For Linux Guests running in LX-Brand Zones on SmartOS hosts
 #        a socket (/native/.zonecontrol/metadata.sock) is used instead
 #        of a serial console.
@@ -273,8 +273,14 @@ class DataSourceSmartOS(sources.DataSource):
         write_boot_content(u_data, u_data_f)
 
         # Handle the cloud-init regular meta
+
+        # The hostname may or may not be qualified with the local domain name.
+        # This follows section 3.14 of RFC 2132.
         if not md['local-hostname']:
-            md['local-hostname'] = md['instance-id']
+            if md['hostname']:
+                md['local-hostname'] = md['hostname']
+            else:
+                md['local-hostname'] = md['instance-id']
 
         ud = None
         if md['user-data']:
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 6fd431f9..b926263f 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -437,6 +437,34 @@ class TestSmartOSDataSource(FilesystemMockingTestCase):
         self.assertEqual(MOCK_RETURNS['hostname'],
                          dsrc.metadata['local-hostname'])
 
+    def test_hostname_if_no_sdc_hostname(self):
+        my_returns = MOCK_RETURNS.copy()
+        my_returns['sdc:hostname'] = 'sdc-' + my_returns['hostname']
+        dsrc = self._get_ds(mockdata=my_returns)
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEqual(my_returns['hostname'],
+                         dsrc.metadata['local-hostname'])
+
+    def test_sdc_hostname_if_no_hostname(self):
+        my_returns = MOCK_RETURNS.copy()
+        my_returns['sdc:hostname'] = 'sdc-' + my_returns['hostname']
+        del my_returns['hostname']
+        dsrc = self._get_ds(mockdata=my_returns)
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEqual(my_returns['sdc:hostname'],
+                         dsrc.metadata['local-hostname'])
+
+    def test_sdc_uuid_if_no_hostname_or_sdc_hostname(self):
+        my_returns = MOCK_RETURNS.copy()
+        del my_returns['hostname']
+        dsrc = self._get_ds(mockdata=my_returns)
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEqual(my_returns['sdc:uuid'],
+                         dsrc.metadata['local-hostname'])
+
     def test_userdata(self):
         dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
-- 
cgit v1.2.3


From 4ed164592fe8cb15758cacf3cb3f8c7d5ab7c82e Mon Sep 17 00:00:00 2001
From: Mike Gerdts <mike.gerdts@joyent.com>
Date: Mon, 23 Apr 2018 09:29:39 -0400
Subject: DataSourceSmartOS: add locking of serial device.

cloud-init and mdata-get each have their own implementation of the
SmartOS metadata protocol.  If cloud-init and other services that call
mdata-get are run concurrently, crosstalk on the serial port can cause
them both to become confused.

This change makes it so that cloud-init uses the same cooperative
locking scheme that's used by mdata-get, thus preventing cross-talk
between mdata-get and cloud-init.

For testing, a VM running on a SmartOS host and pyserial are required.
If the tests are run on a platform other than SmartOS, those that use a
real serial port are skipped.  pyserial remains commented in
requirements.txt because most testers will not be running atop SmartOS.

LP: #1746605
---
 cloudinit/sources/DataSourceSmartOS.py          |  2 +
 tests/unittests/test_datasource/test_smartos.py | 67 ++++++++++++++++++++++++-
 2 files changed, 67 insertions(+), 2 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 0ef10035..4ea00eb1 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -23,6 +23,7 @@
 import base64
 import binascii
 import errno
+import fcntl
 import json
 import os
 import random
@@ -526,6 +527,7 @@ class JoyentMetadataSerialClient(JoyentMetadataClient):
             if not ser.isOpen():
                 raise SystemError("Unable to open %s" % self.device)
             self.fp = ser
+            fcntl.lockf(ser, fcntl.LOCK_EX)
         self._flush()
         self._negotiate()
 
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index b926263f..706e8eb8 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -16,23 +16,27 @@ from __future__ import print_function
 
 from binascii import crc32
 import json
+import multiprocessing
 import os
 import os.path
 import re
 import shutil
+import signal
 import stat
 import tempfile
+import unittest2
 import uuid
 
 from cloudinit import serial
 from cloudinit.sources import DataSourceSmartOS
 from cloudinit.sources.DataSourceSmartOS import (
-    convert_smartos_network_data as convert_net)
+    convert_smartos_network_data as convert_net,
+    SMARTOS_ENV_KVM, SERIAL_DEVICE, get_smartos_environ)
 
 import six
 
 from cloudinit import helpers as c_helpers
-from cloudinit.util import b64e
+from cloudinit.util import (b64e, subp)
 
 from cloudinit.tests.helpers import mock, FilesystemMockingTestCase, TestCase
 
@@ -1023,4 +1027,63 @@ class TestNetworkConversion(TestCase):
         found = convert_net(SDC_NICS_SINGLE_GATEWAY)
         self.assertEqual(expected, found)
 
+
+@unittest2.skipUnless(get_smartos_environ() == SMARTOS_ENV_KVM,
+                      "Only supported on KVM and bhyve guests under SmartOS")
+@unittest2.skipUnless(os.access(SERIAL_DEVICE, os.W_OK),
+                      "Requires write access to " + SERIAL_DEVICE)
+class TestSerialConcurrency(TestCase):
+    """
+       This class tests locking on an actual serial port, and as such can only
+       be run in a kvm or bhyve guest running on a SmartOS host.  A test run on
+       a metadata socket will not be valid because a metadata socket ensures
+       there is only one session over a connection.  In contrast, in the
+       absence of proper locking multiple processes opening the same serial
+       port can corrupt each others' exchanges with the metadata server.
+    """
+    def setUp(self):
+        self.mdata_proc = multiprocessing.Process(target=self.start_mdata_loop)
+        self.mdata_proc.start()
+        super(TestSerialConcurrency, self).setUp()
+
+    def tearDown(self):
+        # os.kill() rather than mdata_proc.terminate() to avoid console spam.
+        os.kill(self.mdata_proc.pid, signal.SIGKILL)
+        self.mdata_proc.join()
+        super(TestSerialConcurrency, self).tearDown()
+
+    def start_mdata_loop(self):
+        """
+           The mdata-get command is repeatedly run in a separate process so
+           that it may try to race with metadata operations performed in the
+           main test process.  Use of mdata-get is better than two processes
+           using the protocol implementation in DataSourceSmartOS because we
+           are testing to be sure that cloud-init and mdata-get respect each
+           others locks.
+        """
+        rcs = list(range(0, 256))
+        while True:
+            subp(['mdata-get', 'sdc:routes'], rcs=rcs)
+
+    def test_all_keys(self):
+        self.assertIsNotNone(self.mdata_proc.pid)
+        ds = DataSourceSmartOS
+        keys = [tup[0] for tup in ds.SMARTOS_ATTRIB_MAP.values()]
+        keys.extend(ds.SMARTOS_ATTRIB_JSON.values())
+
+        client = ds.jmc_client_factory()
+        self.assertIsNotNone(client)
+
+        # The behavior that we are testing for was observed mdata-get running
+        # 10 times at roughly the same time as cloud-init fetched each key
+        # once.  cloud-init would regularly see failures before making it
+        # through all keys once.
+        for _ in range(0, 3):
+            for key in keys:
+                # We don't care about the return value, just that it doesn't
+                # thrown any exceptions.
+                client.get(key)
+
+        self.assertIsNone(self.mdata_proc.exitcode)
+
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From b73559e2f98025e08fdb42544bb1d2e0f92a7a8d Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 25 Apr 2018 16:23:36 -0400
Subject: IBMCloud: improve documentation in datasource.

This adds information to the IBMCloud datasource describing the
6 different scenarios that it is expected to handle.
---
 cloudinit/sources/DataSourceIBMCloud.py | 64 ++++++++++++++++++++++++++++++++-
 1 file changed, 63 insertions(+), 1 deletion(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py
index 02b3d56f..cfa724bf 100644
--- a/cloudinit/sources/DataSourceIBMCloud.py
+++ b/cloudinit/sources/DataSourceIBMCloud.py
@@ -30,11 +30,73 @@ There are 2 different api exposed launch methods.
    mean that 1 in 8^16 (~4 billion) Xen ConfigDrive systems will be
    incorrectly identified as IBMCloud.
 
+The combination of these 2 launch methods and with or without user-data
+creates 6 boot scenarios.
+ A. os_code with user-data
+ B. os_code without user-data
+    Cloud-init is fully operational in this mode.
+
+    There is a block device attached with label 'config-2'.
+    As it differs from OpenStack's config-2, we have to differentiate.
+    We do so by requiring the UUID on the filesystem to be "9796-932E".
+
+    This disk will have the following files. Specifically note, there
+    is no versioned path to the meta-data, only 'latest':
+      openstack/latest/meta_data.json
+      openstack/latest/network_data.json
+      openstack/latest/user_data [optional]
+      openstack/latest/vendor_data.json
+
+    vendor_data.json as of 2018-04 looks like this:
+      {"cloud-init":"#!/bin/bash\necho 'root:$6$<snip>' | chpasswd -e"}
+
+    The only difference between A and B in this mode is the presence
+    of user_data on the config disk.
+
+ C. template, provisioning boot with user-data
+ D. template, provisioning boot without user-data.
+    With ds-identify cloud-init is fully disabled in this mode.
+    Without ds-identify, cloud-init None datasource will be used.
+
+    This is currently identified by the presence of
+    /root/provisioningConfiguration.cfg . That file is placed into the
+    system before it is booted.
+
+    The difference between C and D is the presence of the METADATA disk
+    as described in E below.  There is no METADATA disk attached unless
+    user-data is provided.
+
+ E. template, post-provisioning boot with user-data.
+    Cloud-init is fully operational in this mode.
+
+    This is identified by a block device with filesystem label "METADATA".
+    The looks similar to a version-1 OpenStack config drive.  It will
+    have the following files:
+
+       openstack/latest/user_data
+       openstack/latest/meta_data.json
+       openstack/content/interfaces
+       meta.js
+
+    meta.js contains something similar to user_data.  cloud-init ignores it.
+    cloud-init ignores the 'interfaces' style file here.
+    In this mode, cloud-init has networking code disabled.  It relies
+    on the provisioning boot to have configured networking.
+
+ F. template, post-provisioning boot without user-data.
+    With ds-identify, cloud-init will be fully disabled.
+    Without ds-identify, cloud-init None datasource will be used.
+
+    There is no information available to identify this scenario.
+
+    The user will be able to ssh in as as root with their public keys that
+    have been installed into /root/ssh/.authorized_keys
+    during the provisioning stage.
+
 TODO:
  * is uuid (/sys/hypervisor/uuid) stable for life of an instance?
    it seems it is not the same as data's uuid in the os_code case
    but is in the template case.
-
 """
 import base64
 import json
-- 
cgit v1.2.3


From 4731c8da25ee9bfbcf0ade1d7ffec95814d8622a Mon Sep 17 00:00:00 2001
From: Ryan Harper <ryan.harper@canonical.com>
Date: Thu, 26 Apr 2018 16:35:23 -0400
Subject: net: detect unstable network names and trigger a settle if needed

The cloud-init-local.service expects that any network device name changes
have already been completed by the kernel or udev daemon.

In some situations we've found that the renaming of interfaces from kernel
names (eth0, eth1, etc) to their persistent names (eno1, ens3, enp0s1,
etc) may happen after cloud-init-local has started where it reads values
from sysfs about what network devices are present, and which device to use
as a fallback nic.

Subsequently, cloud-init-local would write out network configuration for a
kernel device name which would no longer be present by the time that
networking services start to bring up the devices. The result is that the
instance does not get networking configured. Prior to use of
systemd-networkd, the Ubuntu 'networking.service' unit included a call to
udevadm settle which is why this race is not seen on a Xenial system.

This change adds the ability to detect if an interface has a stable name,
if if we find one without stable names and stable names have not been
disabled (net.ifnames=0 in /proc/cmdline), then cloud-init will invoke
udevadm settle.

LP: #1766287
---
 cloudinit/config/cc_disk_setup.py       | 12 ++----
 cloudinit/net/__init__.py               | 26 ++++++++++++
 cloudinit/net/tests/test_init.py        |  1 +
 cloudinit/sources/DataSourceAltCloud.py |  5 +--
 cloudinit/tests/test_util.py            | 49 ++++++++++++++++++++++
 cloudinit/util.py                       | 15 +++++++
 tests/unittests/test_net.py             | 73 +++++++++++++++++++++++++++++++--
 7 files changed, 165 insertions(+), 16 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py
index c3e8c484..943089e0 100644
--- a/cloudinit/config/cc_disk_setup.py
+++ b/cloudinit/config/cc_disk_setup.py
@@ -680,13 +680,13 @@ def read_parttbl(device):
     reliable way to probe the partition table.
     """
     blkdev_cmd = [BLKDEV_CMD, '--rereadpt', device]
-    udevadm_settle()
+    util.udevadm_settle()
     try:
         util.subp(blkdev_cmd)
     except Exception as e:
         util.logexc(LOG, "Failed reading the partition table %s" % e)
 
-    udevadm_settle()
+    util.udevadm_settle()
 
 
 def exec_mkpart_mbr(device, layout):
@@ -737,14 +737,10 @@ def exec_mkpart(table_type, device, layout):
     return get_dyn_func("exec_mkpart_%s", table_type, device, layout)
 
 
-def udevadm_settle():
-    util.subp(['udevadm', 'settle'])
-
-
 def assert_and_settle_device(device):
     """Assert that device exists and settle so it is fully recognized."""
     if not os.path.exists(device):
-        udevadm_settle()
+        util.udevadm_settle()
         if not os.path.exists(device):
             raise RuntimeError("Device %s did not exist and was not created "
                                "with a udevamd settle." % device)
@@ -752,7 +748,7 @@ def assert_and_settle_device(device):
     # Whether or not the device existed above, it is possible that udev
     # events that would populate udev database (for reading by lsdname) have
     # not yet finished. So settle again.
-    udevadm_settle()
+    util.udevadm_settle()
 
 
 def mkpart(device, definition):
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 80054546..43226bd0 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -107,6 +107,21 @@ def is_bond(devname):
     return os.path.exists(sys_dev_path(devname, "bonding"))
 
 
+def is_renamed(devname):
+    """
+    /* interface name assignment types (sysfs name_assign_type attribute) */
+    #define NET_NAME_UNKNOWN	0	/* unknown origin (not exposed to user) */
+    #define NET_NAME_ENUM		1	/* enumerated by kernel */
+    #define NET_NAME_PREDICTABLE	2	/* predictably named by the kernel */
+    #define NET_NAME_USER		3	/* provided by user-space */
+    #define NET_NAME_RENAMED	4	/* renamed by user-space */
+    """
+    name_assign_type = read_sys_net_safe(devname, 'name_assign_type')
+    if name_assign_type and name_assign_type in ['3', '4']:
+        return True
+    return False
+
+
 def is_vlan(devname):
     uevent = str(read_sys_net_safe(devname, "uevent"))
     return 'DEVTYPE=vlan' in uevent.splitlines()
@@ -180,6 +195,17 @@ def find_fallback_nic(blacklist_drivers=None):
     if not blacklist_drivers:
         blacklist_drivers = []
 
+    if 'net.ifnames=0' in util.get_cmdline():
+        LOG.debug('Stable ifnames disabled by net.ifnames=0 in /proc/cmdline')
+    else:
+        unstable = [device for device in get_devicelist()
+                    if device != 'lo' and not is_renamed(device)]
+        if len(unstable):
+            LOG.debug('Found unstable nic names: %s; calling udevadm settle',
+                      unstable)
+            msg = 'Waiting for udev events to settle'
+            util.log_time(LOG.debug, msg, func=util.udevadm_settle)
+
     # get list of interfaces that could have connections
     invalid_interfaces = set(['lo'])
     potential_interfaces = set([device for device in get_devicelist()
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
index 276556ee..5c017d15 100644
--- a/cloudinit/net/tests/test_init.py
+++ b/cloudinit/net/tests/test_init.py
@@ -199,6 +199,7 @@ class TestGenerateFallbackConfig(CiTestCase):
         self.sysdir = self.tmp_dir() + '/'
         self.m_sys_path.return_value = self.sysdir
         self.addCleanup(sys_mock.stop)
+        self.add_patch('cloudinit.net.util.udevadm_settle', 'm_settle')
 
     def test_generate_fallback_finds_connected_eth_with_mac(self):
         """generate_fallback_config finds any connected device with a mac."""
diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index e1d0055b..f6e86f34 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -29,7 +29,6 @@ CLOUD_INFO_FILE = '/etc/sysconfig/cloud-info'
 
 # Shell command lists
 CMD_PROBE_FLOPPY = ['modprobe', 'floppy']
-CMD_UDEVADM_SETTLE = ['udevadm', 'settle', '--timeout=5']
 
 META_DATA_NOT_SUPPORTED = {
     'block-device-mapping': {},
@@ -196,9 +195,7 @@ class DataSourceAltCloud(sources.DataSource):
 
         # udevadm settle for floppy device
         try:
-            cmd = CMD_UDEVADM_SETTLE
-            cmd.append('--exit-if-exists=' + floppy_dev)
-            (cmd_out, _err) = util.subp(cmd)
+            (cmd_out, _err) = util.udevadm_settle(exists=floppy_dev, timeout=5)
             LOG.debug('Command: %s\nOutput%s', ' '.join(cmd), cmd_out)
         except ProcessExecutionError as _err:
             util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err)
diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py
index 76eed076..3c05a437 100644
--- a/cloudinit/tests/test_util.py
+++ b/cloudinit/tests/test_util.py
@@ -212,4 +212,53 @@ class TestBlkid(CiTestCase):
                                   capture=True, decode="replace")
 
 
+@mock.patch('cloudinit.util.subp')
+class TestUdevadmSettle(CiTestCase):
+    def test_with_no_params(self, m_subp):
+        """called with no parameters."""
+        util.udevadm_settle()
+        m_subp.called_once_with(mock.call(['udevadm', 'settle']))
+
+    def test_with_exists_and_not_exists(self, m_subp):
+        """with exists=file where file does not exist should invoke subp."""
+        mydev = self.tmp_path("mydev")
+        util.udevadm_settle(exists=mydev)
+        m_subp.called_once_with(
+            ['udevadm', 'settle', '--exit-if-exists=%s' % mydev])
+
+    def test_with_exists_and_file_exists(self, m_subp):
+        """with exists=file where file does exist should not invoke subp."""
+        mydev = self.tmp_path("mydev")
+        util.write_file(mydev, "foo\n")
+        util.udevadm_settle(exists=mydev)
+        self.assertIsNone(m_subp.call_args)
+
+    def test_with_timeout_int(self, m_subp):
+        """timeout can be an integer."""
+        timeout = 9
+        util.udevadm_settle(timeout=timeout)
+        m_subp.called_once_with(
+            ['udevadm', 'settle', '--timeout=%s' % timeout])
+
+    def test_with_timeout_string(self, m_subp):
+        """timeout can be a string."""
+        timeout = "555"
+        util.udevadm_settle(timeout=timeout)
+        m_subp.assert_called_once_with(
+            ['udevadm', 'settle', '--timeout=%s' % timeout])
+
+    def test_with_exists_and_timeout(self, m_subp):
+        """test call with both exists and timeout."""
+        mydev = self.tmp_path("mydev")
+        timeout = "3"
+        util.udevadm_settle(exists=mydev)
+        m_subp.called_once_with(
+            ['udevadm', 'settle', '--exit-if-exists=%s' % mydev,
+             '--timeout=%s' % timeout])
+
+    def test_subp_exception_raises_to_caller(self, m_subp):
+        m_subp.side_effect = util.ProcessExecutionError("BOOM")
+        self.assertRaises(util.ProcessExecutionError, util.udevadm_settle)
+
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 310758dd..2828ca38 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -2727,4 +2727,19 @@ def mount_is_read_write(mount_point):
     mount_opts = result[-1].split(',')
     return mount_opts[0] == 'rw'
 
+
+def udevadm_settle(exists=None, timeout=None):
+    """Invoke udevadm settle with optional exists and timeout parameters"""
+    settle_cmd = ["udevadm", "settle"]
+    if exists:
+        # skip the settle if the requested path already exists
+        if os.path.exists(exists):
+            return
+        settle_cmd.extend(['--exit-if-exists=%s' % exists])
+    if timeout:
+        settle_cmd.extend(['--timeout=%s' % timeout])
+
+    return subp(settle_cmd)
+
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index 84839fd7..fac82678 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -1442,6 +1442,7 @@ DEFAULT_DEV_ATTRS = {
         "address": "07-1C-C6-75-A4-BE",
         "device/driver": None,
         "device/device": None,
+        "name_assign_type": "4",
     }
 }
 
@@ -1489,11 +1490,14 @@ class TestGenerateFallbackConfig(CiTestCase):
             'eth0': {
                 'bridge': False, 'carrier': False, 'dormant': False,
                 'operstate': 'down', 'address': '00:11:22:33:44:55',
-                'device/driver': 'hv_netsvc', 'device/device': '0x3'},
+                'device/driver': 'hv_netsvc', 'device/device': '0x3',
+                'name_assign_type': '4'},
             'eth1': {
                 'bridge': False, 'carrier': False, 'dormant': False,
                 'operstate': 'down', 'address': '00:11:22:33:44:55',
-                'device/driver': 'mlx4_core', 'device/device': '0x7'},
+                'device/driver': 'mlx4_core', 'device/device': '0x7',
+                'name_assign_type': '4'},
+
         }
 
         tmp_dir = self.tmp_dir()
@@ -1549,11 +1553,13 @@ iface eth0 inet dhcp
             'eth1': {
                 'bridge': False, 'carrier': False, 'dormant': False,
                 'operstate': 'down', 'address': '00:11:22:33:44:55',
-                'device/driver': 'hv_netsvc', 'device/device': '0x3'},
+                'device/driver': 'hv_netsvc', 'device/device': '0x3',
+                'name_assign_type': '4'},
             'eth0': {
                 'bridge': False, 'carrier': False, 'dormant': False,
                 'operstate': 'down', 'address': '00:11:22:33:44:55',
-                'device/driver': 'mlx4_core', 'device/device': '0x7'},
+                'device/driver': 'mlx4_core', 'device/device': '0x7',
+                'name_assign_type': '4'},
         }
 
         tmp_dir = self.tmp_dir()
@@ -1602,6 +1608,65 @@ iface eth1 inet dhcp
         ]
         self.assertEqual(", ".join(expected_rule) + '\n', contents.lstrip())
 
+    @mock.patch("cloudinit.util.udevadm_settle")
+    @mock.patch("cloudinit.net.sys_dev_path")
+    @mock.patch("cloudinit.net.read_sys_net")
+    @mock.patch("cloudinit.net.get_devicelist")
+    def test_unstable_names(self, mock_get_devicelist, mock_read_sys_net,
+                            mock_sys_dev_path, mock_settle):
+        """verify that udevadm settle is called when we find unstable names"""
+        devices = {
+            'eth0': {
+                'bridge': False, 'carrier': False, 'dormant': False,
+                'operstate': 'down', 'address': '00:11:22:33:44:55',
+                'device/driver': 'hv_netsvc', 'device/device': '0x3',
+                'name_assign_type': False},
+            'ens4': {
+                'bridge': False, 'carrier': False, 'dormant': False,
+                'operstate': 'down', 'address': '00:11:22:33:44:55',
+                'device/driver': 'mlx4_core', 'device/device': '0x7',
+                'name_assign_type': '4'},
+
+        }
+
+        tmp_dir = self.tmp_dir()
+        _setup_test(tmp_dir, mock_get_devicelist,
+                    mock_read_sys_net, mock_sys_dev_path,
+                    dev_attrs=devices)
+        net.generate_fallback_config(config_driver=True)
+        self.assertEqual(1, mock_settle.call_count)
+
+    @mock.patch("cloudinit.util.get_cmdline")
+    @mock.patch("cloudinit.util.udevadm_settle")
+    @mock.patch("cloudinit.net.sys_dev_path")
+    @mock.patch("cloudinit.net.read_sys_net")
+    @mock.patch("cloudinit.net.get_devicelist")
+    def test_unstable_names_disabled(self, mock_get_devicelist,
+                                     mock_read_sys_net, mock_sys_dev_path,
+                                     mock_settle, m_get_cmdline):
+        """verify udevadm settle not called when cmdline has net.ifnames=0"""
+        devices = {
+            'eth0': {
+                'bridge': False, 'carrier': False, 'dormant': False,
+                'operstate': 'down', 'address': '00:11:22:33:44:55',
+                'device/driver': 'hv_netsvc', 'device/device': '0x3',
+                'name_assign_type': False},
+            'ens4': {
+                'bridge': False, 'carrier': False, 'dormant': False,
+                'operstate': 'down', 'address': '00:11:22:33:44:55',
+                'device/driver': 'mlx4_core', 'device/device': '0x7',
+                'name_assign_type': '4'},
+
+        }
+
+        m_get_cmdline.return_value = 'net.ifnames=0'
+        tmp_dir = self.tmp_dir()
+        _setup_test(tmp_dir, mock_get_devicelist,
+                    mock_read_sys_net, mock_sys_dev_path,
+                    dev_attrs=devices)
+        net.generate_fallback_config(config_driver=True)
+        self.assertEqual(0, mock_settle.call_count)
+
 
 class TestSysConfigRendering(CiTestCase):
 
-- 
cgit v1.2.3


From 6ef92c98c3d2b127b05d6708337efc8a81e00071 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Thu, 26 Apr 2018 16:24:24 -0500
Subject: IBMCloud: recognize provisioning environment during debug boots.

When images are deployed from template in a production environment
the artifacts of the provisioning stage (provisioningConfiguration.cfg)
that cloud-init referenced are cleaned up.  However, when provisioned
in "debug" mode (internal to IBM) the artifacts are left.

This changes the 'is_ibm_provisioning' implementations in both
ds-identify and in the IBM datasource to identify the provisioning
stage more correctly.  The change is to consider provisioning only
if the provisioing file existed and there was no log file or
the log file was older than this boot.

LP: #1767166
---
 cloudinit/sources/DataSourceIBMCloud.py          | 42 +++++++++-----
 cloudinit/tests/helpers.py                       | 13 ++++-
 tests/unittests/test_datasource/test_ibmcloud.py | 50 ++++++++++++++++
 tests/unittests/test_ds_identify.py              | 72 +++++++++++++++++++++---
 tools/ds-identify                                | 21 ++++++-
 5 files changed, 175 insertions(+), 23 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py
index cfa724bf..01106ec0 100644
--- a/cloudinit/sources/DataSourceIBMCloud.py
+++ b/cloudinit/sources/DataSourceIBMCloud.py
@@ -8,17 +8,11 @@ There are 2 different api exposed launch methods.
  * template: This is the legacy method of launching instances.
    When booting from an image template, the system boots first into
    a "provisioning" mode.  There, host <-> guest mechanisms are utilized
-   to execute code in the guest and provision it.
+   to execute code in the guest and configure it.  The configuration
+   includes configuring the system network and possibly installing
+   packages and other software stack.
 
-   Cloud-init will disable itself when it detects that it is in the
-   provisioning mode.  It detects this by the presence of
-   a file '/root/provisioningConfiguration.cfg'.
-
-   When provided with user-data, the "first boot" will contain a
-   ConfigDrive-like disk labeled with 'METADATA'.  If there is no user-data
-   provided, then there is no data-source.
-
-   Cloud-init never does any network configuration in this mode.
+   After the provisioning is finished, the system reboots.
 
  * os_code: Essentially "launch by OS Code" (Operating System Code).
    This is a more modern approach.  There is no specific "provisioning" boot.
@@ -200,8 +194,30 @@ def _is_xen():
     return os.path.exists("/proc/xen")
 
 
-def _is_ibm_provisioning():
-    return os.path.exists("/root/provisioningConfiguration.cfg")
+def _is_ibm_provisioning(
+        prov_cfg="/root/provisioningConfiguration.cfg",
+        inst_log="/root/swinstall.log",
+        boot_ref="/proc/1/environ"):
+    """Return boolean indicating if this boot is ibm provisioning boot."""
+    if os.path.exists(prov_cfg):
+        msg = "config '%s' exists." % prov_cfg
+        result = True
+        if os.path.exists(inst_log):
+            if os.path.exists(boot_ref):
+                result = (os.stat(inst_log).st_mtime >
+                          os.stat(boot_ref).st_mtime)
+                msg += (" log '%s' from %s boot." %
+                        (inst_log, "current" if result else "previous"))
+            else:
+                msg += (" log '%s' existed, but no reference file '%s'." %
+                        (inst_log, boot_ref))
+                result = False
+        else:
+            msg += " log '%s' did not exist." % inst_log
+    else:
+        result, msg = (False, "config '%s' did not exist." % prov_cfg)
+    LOG.debug("ibm_provisioning=%s: %s", result, msg)
+    return result
 
 
 def get_ibm_platform():
@@ -251,7 +267,7 @@ def get_ibm_platform():
         else:
             return (Platforms.TEMPLATE_LIVE_METADATA, metadata_path)
     elif _is_ibm_provisioning():
-            return (Platforms.TEMPLATE_PROVISIONING_NODATA, None)
+        return (Platforms.TEMPLATE_PROVISIONING_NODATA, None)
     return not_found
 
 
diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py
index 4999f1f6..117a9cfe 100644
--- a/cloudinit/tests/helpers.py
+++ b/cloudinit/tests/helpers.py
@@ -8,6 +8,7 @@ import os
 import shutil
 import sys
 import tempfile
+import time
 import unittest
 
 import mock
@@ -263,7 +264,8 @@ class FilesystemMockingTestCase(ResourceUsingTestCase):
             os.path: [('isfile', 1), ('exists', 1),
                       ('islink', 1), ('isdir', 1), ('lexists', 1)],
             os: [('listdir', 1), ('mkdir', 1),
-                 ('lstat', 1), ('symlink', 2)]
+                 ('lstat', 1), ('symlink', 2),
+                 ('stat', 1)]
         }
 
         if hasattr(os, 'scandir'):
@@ -349,6 +351,15 @@ def populate_dir(path, files):
     return ret
 
 
+def populate_dir_with_ts(path, data):
+    """data is {'file': ('contents', mtime)}.  mtime relative to now."""
+    populate_dir(path, dict((k, v[0]) for k, v in data.items()))
+    btime = time.time()
+    for fpath, (_contents, mtime) in data.items():
+        ts = btime + mtime if mtime else btime
+        os.utime(os.path.sep.join((path, fpath)), (ts, ts))
+
+
 def dir2dict(startdir, prefix=None):
     flist = {}
     if prefix is None:
diff --git a/tests/unittests/test_datasource/test_ibmcloud.py b/tests/unittests/test_datasource/test_ibmcloud.py
index 621cfe49..e639ae47 100644
--- a/tests/unittests/test_datasource/test_ibmcloud.py
+++ b/tests/unittests/test_datasource/test_ibmcloud.py
@@ -259,4 +259,54 @@ class TestReadMD(test_helpers.CiTestCase):
                          ret['metadata'])
 
 
+class TestIsIBMProvisioning(test_helpers.FilesystemMockingTestCase):
+    """Test the _is_ibm_provisioning method."""
+    inst_log = "/root/swinstall.log"
+    prov_cfg = "/root/provisioningConfiguration.cfg"
+    boot_ref = "/proc/1/environ"
+    with_logs = True
+
+    def _call_with_root(self, rootd):
+        self.reRoot(rootd)
+        return ibm._is_ibm_provisioning()
+
+    def test_no_config(self):
+        """No provisioning config means not provisioning."""
+        self.assertFalse(self._call_with_root(self.tmp_dir()))
+
+    def test_config_only(self):
+        """A provisioning config without a log means provisioning."""
+        rootd = self.tmp_dir()
+        test_helpers.populate_dir(rootd, {self.prov_cfg: "key=value"})
+        self.assertTrue(self._call_with_root(rootd))
+
+    def test_config_with_old_log(self):
+        """A config with a log from previous boot is not provisioning."""
+        rootd = self.tmp_dir()
+        data = {self.prov_cfg: ("key=value\nkey2=val2\n", -10),
+                self.inst_log: ("log data\n", -30),
+                self.boot_ref: ("PWD=/", 0)}
+        test_helpers.populate_dir_with_ts(rootd, data)
+        self.assertFalse(self._call_with_root(rootd=rootd))
+        self.assertIn("from previous boot", self.logs.getvalue())
+
+    def test_config_with_new_log(self):
+        """A config with a log from this boot is provisioning."""
+        rootd = self.tmp_dir()
+        data = {self.prov_cfg: ("key=value\nkey2=val2\n", -10),
+                self.inst_log: ("log data\n", 30),
+                self.boot_ref: ("PWD=/", 0)}
+        test_helpers.populate_dir_with_ts(rootd, data)
+        self.assertTrue(self._call_with_root(rootd=rootd))
+        self.assertIn("from current boot", self.logs.getvalue())
+
+    def test_config_and_log_no_reference(self):
+        """If the config and log existed, but no reference, assume not."""
+        rootd = self.tmp_dir()
+        test_helpers.populate_dir(
+            rootd, {self.prov_cfg: "key=value", self.inst_log: "log data\n"})
+        self.assertFalse(self._call_with_root(rootd=rootd))
+        self.assertIn("no reference file", self.logs.getvalue())
+
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 53643989..ad7fe41e 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -1,5 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
+from collections import namedtuple
 import copy
 import os
 from uuid import uuid4
@@ -7,7 +8,7 @@ from uuid import uuid4
 from cloudinit import safeyaml
 from cloudinit import util
 from cloudinit.tests.helpers import (
-    CiTestCase, dir2dict, populate_dir)
+    CiTestCase, dir2dict, populate_dir, populate_dir_with_ts)
 
 from cloudinit.sources import DataSourceIBMCloud as dsibm
 
@@ -66,7 +67,6 @@ P_SYS_VENDOR = "sys/class/dmi/id/sys_vendor"
 P_SEED_DIR = "var/lib/cloud/seed"
 P_DSID_CFG = "etc/cloud/ds-identify.cfg"
 
-IBM_PROVISIONING_CHECK_PATH = "/root/provisioningConfiguration.cfg"
 IBM_CONFIG_UUID = "9796-932E"
 
 MOCK_VIRT_IS_KVM = {'name': 'detect_virt', 'RET': 'kvm', 'ret': 0}
@@ -74,11 +74,17 @@ MOCK_VIRT_IS_VMWARE = {'name': 'detect_virt', 'RET': 'vmware', 'ret': 0}
 MOCK_VIRT_IS_XEN = {'name': 'detect_virt', 'RET': 'xen', 'ret': 0}
 MOCK_UNAME_IS_PPC64 = {'name': 'uname', 'out': UNAME_PPC64EL, 'ret': 0}
 
+shell_true = 0
+shell_false = 1
 
-class TestDsIdentify(CiTestCase):
+CallReturn = namedtuple('CallReturn',
+                        ['rc', 'stdout', 'stderr', 'cfg', 'files'])
+
+
+class DsIdentifyBase(CiTestCase):
     dsid_path = os.path.realpath('tools/ds-identify')
 
-    def call(self, rootd=None, mocks=None, args=None, files=None,
+    def call(self, rootd=None, mocks=None, func="main", args=None, files=None,
              policy_dmi=DI_DEFAULT_POLICY,
              policy_no_dmi=DI_DEFAULT_POLICY_NO_DMI,
              ec2_strict_id=DI_EC2_STRICT_ID_DEFAULT):
@@ -135,7 +141,7 @@ class TestDsIdentify(CiTestCase):
                 mocklines.append(write_mock(d))
 
         endlines = [
-            'main %s' % ' '.join(['"%s"' % s for s in args])
+            func + ' ' + ' '.join(['"%s"' % s for s in args])
         ]
 
         with open(wrap, "w") as fp:
@@ -159,7 +165,7 @@ class TestDsIdentify(CiTestCase):
                 cfg = {"_INVALID_YAML": contents,
                        "_EXCEPTION": str(e)}
 
-        return rc, out, err, cfg, dir2dict(rootd)
+        return CallReturn(rc, out, err, cfg, dir2dict(rootd))
 
     def _call_via_dict(self, data, rootd=None, **kwargs):
         # return output of self.call with a dict input like VALID_CFG[item]
@@ -190,6 +196,8 @@ class TestDsIdentify(CiTestCase):
                 _print_run_output(rc, out, err, cfg, files)
         return rc, out, err, cfg, files
 
+
+class TestDsIdentify(DsIdentifyBase):
     def test_wb_print_variables(self):
         """_print_info reports an array of discovered variables to stderr."""
         data = VALID_CFG['Azure-dmi-detection']
@@ -250,7 +258,10 @@ class TestDsIdentify(CiTestCase):
         Template provisioning with user-data has METADATA disk,
         datasource should return not found."""
         data = copy.deepcopy(VALID_CFG['IBMCloud-metadata'])
-        data['files'] = {IBM_PROVISIONING_CHECK_PATH: 'xxx'}
+        # change the 'is_ibm_provisioning' mock to return 1 (false)
+        isprov_m = [m for m in data['mocks']
+                    if m["name"] == "is_ibm_provisioning"][0]
+        isprov_m['ret'] = shell_true
         return self._check_via_dict(data, RC_NOT_FOUND)
 
     def test_ibmcloud_template_userdata(self):
@@ -265,7 +276,8 @@ class TestDsIdentify(CiTestCase):
 
         no disks attached.  Datasource should return not found."""
         data = copy.deepcopy(VALID_CFG['IBMCloud-nodisks'])
-        data['files'] = {IBM_PROVISIONING_CHECK_PATH: 'xxx'}
+        data['mocks'].append(
+            {'name': 'is_ibm_provisioning', 'ret': shell_true})
         return self._check_via_dict(data, RC_NOT_FOUND)
 
     def test_ibmcloud_template_no_userdata(self):
@@ -446,6 +458,47 @@ class TestDsIdentify(CiTestCase):
         self._test_ds_found('Hetzner')
 
 
+class TestIsIBMProvisioning(DsIdentifyBase):
+    """Test the is_ibm_provisioning method in ds-identify."""
+
+    inst_log = "/root/swinstall.log"
+    prov_cfg = "/root/provisioningConfiguration.cfg"
+    boot_ref = "/proc/1/environ"
+    funcname = "is_ibm_provisioning"
+
+    def test_no_config(self):
+        """No provisioning config means not provisioning."""
+        ret = self.call(files={}, func=self.funcname)
+        self.assertEqual(shell_false, ret.rc)
+
+    def test_config_only(self):
+        """A provisioning config without a log means provisioning."""
+        ret = self.call(files={self.prov_cfg: "key=value"}, func=self.funcname)
+        self.assertEqual(shell_true, ret.rc)
+
+    def test_config_with_old_log(self):
+        """A config with a log from previous boot is not provisioning."""
+        rootd = self.tmp_dir()
+        data = {self.prov_cfg: ("key=value\nkey2=val2\n", -10),
+                self.inst_log: ("log data\n", -30),
+                self.boot_ref: ("PWD=/", 0)}
+        populate_dir_with_ts(rootd, data)
+        ret = self.call(rootd=rootd, func=self.funcname)
+        self.assertEqual(shell_false, ret.rc)
+        self.assertIn("from previous boot", ret.stderr)
+
+    def test_config_with_new_log(self):
+        """A config with a log from this boot is provisioning."""
+        rootd = self.tmp_dir()
+        data = {self.prov_cfg: ("key=value\nkey2=val2\n", -10),
+                self.inst_log: ("log data\n", 30),
+                self.boot_ref: ("PWD=/", 0)}
+        populate_dir_with_ts(rootd, data)
+        ret = self.call(rootd=rootd, func=self.funcname)
+        self.assertEqual(shell_true, ret.rc)
+        self.assertIn("from current boot", ret.stderr)
+
+
 def blkid_out(disks=None):
     """Convert a list of disk dictionaries into blkid content."""
     if disks is None:
@@ -639,6 +692,7 @@ VALID_CFG = {
         'ds': 'IBMCloud',
         'mocks': [
             MOCK_VIRT_IS_XEN,
+            {'name': 'is_ibm_provisioning', 'ret': shell_false},
             {'name': 'blkid', 'ret': 0,
              'out': blkid_out(
                  [{'DEVNAME': 'xvda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()},
@@ -652,6 +706,7 @@ VALID_CFG = {
         'ds': 'IBMCloud',
         'mocks': [
             MOCK_VIRT_IS_XEN,
+            {'name': 'is_ibm_provisioning', 'ret': shell_false},
             {'name': 'blkid', 'ret': 0,
              'out': blkid_out(
                  [{'DEVNAME': 'xvda1', 'TYPE': 'ext3', 'PARTUUID': uuid4(),
@@ -669,6 +724,7 @@ VALID_CFG = {
         'ds': 'IBMCloud',
         'mocks': [
             MOCK_VIRT_IS_XEN,
+            {'name': 'is_ibm_provisioning', 'ret': shell_false},
             {'name': 'blkid', 'ret': 0,
              'out': blkid_out(
                  [{'DEVNAME': 'xvda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()},
diff --git a/tools/ds-identify b/tools/ds-identify
index 9a2db5c4..7fff5d1e 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -125,6 +125,7 @@ DI_ON_NOTFOUND=""
 DI_EC2_STRICT_ID_DEFAULT="true"
 
 _IS_IBM_CLOUD=""
+_IS_IBM_PROVISIONING=""
 
 error() {
     set -- "ERROR:" "$@";
@@ -1006,7 +1007,25 @@ dscheck_Hetzner() {
 }
 
 is_ibm_provisioning() {
-    [ -f "${PATH_ROOT}/root/provisioningConfiguration.cfg" ]
+    local pcfg="${PATH_ROOT}/root/provisioningConfiguration.cfg"
+    local logf="${PATH_ROOT}/root/swinstall.log"
+    local is_prov=false msg="config '$pcfg' did not exist."
+    if [ -f "$pcfg" ]; then
+        msg="config '$pcfg' exists."
+        is_prov=true
+        if [ -f "$logf" ]; then
+            if [ "$logf" -nt "$PATH_PROC_1_ENVIRON" ]; then
+                msg="$msg log '$logf' from current boot."
+            else
+                is_prov=false
+                msg="$msg log '$logf' from previous boot."
+            fi
+        else
+            msg="$msg log '$logf' did not exist."
+        fi
+    fi
+    debug 2 "ibm_provisioning=$is_prov: $msg"
+    [ "$is_prov" = "true" ]
 }
 
 is_ibm_cloud() {
-- 
cgit v1.2.3


From 11172924a48a47a7231d19d9cefe628dfddda8bf Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Mon, 30 Apr 2018 13:21:51 -0600
Subject: IBMCloud: Disable config-drive and nocloud only if IBMCloud is
 enabled.

Ubuntu images on IBMCloud for 16.04 have some seed data in
/var/lib/cloud/data/seed/nocloud-net.  In order to have systems with
IBMCloud enabled, we modified ds-identify detection to skip that seed
if the system was on IBMCloud.  That change did not consider the
fact that IBMCloud might not be in the datasource list.

There was similar logic in the ConfigDrive datasource in ds-identify
and the datasource itself.

Config drive is now updated to only check and avoid IBMCloud if IBMCloud
is enabled.  The check in ds-identify for nocloud was dropped.  If a
user provides a nocloud seed on IBMCloud, then that can be used.

This means that systems running Xenial will continue to get their
old datasources.

LP: #1766401
---
 cloudinit/sources/DataSourceConfigDrive.py | 11 +++--
 tests/unittests/test_ds_identify.py        | 77 +++++++++++++++++++++++++++---
 tools/ds-identify                          | 17 +++++--
 3 files changed, 91 insertions(+), 14 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index c7b5fe5f..121cf215 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -69,7 +69,8 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
                 util.logexc(LOG, "Failed reading config drive from %s", sdir)
 
         if not found:
-            for dev in find_candidate_devs():
+            dslist = self.sys_cfg.get('datasource_list')
+            for dev in find_candidate_devs(dslist=dslist):
                 try:
                     # Set mtype if freebsd and turn off sync
                     if dev.startswith("/dev/cd"):
@@ -211,7 +212,7 @@ def write_injected_files(files):
                 util.logexc(LOG, "Failed writing file: %s", filename)
 
 
-def find_candidate_devs(probe_optical=True):
+def find_candidate_devs(probe_optical=True, dslist=None):
     """Return a list of devices that may contain the config drive.
 
     The returned list is sorted by search order where the first item has
@@ -227,6 +228,9 @@ def find_candidate_devs(probe_optical=True):
         * either vfat or iso9660 formated
         * labeled with 'config-2' or 'CONFIG-2'
     """
+    if dslist is None:
+        dslist = []
+
     # query optical drive to get it in blkid cache for 2.6 kernels
     if probe_optical:
         for device in OPTICAL_DEVICES:
@@ -257,7 +261,8 @@ def find_candidate_devs(probe_optical=True):
     devices = [d for d in candidates
                if d in by_label or not util.is_partition(d)]
 
-    if devices:
+    LOG.debug("devices=%s dslist=%s", devices, dslist)
+    if devices and "IBMCloud" in dslist:
         # IBMCloud uses config-2 label, but limited to a single UUID.
         ibm_platform, ibm_path = get_ibm_platform()
         if ibm_path in devices:
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index ad7fe41e..4d8a4360 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -184,17 +184,18 @@ class DsIdentifyBase(CiTestCase):
             data, RC_FOUND, dslist=[data.get('ds'), DS_NONE])
 
     def _check_via_dict(self, data, rc, dslist=None, **kwargs):
-        found_rc, out, err, cfg, files = self._call_via_dict(data, **kwargs)
+        ret = self._call_via_dict(data, **kwargs)
         good = False
         try:
-            self.assertEqual(rc, found_rc)
+            self.assertEqual(rc, ret.rc)
             if dslist is not None:
-                self.assertEqual(dslist, cfg['datasource_list'])
+                self.assertEqual(dslist, ret.cfg['datasource_list'])
             good = True
         finally:
             if not good:
-                _print_run_output(rc, out, err, cfg, files)
-        return rc, out, err, cfg, files
+                _print_run_output(ret.rc, ret.stdout, ret.stderr, ret.cfg,
+                                  ret.files)
+        return ret
 
 
 class TestDsIdentify(DsIdentifyBase):
@@ -245,13 +246,40 @@ class TestDsIdentify(DsIdentifyBase):
     def test_config_drive(self):
         """ConfigDrive datasource has a disk with LABEL=config-2."""
         self._test_ds_found('ConfigDrive')
-        return
 
     def test_config_drive_upper(self):
         """ConfigDrive datasource has a disk with LABEL=CONFIG-2."""
         self._test_ds_found('ConfigDriveUpper')
         return
 
+    def test_config_drive_seed(self):
+        """Config Drive seed directory."""
+        self._test_ds_found('ConfigDrive-seed')
+
+    def test_config_drive_interacts_with_ibmcloud_config_disk(self):
+        """Verify ConfigDrive interaction with IBMCloud.
+
+        If ConfigDrive is enabled and not IBMCloud, then ConfigDrive
+        should claim the ibmcloud 'config-2' disk.
+        If IBMCloud is enabled, then ConfigDrive should skip."""
+        data = copy.deepcopy(VALID_CFG['IBMCloud-config-2'])
+        files = data.get('files', {})
+        if not files:
+            data['files'] = files
+        cfgpath = 'etc/cloud/cloud.cfg.d/99_networklayer_common.cfg'
+
+        # with list including IBMCloud, config drive should be not found.
+        files[cfgpath] = 'datasource_list: [ ConfigDrive, IBMCloud ]\n'
+        ret = self._check_via_dict(data, shell_true)
+        self.assertEqual(
+            ret.cfg.get('datasource_list'), ['IBMCloud', 'None'])
+
+        # But if IBMCloud is not enabled, config drive should claim this.
+        files[cfgpath] = 'datasource_list: [ ConfigDrive, NoCloud ]\n'
+        ret = self._check_via_dict(data, shell_true)
+        self.assertEqual(
+            ret.cfg.get('datasource_list'), ['ConfigDrive', 'None'])
+
     def test_ibmcloud_template_userdata_in_provisioning(self):
         """Template provisioned with user-data during provisioning stage.
 
@@ -307,6 +335,37 @@ class TestDsIdentify(DsIdentifyBase):
         self._check_via_dict(
             data, rc=RC_FOUND, dslist=['ConfigDrive', DS_NONE])
 
+    def test_ibmcloud_with_nocloud_seed(self):
+        """NoCloud seed should be preferred over IBMCloud.
+
+        A nocloud seed should be preferred over IBMCloud even if enabled.
+        Ubuntu 16.04 images have <vlc>/seed/nocloud-net. LP: #1766401."""
+        data = copy.deepcopy(VALID_CFG['IBMCloud-config-2'])
+        files = data.get('files', {})
+        if not files:
+            data['files'] = files
+        files.update(VALID_CFG['NoCloud-seed']['files'])
+        ret = self._check_via_dict(data, shell_true)
+        self.assertEqual(
+            ['NoCloud', 'IBMCloud', 'None'],
+            ret.cfg.get('datasource_list'))
+
+    def test_ibmcloud_with_configdrive_seed(self):
+        """ConfigDrive seed should be preferred over IBMCloud.
+
+        A ConfigDrive seed should be preferred over IBMCloud even if enabled.
+        Ubuntu 16.04 images have a fstab entry that mounts the
+        METADATA disk into <vlc>/seed/config_drive. LP: ##1766401."""
+        data = copy.deepcopy(VALID_CFG['IBMCloud-config-2'])
+        files = data.get('files', {})
+        if not files:
+            data['files'] = files
+        files.update(VALID_CFG['ConfigDrive-seed']['files'])
+        ret = self._check_via_dict(data, shell_true)
+        self.assertEqual(
+            ['ConfigDrive', 'IBMCloud', 'None'],
+            ret.cfg.get('datasource_list'))
+
     def test_policy_disabled(self):
         """A Builtin policy of 'disabled' should return not found.
 
@@ -684,6 +743,12 @@ VALID_CFG = {
              },
         ],
     },
+    'ConfigDrive-seed': {
+        'ds': 'ConfigDrive',
+        'files': {
+            os.path.join(P_SEED_DIR, 'config_drive', 'openstack',
+                         'latest', 'meta_data.json'): 'md\n'},
+    },
     'Hetzner': {
         'ds': 'Hetzner',
         'files': {P_SYS_VENDOR: 'Hetzner\n'},
diff --git a/tools/ds-identify b/tools/ds-identify
index 7fff5d1e..9f0d96f7 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -601,7 +601,6 @@ dscheck_NoCloud() {
         *\ ds=nocloud*) return ${DS_FOUND};;
     esac
 
-    is_ibm_cloud && return ${DS_NOT_FOUND}
     for d in nocloud nocloud-net; do
         check_seed_dir "$d" meta-data user-data && return ${DS_FOUND}
         check_writable_seed_dir "$d" meta-data user-data && return ${DS_FOUND}
@@ -612,11 +611,12 @@ dscheck_NoCloud() {
     return ${DS_NOT_FOUND}
 }
 
+is_ds_enabled() {
+    local name="$1" pad=" ${DI_DSLIST} "
+    [ "${pad#* $name }" != "${pad}" ]
+}
+
 check_configdrive_v2() {
-    is_ibm_cloud && return ${DS_NOT_FOUND}
-    if has_fs_with_label CONFIG-2 config-2; then
-        return ${DS_FOUND}
-    fi
     # look in /config-drive <vlc>/seed/config_drive for a directory
     # openstack/YYYY-MM-DD format with a file meta_data.json
     local d=""
@@ -631,6 +631,13 @@ check_configdrive_v2() {
         debug 1 "config drive seeded directory had only 'latest'"
         return ${DS_FOUND}
     fi
+
+    is_ds_enabled "IBMCloud"
+    debug 1 "is_ds_enabled returned $?: $DI_DSLIST"
+    is_ds_enabled "IBMCloud" && is_ibm_cloud && return ${DS_NOT_FOUND}
+    if has_fs_with_label CONFIG-2 config-2; then
+        return ${DS_FOUND}
+    fi
     return ${DS_NOT_FOUND}
 }
 
-- 
cgit v1.2.3


From aae494c39f4c6f625e7409ca262e657d085dd5d1 Mon Sep 17 00:00:00 2001
From: Joshua Chan <jocha@microsoft.com>
Date: Thu, 3 May 2018 14:50:16 -0600
Subject: azure: Add reported ready marker file.

This change is for Azure VM Preprovisioning. A bug was found when after
azure VMs report ready the first time, during the time when VM is polling
indefinitely for the new ovf-env.xml from Instance Metadata Service
(IMDS), if a reboot happens, we send another report ready signal to the
fabric, which deletes the reprovisioning data on the node.

This marker file is used to fix this issue so that we will only send a
report ready signal to the fabric when no marker file is present. Then,
create a marker file so that when a reboot does occur, we check if a
marker file has been created and decide whether we would like to send the
repot ready signal.

LP: #1765214
---
 cloudinit/sources/DataSourceAzure.py          |  21 +++-
 tests/unittests/test_datasource/test_azure.py | 170 ++++++++++++++++++--------
 2 files changed, 134 insertions(+), 57 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index a71197a6..1b03d460 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -48,6 +48,7 @@ DEFAULT_FS = 'ext4'
 # DMI chassis-asset-tag is set static for all azure instances
 AZURE_CHASSIS_ASSET_TAG = '7783-7084-3265-9085-8269-3286-77'
 REPROVISION_MARKER_FILE = "/var/lib/cloud/data/poll_imds"
+REPORTED_READY_MARKER_FILE = "/var/lib/cloud/data/reported_ready"
 IMDS_URL = "http://169.254.169.254/metadata/reprovisiondata"
 
 
@@ -436,11 +437,12 @@ class DataSourceAzure(sources.DataSource):
             LOG.debug("negotiating already done for %s",
                       self.get_instance_id())
 
-    def _poll_imds(self, report_ready=True):
+    def _poll_imds(self):
         """Poll IMDS for the new provisioning data until we get a valid
         response. Then return the returned JSON object."""
         url = IMDS_URL + "?api-version=2017-04-02"
         headers = {"Metadata": "true"}
+        report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE))
         LOG.debug("Start polling IMDS")
 
         def exc_cb(msg, exception):
@@ -450,13 +452,17 @@ class DataSourceAzure(sources.DataSource):
             # call DHCP and setup the ephemeral network to acquire the new IP.
             return False
 
-        need_report = report_ready
         while True:
             try:
                 with EphemeralDHCPv4() as lease:
-                    if need_report:
+                    if report_ready:
+                        path = REPORTED_READY_MARKER_FILE
+                        LOG.info(
+                            "Creating a marker file to report ready: %s", path)
+                        util.write_file(path, "{pid}: {time}\n".format(
+                            pid=os.getpid(), time=time()))
                         self._report_ready(lease=lease)
-                        need_report = False
+                        report_ready = False
                     return readurl(url, timeout=1, headers=headers,
                                    exception_cb=exc_cb, infinite=True).contents
             except UrlError:
@@ -490,8 +496,10 @@ class DataSourceAzure(sources.DataSource):
         if (cfg.get('PreprovisionedVm') is True or
                 os.path.isfile(path)):
             if not os.path.isfile(path):
-                LOG.info("Creating a marker file to poll imds")
-                util.write_file(path, "%s: %s\n" % (os.getpid(), time()))
+                LOG.info("Creating a marker file to poll imds: %s",
+                         path)
+                util.write_file(path, "{pid}: {time}\n".format(
+                    pid=os.getpid(), time=time()))
             return True
         return False
 
@@ -526,6 +534,7 @@ class DataSourceAzure(sources.DataSource):
                 "Error communicating with Azure fabric; You may experience."
                 "connectivity issues.", exc_info=True)
             return False
+        util.del_file(REPORTED_READY_MARKER_FILE)
         util.del_file(REPROVISION_MARKER_FILE)
         return fabric_data
 
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 88fe76c7..26e8d7d3 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -1125,19 +1125,9 @@ class TestAzureNetExists(CiTestCase):
         self.assertTrue(hasattr(dsaz, "DataSourceAzureNet"))
 
 
-@mock.patch('cloudinit.sources.DataSourceAzure.util.subp')
-@mock.patch.object(dsaz, 'get_hostname')
-@mock.patch.object(dsaz, 'set_hostname')
-class TestAzureDataSourcePreprovisioning(CiTestCase):
-
-    def setUp(self):
-        super(TestAzureDataSourcePreprovisioning, self).setUp()
-        tmp = self.tmp_dir()
-        self.waagent_d = self.tmp_path('/var/lib/waagent', tmp)
-        self.paths = helpers.Paths({'cloud_dir': tmp})
-        dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d
+class TestPreprovisioningReadAzureOvfFlag(CiTestCase):
 
-    def test_read_azure_ovf_with_true_flag(self, *args):
+    def test_read_azure_ovf_with_true_flag(self):
         """The read_azure_ovf method should set the PreprovisionedVM
            cfg flag if the proper setting is present."""
         content = construct_valid_ovf_env(
@@ -1146,7 +1136,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase):
         cfg = ret[2]
         self.assertTrue(cfg['PreprovisionedVm'])
 
-    def test_read_azure_ovf_with_false_flag(self, *args):
+    def test_read_azure_ovf_with_false_flag(self):
         """The read_azure_ovf method should set the PreprovisionedVM
            cfg flag to false if the proper setting is false."""
         content = construct_valid_ovf_env(
@@ -1155,7 +1145,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase):
         cfg = ret[2]
         self.assertFalse(cfg['PreprovisionedVm'])
 
-    def test_read_azure_ovf_without_flag(self, *args):
+    def test_read_azure_ovf_without_flag(self):
         """The read_azure_ovf method should not set the
            PreprovisionedVM cfg flag."""
         content = construct_valid_ovf_env()
@@ -1163,12 +1153,121 @@ class TestAzureDataSourcePreprovisioning(CiTestCase):
         cfg = ret[2]
         self.assertFalse(cfg['PreprovisionedVm'])
 
-    @mock.patch('cloudinit.sources.DataSourceAzure.util.is_FreeBSD')
-    @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network')
-    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
-    @mock.patch('requests.Session.request')
+
+@mock.patch('os.path.isfile')
+class TestPreprovisioningShouldReprovision(CiTestCase):
+
+    def setUp(self):
+        super(TestPreprovisioningShouldReprovision, self).setUp()
+        tmp = self.tmp_dir()
+        self.waagent_d = self.tmp_path('/var/lib/waagent', tmp)
+        self.paths = helpers.Paths({'cloud_dir': tmp})
+        dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d
+
+    @mock.patch('cloudinit.sources.DataSourceAzure.util.write_file')
+    def test__should_reprovision_with_true_cfg(self, isfile, write_f):
+        """The _should_reprovision method should return true with config
+           flag present."""
+        isfile.return_value = False
+        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
+        self.assertTrue(dsa._should_reprovision(
+            (None, None, {'PreprovisionedVm': True}, None)))
+
+    def test__should_reprovision_with_file_existing(self, isfile):
+        """The _should_reprovision method should return True if the sentinal
+           exists."""
+        isfile.return_value = True
+        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
+        self.assertTrue(dsa._should_reprovision(
+            (None, None, {'preprovisionedvm': False}, None)))
+
+    def test__should_reprovision_returns_false(self, isfile):
+        """The _should_reprovision method should return False
+           if config and sentinal are not present."""
+        isfile.return_value = False
+        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
+        self.assertFalse(dsa._should_reprovision((None, None, {}, None)))
+
+    @mock.patch('cloudinit.sources.DataSourceAzure.DataSourceAzure._poll_imds')
+    def test_reprovision_calls__poll_imds(self, _poll_imds, isfile):
+        """_reprovision will poll IMDS."""
+        isfile.return_value = False
+        hostname = "myhost"
+        username = "myuser"
+        odata = {'HostName': hostname, 'UserName': username}
+        _poll_imds.return_value = construct_valid_ovf_env(data=odata)
+        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
+        dsa._reprovision()
+        _poll_imds.assert_called_with()
+
+
+@mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network')
+@mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+@mock.patch('requests.Session.request')
+@mock.patch(
+    'cloudinit.sources.DataSourceAzure.DataSourceAzure._report_ready')
+class TestPreprovisioningPollIMDS(CiTestCase):
+
+    def setUp(self):
+        super(TestPreprovisioningPollIMDS, self).setUp()
+        self.tmp = self.tmp_dir()
+        self.waagent_d = self.tmp_path('/var/lib/waagent', self.tmp)
+        self.paths = helpers.Paths({'cloud_dir': self.tmp})
+        dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d
+
+    @mock.patch('cloudinit.sources.DataSourceAzure.util.write_file')
+    def test_poll_imds_calls_report_ready(self, write_f, report_ready_func,
+                                          fake_resp, m_dhcp, m_net):
+        """The poll_imds will call report_ready after creating marker file."""
+        report_marker = self.tmp_path('report_marker', self.tmp)
+        lease = {
+            'interface': 'eth9', 'fixed-address': '192.168.2.9',
+            'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0',
+            'unknown-245': '624c3620'}
+        m_dhcp.return_value = [lease]
+        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
+        mock_path = (
+            'cloudinit.sources.DataSourceAzure.REPORTED_READY_MARKER_FILE')
+        with mock.patch(mock_path, report_marker):
+            dsa._poll_imds()
+        self.assertEqual(report_ready_func.call_count, 1)
+        report_ready_func.assert_called_with(lease=lease)
+
+    def test_poll_imds_report_ready_false(self, report_ready_func,
+                                          fake_resp, m_dhcp, m_net):
+        """The poll_imds should not call reporting ready
+           when flag is false"""
+        report_marker = self.tmp_path('report_marker', self.tmp)
+        write_file(report_marker, content='dont run report_ready :)')
+        m_dhcp.return_value = [{
+            'interface': 'eth9', 'fixed-address': '192.168.2.9',
+            'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0',
+            'unknown-245': '624c3620'}]
+        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
+        mock_path = (
+            'cloudinit.sources.DataSourceAzure.REPORTED_READY_MARKER_FILE')
+        with mock.patch(mock_path, report_marker):
+            dsa._poll_imds()
+        self.assertEqual(report_ready_func.call_count, 0)
+
+
+@mock.patch('cloudinit.sources.DataSourceAzure.util.subp')
+@mock.patch('cloudinit.sources.DataSourceAzure.util.write_file')
+@mock.patch('cloudinit.sources.DataSourceAzure.util.is_FreeBSD')
+@mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network')
+@mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+@mock.patch('requests.Session.request')
+class TestAzureDataSourcePreprovisioning(CiTestCase):
+
+    def setUp(self):
+        super(TestAzureDataSourcePreprovisioning, self).setUp()
+        tmp = self.tmp_dir()
+        self.waagent_d = self.tmp_path('/var/lib/waagent', tmp)
+        self.paths = helpers.Paths({'cloud_dir': tmp})
+        dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d
+
     def test_poll_imds_returns_ovf_env(self, fake_resp, m_dhcp, m_net,
-                                       m_is_bsd, *args):
+                                       m_is_bsd, write_f, subp):
         """The _poll_imds method should return the ovf_env.xml."""
         m_is_bsd.return_value = False
         m_dhcp.return_value = [{
@@ -1194,12 +1293,8 @@ class TestAzureDataSourcePreprovisioning(CiTestCase):
             prefix_or_mask='255.255.255.0', router='192.168.2.1')
         self.assertEqual(m_net.call_count, 1)
 
-    @mock.patch('cloudinit.sources.DataSourceAzure.util.is_FreeBSD')
-    @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network')
-    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
-    @mock.patch('requests.Session.request')
     def test__reprovision_calls__poll_imds(self, fake_resp, m_dhcp, m_net,
-                                           m_is_bsd, *args):
+                                           m_is_bsd, write_f, subp):
         """The _reprovision method should call poll IMDS."""
         m_is_bsd.return_value = False
         m_dhcp.return_value = [{
@@ -1231,32 +1326,5 @@ class TestAzureDataSourcePreprovisioning(CiTestCase):
             prefix_or_mask='255.255.255.0', router='192.168.2.1')
         self.assertEqual(m_net.call_count, 1)
 
-    @mock.patch('cloudinit.sources.DataSourceAzure.util.write_file')
-    @mock.patch('os.path.isfile')
-    def test__should_reprovision_with_true_cfg(self, isfile, write_f, *args):
-        """The _should_reprovision method should return true with config
-           flag present."""
-        isfile.return_value = False
-        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
-        self.assertTrue(dsa._should_reprovision(
-            (None, None, {'PreprovisionedVm': True}, None)))
-
-    @mock.patch('os.path.isfile')
-    def test__should_reprovision_with_file_existing(self, isfile, *args):
-        """The _should_reprovision method should return True if the sentinal
-           exists."""
-        isfile.return_value = True
-        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
-        self.assertTrue(dsa._should_reprovision(
-            (None, None, {'preprovisionedvm': False}, None)))
-
-    @mock.patch('os.path.isfile')
-    def test__should_reprovision_returns_false(self, isfile, *args):
-        """The _should_reprovision method should return False
-           if config and sentinal are not present."""
-        isfile.return_value = False
-        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
-        self.assertFalse(dsa._should_reprovision((None, None, {}, None)))
-
 
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From 0d7ee5592621d09699d079945ffd6febf16669b2 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Tue, 15 May 2018 13:38:20 -0400
Subject: ds-identify: recognize container-other as a container, test SmartOS.

In playing with a SmartOS container I found that ds-identify did
not identify the container there as a container.  Systemd-detect-virt
identifies it as 'container-other'.

Also here are tests for ds-identify for the SmartOS platform
identification, and some indentation fixes in ds-identify.
---
 cloudinit/sources/DataSourceSmartOS.py |  4 +--
 tests/unittests/test_ds_identify.py    | 51 ++++++++++++++++++++++++++++++++--
 tools/ds-identify                      | 12 ++++----
 3 files changed, 57 insertions(+), 10 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 4ea00eb1..fcb46b14 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -745,7 +745,7 @@ def get_smartos_environ(uname_version=None, product_name=None):
     # report 'BrandZ virtual linux' as the kernel version
     if uname_version is None:
         uname_version = uname[3]
-    if uname_version.lower() == 'brandz virtual linux':
+    if uname_version == 'BrandZ virtual linux':
         return SMARTOS_ENV_LX_BRAND
 
     if product_name is None:
@@ -753,7 +753,7 @@ def get_smartos_environ(uname_version=None, product_name=None):
     else:
         system_type = product_name
 
-    if system_type and 'smartdc' in system_type.lower():
+    if system_type and system_type.startswith('SmartDC'):
         return SMARTOS_ENV_KVM
 
     return None
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 4d8a4360..7f12be59 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -10,7 +10,8 @@ from cloudinit import util
 from cloudinit.tests.helpers import (
     CiTestCase, dir2dict, populate_dir, populate_dir_with_ts)
 
-from cloudinit.sources import DataSourceIBMCloud as dsibm
+from cloudinit.sources import DataSourceIBMCloud as ds_ibm
+from cloudinit.sources import DataSourceSmartOS as ds_smartos
 
 UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu "
                "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux")
@@ -69,8 +70,12 @@ P_DSID_CFG = "etc/cloud/ds-identify.cfg"
 
 IBM_CONFIG_UUID = "9796-932E"
 
+MOCK_VIRT_IS_CONTAINER_OTHER = {'name': 'detect_virt',
+                                'RET': 'container-other', 'ret': 0}
 MOCK_VIRT_IS_KVM = {'name': 'detect_virt', 'RET': 'kvm', 'ret': 0}
 MOCK_VIRT_IS_VMWARE = {'name': 'detect_virt', 'RET': 'vmware', 'ret': 0}
+# currenty' SmartOS hypervisor "bhyve" is unknown by systemd-detect-virt.
+MOCK_VIRT_IS_VM_OTHER = {'name': 'detect_virt', 'RET': 'vm-other', 'ret': 0}
 MOCK_VIRT_IS_XEN = {'name': 'detect_virt', 'RET': 'xen', 'ret': 0}
 MOCK_UNAME_IS_PPC64 = {'name': 'uname', 'out': UNAME_PPC64EL, 'ret': 0}
 
@@ -330,7 +335,7 @@ class TestDsIdentify(DsIdentifyBase):
                 break
         if not offset:
             raise ValueError("Expected to find 'blkid' mock, but did not.")
-        data['mocks'][offset]['out'] = d['out'].replace(dsibm.IBM_CONFIG_UUID,
+        data['mocks'][offset]['out'] = d['out'].replace(ds_ibm.IBM_CONFIG_UUID,
                                                         "DEAD-BEEF")
         self._check_via_dict(
             data, rc=RC_FOUND, dslist=['ConfigDrive', DS_NONE])
@@ -516,6 +521,20 @@ class TestDsIdentify(DsIdentifyBase):
         """Hetzner cloud is identified in sys_vendor."""
         self._test_ds_found('Hetzner')
 
+    def test_smartos_bhyve(self):
+        """SmartOS cloud identified by SmartDC in dmi."""
+        self._test_ds_found('SmartOS-bhyve')
+
+    def test_smartos_lxbrand(self):
+        """SmartOS cloud identified on lxbrand container."""
+        self._test_ds_found('SmartOS-lxbrand')
+
+    def test_smartos_lxbrand_requires_socket(self):
+        """SmartOS cloud should not be identified if no socket file."""
+        mycfg = copy.deepcopy(VALID_CFG['SmartOS-lxbrand'])
+        del mycfg['files'][ds_smartos.METADATA_SOCKFILE]
+        self._check_via_dict(mycfg, rc=RC_NOT_FOUND, policy_dmi="disabled")
+
 
 class TestIsIBMProvisioning(DsIdentifyBase):
     """Test the is_ibm_provisioning method in ds-identify."""
@@ -777,7 +796,7 @@ VALID_CFG = {
                  [{'DEVNAME': 'xvda1', 'TYPE': 'ext3', 'PARTUUID': uuid4(),
                    'UUID': uuid4(), 'LABEL': 'cloudimg-bootfs'},
                   {'DEVNAME': 'xvdb', 'TYPE': 'vfat', 'LABEL': 'config-2',
-                   'UUID': dsibm.IBM_CONFIG_UUID},
+                   'UUID': ds_ibm.IBM_CONFIG_UUID},
                   {'DEVNAME': 'xvda2', 'TYPE': 'ext4',
                    'LABEL': 'cloudimg-rootfs', 'PARTUUID': uuid4(),
                    'UUID': uuid4()},
@@ -798,6 +817,32 @@ VALID_CFG = {
              },
         ],
     },
+    'SmartOS-bhyve': {
+        'ds': 'SmartOS',
+        'mocks': [
+            MOCK_VIRT_IS_VM_OTHER,
+            {'name': 'blkid', 'ret': 0,
+             'out': blkid_out(
+                 [{'DEVNAME': 'vda1', 'TYPE': 'ext4',
+                   'PARTUUID': '49ec635a-01'},
+                  {'DEVNAME': 'vda2', 'TYPE': 'swap',
+                   'LABEL': 'cloudimg-swap', 'PARTUUID': '49ec635a-02'}]),
+             },
+        ],
+        'files': {P_PRODUCT_NAME: 'SmartDC HVM\n'},
+    },
+    'SmartOS-lxbrand': {
+        'ds': 'SmartOS',
+        'mocks': [
+            MOCK_VIRT_IS_CONTAINER_OTHER,
+            {'name': 'uname', 'ret': 0,
+             'out': ("Linux d43da87a-daca-60e8-e6d4-d2ed372662a3 4.3.0 "
+                     "BrandZ virtual linux x86_64 GNU/Linux")},
+            {'name': 'blkid', 'ret': 2, 'out': ''},
+        ],
+        'files': {ds_smartos.METADATA_SOCKFILE: 'would be a socket\n'},
+    }
+
 }
 
 # vi: ts=4 expandtab
diff --git a/tools/ds-identify b/tools/ds-identify
index 435a5bcb..dc7ac3a7 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -261,7 +261,7 @@ read_virt() {
 
 is_container() {
     case "${DI_VIRT}" in
-        lxc|lxc-libvirt|systemd-nspawn|docker|rkt) return 0;;
+        container-other|lxc|lxc-libvirt|systemd-nspawn|docker|rkt) return 0;;
         *) return 1;;
     esac
 }
@@ -990,12 +990,14 @@ dscheck_SmartOS() {
     # joyent cloud has two virt types: kvm and container
     # on kvm, product name on joyent public cloud shows 'SmartDC HVM'
     # on the container platform, uname's version has: BrandZ virtual linux
+    # for container, we also verify that the socketfile exists to protect
+    # against embedded containers (lxd running on brandz)
     local smartdc_kver="BrandZ virtual linux"
+    local metadata_sockfile="${PATH_ROOT}/native/.zonecontrol/metadata.sock"
     dmi_product_name_matches "SmartDC*" && return $DS_FOUND
-    if [ "${DI_UNAME_KERNEL_VERSION}" = "${smartdc_kver}" ] &&
-       [ "${DI_VIRT}" = "container-other" ]; then
-       return ${DS_FOUND}
-    fi
+    [ "${DI_UNAME_KERNEL_VERSION}" = "${smartdc_kver}" ] &&
+        [ -e "${metadata_sockfile}" ] &&
+        return ${DS_FOUND}
     return ${DS_NOT_FOUND}
 }
 
-- 
cgit v1.2.3


From 30e730f7ca111487d243ba9f40c66df6d7a49953 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Thu, 17 May 2018 14:59:54 -0600
Subject: read_file_or_url: move to url_helper, fix bug in its FileResponse.

The result of a read_file_or_url on a file and on a url would differ
in behavior.
  str(UrlResponse) would return UrlResponse.contents.decode('utf-8')
while
  str(FileResponse) would return str(FileResponse.contents)

The difference being "b'foo'" versus "foo".

As part of the general goal of cleaning util, move read_file_or_url
into url_helper.
---
 cloudinit/cmd/main.py                              |  2 +-
 cloudinit/config/cc_phone_home.py                  |  7 +--
 cloudinit/config/schema.py                         |  4 +-
 cloudinit/ec2_utils.py                             | 14 +++---
 cloudinit/sources/DataSourceMAAS.py                |  2 +-
 cloudinit/sources/helpers/azure.py                 |  5 ++-
 cloudinit/tests/test_url_helper.py                 | 28 +++++++++++-
 cloudinit/url_helper.py                            | 29 +++++++++++-
 cloudinit/user_data.py                             |  6 +--
 cloudinit/util.py                                  | 37 ++--------------
 tests/unittests/test__init__.py                    |  8 ++--
 .../unittests/test_datasource/test_azure_helper.py |  2 +-
 .../test_handler/test_handler_apt_conf_v1.py       | 16 +++----
 .../test_handler_apt_configure_sources_list_v1.py  |  7 ---
 .../test_handler/test_handler_apt_source_v1.py     | 27 +++++-------
 .../test_handler/test_handler_apt_source_v3.py     | 27 +++++-------
 tests/unittests/test_handler/test_handler_ntp.py   | 51 +++++++++-------------
 17 files changed, 131 insertions(+), 141 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index 3f2dbb93..d6ba90f4 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -187,7 +187,7 @@ def attempt_cmdline_url(path, network=True, cmdline=None):
     data = None
     header = b'#cloud-config'
     try:
-        resp = util.read_file_or_url(**kwargs)
+        resp = url_helper.read_file_or_url(**kwargs)
         if resp.ok():
             data = resp.contents
             if not resp.contents.startswith(header):
diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py
index 878069b7..3be0d1c1 100644
--- a/cloudinit/config/cc_phone_home.py
+++ b/cloudinit/config/cc_phone_home.py
@@ -41,6 +41,7 @@ keys to post. Available keys are:
 """
 
 from cloudinit import templater
+from cloudinit import url_helper
 from cloudinit import util
 
 from cloudinit.settings import PER_INSTANCE
@@ -136,9 +137,9 @@ def handle(name, cfg, cloud, log, args):
     }
     url = templater.render_string(url, url_params)
     try:
-        util.read_file_or_url(url, data=real_submit_keys,
-                              retries=tries, sec_between=3,
-                              ssl_details=util.fetch_ssl_details(cloud.paths))
+        url_helper.read_file_or_url(
+            url, data=real_submit_keys, retries=tries, sec_between=3,
+            ssl_details=util.fetch_ssl_details(cloud.paths))
     except Exception:
         util.logexc(log, "Failed to post phone home data to %s in %s tries",
                     url, tries)
diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py
index 76826e05..3bad8e22 100644
--- a/cloudinit/config/schema.py
+++ b/cloudinit/config/schema.py
@@ -4,7 +4,7 @@
 from __future__ import print_function
 
 from cloudinit import importer
-from cloudinit.util import find_modules, read_file_or_url
+from cloudinit.util import find_modules, load_file
 
 import argparse
 from collections import defaultdict
@@ -139,7 +139,7 @@ def validate_cloudconfig_file(config_path, schema, annotate=False):
     """
     if not os.path.exists(config_path):
         raise RuntimeError('Configfile {0} does not exist'.format(config_path))
-    content = read_file_or_url('file://{0}'.format(config_path)).contents
+    content = load_file(config_path, decode=False)
     if not content.startswith(CLOUD_CONFIG_HEADER):
         errors = (
             ('header', 'File {0} needs to begin with "{1}"'.format(
diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py
index dc3f0fc3..3b7b17f1 100644
--- a/cloudinit/ec2_utils.py
+++ b/cloudinit/ec2_utils.py
@@ -150,11 +150,9 @@ def get_instance_userdata(api_version='latest',
         # NOT_FOUND occurs) and just in that case returning an empty string.
         exception_cb = functools.partial(_skip_retry_on_codes,
                                          SKIP_USERDATA_CODES)
-        response = util.read_file_or_url(ud_url,
-                                         ssl_details=ssl_details,
-                                         timeout=timeout,
-                                         retries=retries,
-                                         exception_cb=exception_cb)
+        response = url_helper.read_file_or_url(
+            ud_url, ssl_details=ssl_details, timeout=timeout,
+            retries=retries, exception_cb=exception_cb)
         user_data = response.contents
     except url_helper.UrlError as e:
         if e.code not in SKIP_USERDATA_CODES:
@@ -169,9 +167,9 @@ def _get_instance_metadata(tree, api_version='latest',
                            ssl_details=None, timeout=5, retries=5,
                            leaf_decoder=None):
     md_url = url_helper.combine_url(metadata_address, api_version, tree)
-    caller = functools.partial(util.read_file_or_url,
-                               ssl_details=ssl_details, timeout=timeout,
-                               retries=retries)
+    caller = functools.partial(
+        url_helper.read_file_or_url, ssl_details=ssl_details,
+        timeout=timeout, retries=retries)
 
     def mcaller(url):
         return caller(url).contents
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index aa56addb..bcb38544 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -198,7 +198,7 @@ def read_maas_seed_url(seed_url, read_file_or_url=None, timeout=None,
     If version is None, then <version>/ will not be used.
     """
     if read_file_or_url is None:
-        read_file_or_url = util.read_file_or_url
+        read_file_or_url = url_helper.read_file_or_url
 
     if seed_url.endswith("/"):
         seed_url = seed_url[:-1]
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index 90c12df1..e5696b1f 100644
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -14,6 +14,7 @@ from cloudinit import temp_utils
 from contextlib import contextmanager
 from xml.etree import ElementTree
 
+from cloudinit import url_helper
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -55,14 +56,14 @@ class AzureEndpointHttpClient(object):
         if secure:
             headers = self.headers.copy()
             headers.update(self.extra_secure_headers)
-        return util.read_file_or_url(url, headers=headers)
+        return url_helper.read_file_or_url(url, headers=headers)
 
     def post(self, url, data=None, extra_headers=None):
         headers = self.headers
         if extra_headers is not None:
             headers = self.headers.copy()
             headers.update(extra_headers)
-        return util.read_file_or_url(url, data=data, headers=headers)
+        return url_helper.read_file_or_url(url, data=data, headers=headers)
 
 
 class GoalState(object):
diff --git a/cloudinit/tests/test_url_helper.py b/cloudinit/tests/test_url_helper.py
index b778a3a7..113249d9 100644
--- a/cloudinit/tests/test_url_helper.py
+++ b/cloudinit/tests/test_url_helper.py
@@ -1,7 +1,10 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-from cloudinit.url_helper import oauth_headers
+from cloudinit.url_helper import oauth_headers, read_file_or_url
 from cloudinit.tests.helpers import CiTestCase, mock, skipIf
+from cloudinit import util
+
+import httpretty
 
 
 try:
@@ -38,3 +41,26 @@ class TestOAuthHeaders(CiTestCase):
             'url', 'consumer_key', 'token_key', 'token_secret',
             'consumer_secret')
         self.assertEqual('url', return_value)
+
+
+class TestReadFileOrUrl(CiTestCase):
+    def test_read_file_or_url_str_from_file(self):
+        """Test that str(result.contents) on file is text version of contents.
+        It should not be "b'data'", but just "'data'" """
+        tmpf = self.tmp_path("myfile1")
+        data = b'This is my file content\n'
+        util.write_file(tmpf, data, omode="wb")
+        result = read_file_or_url("file://%s" % tmpf)
+        self.assertEqual(result.contents, data)
+        self.assertEqual(str(result), data.decode('utf-8'))
+
+    @httpretty.activate
+    def test_read_file_or_url_str_from_url(self):
+        """Test that str(result.contents) on url is text version of contents.
+        It should not be "b'data'", but just "'data'" """
+        url = 'http://hostname/path'
+        data = b'This is my url content\n'
+        httpretty.register_uri(httpretty.GET, url, data)
+        result = read_file_or_url(url)
+        self.assertEqual(result.contents, data)
+        self.assertEqual(str(result), data.decode('utf-8'))
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 1de07b1c..8067979e 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -15,6 +15,7 @@ import six
 import time
 
 from email.utils import parsedate
+from errno import ENOENT
 from functools import partial
 from itertools import count
 from requests import exceptions
@@ -80,6 +81,32 @@ def combine_url(base, *add_ons):
     return url
 
 
+def read_file_or_url(url, timeout=5, retries=10,
+                     headers=None, data=None, sec_between=1, ssl_details=None,
+                     headers_cb=None, exception_cb=None):
+    url = url.lstrip()
+    if url.startswith("/"):
+        url = "file://%s" % url
+    if url.lower().startswith("file://"):
+        if data:
+            LOG.warning("Unable to post data to file resource %s", url)
+        file_path = url[len("file://"):]
+        try:
+            with open(file_path, "rb") as fp:
+                contents = fp.read()
+        except IOError as e:
+            code = e.errno
+            if e.errno == ENOENT:
+                code = NOT_FOUND
+            raise UrlError(cause=e, code=code, headers=None, url=url)
+        return FileResponse(file_path, contents=contents)
+    else:
+        return readurl(url, timeout=timeout, retries=retries, headers=headers,
+                       headers_cb=headers_cb, data=data,
+                       sec_between=sec_between, ssl_details=ssl_details,
+                       exception_cb=exception_cb)
+
+
 # Made to have same accessors as UrlResponse so that the
 # read_file_or_url can return this or that object and the
 # 'user' of those objects will not need to know the difference.
@@ -96,7 +123,7 @@ class StringResponse(object):
         return True
 
     def __str__(self):
-        return self.contents
+        return self.contents.decode('utf-8')
 
 
 class FileResponse(StringResponse):
diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index cc55daf8..8f6aba1e 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -19,7 +19,7 @@ import six
 
 from cloudinit import handlers
 from cloudinit import log as logging
-from cloudinit.url_helper import UrlError
+from cloudinit.url_helper import read_file_or_url, UrlError
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -224,8 +224,8 @@ class UserDataProcessor(object):
                 content = util.load_file(include_once_fn)
             else:
                 try:
-                    resp = util.read_file_or_url(include_url,
-                                                 ssl_details=self.ssl_details)
+                    resp = read_file_or_url(include_url,
+                                            ssl_details=self.ssl_details)
                     if include_once_on and resp.ok():
                         util.write_file(include_once_fn, resp.contents,
                                         mode=0o600)
diff --git a/cloudinit/util.py b/cloudinit/util.py
index fc30018c..edfedc7d 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -857,37 +857,6 @@ def fetch_ssl_details(paths=None):
     return ssl_details
 
 
-def read_file_or_url(url, timeout=5, retries=10,
-                     headers=None, data=None, sec_between=1, ssl_details=None,
-                     headers_cb=None, exception_cb=None):
-    url = url.lstrip()
-    if url.startswith("/"):
-        url = "file://%s" % url
-    if url.lower().startswith("file://"):
-        if data:
-            LOG.warning("Unable to post data to file resource %s", url)
-        file_path = url[len("file://"):]
-        try:
-            contents = load_file(file_path, decode=False)
-        except IOError as e:
-            code = e.errno
-            if e.errno == ENOENT:
-                code = url_helper.NOT_FOUND
-            raise url_helper.UrlError(cause=e, code=code, headers=None,
-                                      url=url)
-        return url_helper.FileResponse(file_path, contents=contents)
-    else:
-        return url_helper.readurl(url,
-                                  timeout=timeout,
-                                  retries=retries,
-                                  headers=headers,
-                                  headers_cb=headers_cb,
-                                  data=data,
-                                  sec_between=sec_between,
-                                  ssl_details=ssl_details,
-                                  exception_cb=exception_cb)
-
-
 def load_yaml(blob, default=None, allowed=(dict,)):
     loaded = default
     blob = decode_binary(blob)
@@ -925,12 +894,14 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0):
         ud_url = "%s%s%s" % (base, "user-data", ext)
         md_url = "%s%s%s" % (base, "meta-data", ext)
 
-    md_resp = read_file_or_url(md_url, timeout, retries, file_retries)
+    md_resp = url_helper.read_file_or_url(md_url, timeout, retries,
+                                          file_retries)
     md = None
     if md_resp.ok():
         md = load_yaml(decode_binary(md_resp.contents), default={})
 
-    ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries)
+    ud_resp = url_helper.read_file_or_url(ud_url, timeout, retries,
+                                          file_retries)
     ud = None
     if ud_resp.ok():
         ud = ud_resp.contents
diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py
index f1ab02e9..739bbebf 100644
--- a/tests/unittests/test__init__.py
+++ b/tests/unittests/test__init__.py
@@ -182,7 +182,7 @@ class TestCmdlineUrl(CiTestCase):
         self.assertEqual(
             ('url', 'http://example.com'), main.parse_cmdline_url(cmdline))
 
-    @mock.patch('cloudinit.cmd.main.util.read_file_or_url')
+    @mock.patch('cloudinit.cmd.main.url_helper.read_file_or_url')
     def test_invalid_content(self, m_read):
         key = "cloud-config-url"
         url = 'http://example.com/foo'
@@ -196,7 +196,7 @@ class TestCmdlineUrl(CiTestCase):
         self.assertIn(url, msg)
         self.assertFalse(os.path.exists(fpath))
 
-    @mock.patch('cloudinit.cmd.main.util.read_file_or_url')
+    @mock.patch('cloudinit.cmd.main.url_helper.read_file_or_url')
     def test_valid_content(self, m_read):
         url = "http://example.com/foo"
         payload = b"#cloud-config\nmydata: foo\nbar: wark\n"
@@ -210,7 +210,7 @@ class TestCmdlineUrl(CiTestCase):
         self.assertEqual(logging.INFO, lvl)
         self.assertIn(url, msg)
 
-    @mock.patch('cloudinit.cmd.main.util.read_file_or_url')
+    @mock.patch('cloudinit.cmd.main.url_helper.read_file_or_url')
     def test_no_key_found(self, m_read):
         cmdline = "ro mykey=http://example.com/foo root=foo"
         fpath = self.tmp_path("ccpath")
@@ -221,7 +221,7 @@ class TestCmdlineUrl(CiTestCase):
         self.assertFalse(os.path.exists(fpath))
         self.assertEqual(logging.DEBUG, lvl)
 
-    @mock.patch('cloudinit.cmd.main.util.read_file_or_url')
+    @mock.patch('cloudinit.cmd.main.url_helper.read_file_or_url')
     def test_exception_warns(self, m_read):
         url = "http://example.com/foo"
         cmdline = "ro cloud-config-url=%s root=LABEL=bar" % url
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
index b42b073f..af9d3e1a 100644
--- a/tests/unittests/test_datasource/test_azure_helper.py
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -195,7 +195,7 @@ class TestAzureEndpointHttpClient(CiTestCase):
         self.addCleanup(patches.close)
 
         self.read_file_or_url = patches.enter_context(
-            mock.patch.object(azure_helper.util, 'read_file_or_url'))
+            mock.patch.object(azure_helper.url_helper, 'read_file_or_url'))
 
     def test_non_secure_get(self):
         client = azure_helper.AzureEndpointHttpClient(mock.MagicMock())
diff --git a/tests/unittests/test_handler/test_handler_apt_conf_v1.py b/tests/unittests/test_handler/test_handler_apt_conf_v1.py
index 83f962a9..6a4b03ee 100644
--- a/tests/unittests/test_handler/test_handler_apt_conf_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_conf_v1.py
@@ -12,10 +12,6 @@ import shutil
 import tempfile
 
 
-def load_tfile_or_url(*args, **kwargs):
-    return(util.decode_binary(util.read_file_or_url(*args, **kwargs).contents))
-
-
 class TestAptProxyConfig(TestCase):
     def setUp(self):
         super(TestAptProxyConfig, self).setUp()
@@ -36,7 +32,7 @@ class TestAptProxyConfig(TestCase):
         self.assertTrue(os.path.isfile(self.pfile))
         self.assertFalse(os.path.isfile(self.cfile))
 
-        contents = load_tfile_or_url(self.pfile)
+        contents = util.load_file(self.pfile)
         self.assertTrue(self._search_apt_config(contents, "http", "myproxy"))
 
     def test_apt_http_proxy_written(self):
@@ -46,7 +42,7 @@ class TestAptProxyConfig(TestCase):
         self.assertTrue(os.path.isfile(self.pfile))
         self.assertFalse(os.path.isfile(self.cfile))
 
-        contents = load_tfile_or_url(self.pfile)
+        contents = util.load_file(self.pfile)
         self.assertTrue(self._search_apt_config(contents, "http", "myproxy"))
 
     def test_apt_all_proxy_written(self):
@@ -64,7 +60,7 @@ class TestAptProxyConfig(TestCase):
         self.assertTrue(os.path.isfile(self.pfile))
         self.assertFalse(os.path.isfile(self.cfile))
 
-        contents = load_tfile_or_url(self.pfile)
+        contents = util.load_file(self.pfile)
 
         for ptype, pval in values.items():
             self.assertTrue(self._search_apt_config(contents, ptype, pval))
@@ -80,7 +76,7 @@ class TestAptProxyConfig(TestCase):
         cc_apt_configure.apply_apt_config({'proxy': "foo"},
                                           self.pfile, self.cfile)
         self.assertTrue(os.path.isfile(self.pfile))
-        contents = load_tfile_or_url(self.pfile)
+        contents = util.load_file(self.pfile)
         self.assertTrue(self._search_apt_config(contents, "http", "foo"))
 
     def test_config_written(self):
@@ -92,14 +88,14 @@ class TestAptProxyConfig(TestCase):
         self.assertTrue(os.path.isfile(self.cfile))
         self.assertFalse(os.path.isfile(self.pfile))
 
-        self.assertEqual(load_tfile_or_url(self.cfile), payload)
+        self.assertEqual(util.load_file(self.cfile), payload)
 
     def test_config_replaced(self):
         util.write_file(self.pfile, "content doesnt matter")
         cc_apt_configure.apply_apt_config({'conf': "foo"},
                                           self.pfile, self.cfile)
         self.assertTrue(os.path.isfile(self.cfile))
-        self.assertEqual(load_tfile_or_url(self.cfile), "foo")
+        self.assertEqual(util.load_file(self.cfile), "foo")
 
     def test_config_deleted(self):
         # if no 'conf' is provided, delete any previously written file
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
index d2b96f0b..23bd6e10 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
@@ -64,13 +64,6 @@ deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted
 """)
 
 
-def load_tfile_or_url(*args, **kwargs):
-    """load_tfile_or_url
-    load file and return content after decoding
-    """
-    return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)
-
-
 class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
     """TestAptSourceConfigSourceList
     Main Class to test sources list rendering
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py
index 46ca4ce4..a3132fbd 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py
@@ -39,13 +39,6 @@ S0ORP6HXET3+jC8BMG4tBWCTK/XEZw==
 ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
 
 
-def load_tfile_or_url(*args, **kwargs):
-    """load_tfile_or_url
-    load file and return content after decoding
-    """
-    return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)
-
-
 class FakeDistro(object):
     """Fake Distro helper object"""
     def update_package_sources(self):
@@ -125,7 +118,7 @@ class TestAptSourceConfig(TestCase):
 
         self.assertTrue(os.path.isfile(filename))
 
-        contents = load_tfile_or_url(filename)
+        contents = util.load_file(filename)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", "http://archive.ubuntu.com/ubuntu",
                                    "karmic-backports",
@@ -157,13 +150,13 @@ class TestAptSourceConfig(TestCase):
         self.apt_src_basic(self.aptlistfile, cfg)
 
         # extra verify on two extra files of this test
-        contents = load_tfile_or_url(self.aptlistfile2)
+        contents = util.load_file(self.aptlistfile2)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", "http://archive.ubuntu.com/ubuntu",
                                    "precise-backports",
                                    "main universe multiverse restricted"),
                                   contents, flags=re.IGNORECASE))
-        contents = load_tfile_or_url(self.aptlistfile3)
+        contents = util.load_file(self.aptlistfile3)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", "http://archive.ubuntu.com/ubuntu",
                                    "lucid-backports",
@@ -220,7 +213,7 @@ class TestAptSourceConfig(TestCase):
 
         self.assertTrue(os.path.isfile(filename))
 
-        contents = load_tfile_or_url(filename)
+        contents = util.load_file(filename)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", params['MIRROR'], params['RELEASE'],
                                    "multiverse"),
@@ -241,12 +234,12 @@ class TestAptSourceConfig(TestCase):
 
         # extra verify on two extra files of this test
         params = self._get_default_params()
-        contents = load_tfile_or_url(self.aptlistfile2)
+        contents = util.load_file(self.aptlistfile2)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", params['MIRROR'], params['RELEASE'],
                                    "main"),
                                   contents, flags=re.IGNORECASE))
-        contents = load_tfile_or_url(self.aptlistfile3)
+        contents = util.load_file(self.aptlistfile3)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", params['MIRROR'], params['RELEASE'],
                                    "universe"),
@@ -296,7 +289,7 @@ class TestAptSourceConfig(TestCase):
 
         self.assertTrue(os.path.isfile(filename))
 
-        contents = load_tfile_or_url(filename)
+        contents = util.load_file(filename)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb",
                                    ('http://ppa.launchpad.net/smoser/'
@@ -336,14 +329,14 @@ class TestAptSourceConfig(TestCase):
                 'filename': self.aptlistfile3}
 
         self.apt_src_keyid(self.aptlistfile, [cfg1, cfg2, cfg3], 3)
-        contents = load_tfile_or_url(self.aptlistfile2)
+        contents = util.load_file(self.aptlistfile2)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb",
                                    ('http://ppa.launchpad.net/smoser/'
                                     'cloud-init-test/ubuntu'),
                                    "xenial", "universe"),
                                   contents, flags=re.IGNORECASE))
-        contents = load_tfile_or_url(self.aptlistfile3)
+        contents = util.load_file(self.aptlistfile3)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb",
                                    ('http://ppa.launchpad.net/smoser/'
@@ -375,7 +368,7 @@ class TestAptSourceConfig(TestCase):
 
         self.assertTrue(os.path.isfile(filename))
 
-        contents = load_tfile_or_url(filename)
+        contents = util.load_file(filename)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb",
                                    ('http://ppa.launchpad.net/smoser/'
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py
index e486862d..7a64c230 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v3.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py
@@ -49,13 +49,6 @@ ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
 TARGET = None
 
 
-def load_tfile(*args, **kwargs):
-    """load_tfile_or_url
-    load file and return content after decoding
-    """
-    return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)
-
-
 class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
     """TestAptSourceConfig
     Main Class to test apt configs
@@ -119,7 +112,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
 
         self.assertTrue(os.path.isfile(filename))
 
-        contents = load_tfile(filename)
+        contents = util.load_file(filename)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", "http://test.ubuntu.com/ubuntu",
                                    "karmic-backports",
@@ -151,13 +144,13 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
         self._apt_src_basic(self.aptlistfile, cfg)
 
         # extra verify on two extra files of this test
-        contents = load_tfile(self.aptlistfile2)
+        contents = util.load_file(self.aptlistfile2)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", "http://test.ubuntu.com/ubuntu",
                                    "precise-backports",
                                    "main universe multiverse restricted"),
                                   contents, flags=re.IGNORECASE))
-        contents = load_tfile(self.aptlistfile3)
+        contents = util.load_file(self.aptlistfile3)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", "http://test.ubuntu.com/ubuntu",
                                    "lucid-backports",
@@ -174,7 +167,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
 
         self.assertTrue(os.path.isfile(filename))
 
-        contents = load_tfile(filename)
+        contents = util.load_file(filename)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", params['MIRROR'], params['RELEASE'],
                                    "multiverse"),
@@ -201,12 +194,12 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
 
         # extra verify on two extra files of this test
         params = self._get_default_params()
-        contents = load_tfile(self.aptlistfile2)
+        contents = util.load_file(self.aptlistfile2)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", params['MIRROR'], params['RELEASE'],
                                    "main"),
                                   contents, flags=re.IGNORECASE))
-        contents = load_tfile(self.aptlistfile3)
+        contents = util.load_file(self.aptlistfile3)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb", params['MIRROR'], params['RELEASE'],
                                    "universe"),
@@ -240,7 +233,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
 
         self.assertTrue(os.path.isfile(filename))
 
-        contents = load_tfile(filename)
+        contents = util.load_file(filename)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb",
                                    ('http://ppa.launchpad.net/smoser/'
@@ -277,14 +270,14 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
                                    'keyid': "03683F77"}}
 
         self._apt_src_keyid(self.aptlistfile, cfg, 3)
-        contents = load_tfile(self.aptlistfile2)
+        contents = util.load_file(self.aptlistfile2)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb",
                                    ('http://ppa.launchpad.net/smoser/'
                                     'cloud-init-test/ubuntu'),
                                    "xenial", "universe"),
                                   contents, flags=re.IGNORECASE))
-        contents = load_tfile(self.aptlistfile3)
+        contents = util.load_file(self.aptlistfile3)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb",
                                    ('http://ppa.launchpad.net/smoser/'
@@ -310,7 +303,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
 
         self.assertTrue(os.path.isfile(self.aptlistfile))
 
-        contents = load_tfile(self.aptlistfile)
+        contents = util.load_file(self.aptlistfile)
         self.assertTrue(re.search(r"%s %s %s %s\n" %
                                   ("deb",
                                    ('http://ppa.launchpad.net/smoser/'
diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
index 6da4564e..6fe3659d 100644
--- a/tests/unittests/test_handler/test_handler_ntp.py
+++ b/tests/unittests/test_handler/test_handler_ntp.py
@@ -155,9 +155,9 @@ class TestNtp(FilesystemMockingTestCase):
                                              path=confpath,
                                              template_fn=template_fn,
                                              template=None)
-        content = util.read_file_or_url('file://' + confpath).contents
         self.assertEqual(
-            "servers []\npools ['10.0.0.1', '10.0.0.2']\n", content.decode())
+            "servers []\npools ['10.0.0.1', '10.0.0.2']\n",
+            util.load_file(confpath))
 
     def test_write_ntp_config_template_defaults_pools_w_empty_lists(self):
         """write_ntp_config_template defaults pools servers upon empty config.
@@ -176,10 +176,9 @@ class TestNtp(FilesystemMockingTestCase):
                                              path=confpath,
                                              template_fn=template_fn,
                                              template=None)
-        content = util.read_file_or_url('file://' + confpath).contents
         self.assertEqual(
             "servers []\npools {0}\n".format(pools),
-            content.decode())
+            util.load_file(confpath))
 
     def test_defaults_pools_empty_lists_sles(self):
         """write_ntp_config_template defaults opensuse pools upon empty config.
@@ -196,11 +195,11 @@ class TestNtp(FilesystemMockingTestCase):
                                          path=confpath,
                                          template_fn=template_fn,
                                          template=None)
-        content = util.read_file_or_url('file://' + confpath).contents
         for pool in default_pools:
             self.assertIn('opensuse', pool)
         self.assertEqual(
-            "servers []\npools {0}\n".format(default_pools), content.decode())
+            "servers []\npools {0}\n".format(default_pools),
+            util.load_file(confpath))
         self.assertIn(
             "Adding distro default ntp pool servers: {0}".format(
                 ",".join(default_pools)),
@@ -217,10 +216,9 @@ class TestNtp(FilesystemMockingTestCase):
                                          path=confpath,
                                          template_fn=template_fn,
                                          template=None)
-        content = util.read_file_or_url('file://' + confpath).contents
         self.assertEqual(
             "[Time]\nNTP=%s %s \n" % (" ".join(servers), " ".join(pools)),
-            content.decode())
+            util.load_file(confpath))
 
     def test_distro_ntp_client_configs(self):
         """Test we have updated ntp client configs on different distros"""
@@ -267,17 +265,17 @@ class TestNtp(FilesystemMockingTestCase):
                 cc_ntp.write_ntp_config_template(distro, servers=servers,
                                                  pools=pools, path=confpath,
                                                  template_fn=template_fn)
-                content = util.read_file_or_url('file://' + confpath).contents
+                content = util.load_file(confpath)
                 if client in ['ntp', 'chrony']:
                     expected_servers = '\n'.join([
                         'server {0} iburst'.format(srv) for srv in servers])
                     print('distro=%s client=%s' % (distro, client))
-                    self.assertIn(expected_servers, content.decode('utf-8'),
+                    self.assertIn(expected_servers, content,
                                   ('failed to render {0} conf'
                                    ' for distro:{1}'.format(client, distro)))
                     expected_pools = '\n'.join([
                         'pool {0} iburst'.format(pool) for pool in pools])
-                    self.assertIn(expected_pools, content.decode('utf-8'),
+                    self.assertIn(expected_pools, content,
                                   ('failed to render {0} conf'
                                    ' for distro:{1}'.format(client, distro)))
                 elif client == 'systemd-timesyncd':
@@ -286,7 +284,7 @@ class TestNtp(FilesystemMockingTestCase):
                         "# See timesyncd.conf(5) for details.\n\n" +
                         "[Time]\nNTP=%s %s \n" % (" ".join(servers),
                                                   " ".join(pools)))
-                    self.assertEqual(expected_content, content.decode())
+                    self.assertEqual(expected_content, content)
 
     def test_no_ntpcfg_does_nothing(self):
         """When no ntp section is defined handler logs a warning and noops."""
@@ -308,10 +306,10 @@ class TestNtp(FilesystemMockingTestCase):
                 confpath = ntpconfig['confpath']
                 m_select.return_value = ntpconfig
                 cc_ntp.handle('cc_ntp', valid_empty_config, mycloud, None, [])
-                content = util.read_file_or_url('file://' + confpath).contents
                 pools = cc_ntp.generate_server_names(mycloud.distro.name)
                 self.assertEqual(
-                    "servers []\npools {0}\n".format(pools), content.decode())
+                    "servers []\npools {0}\n".format(pools),
+                    util.load_file(confpath))
             self.assertNotIn('Invalid config:', self.logs.getvalue())
 
     @skipUnlessJsonSchema()
@@ -333,9 +331,8 @@ class TestNtp(FilesystemMockingTestCase):
                 "Invalid config:\nntp.pools.0: 123 is not of type 'string'\n"
                 "ntp.servers.1: None is not of type 'string'",
                 self.logs.getvalue())
-            content = util.read_file_or_url('file://' + confpath).contents
             self.assertEqual("servers ['valid', None]\npools [123]\n",
-                             content.decode())
+                             util.load_file(confpath))
 
     @skipUnlessJsonSchema()
     @mock.patch('cloudinit.config.cc_ntp.select_ntp_client')
@@ -357,9 +354,8 @@ class TestNtp(FilesystemMockingTestCase):
                 "Invalid config:\nntp.pools: 123 is not of type 'array'\n"
                 "ntp.servers: 'non-array' is not of type 'array'",
                 self.logs.getvalue())
-            content = util.read_file_or_url('file://' + confpath).contents
             self.assertEqual("servers non-array\npools 123\n",
-                             content.decode())
+                             util.load_file(confpath))
 
     @skipUnlessJsonSchema()
     @mock.patch('cloudinit.config.cc_ntp.select_ntp_client')
@@ -381,10 +377,9 @@ class TestNtp(FilesystemMockingTestCase):
                 "Invalid config:\nntp: Additional properties are not allowed "
                 "('invalidkey' was unexpected)",
                 self.logs.getvalue())
-            content = util.read_file_or_url('file://' + confpath).contents
             self.assertEqual(
                 "servers []\npools ['0.mycompany.pool.ntp.org']\n",
-                content.decode())
+                util.load_file(confpath))
 
     @skipUnlessJsonSchema()
     @mock.patch('cloudinit.config.cc_ntp.select_ntp_client')
@@ -407,10 +402,10 @@ class TestNtp(FilesystemMockingTestCase):
                 " has non-unique elements\nntp.servers: "
                 "['10.0.0.1', '10.0.0.1'] has non-unique elements",
                 self.logs.getvalue())
-            content = util.read_file_or_url('file://' + confpath).contents
             self.assertEqual(
                 "servers ['10.0.0.1', '10.0.0.1']\n"
-                "pools ['0.mypool.org', '0.mypool.org']\n", content.decode())
+                "pools ['0.mypool.org', '0.mypool.org']\n",
+                util.load_file(confpath))
 
     @mock.patch('cloudinit.config.cc_ntp.select_ntp_client')
     def test_ntp_handler_timesyncd(self, m_select):
@@ -426,10 +421,9 @@ class TestNtp(FilesystemMockingTestCase):
             confpath = ntpconfig['confpath']
             m_select.return_value = ntpconfig
             cc_ntp.handle('cc_ntp', cfg, mycloud, None, [])
-            content = util.read_file_or_url('file://' + confpath).contents
             self.assertEqual(
                 "[Time]\nNTP=192.168.2.1 192.168.2.2 0.mypool.org \n",
-                content.decode())
+                util.load_file(confpath))
 
     @mock.patch('cloudinit.config.cc_ntp.select_ntp_client')
     def test_ntp_handler_enabled_false(self, m_select):
@@ -466,10 +460,9 @@ class TestNtp(FilesystemMockingTestCase):
                 m_util.subp.assert_called_with(
                     ['systemctl', 'reload-or-restart',
                      service_name], capture=True)
-            content = util.read_file_or_url('file://' + confpath).contents
             self.assertEqual(
                 "servers []\npools {0}\n".format(pools),
-                content.decode())
+                util.load_file(confpath))
 
     def test_opensuse_picks_chrony(self):
         """Test opensuse picks chrony or ntp on certain distro versions"""
@@ -638,10 +631,9 @@ class TestNtp(FilesystemMockingTestCase):
             mock_path = 'cloudinit.config.cc_ntp.temp_utils._TMPDIR'
             with mock.patch(mock_path, self.new_root):
                 cc_ntp.handle('notimportant', cfg, mycloud, None, None)
-            content = util.read_file_or_url('file://' + confpath).contents
             self.assertEqual(
                 "servers []\npools ['mypool.org']\n%s" % custom,
-                content.decode())
+                util.load_file(confpath))
 
     @mock.patch('cloudinit.config.cc_ntp.supplemental_schema_validation')
     @mock.patch('cloudinit.config.cc_ntp.reload_ntp')
@@ -675,10 +667,9 @@ class TestNtp(FilesystemMockingTestCase):
             with mock.patch(mock_path, self.new_root):
                 cc_ntp.handle('notimportant',
                               {'ntp': cfg}, mycloud, None, None)
-            content = util.read_file_or_url('file://' + confpath).contents
             self.assertEqual(
                 "servers []\npools ['mypool.org']\n%s" % custom,
-                content.decode())
+                util.load_file(confpath))
         m_schema.assert_called_with(expected_merged_cfg)
 
 
-- 
cgit v1.2.3


From aa4eeb80839382117e1813e396dc53aa634fd7ba Mon Sep 17 00:00:00 2001
From: Paul Meyer <paulmey@microsoft.com>
Date: Wed, 23 May 2018 15:45:39 -0400
Subject: Azure: Ignore NTFS mount errors when checking ephemeral drive

The Azure data source provides a method to check whether a NTFS partition
on the ephemeral disk is safe for reformatting to ext4. The method checks
to see if there are customer data files on the disk. However, mounting
the partition fails on systems that do not have the capability of
mounting NTFS. Note that in this case, it is also very unlikely that the
NTFS partition would have been used by the system (since it can't mount
it). The only case would be where an update to the system removed the
capability to mount NTFS, the likelihood of which is also very small.
This change allows the reformatting of the ephemeral disk to ext4 on
systems where mounting NTFS is not supported.
---
 cloudinit/sources/DataSourceAzure.py          |  63 ++++++++++++----
 cloudinit/util.py                             |   5 +-
 tests/unittests/test_datasource/test_azure.py | 105 +++++++++++++++++++++-----
 3 files changed, 138 insertions(+), 35 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 1b03d460..7007d9ea 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -208,6 +208,7 @@ BUILTIN_CLOUD_CONFIG = {
 }
 
 DS_CFG_PATH = ['datasource', DS_NAME]
+DS_CFG_KEY_PRESERVE_NTFS = 'never_destroy_ntfs'
 DEF_EPHEMERAL_LABEL = 'Temporary Storage'
 
 # The redacted password fails to meet password complexity requirements
@@ -394,14 +395,9 @@ class DataSourceAzure(sources.DataSource):
         if found == ddir:
             LOG.debug("using files cached in %s", ddir)
 
-        # azure / hyper-v provides random data here
-        # TODO. find the seed on FreeBSD platform
-        # now update ds_cfg to reflect contents pass in config
-        if not util.is_FreeBSD():
-            seed = util.load_file("/sys/firmware/acpi/tables/OEM0",
-                                  quiet=True, decode=False)
-            if seed:
-                self.metadata['random_seed'] = seed
+        seed = _get_random_seed()
+        if seed:
+            self.metadata['random_seed'] = seed
 
         user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {})
         self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg])
@@ -539,7 +535,9 @@ class DataSourceAzure(sources.DataSource):
         return fabric_data
 
     def activate(self, cfg, is_new_instance):
-        address_ephemeral_resize(is_new_instance=is_new_instance)
+        address_ephemeral_resize(is_new_instance=is_new_instance,
+                                 preserve_ntfs=self.ds_cfg.get(
+                                     DS_CFG_KEY_PRESERVE_NTFS, False))
         return
 
     @property
@@ -583,17 +581,29 @@ def _has_ntfs_filesystem(devpath):
     return os.path.realpath(devpath) in ntfs_devices
 
 
-def can_dev_be_reformatted(devpath):
-    """Determine if block device devpath is newly formatted ephemeral.
+def can_dev_be_reformatted(devpath, preserve_ntfs):
+    """Determine if the ephemeral drive at devpath should be reformatted.
 
-    A newly formatted disk will:
+    A fresh ephemeral disk is formatted by Azure and will:
       a.) have a partition table (dos or gpt)
       b.) have 1 partition that is ntfs formatted, or
           have 2 partitions with the second partition ntfs formatted.
           (larger instances with >2TB ephemeral disk have gpt, and will
            have a microsoft reserved partition as part 1.  LP: #1686514)
       c.) the ntfs partition will have no files other than possibly
-          'dataloss_warning_readme.txt'"""
+          'dataloss_warning_readme.txt'
+
+    User can indicate that NTFS should never be destroyed by setting
+    DS_CFG_KEY_PRESERVE_NTFS in dscfg.
+    If data is found on NTFS, user is warned to set DS_CFG_KEY_PRESERVE_NTFS
+    to make sure cloud-init does not accidentally wipe their data.
+    If cloud-init cannot mount the disk to check for data, destruction
+    will be allowed, unless the dscfg key is set."""
+    if preserve_ntfs:
+        msg = ('config says to never destroy NTFS (%s.%s), skipping checks' %
+               (".".join(DS_CFG_PATH), DS_CFG_KEY_PRESERVE_NTFS))
+        return False, msg
+
     if not os.path.exists(devpath):
         return False, 'device %s does not exist' % devpath
 
@@ -626,18 +636,27 @@ def can_dev_be_reformatted(devpath):
     bmsg = ('partition %s (%s) on device %s was ntfs formatted' %
             (cand_part, cand_path, devpath))
     try:
-        file_count = util.mount_cb(cand_path, count_files)
+        file_count = util.mount_cb(cand_path, count_files, mtype="ntfs",
+                                   update_env_for_mount={'LANG': 'C'})
     except util.MountFailedError as e:
+        if "mount: unknown filesystem type 'ntfs'" in str(e):
+            return True, (bmsg + ' but this system cannot mount NTFS,'
+                          ' assuming there are no important files.'
+                          ' Formatting allowed.')
         return False, bmsg + ' but mount of %s failed: %s' % (cand_part, e)
 
     if file_count != 0:
+        LOG.warning("it looks like you're using NTFS on the ephemeral disk, "
+                    'to ensure that filesystem does not get wiped, set '
+                    '%s.%s in config', '.'.join(DS_CFG_PATH),
+                    DS_CFG_KEY_PRESERVE_NTFS)
         return False, bmsg + ' but had %d files on it.' % file_count
 
     return True, bmsg + ' and had no important files. Safe for reformatting.'
 
 
 def address_ephemeral_resize(devpath=RESOURCE_DISK_PATH, maxwait=120,
-                             is_new_instance=False):
+                             is_new_instance=False, preserve_ntfs=False):
     # wait for ephemeral disk to come up
     naplen = .2
     missing = util.wait_for_files([devpath], maxwait=maxwait, naplen=naplen,
@@ -653,7 +672,7 @@ def address_ephemeral_resize(devpath=RESOURCE_DISK_PATH, maxwait=120,
     if is_new_instance:
         result, msg = (True, "First instance boot.")
     else:
-        result, msg = can_dev_be_reformatted(devpath)
+        result, msg = can_dev_be_reformatted(devpath, preserve_ntfs)
 
     LOG.debug("reformattable=%s: %s", result, msg)
     if not result:
@@ -967,6 +986,18 @@ def _check_freebsd_cdrom(cdrom_dev):
     return False
 
 
+def _get_random_seed():
+    """Return content random seed file if available, otherwise,
+       return None."""
+    # azure / hyper-v provides random data here
+    # TODO. find the seed on FreeBSD platform
+    # now update ds_cfg to reflect contents pass in config
+    if util.is_FreeBSD():
+        return None
+    return util.load_file("/sys/firmware/acpi/tables/OEM0",
+                          quiet=True, decode=False)
+
+
 def list_possible_azure_ds_devs():
     devlist = []
     if util.is_FreeBSD():
diff --git a/cloudinit/util.py b/cloudinit/util.py
index edfedc7d..653ed6ea 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -1581,7 +1581,8 @@ def mounts():
     return mounted
 
 
-def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True):
+def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True,
+             update_env_for_mount=None):
     """
     Mount the device, call method 'callback' passing the directory
     in which it was mounted, then unmount.  Return whatever 'callback'
@@ -1643,7 +1644,7 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True):
                         mountcmd.extend(['-t', mtype])
                     mountcmd.append(device)
                     mountcmd.append(tmpd)
-                    subp(mountcmd)
+                    subp(mountcmd, update_env=update_env_for_mount)
                     umount = tmpd  # This forces it to be unmounted (when set)
                     mountpoint = tmpd
                     break
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 26e8d7d3..e82716eb 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -1,10 +1,10 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit import helpers
-from cloudinit.util import b64e, decode_binary, load_file, write_file
 from cloudinit.sources import DataSourceAzure as dsaz
-from cloudinit.util import find_freebsd_part
-from cloudinit.util import get_path_dev_freebsd
+from cloudinit.util import (b64e, decode_binary, load_file, write_file,
+                            find_freebsd_part, get_path_dev_freebsd,
+                            MountFailedError)
 from cloudinit.version import version_string as vs
 from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock,
                                      ExitStack, PY26, SkipTest)
@@ -95,6 +95,8 @@ class TestAzureDataSource(CiTestCase):
         self.patches = ExitStack()
         self.addCleanup(self.patches.close)
 
+        self.patches.enter_context(mock.patch.object(dsaz, '_get_random_seed'))
+
         super(TestAzureDataSource, self).setUp()
 
     def apply_patches(self, patches):
@@ -335,6 +337,18 @@ fdescfs            /dev/fd          fdescfs rw              0 0
         self.assertTrue(ret)
         self.assertEqual(data['agent_invoked'], '_COMMAND')
 
+    def test_sys_cfg_set_never_destroy_ntfs(self):
+        sys_cfg = {'datasource': {'Azure': {
+            'never_destroy_ntfs': 'user-supplied-value'}}}
+        data = {'ovfcontent': construct_valid_ovf_env(data={}),
+                'sys_cfg': sys_cfg}
+
+        dsrc = self._get_ds(data)
+        ret = self._get_and_setup(dsrc)
+        self.assertTrue(ret)
+        self.assertEqual(dsrc.ds_cfg.get(dsaz.DS_CFG_KEY_PRESERVE_NTFS),
+                         'user-supplied-value')
+
     def test_username_used(self):
         odata = {'HostName': "myhost", 'UserName': "myuser"}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
@@ -676,6 +690,8 @@ class TestAzureBounce(CiTestCase):
                               mock.MagicMock(return_value={})))
         self.patches.enter_context(
             mock.patch.object(dsaz.util, 'which', lambda x: True))
+        self.patches.enter_context(
+            mock.patch.object(dsaz, '_get_random_seed'))
 
         def _dmi_mocks(key):
             if key == 'system-uuid':
@@ -957,7 +973,9 @@ class TestCanDevBeReformatted(CiTestCase):
             # return sorted by partition number
             return sorted(ret, key=lambda d: d[0])
 
-        def mount_cb(device, callback):
+        def mount_cb(device, callback, mtype, update_env_for_mount):
+            self.assertEqual('ntfs', mtype)
+            self.assertEqual('C', update_env_for_mount.get('LANG'))
             p = self.tmp_dir()
             for f in bypath.get(device).get('files', []):
                 write_file(os.path.join(p, f), content=f)
@@ -988,14 +1006,16 @@ class TestCanDevBeReformatted(CiTestCase):
                     '/dev/sda2': {'num': 2},
                     '/dev/sda3': {'num': 3},
                 }}})
-        value, msg = dsaz.can_dev_be_reformatted("/dev/sda")
+        value, msg = dsaz.can_dev_be_reformatted("/dev/sda",
+                                                 preserve_ntfs=False)
         self.assertFalse(value)
         self.assertIn("3 or more", msg.lower())
 
     def test_no_partitions_is_false(self):
         """A disk with no partitions can not be formatted."""
         self.patchup({'/dev/sda': {}})
-        value, msg = dsaz.can_dev_be_reformatted("/dev/sda")
+        value, msg = dsaz.can_dev_be_reformatted("/dev/sda",
+                                                 preserve_ntfs=False)
         self.assertFalse(value)
         self.assertIn("not partitioned", msg.lower())
 
@@ -1007,7 +1027,8 @@ class TestCanDevBeReformatted(CiTestCase):
                     '/dev/sda1': {'num': 1},
                     '/dev/sda2': {'num': 2, 'fs': 'ext4', 'files': []},
                 }}})
-        value, msg = dsaz.can_dev_be_reformatted("/dev/sda")
+        value, msg = dsaz.can_dev_be_reformatted("/dev/sda",
+                                                 preserve_ntfs=False)
         self.assertFalse(value)
         self.assertIn("not ntfs", msg.lower())
 
@@ -1020,7 +1041,8 @@ class TestCanDevBeReformatted(CiTestCase):
                     '/dev/sda2': {'num': 2, 'fs': 'ntfs',
                                   'files': ['secret.txt']},
                 }}})
-        value, msg = dsaz.can_dev_be_reformatted("/dev/sda")
+        value, msg = dsaz.can_dev_be_reformatted("/dev/sda",
+                                                 preserve_ntfs=False)
         self.assertFalse(value)
         self.assertIn("files on it", msg.lower())
 
@@ -1032,7 +1054,8 @@ class TestCanDevBeReformatted(CiTestCase):
                     '/dev/sda1': {'num': 1},
                     '/dev/sda2': {'num': 2, 'fs': 'ntfs', 'files': []},
                 }}})
-        value, msg = dsaz.can_dev_be_reformatted("/dev/sda")
+        value, msg = dsaz.can_dev_be_reformatted("/dev/sda",
+                                                 preserve_ntfs=False)
         self.assertTrue(value)
         self.assertIn("safe for", msg.lower())
 
@@ -1043,7 +1066,8 @@ class TestCanDevBeReformatted(CiTestCase):
                 'partitions': {
                     '/dev/sda1': {'num': 1, 'fs': 'zfs'},
                 }}})
-        value, msg = dsaz.can_dev_be_reformatted("/dev/sda")
+        value, msg = dsaz.can_dev_be_reformatted("/dev/sda",
+                                                 preserve_ntfs=False)
         self.assertFalse(value)
         self.assertIn("not ntfs", msg.lower())
 
@@ -1055,9 +1079,14 @@ class TestCanDevBeReformatted(CiTestCase):
                     '/dev/sda1': {'num': 1, 'fs': 'ntfs',
                                   'files': ['file1.txt', 'file2.exe']},
                 }}})
-        value, msg = dsaz.can_dev_be_reformatted("/dev/sda")
-        self.assertFalse(value)
-        self.assertIn("files on it", msg.lower())
+        with mock.patch.object(dsaz.LOG, 'warning') as warning:
+            value, msg = dsaz.can_dev_be_reformatted("/dev/sda",
+                                                     preserve_ntfs=False)
+            wmsg = warning.call_args[0][0]
+            self.assertIn("looks like you're using NTFS on the ephemeral disk",
+                          wmsg)
+            self.assertFalse(value)
+            self.assertIn("files on it", msg.lower())
 
     def test_one_partition_ntfs_empty_is_true(self):
         """1 mountable ntfs partition and no files can be formatted."""
@@ -1066,7 +1095,8 @@ class TestCanDevBeReformatted(CiTestCase):
                 'partitions': {
                     '/dev/sda1': {'num': 1, 'fs': 'ntfs', 'files': []}
                 }}})
-        value, msg = dsaz.can_dev_be_reformatted("/dev/sda")
+        value, msg = dsaz.can_dev_be_reformatted("/dev/sda",
+                                                 preserve_ntfs=False)
         self.assertTrue(value)
         self.assertIn("safe for", msg.lower())
 
@@ -1078,7 +1108,8 @@ class TestCanDevBeReformatted(CiTestCase):
                     '/dev/sda1': {'num': 1, 'fs': 'ntfs',
                                   'files': ['dataloss_warning_readme.txt']}
                 }}})
-        value, msg = dsaz.can_dev_be_reformatted("/dev/sda")
+        value, msg = dsaz.can_dev_be_reformatted("/dev/sda",
+                                                 preserve_ntfs=False)
         self.assertTrue(value)
         self.assertIn("safe for", msg.lower())
 
@@ -1093,7 +1124,8 @@ class TestCanDevBeReformatted(CiTestCase):
                         'num': 1, 'fs': 'ntfs', 'files': [self.warning_file],
                         'realpath': '/dev/sdb1'}
                 }}})
-        value, msg = dsaz.can_dev_be_reformatted(epath)
+        value, msg = dsaz.can_dev_be_reformatted(epath,
+                                                 preserve_ntfs=False)
         self.assertTrue(value)
         self.assertIn("safe for", msg.lower())
 
@@ -1112,10 +1144,49 @@ class TestCanDevBeReformatted(CiTestCase):
                     epath + '-part3': {'num': 3, 'fs': 'ext',
                                        'realpath': '/dev/sdb3'}
                 }}})
-        value, msg = dsaz.can_dev_be_reformatted(epath)
+        value, msg = dsaz.can_dev_be_reformatted(epath,
+                                                 preserve_ntfs=False)
         self.assertFalse(value)
         self.assertIn("3 or more", msg.lower())
 
+    def test_ntfs_mount_errors_true(self):
+        """can_dev_be_reformatted does not fail if NTFS is unknown fstype."""
+        self.patchup({
+            '/dev/sda': {
+                'partitions': {
+                    '/dev/sda1': {'num': 1, 'fs': 'ntfs', 'files': []}
+                }}})
+
+        err = ("Unexpected error while running command.\n",
+               "Command: ['mount', '-o', 'ro,sync', '-t', 'auto', ",
+               "'/dev/sda1', '/fake-tmp/dir']\n"
+               "Exit code: 32\n"
+               "Reason: -\n"
+               "Stdout: -\n"
+               "Stderr: mount: unknown filesystem type 'ntfs'")
+        self.m_mount_cb.side_effect = MountFailedError(
+            'Failed mounting %s to %s due to: %s' %
+            ('/dev/sda', '/fake-tmp/dir', err))
+
+        value, msg = dsaz.can_dev_be_reformatted('/dev/sda',
+                                                 preserve_ntfs=False)
+        self.assertTrue(value)
+        self.assertIn('cannot mount NTFS, assuming', msg)
+
+    def test_never_destroy_ntfs_config_false(self):
+        """Normally formattable situation with never_destroy_ntfs set."""
+        self.patchup({
+            '/dev/sda': {
+                'partitions': {
+                    '/dev/sda1': {'num': 1, 'fs': 'ntfs',
+                                  'files': ['dataloss_warning_readme.txt']}
+                }}})
+        value, msg = dsaz.can_dev_be_reformatted("/dev/sda",
+                                                 preserve_ntfs=True)
+        self.assertFalse(value)
+        self.assertIn("config says to never destroy NTFS "
+                      "(datasource.Azure.never_destroy_ntfs)", msg)
+
 
 class TestAzureNetExists(CiTestCase):
 
-- 
cgit v1.2.3


From cd1de5f47ab6b82f2c6fd61a5f6681f33b3e5705 Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Wed, 23 May 2018 16:08:43 -0600
Subject: openstack: Allow discovery in init-local using dhclient in a sandbox.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Network has not yet been configured in the init-local stage so the
openstack datasource will use dhcp-client to temporarily obtain an ipv4
address and query the metadata service at http://169.254.169.254 to get
network_data.json configuration. If present, the datasource will return
network_config version 1 config based on that network_data.json content.
Previously OpenStack datasource only setup dhcp on the fallback interface
so this represents a change in behavior to react to the full config
provided by openstack.

Also significant to OpenStack is the separation of a _crawl_data operation
from get_data(). crawl_data walks the available metadata services and
returns a dict of discovered content. get_data consumes the crawled_data,
 caches it in the datasource and reacts to that data.
/run/cloud-init/instance-data.json now published network_data.json or
ec2_metadata key if that data is present on any datasource.

The main reasons for the separation of crawl from get_data:
 * Enable performance metrics of cloud-init's metadata crawls on each
 * Enable cloud-init modules and scripts to query and consume metadata
   content which may have updated/changed after cloud-init's initial cache
   during instance boot. (Think hotplug)

Also generalize common logic to base DataSource class/module:
 * Move to a common UNSET variable up into base datasource module fix EC2,
   ConfigDrive, OpenStack, SmartOS to use the global.
 * Drop get_url_settings from Ec2, CloudStack and OpenStack and generalize
   DataSource.get_url_params(). Allow subclasses to override url_max_wait,
   url_timeout and url_retries params.
 * Rename get_network_metadata bool to perform_dhcp_setup as it designates
   whether EphemeralDHCPv4 setup is required before crawling metadata.

LP: #1749717
---
 cloudinit/sources/DataSourceCloudStack.py         |  31 ++---
 cloudinit/sources/DataSourceConfigDrive.py        |   4 +-
 cloudinit/sources/DataSourceEc2.py                |  48 ++-----
 cloudinit/sources/DataSourceOpenStack.py          | 161 ++++++++++++++--------
 cloudinit/sources/DataSourceSmartOS.py            |   9 +-
 cloudinit/sources/__init__.py                     |  76 ++++++++++
 cloudinit/sources/tests/test_init.py              |  89 +++++++++++-
 tests/unittests/test_datasource/test_common.py    |   1 +
 tests/unittests/test_datasource/test_openstack.py | 121 +++++++++++++++-
 9 files changed, 416 insertions(+), 124 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index 0df545fc..d4b758f2 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -68,6 +68,10 @@ class DataSourceCloudStack(sources.DataSource):
 
     dsname = 'CloudStack'
 
+    # Setup read_url parameters per get_url_params.
+    url_max_wait = 120
+    url_timeout = 50
+
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
         self.seed_dir = os.path.join(paths.seed_dir, 'cs')
@@ -80,33 +84,18 @@ class DataSourceCloudStack(sources.DataSource):
         self.metadata_address = "http://%s/" % (self.vr_addr,)
         self.cfg = {}
 
-    def _get_url_settings(self):
-        mcfg = self.ds_cfg
-        max_wait = 120
-        try:
-            max_wait = int(mcfg.get("max_wait", max_wait))
-        except Exception:
-            util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
+    def wait_for_metadata_service(self):
+        url_params = self.get_url_params()
 
-        if max_wait == 0:
+        if url_params.max_wait_seconds <= 0:
             return False
 
-        timeout = 50
-        try:
-            timeout = int(mcfg.get("timeout", timeout))
-        except Exception:
-            util.logexc(LOG, "Failed to get timeout, using %s", timeout)
-
-        return (max_wait, timeout)
-
-    def wait_for_metadata_service(self):
-        (max_wait, timeout) = self._get_url_settings()
-
         urls = [uhelp.combine_url(self.metadata_address,
                                   'latest/meta-data/instance-id')]
         start_time = time.time()
-        url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
-                                 timeout=timeout, status_cb=LOG.warn)
+        url = uhelp.wait_for_url(
+            urls=urls, max_wait=url_params.max_wait_seconds,
+            timeout=url_params.timeout_seconds, status_cb=LOG.warn)
 
         if url:
             LOG.debug("Using metadata source: '%s'", url)
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 121cf215..4cb28977 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -43,7 +43,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
         self.version = None
         self.ec2_metadata = None
         self._network_config = None
-        self.network_json = None
+        self.network_json = sources.UNSET
         self.network_eni = None
         self.known_macs = None
         self.files = {}
@@ -149,7 +149,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
     @property
     def network_config(self):
         if self._network_config is None:
-            if self.network_json is not None:
+            if self.network_json not in (None, sources.UNSET):
                 LOG.debug("network config provided via network_json")
                 self._network_config = openstack.convert_net_json(
                     self.network_json, known_macs=self.known_macs)
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 21e9ef84..968ab3f7 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -27,8 +27,6 @@ SKIP_METADATA_URL_CODES = frozenset([uhelp.NOT_FOUND])
 STRICT_ID_PATH = ("datasource", "Ec2", "strict_id")
 STRICT_ID_DEFAULT = "warn"
 
-_unset = "_unset"
-
 
 class Platforms(object):
     # TODO Rename and move to cloudinit.cloud.CloudNames
@@ -59,15 +57,16 @@ class DataSourceEc2(sources.DataSource):
     # for extended metadata content. IPv6 support comes in 2016-09-02
     extended_metadata_versions = ['2016-09-02']
 
+    # Setup read_url parameters per get_url_params.
+    url_max_wait = 120
+    url_timeout = 50
+
     _cloud_platform = None
 
-    _network_config = _unset  # Used for caching calculated network config v1
+    _network_config = sources.UNSET  # Used to cache calculated network cfg v1
 
     # Whether we want to get network configuration from the metadata service.
-    get_network_metadata = False
-
-    # Track the discovered fallback nic for use in configuration generation.
-    _fallback_interface = None
+    perform_dhcp_setup = False
 
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceEc2, self).__init__(sys_cfg, distro, paths)
@@ -98,7 +97,7 @@ class DataSourceEc2(sources.DataSource):
         elif self.cloud_platform == Platforms.NO_EC2_METADATA:
             return False
 
-        if self.get_network_metadata:  # Setup networking in init-local stage.
+        if self.perform_dhcp_setup:  # Setup networking in init-local stage.
             if util.is_FreeBSD():
                 LOG.debug("FreeBSD doesn't support running dhclient with -sf")
                 return False
@@ -158,27 +157,11 @@ class DataSourceEc2(sources.DataSource):
         else:
             return self.metadata['instance-id']
 
-    def _get_url_settings(self):
-        mcfg = self.ds_cfg
-        max_wait = 120
-        try:
-            max_wait = int(mcfg.get("max_wait", max_wait))
-        except Exception:
-            util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
-
-        timeout = 50
-        try:
-            timeout = max(0, int(mcfg.get("timeout", timeout)))
-        except Exception:
-            util.logexc(LOG, "Failed to get timeout, using %s", timeout)
-
-        return (max_wait, timeout)
-
     def wait_for_metadata_service(self):
         mcfg = self.ds_cfg
 
-        (max_wait, timeout) = self._get_url_settings()
-        if max_wait <= 0:
+        url_params = self.get_url_params()
+        if url_params.max_wait_seconds <= 0:
             return False
 
         # Remove addresses from the list that wont resolve.
@@ -205,7 +188,8 @@ class DataSourceEc2(sources.DataSource):
 
         start_time = time.time()
         url = uhelp.wait_for_url(
-            urls=urls, max_wait=max_wait, timeout=timeout, status_cb=LOG.warn)
+            urls=urls, max_wait=url_params.max_wait_seconds,
+            timeout=url_params.timeout_seconds, status_cb=LOG.warn)
 
         if url:
             self.metadata_address = url2base[url]
@@ -310,11 +294,11 @@ class DataSourceEc2(sources.DataSource):
     @property
     def network_config(self):
         """Return a network config dict for rendering ENI or netplan files."""
-        if self._network_config != _unset:
+        if self._network_config != sources.UNSET:
             return self._network_config
 
         if self.metadata is None:
-            # this would happen if get_data hadn't been called. leave as _unset
+            # this would happen if get_data hadn't been called. leave as UNSET
             LOG.warning(
                 "Unexpected call to network_config when metadata is None.")
             return None
@@ -353,9 +337,7 @@ class DataSourceEc2(sources.DataSource):
                 self._fallback_interface = _legacy_fbnic
                 self.fallback_nic = None
             else:
-                self._fallback_interface = net.find_fallback_nic()
-                if self._fallback_interface is None:
-                    LOG.warning("Did not find a fallback interface on EC2.")
+                return super(DataSourceEc2, self).fallback_interface
         return self._fallback_interface
 
     def _crawl_metadata(self):
@@ -390,7 +372,7 @@ class DataSourceEc2Local(DataSourceEc2):
     metadata service. If the metadata service provides network configuration
     then render the network configuration for that instance based on metadata.
     """
-    get_network_metadata = True  # Get metadata network config if present
+    perform_dhcp_setup = True  # Use dhcp before querying metadata
 
     def get_data(self):
         supported_platforms = (Platforms.AWS,)
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index fb166ae1..1a12a3f1 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -7,6 +7,7 @@
 import time
 
 from cloudinit import log as logging
+from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError
 from cloudinit import sources
 from cloudinit import url_helper
 from cloudinit import util
@@ -27,46 +28,25 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
 
     dsname = "OpenStack"
 
+    _network_config = sources.UNSET  # Used to cache calculated network cfg v1
+
+    # Whether we want to get network configuration from the metadata service.
+    perform_dhcp_setup = False
+
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths)
         self.metadata_address = None
         self.ssl_details = util.fetch_ssl_details(self.paths)
         self.version = None
         self.files = {}
-        self.ec2_metadata = None
+        self.ec2_metadata = sources.UNSET
+        self.network_json = sources.UNSET
 
     def __str__(self):
         root = sources.DataSource.__str__(self)
         mstr = "%s [%s,ver=%s]" % (root, self.dsmode, self.version)
         return mstr
 
-    def _get_url_settings(self):
-        # TODO(harlowja): this is shared with ec2 datasource, we should just
-        # move it to a shared location instead...
-        # Note: the defaults here are different though.
-
-        # max_wait < 0 indicates do not wait
-        max_wait = -1
-        timeout = 10
-        retries = 5
-
-        try:
-            max_wait = int(self.ds_cfg.get("max_wait", max_wait))
-        except Exception:
-            util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
-
-        try:
-            timeout = max(0, int(self.ds_cfg.get("timeout", timeout)))
-        except Exception:
-            util.logexc(LOG, "Failed to get timeout, using %s", timeout)
-
-        try:
-            retries = int(self.ds_cfg.get("retries", retries))
-        except Exception:
-            util.logexc(LOG, "Failed to get retries. using %s", retries)
-
-        return (max_wait, timeout, retries)
-
     def wait_for_metadata_service(self):
         urls = self.ds_cfg.get("metadata_urls", [DEF_MD_URL])
         filtered = [x for x in urls if util.is_resolvable_url(x)]
@@ -86,10 +66,11 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
             md_urls.append(md_url)
             url2base[md_url] = url
 
-        (max_wait, timeout, _retries) = self._get_url_settings()
+        url_params = self.get_url_params()
         start_time = time.time()
-        avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait,
-                                            timeout=timeout)
+        avail_url = url_helper.wait_for_url(
+            urls=md_urls, max_wait=url_params.max_wait_seconds,
+            timeout=url_params.timeout_seconds)
         if avail_url:
             LOG.debug("Using metadata source: '%s'", url2base[avail_url])
         else:
@@ -99,38 +80,64 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
         self.metadata_address = url2base.get(avail_url)
         return bool(avail_url)
 
-    def _get_data(self):
-        try:
-            if not self.wait_for_metadata_service():
-                return False
-        except IOError:
-            return False
+    def check_instance_id(self, sys_cfg):
+        # quickly (local check only) if self.instance_id is still valid
+        return sources.instance_id_matches_system_uuid(self.get_instance_id())
 
-        (_max_wait, timeout, retries) = self._get_url_settings()
+    @property
+    def network_config(self):
+        """Return a network config dict for rendering ENI or netplan files."""
+        if self._network_config != sources.UNSET:
+            return self._network_config
+
+        # RELEASE_BLOCKER: SRU to Xenial and Artful SRU should not provide
+        # network_config by default unless configured in /etc/cloud/cloud.cfg*.
+        # Patch Xenial and Artful before release to default to False.
+        if util.is_false(self.ds_cfg.get('apply_network_config', True)):
+            self._network_config = None
+            return self._network_config
+        if self.network_json == sources.UNSET:
+            # this would happen if get_data hadn't been called. leave as UNSET
+            LOG.warning(
+                'Unexpected call to network_config when network_json is None.')
+            return None
+
+        LOG.debug('network config provided via network_json')
+        self._network_config = openstack.convert_net_json(
+            self.network_json, known_macs=None)
+        return self._network_config
 
-        try:
-            results = util.log_time(LOG.debug,
-                                    'Crawl of openstack metadata service',
-                                    read_metadata_service,
-                                    args=[self.metadata_address],
-                                    kwargs={'ssl_details': self.ssl_details,
-                                            'retries': retries,
-                                            'timeout': timeout})
-        except openstack.NonReadable:
-            return False
-        except (openstack.BrokenMetadata, IOError):
-            util.logexc(LOG, "Broken metadata address %s",
-                        self.metadata_address)
-            return False
+    def _get_data(self):
+        """Crawl metadata, parse and persist that data for this instance.
+
+        @return: True when metadata discovered indicates OpenStack datasource.
+            False when unable to contact metadata service or when metadata
+            format is invalid or disabled.
+        """
+        if self.perform_dhcp_setup:  # Setup networking in init-local stage.
+            try:
+                with EphemeralDHCPv4(self.fallback_interface):
+                    results = util.log_time(
+                        logfunc=LOG.debug, msg='Crawl of metadata service',
+                        func=self._crawl_metadata)
+            except (NoDHCPLeaseError, sources.InvalidMetaDataException) as e:
+                util.logexc(LOG, str(e))
+                return False
+        else:
+            try:
+                results = self._crawl_metadata()
+            except sources.InvalidMetaDataException as e:
+                util.logexc(LOG, str(e))
+                return False
 
         self.dsmode = self._determine_dsmode([results.get('dsmode')])
         if self.dsmode == sources.DSMODE_DISABLED:
             return False
-
         md = results.get('metadata', {})
         md = util.mergemanydict([md, DEFAULT_METADATA])
         self.metadata = md
         self.ec2_metadata = results.get('ec2-metadata')
+        self.network_json = results.get('networkdata')
         self.userdata_raw = results.get('userdata')
         self.version = results['version']
         self.files.update(results.get('files', {}))
@@ -145,9 +152,50 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
 
         return True
 
-    def check_instance_id(self, sys_cfg):
-        # quickly (local check only) if self.instance_id is still valid
-        return sources.instance_id_matches_system_uuid(self.get_instance_id())
+    def _crawl_metadata(self):
+        """Crawl metadata service when available.
+
+        @returns: Dictionary with all metadata discovered for this datasource.
+        @raise: InvalidMetaDataException on unreadable or broken
+            metadata.
+        """
+        try:
+            if not self.wait_for_metadata_service():
+                raise sources.InvalidMetaDataException(
+                    'No active metadata service found')
+        except IOError as e:
+            raise sources.InvalidMetaDataException(
+                'IOError contacting metadata service: {error}'.format(
+                    error=str(e)))
+
+        url_params = self.get_url_params()
+
+        try:
+            result = util.log_time(
+                LOG.debug, 'Crawl of openstack metadata service',
+                read_metadata_service, args=[self.metadata_address],
+                kwargs={'ssl_details': self.ssl_details,
+                        'retries': url_params.num_retries,
+                        'timeout': url_params.timeout_seconds})
+        except openstack.NonReadable as e:
+            raise sources.InvalidMetaDataException(str(e))
+        except (openstack.BrokenMetadata, IOError):
+            msg = 'Broken metadata address {addr}'.format(
+                addr=self.metadata_address)
+            raise sources.InvalidMetaDataException(msg)
+        return result
+
+
+class DataSourceOpenStackLocal(DataSourceOpenStack):
+    """Run in init-local using a dhcp discovery prior to metadata crawl.
+
+    In init-local, no network is available. This subclass sets up minimal
+    networking with dhclient on a viable nic so that it can talk to the
+    metadata service. If the metadata service provides network configuration
+    then render the network configuration for that instance based on metadata.
+    """
+
+    perform_dhcp_setup = True  # Get metadata network config if present
 
 
 def read_metadata_service(base_url, ssl_details=None,
@@ -159,6 +207,7 @@ def read_metadata_service(base_url, ssl_details=None,
 
 # Used to match classes to dependencies
 datasources = [
+    (DataSourceOpenStackLocal, (sources.DEP_FILESYSTEM,)),
     (DataSourceOpenStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
 ]
 
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index fcb46b14..c91e4d59 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -165,9 +165,8 @@ class DataSourceSmartOS(sources.DataSource):
 
     dsname = "Joyent"
 
-    _unset = "_unset"
-    smartos_type = _unset
-    md_client = _unset
+    smartos_type = sources.UNSET
+    md_client = sources.UNSET
 
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
@@ -189,12 +188,12 @@ class DataSourceSmartOS(sources.DataSource):
         return "%s [client=%s]" % (root, self.md_client)
 
     def _init(self):
-        if self.smartos_type == self._unset:
+        if self.smartos_type == sources.UNSET:
             self.smartos_type = get_smartos_environ()
             if self.smartos_type is None:
                 self.md_client = None
 
-        if self.md_client == self._unset:
+        if self.md_client == sources.UNSET:
             self.md_client = jmc_client_factory(
                 smartos_type=self.smartos_type,
                 metadata_sockfile=self.ds_cfg['metadata_sockfile'],
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index df0b374a..90d74575 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -9,6 +9,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import abc
+from collections import namedtuple
 import copy
 import json
 import os
@@ -17,6 +18,7 @@ import six
 from cloudinit.atomic_helper import write_json
 from cloudinit import importer
 from cloudinit import log as logging
+from cloudinit import net
 from cloudinit import type_utils
 from cloudinit import user_data as ud
 from cloudinit import util
@@ -41,6 +43,8 @@ INSTANCE_JSON_FILE = 'instance-data.json'
 # Key which can be provide a cloud's official product name to cloud-init
 METADATA_CLOUD_NAME_KEY = 'cloud-name'
 
+UNSET = "_unset"
+
 LOG = logging.getLogger(__name__)
 
 
@@ -48,6 +52,11 @@ class DataSourceNotFoundException(Exception):
     pass
 
 
+class InvalidMetaDataException(Exception):
+    """Raised when metadata is broken, unavailable or disabled."""
+    pass
+
+
 def process_base64_metadata(metadata, key_path=''):
     """Strip ci-b64 prefix and return metadata with base64-encoded-keys set."""
     md_copy = copy.deepcopy(metadata)
@@ -68,6 +77,10 @@ def process_base64_metadata(metadata, key_path=''):
     return md_copy
 
 
+URLParams = namedtuple(
+    'URLParms', ['max_wait_seconds', 'timeout_seconds', 'num_retries'])
+
+
 @six.add_metaclass(abc.ABCMeta)
 class DataSource(object):
 
@@ -81,6 +94,14 @@ class DataSource(object):
     # Cached cloud_name as determined by _get_cloud_name
     _cloud_name = None
 
+    # Track the discovered fallback nic for use in configuration generation.
+    _fallback_interface = None
+
+    # read_url_params
+    url_max_wait = -1   # max_wait < 0 means do not wait
+    url_timeout = 10    # timeout for each metadata url read attempt
+    url_retries = 5     # number of times to retry url upon 404
+
     def __init__(self, sys_cfg, distro, paths, ud_proc=None):
         self.sys_cfg = sys_cfg
         self.distro = distro
@@ -128,6 +149,14 @@ class DataSource(object):
                 'meta-data': self.metadata,
                 'user-data': self.get_userdata_raw(),
                 'vendor-data': self.get_vendordata_raw()}}
+        if hasattr(self, 'network_json'):
+            network_json = getattr(self, 'network_json')
+            if network_json != UNSET:
+                instance_data['ds']['network_json'] = network_json
+        if hasattr(self, 'ec2_metadata'):
+            ec2_metadata = getattr(self, 'ec2_metadata')
+            if ec2_metadata != UNSET:
+                instance_data['ds']['ec2_metadata'] = ec2_metadata
         instance_data.update(
             self._get_standardized_metadata())
         try:
@@ -149,6 +178,42 @@ class DataSource(object):
             'Subclasses of DataSource must implement _get_data which'
             ' sets self.metadata, vendordata_raw and userdata_raw.')
 
+    def get_url_params(self):
+        """Return the Datasource's prefered url_read parameters.
+
+        Subclasses may override url_max_wait, url_timeout, url_retries.
+
+        @return: A URLParams object with max_wait_seconds, timeout_seconds,
+            num_retries.
+        """
+        max_wait = self.url_max_wait
+        try:
+            max_wait = int(self.ds_cfg.get("max_wait", self.url_max_wait))
+        except ValueError:
+            util.logexc(
+                LOG, "Config max_wait '%s' is not an int, using default '%s'",
+                self.ds_cfg.get("max_wait"), max_wait)
+
+        timeout = self.url_timeout
+        try:
+            timeout = max(
+                0, int(self.ds_cfg.get("timeout", self.url_timeout)))
+        except ValueError:
+            timeout = self.url_timeout
+            util.logexc(
+                LOG, "Config timeout '%s' is not an int, using default '%s'",
+                self.ds_cfg.get('timeout'), timeout)
+
+        retries = self.url_retries
+        try:
+            retries = int(self.ds_cfg.get("retries", self.url_retries))
+        except Exception:
+            util.logexc(
+                LOG, "Config retries '%s' is not an int, using default '%s'",
+                self.ds_cfg.get('retries'), retries)
+
+        return URLParams(max_wait, timeout, retries)
+
     def get_userdata(self, apply_filter=False):
         if self.userdata is None:
             self.userdata = self.ud_proc.process(self.get_userdata_raw())
@@ -161,6 +226,17 @@ class DataSource(object):
             self.vendordata = self.ud_proc.process(self.get_vendordata_raw())
         return self.vendordata
 
+    @property
+    def fallback_interface(self):
+        """Determine the network interface used during local network config."""
+        if self._fallback_interface is None:
+            self._fallback_interface = net.find_fallback_nic()
+            if self._fallback_interface is None:
+                LOG.warning(
+                    "Did not find a fallback interface on %s.",
+                    self.cloud_name)
+        return self._fallback_interface
+
     @property
     def cloud_name(self):
         """Return lowercase cloud name as determined by the datasource.
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index 452e9219..d5bc98a4 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -17,6 +17,7 @@ from cloudinit import util
 class DataSourceTestSubclassNet(DataSource):
 
     dsname = 'MyTestSubclass'
+    url_max_wait = 55
 
     def __init__(self, sys_cfg, distro, paths, custom_userdata=None):
         super(DataSourceTestSubclassNet, self).__init__(
@@ -70,8 +71,7 @@ class TestDataSource(CiTestCase):
         """Init uses DataSource.dsname for sourcing ds_cfg."""
         sys_cfg = {'datasource': {'MyTestSubclass': {'key2': False}}}
         distro = 'distrotest'  # generally should be a Distro object
-        paths = Paths({})
-        datasource = DataSourceTestSubclassNet(sys_cfg, distro, paths)
+        datasource = DataSourceTestSubclassNet(sys_cfg, distro, self.paths)
         self.assertEqual({'key2': False}, datasource.ds_cfg)
 
     def test_str_is_classname(self):
@@ -81,6 +81,91 @@ class TestDataSource(CiTestCase):
             'DataSourceTestSubclassNet',
             str(DataSourceTestSubclassNet('', '', self.paths)))
 
+    def test_datasource_get_url_params_defaults(self):
+        """get_url_params default url config settings for the datasource."""
+        params = self.datasource.get_url_params()
+        self.assertEqual(params.max_wait_seconds, self.datasource.url_max_wait)
+        self.assertEqual(params.timeout_seconds, self.datasource.url_timeout)
+        self.assertEqual(params.num_retries, self.datasource.url_retries)
+
+    def test_datasource_get_url_params_subclassed(self):
+        """Subclasses can override get_url_params defaults."""
+        sys_cfg = {'datasource': {'MyTestSubclass': {'key2': False}}}
+        distro = 'distrotest'  # generally should be a Distro object
+        datasource = DataSourceTestSubclassNet(sys_cfg, distro, self.paths)
+        expected = (datasource.url_max_wait, datasource.url_timeout,
+                    datasource.url_retries)
+        url_params = datasource.get_url_params()
+        self.assertNotEqual(self.datasource.get_url_params(), url_params)
+        self.assertEqual(expected, url_params)
+
+    def test_datasource_get_url_params_ds_config_override(self):
+        """Datasource configuration options can override url param defaults."""
+        sys_cfg = {
+            'datasource': {
+                'MyTestSubclass': {
+                    'max_wait': '1', 'timeout': '2', 'retries': '3'}}}
+        datasource = DataSourceTestSubclassNet(
+            sys_cfg, self.distro, self.paths)
+        expected = (1, 2, 3)
+        url_params = datasource.get_url_params()
+        self.assertNotEqual(
+            (datasource.url_max_wait, datasource.url_timeout,
+             datasource.url_retries),
+            url_params)
+        self.assertEqual(expected, url_params)
+
+    def test_datasource_get_url_params_is_zero_or_greater(self):
+        """get_url_params ignores timeouts with a value below 0."""
+        # Set an override that is below 0 which gets ignored.
+        sys_cfg = {'datasource': {'_undef': {'timeout': '-1'}}}
+        datasource = DataSource(sys_cfg, self.distro, self.paths)
+        (_max_wait, timeout, _retries) = datasource.get_url_params()
+        self.assertEqual(0, timeout)
+
+    def test_datasource_get_url_uses_defaults_on_errors(self):
+        """On invalid system config values for url_params defaults are used."""
+        # All invalid values should be logged
+        sys_cfg = {'datasource': {
+            '_undef': {
+                'max_wait': 'nope', 'timeout': 'bug', 'retries': 'nonint'}}}
+        datasource = DataSource(sys_cfg, self.distro, self.paths)
+        url_params = datasource.get_url_params()
+        expected = (datasource.url_max_wait, datasource.url_timeout,
+                    datasource.url_retries)
+        self.assertEqual(expected, url_params)
+        logs = self.logs.getvalue()
+        expected_logs = [
+            "Config max_wait 'nope' is not an int, using default '-1'",
+            "Config timeout 'bug' is not an int, using default '10'",
+            "Config retries 'nonint' is not an int, using default '5'",
+        ]
+        for log in expected_logs:
+            self.assertIn(log, logs)
+
+    @mock.patch('cloudinit.sources.net.find_fallback_nic')
+    def test_fallback_interface_is_discovered(self, m_get_fallback_nic):
+        """The fallback_interface is discovered via find_fallback_nic."""
+        m_get_fallback_nic.return_value = 'nic9'
+        self.assertEqual('nic9', self.datasource.fallback_interface)
+
+    @mock.patch('cloudinit.sources.net.find_fallback_nic')
+    def test_fallback_interface_logs_undiscovered(self, m_get_fallback_nic):
+        """Log a warning when fallback_interface can not discover the nic."""
+        self.datasource._cloud_name = 'MySupahCloud'
+        m_get_fallback_nic.return_value = None  # Couldn't discover nic
+        self.assertIsNone(self.datasource.fallback_interface)
+        self.assertEqual(
+            'WARNING: Did not find a fallback interface on MySupahCloud.\n',
+            self.logs.getvalue())
+
+    @mock.patch('cloudinit.sources.net.find_fallback_nic')
+    def test_wb_fallback_interface_is_cached(self, m_get_fallback_nic):
+        """The fallback_interface is cached and won't be rediscovered."""
+        self.datasource._fallback_interface = 'nic10'
+        self.assertEqual('nic10', self.datasource.fallback_interface)
+        m_get_fallback_nic.assert_not_called()
+
     def test__get_data_unimplemented(self):
         """Raise an error when _get_data is not implemented."""
         with self.assertRaises(NotImplementedError) as context_manager:
diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
index ec333888..0d35dc29 100644
--- a/tests/unittests/test_datasource/test_common.py
+++ b/tests/unittests/test_datasource/test_common.py
@@ -40,6 +40,7 @@ DEFAULT_LOCAL = [
     OVF.DataSourceOVF,
     SmartOS.DataSourceSmartOS,
     Ec2.DataSourceEc2Local,
+    OpenStack.DataSourceOpenStackLocal,
 ]
 
 DEFAULT_NETWORK = [
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index bb180c08..fad73b21 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -16,7 +16,7 @@ from six import StringIO
 
 from cloudinit import helpers
 from cloudinit import settings
-from cloudinit.sources import convert_vendordata
+from cloudinit.sources import convert_vendordata, UNSET
 from cloudinit.sources import DataSourceOpenStack as ds
 from cloudinit.sources.helpers import openstack
 from cloudinit import util
@@ -129,6 +129,8 @@ def _read_metadata_service():
 
 
 class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
+
+    with_logs = True
     VERSION = 'latest'
 
     def setUp(self):
@@ -223,11 +225,11 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, {}, {}, os_files)
         self.assertRaises(openstack.BrokenMetadata, _read_metadata_service)
 
-    def test_datasource(self):
+    @test_helpers.mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    def test_datasource(self, m_dhcp):
         _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES)
-        ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN,
-                                       None,
-                                       helpers.Paths({'run_dir': self.tmp}))
+        ds_os = ds.DataSourceOpenStack(
+            settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp}))
         self.assertIsNone(ds_os.version)
         found = ds_os.get_data()
         self.assertTrue(found)
@@ -241,6 +243,36 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         self.assertEqual(2, len(ds_os.files))
         self.assertEqual(VENDOR_DATA, ds_os.vendordata_pure)
         self.assertIsNone(ds_os.vendordata_raw)
+        m_dhcp.assert_not_called()
+
+    @hp.activate
+    @test_helpers.mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network')
+    @test_helpers.mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    def test_local_datasource(self, m_dhcp, m_net):
+        """OpenStackLocal calls EphemeralDHCPNetwork and gets instance data."""
+        _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES)
+        ds_os_local = ds.DataSourceOpenStackLocal(
+            settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp}))
+        ds_os_local._fallback_interface = 'eth9'  # Monkey patch for dhcp
+        m_dhcp.return_value = [{
+            'interface': 'eth9', 'fixed-address': '192.168.2.9',
+            'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0',
+            'broadcast-address': '192.168.2.255'}]
+
+        self.assertIsNone(ds_os_local.version)
+        found = ds_os_local.get_data()
+        self.assertTrue(found)
+        self.assertEqual(2, ds_os_local.version)
+        md = dict(ds_os_local.metadata)
+        md.pop('instance-id', None)
+        md.pop('local-hostname', None)
+        self.assertEqual(OSTACK_META, md)
+        self.assertEqual(EC2_META, ds_os_local.ec2_metadata)
+        self.assertEqual(USER_DATA, ds_os_local.userdata_raw)
+        self.assertEqual(2, len(ds_os_local.files))
+        self.assertEqual(VENDOR_DATA, ds_os_local.vendordata_pure)
+        self.assertIsNone(ds_os_local.vendordata_raw)
+        m_dhcp.assert_called_with('eth9')
 
     def test_bad_datasource_meta(self):
         os_files = copy.deepcopy(OS_FILES)
@@ -255,6 +287,10 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         found = ds_os.get_data()
         self.assertFalse(found)
         self.assertIsNone(ds_os.version)
+        self.assertIn(
+            'InvalidMetaDataException: Broken metadata address'
+            ' http://169.254.169.25',
+            self.logs.getvalue())
 
     def test_no_datasource(self):
         os_files = copy.deepcopy(OS_FILES)
@@ -274,6 +310,52 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         self.assertFalse(found)
         self.assertIsNone(ds_os.version)
 
+    def test_network_config_disabled_by_datasource_config(self):
+        """The network_config can be disabled from datasource config."""
+        mock_path = (
+            'cloudinit.sources.DataSourceOpenStack.openstack.'
+            'convert_net_json')
+        ds_os = ds.DataSourceOpenStack(
+            settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp}))
+        ds_os.ds_cfg = {'apply_network_config': False}
+        sample_json = {'links': [{'ethernet_mac_address': 'mymac'}],
+                       'networks': [], 'services': []}
+        ds_os.network_json = sample_json  # Ignore this content from metadata
+        with test_helpers.mock.patch(mock_path) as m_convert_json:
+            self.assertIsNone(ds_os.network_config)
+        m_convert_json.assert_not_called()
+
+    def test_network_config_from_network_json(self):
+        """The datasource gets network_config from network_data.json."""
+        mock_path = (
+            'cloudinit.sources.DataSourceOpenStack.openstack.'
+            'convert_net_json')
+        example_cfg = {'version': 1, 'config': []}
+        ds_os = ds.DataSourceOpenStack(
+            settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp}))
+        sample_json = {'links': [{'ethernet_mac_address': 'mymac'}],
+                       'networks': [], 'services': []}
+        ds_os.network_json = sample_json
+        with test_helpers.mock.patch(mock_path) as m_convert_json:
+            m_convert_json.return_value = example_cfg
+            self.assertEqual(example_cfg, ds_os.network_config)
+        self.assertIn(
+            'network config provided via network_json', self.logs.getvalue())
+        m_convert_json.assert_called_with(sample_json, known_macs=None)
+
+    def test_network_config_cached(self):
+        """The datasource caches the network_config property."""
+        mock_path = (
+            'cloudinit.sources.DataSourceOpenStack.openstack.'
+            'convert_net_json')
+        example_cfg = {'version': 1, 'config': []}
+        ds_os = ds.DataSourceOpenStack(
+            settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp}))
+        ds_os._network_config = example_cfg
+        with test_helpers.mock.patch(mock_path) as m_convert_json:
+            self.assertEqual(example_cfg, ds_os.network_config)
+        m_convert_json.assert_not_called()
+
     def test_disabled_datasource(self):
         os_files = copy.deepcopy(OS_FILES)
         os_meta = copy.deepcopy(OSTACK_META)
@@ -296,6 +378,35 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         self.assertFalse(found)
         self.assertIsNone(ds_os.version)
 
+    @hp.activate
+    def test_wb__crawl_metadata_does_not_persist(self):
+        """_crawl_metadata returns current metadata and does not cache."""
+        _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES)
+        ds_os = ds.DataSourceOpenStack(
+            settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp}))
+        crawled_data = ds_os._crawl_metadata()
+        self.assertEqual(UNSET, ds_os.ec2_metadata)
+        self.assertIsNone(ds_os.userdata_raw)
+        self.assertEqual(0, len(ds_os.files))
+        self.assertIsNone(ds_os.vendordata_raw)
+        self.assertEqual(
+            ['dsmode', 'ec2-metadata', 'files', 'metadata', 'networkdata',
+             'userdata', 'vendordata', 'version'],
+            sorted(crawled_data.keys()))
+        self.assertEqual('local', crawled_data['dsmode'])
+        self.assertEqual(EC2_META, crawled_data['ec2-metadata'])
+        self.assertEqual(2, len(crawled_data['files']))
+        md = copy.deepcopy(crawled_data['metadata'])
+        md.pop('instance-id')
+        md.pop('local-hostname')
+        self.assertEqual(OSTACK_META, md)
+        self.assertEqual(
+            json.loads(OS_FILES['openstack/latest/network_data.json']),
+            crawled_data['networkdata'])
+        self.assertEqual(USER_DATA, crawled_data['userdata'])
+        self.assertEqual(VENDOR_DATA, crawled_data['vendordata'])
+        self.assertEqual(2, crawled_data['version'])
+
 
 class TestVendorDataLoading(test_helpers.TestCase):
     def cvj(self, data):
-- 
cgit v1.2.3


From 3f99f4aba8af559f99f1d46b2b46bea7622da1d0 Mon Sep 17 00:00:00 2001
From: Dan McDonald <danmcd@joyent.com>
Date: Thu, 24 May 2018 10:54:18 -0400
Subject: Enable SmartOS network metadata to work with netplan via per-subnet
 routes

- Updated datadict reference URL
- Store sdc:routes metadata in DatasourceSmartOS
- Map sdc:routes values to per-interface subnet configuration
- Added unittest

Co-authored-by: Mike Gerdts <mike.gerdts@joyent.com>

LP: #1763512
---
 cloudinit/sources/DataSourceSmartOS.py          | 46 ++++++++++++++++++++++---
 tests/unittests/test_datasource/test_smartos.py | 26 ++++++++++++++
 2 files changed, 67 insertions(+), 5 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index c91e4d59..f92e8b5c 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -17,7 +17,7 @@
 #        of a serial console.
 #
 #   Certain behavior is defined by the DataDictionary
-#       http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html
+#       https://eng.joyent.com/mdata/datadict.html
 #       Comments with "@datadictionary" are snippets of the definition
 
 import base64
@@ -298,6 +298,7 @@ class DataSourceSmartOS(sources.DataSource):
         self.userdata_raw = ud
         self.vendordata_raw = md['vendor-data']
         self.network_data = md['network-data']
+        self.routes_data = md['routes']
 
         self._set_provisioned()
         return True
@@ -321,7 +322,8 @@ class DataSourceSmartOS(sources.DataSource):
                     convert_smartos_network_data(
                         network_data=self.network_data,
                         dns_servers=self.metadata['dns_servers'],
-                        dns_domain=self.metadata['dns_domain']))
+                        dns_domain=self.metadata['dns_domain'],
+                        routes=self.routes_data))
         return self._network_config
 
 
@@ -760,7 +762,8 @@ def get_smartos_environ(uname_version=None, product_name=None):
 
 # Convert SMARTOS 'sdc:nics' data to network_config yaml
 def convert_smartos_network_data(network_data=None,
-                                 dns_servers=None, dns_domain=None):
+                                 dns_servers=None, dns_domain=None,
+                                 routes=None):
     """Return a dictionary of network_config by parsing provided
        SMARTOS sdc:nics configuration data
 
@@ -778,6 +781,10 @@ def convert_smartos_network_data(network_data=None,
     keys are related to ip configuration.  For each ip in the 'ips' list
     we create a subnet entry under 'subnets' pairing the ip to a one in
     the 'gateways' list.
+
+    Each route in sdc:routes is mapped to a route on each interface.
+    The sdc:routes properties 'dst' and 'gateway' map to 'network' and
+    'gateway'.  The 'linklocal' sdc:routes property is ignored.
     """
 
     valid_keys = {
@@ -800,6 +807,10 @@ def convert_smartos_network_data(network_data=None,
             'scope',
             'type',
         ],
+        'route': [
+            'network',
+            'gateway',
+        ],
     }
 
     if dns_servers:
@@ -814,6 +825,9 @@ def convert_smartos_network_data(network_data=None,
     else:
         dns_domain = []
 
+    if not routes:
+        routes = []
+
     def is_valid_ipv4(addr):
         return '.' in addr
 
@@ -840,6 +854,7 @@ def convert_smartos_network_data(network_data=None,
             if ip == "dhcp":
                 subnet = {'type': 'dhcp4'}
             else:
+                routeents = []
                 subnet = dict((k, v) for k, v in nic.items()
                               if k in valid_keys['subnet'])
                 subnet.update({
@@ -861,6 +876,25 @@ def convert_smartos_network_data(network_data=None,
                             pgws[proto]['gw'] = gateways[0]
                             subnet.update({'gateway': pgws[proto]['gw']})
 
+                for route in routes:
+                    rcfg = dict((k, v) for k, v in route.items()
+                                if k in valid_keys['route'])
+                    # Linux uses the value of 'gateway' to determine
+                    # automatically if the route is a forward/next-hop
+                    # (non-local IP for gateway) or an interface/resolver
+                    # (local IP for gateway).  So we can ignore the
+                    # 'interface' attribute of sdc:routes, because SDC
+                    # guarantees that the gateway is a local IP for
+                    # "interface=true".
+                    #
+                    # Eventually we should be smart and compare "gateway"
+                    # to see if it's in the prefix.  We can then smartly
+                    # add or not-add this route.  But for now,
+                    # when in doubt, use brute force! Routes for everyone!
+                    rcfg.update({'network': route['dst']})
+                    routeents.append(rcfg)
+                    subnet.update({'routes': routeents})
+
             subnets.append(subnet)
         cfg.update({'subnets': subnets})
         config.append(cfg)
@@ -904,12 +938,14 @@ if __name__ == "__main__":
             keyname = SMARTOS_ATTRIB_JSON[key]
             data[key] = client.get_json(keyname)
         elif key == "network_config":
-            for depkey in ('network-data', 'dns_servers', 'dns_domain'):
+            for depkey in ('network-data', 'dns_servers', 'dns_domain',
+                           'routes'):
                 load_key(client, depkey, data)
             data[key] = convert_smartos_network_data(
                 network_data=data['network-data'],
                 dns_servers=data['dns_servers'],
-                dns_domain=data['dns_domain'])
+                dns_domain=data['dns_domain'],
+                routes=data['routes'])
         else:
             if key in SMARTOS_ATTRIB_MAP:
                 keyname, strip = SMARTOS_ATTRIB_MAP[key]
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 706e8eb8..dca0b3d4 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -1027,6 +1027,32 @@ class TestNetworkConversion(TestCase):
         found = convert_net(SDC_NICS_SINGLE_GATEWAY)
         self.assertEqual(expected, found)
 
+    def test_routes_on_all_nics(self):
+        routes = [
+            {'linklocal': False, 'dst': '3.0.0.0/8', 'gateway': '8.12.42.3'},
+            {'linklocal': False, 'dst': '4.0.0.0/8', 'gateway': '10.210.1.4'}]
+        expected = {
+            'version': 1,
+            'config': [
+                {'mac_address': '90:b8:d0:d8:82:b4', 'mtu': 1500,
+                 'name': 'net0', 'type': 'physical',
+                 'subnets': [{'address': '8.12.42.26/24',
+                              'gateway': '8.12.42.1', 'type': 'static',
+                              'routes': [{'network': '3.0.0.0/8',
+                                          'gateway': '8.12.42.3'},
+                                         {'network': '4.0.0.0/8',
+                                         'gateway': '10.210.1.4'}]}]},
+                {'mac_address': '90:b8:d0:0a:51:31', 'mtu': 1500,
+                 'name': 'net1', 'type': 'physical',
+                 'subnets': [{'address': '10.210.1.27/24', 'type': 'static',
+                              'routes': [{'network': '3.0.0.0/8',
+                                          'gateway': '8.12.42.3'},
+                                         {'network': '4.0.0.0/8',
+                                         'gateway': '10.210.1.4'}]}]}]}
+        found = convert_net(SDC_NICS_SINGLE_GATEWAY, routes=routes)
+        self.maxDiff = None
+        self.assertEqual(expected, found)
+
 
 @unittest2.skipUnless(get_smartos_environ() == SMARTOS_ENV_KVM,
                       "Only supported on KVM and bhyve guests under SmartOS")
-- 
cgit v1.2.3


From 4ba4639b86edad7ec89a55a61b7d9075f92d2166 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Tue, 29 May 2018 13:19:27 -0400
Subject: pyflakes: fix unused variable references identified by pyflakes
 2.0.0.

A newer version of pyflakes (2.0.0) was released.
It identifed some unused variables that version 1.6.0 did not identify.
The change here merely fixes those unused variables.
---
 cloudinit/config/cc_mounts.py             |  2 +-
 cloudinit/handlers/upstart_job.py         |  2 +-
 cloudinit/sources/DataSourceAltCloud.py   | 16 ++++++++--------
 cloudinit/sources/DataSourceNoCloud.py    |  4 ++--
 cloudinit/sources/DataSourceOpenNebula.py |  2 +-
 cloudinit/util.py                         |  4 ++--
 6 files changed, 15 insertions(+), 15 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py
index eca6ea3f..339baba9 100644
--- a/cloudinit/config/cc_mounts.py
+++ b/cloudinit/config/cc_mounts.py
@@ -233,7 +233,7 @@ def setup_swapfile(fname, size=None, maxsize=None):
     if str(size).lower() == "auto":
         try:
             memsize = util.read_meminfo()['total']
-        except IOError as e:
+        except IOError:
             LOG.debug("Not creating swap: failed to read meminfo")
             return
 
diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py
index 1ca92d4b..dc338769 100644
--- a/cloudinit/handlers/upstart_job.py
+++ b/cloudinit/handlers/upstart_job.py
@@ -97,7 +97,7 @@ def _has_suitable_upstart():
             else:
                 util.logexc(LOG, "dpkg --compare-versions failed [%s]",
                             e.exit_code)
-        except Exception as e:
+        except Exception:
             util.logexc(LOG, "dpkg --compare-versions failed")
         return False
     else:
diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index f6e86f34..24fd65ff 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -184,11 +184,11 @@ class DataSourceAltCloud(sources.DataSource):
             cmd = CMD_PROBE_FLOPPY
             (cmd_out, _err) = util.subp(cmd)
             LOG.debug('Command: %s\nOutput%s', ' '.join(cmd), cmd_out)
-        except ProcessExecutionError as _err:
-            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err)
+        except ProcessExecutionError as e:
+            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), e)
             return False
-        except OSError as _err:
-            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err)
+        except OSError as e:
+            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), e)
             return False
 
         floppy_dev = '/dev/fd0'
@@ -197,11 +197,11 @@ class DataSourceAltCloud(sources.DataSource):
         try:
             (cmd_out, _err) = util.udevadm_settle(exists=floppy_dev, timeout=5)
             LOG.debug('Command: %s\nOutput%s', ' '.join(cmd), cmd_out)
-        except ProcessExecutionError as _err:
-            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err)
+        except ProcessExecutionError as e:
+            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), e)
             return False
-        except OSError as _err:
-            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err)
+        except OSError as e:
+            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), e)
             return False
 
         try:
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 5d3a8ddb..2daea59d 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -78,7 +78,7 @@ class DataSourceNoCloud(sources.DataSource):
                 LOG.debug("Using seeded data from %s", path)
                 mydata = _merge_new_seed(mydata, seeded)
                 break
-            except ValueError as e:
+            except ValueError:
                 pass
 
         # If the datasource config had a 'seedfrom' entry, then that takes
@@ -117,7 +117,7 @@ class DataSourceNoCloud(sources.DataSource):
                     try:
                         seeded = util.mount_cb(dev, _pp2d_callback,
                                                pp2d_kwargs)
-                    except ValueError as e:
+                    except ValueError:
                         if dev in label_list:
                             LOG.warning("device %s with label=%s not a"
                                         "valid seed.", dev, label)
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index d4a41116..16c10785 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -378,7 +378,7 @@ def read_context_disk_dir(source_dir, asuser=None):
         if asuser is not None:
             try:
                 pwd.getpwnam(asuser)
-            except KeyError as e:
+            except KeyError:
                 raise BrokenContextDiskDir(
                     "configured user '{user}' does not exist".format(
                         user=asuser))
diff --git a/cloudinit/util.py b/cloudinit/util.py
index c0473b8c..6ea6ca76 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -2531,8 +2531,8 @@ def _call_dmidecode(key, dmidecode_path):
         if result.replace(".", "") == "":
             return ""
         return result
-    except (IOError, OSError) as _err:
-        LOG.debug('failed dmidecode cmd: %s\n%s', cmd, _err)
+    except (IOError, OSError) as e:
+        LOG.debug('failed dmidecode cmd: %s\n%s', cmd, e)
         return None
 
 
-- 
cgit v1.2.3


From 1efa8a0a030794cec68197100f31a856d0d264ab Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Fri, 15 Jun 2018 19:33:30 -0600
Subject: openstack: avoid unneeded metadata probe on non-openstack platforms

OpenStack datasource is now discovered in init-local stage. In order to
probe whether OpenStack metadata is present, it performs a costly
sandboxed dhclient setup and metadata probe against http://169.254.169.254
for openstack data.

Cloud-init properly detects non-OpenStack on EC2, but it spends precious
time probing the metadata service also resulting in a confusing WARNING
log about 'metadata not present'. To avoid the wasted cycles, and
confusing warning, get_data will call a detect_openstack function to
quickly determine whether the platform looks like OpenStack before trying
to setup network to probe and crawl the metadata service.

LP: #1776701
---
 cloudinit/sources/DataSourceOpenStack.py          |  23 ++++
 cloudinit/util.py                                 |  13 ++-
 doc/rtd/topics/datasources/openstack.rst          |  15 +++
 tests/unittests/test_datasource/test_openstack.py | 124 +++++++++++++++++++---
 tests/unittests/test_util.py                      |  23 ++++
 5 files changed, 182 insertions(+), 16 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index 1a12a3f1..365af96a 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -23,6 +23,13 @@ DEFAULT_METADATA = {
     "instance-id": DEFAULT_IID,
 }
 
+# OpenStack DMI constants
+DMI_PRODUCT_NOVA = 'OpenStack Nova'
+DMI_PRODUCT_COMPUTE = 'OpenStack Compute'
+VALID_DMI_PRODUCT_NAMES = [DMI_PRODUCT_NOVA, DMI_PRODUCT_COMPUTE]
+DMI_ASSET_TAG_OPENTELEKOM = 'OpenTelekomCloud'
+VALID_DMI_ASSET_TAGS = [DMI_ASSET_TAG_OPENTELEKOM]
+
 
 class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
 
@@ -114,6 +121,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
             False when unable to contact metadata service or when metadata
             format is invalid or disabled.
         """
+        if not detect_openstack():
+            return False
         if self.perform_dhcp_setup:  # Setup networking in init-local stage.
             try:
                 with EphemeralDHCPv4(self.fallback_interface):
@@ -205,6 +214,20 @@ def read_metadata_service(base_url, ssl_details=None,
     return reader.read_v2()
 
 
+def detect_openstack():
+    """Return True when a potential OpenStack platform is detected."""
+    if not util.is_x86():
+        return True  # Non-Intel cpus don't properly report dmi product names
+    product_name = util.read_dmi_data('system-product-name')
+    if product_name in VALID_DMI_PRODUCT_NAMES:
+        return True
+    elif util.read_dmi_data('chassis-asset-tag') in VALID_DMI_ASSET_TAGS:
+        return True
+    elif util.get_proc_env(1).get('product_name') == DMI_PRODUCT_NOVA:
+        return True
+    return False
+
+
 # Used to match classes to dependencies
 datasources = [
     (DataSourceOpenStackLocal, (sources.DEP_FILESYSTEM,)),
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 26a41122..6da95113 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -2629,6 +2629,16 @@ def _call_dmidecode(key, dmidecode_path):
         return None
 
 
+def is_x86(uname_arch=None):
+    """Return True if platform is x86-based"""
+    if uname_arch is None:
+        uname_arch = os.uname()[4]
+    x86_arch_match = (
+        uname_arch == 'x86_64' or
+        (uname_arch[0] == 'i' and uname_arch[2:] == '86'))
+    return x86_arch_match
+
+
 def read_dmi_data(key):
     """
     Wrapper for reading DMI data.
@@ -2656,8 +2666,7 @@ def read_dmi_data(key):
 
     # running dmidecode can be problematic on some arches (LP: #1243287)
     uname_arch = os.uname()[4]
-    if not (uname_arch == "x86_64" or
-            (uname_arch.startswith("i") and uname_arch[2:] == "86") or
+    if not (is_x86(uname_arch) or
             uname_arch == 'aarch64' or
             uname_arch == 'amd64'):
         LOG.debug("dmidata is not supported on %s", uname_arch)
diff --git a/doc/rtd/topics/datasources/openstack.rst b/doc/rtd/topics/datasources/openstack.rst
index 0ea89943..421da08f 100644
--- a/doc/rtd/topics/datasources/openstack.rst
+++ b/doc/rtd/topics/datasources/openstack.rst
@@ -7,6 +7,21 @@ This datasource supports reading data from the
 `OpenStack Metadata Service
 <https://docs.openstack.org/nova/latest/admin/networking-nova.html#metadata-service>`_.
 
+Discovery
+-------------
+To determine whether a platform looks like it may be OpenStack, cloud-init
+checks the following environment attributes as a potential OpenStack platform:
+
+ * Maybe OpenStack if
+
+   * **non-x86 cpu architecture**: because DMI data is buggy on some arches
+ * Is OpenStack **if x86 architecture and ANY** of the following
+
+   * **/proc/1/environ**: Nova-lxd contains *product_name=OpenStack Nova*
+   * **DMI product_name**: Either *Openstack Nova* or *OpenStack Compute*
+   * **DMI chassis_asset_tag** is *OpenTelekomCloud*
+
+
 Configuration
 -------------
 The following configuration can be set for the datasource in system
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index fad73b21..585acc33 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -69,6 +69,8 @@ EC2_VERSIONS = [
     'latest',
 ]
 
+MOCK_PATH = 'cloudinit.sources.DataSourceOpenStack.'
+
 
 # TODO _register_uris should leverage test_ec2.register_mock_metaserver.
 def _register_uris(version, ec2_files, ec2_meta, os_files):
@@ -231,7 +233,10 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         ds_os = ds.DataSourceOpenStack(
             settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp}))
         self.assertIsNone(ds_os.version)
-        found = ds_os.get_data()
+        mock_path = MOCK_PATH + 'detect_openstack'
+        with test_helpers.mock.patch(mock_path) as m_detect_os:
+            m_detect_os.return_value = True
+            found = ds_os.get_data()
         self.assertTrue(found)
         self.assertEqual(2, ds_os.version)
         md = dict(ds_os.metadata)
@@ -260,7 +265,10 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             'broadcast-address': '192.168.2.255'}]
 
         self.assertIsNone(ds_os_local.version)
-        found = ds_os_local.get_data()
+        mock_path = MOCK_PATH + 'detect_openstack'
+        with test_helpers.mock.patch(mock_path) as m_detect_os:
+            m_detect_os.return_value = True
+            found = ds_os_local.get_data()
         self.assertTrue(found)
         self.assertEqual(2, ds_os_local.version)
         md = dict(ds_os_local.metadata)
@@ -284,7 +292,10 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
                                        None,
                                        helpers.Paths({'run_dir': self.tmp}))
         self.assertIsNone(ds_os.version)
-        found = ds_os.get_data()
+        mock_path = MOCK_PATH + 'detect_openstack'
+        with test_helpers.mock.patch(mock_path) as m_detect_os:
+            m_detect_os.return_value = True
+            found = ds_os.get_data()
         self.assertFalse(found)
         self.assertIsNone(ds_os.version)
         self.assertIn(
@@ -306,15 +317,16 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             'timeout': 0,
         }
         self.assertIsNone(ds_os.version)
-        found = ds_os.get_data()
+        mock_path = MOCK_PATH + 'detect_openstack'
+        with test_helpers.mock.patch(mock_path) as m_detect_os:
+            m_detect_os.return_value = True
+            found = ds_os.get_data()
         self.assertFalse(found)
         self.assertIsNone(ds_os.version)
 
     def test_network_config_disabled_by_datasource_config(self):
         """The network_config can be disabled from datasource config."""
-        mock_path = (
-            'cloudinit.sources.DataSourceOpenStack.openstack.'
-            'convert_net_json')
+        mock_path = MOCK_PATH + 'openstack.convert_net_json'
         ds_os = ds.DataSourceOpenStack(
             settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp}))
         ds_os.ds_cfg = {'apply_network_config': False}
@@ -327,9 +339,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
 
     def test_network_config_from_network_json(self):
         """The datasource gets network_config from network_data.json."""
-        mock_path = (
-            'cloudinit.sources.DataSourceOpenStack.openstack.'
-            'convert_net_json')
+        mock_path = MOCK_PATH + 'openstack.convert_net_json'
         example_cfg = {'version': 1, 'config': []}
         ds_os = ds.DataSourceOpenStack(
             settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp}))
@@ -345,9 +355,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
 
     def test_network_config_cached(self):
         """The datasource caches the network_config property."""
-        mock_path = (
-            'cloudinit.sources.DataSourceOpenStack.openstack.'
-            'convert_net_json')
+        mock_path = MOCK_PATH + 'openstack.convert_net_json'
         example_cfg = {'version': 1, 'config': []}
         ds_os = ds.DataSourceOpenStack(
             settings.CFG_BUILTIN, None, helpers.Paths({'run_dir': self.tmp}))
@@ -374,7 +382,10 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             'timeout': 0,
         }
         self.assertIsNone(ds_os.version)
-        found = ds_os.get_data()
+        mock_path = MOCK_PATH + 'detect_openstack'
+        with test_helpers.mock.patch(mock_path) as m_detect_os:
+            m_detect_os.return_value = True
+            found = ds_os.get_data()
         self.assertFalse(found)
         self.assertIsNone(ds_os.version)
 
@@ -438,4 +449,89 @@ class TestVendorDataLoading(test_helpers.TestCase):
         data = {'foo': 'bar', 'cloud-init': ['VD_1', 'VD_2']}
         self.assertEqual(self.cvj(data), data['cloud-init'])
 
+
+@test_helpers.mock.patch(MOCK_PATH + 'util.is_x86')
+class TestDetectOpenStack(test_helpers.CiTestCase):
+
+    def test_detect_openstack_non_intel_x86(self, m_is_x86):
+        """Return True on non-intel platforms because dmi isn't conclusive."""
+        m_is_x86.return_value = False
+        self.assertTrue(
+            ds.detect_openstack(), 'Expected detect_openstack == True')
+
+    @test_helpers.mock.patch(MOCK_PATH + 'util.get_proc_env')
+    @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data')
+    def test_not_detect_openstack_intel_x86_ec2(self, m_dmi, m_proc_env,
+                                                m_is_x86):
+        """Return False on EC2 platforms."""
+        m_is_x86.return_value = True
+        # No product_name in proc/1/environ
+        m_proc_env.return_value = {'HOME': '/'}
+
+        def fake_dmi_read(dmi_key):
+            if dmi_key == 'system-product-name':
+                return 'HVM domU'  # Nothing 'openstackish' on EC2
+            if dmi_key == 'chassis-asset-tag':
+                return ''  # Empty string on EC2
+            assert False, 'Unexpected dmi read of %s' % dmi_key
+
+        m_dmi.side_effect = fake_dmi_read
+        self.assertFalse(
+            ds.detect_openstack(), 'Expected detect_openstack == False on EC2')
+        m_proc_env.assert_called_with(1)
+
+    @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data')
+    def test_detect_openstack_intel_product_name_compute(self, m_dmi,
+                                                         m_is_x86):
+        """Return True on OpenStack compute and nova instances."""
+        m_is_x86.return_value = True
+        openstack_product_names = ['OpenStack Nova', 'OpenStack Compute']
+
+        for product_name in openstack_product_names:
+            m_dmi.return_value = product_name
+            self.assertTrue(
+                ds.detect_openstack(), 'Failed to detect_openstack')
+
+    @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data')
+    def test_detect_openstack_opentelekomcloud_chassis_asset_tag(self, m_dmi,
+                                                                 m_is_x86):
+        """Return True on OpenStack reporting OpenTelekomCloud asset-tag."""
+        m_is_x86.return_value = True
+
+        def fake_dmi_read(dmi_key):
+            if dmi_key == 'system-product-name':
+                return 'HVM domU'  # Nothing 'openstackish' on OpenTelekomCloud
+            if dmi_key == 'chassis-asset-tag':
+                return 'OpenTelekomCloud'
+            assert False, 'Unexpected dmi read of %s' % dmi_key
+
+        m_dmi.side_effect = fake_dmi_read
+        self.assertTrue(
+            ds.detect_openstack(),
+            'Expected detect_openstack == True on OpenTelekomCloud')
+
+    @test_helpers.mock.patch(MOCK_PATH + 'util.get_proc_env')
+    @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data')
+    def test_detect_openstack_by_proc_1_environ(self, m_dmi, m_proc_env,
+                                                m_is_x86):
+        """Return True when nova product_name specified in /proc/1/environ."""
+        m_is_x86.return_value = True
+        # Nova product_name in proc/1/environ
+        m_proc_env.return_value = {
+            'HOME': '/', 'product_name': 'OpenStack Nova'}
+
+        def fake_dmi_read(dmi_key):
+            if dmi_key == 'system-product-name':
+                return 'HVM domU'  # Nothing 'openstackish'
+            if dmi_key == 'chassis-asset-tag':
+                return ''  # Nothin 'openstackish'
+            assert False, 'Unexpected dmi read of %s' % dmi_key
+
+        m_dmi.side_effect = fake_dmi_read
+        self.assertTrue(
+            ds.detect_openstack(),
+            'Expected detect_openstack == True on OpenTelekomCloud')
+        m_proc_env.assert_called_with(1)
+
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index 20479f66..7a203ce2 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -468,6 +468,29 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase):
         self.assertIsNone(ret)
 
 
+class TestIsX86(helpers.CiTestCase):
+
+    def test_is_x86_matches_x86_types(self):
+        """is_x86 returns True if CPU architecture matches."""
+        matched_arches = ['x86_64', 'i386', 'i586', 'i686']
+        for arch in matched_arches:
+            self.assertTrue(
+                util.is_x86(arch), 'Expected is_x86 for arch "%s"' % arch)
+
+    def test_is_x86_unmatched_types(self):
+        """is_x86 returns Fale on non-intel x86 architectures."""
+        unmatched_arches = ['ia64', '9000/800', 'arm64v71']
+        for arch in unmatched_arches:
+            self.assertFalse(
+                util.is_x86(arch), 'Expected not is_x86 for arch "%s"' % arch)
+
+    @mock.patch('cloudinit.util.os.uname')
+    def test_is_x86_calls_uname_for_architecture(self, m_uname):
+        """is_x86 returns True if platform from uname matches."""
+        m_uname.return_value = [0, 1, 2, 3, 'x86_64']
+        self.assertTrue(util.is_x86())
+
+
 class TestReadDMIData(helpers.FilesystemMockingTestCase):
 
     def setUp(self):
-- 
cgit v1.2.3


From be9ecc12823607b4709b64408aee137bfdfc7d01 Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Sun, 1 Jul 2018 16:46:23 -0600
Subject: update_metadata: a datasource can support network re-config every
 boot

Very basic type definitions are now defined to distinguish 'boot'
events from 'new instance (first boot)'. Event types will now be handed
to a datasource.update_metadata method which can determine whether
to refresh its metadata and re-render configuration based on that
source event.

A datasource can 'subscribe' to an event by setting up the update_events
attribute on the datasource class which describe what config scope is
updated by a list of matching events. By default datasources will have
the following update_events: {'network': [EventType.BOOT_NEW_INSTANCE]}
This setting says the datasource will re-write network configuration only
on first boot of a new instance or when the instance id changes.

New methods are now present on the datasource:
  - clear_cached_attrs: Resets cached datasource attributes to values
    listed in datasource.cached_attr_defaults. This is performed prior to
    processing a fresh metadata process to avoid keeping old/invalid
    cached data around.
  - update_metadata: accepts source_event_types to determine if the
    metadata should be crawled again and processed
---
 cloudinit/event.py                   |  17 +++
 cloudinit/sources/__init__.py        |  78 +++++++++++-
 cloudinit/sources/tests/test_init.py |  83 ++++++++++++-
 cloudinit/stages.py                  |  14 ++-
 cloudinit/tests/test_stages.py       | 231 +++++++++++++++++++++++++++++++++++
 5 files changed, 417 insertions(+), 6 deletions(-)
 create mode 100644 cloudinit/event.py
 create mode 100644 cloudinit/tests/test_stages.py

(limited to 'cloudinit/sources')

diff --git a/cloudinit/event.py b/cloudinit/event.py
new file mode 100644
index 00000000..f7b311fb
--- /dev/null
+++ b/cloudinit/event.py
@@ -0,0 +1,17 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Classes and functions related to event handling."""
+
+
+# Event types which can generate maintenance requests for cloud-init.
+class EventType(object):
+    BOOT = "System boot"
+    BOOT_NEW_INSTANCE = "New instance first boot"
+
+    # TODO: Cloud-init will grow support for the follow event types:
+    # UDEV
+    # METADATA_CHANGE
+    # USER_REQUEST
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 90d74575..f424316a 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -19,6 +19,7 @@ from cloudinit.atomic_helper import write_json
 from cloudinit import importer
 from cloudinit import log as logging
 from cloudinit import net
+from cloudinit.event import EventType
 from cloudinit import type_utils
 from cloudinit import user_data as ud
 from cloudinit import util
@@ -102,6 +103,25 @@ class DataSource(object):
     url_timeout = 10    # timeout for each metadata url read attempt
     url_retries = 5     # number of times to retry url upon 404
 
+    # The datasource defines a list of supported EventTypes during which
+    # the datasource can react to changes in metadata and regenerate
+    # network configuration on metadata changes.
+    # A datasource which supports writing network config on each system boot
+    # would set update_events = {'network': [EventType.BOOT]}
+
+    # Default: generate network config on new instance id (first boot).
+    update_events = {'network': [EventType.BOOT_NEW_INSTANCE]}
+
+    # N-tuple listing default values for any metadata-related class
+    # attributes cached on an instance by a process_data runs. These attribute
+    # values are reset via clear_cached_attrs during any update_metadata call.
+    cached_attr_defaults = (
+        ('ec2_metadata', UNSET), ('network_json', UNSET),
+        ('metadata', {}), ('userdata', None), ('userdata_raw', None),
+        ('vendordata', None), ('vendordata_raw', None))
+
+    _dirty_cache = False
+
     def __init__(self, sys_cfg, distro, paths, ud_proc=None):
         self.sys_cfg = sys_cfg
         self.distro = distro
@@ -134,11 +154,31 @@ class DataSource(object):
             'region': self.region,
             'availability-zone': self.availability_zone}}
 
+    def clear_cached_attrs(self, attr_defaults=()):
+        """Reset any cached metadata attributes to datasource defaults.
+
+        @param attr_defaults: Optional tuple of (attr, value) pairs to
+           set instead of cached_attr_defaults.
+        """
+        if not self._dirty_cache:
+            return
+        if attr_defaults:
+            attr_values = attr_defaults
+        else:
+            attr_values = self.cached_attr_defaults
+
+        for attribute, value in attr_values:
+            if hasattr(self, attribute):
+                setattr(self, attribute, value)
+        if not attr_defaults:
+            self._dirty_cache = False
+
     def get_data(self):
         """Datasources implement _get_data to setup metadata and userdata_raw.
 
         Minimally, the datasource should return a boolean True on success.
         """
+        self._dirty_cache = True
         return_value = self._get_data()
         json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE)
         if not return_value:
@@ -174,6 +214,7 @@ class DataSource(object):
         return return_value
 
     def _get_data(self):
+        """Walk metadata sources, process crawled data and save attributes."""
         raise NotImplementedError(
             'Subclasses of DataSource must implement _get_data which'
             ' sets self.metadata, vendordata_raw and userdata_raw.')
@@ -416,6 +457,41 @@ class DataSource(object):
     def get_package_mirror_info(self):
         return self.distro.get_package_mirror_info(data_source=self)
 
+    def update_metadata(self, source_event_types):
+        """Refresh cached metadata if the datasource supports this event.
+
+        The datasource has a list of update_events which
+        trigger refreshing all cached metadata as well as refreshing the
+        network configuration.
+
+        @param source_event_types: List of EventTypes which may trigger a
+            metadata update.
+
+        @return True if the datasource did successfully update cached metadata
+            due to source_event_type.
+        """
+        supported_events = {}
+        for event in source_event_types:
+            for update_scope, update_events in self.update_events.items():
+                if event in update_events:
+                    if not supported_events.get(update_scope):
+                        supported_events[update_scope] = []
+                    supported_events[update_scope].append(event)
+        for scope, matched_events in supported_events.items():
+            LOG.debug(
+                "Update datasource metadata and %s config due to events: %s",
+                scope, ', '.join(matched_events))
+            # Each datasource has a cached config property which needs clearing
+            # Once cleared that config property will be regenerated from
+            # current metadata.
+            self.clear_cached_attrs((('_%s_config' % scope, UNSET),))
+        if supported_events:
+            self.clear_cached_attrs()
+            result = self.get_data()
+            if result:
+                return True
+        return False
+
     def check_instance_id(self, sys_cfg):
         # quickly (local check only) if self.instance_id is still
         return False
@@ -520,7 +596,7 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter):
             with myrep:
                 LOG.debug("Seeing if we can get any data from %s", cls)
                 s = cls(sys_cfg, distro, paths)
-                if s.get_data():
+                if s.update_metadata([EventType.BOOT_NEW_INSTANCE]):
                     myrep.message = "found %s data from %s" % (mode, name)
                     return (s, type_utils.obj_name(cls))
         except Exception:
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index d5bc98a4..dcd221be 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -5,10 +5,11 @@ import os
 import six
 import stat
 
+from cloudinit.event import EventType
 from cloudinit.helpers import Paths
 from cloudinit import importer
 from cloudinit.sources import (
-    INSTANCE_JSON_FILE, DataSource)
+    INSTANCE_JSON_FILE, DataSource, UNSET)
 from cloudinit.tests.helpers import CiTestCase, skipIf, mock
 from cloudinit.user_data import UserDataProcessor
 from cloudinit import util
@@ -381,3 +382,83 @@ class TestDataSource(CiTestCase):
                     get_args(grandchild.get_hostname),  # pylint: disable=W1505
                     '%s does not implement DataSource.get_hostname params'
                     % grandchild)
+
+    def test_clear_cached_attrs_resets_cached_attr_class_attributes(self):
+        """Class attributes listed in cached_attr_defaults are reset."""
+        count = 0
+        # Setup values for all cached class attributes
+        for attr, value in self.datasource.cached_attr_defaults:
+            setattr(self.datasource, attr, count)
+            count += 1
+        self.datasource._dirty_cache = True
+        self.datasource.clear_cached_attrs()
+        for attr, value in self.datasource.cached_attr_defaults:
+            self.assertEqual(value, getattr(self.datasource, attr))
+
+    def test_clear_cached_attrs_noops_on_clean_cache(self):
+        """Class attributes listed in cached_attr_defaults are reset."""
+        count = 0
+        # Setup values for all cached class attributes
+        for attr, _ in self.datasource.cached_attr_defaults:
+            setattr(self.datasource, attr, count)
+            count += 1
+        self.datasource._dirty_cache = False   # Fake clean cache
+        self.datasource.clear_cached_attrs()
+        count = 0
+        for attr, _ in self.datasource.cached_attr_defaults:
+            self.assertEqual(count, getattr(self.datasource, attr))
+            count += 1
+
+    def test_clear_cached_attrs_skips_non_attr_class_attributes(self):
+        """Skip any cached_attr_defaults which aren't class attributes."""
+        self.datasource._dirty_cache = True
+        self.datasource.clear_cached_attrs()
+        for attr in ('ec2_metadata', 'network_json'):
+            self.assertFalse(hasattr(self.datasource, attr))
+
+    def test_clear_cached_attrs_of_custom_attrs(self):
+        """Custom attr_values can be passed to clear_cached_attrs."""
+        self.datasource._dirty_cache = True
+        cached_attr_name = self.datasource.cached_attr_defaults[0][0]
+        setattr(self.datasource, cached_attr_name, 'himom')
+        self.datasource.myattr = 'orig'
+        self.datasource.clear_cached_attrs(
+            attr_defaults=(('myattr', 'updated'),))
+        self.assertEqual('himom', getattr(self.datasource, cached_attr_name))
+        self.assertEqual('updated', self.datasource.myattr)
+
+    def test_update_metadata_only_acts_on_supported_update_events(self):
+        """update_metadata won't get_data on unsupported update events."""
+        self.assertEqual(
+            {'network': [EventType.BOOT_NEW_INSTANCE]},
+            self.datasource.update_events)
+
+        def fake_get_data():
+            raise Exception('get_data should not be called')
+
+        self.datasource.get_data = fake_get_data
+        self.assertFalse(
+            self.datasource.update_metadata(
+                source_event_types=[EventType.BOOT]))
+
+    def test_update_metadata_returns_true_on_supported_update_event(self):
+        """update_metadata returns get_data response on supported events."""
+
+        def fake_get_data():
+            return True
+
+        self.datasource.get_data = fake_get_data
+        self.datasource._network_config = 'something'
+        self.datasource._dirty_cache = True
+        self.assertTrue(
+            self.datasource.update_metadata(
+                source_event_types=[
+                    EventType.BOOT, EventType.BOOT_NEW_INSTANCE]))
+        self.assertEqual(UNSET, self.datasource._network_config)
+        self.assertIn(
+            "DEBUG: Update datasource metadata and network config due to"
+            " events: New instance first boot",
+            self.logs.getvalue())
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 286607bf..c132b57d 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -22,6 +22,8 @@ from cloudinit.handlers import cloud_config as cc_part
 from cloudinit.handlers import shell_script as ss_part
 from cloudinit.handlers import upstart_job as up_part
 
+from cloudinit.event import EventType
+
 from cloudinit import cloud
 from cloudinit import config
 from cloudinit import distros
@@ -648,10 +650,14 @@ class Init(object):
         except Exception as e:
             LOG.warning("Failed to rename devices: %s", e)
 
-        if (self.datasource is not NULL_DATA_SOURCE and
-                not self.is_new_instance()):
-            LOG.debug("not a new instance. network config is not applied.")
-            return
+        if self.datasource is not NULL_DATA_SOURCE:
+            if not self.is_new_instance():
+                if not self.datasource.update_metadata([EventType.BOOT]):
+                    LOG.debug(
+                        "No network config applied. Neither a new instance"
+                        " nor datasource network update on '%s' event",
+                        EventType.BOOT)
+                    return
 
         LOG.info("Applying network configuration from %s bringup=%s: %s",
                  src, bring_up, netcfg)
diff --git a/cloudinit/tests/test_stages.py b/cloudinit/tests/test_stages.py
new file mode 100644
index 00000000..94b6b255
--- /dev/null
+++ b/cloudinit/tests/test_stages.py
@@ -0,0 +1,231 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Tests related to cloudinit.stages module."""
+
+import os
+
+from cloudinit import stages
+from cloudinit import sources
+
+from cloudinit.event import EventType
+from cloudinit.util import write_file
+
+from cloudinit.tests.helpers import CiTestCase, mock
+
+TEST_INSTANCE_ID = 'i-testing'
+
+
+class FakeDataSource(sources.DataSource):
+
+    def __init__(self, paths=None, userdata=None, vendordata=None,
+                 network_config=''):
+        super(FakeDataSource, self).__init__({}, None, paths=paths)
+        self.metadata = {'instance-id': TEST_INSTANCE_ID}
+        self.userdata_raw = userdata
+        self.vendordata_raw = vendordata
+        self._network_config = None
+        if network_config:   # Permit for None value to setup attribute
+            self._network_config = network_config
+
+    @property
+    def network_config(self):
+        return self._network_config
+
+    def _get_data(self):
+        return True
+
+
+class TestInit(CiTestCase):
+    with_logs = True
+
+    def setUp(self):
+        super(TestInit, self).setUp()
+        self.tmpdir = self.tmp_dir()
+        self.init = stages.Init()
+        # Setup fake Paths for Init to reference
+        self.init._cfg = {'system_info': {
+            'distro': 'ubuntu', 'paths': {'cloud_dir': self.tmpdir,
+                                          'run_dir': self.tmpdir}}}
+        self.init.datasource = FakeDataSource(paths=self.init.paths)
+
+    def test_wb__find_networking_config_disabled(self):
+        """find_networking_config returns no config when disabled."""
+        disable_file = os.path.join(
+            self.init.paths.get_cpath('data'), 'upgraded-network')
+        write_file(disable_file, '')
+        self.assertEqual(
+            (None, disable_file),
+            self.init._find_networking_config())
+
+    @mock.patch('cloudinit.stages.cmdline.read_kernel_cmdline_config')
+    def test_wb__find_networking_config_disabled_by_kernel(self, m_cmdline):
+        """find_networking_config returns when disabled by kernel cmdline."""
+        m_cmdline.return_value = {'config': 'disabled'}
+        self.assertEqual(
+            (None, 'cmdline'),
+            self.init._find_networking_config())
+        self.assertEqual('DEBUG: network config disabled by cmdline\n',
+                         self.logs.getvalue())
+
+    @mock.patch('cloudinit.stages.cmdline.read_kernel_cmdline_config')
+    def test_wb__find_networking_config_disabled_by_datasrc(self, m_cmdline):
+        """find_networking_config returns when disabled by datasource cfg."""
+        m_cmdline.return_value = {}  # Kernel doesn't disable networking
+        self.init._cfg = {'system_info': {'paths': {'cloud_dir': self.tmpdir}},
+                          'network': {}}  # system config doesn't disable
+
+        self.init.datasource = FakeDataSource(
+            network_config={'config': 'disabled'})
+        self.assertEqual(
+            (None, 'ds'),
+            self.init._find_networking_config())
+        self.assertEqual('DEBUG: network config disabled by ds\n',
+                         self.logs.getvalue())
+
+    @mock.patch('cloudinit.stages.cmdline.read_kernel_cmdline_config')
+    def test_wb__find_networking_config_disabled_by_sysconfig(self, m_cmdline):
+        """find_networking_config returns when disabled by system config."""
+        m_cmdline.return_value = {}  # Kernel doesn't disable networking
+        self.init._cfg = {'system_info': {'paths': {'cloud_dir': self.tmpdir}},
+                          'network': {'config': 'disabled'}}
+        self.assertEqual(
+            (None, 'system_cfg'),
+            self.init._find_networking_config())
+        self.assertEqual('DEBUG: network config disabled by system_cfg\n',
+                         self.logs.getvalue())
+
+    @mock.patch('cloudinit.stages.cmdline.read_kernel_cmdline_config')
+    def test_wb__find_networking_config_returns_kernel(self, m_cmdline):
+        """find_networking_config returns kernel cmdline config if present."""
+        expected_cfg = {'config': ['fakekernel']}
+        m_cmdline.return_value = expected_cfg
+        self.init._cfg = {'system_info': {'paths': {'cloud_dir': self.tmpdir}},
+                          'network': {'config': ['fakesys_config']}}
+        self.init.datasource = FakeDataSource(
+            network_config={'config': ['fakedatasource']})
+        self.assertEqual(
+            (expected_cfg, 'cmdline'),
+            self.init._find_networking_config())
+
+    @mock.patch('cloudinit.stages.cmdline.read_kernel_cmdline_config')
+    def test_wb__find_networking_config_returns_system_cfg(self, m_cmdline):
+        """find_networking_config returns system config when present."""
+        m_cmdline.return_value = {}  # No kernel network config
+        expected_cfg = {'config': ['fakesys_config']}
+        self.init._cfg = {'system_info': {'paths': {'cloud_dir': self.tmpdir}},
+                          'network': expected_cfg}
+        self.init.datasource = FakeDataSource(
+            network_config={'config': ['fakedatasource']})
+        self.assertEqual(
+            (expected_cfg, 'system_cfg'),
+            self.init._find_networking_config())
+
+    @mock.patch('cloudinit.stages.cmdline.read_kernel_cmdline_config')
+    def test_wb__find_networking_config_returns_datasrc_cfg(self, m_cmdline):
+        """find_networking_config returns datasource net config if present."""
+        m_cmdline.return_value = {}  # No kernel network config
+        # No system config for network in setUp
+        expected_cfg = {'config': ['fakedatasource']}
+        self.init.datasource = FakeDataSource(network_config=expected_cfg)
+        self.assertEqual(
+            (expected_cfg, 'ds'),
+            self.init._find_networking_config())
+
+    @mock.patch('cloudinit.stages.cmdline.read_kernel_cmdline_config')
+    def test_wb__find_networking_config_returns_fallback(self, m_cmdline):
+        """find_networking_config returns fallback config if not defined."""
+        m_cmdline.return_value = {}  # Kernel doesn't disable networking
+        # Neither datasource nor system_info disable or provide network
+
+        fake_cfg = {'config': [{'type': 'physical', 'name': 'eth9'}],
+                    'version': 1}
+
+        def fake_generate_fallback():
+            return fake_cfg
+
+        # Monkey patch distro which gets cached on self.init
+        distro = self.init.distro
+        distro.generate_fallback_config = fake_generate_fallback
+        self.assertEqual(
+            (fake_cfg, 'fallback'),
+            self.init._find_networking_config())
+        self.assertNotIn('network config disabled', self.logs.getvalue())
+
+    def test_apply_network_config_disabled(self):
+        """Log when network is disabled by upgraded-network."""
+        disable_file = os.path.join(
+            self.init.paths.get_cpath('data'), 'upgraded-network')
+
+        def fake_network_config():
+            return (None, disable_file)
+
+        self.init._find_networking_config = fake_network_config
+
+        self.init.apply_network_config(True)
+        self.assertIn(
+            'INFO: network config is disabled by %s' % disable_file,
+            self.logs.getvalue())
+
+    @mock.patch('cloudinit.distros.ubuntu.Distro')
+    def test_apply_network_on_new_instance(self, m_ubuntu):
+        """Call distro apply_network_config methods on is_new_instance."""
+        net_cfg = {
+            'version': 1, 'config': [
+                {'subnets': [{'type': 'dhcp'}], 'type': 'physical',
+                 'name': 'eth9', 'mac_address': '42:42:42:42:42:42'}]}
+
+        def fake_network_config():
+            return net_cfg, 'fallback'
+
+        self.init._find_networking_config = fake_network_config
+        self.init.apply_network_config(True)
+        self.init.distro.apply_network_config_names.assert_called_with(net_cfg)
+        self.init.distro.apply_network_config.assert_called_with(
+            net_cfg, bring_up=True)
+
+    @mock.patch('cloudinit.distros.ubuntu.Distro')
+    def test_apply_network_on_same_instance_id(self, m_ubuntu):
+        """Only call distro.apply_network_config_names on same instance id."""
+        old_instance_id = os.path.join(
+            self.init.paths.get_cpath('data'), 'instance-id')
+        write_file(old_instance_id, TEST_INSTANCE_ID)
+        net_cfg = {
+            'version': 1, 'config': [
+                {'subnets': [{'type': 'dhcp'}], 'type': 'physical',
+                 'name': 'eth9', 'mac_address': '42:42:42:42:42:42'}]}
+
+        def fake_network_config():
+            return net_cfg, 'fallback'
+
+        self.init._find_networking_config = fake_network_config
+        self.init.apply_network_config(True)
+        self.init.distro.apply_network_config_names.assert_called_with(net_cfg)
+        self.init.distro.apply_network_config.assert_not_called()
+        self.assertIn(
+            'No network config applied. Neither a new instance'
+            " nor datasource network update on '%s' event" % EventType.BOOT,
+            self.logs.getvalue())
+
+    @mock.patch('cloudinit.distros.ubuntu.Distro')
+    def test_apply_network_on_datasource_allowed_event(self, m_ubuntu):
+        """Apply network if datasource.update_metadata permits BOOT event."""
+        old_instance_id = os.path.join(
+            self.init.paths.get_cpath('data'), 'instance-id')
+        write_file(old_instance_id, TEST_INSTANCE_ID)
+        net_cfg = {
+            'version': 1, 'config': [
+                {'subnets': [{'type': 'dhcp'}], 'type': 'physical',
+                 'name': 'eth9', 'mac_address': '42:42:42:42:42:42'}]}
+
+        def fake_network_config():
+            return net_cfg, 'fallback'
+
+        self.init._find_networking_config = fake_network_config
+        self.init.datasource = FakeDataSource(paths=self.init.paths)
+        self.init.datasource.update_events = {'network': [EventType.BOOT]}
+        self.init.apply_network_config(True)
+        self.init.distro.apply_network_config_names.assert_called_with(net_cfg)
+        self.init.distro.apply_network_config.assert_called_with(
+            net_cfg, bring_up=True)
+
+# vi: ts=4 expandtab
-- 
cgit v1.2.3


From 2a9d6203d946484cbe24297fe0e48a42a800756a Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Sat, 21 Jul 2018 15:30:58 +0000
Subject: pylint: Fix pylint warnings reported in pylint 2.0.0.

Pylint 2.0.0 was recently released and complains more about
logging-not-lazy than it used to.  I've fixed those warnings, here.

The changes in rh_subscription are more extensive.  pylint may be
complaining incorrectly there, but the tests were not correctly un-doing
all of their mock/patching.  This cleans those up and makes pylint happy.
---
 cloudinit/config/cc_lxd.py              |  16 +--
 cloudinit/config/cc_rh_subscription.py  |  43 ++++----
 cloudinit/sources/DataSourceSmartOS.py  |   2 +-
 cloudinit/warnings.py                   |   2 +-
 tests/unittests/test_rh_subscription.py | 185 ++++++++++++++++----------------
 5 files changed, 124 insertions(+), 124 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py
index ac72ac4a..a604825a 100644
--- a/cloudinit/config/cc_lxd.py
+++ b/cloudinit/config/cc_lxd.py
@@ -276,27 +276,27 @@ def maybe_cleanup_default(net_name, did_init, create, attach,
     if net_name != _DEFAULT_NETWORK_NAME or not did_init:
         return
 
-    fail_assume_enoent = " failed. Assuming it did not exist."
-    succeeded = " succeeded."
+    fail_assume_enoent = "failed. Assuming it did not exist."
+    succeeded = "succeeded."
     if create:
-        msg = "Deletion of lxd network '%s'" % net_name
+        msg = "Deletion of lxd network '%s' %s"
         try:
             _lxc(["network", "delete", net_name])
-            LOG.debug(msg + succeeded)
+            LOG.debug(msg, net_name, succeeded)
         except util.ProcessExecutionError as e:
             if e.exit_code != 1:
                 raise e
-            LOG.debug(msg + fail_assume_enoent)
+            LOG.debug(msg, net_name, fail_assume_enoent)
 
     if attach:
-        msg = "Removal of device '%s' from profile '%s'" % (nic_name, profile)
+        msg = "Removal of device '%s' from profile '%s' %s"
         try:
             _lxc(["profile", "device", "remove", profile, nic_name])
-            LOG.debug(msg + succeeded)
+            LOG.debug(msg, nic_name, profile, succeeded)
         except util.ProcessExecutionError as e:
             if e.exit_code != 1:
                 raise e
-            LOG.debug(msg + fail_assume_enoent)
+            LOG.debug(msg, nic_name, profile, fail_assume_enoent)
 
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py
index 1c679430..edee01e5 100644
--- a/cloudinit/config/cc_rh_subscription.py
+++ b/cloudinit/config/cc_rh_subscription.py
@@ -126,7 +126,6 @@ class SubscriptionManager(object):
         self.enable_repo = self.rhel_cfg.get('enable-repo')
         self.disable_repo = self.rhel_cfg.get('disable-repo')
         self.servicelevel = self.rhel_cfg.get('service-level')
-        self.subman = ['subscription-manager']
 
     def log_success(self, msg):
         '''Simple wrapper for logging info messages. Useful for unittests'''
@@ -173,21 +172,12 @@ class SubscriptionManager(object):
         cmd = ['identity']
 
         try:
-            self._sub_man_cli(cmd)
+            _sub_man_cli(cmd)
         except util.ProcessExecutionError:
             return False
 
         return True
 
-    def _sub_man_cli(self, cmd, logstring_val=False):
-        '''
-        Uses the prefered cloud-init subprocess def of util.subp
-        and runs subscription-manager.  Breaking this to a
-        separate function for later use in mocking and unittests
-        '''
-        cmd = self.subman + cmd
-        return util.subp(cmd, logstring=logstring_val)
-
     def rhn_register(self):
         '''
         Registers the system by userid and password or activation key
@@ -209,7 +199,7 @@ class SubscriptionManager(object):
                 cmd.append("--serverurl={0}".format(self.server_hostname))
 
             try:
-                return_out = self._sub_man_cli(cmd, logstring_val=True)[0]
+                return_out = _sub_man_cli(cmd, logstring_val=True)[0]
             except util.ProcessExecutionError as e:
                 if e.stdout == "":
                     self.log_warn("Registration failed due "
@@ -232,7 +222,7 @@ class SubscriptionManager(object):
 
             # Attempting to register the system only
             try:
-                return_out = self._sub_man_cli(cmd, logstring_val=True)[0]
+                return_out = _sub_man_cli(cmd, logstring_val=True)[0]
             except util.ProcessExecutionError as e:
                 if e.stdout == "":
                     self.log_warn("Registration failed due "
@@ -255,7 +245,7 @@ class SubscriptionManager(object):
                .format(self.servicelevel)]
 
         try:
-            return_out = self._sub_man_cli(cmd)[0]
+            return_out = _sub_man_cli(cmd)[0]
         except util.ProcessExecutionError as e:
             if e.stdout.rstrip() != '':
                 for line in e.stdout.split("\n"):
@@ -273,7 +263,7 @@ class SubscriptionManager(object):
     def _set_auto_attach(self):
         cmd = ['attach', '--auto']
         try:
-            return_out = self._sub_man_cli(cmd)[0]
+            return_out = _sub_man_cli(cmd)[0]
         except util.ProcessExecutionError as e:
             self.log_warn("Auto-attach failed with: {0}".format(e))
             return False
@@ -292,12 +282,12 @@ class SubscriptionManager(object):
 
         # Get all available pools
         cmd = ['list', '--available', '--pool-only']
-        results = self._sub_man_cli(cmd)[0]
+        results = _sub_man_cli(cmd)[0]
         available = (results.rstrip()).split("\n")
 
         # Get all consumed pools
         cmd = ['list', '--consumed', '--pool-only']
-        results = self._sub_man_cli(cmd)[0]
+        results = _sub_man_cli(cmd)[0]
         consumed = (results.rstrip()).split("\n")
 
         return available, consumed
@@ -309,14 +299,14 @@ class SubscriptionManager(object):
         '''
 
         cmd = ['repos', '--list-enabled']
-        return_out = self._sub_man_cli(cmd)[0]
+        return_out = _sub_man_cli(cmd)[0]
         active_repos = []
         for repo in return_out.split("\n"):
             if "Repo ID:" in repo:
                 active_repos.append((repo.split(':')[1]).strip())
 
         cmd = ['repos', '--list-disabled']
-        return_out = self._sub_man_cli(cmd)[0]
+        return_out = _sub_man_cli(cmd)[0]
 
         inactive_repos = []
         for repo in return_out.split("\n"):
@@ -346,7 +336,7 @@ class SubscriptionManager(object):
         if len(pool_list) > 0:
             cmd.extend(pool_list)
             try:
-                self._sub_man_cli(cmd)
+                _sub_man_cli(cmd)
                 self.log.debug("Attached the following pools to your "
                                "system: %s", (", ".join(pool_list))
                                .replace('--pool=', ''))
@@ -423,7 +413,7 @@ class SubscriptionManager(object):
             cmd.extend(enable_list)
 
         try:
-            self._sub_man_cli(cmd)
+            _sub_man_cli(cmd)
         except util.ProcessExecutionError as e:
             self.log_warn("Unable to alter repos due to {0}".format(e))
             return False
@@ -439,4 +429,15 @@ class SubscriptionManager(object):
     def is_configured(self):
         return bool((self.userid and self.password) or self.activation_key)
 
+
+def _sub_man_cli(cmd, logstring_val=False):
+    '''
+    Uses the prefered cloud-init subprocess def of util.subp
+    and runs subscription-manager.  Breaking this to a
+    separate function for later use in mocking and unittests
+    '''
+    return util.subp(['subscription-manager'] + cmd,
+                     logstring=logstring_val)
+
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index f92e8b5c..ad8cfb9c 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -564,7 +564,7 @@ class JoyentMetadataSerialClient(JoyentMetadataClient):
                     continue
                 LOG.warning('Unexpected response "%s" during flush', response)
             except JoyentMetadataTimeoutException:
-                LOG.warning('Timeout while initializing metadata client. ' +
+                LOG.warning('Timeout while initializing metadata client. '
                             'Is the host metadata service running?')
         LOG.debug('Got "invalid command".  Flush complete.')
         self.fp.timeout = timeout
diff --git a/cloudinit/warnings.py b/cloudinit/warnings.py
index f9f7a63c..1da90c40 100644
--- a/cloudinit/warnings.py
+++ b/cloudinit/warnings.py
@@ -130,7 +130,7 @@ def show_warning(name, cfg=None, sleep=None, mode=True, **kwargs):
         os.path.join(_get_warn_dir(cfg), name),
         topline + "\n".join(fmtlines) + "\n" + topline)
 
-    LOG.warning(topline + "\n".join(fmtlines) + "\n" + closeline)
+    LOG.warning("%s%s\n%s", topline, "\n".join(fmtlines), closeline)
 
     if sleep:
         LOG.debug("sleeping %d seconds for warning '%s'", sleep, name)
diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py
index 22718108..4cd27eed 100644
--- a/tests/unittests/test_rh_subscription.py
+++ b/tests/unittests/test_rh_subscription.py
@@ -8,10 +8,16 @@ import logging
 from cloudinit.config import cc_rh_subscription
 from cloudinit import util
 
-from cloudinit.tests.helpers import TestCase, mock
+from cloudinit.tests.helpers import CiTestCase, mock
 
+SUBMGR = cc_rh_subscription.SubscriptionManager
+SUB_MAN_CLI = 'cloudinit.config.cc_rh_subscription._sub_man_cli'
+
+
+@mock.patch(SUB_MAN_CLI)
+class GoodTests(CiTestCase):
+    with_logs = True
 
-class GoodTests(TestCase):
     def setUp(self):
         super(GoodTests, self).setUp()
         self.name = "cc_rh_subscription"
@@ -19,7 +25,6 @@ class GoodTests(TestCase):
         self.log = logging.getLogger("good_tests")
         self.args = []
         self.handle = cc_rh_subscription.handle
-        self.SM = cc_rh_subscription.SubscriptionManager
 
         self.config = {'rh_subscription':
                        {'username': 'scooby@do.com',
@@ -35,55 +40,47 @@ class GoodTests(TestCase):
                              'disable-repo': ['repo4', 'repo5']
                              }}
 
-    def test_already_registered(self):
+    def test_already_registered(self, m_sman_cli):
         '''
         Emulates a system that is already registered. Ensure it gets
         a non-ProcessExecution error from is_registered()
         '''
-        with mock.patch.object(cc_rh_subscription.SubscriptionManager,
-                               '_sub_man_cli') as mockobj:
-            self.SM.log_success = mock.MagicMock()
-            self.handle(self.name, self.config, self.cloud_init,
-                        self.log, self.args)
-            self.assertEqual(self.SM.log_success.call_count, 1)
-            self.assertEqual(mockobj.call_count, 1)
-
-    def test_simple_registration(self):
+        self.handle(self.name, self.config, self.cloud_init,
+                    self.log, self.args)
+        self.assertEqual(m_sman_cli.call_count, 1)
+        self.assertIn('System is already registered', self.logs.getvalue())
+
+    def test_simple_registration(self, m_sman_cli):
         '''
         Simple registration with username and password
         '''
-        self.SM.log_success = mock.MagicMock()
         reg = "The system has been registered with ID:" \
               " 12345678-abde-abcde-1234-1234567890abc"
-        self.SM._sub_man_cli = mock.MagicMock(
-            side_effect=[util.ProcessExecutionError, (reg, 'bar')])
+        m_sman_cli.side_effect = [util.ProcessExecutionError, (reg, 'bar')]
         self.handle(self.name, self.config, self.cloud_init,
                     self.log, self.args)
-        self.assertIn(mock.call(['identity']),
-                      self.SM._sub_man_cli.call_args_list)
+        self.assertIn(mock.call(['identity']), m_sman_cli.call_args_list)
         self.assertIn(mock.call(['register', '--username=scooby@do.com',
                                  '--password=scooby-snacks'],
                                 logstring_val=True),
-                      self.SM._sub_man_cli.call_args_list)
-
-        self.assertEqual(self.SM.log_success.call_count, 1)
-        self.assertEqual(self.SM._sub_man_cli.call_count, 2)
+                      m_sman_cli.call_args_list)
+        self.assertIn('rh_subscription plugin completed successfully',
+                      self.logs.getvalue())
+        self.assertEqual(m_sman_cli.call_count, 2)
 
     @mock.patch.object(cc_rh_subscription.SubscriptionManager, "_getRepos")
-    @mock.patch.object(cc_rh_subscription.SubscriptionManager, "_sub_man_cli")
-    def test_update_repos_disable_with_none(self, m_sub_man_cli, m_get_repos):
+    def test_update_repos_disable_with_none(self, m_get_repos, m_sman_cli):
         cfg = copy.deepcopy(self.config)
         m_get_repos.return_value = ([], ['repo1'])
-        m_sub_man_cli.return_value = (b'', b'')
         cfg['rh_subscription'].update(
             {'enable-repo': ['repo1'], 'disable-repo': None})
         mysm = cc_rh_subscription.SubscriptionManager(cfg)
         self.assertEqual(True, mysm.update_repos())
         m_get_repos.assert_called_with()
-        self.assertEqual(m_sub_man_cli.call_args_list,
+        self.assertEqual(m_sman_cli.call_args_list,
                          [mock.call(['repos', '--enable=repo1'])])
 
-    def test_full_registration(self):
+    def test_full_registration(self, m_sman_cli):
         '''
         Registration with auto-attach, service-level, adding pools,
         and enabling and disabling yum repos
@@ -93,26 +90,28 @@ class GoodTests(TestCase):
         call_lists.append(['repos', '--disable=repo5', '--enable=repo2',
                            '--enable=repo3'])
         call_lists.append(['attach', '--auto', '--servicelevel=self-support'])
-        self.SM.log_success = mock.MagicMock()
         reg = "The system has been registered with ID:" \
               " 12345678-abde-abcde-1234-1234567890abc"
-        self.SM._sub_man_cli = mock.MagicMock(
-            side_effect=[util.ProcessExecutionError, (reg, 'bar'),
-                         ('Service level set to: self-support', ''),
-                         ('pool1\npool3\n', ''), ('pool2\n', ''), ('', ''),
-                         ('Repo ID: repo1\nRepo ID: repo5\n', ''),
-                         ('Repo ID: repo2\nRepo ID: repo3\nRepo ID: '
-                          'repo4', ''),
-                         ('', '')])
+        m_sman_cli.side_effect = [
+            util.ProcessExecutionError,
+            (reg, 'bar'),
+            ('Service level set to: self-support', ''),
+            ('pool1\npool3\n', ''), ('pool2\n', ''), ('', ''),
+            ('Repo ID: repo1\nRepo ID: repo5\n', ''),
+            ('Repo ID: repo2\nRepo ID: repo3\nRepo ID: repo4', ''),
+            ('', '')]
         self.handle(self.name, self.config_full, self.cloud_init,
                     self.log, self.args)
+        self.assertEqual(m_sman_cli.call_count, 9)
         for call in call_lists:
-            self.assertIn(mock.call(call), self.SM._sub_man_cli.call_args_list)
-        self.assertEqual(self.SM.log_success.call_count, 1)
-        self.assertEqual(self.SM._sub_man_cli.call_count, 9)
+            self.assertIn(mock.call(call), m_sman_cli.call_args_list)
+        self.assertIn("rh_subscription plugin completed successfully",
+                      self.logs.getvalue())
 
 
-class TestBadInput(TestCase):
+@mock.patch(SUB_MAN_CLI)
+class TestBadInput(CiTestCase):
+    with_logs = True
     name = "cc_rh_subscription"
     cloud_init = None
     log = logging.getLogger("bad_tests")
@@ -155,81 +154,81 @@ class TestBadInput(TestCase):
         super(TestBadInput, self).setUp()
         self.handle = cc_rh_subscription.handle
 
-    def test_no_password(self):
-        '''
-        Attempt to register without the password key/value
-        '''
-        self.SM._sub_man_cli = mock.MagicMock(
-            side_effect=[util.ProcessExecutionError, (self.reg, 'bar')])
+    def assert_logged_warnings(self, warnings):
+        logs = self.logs.getvalue()
+        missing = [w for w in warnings if "WARNING: " + w not in logs]
+        self.assertEqual([], missing, "Missing expected warnings.")
+
+    def test_no_password(self, m_sman_cli):
+        '''Attempt to register without the password key/value.'''
+        m_sman_cli.side_effect = [util.ProcessExecutionError,
+                                  (self.reg, 'bar')]
         self.handle(self.name, self.config_no_password, self.cloud_init,
                     self.log, self.args)
-        self.assertEqual(self.SM._sub_man_cli.call_count, 0)
+        self.assertEqual(m_sman_cli.call_count, 0)
 
-    def test_no_org(self):
-        '''
-        Attempt to register without the org key/value
-        '''
-        self.input_is_missing_data(self.config_no_key)
-
-    def test_service_level_without_auto(self):
-        '''
-        Attempt to register using service-level without the auto-attach key
-        '''
-        self.SM.log_warn = mock.MagicMock()
-        self.SM._sub_man_cli = mock.MagicMock(
-            side_effect=[util.ProcessExecutionError, (self.reg, 'bar')])
+    def test_no_org(self, m_sman_cli):
+        '''Attempt to register without the org key/value.'''
+        m_sman_cli.side_effect = [util.ProcessExecutionError]
+        self.handle(self.name, self.config_no_key, self.cloud_init,
+                    self.log, self.args)
+        m_sman_cli.assert_called_with(['identity'])
+        self.assertEqual(m_sman_cli.call_count, 1)
+        self.assert_logged_warnings((
+            'Unable to register system due to incomplete information.',
+            'Use either activationkey and org *or* userid and password',
+            'Registration failed or did not run completely',
+            'rh_subscription plugin did not complete successfully'))
+
+    def test_service_level_without_auto(self, m_sman_cli):
+        '''Attempt to register using service-level without auto-attach key.'''
+        m_sman_cli.side_effect = [util.ProcessExecutionError,
+                                  (self.reg, 'bar')]
         self.handle(self.name, self.config_service, self.cloud_init,
                     self.log, self.args)
-        self.assertEqual(self.SM._sub_man_cli.call_count, 1)
-        self.assertEqual(self.SM.log_warn.call_count, 2)
+        self.assertEqual(m_sman_cli.call_count, 1)
+        self.assert_logged_warnings((
+            'The service-level key must be used in conjunction with ',
+            'rh_subscription plugin did not complete successfully'))
 
-    def test_pool_not_a_list(self):
+    def test_pool_not_a_list(self, m_sman_cli):
         '''
         Register with pools that are not in the format of a list
         '''
-        self.SM.log_warn = mock.MagicMock()
-        self.SM._sub_man_cli = mock.MagicMock(
-            side_effect=[util.ProcessExecutionError, (self.reg, 'bar')])
+        m_sman_cli.side_effect = [util.ProcessExecutionError,
+                                  (self.reg, 'bar')]
         self.handle(self.name, self.config_badpool, self.cloud_init,
                     self.log, self.args)
-        self.assertEqual(self.SM._sub_man_cli.call_count, 2)
-        self.assertEqual(self.SM.log_warn.call_count, 2)
+        self.assertEqual(m_sman_cli.call_count, 2)
+        self.assert_logged_warnings((
+            'Pools must in the format of a list',
+            'rh_subscription plugin did not complete successfully'))
 
-    def test_repo_not_a_list(self):
+    def test_repo_not_a_list(self, m_sman_cli):
         '''
         Register with repos that are not in the format of a list
         '''
-        self.SM.log_warn = mock.MagicMock()
-        self.SM._sub_man_cli = mock.MagicMock(
-            side_effect=[util.ProcessExecutionError, (self.reg, 'bar')])
+        m_sman_cli.side_effect = [util.ProcessExecutionError,
+                                  (self.reg, 'bar')]
         self.handle(self.name, self.config_badrepo, self.cloud_init,
                     self.log, self.args)
-        self.assertEqual(self.SM.log_warn.call_count, 3)
-        self.assertEqual(self.SM._sub_man_cli.call_count, 2)
+        self.assertEqual(m_sman_cli.call_count, 2)
+        self.assert_logged_warnings((
+            'Repo IDs must in the format of a list.',
+            'Unable to add or remove repos',
+            'rh_subscription plugin did not complete successfully'))
 
-    def test_bad_key_value(self):
+    def test_bad_key_value(self, m_sman_cli):
         '''
         Attempt to register with a key that we don't know
         '''
-        self.SM.log_warn = mock.MagicMock()
-        self.SM._sub_man_cli = mock.MagicMock(
-            side_effect=[util.ProcessExecutionError, (self.reg, 'bar')])
+        m_sman_cli.side_effect = [util.ProcessExecutionError,
+                                  (self.reg, 'bar')]
         self.handle(self.name, self.config_badkey, self.cloud_init,
                     self.log, self.args)
-        self.assertEqual(self.SM.log_warn.call_count, 2)
-        self.assertEqual(self.SM._sub_man_cli.call_count, 1)
-
-    def input_is_missing_data(self, config):
-        '''
-        Helper def for tests that having missing information
-        '''
-        self.SM.log_warn = mock.MagicMock()
-        self.SM._sub_man_cli = mock.MagicMock(
-            side_effect=[util.ProcessExecutionError])
-        self.handle(self.name, config, self.cloud_init,
-                    self.log, self.args)
-        self.SM._sub_man_cli.assert_called_with(['identity'])
-        self.assertEqual(self.SM.log_warn.call_count, 4)
-        self.assertEqual(self.SM._sub_man_cli.call_count, 1)
+        self.assertEqual(m_sman_cli.call_count, 1)
+        self.assert_logged_warnings((
+            'fookey is not a valid key for rh_subscription. Valid keys are:',
+            'rh_subscription plugin did not complete successfully'))
 
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From 3146c96a7a740427ac6e65f9b29239f247ecc617 Mon Sep 17 00:00:00 2001
From: Mike Gerdts <mike.gerdts@joyent.com>
Date: Thu, 26 Jul 2018 15:31:52 +0000
Subject: update_metadata re-config on every boot comments and tests not quite
 right

The comment in update_metadata() that explains how a datasource should
enable network reconfig on every boot presumes that
EventType.BOOT_NEW_INSTANCE is a subset of EventType.BOOT. That's not
the case, and as such a datasource that needs to configure networking
when it is a new instance and every boot needs to include both event
types.

To make the situation above easier to debug, update_metadata() now
logs when it returns false.

To make it so that datasources do not need to test before appending to
the update_events['network'], it is changed from a list to a set.

test_update_metadata_only_acts_on_supported_update_events is updated
to allow datasources to support EventType.BOOT.

Author: Mike Gerdts <mike.gerdts@joyent.com>
---
 cloudinit/sources/__init__.py        | 12 +++++++-----
 cloudinit/sources/tests/test_init.py |  3 ++-
 2 files changed, 9 insertions(+), 6 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index f424316a..06e613f8 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -103,14 +103,14 @@ class DataSource(object):
     url_timeout = 10    # timeout for each metadata url read attempt
     url_retries = 5     # number of times to retry url upon 404
 
-    # The datasource defines a list of supported EventTypes during which
+    # The datasource defines a set of supported EventTypes during which
     # the datasource can react to changes in metadata and regenerate
     # network configuration on metadata changes.
     # A datasource which supports writing network config on each system boot
-    # would set update_events = {'network': [EventType.BOOT]}
+    # would call update_events['network'].add(EventType.BOOT).
 
     # Default: generate network config on new instance id (first boot).
-    update_events = {'network': [EventType.BOOT_NEW_INSTANCE]}
+    update_events = {'network': set([EventType.BOOT_NEW_INSTANCE])}
 
     # N-tuple listing default values for any metadata-related class
     # attributes cached on an instance by a process_data runs. These attribute
@@ -475,8 +475,8 @@ class DataSource(object):
             for update_scope, update_events in self.update_events.items():
                 if event in update_events:
                     if not supported_events.get(update_scope):
-                        supported_events[update_scope] = []
-                    supported_events[update_scope].append(event)
+                        supported_events[update_scope] = set()
+                    supported_events[update_scope].add(event)
         for scope, matched_events in supported_events.items():
             LOG.debug(
                 "Update datasource metadata and %s config due to events: %s",
@@ -490,6 +490,8 @@ class DataSource(object):
             result = self.get_data()
             if result:
                 return True
+        LOG.debug("Datasource %s not updated for events: %s", self,
+                  ', '.join(source_event_types))
         return False
 
     def check_instance_id(self, sys_cfg):
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index dcd221be..9e939c1e 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -429,8 +429,9 @@ class TestDataSource(CiTestCase):
 
     def test_update_metadata_only_acts_on_supported_update_events(self):
         """update_metadata won't get_data on unsupported update events."""
+        self.datasource.update_events['network'].discard(EventType.BOOT)
         self.assertEqual(
-            {'network': [EventType.BOOT_NEW_INSTANCE]},
+            {'network': set([EventType.BOOT_NEW_INSTANCE])},
             self.datasource.update_events)
 
         def fake_get_data():
-- 
cgit v1.2.3


From 3cee0bf85fbf12d272422c8eeed63bf06e64570b Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Tue, 31 Jul 2018 18:44:12 +0000
Subject: oracle: fix detect_openstack to report True on OracleCloud.com DMI
 data

The OpenStack datasource in 18.3 changed to detect data in the
init-local stage instead of init-network and attempted to redetect
OpenStackLocal datasource on Oracle across reboots. The function
detect_openstack was added to quickly detect whether a platform is
OpenStack based on dmi product_name or chassis_asset_tag and it was
a bit too strict for Oracle in checking for 'OpenStack Nova'/'Compute'
DMI product_name.

Oracle's DMI product_name reports 'SAtandard PC (i440FX + PIIX, 1996)'
and DMI chassis_asset_tag is 'OracleCloud.com'.

detect_openstack function now adds 'OracleCloud.com' as a supported value
'OracleCloud.com' to valid chassis-asset-tags for the OpenStack
datasource.

LP: #1784685
---
 cloudinit/sources/DataSourceOpenStack.py          |  3 ++-
 tests/unittests/test_datasource/test_openstack.py | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index 365af96a..b9ade90d 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -28,7 +28,8 @@ DMI_PRODUCT_NOVA = 'OpenStack Nova'
 DMI_PRODUCT_COMPUTE = 'OpenStack Compute'
 VALID_DMI_PRODUCT_NAMES = [DMI_PRODUCT_NOVA, DMI_PRODUCT_COMPUTE]
 DMI_ASSET_TAG_OPENTELEKOM = 'OpenTelekomCloud'
-VALID_DMI_ASSET_TAGS = [DMI_ASSET_TAG_OPENTELEKOM]
+DMI_ASSET_TAG_ORACLE_CLOUD = 'OracleCloud.com'
+VALID_DMI_ASSET_TAGS = [DMI_ASSET_TAG_OPENTELEKOM, DMI_ASSET_TAG_ORACLE_CLOUD]
 
 
 class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index 585acc33..d862f4bc 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -510,6 +510,24 @@ class TestDetectOpenStack(test_helpers.CiTestCase):
             ds.detect_openstack(),
             'Expected detect_openstack == True on OpenTelekomCloud')
 
+    @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data')
+    def test_detect_openstack_oraclecloud_chassis_asset_tag(self, m_dmi,
+                                                            m_is_x86):
+        """Return True on OpenStack reporting Oracle cloud asset-tag."""
+        m_is_x86.return_value = True
+
+        def fake_dmi_read(dmi_key):
+            if dmi_key == 'system-product-name':
+                return 'Standard PC (i440FX + PIIX, 1996)'  # No match
+            if dmi_key == 'chassis-asset-tag':
+                return 'OracleCloud.com'
+            assert False, 'Unexpected dmi read of %s' % dmi_key
+
+        m_dmi.side_effect = fake_dmi_read
+        self.assertTrue(
+            ds.detect_openstack(),
+            'Expected detect_openstack == True on OracleCloud.com')
+
     @test_helpers.mock.patch(MOCK_PATH + 'util.get_proc_env')
     @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data')
     def test_detect_openstack_by_proc_1_environ(self, m_dmi, m_proc_env,
-- 
cgit v1.2.3


From 01cfa711bcf31c6c0019d1e936b5b07aa6abc2bc Mon Sep 17 00:00:00 2001
From: Akihiko Ota <skywalker.37th@gmail.com>
Date: Fri, 3 Aug 2018 20:44:59 +0000
Subject: OpenNebula: Fix null gateway6

The OpenNebula data source generates an invalid netplan yaml file
if the IPv6 gateway is not defined in context.sh.

LP: #1768547
---
 cloudinit/sources/DataSourceOpenNebula.py          |   2 +-
 tests/unittests/test_datasource/test_opennebula.py | 408 ++++++++++++++++++++-
 2 files changed, 407 insertions(+), 3 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 16c10785..77ccd128 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -232,7 +232,7 @@ class OpenNebulaNetwork(object):
 
             # Set IPv6 default gateway
             gateway6 = self.get_gateway6(c_dev)
-            if gateway:
+            if gateway6:
                 devconf['gateway6'] = gateway6
 
             # Set DNS servers and search domains
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index ab42f344..36b4d771 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -354,6 +354,412 @@ class TestOpenNebulaNetwork(unittest.TestCase):
 
     system_nics = ('eth0', 'ens3')
 
+    def test_context_devname(self):
+        """Verify context_devname correctly returns mac and name."""
+        context = {
+            'ETH0_MAC': '02:00:0a:12:01:01',
+            'ETH1_MAC': '02:00:0a:12:0f:0f', }
+        expected = {
+            '02:00:0a:12:01:01': 'ETH0',
+            '02:00:0a:12:0f:0f': 'ETH1', }
+        net = ds.OpenNebulaNetwork(context)
+        self.assertEqual(expected, net.context_devname)
+
+    def test_get_nameservers(self):
+        """
+        Verify get_nameservers('device') correctly returns DNS server addresses
+        and search domains.
+        """
+        context = {
+            'DNS': '1.2.3.8',
+            'ETH0_DNS': '1.2.3.6 1.2.3.7',
+            'ETH0_SEARCH_DOMAIN': 'example.com example.org', }
+        expected = {
+            'addresses': ['1.2.3.6', '1.2.3.7', '1.2.3.8'],
+            'search': ['example.com', 'example.org']}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_nameservers('eth0')
+        self.assertEqual(expected, val)
+
+    def test_get_mtu(self):
+        """Verify get_mtu('device') correctly returns MTU size."""
+        context = {'ETH0_MTU': '1280'}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_mtu('eth0')
+        self.assertEqual('1280', val)
+
+    def test_get_ip(self):
+        """Verify get_ip('device') correctly returns IPv4 address."""
+        context = {'ETH0_IP': PUBLIC_IP}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_ip('eth0', MACADDR)
+        self.assertEqual(PUBLIC_IP, val)
+
+    def test_get_ip_emptystring(self):
+        """
+        Verify get_ip('device') correctly returns IPv4 address.
+        It returns IP address created by MAC address if ETH0_IP has empty
+        string.
+        """
+        context = {'ETH0_IP': ''}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_ip('eth0', MACADDR)
+        self.assertEqual(IP_BY_MACADDR, val)
+
+    def test_get_ip6(self):
+        """
+        Verify get_ip6('device') correctly returns IPv6 address.
+        In this case, IPv6 address is Given by ETH0_IP6.
+        """
+        context = {
+            'ETH0_IP6': IP6_GLOBAL,
+            'ETH0_IP6_ULA': '', }
+        expected = [IP6_GLOBAL]
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_ip6('eth0')
+        self.assertEqual(expected, val)
+
+    def test_get_ip6_ula(self):
+        """
+        Verify get_ip6('device') correctly returns IPv6 address.
+        In this case, IPv6 address is Given by ETH0_IP6_ULA.
+        """
+        context = {
+            'ETH0_IP6': '',
+            'ETH0_IP6_ULA': IP6_ULA, }
+        expected = [IP6_ULA]
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_ip6('eth0')
+        self.assertEqual(expected, val)
+
+    def test_get_ip6_dual(self):
+        """
+        Verify get_ip6('device') correctly returns IPv6 address.
+        In this case, IPv6 addresses are Given by ETH0_IP6 and ETH0_IP6_ULA.
+        """
+        context = {
+            'ETH0_IP6': IP6_GLOBAL,
+            'ETH0_IP6_ULA': IP6_ULA, }
+        expected = [IP6_GLOBAL, IP6_ULA]
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_ip6('eth0')
+        self.assertEqual(expected, val)
+
+    def test_get_ip6_prefix(self):
+        """
+        Verify get_ip6_prefix('device') correctly returns IPv6 prefix.
+        """
+        context = {'ETH0_IP6_PREFIX_LENGTH': IP6_PREFIX}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_ip6_prefix('eth0')
+        self.assertEqual(IP6_PREFIX, val)
+
+    def test_get_ip6_prefix_emptystring(self):
+        """
+        Verify get_ip6_prefix('device') correctly returns IPv6 prefix.
+        It returns default value '64' if ETH0_IP6_PREFIX_LENGTH has empty
+        string.
+        """
+        context = {'ETH0_IP6_PREFIX_LENGTH': ''}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_ip6_prefix('eth0')
+        self.assertEqual('64', val)
+
+    def test_get_gateway(self):
+        """
+        Verify get_gateway('device') correctly returns IPv4 default gateway
+        address.
+        """
+        context = {'ETH0_GATEWAY': '1.2.3.5'}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_gateway('eth0')
+        self.assertEqual('1.2.3.5', val)
+
+    def test_get_gateway6(self):
+        """
+        Verify get_gateway6('device') correctly returns IPv6 default gateway
+        address.
+        """
+        context = {'ETH0_GATEWAY6': IP6_GW}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_gateway6('eth0')
+        self.assertEqual(IP6_GW, val)
+
+    def test_get_mask(self):
+        """
+        Verify get_mask('device') correctly returns IPv4 subnet mask.
+        """
+        context = {'ETH0_MASK': '255.255.0.0'}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_mask('eth0')
+        self.assertEqual('255.255.0.0', val)
+
+    def test_get_mask_emptystring(self):
+        """
+        Verify get_mask('device') correctly returns IPv4 subnet mask.
+        It returns default value '255.255.255.0' if ETH0_MASK has empty string.
+        """
+        context = {'ETH0_MASK': ''}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_mask('eth0')
+        self.assertEqual('255.255.255.0', val)
+
+    def test_get_network(self):
+        """
+        Verify get_network('device') correctly returns IPv4 network address.
+        """
+        context = {'ETH0_NETWORK': '1.2.3.0'}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_network('eth0', MACADDR)
+        self.assertEqual('1.2.3.0', val)
+
+    def test_get_network_emptystring(self):
+        """
+        Verify get_network('device') correctly returns IPv4 network address.
+        It returns network address created by MAC address if ETH0_NETWORK has
+        empty string.
+        """
+        context = {'ETH0_NETWORK': ''}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_network('eth0', MACADDR)
+        self.assertEqual('10.18.1.0', val)
+
+    def test_get_field(self):
+        """
+        Verify get_field('device', 'name') returns *context* value.
+        """
+        context = {'ETH9_DUMMY': 'DUMMY_VALUE'}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_field('eth9', 'dummy')
+        self.assertEqual('DUMMY_VALUE', val)
+
+    def test_get_field_withdefaultvalue(self):
+        """
+        Verify get_field('device', 'name', 'default value') returns *context*
+        value.
+        """
+        context = {'ETH9_DUMMY': 'DUMMY_VALUE'}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_field('eth9', 'dummy', 'DEFAULT_VALUE')
+        self.assertEqual('DUMMY_VALUE', val)
+
+    def test_get_field_withdefaultvalue_emptycontext(self):
+        """
+        Verify get_field('device', 'name', 'default value') returns *default*
+        value if context value is empty string.
+        """
+        context = {'ETH9_DUMMY': ''}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_field('eth9', 'dummy', 'DEFAULT_VALUE')
+        self.assertEqual('DEFAULT_VALUE', val)
+
+    def test_get_field_emptycontext(self):
+        """
+        Verify get_field('device', 'name') returns None if context value is
+        empty string.
+        """
+        context = {'ETH9_DUMMY': ''}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_field('eth9', 'dummy')
+        self.assertEqual(None, val)
+
+    def test_get_field_nonecontext(self):
+        """
+        Verify get_field('device', 'name') returns None if context value is
+        None.
+        """
+        context = {'ETH9_DUMMY': None}
+        net = ds.OpenNebulaNetwork(context)
+        val = net.get_field('eth9', 'dummy')
+        self.assertEqual(None, val)
+
+    @mock.patch(DS_PATH + ".get_physical_nics_by_mac")
+    def test_gen_conf_gateway(self, m_get_phys_by_mac):
+        """Test rendering with/without IPv4 gateway"""
+        self.maxDiff = None
+        # empty ETH0_GATEWAY
+        context = {
+            'ETH0_MAC': '02:00:0a:12:01:01',
+            'ETH0_GATEWAY': '', }
+        for nic in self.system_nics:
+            expected = {
+                'version': 2,
+                'ethernets': {
+                    nic: {
+                        'match': {'macaddress': MACADDR},
+                        'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
+            m_get_phys_by_mac.return_value = {MACADDR: nic}
+            net = ds.OpenNebulaNetwork(context)
+            self.assertEqual(net.gen_conf(), expected)
+
+        # set ETH0_GATEWAY
+        context = {
+            'ETH0_MAC': '02:00:0a:12:01:01',
+            'ETH0_GATEWAY': '1.2.3.5', }
+        for nic in self.system_nics:
+            expected = {
+                'version': 2,
+                'ethernets': {
+                    nic: {
+                        'gateway4': '1.2.3.5',
+                        'match': {'macaddress': MACADDR},
+                        'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
+            m_get_phys_by_mac.return_value = {MACADDR: nic}
+            net = ds.OpenNebulaNetwork(context)
+            self.assertEqual(net.gen_conf(), expected)
+
+    @mock.patch(DS_PATH + ".get_physical_nics_by_mac")
+    def test_gen_conf_gateway6(self, m_get_phys_by_mac):
+        """Test rendering with/without IPv6 gateway"""
+        self.maxDiff = None
+        # empty ETH0_GATEWAY6
+        context = {
+            'ETH0_MAC': '02:00:0a:12:01:01',
+            'ETH0_GATEWAY6': '', }
+        for nic in self.system_nics:
+            expected = {
+                'version': 2,
+                'ethernets': {
+                    nic: {
+                        'match': {'macaddress': MACADDR},
+                        'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
+            m_get_phys_by_mac.return_value = {MACADDR: nic}
+            net = ds.OpenNebulaNetwork(context)
+            self.assertEqual(net.gen_conf(), expected)
+
+        # set ETH0_GATEWAY6
+        context = {
+            'ETH0_MAC': '02:00:0a:12:01:01',
+            'ETH0_GATEWAY6': IP6_GW, }
+        for nic in self.system_nics:
+            expected = {
+                'version': 2,
+                'ethernets': {
+                    nic: {
+                        'gateway6': IP6_GW,
+                        'match': {'macaddress': MACADDR},
+                        'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
+            m_get_phys_by_mac.return_value = {MACADDR: nic}
+            net = ds.OpenNebulaNetwork(context)
+            self.assertEqual(net.gen_conf(), expected)
+
+    @mock.patch(DS_PATH + ".get_physical_nics_by_mac")
+    def test_gen_conf_ipv6address(self, m_get_phys_by_mac):
+        """Test rendering with/without IPv6 address"""
+        self.maxDiff = None
+        # empty ETH0_IP6, ETH0_IP6_ULA, ETH0_IP6_PREFIX_LENGTH
+        context = {
+            'ETH0_MAC': '02:00:0a:12:01:01',
+            'ETH0_IP6': '',
+            'ETH0_IP6_ULA': '',
+            'ETH0_IP6_PREFIX_LENGTH': '', }
+        for nic in self.system_nics:
+            expected = {
+                'version': 2,
+                'ethernets': {
+                    nic: {
+                        'match': {'macaddress': MACADDR},
+                        'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
+            m_get_phys_by_mac.return_value = {MACADDR: nic}
+            net = ds.OpenNebulaNetwork(context)
+            self.assertEqual(net.gen_conf(), expected)
+
+        # set ETH0_IP6, ETH0_IP6_ULA, ETH0_IP6_PREFIX_LENGTH
+        context = {
+            'ETH0_MAC': '02:00:0a:12:01:01',
+            'ETH0_IP6': IP6_GLOBAL,
+            'ETH0_IP6_PREFIX_LENGTH': IP6_PREFIX,
+            'ETH0_IP6_ULA': IP6_ULA, }
+        for nic in self.system_nics:
+            expected = {
+                'version': 2,
+                'ethernets': {
+                    nic: {
+                        'match': {'macaddress': MACADDR},
+                        'addresses': [
+                            IP_BY_MACADDR + '/' + IP4_PREFIX,
+                            IP6_GLOBAL + '/' + IP6_PREFIX,
+                            IP6_ULA + '/' + IP6_PREFIX]}}}
+            m_get_phys_by_mac.return_value = {MACADDR: nic}
+            net = ds.OpenNebulaNetwork(context)
+            self.assertEqual(net.gen_conf(), expected)
+
+    @mock.patch(DS_PATH + ".get_physical_nics_by_mac")
+    def test_gen_conf_dns(self, m_get_phys_by_mac):
+        """Test rendering with/without DNS server, search domain"""
+        self.maxDiff = None
+        # empty DNS, ETH0_DNS, ETH0_SEARCH_DOMAIN
+        context = {
+            'ETH0_MAC': '02:00:0a:12:01:01',
+            'DNS': '',
+            'ETH0_DNS': '',
+            'ETH0_SEARCH_DOMAIN': '', }
+        for nic in self.system_nics:
+            expected = {
+                'version': 2,
+                'ethernets': {
+                    nic: {
+                        'match': {'macaddress': MACADDR},
+                        'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
+            m_get_phys_by_mac.return_value = {MACADDR: nic}
+            net = ds.OpenNebulaNetwork(context)
+            self.assertEqual(net.gen_conf(), expected)
+
+        # set DNS, ETH0_DNS, ETH0_SEARCH_DOMAIN
+        context = {
+            'ETH0_MAC': '02:00:0a:12:01:01',
+            'DNS': '1.2.3.8',
+            'ETH0_DNS': '1.2.3.6 1.2.3.7',
+            'ETH0_SEARCH_DOMAIN': 'example.com example.org', }
+        for nic in self.system_nics:
+            expected = {
+                'version': 2,
+                'ethernets': {
+                    nic: {
+                        'nameservers': {
+                            'addresses': ['1.2.3.6', '1.2.3.7', '1.2.3.8'],
+                            'search': ['example.com', 'example.org']},
+                        'match': {'macaddress': MACADDR},
+                        'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
+            m_get_phys_by_mac.return_value = {MACADDR: nic}
+            net = ds.OpenNebulaNetwork(context)
+            self.assertEqual(net.gen_conf(), expected)
+
+    @mock.patch(DS_PATH + ".get_physical_nics_by_mac")
+    def test_gen_conf_mtu(self, m_get_phys_by_mac):
+        """Test rendering with/without MTU"""
+        self.maxDiff = None
+        # empty ETH0_MTU
+        context = {
+            'ETH0_MAC': '02:00:0a:12:01:01',
+            'ETH0_MTU': '', }
+        for nic in self.system_nics:
+            expected = {
+                'version': 2,
+                'ethernets': {
+                    nic: {
+                        'match': {'macaddress': MACADDR},
+                        'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
+            m_get_phys_by_mac.return_value = {MACADDR: nic}
+            net = ds.OpenNebulaNetwork(context)
+            self.assertEqual(net.gen_conf(), expected)
+
+        # set ETH0_MTU
+        context = {
+            'ETH0_MAC': '02:00:0a:12:01:01',
+            'ETH0_MTU': '1280', }
+        for nic in self.system_nics:
+            expected = {
+                'version': 2,
+                'ethernets': {
+                    nic: {
+                        'mtu': '1280',
+                        'match': {'macaddress': MACADDR},
+                        'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
+            m_get_phys_by_mac.return_value = {MACADDR: nic}
+            net = ds.OpenNebulaNetwork(context)
+            self.assertEqual(net.gen_conf(), expected)
+
     @mock.patch(DS_PATH + ".get_physical_nics_by_mac")
     def test_eth0(self, m_get_phys_by_mac):
         for nic in self.system_nics:
@@ -395,7 +801,6 @@ class TestOpenNebulaNetwork(unittest.TestCase):
                         'match': {'macaddress': MACADDR},
                         'addresses': [IP_BY_MACADDR + '/16'],
                         'gateway4': '1.2.3.5',
-                        'gateway6': None,
                         'nameservers': {
                             'addresses': ['1.2.3.6', '1.2.3.7', '1.2.3.8']}}}}
 
@@ -494,7 +899,6 @@ class TestOpenNebulaNetwork(unittest.TestCase):
                     'match': {'macaddress': MAC_1},
                     'addresses': ['10.3.1.3/16'],
                     'gateway4': '10.3.0.1',
-                    'gateway6': None,
                     'nameservers': {
                         'addresses': ['10.3.1.2', '1.2.3.8'],
                         'search': [
-- 
cgit v1.2.3


From 51f49dc1da0c045d01ddc977874c9bed70cb510f Mon Sep 17 00:00:00 2001
From: Louis Bouchard <lbouchard@online.net>
Date: Fri, 17 Aug 2018 18:43:44 +0000
Subject: Scaleway: Add network configuration to the DataSource

DEP_NETWORK is removed since the network_config must
run at each boot. New EventType.BOOT event is used
for that.

Network is brought up early to fetch the metadata which
is required to configure the network (ipv4 and/or v6).

Adds unittests for the following and fixes test_common for
LOCAL and NETWORK sets.
---
 cloudinit/sources/DataSourceScaleway.py          | 54 +++++++++++++---
 tests/unittests/test_datasource/test_common.py   |  2 +-
 tests/unittests/test_datasource/test_scaleway.py | 79 +++++++++++++++++++++++-
 3 files changed, 124 insertions(+), 11 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceScaleway.py b/cloudinit/sources/DataSourceScaleway.py
index e2502b02..9dc4ab23 100644
--- a/cloudinit/sources/DataSourceScaleway.py
+++ b/cloudinit/sources/DataSourceScaleway.py
@@ -29,7 +29,9 @@ from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import url_helper
 from cloudinit import util
-
+from cloudinit import net
+from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError
+from cloudinit.event import EventType
 
 LOG = logging.getLogger(__name__)
 
@@ -168,8 +170,8 @@ def query_data_api(api_type, api_address, retries, timeout):
 
 
 class DataSourceScaleway(sources.DataSource):
-
     dsname = "Scaleway"
+    update_events = {'network': [EventType.BOOT_NEW_INSTANCE, EventType.BOOT]}
 
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceScaleway, self).__init__(sys_cfg, distro, paths)
@@ -185,11 +187,10 @@ class DataSourceScaleway(sources.DataSource):
 
         self.retries = int(self.ds_cfg.get('retries', DEF_MD_RETRIES))
         self.timeout = int(self.ds_cfg.get('timeout', DEF_MD_TIMEOUT))
+        self._fallback_interface = None
+        self._network_config = None
 
-    def _get_data(self):
-        if not on_scaleway():
-            return False
-
+    def _crawl_metadata(self):
         resp = url_helper.readurl(self.metadata_address,
                                   timeout=self.timeout,
                                   retries=self.retries)
@@ -203,8 +204,47 @@ class DataSourceScaleway(sources.DataSource):
             'vendor-data', self.vendordata_address,
             self.retries, self.timeout
         )
+
+    def _get_data(self):
+        if not on_scaleway():
+            return False
+
+        if self._fallback_interface is None:
+            self._fallback_interface = net.find_fallback_nic()
+        try:
+            with EphemeralDHCPv4(self._fallback_interface):
+                util.log_time(
+                    logfunc=LOG.debug, msg='Crawl of metadata service',
+                    func=self._crawl_metadata)
+        except (NoDHCPLeaseError) as e:
+            util.logexc(LOG, str(e))
+            return False
         return True
 
+    @property
+    def network_config(self):
+        """
+        Configure networking according to data received from the
+        metadata API.
+        """
+        if self._network_config:
+            return self._network_config
+
+        if self._fallback_interface is None:
+            self._fallback_interface = net.find_fallback_nic()
+
+        netcfg = {'type': 'physical', 'name': '%s' % self._fallback_interface}
+        subnets = [{'type': 'dhcp4'}]
+        if self.metadata['ipv6']:
+            subnets += [{'type': 'static',
+                         'address': '%s' % self.metadata['ipv6']['address'],
+                         'gateway': '%s' % self.metadata['ipv6']['gateway'],
+                         'netmask': '%s' % self.metadata['ipv6']['netmask'],
+                         }]
+        netcfg['subnets'] = subnets
+        self._network_config = {'version': 1, 'config': [netcfg]}
+        return self._network_config
+
     @property
     def launch_index(self):
         return None
@@ -228,7 +268,7 @@ class DataSourceScaleway(sources.DataSource):
 
 
 datasources = [
-    (DataSourceScaleway, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+    (DataSourceScaleway, (sources.DEP_FILESYSTEM,)),
 ]
 
 
diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
index 0d35dc29..1a5a3db2 100644
--- a/tests/unittests/test_datasource/test_common.py
+++ b/tests/unittests/test_datasource/test_common.py
@@ -41,6 +41,7 @@ DEFAULT_LOCAL = [
     SmartOS.DataSourceSmartOS,
     Ec2.DataSourceEc2Local,
     OpenStack.DataSourceOpenStackLocal,
+    Scaleway.DataSourceScaleway,
 ]
 
 DEFAULT_NETWORK = [
@@ -55,7 +56,6 @@ DEFAULT_NETWORK = [
     NoCloud.DataSourceNoCloudNet,
     OpenStack.DataSourceOpenStack,
     OVF.DataSourceOVFNet,
-    Scaleway.DataSourceScaleway,
 ]
 
 
diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py
index e4e9bb20..c2bc7a00 100644
--- a/tests/unittests/test_datasource/test_scaleway.py
+++ b/tests/unittests/test_datasource/test_scaleway.py
@@ -176,11 +176,18 @@ class TestDataSourceScaleway(HttprettyTestCase):
         self.vendordata_url = \
             DataSourceScaleway.BUILTIN_DS_CONFIG['vendordata_url']
 
+        self.add_patch('cloudinit.sources.DataSourceScaleway.on_scaleway',
+                       '_m_on_scaleway', return_value=True)
+        self.add_patch(
+            'cloudinit.sources.DataSourceScaleway.net.find_fallback_nic',
+            '_m_find_fallback_nic', return_value='scalewaynic0')
+
+    @mock.patch('cloudinit.sources.DataSourceScaleway.EphemeralDHCPv4')
     @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter',
                 get_source_address_adapter)
     @mock.patch('cloudinit.util.get_cmdline')
     @mock.patch('time.sleep', return_value=None)
-    def test_metadata_ok(self, sleep, m_get_cmdline):
+    def test_metadata_ok(self, sleep, m_get_cmdline, dhcpv4):
         """
         get_data() returns metadata, user data and vendor data.
         """
@@ -211,11 +218,12 @@ class TestDataSourceScaleway(HttprettyTestCase):
         self.assertIsNone(self.datasource.region)
         self.assertEqual(sleep.call_count, 0)
 
+    @mock.patch('cloudinit.sources.DataSourceScaleway.EphemeralDHCPv4')
     @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter',
                 get_source_address_adapter)
     @mock.patch('cloudinit.util.get_cmdline')
     @mock.patch('time.sleep', return_value=None)
-    def test_metadata_404(self, sleep, m_get_cmdline):
+    def test_metadata_404(self, sleep, m_get_cmdline, dhcpv4):
         """
         get_data() returns metadata, but no user data nor vendor data.
         """
@@ -234,11 +242,12 @@ class TestDataSourceScaleway(HttprettyTestCase):
         self.assertIsNone(self.datasource.get_vendordata_raw())
         self.assertEqual(sleep.call_count, 0)
 
+    @mock.patch('cloudinit.sources.DataSourceScaleway.EphemeralDHCPv4')
     @mock.patch('cloudinit.sources.DataSourceScaleway.SourceAddressAdapter',
                 get_source_address_adapter)
     @mock.patch('cloudinit.util.get_cmdline')
     @mock.patch('time.sleep', return_value=None)
-    def test_metadata_rate_limit(self, sleep, m_get_cmdline):
+    def test_metadata_rate_limit(self, sleep, m_get_cmdline, dhcpv4):
         """
         get_data() is rate limited two times by the metadata API when fetching
         user data.
@@ -262,3 +271,67 @@ class TestDataSourceScaleway(HttprettyTestCase):
         self.assertEqual(self.datasource.get_userdata_raw(),
                          DataResponses.FAKE_USER_DATA)
         self.assertEqual(sleep.call_count, 2)
+
+    @mock.patch('cloudinit.sources.DataSourceScaleway.net.find_fallback_nic')
+    @mock.patch('cloudinit.util.get_cmdline')
+    def test_network_config_ok(self, m_get_cmdline, fallback_nic):
+        """
+        network_config will only generate IPv4 config if no ipv6 data is
+        available in the metadata
+        """
+        m_get_cmdline.return_value = 'scaleway'
+        fallback_nic.return_value = 'ens2'
+        self.datasource.metadata['ipv6'] = None
+
+        netcfg = self.datasource.network_config
+        resp = {'version': 1,
+                'config': [{
+                     'type': 'physical',
+                     'name': 'ens2',
+                     'subnets': [{'type': 'dhcp4'}]}]
+                }
+        self.assertEqual(netcfg, resp)
+
+    @mock.patch('cloudinit.sources.DataSourceScaleway.net.find_fallback_nic')
+    @mock.patch('cloudinit.util.get_cmdline')
+    def test_network_config_ipv6_ok(self, m_get_cmdline, fallback_nic):
+        """
+        network_config will only generate IPv4/v6 configs if ipv6 data is
+        available in the metadata
+        """
+        m_get_cmdline.return_value = 'scaleway'
+        fallback_nic.return_value = 'ens2'
+        self.datasource.metadata['ipv6'] = {
+                'address': '2000:abc:4444:9876::42:999',
+                'gateway': '2000:abc:4444:9876::42:000',
+                'netmask': '127',
+                }
+
+        netcfg = self.datasource.network_config
+        resp = {'version': 1,
+                'config': [{
+                     'type': 'physical',
+                     'name': 'ens2',
+                     'subnets': [{'type': 'dhcp4'},
+                                 {'type': 'static',
+                                  'address': '2000:abc:4444:9876::42:999',
+                                  'gateway': '2000:abc:4444:9876::42:000',
+                                  'netmask': '127', }
+                                 ]
+
+                     }]
+                }
+        self.assertEqual(netcfg, resp)
+
+    @mock.patch('cloudinit.sources.DataSourceScaleway.net.find_fallback_nic')
+    @mock.patch('cloudinit.util.get_cmdline')
+    def test_network_config_existing(self, m_get_cmdline, fallback_nic):
+        """
+        network_config() should return the same data if a network config
+        already exists
+        """
+        m_get_cmdline.return_value = 'scaleway'
+        self.datasource._network_config = '0xdeadbeef'
+
+        netcfg = self.datasource.network_config
+        self.assertEqual(netcfg, '0xdeadbeef')
-- 
cgit v1.2.3


From 47548df9ded4ad4088d3d846f1876b29b16aa7d1 Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Fri, 17 Aug 2018 20:24:58 +0000
Subject: azure: allow azure to generate network configuration from IMDS per
 boot.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Azure datasource now queries IMDS metadata service for network
configuration at link local address
http://169.254.169.254/metadata/instance?api-version=2017-12-01. The
azure metadata service presents a list of macs and allocated ip addresses
associated with this instance. Azure will now also regenerate network
configuration on every boot because it subscribes to EventType.BOOT
maintenance events as well as the 'first boot'
EventType.BOOT_NEW_INSTANCE.

For testing add azure-imds --kind to cloud-init devel net_convert tool
for debugging IMDS metadata.

Also refactor _get_data into 3 discrete methods:
  - is_platform_viable: check quickly whether the datasource is
    potentially compatible with the platform on which is is running
  - crawl_metadata: walk all potential metadata candidates, returning a
    structured dict of all metadata and userdata. Raise InvalidMetaData on
    error.
  - _get_data: call crawl_metadata and process results or error. Cache
    instance data on class attributes: metadata, userdata_raw etc.
---
 cloudinit/cmd/devel/net_convert.py            |   9 +-
 cloudinit/sources/DataSourceAzure.py          | 256 +++++++++++++++--
 tests/unittests/test_datasource/test_azure.py | 399 ++++++++++++++++++++++++--
 3 files changed, 605 insertions(+), 59 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/cmd/devel/net_convert.py b/cloudinit/cmd/devel/net_convert.py
index 1ec08a3c..271dc5ed 100755
--- a/cloudinit/cmd/devel/net_convert.py
+++ b/cloudinit/cmd/devel/net_convert.py
@@ -8,6 +8,7 @@ import sys
 import yaml
 
 from cloudinit.sources.helpers import openstack
+from cloudinit.sources import DataSourceAzure as azure
 
 from cloudinit.net import eni, netplan, network_state, sysconfig
 from cloudinit import log
@@ -28,7 +29,8 @@ def get_parser(parser=None):
     parser.add_argument("-p", "--network-data", type=open,
                         metavar="PATH", required=True)
     parser.add_argument("-k", "--kind",
-                        choices=['eni', 'network_data.json', 'yaml'],
+                        choices=['eni', 'network_data.json', 'yaml',
+                                 'azure-imds'],
                         required=True)
     parser.add_argument("-d", "--directory",
                         metavar="PATH",
@@ -78,10 +80,13 @@ def handle_args(name, args):
                 ["Input YAML",
                  yaml.dump(pre_ns, default_flow_style=False, indent=4), ""]))
         ns = network_state.parse_net_config_data(pre_ns)
-    else:
+    elif args.kind == 'network_data.json':
         pre_ns = openstack.convert_net_json(
             json.loads(net_data), known_macs=known_macs)
         ns = network_state.parse_net_config_data(pre_ns)
+    elif args.kind == 'azure-imds':
+        pre_ns = azure.parse_network_config(json.loads(net_data))
+        ns = network_state.parse_net_config_data(pre_ns)
 
     if not ns:
         raise RuntimeError("No valid network_state object created from"
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 7007d9ea..783445e1 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -8,6 +8,7 @@ import base64
 import contextlib
 import crypt
 from functools import partial
+import json
 import os
 import os.path
 import re
@@ -17,6 +18,7 @@ import xml.etree.ElementTree as ET
 
 from cloudinit import log as logging
 from cloudinit import net
+from cloudinit.event import EventType
 from cloudinit.net.dhcp import EphemeralDHCPv4
 from cloudinit import sources
 from cloudinit.sources.helpers.azure import get_metadata_from_fabric
@@ -49,7 +51,17 @@ DEFAULT_FS = 'ext4'
 AZURE_CHASSIS_ASSET_TAG = '7783-7084-3265-9085-8269-3286-77'
 REPROVISION_MARKER_FILE = "/var/lib/cloud/data/poll_imds"
 REPORTED_READY_MARKER_FILE = "/var/lib/cloud/data/reported_ready"
-IMDS_URL = "http://169.254.169.254/metadata/reprovisiondata"
+AGENT_SEED_DIR = '/var/lib/waagent'
+IMDS_URL = "http://169.254.169.254/metadata/"
+
+# List of static scripts and network config artifacts created by
+# stock ubuntu suported images.
+UBUNTU_EXTENDED_NETWORK_SCRIPTS = [
+    '/etc/netplan/90-azure-hotplug.yaml',
+    '/usr/local/sbin/ephemeral_eth.sh',
+    '/etc/udev/rules.d/10-net-device-added.rules',
+    '/run/network/interfaces.ephemeral.d',
+]
 
 
 def find_storvscid_from_sysctl_pnpinfo(sysctl_out, deviceid):
@@ -185,7 +197,7 @@ if util.is_FreeBSD():
 
 BUILTIN_DS_CONFIG = {
     'agent_command': AGENT_START_BUILTIN,
-    'data_dir': "/var/lib/waagent",
+    'data_dir': AGENT_SEED_DIR,
     'set_hostname': True,
     'hostname_bounce': {
         'interface': DEFAULT_PRIMARY_NIC,
@@ -252,6 +264,7 @@ class DataSourceAzure(sources.DataSource):
 
     dsname = 'Azure'
     _negotiated = False
+    _metadata_imds = sources.UNSET
 
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
@@ -263,6 +276,8 @@ class DataSourceAzure(sources.DataSource):
             BUILTIN_DS_CONFIG])
         self.dhclient_lease_file = self.ds_cfg.get('dhclient_lease_file')
         self._network_config = None
+        # Regenerate network config new_instance boot and every boot
+        self.update_events['network'].add(EventType.BOOT)
 
     def __str__(self):
         root = sources.DataSource.__str__(self)
@@ -336,15 +351,17 @@ class DataSourceAzure(sources.DataSource):
         metadata['public-keys'] = key_value or pubkeys_from_crt_files(fp_files)
         return metadata
 
-    def _get_data(self):
+    def crawl_metadata(self):
+        """Walk all instance metadata sources returning a dict on success.
+
+        @return: A dictionary of any metadata content for this instance.
+        @raise: InvalidMetaDataException when the expected metadata service is
+            unavailable, broken or disabled.
+        """
+        crawled_data = {}
         # azure removes/ejects the cdrom containing the ovf-env.xml
         # file on reboot.  So, in order to successfully reboot we
         # need to look in the datadir and consider that valid
-        asset_tag = util.read_dmi_data('chassis-asset-tag')
-        if asset_tag != AZURE_CHASSIS_ASSET_TAG:
-            LOG.debug("Non-Azure DMI asset tag '%s' discovered.", asset_tag)
-            return False
-
         ddir = self.ds_cfg['data_dir']
 
         candidates = [self.seed_dir]
@@ -373,41 +390,84 @@ class DataSourceAzure(sources.DataSource):
             except NonAzureDataSource:
                 continue
             except BrokenAzureDataSource as exc:
-                raise exc
+                msg = 'BrokenAzureDataSource: %s' % exc
+                raise sources.InvalidMetaDataException(msg)
             except util.MountFailedError:
                 LOG.warning("%s was not mountable", cdev)
                 continue
 
             if reprovision or self._should_reprovision(ret):
                 ret = self._reprovision()
-            (md, self.userdata_raw, cfg, files) = ret
+            imds_md = get_metadata_from_imds(
+                self.fallback_interface, retries=3)
+            (md, userdata_raw, cfg, files) = ret
             self.seed = cdev
-            self.metadata = util.mergemanydict([md, DEFAULT_METADATA])
-            self.cfg = util.mergemanydict([cfg, BUILTIN_CLOUD_CONFIG])
+            crawled_data.update({
+                'cfg': cfg,
+                'files': files,
+                'metadata': util.mergemanydict(
+                    [md, {'imds': imds_md}]),
+                'userdata_raw': userdata_raw})
             found = cdev
 
             LOG.debug("found datasource in %s", cdev)
             break
 
         if not found:
-            return False
+            raise sources.InvalidMetaDataException('No Azure metadata found')
 
         if found == ddir:
             LOG.debug("using files cached in %s", ddir)
 
         seed = _get_random_seed()
         if seed:
-            self.metadata['random_seed'] = seed
+            crawled_data['metadata']['random_seed'] = seed
+        crawled_data['metadata']['instance-id'] = util.read_dmi_data(
+            'system-uuid')
+        return crawled_data
+
+    def _is_platform_viable(self):
+        """Check platform environment to report if this datasource may run."""
+        return _is_platform_viable(self.seed_dir)
+
+    def clear_cached_attrs(self, attr_defaults=()):
+        """Reset any cached class attributes to defaults."""
+        super(DataSourceAzure, self).clear_cached_attrs(attr_defaults)
+        self._metadata_imds = sources.UNSET
+
+    def _get_data(self):
+        """Crawl and process datasource metadata caching metadata as attrs.
+
+        @return: True on success, False on error, invalid or disabled
+            datasource.
+        """
+        if not self._is_platform_viable():
+            return False
+        try:
+            crawled_data = util.log_time(
+                        logfunc=LOG.debug, msg='Crawl of metadata service',
+                        func=self.crawl_metadata)
+        except sources.InvalidMetaDataException as e:
+            LOG.warning('Could not crawl Azure metadata: %s', e)
+            return False
+        if self.distro and self.distro.name == 'ubuntu':
+            maybe_remove_ubuntu_network_config_scripts()
+
+        # Process crawled data and augment with various config defaults
+        self.cfg = util.mergemanydict(
+            [crawled_data['cfg'], BUILTIN_CLOUD_CONFIG])
+        self._metadata_imds = crawled_data['metadata']['imds']
+        self.metadata = util.mergemanydict(
+            [crawled_data['metadata'], DEFAULT_METADATA])
+        self.userdata_raw = crawled_data['userdata_raw']
 
         user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {})
         self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg])
 
         # walinux agent writes files world readable, but expects
         # the directory to be protected.
-        write_files(ddir, files, dirmode=0o700)
-
-        self.metadata['instance-id'] = util.read_dmi_data('system-uuid')
-
+        write_files(
+            self.ds_cfg['data_dir'], crawled_data['files'], dirmode=0o700)
         return True
 
     def device_name_to_device(self, name):
@@ -436,7 +496,7 @@ class DataSourceAzure(sources.DataSource):
     def _poll_imds(self):
         """Poll IMDS for the new provisioning data until we get a valid
         response. Then return the returned JSON object."""
-        url = IMDS_URL + "?api-version=2017-04-02"
+        url = IMDS_URL + "reprovisiondata?api-version=2017-04-02"
         headers = {"Metadata": "true"}
         report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE))
         LOG.debug("Start polling IMDS")
@@ -487,7 +547,7 @@ class DataSourceAzure(sources.DataSource):
         jump back into the polling loop in order to retrieve the ovf_env."""
         if not ret:
             return False
-        (_md, self.userdata_raw, cfg, _files) = ret
+        (_md, _userdata_raw, cfg, _files) = ret
         path = REPROVISION_MARKER_FILE
         if (cfg.get('PreprovisionedVm') is True or
                 os.path.isfile(path)):
@@ -543,22 +603,15 @@ class DataSourceAzure(sources.DataSource):
     @property
     def network_config(self):
         """Generate a network config like net.generate_fallback_network() with
-           the following execptions.
+           the following exceptions.
 
            1. Probe the drivers of the net-devices present and inject them in
               the network configuration under params: driver: <driver> value
            2. Generate a fallback network config that does not include any of
               the blacklisted devices.
         """
-        blacklist = ['mlx4_core']
         if not self._network_config:
-            LOG.debug('Azure: generating fallback configuration')
-            # generate a network config, blacklist picking any mlx4_core devs
-            netconfig = net.generate_fallback_config(
-                blacklist_drivers=blacklist, config_driver=True)
-
-            self._network_config = netconfig
-
+            self._network_config = parse_network_config(self._metadata_imds)
         return self._network_config
 
 
@@ -1025,6 +1078,151 @@ def load_azure_ds_dir(source_dir):
     return (md, ud, cfg, {'ovf-env.xml': contents})
 
 
+def parse_network_config(imds_metadata):
+    """Convert imds_metadata dictionary to network v2 configuration.
+
+    Parses network configuration from imds metadata if present or generate
+    fallback network config excluding mlx4_core devices.
+
+    @param: imds_metadata: Dict of content read from IMDS network service.
+    @return: Dictionary containing network version 2 standard configuration.
+    """
+    if imds_metadata != sources.UNSET and imds_metadata:
+        netconfig = {'version': 2, 'ethernets': {}}
+        LOG.debug('Azure: generating network configuration from IMDS')
+        network_metadata = imds_metadata['network']
+        for idx, intf in enumerate(network_metadata['interface']):
+            nicname = 'eth{idx}'.format(idx=idx)
+            dev_config = {}
+            for addr4 in intf['ipv4']['ipAddress']:
+                privateIpv4 = addr4['privateIpAddress']
+                if privateIpv4:
+                    if dev_config.get('dhcp4', False):
+                        # Append static address config for nic > 1
+                        netPrefix = intf['ipv4']['subnet'][0].get(
+                            'prefix', '24')
+                        if not dev_config.get('addresses'):
+                            dev_config['addresses'] = []
+                        dev_config['addresses'].append(
+                            '{ip}/{prefix}'.format(
+                                ip=privateIpv4, prefix=netPrefix))
+                    else:
+                        dev_config['dhcp4'] = True
+            for addr6 in intf['ipv6']['ipAddress']:
+                privateIpv6 = addr6['privateIpAddress']
+                if privateIpv6:
+                    dev_config['dhcp6'] = True
+                    break
+            if dev_config:
+                mac = ':'.join(re.findall(r'..', intf['macAddress']))
+                dev_config.update(
+                    {'match': {'macaddress': mac.lower()},
+                     'set-name': nicname})
+                netconfig['ethernets'][nicname] = dev_config
+    else:
+        blacklist = ['mlx4_core']
+        LOG.debug('Azure: generating fallback configuration')
+        # generate a network config, blacklist picking mlx4_core devs
+        netconfig = net.generate_fallback_config(
+            blacklist_drivers=blacklist, config_driver=True)
+    return netconfig
+
+
+def get_metadata_from_imds(fallback_nic, retries):
+    """Query Azure's network metadata service, returning a dictionary.
+
+    If network is not up, setup ephemeral dhcp on fallback_nic to talk to the
+    IMDS. For more info on IMDS:
+        https://docs.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service
+
+    @param fallback_nic: String. The name of the nic which requires active
+        network in order to query IMDS.
+    @param retries: The number of retries of the IMDS_URL.
+
+    @return: A dict of instance metadata containing compute and network
+        info.
+    """
+    kwargs = {'logfunc': LOG.debug,
+              'msg': 'Crawl of Azure Instance Metadata Service (IMDS)',
+              'func': _get_metadata_from_imds, 'args': (retries,)}
+    if net.is_up(fallback_nic):
+        return util.log_time(**kwargs)
+    else:
+        with EphemeralDHCPv4(fallback_nic):
+            return util.log_time(**kwargs)
+
+
+def _get_metadata_from_imds(retries):
+
+    def retry_on_url_error(msg, exception):
+        if isinstance(exception, UrlError) and exception.code == 404:
+            return True  # Continue retries
+        return False  # Stop retries on all other exceptions
+
+    url = IMDS_URL + "instance?api-version=2017-12-01"
+    headers = {"Metadata": "true"}
+    try:
+        response = readurl(
+            url, timeout=1, headers=headers, retries=retries,
+            exception_cb=retry_on_url_error)
+    except Exception as e:
+        LOG.debug('Ignoring IMDS instance metadata: %s', e)
+        return {}
+    try:
+        return util.load_json(str(response))
+    except json.decoder.JSONDecodeError:
+        LOG.warning(
+            'Ignoring non-json IMDS instance metadata: %s', str(response))
+    return {}
+
+
+def maybe_remove_ubuntu_network_config_scripts(paths=None):
+    """Remove Azure-specific ubuntu network config for non-primary nics.
+
+    @param paths: List of networking scripts or directories to remove when
+        present.
+
+    In certain supported ubuntu images, static udev rules or netplan yaml
+    config is delivered in the base ubuntu image to support dhcp on any
+    additional interfaces which get attached by a customer at some point
+    after initial boot. Since the Azure datasource can now regenerate
+    network configuration as metadata reports these new devices, we no longer
+    want the udev rules or netplan's 90-azure-hotplug.yaml to configure
+    networking on eth1 or greater as it might collide with cloud-init's
+    configuration.
+
+    Remove the any existing extended network scripts if the datasource is
+    enabled to write network per-boot.
+    """
+    if not paths:
+        paths = UBUNTU_EXTENDED_NETWORK_SCRIPTS
+    logged = False
+    for path in paths:
+        if os.path.exists(path):
+            if not logged:
+                LOG.info(
+                    'Removing Ubuntu extended network scripts because'
+                    ' cloud-init updates Azure network configuration on the'
+                    ' following event: %s.',
+                    EventType.BOOT)
+                logged = True
+            if os.path.isdir(path):
+                util.del_dir(path)
+            else:
+                util.del_file(path)
+
+
+def _is_platform_viable(seed_dir):
+    """Check platform environment to report if this datasource may run."""
+    asset_tag = util.read_dmi_data('chassis-asset-tag')
+    if asset_tag == AZURE_CHASSIS_ASSET_TAG:
+        return True
+    LOG.debug("Non-Azure DMI asset tag '%s' discovered.", asset_tag)
+    if os.path.exists(os.path.join(seed_dir, 'ovf-env.xml')):
+        return True
+    return False
+
+
 class BrokenAzureDataSource(Exception):
     pass
 
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index e82716eb..4e428b71 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -1,15 +1,21 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
+from cloudinit import distros
 from cloudinit import helpers
-from cloudinit.sources import DataSourceAzure as dsaz
+from cloudinit import url_helper
+from cloudinit.sources import (
+    UNSET, DataSourceAzure as dsaz, InvalidMetaDataException)
 from cloudinit.util import (b64e, decode_binary, load_file, write_file,
                             find_freebsd_part, get_path_dev_freebsd,
                             MountFailedError)
 from cloudinit.version import version_string as vs
-from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock,
-                                     ExitStack, PY26, SkipTest)
+from cloudinit.tests.helpers import (
+    HttprettyTestCase, CiTestCase, populate_dir, mock, wrap_and_call,
+    ExitStack, PY26, SkipTest)
 
 import crypt
+import httpretty
+import json
 import os
 import stat
 import xml.etree.ElementTree as ET
@@ -77,6 +83,106 @@ def construct_valid_ovf_env(data=None, pubkeys=None,
     return content
 
 
+NETWORK_METADATA = {
+    "network": {
+        "interface": [
+            {
+                "macAddress": "000D3A047598",
+                "ipv6": {
+                    "ipAddress": []
+                },
+                "ipv4": {
+                    "subnet": [
+                        {
+                           "prefix": "24",
+                           "address": "10.0.0.0"
+                        }
+                    ],
+                    "ipAddress": [
+                        {
+                           "privateIpAddress": "10.0.0.4",
+                           "publicIpAddress": "104.46.124.81"
+                        }
+                    ]
+                }
+            }
+        ]
+    }
+}
+
+
+class TestGetMetadataFromIMDS(HttprettyTestCase):
+
+    with_logs = True
+
+    def setUp(self):
+        super(TestGetMetadataFromIMDS, self).setUp()
+        self.network_md_url = dsaz.IMDS_URL + "instance?api-version=2017-12-01"
+
+    @mock.patch('cloudinit.sources.DataSourceAzure.readurl')
+    @mock.patch('cloudinit.sources.DataSourceAzure.EphemeralDHCPv4')
+    @mock.patch('cloudinit.sources.DataSourceAzure.net.is_up')
+    def test_get_metadata_does_not_dhcp_if_network_is_up(
+            self, m_net_is_up, m_dhcp, m_readurl):
+        """Do not perform DHCP setup when nic is already up."""
+        m_net_is_up.return_value = True
+        m_readurl.return_value = url_helper.StringResponse(
+            json.dumps(NETWORK_METADATA).encode('utf-8'))
+        self.assertEqual(
+            NETWORK_METADATA,
+            dsaz.get_metadata_from_imds('eth9', retries=3))
+
+        m_net_is_up.assert_called_with('eth9')
+        m_dhcp.assert_not_called()
+        self.assertIn(
+            "Crawl of Azure Instance Metadata Service (IMDS) took",  # log_time
+            self.logs.getvalue())
+
+    @mock.patch('cloudinit.sources.DataSourceAzure.readurl')
+    @mock.patch('cloudinit.sources.DataSourceAzure.EphemeralDHCPv4')
+    @mock.patch('cloudinit.sources.DataSourceAzure.net.is_up')
+    def test_get_metadata_performs_dhcp_when_network_is_down(
+            self, m_net_is_up, m_dhcp, m_readurl):
+        """Perform DHCP setup when nic is not up."""
+        m_net_is_up.return_value = False
+        m_readurl.return_value = url_helper.StringResponse(
+            json.dumps(NETWORK_METADATA).encode('utf-8'))
+
+        self.assertEqual(
+            NETWORK_METADATA,
+            dsaz.get_metadata_from_imds('eth9', retries=2))
+
+        m_net_is_up.assert_called_with('eth9')
+        m_dhcp.assert_called_with('eth9')
+        self.assertIn(
+            "Crawl of Azure Instance Metadata Service (IMDS) took",  # log_time
+            self.logs.getvalue())
+
+        m_readurl.assert_called_with(
+            self.network_md_url, exception_cb=mock.ANY,
+            headers={'Metadata': 'true'}, retries=2, timeout=1)
+
+    @mock.patch('cloudinit.url_helper.time.sleep')
+    @mock.patch('cloudinit.sources.DataSourceAzure.net.is_up')
+    def test_get_metadata_from_imds_empty_when_no_imds_present(
+            self, m_net_is_up, m_sleep):
+        """Return empty dict when IMDS network metadata is absent."""
+        httpretty.register_uri(
+            httpretty.GET,
+            dsaz.IMDS_URL + 'instance?api-version=2017-12-01',
+            body={}, status=404)
+
+        m_net_is_up.return_value = True  # skips dhcp
+
+        self.assertEqual({}, dsaz.get_metadata_from_imds('eth9', retries=2))
+
+        m_net_is_up.assert_called_with('eth9')
+        self.assertEqual([mock.call(1), mock.call(1)], m_sleep.call_args_list)
+        self.assertIn(
+            "Crawl of Azure Instance Metadata Service (IMDS) took",  # log_time
+            self.logs.getvalue())
+
+
 class TestAzureDataSource(CiTestCase):
 
     with_logs = True
@@ -95,8 +201,19 @@ class TestAzureDataSource(CiTestCase):
         self.patches = ExitStack()
         self.addCleanup(self.patches.close)
 
-        self.patches.enter_context(mock.patch.object(dsaz, '_get_random_seed'))
-
+        self.patches.enter_context(mock.patch.object(
+            dsaz, '_get_random_seed', return_value='wild'))
+        self.m_get_metadata_from_imds = self.patches.enter_context(
+            mock.patch.object(
+                dsaz, 'get_metadata_from_imds',
+                mock.MagicMock(return_value=NETWORK_METADATA)))
+        self.m_fallback_nic = self.patches.enter_context(
+            mock.patch('cloudinit.sources.net.find_fallback_nic',
+                       return_value='eth9'))
+        self.m_remove_ubuntu_network_scripts = self.patches.enter_context(
+            mock.patch.object(
+                dsaz, 'maybe_remove_ubuntu_network_config_scripts',
+                mock.MagicMock()))
         super(TestAzureDataSource, self).setUp()
 
     def apply_patches(self, patches):
@@ -137,7 +254,7 @@ scbus-1 on xpt0 bus 0
         ])
         return dsaz
 
-    def _get_ds(self, data, agent_command=None):
+    def _get_ds(self, data, agent_command=None, distro=None):
 
         def dsdevs():
             return data.get('dsdevs', [])
@@ -186,8 +303,11 @@ scbus-1 on xpt0 bus 0
                 side_effect=_wait_for_files)),
         ])
 
+        if distro is not None:
+            distro_cls = distros.fetch(distro)
+            distro = distro_cls(distro, data.get('sys_cfg', {}), self.paths)
         dsrc = dsaz.DataSourceAzure(
-            data.get('sys_cfg', {}), distro=None, paths=self.paths)
+            data.get('sys_cfg', {}), distro=distro, paths=self.paths)
         if agent_command is not None:
             dsrc.ds_cfg['agent_command'] = agent_command
 
@@ -260,29 +380,20 @@ fdescfs            /dev/fd          fdescfs rw              0 0
             res = get_path_dev_freebsd('/etc', mnt_list)
             self.assertIsNotNone(res)
 
-    @mock.patch('cloudinit.sources.DataSourceAzure.util.read_dmi_data')
-    def test_non_azure_dmi_chassis_asset_tag(self, m_read_dmi_data):
-        """Report non-azure when DMI's chassis asset tag doesn't match.
-
-        Return False when the asset tag doesn't match Azure's static
-        AZURE_CHASSIS_ASSET_TAG.
-        """
+    @mock.patch('cloudinit.sources.DataSourceAzure._is_platform_viable')
+    def test_call_is_platform_viable_seed(self, m_is_platform_viable):
+        """Check seed_dir using _is_platform_viable and return False."""
         # Return a non-matching asset tag value
-        nonazure_tag = dsaz.AZURE_CHASSIS_ASSET_TAG + 'X'
-        m_read_dmi_data.return_value = nonazure_tag
+        m_is_platform_viable.return_value = False
         dsrc = dsaz.DataSourceAzure(
             {}, distro=None, paths=self.paths)
         self.assertFalse(dsrc.get_data())
-        self.assertEqual(
-            "DEBUG: Non-Azure DMI asset tag '{0}' discovered.\n".format(
-                nonazure_tag),
-            self.logs.getvalue())
+        m_is_platform_viable.assert_called_with(dsrc.seed_dir)
 
     def test_basic_seed_dir(self):
         odata = {'HostName': "myhost", 'UserName': "myuser"}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata),
                 'sys_cfg': {}}
-
         dsrc = self._get_ds(data)
         ret = dsrc.get_data()
         self.assertTrue(ret)
@@ -291,6 +402,82 @@ fdescfs            /dev/fd          fdescfs rw              0 0
         self.assertTrue(os.path.isfile(
             os.path.join(self.waagent_d, 'ovf-env.xml')))
 
+    def test_get_data_non_ubuntu_will_not_remove_network_scripts(self):
+        """get_data on non-Ubuntu will not remove ubuntu net scripts."""
+        odata = {'HostName': "myhost", 'UserName': "myuser"}
+        data = {'ovfcontent': construct_valid_ovf_env(data=odata),
+                'sys_cfg': {}}
+
+        dsrc = self._get_ds(data, distro='debian')
+        dsrc.get_data()
+        self.m_remove_ubuntu_network_scripts.assert_not_called()
+
+    def test_get_data_on_ubuntu_will_remove_network_scripts(self):
+        """get_data will remove ubuntu net scripts on Ubuntu distro."""
+        odata = {'HostName': "myhost", 'UserName': "myuser"}
+        data = {'ovfcontent': construct_valid_ovf_env(data=odata),
+                'sys_cfg': {}}
+
+        dsrc = self._get_ds(data, distro='ubuntu')
+        dsrc.get_data()
+        self.m_remove_ubuntu_network_scripts.assert_called_once_with()
+
+    def test_crawl_metadata_returns_structured_data_and_caches_nothing(self):
+        """Return all structured metadata and cache no class attributes."""
+        yaml_cfg = "{agent_command: my_command}\n"
+        odata = {'HostName': "myhost", 'UserName': "myuser",
+                 'UserData': {'text': 'FOOBAR', 'encoding': 'plain'},
+                 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}}
+        data = {'ovfcontent': construct_valid_ovf_env(data=odata),
+                'sys_cfg': {}}
+        dsrc = self._get_ds(data)
+        expected_cfg = {
+            'PreprovisionedVm': False,
+            'datasource': {'Azure': {'agent_command': 'my_command'}},
+            'system_info': {'default_user': {'name': u'myuser'}}}
+        expected_metadata = {
+            'azure_data': {
+                'configurationsettype': 'LinuxProvisioningConfiguration'},
+            'imds': {'network': {'interface': [{
+                'ipv4': {'ipAddress': [
+                     {'privateIpAddress': '10.0.0.4',
+                      'publicIpAddress': '104.46.124.81'}],
+                      'subnet': [{'address': '10.0.0.0', 'prefix': '24'}]},
+                'ipv6': {'ipAddress': []},
+                'macAddress': '000D3A047598'}]}},
+            'instance-id': 'test-instance-id',
+            'local-hostname': u'myhost',
+            'random_seed': 'wild'}
+
+        crawled_metadata = dsrc.crawl_metadata()
+
+        self.assertItemsEqual(
+            crawled_metadata.keys(),
+            ['cfg', 'files', 'metadata', 'userdata_raw'])
+        self.assertEqual(crawled_metadata['cfg'], expected_cfg)
+        self.assertEqual(
+            list(crawled_metadata['files'].keys()), ['ovf-env.xml'])
+        self.assertIn(
+            b'<HostName>myhost</HostName>',
+            crawled_metadata['files']['ovf-env.xml'])
+        self.assertEqual(crawled_metadata['metadata'], expected_metadata)
+        self.assertEqual(crawled_metadata['userdata_raw'], 'FOOBAR')
+        self.assertEqual(dsrc.userdata_raw, None)
+        self.assertEqual(dsrc.metadata, {})
+        self.assertEqual(dsrc._metadata_imds, UNSET)
+        self.assertFalse(os.path.isfile(
+            os.path.join(self.waagent_d, 'ovf-env.xml')))
+
+    def test_crawl_metadata_raises_invalid_metadata_on_error(self):
+        """crawl_metadata raises an exception on invalid ovf-env.xml."""
+        data = {'ovfcontent': "BOGUS", 'sys_cfg': {}}
+        dsrc = self._get_ds(data)
+        error_msg = ('BrokenAzureDataSource: Invalid ovf-env.xml:'
+                     ' syntax error: line 1, column 0')
+        with self.assertRaises(InvalidMetaDataException) as cm:
+            dsrc.crawl_metadata()
+        self.assertEqual(str(cm.exception), error_msg)
+
     def test_waagent_d_has_0700_perms(self):
         # we expect /var/lib/waagent to be created 0700
         dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()})
@@ -314,6 +501,20 @@ fdescfs            /dev/fd          fdescfs rw              0 0
         self.assertTrue(ret)
         self.assertEqual(data['agent_invoked'], cfg['agent_command'])
 
+    def test_network_config_set_from_imds(self):
+        """Datasource.network_config returns IMDS network data."""
+        odata = {}
+        data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
+        expected_network_config = {
+            'ethernets': {
+                'eth0': {'set-name': 'eth0',
+                         'match': {'macaddress': '00:0d:3a:04:75:98'},
+                         'dhcp4': True}},
+            'version': 2}
+        dsrc = self._get_ds(data)
+        dsrc.get_data()
+        self.assertEqual(expected_network_config, dsrc.network_config)
+
     def test_user_cfg_set_agent_command(self):
         # set dscfg in via base64 encoded yaml
         cfg = {'agent_command': "my_command"}
@@ -579,12 +780,34 @@ fdescfs            /dev/fd          fdescfs rw              0 0
         self.assertEqual(
             [mock.call("/dev/cd0")], m_check_fbsd_cdrom.call_args_list)
 
+    @mock.patch('cloudinit.net.generate_fallback_config')
+    def test_imds_network_config(self, mock_fallback):
+        """Network config is generated from IMDS network data when present."""
+        odata = {'HostName': "myhost", 'UserName': "myuser"}
+        data = {'ovfcontent': construct_valid_ovf_env(data=odata),
+                'sys_cfg': {}}
+
+        dsrc = self._get_ds(data)
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+
+        expected_cfg = {
+            'ethernets': {
+                'eth0': {'dhcp4': True,
+                         'match': {'macaddress': '00:0d:3a:04:75:98'},
+                         'set-name': 'eth0'}},
+            'version': 2}
+
+        self.assertEqual(expected_cfg, dsrc.network_config)
+        mock_fallback.assert_not_called()
+
     @mock.patch('cloudinit.net.get_interface_mac')
     @mock.patch('cloudinit.net.get_devicelist')
     @mock.patch('cloudinit.net.device_driver')
     @mock.patch('cloudinit.net.generate_fallback_config')
-    def test_network_config(self, mock_fallback, mock_dd,
-                            mock_devlist, mock_get_mac):
+    def test_fallback_network_config(self, mock_fallback, mock_dd,
+                                     mock_devlist, mock_get_mac):
+        """On absent IMDS network data, generate network fallback config."""
         odata = {'HostName': "myhost", 'UserName': "myuser"}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata),
                 'sys_cfg': {}}
@@ -605,6 +828,8 @@ fdescfs            /dev/fd          fdescfs rw              0 0
         mock_get_mac.return_value = '00:11:22:33:44:55'
 
         dsrc = self._get_ds(data)
+        # Represent empty response from network imds
+        self.m_get_metadata_from_imds.return_value = {}
         ret = dsrc.get_data()
         self.assertTrue(ret)
 
@@ -617,8 +842,9 @@ fdescfs            /dev/fd          fdescfs rw              0 0
     @mock.patch('cloudinit.net.get_devicelist')
     @mock.patch('cloudinit.net.device_driver')
     @mock.patch('cloudinit.net.generate_fallback_config')
-    def test_network_config_blacklist(self, mock_fallback, mock_dd,
-                                      mock_devlist, mock_get_mac):
+    def test_fallback_network_config_blacklist(self, mock_fallback, mock_dd,
+                                               mock_devlist, mock_get_mac):
+        """On absent network metadata, blacklist mlx from fallback config."""
         odata = {'HostName': "myhost", 'UserName': "myuser"}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata),
                 'sys_cfg': {}}
@@ -649,6 +875,8 @@ fdescfs            /dev/fd          fdescfs rw              0 0
         mock_get_mac.return_value = '00:11:22:33:44:55'
 
         dsrc = self._get_ds(data)
+        # Represent empty response from network imds
+        self.m_get_metadata_from_imds.return_value = {}
         ret = dsrc.get_data()
         self.assertTrue(ret)
 
@@ -689,9 +917,12 @@ class TestAzureBounce(CiTestCase):
             mock.patch.object(dsaz, 'get_metadata_from_fabric',
                               mock.MagicMock(return_value={})))
         self.patches.enter_context(
-            mock.patch.object(dsaz.util, 'which', lambda x: True))
+            mock.patch.object(dsaz, 'get_metadata_from_imds',
+                              mock.MagicMock(return_value={})))
         self.patches.enter_context(
-            mock.patch.object(dsaz, '_get_random_seed'))
+            mock.patch.object(dsaz.util, 'which', lambda x: True))
+        self.patches.enter_context(mock.patch.object(
+            dsaz, '_get_random_seed', return_value='wild'))
 
         def _dmi_mocks(key):
             if key == 'system-uuid':
@@ -719,9 +950,12 @@ class TestAzureBounce(CiTestCase):
             mock.patch.object(dsaz, 'set_hostname'))
         self.subp = self.patches.enter_context(
             mock.patch('cloudinit.sources.DataSourceAzure.util.subp'))
+        self.find_fallback_nic = self.patches.enter_context(
+            mock.patch('cloudinit.net.find_fallback_nic', return_value='eth9'))
 
     def tearDown(self):
         self.patches.close()
+        super(TestAzureBounce, self).tearDown()
 
     def _get_ds(self, ovfcontent=None, agent_command=None):
         if ovfcontent is not None:
@@ -927,7 +1161,7 @@ class TestLoadAzureDsDir(CiTestCase):
             str(context_manager.exception))
 
 
-class TestReadAzureOvf(TestCase):
+class TestReadAzureOvf(CiTestCase):
 
     def test_invalid_xml_raises_non_azure_ds(self):
         invalid_xml = "<foo>" + construct_valid_ovf_env(data={})
@@ -1188,6 +1422,25 @@ class TestCanDevBeReformatted(CiTestCase):
                       "(datasource.Azure.never_destroy_ntfs)", msg)
 
 
+class TestClearCachedData(CiTestCase):
+
+    def test_clear_cached_attrs_clears_imds(self):
+        """All class attributes are reset to defaults, including imds data."""
+        tmp = self.tmp_dir()
+        paths = helpers.Paths(
+            {'cloud_dir': tmp, 'run_dir': tmp})
+        dsrc = dsaz.DataSourceAzure({}, distro=None, paths=paths)
+        clean_values = [dsrc.metadata, dsrc.userdata, dsrc._metadata_imds]
+        dsrc.metadata = 'md'
+        dsrc.userdata = 'ud'
+        dsrc._metadata_imds = 'imds'
+        dsrc._dirty_cache = True
+        dsrc.clear_cached_attrs()
+        self.assertEqual(
+            [dsrc.metadata, dsrc.userdata, dsrc._metadata_imds],
+            clean_values)
+
+
 class TestAzureNetExists(CiTestCase):
 
     def test_azure_net_must_exist_for_legacy_objpkl(self):
@@ -1398,4 +1651,94 @@ class TestAzureDataSourcePreprovisioning(CiTestCase):
         self.assertEqual(m_net.call_count, 1)
 
 
+class TestRemoveUbuntuNetworkConfigScripts(CiTestCase):
+
+    with_logs = True
+
+    def setUp(self):
+        super(TestRemoveUbuntuNetworkConfigScripts, self).setUp()
+        self.tmp = self.tmp_dir()
+
+    def test_remove_network_scripts_removes_both_files_and_directories(self):
+        """Any files or directories in paths are removed when present."""
+        file1 = self.tmp_path('file1', dir=self.tmp)
+        subdir = self.tmp_path('sub1', dir=self.tmp)
+        subfile = self.tmp_path('leaf1', dir=subdir)
+        write_file(file1, 'file1content')
+        write_file(subfile, 'leafcontent')
+        dsaz.maybe_remove_ubuntu_network_config_scripts(paths=[subdir, file1])
+
+        for path in (file1, subdir, subfile):
+            self.assertFalse(os.path.exists(path),
+                             'Found unremoved: %s' % path)
+
+        expected_logs = [
+            'INFO: Removing Ubuntu extended network scripts because cloud-init'
+            ' updates Azure network configuration on the following event:'
+            ' System boot.',
+            'Recursively deleting %s' % subdir,
+            'Attempting to remove %s' % file1]
+        for log in expected_logs:
+            self.assertIn(log, self.logs.getvalue())
+
+    def test_remove_network_scripts_only_attempts_removal_if_path_exists(self):
+        """Any files or directories absent are skipped without error."""
+        dsaz.maybe_remove_ubuntu_network_config_scripts(paths=[
+            self.tmp_path('nodirhere/', dir=self.tmp),
+            self.tmp_path('notfilehere', dir=self.tmp)])
+        self.assertNotIn('/not/a', self.logs.getvalue())  # No delete logs
+
+    @mock.patch('cloudinit.sources.DataSourceAzure.os.path.exists')
+    def test_remove_network_scripts_default_removes_stock_scripts(self,
+                                                                  m_exists):
+        """Azure's stock ubuntu image scripts and artifacts are removed."""
+        # Report path absent on all to avoid delete operation
+        m_exists.return_value = False
+        dsaz.maybe_remove_ubuntu_network_config_scripts()
+        calls = m_exists.call_args_list
+        for path in dsaz.UBUNTU_EXTENDED_NETWORK_SCRIPTS:
+            self.assertIn(mock.call(path), calls)
+
+
+class TestWBIsPlatformViable(CiTestCase):
+    """White box tests for _is_platform_viable."""
+    with_logs = True
+
+    @mock.patch('cloudinit.sources.DataSourceAzure.util.read_dmi_data')
+    def test_true_on_non_azure_chassis(self, m_read_dmi_data):
+        """Return True if DMI chassis-asset-tag is AZURE_CHASSIS_ASSET_TAG."""
+        m_read_dmi_data.return_value = dsaz.AZURE_CHASSIS_ASSET_TAG
+        self.assertTrue(dsaz._is_platform_viable('doesnotmatter'))
+
+    @mock.patch('cloudinit.sources.DataSourceAzure.os.path.exists')
+    @mock.patch('cloudinit.sources.DataSourceAzure.util.read_dmi_data')
+    def test_true_on_azure_ovf_env_in_seed_dir(self, m_read_dmi_data, m_exist):
+        """Return True if ovf-env.xml exists in known seed dirs."""
+        # Non-matching Azure chassis-asset-tag
+        m_read_dmi_data.return_value = dsaz.AZURE_CHASSIS_ASSET_TAG + 'X'
+
+        m_exist.return_value = True
+        self.assertTrue(dsaz._is_platform_viable('/some/seed/dir'))
+        m_exist.called_once_with('/other/seed/dir')
+
+    def test_false_on_no_matching_azure_criteria(self):
+        """Report non-azure on unmatched asset tag, ovf-env absent and no dev.
+
+        Return False when the asset tag doesn't match Azure's static
+        AZURE_CHASSIS_ASSET_TAG, no ovf-env.xml files exist in known seed dirs
+        and no devices have a label starting with prefix 'rd_rdfe_'.
+        """
+        self.assertFalse(wrap_and_call(
+            'cloudinit.sources.DataSourceAzure',
+            {'os.path.exists': False,
+             # Non-matching Azure chassis-asset-tag
+             'util.read_dmi_data': dsaz.AZURE_CHASSIS_ASSET_TAG + 'X',
+             'util.which': None},
+            dsaz._is_platform_viable, 'doesnotmatter'))
+        self.assertIn(
+            "DEBUG: Non-Azure DMI asset tag '{0}' discovered.\n".format(
+                dsaz.AZURE_CHASSIS_ASSET_TAG + 'X'),
+            self.logs.getvalue())
+
+
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From aaffd59431fe05932a66016db941fe197c4e7620 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Fri, 17 Aug 2018 20:25:31 +0000
Subject: Add datasource Oracle Compute Infrastructure (OCI).

This adds a Oracle specific datasource that functions with OCI.
It is a simplified version of the OpenStack metadata server
with support for vendor-data.

It does not support the OCI-C (classic) platform.

Also here is a move of BrokenMetadata to common 'sources'
as this was the third occurrence of that class.
---
 .pylintrc                                         |   3 +-
 cloudinit/apport.py                               |   1 +
 cloudinit/settings.py                             |   1 +
 cloudinit/sources/DataSourceIBMCloud.py           |  13 +-
 cloudinit/sources/DataSourceOpenStack.py          |  12 +-
 cloudinit/sources/DataSourceOracle.py             | 233 +++++++++++++++
 cloudinit/sources/__init__.py                     |   4 +
 cloudinit/sources/helpers/openstack.py            |   6 +-
 cloudinit/sources/tests/test_oracle.py            | 331 ++++++++++++++++++++++
 doc/rtd/topics/datasources.rst                    |   1 +
 doc/rtd/topics/datasources/oracle.rst             |  26 ++
 tests/unittests/test_datasource/test_common.py    |   2 +
 tests/unittests/test_datasource/test_openstack.py |  13 +-
 tests/unittests/test_ds_identify.py               |  19 ++
 tools/ds-identify                                 |   8 +-
 15 files changed, 650 insertions(+), 23 deletions(-)
 create mode 100644 cloudinit/sources/DataSourceOracle.py
 create mode 100644 cloudinit/sources/tests/test_oracle.py
 create mode 100644 doc/rtd/topics/datasources/oracle.rst

(limited to 'cloudinit/sources')

diff --git a/.pylintrc b/.pylintrc
index 3bfa0c81..e376b48b 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -61,7 +61,8 @@ ignored-modules=
 # List of class names for which member attributes should not be checked (useful
 # for classes with dynamically set attributes). This supports the use of
 # qualified names.
-ignored-classes=optparse.Values,thread._local
+# argparse.Namespace from https://github.com/PyCQA/pylint/issues/2413
+ignored-classes=argparse.Namespace,optparse.Values,thread._local
 
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
diff --git a/cloudinit/apport.py b/cloudinit/apport.py
index 130ff269..22cb7fde 100644
--- a/cloudinit/apport.py
+++ b/cloudinit/apport.py
@@ -30,6 +30,7 @@ KNOWN_CLOUD_NAMES = [
     'NoCloud',
     'OpenNebula',
     'OpenStack',
+    'Oracle',
     'OVF',
     'OpenTelekomCloud',
     'Scaleway',
diff --git a/cloudinit/settings.py b/cloudinit/settings.py
index dde5749d..ea367cb7 100644
--- a/cloudinit/settings.py
+++ b/cloudinit/settings.py
@@ -38,6 +38,7 @@ CFG_BUILTIN = {
         'Scaleway',
         'Hetzner',
         'IBMCloud',
+        'Oracle',
         # At the end to act as a 'catch' when none of the above work...
         'None',
     ],
diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py
index 01106ec0..a5358148 100644
--- a/cloudinit/sources/DataSourceIBMCloud.py
+++ b/cloudinit/sources/DataSourceIBMCloud.py
@@ -295,7 +295,7 @@ def read_md():
             results = metadata_from_dir(path)
         else:
             results = util.mount_cb(path, metadata_from_dir)
-    except BrokenMetadata as e:
+    except sources.BrokenMetadata as e:
         raise RuntimeError(
             "Failed reading IBM config disk (platform=%s path=%s): %s" %
             (platform, path, e))
@@ -304,10 +304,6 @@ def read_md():
     return ret
 
 
-class BrokenMetadata(IOError):
-    pass
-
-
 def metadata_from_dir(source_dir):
     """Walk source_dir extracting standardized metadata.
 
@@ -352,12 +348,13 @@ def metadata_from_dir(source_dir):
             try:
                 data = transl(raw)
             except Exception as e:
-                raise BrokenMetadata("Failed decoding %s: %s" % (path, e))
+                raise sources.BrokenMetadata(
+                    "Failed decoding %s: %s" % (path, e))
 
         results[name] = data
 
     if results.get('metadata_raw') is None:
-        raise BrokenMetadata(
+        raise sources.BrokenMetadata(
             "%s missing required file 'meta_data.json'" % source_dir)
 
     results['metadata'] = {}
@@ -368,7 +365,7 @@ def metadata_from_dir(source_dir):
         try:
             md['random_seed'] = base64.b64decode(md_raw['random_seed'])
         except (ValueError, TypeError) as e:
-            raise BrokenMetadata(
+            raise sources.BrokenMetadata(
                 "Badly formatted metadata random_seed entry: %s" % e)
 
     renames = (
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index b9ade90d..4a015240 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -13,6 +13,7 @@ from cloudinit import url_helper
 from cloudinit import util
 
 from cloudinit.sources.helpers import openstack
+from cloudinit.sources import DataSourceOracle as oracle
 
 LOG = logging.getLogger(__name__)
 
@@ -28,8 +29,7 @@ DMI_PRODUCT_NOVA = 'OpenStack Nova'
 DMI_PRODUCT_COMPUTE = 'OpenStack Compute'
 VALID_DMI_PRODUCT_NAMES = [DMI_PRODUCT_NOVA, DMI_PRODUCT_COMPUTE]
 DMI_ASSET_TAG_OPENTELEKOM = 'OpenTelekomCloud'
-DMI_ASSET_TAG_ORACLE_CLOUD = 'OracleCloud.com'
-VALID_DMI_ASSET_TAGS = [DMI_ASSET_TAG_OPENTELEKOM, DMI_ASSET_TAG_ORACLE_CLOUD]
+VALID_DMI_ASSET_TAGS = [DMI_ASSET_TAG_OPENTELEKOM]
 
 
 class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
@@ -122,8 +122,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
             False when unable to contact metadata service or when metadata
             format is invalid or disabled.
         """
-        if not detect_openstack():
+        oracle_considered = 'Oracle' in self.sys_cfg.get('datasource_list')
+        if not detect_openstack(accept_oracle=not oracle_considered):
             return False
+
         if self.perform_dhcp_setup:  # Setup networking in init-local stage.
             try:
                 with EphemeralDHCPv4(self.fallback_interface):
@@ -215,7 +217,7 @@ def read_metadata_service(base_url, ssl_details=None,
     return reader.read_v2()
 
 
-def detect_openstack():
+def detect_openstack(accept_oracle=False):
     """Return True when a potential OpenStack platform is detected."""
     if not util.is_x86():
         return True  # Non-Intel cpus don't properly report dmi product names
@@ -224,6 +226,8 @@ def detect_openstack():
         return True
     elif util.read_dmi_data('chassis-asset-tag') in VALID_DMI_ASSET_TAGS:
         return True
+    elif accept_oracle and oracle._is_platform_viable():
+        return True
     elif util.get_proc_env(1).get('product_name') == DMI_PRODUCT_NOVA:
         return True
     return False
diff --git a/cloudinit/sources/DataSourceOracle.py b/cloudinit/sources/DataSourceOracle.py
new file mode 100644
index 00000000..fab39af3
--- /dev/null
+++ b/cloudinit/sources/DataSourceOracle.py
@@ -0,0 +1,233 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+"""Datasource for Oracle (OCI/Oracle Cloud Infrastructure)
+
+OCI provides a OpenStack like metadata service which provides only
+'2013-10-17' and 'latest' versions..
+
+Notes:
+ * This datasource does not support the OCI-Classic. OCI-Classic
+   provides an EC2 lookalike metadata service.
+ * The uuid provided in DMI data is not the same as the meta-data provided
+   instance-id, but has an equivalent lifespan.
+ * We do need to support upgrade from an instance that cloud-init
+   identified as OpenStack.
+ * Both bare-metal and vms use iscsi root
+ * Both bare-metal and vms provide chassis-asset-tag of OracleCloud.com
+"""
+
+from cloudinit.url_helper import combine_url, readurl, UrlError
+from cloudinit.net import dhcp
+from cloudinit import net
+from cloudinit import sources
+from cloudinit import util
+from cloudinit.net import cmdline
+from cloudinit import log as logging
+
+import json
+import re
+
+LOG = logging.getLogger(__name__)
+
+CHASSIS_ASSET_TAG = "OracleCloud.com"
+METADATA_ENDPOINT = "http://169.254.169.254/openstack/"
+
+
+class DataSourceOracle(sources.DataSource):
+
+    dsname = 'Oracle'
+    system_uuid = None
+    vendordata_pure = None
+    _network_config = sources.UNSET
+
+    def _is_platform_viable(self):
+        """Check platform environment to report if this datasource may run."""
+        return _is_platform_viable()
+
+    def _get_data(self):
+        if not self._is_platform_viable():
+            return False
+
+        # network may be configured if iscsi root.  If that is the case
+        # then read_kernel_cmdline_config will return non-None.
+        if _is_iscsi_root():
+            data = self.crawl_metadata()
+        else:
+            with dhcp.EphemeralDHCPv4(net.find_fallback_nic()):
+                data = self.crawl_metadata()
+
+        self._crawled_metadata = data
+        vdata = data['2013-10-17']
+
+        self.userdata_raw = vdata.get('user_data')
+        self.system_uuid = vdata['system_uuid']
+
+        vd = vdata.get('vendor_data')
+        if vd:
+            self.vendordata_pure = vd
+            try:
+                self.vendordata_raw = sources.convert_vendordata(vd)
+            except ValueError as e:
+                LOG.warning("Invalid content in vendor-data: %s", e)
+                self.vendordata_raw = None
+
+        mdcopies = ('public_keys',)
+        md = dict([(k, vdata['meta_data'].get(k))
+                   for k in mdcopies if k in vdata['meta_data']])
+
+        mdtrans = (
+            # oracle meta_data.json name, cloudinit.datasource.metadata name
+            ('availability_zone', 'availability-zone'),
+            ('hostname', 'local-hostname'),
+            ('launch_index', 'launch-index'),
+            ('uuid', 'instance-id'),
+        )
+        for dsname, ciname in mdtrans:
+            if dsname in vdata['meta_data']:
+                md[ciname] = vdata['meta_data'][dsname]
+
+        self.metadata = md
+        return True
+
+    def crawl_metadata(self):
+        return read_metadata()
+
+    def check_instance_id(self, sys_cfg):
+        """quickly check (local only) if self.instance_id is still valid
+
+        On Oracle, the dmi-provided system uuid differs from the instance-id
+        but has the same life-span."""
+        return sources.instance_id_matches_system_uuid(self.system_uuid)
+
+    def get_public_ssh_keys(self):
+        return sources.normalize_pubkey_data(self.metadata.get('public_keys'))
+
+    @property
+    def network_config(self):
+        """Network config is read from initramfs provided files
+        If none is present, then we fall back to fallback configuration.
+
+        One thing to note here is that this method is not currently
+        considered at all if there is is kernel/initramfs provided
+        data.  In that case, stages considers that the cmdline data
+        overrides datasource provided data and does not consult here.
+
+        We nonetheless return cmdline provided config if present
+        and fallback to generate fallback."""
+        if self._network_config == sources.UNSET:
+            cmdline_cfg = cmdline.read_kernel_cmdline_config()
+            if cmdline_cfg:
+                self._network_config = cmdline_cfg
+            else:
+                self._network_config = self.distro.generate_fallback_config()
+        return self._network_config
+
+
+def _read_system_uuid():
+    sys_uuid = util.read_dmi_data('system-uuid')
+    return None if sys_uuid is None else sys_uuid.lower()
+
+
+def _is_platform_viable():
+    asset_tag = util.read_dmi_data('chassis-asset-tag')
+    return asset_tag == CHASSIS_ASSET_TAG
+
+
+def _is_iscsi_root():
+    return bool(cmdline.read_kernel_cmdline_config())
+
+
+def _load_index(content):
+    """Return a list entries parsed from content.
+
+    OpenStack's metadata service returns a newline delimited list
+    of items.  Oracle's implementation has html formatted list of links.
+    The parser here just grabs targets from <a href="target">
+    and throws away "../".
+
+    Oracle has accepted that to be buggy and may fix in the future
+    to instead return a '\n' delimited plain text list.  This function
+    will continue to work if that change is made."""
+    if not content.lower().startswith("<html>"):
+        return content.splitlines()
+    items = re.findall(
+        r'href="(?P<target>[^"]*)"', content, re.MULTILINE | re.IGNORECASE)
+    return [i for i in items if not i.startswith(".")]
+
+
+def read_metadata(endpoint_base=METADATA_ENDPOINT, sys_uuid=None,
+                  version='2013-10-17'):
+    """Read metadata, return a dictionary.
+
+    Each path listed in the index will be represented in the dictionary.
+    If the path ends in .json, then the content will be decoded and
+    populated into the dictionary.
+
+    The system uuid (/sys/class/dmi/id/product_uuid) is also populated.
+    Example: given paths = ('user_data', 'meta_data.json')
+    This would return:
+      {version: {'user_data': b'blob', 'meta_data': json.loads(blob.decode())
+                 'system_uuid': '3b54f2e0-3ab2-458d-b770-af9926eee3b2'}}
+    """
+    endpoint = combine_url(endpoint_base, version) + "/"
+    if sys_uuid is None:
+        sys_uuid = _read_system_uuid()
+    if not sys_uuid:
+        raise sources.BrokenMetadata("Failed to read system uuid.")
+
+    try:
+        resp = readurl(endpoint)
+        if not resp.ok():
+            raise sources.BrokenMetadata(
+                "Bad response from %s: %s" % (endpoint, resp.code))
+    except UrlError as e:
+        raise sources.BrokenMetadata(
+            "Failed to read index at %s: %s" % (endpoint, e))
+
+    entries = _load_index(resp.contents.decode('utf-8'))
+    LOG.debug("index url %s contained: %s", endpoint, entries)
+
+    # meta_data.json is required.
+    mdj = 'meta_data.json'
+    if mdj not in entries:
+        raise sources.BrokenMetadata(
+            "Required field '%s' missing in index at %s" % (mdj, endpoint))
+
+    ret = {'system_uuid': sys_uuid}
+    for path in entries:
+        response = readurl(combine_url(endpoint, path))
+        if path.endswith(".json"):
+            ret[path.rpartition(".")[0]] = (
+                json.loads(response.contents.decode('utf-8')))
+        else:
+            ret[path] = response.contents
+
+    return {version: ret}
+
+
+# Used to match classes to dependencies
+datasources = [
+    (DataSourceOracle, (sources.DEP_FILESYSTEM,)),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
+
+
+if __name__ == "__main__":
+    import argparse
+    import os
+
+    parser = argparse.ArgumentParser(description='Query Oracle Cloud Metadata')
+    parser.add_argument("--endpoint", metavar="URL",
+                        help="The url of the metadata service.",
+                        default=METADATA_ENDPOINT)
+    args = parser.parse_args()
+    sys_uuid = "uuid-not-available-not-root" if os.geteuid() != 0 else None
+
+    data = read_metadata(endpoint_base=args.endpoint, sys_uuid=sys_uuid)
+    data['is_platform_viable'] = _is_platform_viable()
+    print(util.json_dumps(data))
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 06e613f8..41fde9ba 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -671,6 +671,10 @@ def convert_vendordata(data, recurse=True):
     raise ValueError("Unknown data type for vendordata: %s" % type(data))
 
 
+class BrokenMetadata(IOError):
+    pass
+
+
 # 'depends' is a list of dependencies (DEP_FILESYSTEM)
 # ds_list is a list of 2 item lists
 # ds_list = [
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index a4cf0667..8f9c1441 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -21,6 +21,8 @@ from cloudinit import sources
 from cloudinit import url_helper
 from cloudinit import util
 
+from cloudinit.sources import BrokenMetadata
+
 # See https://docs.openstack.org/user-guide/cli-config-drive.html
 
 LOG = logging.getLogger(__name__)
@@ -68,10 +70,6 @@ class NonReadable(IOError):
     pass
 
 
-class BrokenMetadata(IOError):
-    pass
-
-
 class SourceMixin(object):
     def _ec2_name_to_device(self, name):
         if not self.ec2_metadata:
diff --git a/cloudinit/sources/tests/test_oracle.py b/cloudinit/sources/tests/test_oracle.py
new file mode 100644
index 00000000..7599126c
--- /dev/null
+++ b/cloudinit/sources/tests/test_oracle.py
@@ -0,0 +1,331 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit.sources import DataSourceOracle as oracle
+from cloudinit.sources import BrokenMetadata
+from cloudinit import helpers
+
+from cloudinit.tests import helpers as test_helpers
+
+from textwrap import dedent
+import argparse
+import httpretty
+import json
+import mock
+import os
+import six
+import uuid
+
+DS_PATH = "cloudinit.sources.DataSourceOracle"
+MD_VER = "2013-10-17"
+
+
+class TestDataSourceOracle(test_helpers.CiTestCase):
+    """Test datasource DataSourceOracle."""
+
+    ds_class = oracle.DataSourceOracle
+
+    my_uuid = str(uuid.uuid4())
+    my_md = {"uuid": "ocid1.instance.oc1.phx.abyhqlj",
+             "name": "ci-vm1", "availability_zone": "phx-ad-3",
+             "hostname": "ci-vm1hostname",
+             "launch_index": 0, "files": [],
+             "public_keys": {"0": "ssh-rsa AAAAB3N...== user@host"},
+             "meta": {}}
+
+    def _patch_instance(self, inst, patches):
+        """Patch an instance of a class 'inst'.
+        for each name, kwargs in patches:
+             inst.name = mock.Mock(**kwargs)
+        returns a namespace object that has
+             namespace.name = mock.Mock(**kwargs)
+        Do not bother with cleanup as instance is assumed transient."""
+        mocks = argparse.Namespace()
+        for name, kwargs in patches.items():
+            imock = mock.Mock(name=name, spec=getattr(inst, name), **kwargs)
+            setattr(mocks, name, imock)
+            setattr(inst, name, imock)
+        return mocks
+
+    def _get_ds(self, sys_cfg=None, distro=None, paths=None, ud_proc=None,
+                patches=None):
+        if sys_cfg is None:
+            sys_cfg = {}
+        if patches is None:
+            patches = {}
+        if paths is None:
+            tmpd = self.tmp_dir()
+            dirs = {'cloud_dir': self.tmp_path('cloud_dir', tmpd),
+                    'run_dir': self.tmp_path('run_dir')}
+            for d in dirs.values():
+                os.mkdir(d)
+            paths = helpers.Paths(dirs)
+
+        ds = self.ds_class(sys_cfg=sys_cfg, distro=distro,
+                           paths=paths, ud_proc=ud_proc)
+
+        return ds, self._patch_instance(ds, patches)
+
+    def test_platform_not_viable_returns_false(self):
+        ds, mocks = self._get_ds(
+            patches={'_is_platform_viable': {'return_value': False}})
+        self.assertFalse(ds._get_data())
+        mocks._is_platform_viable.assert_called_once_with()
+
+    @mock.patch(DS_PATH + "._is_iscsi_root", return_value=True)
+    def test_without_userdata(self, m_is_iscsi_root):
+        """If no user-data is provided, it should not be in return dict."""
+        ds, mocks = self._get_ds(patches={
+            '_is_platform_viable': {'return_value': True},
+            'crawl_metadata': {
+                'return_value': {
+                    MD_VER: {'system_uuid': self.my_uuid,
+                             'meta_data': self.my_md}}}})
+        self.assertTrue(ds._get_data())
+        mocks._is_platform_viable.assert_called_once_with()
+        mocks.crawl_metadata.assert_called_once_with()
+        self.assertEqual(self.my_uuid, ds.system_uuid)
+        self.assertEqual(self.my_md['availability_zone'], ds.availability_zone)
+        self.assertIn(self.my_md["public_keys"]["0"], ds.get_public_ssh_keys())
+        self.assertEqual(self.my_md['uuid'], ds.get_instance_id())
+        self.assertIsNone(ds.userdata_raw)
+
+    @mock.patch(DS_PATH + "._is_iscsi_root", return_value=True)
+    def test_with_vendordata(self, m_is_iscsi_root):
+        """Test with vendor data."""
+        vd = {'cloud-init': '#cloud-config\nkey: value'}
+        ds, mocks = self._get_ds(patches={
+            '_is_platform_viable': {'return_value': True},
+            'crawl_metadata': {
+                'return_value': {
+                    MD_VER: {'system_uuid': self.my_uuid,
+                             'meta_data': self.my_md,
+                             'vendor_data': vd}}}})
+        self.assertTrue(ds._get_data())
+        mocks._is_platform_viable.assert_called_once_with()
+        mocks.crawl_metadata.assert_called_once_with()
+        self.assertEqual(vd, ds.vendordata_pure)
+        self.assertEqual(vd['cloud-init'], ds.vendordata_raw)
+
+    @mock.patch(DS_PATH + "._is_iscsi_root", return_value=True)
+    def test_with_userdata(self, m_is_iscsi_root):
+        """Ensure user-data is populated if present and is binary."""
+        my_userdata = b'abcdefg'
+        ds, mocks = self._get_ds(patches={
+            '_is_platform_viable': {'return_value': True},
+            'crawl_metadata': {
+                'return_value': {
+                    MD_VER: {'system_uuid': self.my_uuid,
+                             'meta_data': self.my_md,
+                             'user_data': my_userdata}}}})
+        self.assertTrue(ds._get_data())
+        mocks._is_platform_viable.assert_called_once_with()
+        mocks.crawl_metadata.assert_called_once_with()
+        self.assertEqual(self.my_uuid, ds.system_uuid)
+        self.assertIn(self.my_md["public_keys"]["0"], ds.get_public_ssh_keys())
+        self.assertEqual(self.my_md['uuid'], ds.get_instance_id())
+        self.assertEqual(my_userdata, ds.userdata_raw)
+
+    @mock.patch(DS_PATH + ".cmdline.read_kernel_cmdline_config")
+    @mock.patch(DS_PATH + "._is_iscsi_root", return_value=True)
+    def test_network_cmdline(self, m_is_iscsi_root, m_cmdline_config):
+        """network_config should read kernel cmdline."""
+        distro = mock.MagicMock()
+        ds, _ = self._get_ds(distro=distro, patches={
+            '_is_platform_viable': {'return_value': True},
+            'crawl_metadata': {
+                'return_value': {
+                    MD_VER: {'system_uuid': self.my_uuid,
+                             'meta_data': self.my_md}}}})
+        ncfg = {'version': 1, 'config': [{'a': 'b'}]}
+        m_cmdline_config.return_value = ncfg
+        self.assertTrue(ds._get_data())
+        self.assertEqual(ncfg, ds.network_config)
+        m_cmdline_config.assert_called_once_with()
+        self.assertFalse(distro.generate_fallback_config.called)
+
+    @mock.patch(DS_PATH + ".cmdline.read_kernel_cmdline_config")
+    @mock.patch(DS_PATH + "._is_iscsi_root", return_value=True)
+    def test_network_fallback(self, m_is_iscsi_root, m_cmdline_config):
+        """test that fallback network is generated if no kernel cmdline."""
+        distro = mock.MagicMock()
+        ds, _ = self._get_ds(distro=distro, patches={
+            '_is_platform_viable': {'return_value': True},
+            'crawl_metadata': {
+                'return_value': {
+                    MD_VER: {'system_uuid': self.my_uuid,
+                             'meta_data': self.my_md}}}})
+        ncfg = {'version': 1, 'config': [{'a': 'b'}]}
+        m_cmdline_config.return_value = None
+        self.assertTrue(ds._get_data())
+        ncfg = {'version': 1, 'config': [{'distro1': 'value'}]}
+        distro.generate_fallback_config.return_value = ncfg
+        self.assertEqual(ncfg, ds.network_config)
+        m_cmdline_config.assert_called_once_with()
+        distro.generate_fallback_config.assert_called_once_with()
+        self.assertEqual(1, m_cmdline_config.call_count)
+
+        # test that the result got cached, and the methods not re-called.
+        self.assertEqual(ncfg, ds.network_config)
+        self.assertEqual(1, m_cmdline_config.call_count)
+
+
+@mock.patch(DS_PATH + "._read_system_uuid", return_value=str(uuid.uuid4()))
+class TestReadMetaData(test_helpers.HttprettyTestCase):
+    """Test the read_metadata which interacts with http metadata service."""
+
+    mdurl = oracle.METADATA_ENDPOINT
+    my_md = {"uuid": "ocid1.instance.oc1.phx.abyhqlj",
+             "name": "ci-vm1", "availability_zone": "phx-ad-3",
+             "hostname": "ci-vm1hostname",
+             "launch_index": 0, "files": [],
+             "public_keys": {"0": "ssh-rsa AAAAB3N...== user@host"},
+             "meta": {}}
+
+    def populate_md(self, data):
+        """call httppretty.register_url for each item dict 'data',
+           including valid indexes. Text values converted to bytes."""
+        httpretty.register_uri(
+            httpretty.GET, self.mdurl + MD_VER + "/",
+            '\n'.join(data.keys()).encode('utf-8'))
+        for k, v in data.items():
+            httpretty.register_uri(
+                httpretty.GET, self.mdurl + MD_VER + "/" + k,
+                v if not isinstance(v, six.text_type) else v.encode('utf-8'))
+
+    def test_broken_no_sys_uuid(self, m_read_system_uuid):
+        """Datasource requires ability to read system_uuid and true return."""
+        m_read_system_uuid.return_value = None
+        self.assertRaises(BrokenMetadata, oracle.read_metadata)
+
+    def test_broken_no_metadata_json(self, m_read_system_uuid):
+        """Datasource requires meta_data.json."""
+        httpretty.register_uri(
+            httpretty.GET, self.mdurl + MD_VER + "/",
+            '\n'.join(['user_data']).encode('utf-8'))
+        with self.assertRaises(BrokenMetadata) as cm:
+            oracle.read_metadata()
+        self.assertIn("Required field 'meta_data.json' missing",
+                      str(cm.exception))
+
+    def test_with_userdata(self, m_read_system_uuid):
+        data = {'user_data': b'#!/bin/sh\necho hi world\n',
+                'meta_data.json': json.dumps(self.my_md)}
+        self.populate_md(data)
+        result = oracle.read_metadata()[MD_VER]
+        self.assertEqual(data['user_data'], result['user_data'])
+        self.assertEqual(self.my_md, result['meta_data'])
+
+    def test_without_userdata(self, m_read_system_uuid):
+        data = {'meta_data.json': json.dumps(self.my_md)}
+        self.populate_md(data)
+        result = oracle.read_metadata()[MD_VER]
+        self.assertNotIn('user_data', result)
+        self.assertEqual(self.my_md, result['meta_data'])
+
+    def test_unknown_fields_included(self, m_read_system_uuid):
+        """Unknown fields listed in index should be included.
+        And those ending in .json should be decoded."""
+        some_data = {'key1': 'data1', 'subk1': {'subd1': 'subv'}}
+        some_vendor_data = {'cloud-init': 'foo'}
+        data = {'meta_data.json': json.dumps(self.my_md),
+                'some_data.json': json.dumps(some_data),
+                'vendor_data.json': json.dumps(some_vendor_data),
+                'other_blob': b'this is blob'}
+        self.populate_md(data)
+        result = oracle.read_metadata()[MD_VER]
+        self.assertNotIn('user_data', result)
+        self.assertEqual(self.my_md, result['meta_data'])
+        self.assertEqual(some_data, result['some_data'])
+        self.assertEqual(some_vendor_data, result['vendor_data'])
+        self.assertEqual(data['other_blob'], result['other_blob'])
+
+
+class TestIsPlatformViable(test_helpers.CiTestCase):
+    @mock.patch(DS_PATH + ".util.read_dmi_data",
+                return_value=oracle.CHASSIS_ASSET_TAG)
+    def test_expected_viable(self, m_read_dmi_data):
+        """System with known chassis tag is viable."""
+        self.assertTrue(oracle._is_platform_viable())
+        m_read_dmi_data.assert_has_calls([mock.call('chassis-asset-tag')])
+
+    @mock.patch(DS_PATH + ".util.read_dmi_data", return_value=None)
+    def test_expected_not_viable_dmi_data_none(self, m_read_dmi_data):
+        """System without known chassis tag is not viable."""
+        self.assertFalse(oracle._is_platform_viable())
+        m_read_dmi_data.assert_has_calls([mock.call('chassis-asset-tag')])
+
+    @mock.patch(DS_PATH + ".util.read_dmi_data", return_value="LetsGoCubs")
+    def test_expected_not_viable_other(self, m_read_dmi_data):
+        """System with unnown chassis tag is not viable."""
+        self.assertFalse(oracle._is_platform_viable())
+        m_read_dmi_data.assert_has_calls([mock.call('chassis-asset-tag')])
+
+
+class TestLoadIndex(test_helpers.CiTestCase):
+    """_load_index handles parsing of an index into a proper list.
+    The tests here guarantee correct parsing of html version or
+    a fixed version.  See the function docstring for more doc."""
+
+    _known_html_api_versions = dedent("""\
+        <html>
+        <head><title>Index of /openstack/</title></head>
+        <body bgcolor="white">
+        <h1>Index of /openstack/</h1><hr><pre><a href="../">../</a>
+        <a href="2013-10-17/">2013-10-17/</a>   27-Jun-2018 12:22  -
+        <a href="latest/">latest/</a>           27-Jun-2018 12:22  -
+        </pre><hr></body>
+        </html>""")
+
+    _known_html_contents = dedent("""\
+        <html>
+        <head><title>Index of /openstack/2013-10-17/</title></head>
+        <body bgcolor="white">
+        <h1>Index of /openstack/2013-10-17/</h1><hr><pre><a href="../">../</a>
+        <a href="meta_data.json">meta_data.json</a>  27-Jun-2018 12:22  679
+        <a href="user_data">user_data</a>            27-Jun-2018 12:22  146
+        </pre><hr></body>
+        </html>""")
+
+    def test_parse_html(self):
+        """Test parsing of lower case html."""
+        self.assertEqual(
+            ['2013-10-17/', 'latest/'],
+            oracle._load_index(self._known_html_api_versions))
+        self.assertEqual(
+            ['meta_data.json', 'user_data'],
+            oracle._load_index(self._known_html_contents))
+
+    def test_parse_html_upper(self):
+        """Test parsing of upper case html, although known content is lower."""
+        def _toupper(data):
+            return data.replace("<a", "<A").replace("html>", "HTML>")
+
+        self.assertEqual(
+            ['2013-10-17/', 'latest/'],
+            oracle._load_index(_toupper(self._known_html_api_versions)))
+        self.assertEqual(
+            ['meta_data.json', 'user_data'],
+            oracle._load_index(_toupper(self._known_html_contents)))
+
+    def test_parse_newline_list_with_endl(self):
+        """Test parsing of newline separated list with ending newline."""
+        self.assertEqual(
+            ['2013-10-17/', 'latest/'],
+            oracle._load_index("\n".join(["2013-10-17/", "latest/", ""])))
+        self.assertEqual(
+            ['meta_data.json', 'user_data'],
+            oracle._load_index("\n".join(["meta_data.json", "user_data", ""])))
+
+    def test_parse_newline_list_without_endl(self):
+        """Test parsing of newline separated list with no ending newline.
+
+        Actual openstack implementation does not include trailing newline."""
+        self.assertEqual(
+            ['2013-10-17/', 'latest/'],
+            oracle._load_index("\n".join(["2013-10-17/", "latest/"])))
+        self.assertEqual(
+            ['meta_data.json', 'user_data'],
+            oracle._load_index("\n".join(["meta_data.json", "user_data"])))
+
+
+# vi: ts=4 expandtab
diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst
index 30e57d85..83034589 100644
--- a/doc/rtd/topics/datasources.rst
+++ b/doc/rtd/topics/datasources.rst
@@ -189,6 +189,7 @@ Follow for more information.
    datasources/nocloud.rst
    datasources/opennebula.rst
    datasources/openstack.rst
+   datasources/oracle.rst
    datasources/ovf.rst
    datasources/smartos.rst
    datasources/fallback.rst
diff --git a/doc/rtd/topics/datasources/oracle.rst b/doc/rtd/topics/datasources/oracle.rst
new file mode 100644
index 00000000..f2383cee
--- /dev/null
+++ b/doc/rtd/topics/datasources/oracle.rst
@@ -0,0 +1,26 @@
+.. _datasource_oracle:
+
+Oracle
+======
+
+This datasource reads metadata, vendor-data and user-data from
+`Oracle Compute Infrastructure`_ (OCI).
+
+Oracle Platform
+---------------
+OCI provides bare metal and virtual machines.  In both cases, 
+the platform identifies itself via DMI data in the chassis asset tag
+with the string 'OracleCloud.com'.
+
+Oracle's platform provides a metadata service that mimics the 2013-10-17
+version of OpenStack metadata service.  Initially support for Oracle
+was done via the OpenStack datasource.
+
+Cloud-init has a specific datasource for Oracle in order to:
+ a. allow and support future growth of the OCI platform.
+ b. address small differences between OpenStack and Oracle metadata
+    implementation.
+
+
+.. _Oracle Compute Infrastructure: https://cloud.oracle.com/
+.. vi: textwidth=78
diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
index 1a5a3db2..6b01a4ea 100644
--- a/tests/unittests/test_datasource/test_common.py
+++ b/tests/unittests/test_datasource/test_common.py
@@ -20,6 +20,7 @@ from cloudinit.sources import (
     DataSourceNoCloud as NoCloud,
     DataSourceOpenNebula as OpenNebula,
     DataSourceOpenStack as OpenStack,
+    DataSourceOracle as Oracle,
     DataSourceOVF as OVF,
     DataSourceScaleway as Scaleway,
     DataSourceSmartOS as SmartOS,
@@ -37,6 +38,7 @@ DEFAULT_LOCAL = [
     IBMCloud.DataSourceIBMCloud,
     NoCloud.DataSourceNoCloud,
     OpenNebula.DataSourceOpenNebula,
+    Oracle.DataSourceOracle,
     OVF.DataSourceOVF,
     SmartOS.DataSourceSmartOS,
     Ec2.DataSourceEc2Local,
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index d862f4bc..6e1e971b 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -16,7 +16,7 @@ from six import StringIO
 
 from cloudinit import helpers
 from cloudinit import settings
-from cloudinit.sources import convert_vendordata, UNSET
+from cloudinit.sources import BrokenMetadata, convert_vendordata, UNSET
 from cloudinit.sources import DataSourceOpenStack as ds
 from cloudinit.sources.helpers import openstack
 from cloudinit import util
@@ -186,7 +186,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('meta_data.json'):
                 os_files[k] = json.dumps(os_meta)
         _register_uris(self.VERSION, {}, {}, os_files)
-        self.assertRaises(openstack.BrokenMetadata, _read_metadata_service)
+        self.assertRaises(BrokenMetadata, _read_metadata_service)
 
     def test_userdata_empty(self):
         os_files = copy.deepcopy(OS_FILES)
@@ -217,7 +217,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('vendor_data.json'):
                 os_files[k] = '{'  # some invalid json
         _register_uris(self.VERSION, {}, {}, os_files)
-        self.assertRaises(openstack.BrokenMetadata, _read_metadata_service)
+        self.assertRaises(BrokenMetadata, _read_metadata_service)
 
     def test_metadata_invalid(self):
         os_files = copy.deepcopy(OS_FILES)
@@ -225,7 +225,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('meta_data.json'):
                 os_files[k] = '{'  # some invalid json
         _register_uris(self.VERSION, {}, {}, os_files)
-        self.assertRaises(openstack.BrokenMetadata, _read_metadata_service)
+        self.assertRaises(BrokenMetadata, _read_metadata_service)
 
     @test_helpers.mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
     def test_datasource(self, m_dhcp):
@@ -525,8 +525,11 @@ class TestDetectOpenStack(test_helpers.CiTestCase):
 
         m_dmi.side_effect = fake_dmi_read
         self.assertTrue(
-            ds.detect_openstack(),
+            ds.detect_openstack(accept_oracle=True),
             'Expected detect_openstack == True on OracleCloud.com')
+        self.assertFalse(
+            ds.detect_openstack(accept_oracle=False),
+            'Expected detect_openstack == False.')
 
     @test_helpers.mock.patch(MOCK_PATH + 'util.get_proc_env')
     @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data')
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 64d9f9f8..e08e7908 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -12,6 +12,7 @@ from cloudinit.tests.helpers import (
 
 from cloudinit.sources import DataSourceIBMCloud as ds_ibm
 from cloudinit.sources import DataSourceSmartOS as ds_smartos
+from cloudinit.sources import DataSourceOracle as ds_oracle
 
 UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu "
                "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux")
@@ -598,6 +599,18 @@ class TestIsIBMProvisioning(DsIdentifyBase):
         self.assertIn("from current boot", ret.stderr)
 
 
+class TestOracle(DsIdentifyBase):
+    def test_found_by_chassis(self):
+        """Simple positive test of Oracle by chassis id."""
+        self._test_ds_found('Oracle')
+
+    def test_not_found(self):
+        """Simple negative test of Oracle."""
+        mycfg = copy.deepcopy(VALID_CFG['Oracle'])
+        mycfg['files'][P_CHASSIS_ASSET_TAG] = "Not Oracle"
+        self._check_via_dict(mycfg, rc=RC_NOT_FOUND)
+
+
 def blkid_out(disks=None):
     """Convert a list of disk dictionaries into blkid content."""
     if disks is None:
@@ -838,6 +851,12 @@ VALID_CFG = {
              },
         ],
     },
+    'Oracle': {
+        'ds': 'Oracle',
+        'files': {
+            P_CHASSIS_ASSET_TAG: ds_oracle.CHASSIS_ASSET_TAG + '\n',
+        }
+    },
     'SmartOS-bhyve': {
         'ds': 'SmartOS',
         'mocks': [
diff --git a/tools/ds-identify b/tools/ds-identify
index ce0477a5..fcc60149 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -116,7 +116,7 @@ DI_DSNAME=""
 # be searched if there is no setting found in config.
 DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \
 CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \
-OVF SmartOS Scaleway Hetzner IBMCloud"
+OVF SmartOS Scaleway Hetzner IBMCloud Oracle"
 DI_DSLIST=""
 DI_MODE=""
 DI_ON_FOUND=""
@@ -1036,6 +1036,12 @@ dscheck_Hetzner() {
     return ${DS_NOT_FOUND}
 }
 
+dscheck_Oracle() {
+    local asset_tag="OracleCloud.com"
+    dmi_chassis_asset_tag_matches "${asset_tag}" && return ${DS_FOUND}
+    return ${DS_NOT_FOUND}
+}
+
 is_ibm_provisioning() {
     local pcfg="${PATH_ROOT}/root/provisioningConfiguration.cfg"
     local logf="${PATH_ROOT}/root/swinstall.log"
-- 
cgit v1.2.3


From 9c35f9762028b8bf15cdcd6b42c0fafc233ddda3 Mon Sep 17 00:00:00 2001
From: Pengpeng Sun <pengpengs@vmware.com>
Date: Tue, 28 Aug 2018 22:52:28 +0000
Subject: VMWare: Fix a network config bug in vm with static IPv4 and no
 gateway.

The issue is when customize a VM with static IPv4 and without gateway, it
will still extend route list and will loop a gateways list which is None.
This fix is to make sure when no gateway is here, it will not extend route
list.

LP: #1766538
---
 cloudinit/sources/helpers/vmware/imc/config_nic.py |   2 +-
 tests/unittests/test_vmware_config_file.py         | 115 +++++++++++++++++++++
 2 files changed, 116 insertions(+), 1 deletion(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py
index 3ef8c624..e1890e23 100644
--- a/cloudinit/sources/helpers/vmware/imc/config_nic.py
+++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py
@@ -164,7 +164,7 @@ class NicConfigurator(object):
             return ([subnet], route_list)
 
         # Add routes if there is no primary nic
-        if not self._primaryNic:
+        if not self._primaryNic and v4.gateways:
             route_list.extend(self.gen_ipv4_route(nic,
                                                   v4.gateways,
                                                   v4.netmask))
diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py
index 036f6879..602dedb0 100644
--- a/tests/unittests/test_vmware_config_file.py
+++ b/tests/unittests/test_vmware_config_file.py
@@ -2,11 +2,15 @@
 # Copyright (C) 2016 VMware INC.
 #
 # Author: Sankar Tanguturi <stanguturi@vmware.com>
+#         Pengpeng Sun <pengpengs@vmware.com>
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import logging
+import os
 import sys
+import tempfile
+import textwrap
 
 from cloudinit.sources.DataSourceOVF import get_network_config_from_conf
 from cloudinit.sources.DataSourceOVF import read_vmware_imc
@@ -343,4 +347,115 @@ class TestVmwareConfigFile(CiTestCase):
         conf = Config(cf)
         self.assertEqual("test-script", conf.custom_script_name)
 
+
+class TestVmwareNetConfig(CiTestCase):
+    """Test conversion of vmware config to cloud-init config."""
+
+    def _get_NicConfigurator(self, text):
+        fp = None
+        try:
+            with tempfile.NamedTemporaryFile(mode="w", dir=self.tmp_dir(),
+                                             delete=False) as fp:
+                fp.write(text)
+                fp.close()
+            cfg = Config(ConfigFile(fp.name))
+            return NicConfigurator(cfg.nics, use_system_devices=False)
+        finally:
+            if fp:
+                os.unlink(fp.name)
+
+    def test_non_primary_nic_without_gateway(self):
+        """A non primary nic set is not required to have a gateway."""
+        config = textwrap.dedent("""\
+            [NETWORK]
+            NETWORKING = yes
+            BOOTPROTO = dhcp
+            HOSTNAME = myhost1
+            DOMAINNAME = eng.vmware.com
+
+            [NIC-CONFIG]
+            NICS = NIC1
+
+            [NIC1]
+            MACADDR = 00:50:56:a6:8c:08
+            ONBOOT = yes
+            IPv4_MODE = BACKWARDS_COMPATIBLE
+            BOOTPROTO = static
+            IPADDR = 10.20.87.154
+            NETMASK = 255.255.252.0
+            """)
+        nc = self._get_NicConfigurator(config)
+        self.assertEqual(
+            [{'type': 'physical', 'name': 'NIC1',
+              'mac_address': '00:50:56:a6:8c:08',
+              'subnets': [
+                  {'control': 'auto', 'type': 'static',
+                   'address': '10.20.87.154', 'netmask': '255.255.252.0'}]}],
+            nc.generate())
+
+    def test_non_primary_nic_with_gateway(self):
+        """A non primary nic set can have a gateway."""
+        config = textwrap.dedent("""\
+            [NETWORK]
+            NETWORKING = yes
+            BOOTPROTO = dhcp
+            HOSTNAME = myhost1
+            DOMAINNAME = eng.vmware.com
+
+            [NIC-CONFIG]
+            NICS = NIC1
+
+            [NIC1]
+            MACADDR = 00:50:56:a6:8c:08
+            ONBOOT = yes
+            IPv4_MODE = BACKWARDS_COMPATIBLE
+            BOOTPROTO = static
+            IPADDR = 10.20.87.154
+            NETMASK = 255.255.252.0
+            GATEWAY = 10.20.87.253
+            """)
+        nc = self._get_NicConfigurator(config)
+        self.assertEqual(
+            [{'type': 'physical', 'name': 'NIC1',
+              'mac_address': '00:50:56:a6:8c:08',
+              'subnets': [
+                  {'control': 'auto', 'type': 'static',
+                   'address': '10.20.87.154', 'netmask': '255.255.252.0'}]},
+             {'type': 'route', 'destination': '10.20.84.0/22',
+              'gateway': '10.20.87.253', 'metric': 10000}],
+            nc.generate())
+
+    def test_a_primary_nic_with_gateway(self):
+        """A primary nic set can have a gateway."""
+        config = textwrap.dedent("""\
+            [NETWORK]
+            NETWORKING = yes
+            BOOTPROTO = dhcp
+            HOSTNAME = myhost1
+            DOMAINNAME = eng.vmware.com
+
+            [NIC-CONFIG]
+            NICS = NIC1
+
+            [NIC1]
+            MACADDR = 00:50:56:a6:8c:08
+            ONBOOT = yes
+            IPv4_MODE = BACKWARDS_COMPATIBLE
+            BOOTPROTO = static
+            IPADDR = 10.20.87.154
+            NETMASK = 255.255.252.0
+            PRIMARY = true
+            GATEWAY = 10.20.87.253
+            """)
+        nc = self._get_NicConfigurator(config)
+        self.assertEqual(
+            [{'type': 'physical', 'name': 'NIC1',
+              'mac_address': '00:50:56:a6:8c:08',
+              'subnets': [
+                  {'control': 'auto', 'type': 'static',
+                   'address': '10.20.87.154', 'netmask': '255.255.252.0',
+                   'gateway': '10.20.87.253'}]}],
+            nc.generate())
+
+
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From a8dcad9ac62bb1d2a4f7489960395bad6cac9382 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 5 Sep 2018 16:02:25 +0000
Subject: tests: Disallow use of util.subp except for where needed.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

In many cases, cloud-init uses 'util.subp' to run a subprocess.
This is not really desirable in our unit tests as it makes the tests
dependent upon existance of those utilities.

The change here is to modify the base test case class (CiTestCase) to
raise exception any time subp is called.  Then, fix all callers.
For cases where subp is necessary or actually desired, we can use it
via
  a.) context hander CiTestCase.allow_subp(value)
  b.) class level self.allowed_subp = value

Both cases the value is a list of acceptable executable names that
will be called (essentially argv[0]).

Some cleanups in AltCloud were done as the code was being updated.
---
 cloudinit/analyze/tests/test_dump.py               | 86 ++++++++------------
 cloudinit/cmd/tests/test_status.py                 |  6 +-
 cloudinit/config/tests/test_snap.py                |  7 +-
 cloudinit/config/tests/test_ubuntu_advantage.py    |  7 +-
 cloudinit/net/tests/test_init.py                   |  2 +
 cloudinit/sources/DataSourceAltCloud.py            | 24 +++---
 cloudinit/sources/DataSourceSmartOS.py             | 29 ++++---
 cloudinit/tests/helpers.py                         | 43 ++++++++++
 tests/unittests/test_datasource/test_altcloud.py   | 44 +++++-----
 tests/unittests/test_datasource/test_cloudsigma.py |  3 +
 .../unittests/test_datasource/test_configdrive.py  |  3 +
 tests/unittests/test_datasource/test_nocloud.py    |  2 +
 tests/unittests/test_datasource/test_opennebula.py |  1 +
 tests/unittests/test_datasource/test_ovf.py        |  8 +-
 tests/unittests/test_datasource/test_smartos.py    | 94 +++++++++++++++-------
 tests/unittests/test_ds_identify.py                |  1 +
 .../test_handler/test_handler_apt_source_v3.py     | 11 ++-
 .../unittests/test_handler/test_handler_bootcmd.py | 10 ++-
 tests/unittests/test_handler/test_handler_chef.py  | 18 +++--
 .../test_handler/test_handler_resizefs.py          |  8 +-
 tests/unittests/test_handler/test_schema.py        | 12 ++-
 tests/unittests/test_net.py                        | 18 ++++-
 tests/unittests/test_util.py                       | 27 ++++---
 23 files changed, 289 insertions(+), 175 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/analyze/tests/test_dump.py b/cloudinit/analyze/tests/test_dump.py
index f4c42841..db2a667b 100644
--- a/cloudinit/analyze/tests/test_dump.py
+++ b/cloudinit/analyze/tests/test_dump.py
@@ -5,8 +5,8 @@ from textwrap import dedent
 
 from cloudinit.analyze.dump import (
     dump_events, parse_ci_logline, parse_timestamp)
-from cloudinit.util import subp, write_file
-from cloudinit.tests.helpers import CiTestCase
+from cloudinit.util import which, write_file
+from cloudinit.tests.helpers import CiTestCase, mock, skipIf
 
 
 class TestParseTimestamp(CiTestCase):
@@ -15,21 +15,9 @@ class TestParseTimestamp(CiTestCase):
         """Logs with cloud-init detailed formats will be properly parsed."""
         trusty_fmt = '%Y-%m-%d %H:%M:%S,%f'
         trusty_stamp = '2016-09-12 14:39:20,839'
-
-        parsed = parse_timestamp(trusty_stamp)
-
-        # convert ourselves
         dt = datetime.strptime(trusty_stamp, trusty_fmt)
-        expected = float(dt.strftime('%s.%f'))
-
-        # use date(1)
-        out, _err = subp(['date', '+%s.%3N', '-d', trusty_stamp])
-        timestamp = out.strip()
-        date_ts = float(timestamp)
-
-        self.assertEqual(expected, parsed)
-        self.assertEqual(expected, date_ts)
-        self.assertEqual(date_ts, parsed)
+        self.assertEqual(
+            float(dt.strftime('%s.%f')), parse_timestamp(trusty_stamp))
 
     def test_parse_timestamp_handles_syslog_adding_year(self):
         """Syslog timestamps lack a year. Add year and properly parse."""
@@ -39,17 +27,9 @@ class TestParseTimestamp(CiTestCase):
         # convert stamp ourselves by adding the missing year value
         year = datetime.now().year
         dt = datetime.strptime(syslog_stamp + " " + str(year), syslog_fmt)
-        expected = float(dt.strftime('%s.%f'))
-        parsed = parse_timestamp(syslog_stamp)
-
-        # use date(1)
-        out, _ = subp(['date', '+%s.%3N', '-d', syslog_stamp])
-        timestamp = out.strip()
-        date_ts = float(timestamp)
-
-        self.assertEqual(expected, parsed)
-        self.assertEqual(expected, date_ts)
-        self.assertEqual(date_ts, parsed)
+        self.assertEqual(
+            float(dt.strftime('%s.%f')),
+            parse_timestamp(syslog_stamp))
 
     def test_parse_timestamp_handles_journalctl_format_adding_year(self):
         """Journalctl precise timestamps lack a year. Add year and parse."""
@@ -59,37 +39,22 @@ class TestParseTimestamp(CiTestCase):
         # convert stamp ourselves by adding the missing year value
         year = datetime.now().year
         dt = datetime.strptime(journal_stamp + " " + str(year), journal_fmt)
-        expected = float(dt.strftime('%s.%f'))
-        parsed = parse_timestamp(journal_stamp)
-
-        # use date(1)
-        out, _ = subp(['date', '+%s.%6N', '-d', journal_stamp])
-        timestamp = out.strip()
-        date_ts = float(timestamp)
-
-        self.assertEqual(expected, parsed)
-        self.assertEqual(expected, date_ts)
-        self.assertEqual(date_ts, parsed)
+        self.assertEqual(
+            float(dt.strftime('%s.%f')), parse_timestamp(journal_stamp))
 
+    @skipIf(not which("date"), "'date' command not available.")
     def test_parse_unexpected_timestamp_format_with_date_command(self):
-        """Dump sends unexpected timestamp formats to data for processing."""
+        """Dump sends unexpected timestamp formats to date for processing."""
         new_fmt = '%H:%M %m/%d %Y'
         new_stamp = '17:15 08/08'
-
         # convert stamp ourselves by adding the missing year value
         year = datetime.now().year
         dt = datetime.strptime(new_stamp + " " + str(year), new_fmt)
-        expected = float(dt.strftime('%s.%f'))
-        parsed = parse_timestamp(new_stamp)
 
         # use date(1)
-        out, _ = subp(['date', '+%s.%6N', '-d', new_stamp])
-        timestamp = out.strip()
-        date_ts = float(timestamp)
-
-        self.assertEqual(expected, parsed)
-        self.assertEqual(expected, date_ts)
-        self.assertEqual(date_ts, parsed)
+        with self.allow_subp(["date"]):
+            self.assertEqual(
+                float(dt.strftime('%s.%f')), parse_timestamp(new_stamp))
 
 
 class TestParseCILogLine(CiTestCase):
@@ -135,7 +100,9 @@ class TestParseCILogLine(CiTestCase):
             'timestamp': timestamp}
         self.assertEqual(expected, parse_ci_logline(line))
 
-    def test_parse_logline_returns_event_for_finish_events(self):
+    @mock.patch("cloudinit.analyze.dump.parse_timestamp_from_date")
+    def test_parse_logline_returns_event_for_finish_events(self,
+                                                           m_parse_from_date):
         """parse_ci_logline returns a finish event for a parsed log line."""
         line = ('2016-08-30 21:53:25.972325+00:00 y1 [CLOUDINIT]'
                 ' handlers.py[DEBUG]: finish: modules-final: SUCCESS: running'
@@ -147,7 +114,10 @@ class TestParseCILogLine(CiTestCase):
             'origin': 'cloudinit',
             'result': 'SUCCESS',
             'timestamp': 1472594005.972}
+        m_parse_from_date.return_value = "1472594005.972"
         self.assertEqual(expected, parse_ci_logline(line))
+        m_parse_from_date.assert_has_calls(
+            [mock.call("2016-08-30 21:53:25.972325+00:00")])
 
 
 SAMPLE_LOGS = dedent("""\
@@ -162,10 +132,16 @@ Nov 03 06:51:06.074410 x2 cloud-init[106]: [CLOUDINIT] util.py[DEBUG]:\
 class TestDumpEvents(CiTestCase):
     maxDiff = None
 
-    def test_dump_events_with_rawdata(self):
+    @mock.patch("cloudinit.analyze.dump.parse_timestamp_from_date")
+    def test_dump_events_with_rawdata(self, m_parse_from_date):
         """Rawdata is split and parsed into a tuple of events and data"""
+        m_parse_from_date.return_value = "1472594005.972"
         events, data = dump_events(rawdata=SAMPLE_LOGS)
         expected_data = SAMPLE_LOGS.splitlines()
+        self.assertEqual(
+            [mock.call("2016-08-30 21:53:25.972325+00:00")],
+            m_parse_from_date.call_args_list)
+        self.assertEqual(expected_data, data)
         year = datetime.now().year
         dt1 = datetime.strptime(
             'Nov 03 06:51:06.074410 %d' % year, '%b %d %H:%M:%S.%f %Y')
@@ -183,12 +159,14 @@ class TestDumpEvents(CiTestCase):
             'result': 'SUCCESS',
             'timestamp': 1472594005.972}]
         self.assertEqual(expected_events, events)
-        self.assertEqual(expected_data, data)
 
-    def test_dump_events_with_cisource(self):
+    @mock.patch("cloudinit.analyze.dump.parse_timestamp_from_date")
+    def test_dump_events_with_cisource(self, m_parse_from_date):
         """Cisource file is read and parsed into a tuple of events and data."""
         tmpfile = self.tmp_path('logfile')
         write_file(tmpfile, SAMPLE_LOGS)
+        m_parse_from_date.return_value = 1472594005.972
+
         events, data = dump_events(cisource=open(tmpfile))
         year = datetime.now().year
         dt1 = datetime.strptime(
@@ -208,3 +186,5 @@ class TestDumpEvents(CiTestCase):
             'timestamp': 1472594005.972}]
         self.assertEqual(expected_events, events)
         self.assertEqual(SAMPLE_LOGS.splitlines(), [d.strip() for d in data])
+        m_parse_from_date.assert_has_calls(
+            [mock.call("2016-08-30 21:53:25.972325+00:00")])
diff --git a/cloudinit/cmd/tests/test_status.py b/cloudinit/cmd/tests/test_status.py
index 37a89936..aded8580 100644
--- a/cloudinit/cmd/tests/test_status.py
+++ b/cloudinit/cmd/tests/test_status.py
@@ -39,7 +39,8 @@ class TestStatus(CiTestCase):
         ensure_file(self.disable_file)  # Create the ignored disable file
         (is_disabled, reason) = wrap_and_call(
             'cloudinit.cmd.status',
-            {'uses_systemd': False},
+            {'uses_systemd': False,
+             'get_cmdline': "root=/dev/my-root not-important"},
             status._is_cloudinit_disabled, self.disable_file, self.paths)
         self.assertFalse(
             is_disabled, 'expected enabled cloud-init on sysvinit')
@@ -50,7 +51,8 @@ class TestStatus(CiTestCase):
         ensure_file(self.disable_file)  # Create observed disable file
         (is_disabled, reason) = wrap_and_call(
             'cloudinit.cmd.status',
-            {'uses_systemd': True},
+            {'uses_systemd': True,
+             'get_cmdline': "root=/dev/my-root not-important"},
             status._is_cloudinit_disabled, self.disable_file, self.paths)
         self.assertTrue(is_disabled, 'expected disabled cloud-init')
         self.assertEqual(
diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py
index 34c80f1e..3c472891 100644
--- a/cloudinit/config/tests/test_snap.py
+++ b/cloudinit/config/tests/test_snap.py
@@ -162,6 +162,7 @@ class TestAddAssertions(CiTestCase):
 class TestRunCommands(CiTestCase):
 
     with_logs = True
+    allowed_subp = [CiTestCase.SUBP_SHELL_TRUE]
 
     def setUp(self):
         super(TestRunCommands, self).setUp()
@@ -424,8 +425,10 @@ class TestHandle(CiTestCase):
             'snap': {'commands': ['echo "HI" >> %s' % outfile,
                                   'echo "MOM" >> %s' % outfile]}}
         mock_path = 'cloudinit.config.cc_snap.sys.stderr'
-        with mock.patch(mock_path, new_callable=StringIO):
-            handle('snap', cfg=cfg, cloud=None, log=self.logger, args=None)
+        with self.allow_subp([CiTestCase.SUBP_SHELL_TRUE]):
+            with mock.patch(mock_path, new_callable=StringIO):
+                handle('snap', cfg=cfg, cloud=None, log=self.logger, args=None)
+
         self.assertEqual('HI\nMOM\n', util.load_file(outfile))
 
     @mock.patch('cloudinit.config.cc_snap.util.subp')
diff --git a/cloudinit/config/tests/test_ubuntu_advantage.py b/cloudinit/config/tests/test_ubuntu_advantage.py
index f1beeff8..b7cf9bee 100644
--- a/cloudinit/config/tests/test_ubuntu_advantage.py
+++ b/cloudinit/config/tests/test_ubuntu_advantage.py
@@ -23,6 +23,7 @@ class FakeCloud(object):
 class TestRunCommands(CiTestCase):
 
     with_logs = True
+    allowed_subp = [CiTestCase.SUBP_SHELL_TRUE]
 
     def setUp(self):
         super(TestRunCommands, self).setUp()
@@ -234,8 +235,10 @@ class TestHandle(CiTestCase):
             'ubuntu-advantage': {'commands': ['echo "HI" >> %s' % outfile,
                                               'echo "MOM" >> %s' % outfile]}}
         mock_path = '%s.sys.stderr' % MPATH
-        with mock.patch(mock_path, new_callable=StringIO):
-            handle('nomatter', cfg=cfg, cloud=None, log=self.logger, args=None)
+        with self.allow_subp([CiTestCase.SUBP_SHELL_TRUE]):
+            with mock.patch(mock_path, new_callable=StringIO):
+                handle('nomatter', cfg=cfg, cloud=None, log=self.logger,
+                       args=None)
         self.assertEqual('HI\nMOM\n', util.load_file(outfile))
 
 
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
index 5c017d15..8b444f14 100644
--- a/cloudinit/net/tests/test_init.py
+++ b/cloudinit/net/tests/test_init.py
@@ -199,6 +199,8 @@ class TestGenerateFallbackConfig(CiTestCase):
         self.sysdir = self.tmp_dir() + '/'
         self.m_sys_path.return_value = self.sysdir
         self.addCleanup(sys_mock.stop)
+        self.add_patch('cloudinit.net.util.is_container', 'm_is_container',
+                       return_value=False)
         self.add_patch('cloudinit.net.util.udevadm_settle', 'm_settle')
 
     def test_generate_fallback_finds_connected_eth_with_mac(self):
diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index 24fd65ff..8cd312d0 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -181,27 +181,18 @@ class DataSourceAltCloud(sources.DataSource):
 
         # modprobe floppy
         try:
-            cmd = CMD_PROBE_FLOPPY
-            (cmd_out, _err) = util.subp(cmd)
-            LOG.debug('Command: %s\nOutput%s', ' '.join(cmd), cmd_out)
+            modprobe_floppy()
         except ProcessExecutionError as e:
-            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), e)
-            return False
-        except OSError as e:
-            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), e)
+            util.logexc(LOG, 'Failed modprobe: %s', e)
             return False
 
         floppy_dev = '/dev/fd0'
 
         # udevadm settle for floppy device
         try:
-            (cmd_out, _err) = util.udevadm_settle(exists=floppy_dev, timeout=5)
-            LOG.debug('Command: %s\nOutput%s', ' '.join(cmd), cmd_out)
-        except ProcessExecutionError as e:
-            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), e)
-            return False
-        except OSError as e:
-            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), e)
+            util.udevadm_settle(exists=floppy_dev, timeout=5)
+        except (ProcessExecutionError, OSError) as e:
+            util.logexc(LOG, 'Failed udevadm_settle: %s\n', e)
             return False
 
         try:
@@ -258,6 +249,11 @@ class DataSourceAltCloud(sources.DataSource):
             return False
 
 
+def modprobe_floppy():
+    out, _err = util.subp(CMD_PROBE_FLOPPY)
+    LOG.debug('Command: %s\nOutput%s', ' '.join(CMD_PROBE_FLOPPY), out)
+
+
 # Used to match classes to dependencies
 # Source DataSourceAltCloud does not really depend on networking.
 # In the future 'dsmode' like behavior can be added to offer user
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index ad8cfb9c..593ac91a 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -683,6 +683,18 @@ def jmc_client_factory(
     raise ValueError("Unknown value for smartos_type: %s" % smartos_type)
 
 
+def identify_file(content_f):
+    cmd = ["file", "--brief", "--mime-type", content_f]
+    f_type = None
+    try:
+        (f_type, _err) = util.subp(cmd)
+        LOG.debug("script %s mime type is %s", content_f, f_type)
+    except util.ProcessExecutionError as e:
+        util.logexc(
+            LOG, ("Failed to identify script type for %s" % content_f, e))
+    return None if f_type is None else f_type.strip()
+
+
 def write_boot_content(content, content_f, link=None, shebang=False,
                        mode=0o400):
     """
@@ -715,18 +727,11 @@ def write_boot_content(content, content_f, link=None, shebang=False,
     util.write_file(content_f, content, mode=mode)
 
     if shebang and not content.startswith("#!"):
-        try:
-            cmd = ["file", "--brief", "--mime-type", content_f]
-            (f_type, _err) = util.subp(cmd)
-            LOG.debug("script %s mime type is %s", content_f, f_type)
-            if f_type.strip() == "text/plain":
-                new_content = "\n".join(["#!/bin/bash", content])
-                util.write_file(content_f, new_content, mode=mode)
-                LOG.debug("added shebang to file %s", content_f)
-
-        except Exception as e:
-            util.logexc(LOG, ("Failed to identify script type for %s" %
-                              content_f, e))
+        f_type = identify_file(content_f)
+        if f_type == "text/plain":
+            util.write_file(
+                content_f, "\n".join(["#!/bin/bash", content]), mode=mode)
+            LOG.debug("added shebang to file %s", content_f)
 
     if link:
         try:
diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py
index 9a21426e..e7e4182f 100644
--- a/cloudinit/tests/helpers.py
+++ b/cloudinit/tests/helpers.py
@@ -33,6 +33,8 @@ from cloudinit import helpers as ch
 from cloudinit.sources import DataSourceNone
 from cloudinit import util
 
+_real_subp = util.subp
+
 # Used for skipping tests
 SkipTest = unittest2.SkipTest
 skipIf = unittest2.skipIf
@@ -143,6 +145,17 @@ class CiTestCase(TestCase):
     # Subclass overrides for specific test behavior
     # Whether or not a unit test needs logfile setup
     with_logs = False
+    allowed_subp = False
+    SUBP_SHELL_TRUE = "shell=true"
+
+    @contextmanager
+    def allow_subp(self, allowed_subp):
+        orig = self.allowed_subp
+        try:
+            self.allowed_subp = allowed_subp
+            yield
+        finally:
+            self.allowed_subp = orig
 
     def setUp(self):
         super(CiTestCase, self).setUp()
@@ -155,11 +168,41 @@ class CiTestCase(TestCase):
             handler.setFormatter(formatter)
             self.old_handlers = self.logger.handlers
             self.logger.handlers = [handler]
+        if self.allowed_subp is True:
+            util.subp = _real_subp
+        else:
+            util.subp = self._fake_subp
+
+    def _fake_subp(self, *args, **kwargs):
+        if 'args' in kwargs:
+            cmd = kwargs['args']
+        else:
+            cmd = args[0]
+
+        if not isinstance(cmd, six.string_types):
+            cmd = cmd[0]
+        pass_through = False
+        if not isinstance(self.allowed_subp, (list, bool)):
+            raise TypeError("self.allowed_subp supports list or bool.")
+        if isinstance(self.allowed_subp, bool):
+            pass_through = self.allowed_subp
+        else:
+            pass_through = (
+                (cmd in self.allowed_subp) or
+                (self.SUBP_SHELL_TRUE in self.allowed_subp and
+                 kwargs.get('shell')))
+        if pass_through:
+            return _real_subp(*args, **kwargs)
+        raise Exception(
+            "called subp. set self.allowed_subp=True to allow\n subp(%s)" %
+            ', '.join([str(repr(a)) for a in args] +
+                      ["%s=%s" % (k, repr(v)) for k, v in kwargs.items()]))
 
     def tearDown(self):
         if self.with_logs:
             # Remove the handler we setup
             logging.getLogger().handlers = self.old_handlers
+        util.subp = _real_subp
         super(CiTestCase, self).tearDown()
 
     def tmp_dir(self, dir=None, cleanup=True):
diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py
index 3253f3ad..ff35904e 100644
--- a/tests/unittests/test_datasource/test_altcloud.py
+++ b/tests/unittests/test_datasource/test_altcloud.py
@@ -262,64 +262,56 @@ class TestUserDataRhevm(CiTestCase):
     '''
     Test to exercise method: DataSourceAltCloud.user_data_rhevm()
     '''
-    cmd_pass = ['true']
-    cmd_fail = ['false']
-    cmd_not_found = ['bogus bad command']
-
     def setUp(self):
         '''Set up.'''
         self.paths = helpers.Paths({'cloud_dir': '/tmp'})
-        self.mount_dir = tempfile.mkdtemp()
+        self.mount_dir = self.tmp_dir()
         _write_user_data_files(self.mount_dir, 'test user data')
-
-    def tearDown(self):
-        # Reset
-
-        _remove_user_data_files(self.mount_dir)
-
-        # Attempt to remove the temp dir ignoring errors
-        try:
-            shutil.rmtree(self.mount_dir)
-        except OSError:
-            pass
-
-        dsac.CLOUD_INFO_FILE = '/etc/sysconfig/cloud-info'
-        dsac.CMD_PROBE_FLOPPY = ['modprobe', 'floppy']
-        dsac.CMD_UDEVADM_SETTLE = ['udevadm', 'settle',
-                                   '--quiet', '--timeout=5']
+        self.add_patch(
+            'cloudinit.sources.DataSourceAltCloud.modprobe_floppy',
+            'm_modprobe_floppy', return_value=None)
+        self.add_patch(
+            'cloudinit.sources.DataSourceAltCloud.util.udevadm_settle',
+            'm_udevadm_settle', return_value=('', ''))
+        self.add_patch(
+            'cloudinit.sources.DataSourceAltCloud.util.mount_cb',
+            'm_mount_cb')
 
     def test_mount_cb_fails(self):
         '''Test user_data_rhevm() where mount_cb fails.'''
 
-        dsac.CMD_PROBE_FLOPPY = self.cmd_pass
+        self.m_mount_cb.side_effect = util.MountFailedError("Failed Mount")
         dsrc = dsac.DataSourceAltCloud({}, None, self.paths)
         self.assertEqual(False, dsrc.user_data_rhevm())
 
     def test_modprobe_fails(self):
         '''Test user_data_rhevm() where modprobe fails.'''
 
-        dsac.CMD_PROBE_FLOPPY = self.cmd_fail
+        self.m_modprobe_floppy.side_effect = util.ProcessExecutionError(
+            "Failed modprobe")
         dsrc = dsac.DataSourceAltCloud({}, None, self.paths)
         self.assertEqual(False, dsrc.user_data_rhevm())
 
     def test_no_modprobe_cmd(self):
         '''Test user_data_rhevm() with no modprobe command.'''
 
-        dsac.CMD_PROBE_FLOPPY = self.cmd_not_found
+        self.m_modprobe_floppy.side_effect = util.ProcessExecutionError(
+            "No such file or dir")
         dsrc = dsac.DataSourceAltCloud({}, None, self.paths)
         self.assertEqual(False, dsrc.user_data_rhevm())
 
     def test_udevadm_fails(self):
         '''Test user_data_rhevm() where udevadm fails.'''
 
-        dsac.CMD_UDEVADM_SETTLE = self.cmd_fail
+        self.m_udevadm_settle.side_effect = util.ProcessExecutionError(
+            "Failed settle.")
         dsrc = dsac.DataSourceAltCloud({}, None, self.paths)
         self.assertEqual(False, dsrc.user_data_rhevm())
 
     def test_no_udevadm_cmd(self):
         '''Test user_data_rhevm() with no udevadm command.'''
 
-        dsac.CMD_UDEVADM_SETTLE = self.cmd_not_found
+        self.m_udevadm_settle.side_effect = OSError("No such file or dir")
         dsrc = dsac.DataSourceAltCloud({}, None, self.paths)
         self.assertEqual(False, dsrc.user_data_rhevm())
 
diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py
index f6a59b6b..380ad1b5 100644
--- a/tests/unittests/test_datasource/test_cloudsigma.py
+++ b/tests/unittests/test_datasource/test_cloudsigma.py
@@ -42,6 +42,9 @@ class CepkoMock(Cepko):
 class DataSourceCloudSigmaTest(test_helpers.CiTestCase):
     def setUp(self):
         super(DataSourceCloudSigmaTest, self).setUp()
+        self.add_patch(
+            "cloudinit.sources.DataSourceCloudSigma.util.is_container",
+            "m_is_container", return_value=False)
         self.paths = helpers.Paths({'run_dir': self.tmp_dir()})
         self.datasource = DataSourceCloudSigma.DataSourceCloudSigma(
             "", "", paths=self.paths)
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index 7e6fcbbf..b0abfc5f 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -224,6 +224,9 @@ class TestConfigDriveDataSource(CiTestCase):
 
     def setUp(self):
         super(TestConfigDriveDataSource, self).setUp()
+        self.add_patch(
+            "cloudinit.sources.DataSourceConfigDrive.util.find_devs_with",
+            "m_find_devs_with", return_value=[])
         self.tmp = self.tmp_dir()
 
     def test_ec2_metadata(self):
diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py
index cdbd1e1a..21931eb7 100644
--- a/tests/unittests/test_datasource/test_nocloud.py
+++ b/tests/unittests/test_datasource/test_nocloud.py
@@ -25,6 +25,8 @@ class TestNoCloudDataSource(CiTestCase):
 
         self.mocks.enter_context(
             mock.patch.object(util, 'get_cmdline', return_value=self.cmdline))
+        self.mocks.enter_context(
+            mock.patch.object(util, 'read_dmi_data', return_value=None))
 
     def test_nocloud_seed_dir(self):
         md = {'instance-id': 'IID', 'dsmode': 'local'}
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index 36b4d771..61591017 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -43,6 +43,7 @@ DS_PATH = "cloudinit.sources.DataSourceOpenNebula"
 
 class TestOpenNebulaDataSource(CiTestCase):
     parsed_user = None
+    allowed_subp = ['bash']
 
     def setUp(self):
         super(TestOpenNebulaDataSource, self).setUp()
diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py
index fc4eb36e..9d52eb99 100644
--- a/tests/unittests/test_datasource/test_ovf.py
+++ b/tests/unittests/test_datasource/test_ovf.py
@@ -124,7 +124,9 @@ class TestDatasourceOVF(CiTestCase):
         ds = self.datasource(sys_cfg={}, distro={}, paths=paths)
         retcode = wrap_and_call(
             'cloudinit.sources.DataSourceOVF',
-            {'util.read_dmi_data': None},
+            {'util.read_dmi_data': None,
+             'transport_iso9660': (False, None, None),
+             'transport_vmware_guestd': (False, None, None)},
             ds.get_data)
         self.assertFalse(retcode, 'Expected False return from ds.get_data')
         self.assertIn(
@@ -138,7 +140,9 @@ class TestDatasourceOVF(CiTestCase):
             paths=paths)
         retcode = wrap_and_call(
             'cloudinit.sources.DataSourceOVF',
-            {'util.read_dmi_data': 'vmware'},
+            {'util.read_dmi_data': 'vmware',
+             'transport_iso9660': (False, None, None),
+             'transport_vmware_guestd': (False, None, None)},
             ds.get_data)
         self.assertFalse(retcode, 'Expected False return from ds.get_data')
         self.assertIn(
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index dca0b3d4..46d67b94 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -20,10 +20,8 @@ import multiprocessing
 import os
 import os.path
 import re
-import shutil
 import signal
 import stat
-import tempfile
 import unittest2
 import uuid
 
@@ -31,15 +29,27 @@ from cloudinit import serial
 from cloudinit.sources import DataSourceSmartOS
 from cloudinit.sources.DataSourceSmartOS import (
     convert_smartos_network_data as convert_net,
-    SMARTOS_ENV_KVM, SERIAL_DEVICE, get_smartos_environ)
+    SMARTOS_ENV_KVM, SERIAL_DEVICE, get_smartos_environ,
+    identify_file)
 
 import six
 
 from cloudinit import helpers as c_helpers
-from cloudinit.util import (b64e, subp)
+from cloudinit.util import (
+    b64e, subp, ProcessExecutionError, which, write_file)
 
-from cloudinit.tests.helpers import mock, FilesystemMockingTestCase, TestCase
+from cloudinit.tests.helpers import (
+    CiTestCase, mock, FilesystemMockingTestCase, skipIf)
 
+
+try:
+    import serial as _pyserial
+    assert _pyserial  # avoid pyflakes error F401: import unused
+    HAS_PYSERIAL = True
+except ImportError:
+    HAS_PYSERIAL = False
+
+DSMOS = 'cloudinit.sources.DataSourceSmartOS'
 SDC_NICS = json.loads("""
 [
     {
@@ -366,37 +376,33 @@ class PsuedoJoyentClient(object):
 
 
 class TestSmartOSDataSource(FilesystemMockingTestCase):
+    jmc_cfact = None
+    get_smartos_environ = None
+
     def setUp(self):
         super(TestSmartOSDataSource, self).setUp()
 
-        dsmos = 'cloudinit.sources.DataSourceSmartOS'
-        patcher = mock.patch(dsmos + ".jmc_client_factory")
-        self.jmc_cfact = patcher.start()
-        self.addCleanup(patcher.stop)
-        patcher = mock.patch(dsmos + ".get_smartos_environ")
-        self.get_smartos_environ = patcher.start()
-        self.addCleanup(patcher.stop)
-
-        self.tmp = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.tmp)
-        self.paths = c_helpers.Paths(
-            {'cloud_dir': self.tmp, 'run_dir': self.tmp})
-
-        self.legacy_user_d = os.path.join(self.tmp, 'legacy_user_tmp')
+        self.add_patch(DSMOS + ".get_smartos_environ", "get_smartos_environ")
+        self.add_patch(DSMOS + ".jmc_client_factory", "jmc_cfact")
+        self.legacy_user_d = self.tmp_path('legacy_user_tmp')
         os.mkdir(self.legacy_user_d)
-
-        self.orig_lud = DataSourceSmartOS.LEGACY_USER_D
-        DataSourceSmartOS.LEGACY_USER_D = self.legacy_user_d
-
-    def tearDown(self):
-        DataSourceSmartOS.LEGACY_USER_D = self.orig_lud
-        super(TestSmartOSDataSource, self).tearDown()
+        self.add_patch(DSMOS + ".LEGACY_USER_D", "m_legacy_user_d",
+                       autospec=False, new=self.legacy_user_d)
+        self.add_patch(DSMOS + ".identify_file", "m_identify_file",
+                       return_value="text/plain")
 
     def _get_ds(self, mockdata=None, mode=DataSourceSmartOS.SMARTOS_ENV_KVM,
                 sys_cfg=None, ds_cfg=None):
         self.jmc_cfact.return_value = PsuedoJoyentClient(mockdata)
         self.get_smartos_environ.return_value = mode
 
+        tmpd = self.tmp_dir()
+        dirs = {'cloud_dir': self.tmp_path('cloud_dir', tmpd),
+                'run_dir': self.tmp_path('run_dir')}
+        for d in dirs.values():
+            os.mkdir(d)
+        paths = c_helpers.Paths(dirs)
+
         if sys_cfg is None:
             sys_cfg = {}
 
@@ -405,7 +411,7 @@ class TestSmartOSDataSource(FilesystemMockingTestCase):
             sys_cfg['datasource']['SmartOS'] = ds_cfg
 
         return DataSourceSmartOS.DataSourceSmartOS(
-            sys_cfg, distro=None, paths=self.paths)
+            sys_cfg, distro=None, paths=paths)
 
     def test_no_base64(self):
         ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True}
@@ -493,6 +499,7 @@ class TestSmartOSDataSource(FilesystemMockingTestCase):
                          dsrc.metadata['user-script'])
 
         legacy_script_f = "%s/user-script" % self.legacy_user_d
+        print("legacy_script_f=%s" % legacy_script_f)
         self.assertTrue(os.path.exists(legacy_script_f))
         self.assertTrue(os.path.islink(legacy_script_f))
         user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:]
@@ -640,6 +647,28 @@ class TestSmartOSDataSource(FilesystemMockingTestCase):
                          mydscfg['disk_aliases']['FOO'])
 
 
+class TestIdentifyFile(CiTestCase):
+    """Test the 'identify_file' utility."""
+    @skipIf(not which("file"), "command 'file' not available.")
+    def test_file_happy_path(self):
+        """Test file is available and functional on plain text."""
+        fname = self.tmp_path("myfile")
+        write_file(fname, "plain text content here\n")
+        with self.allow_subp(["file"]):
+            self.assertEqual("text/plain", identify_file(fname))
+
+    @mock.patch(DSMOS + ".util.subp")
+    def test_returns_none_on_error(self, m_subp):
+        """On 'file' execution error, None should be returned."""
+        m_subp.side_effect = ProcessExecutionError("FILE_FAILED", exit_code=99)
+        fname = self.tmp_path("myfile")
+        write_file(fname, "plain text content here\n")
+        self.assertEqual(None, identify_file(fname))
+        self.assertEqual(
+            [mock.call(["file", "--brief", "--mime-type", fname])],
+            m_subp.call_args_list)
+
+
 class ShortReader(object):
     """Implements a 'read' interface for bytes provided.
     much like io.BytesIO but the 'endbyte' acts as if EOF.
@@ -893,7 +922,7 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase):
         self.assertEqual(client.list(), [])
 
 
-class TestNetworkConversion(TestCase):
+class TestNetworkConversion(CiTestCase):
     def test_convert_simple(self):
         expected = {
             'version': 1,
@@ -1058,7 +1087,8 @@ class TestNetworkConversion(TestCase):
                       "Only supported on KVM and bhyve guests under SmartOS")
 @unittest2.skipUnless(os.access(SERIAL_DEVICE, os.W_OK),
                       "Requires write access to " + SERIAL_DEVICE)
-class TestSerialConcurrency(TestCase):
+@unittest2.skipUnless(HAS_PYSERIAL is True, "pyserial not available")
+class TestSerialConcurrency(CiTestCase):
     """
        This class tests locking on an actual serial port, and as such can only
        be run in a kvm or bhyve guest running on a SmartOS host.  A test run on
@@ -1066,7 +1096,11 @@ class TestSerialConcurrency(TestCase):
        there is only one session over a connection.  In contrast, in the
        absence of proper locking multiple processes opening the same serial
        port can corrupt each others' exchanges with the metadata server.
+
+       This takes on the order of 2 to 3 minutes to run.
     """
+    allowed_subp = ['mdata-get']
+
     def setUp(self):
         self.mdata_proc = multiprocessing.Process(target=self.start_mdata_loop)
         self.mdata_proc.start()
@@ -1097,7 +1131,7 @@ class TestSerialConcurrency(TestCase):
         keys = [tup[0] for tup in ds.SMARTOS_ATTRIB_MAP.values()]
         keys.extend(ds.SMARTOS_ATTRIB_JSON.values())
 
-        client = ds.jmc_client_factory()
+        client = ds.jmc_client_factory(smartos_type=SMARTOS_ENV_KVM)
         self.assertIsNotNone(client)
 
         # The behavior that we are testing for was observed mdata-get running
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index e08e7908..46778e95 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -89,6 +89,7 @@ CallReturn = namedtuple('CallReturn',
 
 class DsIdentifyBase(CiTestCase):
     dsid_path = os.path.realpath('tools/ds-identify')
+    allowed_subp = ['sh']
 
     def call(self, rootd=None, mocks=None, func="main", args=None, files=None,
              policy_dmi=DI_DEFAULT_POLICY,
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py
index 7a64c230..a81c67c7 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v3.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py
@@ -48,6 +48,10 @@ ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
 
 TARGET = None
 
+MOCK_LSB_RELEASE_DATA = {
+    'id': 'Ubuntu', 'description': 'Ubuntu 18.04.1 LTS',
+    'release': '18.04', 'codename': 'bionic'}
+
 
 class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
     """TestAptSourceConfig
@@ -64,6 +68,9 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
         self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list")
         self.join = os.path.join
         self.matcher = re.compile(ADD_APT_REPO_MATCH).search
+        self.add_patch(
+            'cloudinit.config.cc_apt_configure.util.lsb_release',
+            'm_lsb_release', return_value=MOCK_LSB_RELEASE_DATA.copy())
 
     @staticmethod
     def _add_apt_sources(*args, **kwargs):
@@ -76,7 +83,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
         Get the most basic default mrror and release info to be used in tests
         """
         params = {}
-        params['RELEASE'] = util.lsb_release()['codename']
+        params['RELEASE'] = MOCK_LSB_RELEASE_DATA['release']
         arch = 'amd64'
         params['MIRROR'] = cc_apt_configure.\
             get_default_mirrors(arch)["PRIMARY"]
@@ -464,7 +471,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
                              'uri':
                              'http://testsec.ubuntu.com/%s/' % component}]}
         post = ("%s_dists_%s-updates_InRelease" %
-                (component, util.lsb_release()['codename']))
+                (component, MOCK_LSB_RELEASE_DATA['codename']))
         fromfn = ("%s/%s_%s" % (pre, archive, post))
         tofn = ("%s/test.ubuntu.com_%s" % (pre, post))
 
diff --git a/tests/unittests/test_handler/test_handler_bootcmd.py b/tests/unittests/test_handler/test_handler_bootcmd.py
index b1375269..a76760fa 100644
--- a/tests/unittests/test_handler/test_handler_bootcmd.py
+++ b/tests/unittests/test_handler/test_handler_bootcmd.py
@@ -118,7 +118,8 @@ class TestBootcmd(CiTestCase):
             'echo {0} $INSTANCE_ID > {1}'.format(my_id, out_file)]}
 
         with mock.patch(self._etmpfile_path, FakeExtendedTempFile):
-            handle('cc_bootcmd', valid_config, cc, LOG, [])
+            with self.allow_subp(['/bin/sh']):
+                handle('cc_bootcmd', valid_config, cc, LOG, [])
         self.assertEqual(my_id + ' iid-datasource-none\n',
                          util.load_file(out_file))
 
@@ -128,12 +129,13 @@ class TestBootcmd(CiTestCase):
         valid_config = {'bootcmd': ['exit 1']}  # Script with error
 
         with mock.patch(self._etmpfile_path, FakeExtendedTempFile):
-            with self.assertRaises(util.ProcessExecutionError) as ctxt_manager:
-                handle('does-not-matter', valid_config, cc, LOG, [])
+            with self.allow_subp(['/bin/sh']):
+                with self.assertRaises(util.ProcessExecutionError) as ctxt:
+                    handle('does-not-matter', valid_config, cc, LOG, [])
         self.assertIn(
             'Unexpected error while running command.\n'
             "Command: ['/bin/sh',",
-            str(ctxt_manager.exception))
+            str(ctxt.exception))
         self.assertIn(
             'Failed to run bootcmd module does-not-matter',
             self.logs.getvalue())
diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py
index f4bbd66d..b16532ea 100644
--- a/tests/unittests/test_handler/test_handler_chef.py
+++ b/tests/unittests/test_handler/test_handler_chef.py
@@ -36,13 +36,21 @@ class TestInstallChefOmnibus(HttprettyTestCase):
 
     @mock.patch("cloudinit.config.cc_chef.OMNIBUS_URL", OMNIBUS_URL_HTTP)
     def test_install_chef_from_omnibus_runs_chef_url_content(self):
-        """install_chef_from_omnibus runs downloaded OMNIBUS_URL as script."""
-        chef_outfile = self.tmp_path('chef.out', self.new_root)
-        response = '#!/bin/bash\necho "Hi Mom" > {0}'.format(chef_outfile)
+        """install_chef_from_omnibus calls subp_blob_in_tempfile."""
+        response = b'#!/bin/bash\necho "Hi Mom"'
         httpretty.register_uri(
             httpretty.GET, cc_chef.OMNIBUS_URL, body=response, status=200)
-        cc_chef.install_chef_from_omnibus()
-        self.assertEqual('Hi Mom\n', util.load_file(chef_outfile))
+        ret = (None, None)  # stdout, stderr but capture=False
+
+        with mock.patch("cloudinit.config.cc_chef.util.subp_blob_in_tempfile",
+                        return_value=ret) as m_subp_blob:
+            cc_chef.install_chef_from_omnibus()
+        # admittedly whitebox, but assuming subp_blob_in_tempfile works
+        # this should be fine.
+        self.assertEqual(
+            [mock.call(blob=response, args=[], basename='chef-omnibus-install',
+                       capture=False)],
+            m_subp_blob.call_args_list)
 
     @mock.patch('cloudinit.config.cc_chef.url_helper.readurl')
     @mock.patch('cloudinit.config.cc_chef.util.subp_blob_in_tempfile')
diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py
index f92175fd..feca56c2 100644
--- a/tests/unittests/test_handler/test_handler_resizefs.py
+++ b/tests/unittests/test_handler/test_handler_resizefs.py
@@ -150,10 +150,12 @@ class TestResizefs(CiTestCase):
         self.assertEqual(('growfs', '-y', devpth),
                          _resize_ufs(mount_point, devpth))
 
+    @mock.patch('cloudinit.util.is_container', return_value=False)
     @mock.patch('cloudinit.util.get_mount_info')
     @mock.patch('cloudinit.util.get_device_info_from_zpool')
     @mock.patch('cloudinit.util.parse_mount')
-    def test_handle_zfs_root(self, mount_info, zpool_info, parse_mount):
+    def test_handle_zfs_root(self, mount_info, zpool_info, parse_mount,
+                             is_container):
         devpth = 'vmzroot/ROOT/freebsd'
         disk = 'gpt/system'
         fs_type = 'zfs'
@@ -354,8 +356,10 @@ class TestMaybeGetDevicePathAsWritableBlock(CiTestCase):
             ('btrfs', 'filesystem', 'resize', 'max', '/'),
             _resize_btrfs("/", "/dev/sda1"))
 
+    @mock.patch('cloudinit.util.is_container', return_value=True)
     @mock.patch('cloudinit.util.is_FreeBSD')
-    def test_maybe_get_writable_device_path_zfs_freebsd(self, freebsd):
+    def test_maybe_get_writable_device_path_zfs_freebsd(self, freebsd,
+                                                        m_is_container):
         freebsd.return_value = True
         info = 'dev=gpt/system mnt_point=/ path=/'
         devpth = maybe_get_writable_device_path('gpt/system', info, LOG)
diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
index fb266faf..1bad07f6 100644
--- a/tests/unittests/test_handler/test_schema.py
+++ b/tests/unittests/test_handler/test_schema.py
@@ -4,7 +4,7 @@ from cloudinit.config.schema import (
     CLOUD_CONFIG_HEADER, SchemaValidationError, annotated_cloudconfig_file,
     get_schema_doc, get_schema, validate_cloudconfig_file,
     validate_cloudconfig_schema, main)
-from cloudinit.util import subp, write_file
+from cloudinit.util import write_file
 
 from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJsonSchema
 
@@ -406,8 +406,14 @@ class CloudTestsIntegrationTest(CiTestCase):
         integration_testdir = os.path.sep.join(
             [testsdir, 'cloud_tests', 'testcases'])
         errors = []
-        out, _ = subp(['find', integration_testdir, '-name', '*yaml'])
-        for filename in out.splitlines():
+
+        yaml_files = []
+        for root, _dirnames, filenames in os.walk(integration_testdir):
+            yaml_files.extend([os.path.join(root, f)
+                               for f in filenames if f.endswith(".yaml")])
+        self.assertTrue(len(yaml_files) > 0)
+
+        for filename in yaml_files:
             test_cfg = safe_load(open(filename))
             cloud_config = test_cfg.get('cloud_config')
             if cloud_config:
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index 05d5c72c..f3165da9 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -1653,6 +1653,12 @@ def _setup_test(tmp_dir, mock_get_devicelist, mock_read_sys_net,
 
 class TestGenerateFallbackConfig(CiTestCase):
 
+    def setUp(self):
+        super(TestGenerateFallbackConfig, self).setUp()
+        self.add_patch(
+            "cloudinit.util.get_cmdline", "m_get_cmdline",
+            return_value="root=/dev/sda1")
+
     @mock.patch("cloudinit.net.sys_dev_path")
     @mock.patch("cloudinit.net.read_sys_net")
     @mock.patch("cloudinit.net.get_devicelist")
@@ -1895,12 +1901,13 @@ class TestRhelSysConfigRendering(CiTestCase):
         if missing:
             raise AssertionError("Missing headers in: %s" % missing)
 
+    @mock.patch("cloudinit.net.util.get_cmdline", return_value="root=myroot")
     @mock.patch("cloudinit.net.sys_dev_path")
     @mock.patch("cloudinit.net.read_sys_net")
     @mock.patch("cloudinit.net.get_devicelist")
     def test_default_generation(self, mock_get_devicelist,
                                 mock_read_sys_net,
-                                mock_sys_dev_path):
+                                mock_sys_dev_path, m_get_cmdline):
         tmp_dir = self.tmp_dir()
         _setup_test(tmp_dir, mock_get_devicelist,
                     mock_read_sys_net, mock_sys_dev_path)
@@ -2183,12 +2190,13 @@ class TestOpenSuseSysConfigRendering(CiTestCase):
         if missing:
             raise AssertionError("Missing headers in: %s" % missing)
 
+    @mock.patch("cloudinit.net.util.get_cmdline", return_value="root=myroot")
     @mock.patch("cloudinit.net.sys_dev_path")
     @mock.patch("cloudinit.net.read_sys_net")
     @mock.patch("cloudinit.net.get_devicelist")
     def test_default_generation(self, mock_get_devicelist,
                                 mock_read_sys_net,
-                                mock_sys_dev_path):
+                                mock_sys_dev_path, m_get_cmdline):
         tmp_dir = self.tmp_dir()
         _setup_test(tmp_dir, mock_get_devicelist,
                     mock_read_sys_net, mock_sys_dev_path)
@@ -2429,12 +2437,13 @@ USERCTL=no
 
 class TestEniNetRendering(CiTestCase):
 
+    @mock.patch("cloudinit.net.util.get_cmdline", return_value="root=myroot")
     @mock.patch("cloudinit.net.sys_dev_path")
     @mock.patch("cloudinit.net.read_sys_net")
     @mock.patch("cloudinit.net.get_devicelist")
     def test_default_generation(self, mock_get_devicelist,
                                 mock_read_sys_net,
-                                mock_sys_dev_path):
+                                mock_sys_dev_path, m_get_cmdline):
         tmp_dir = self.tmp_dir()
         _setup_test(tmp_dir, mock_get_devicelist,
                     mock_read_sys_net, mock_sys_dev_path)
@@ -2482,6 +2491,7 @@ iface eth0 inet dhcp
 
 class TestNetplanNetRendering(CiTestCase):
 
+    @mock.patch("cloudinit.net.util.get_cmdline", return_value="root=myroot")
     @mock.patch("cloudinit.net.netplan._clean_default")
     @mock.patch("cloudinit.net.sys_dev_path")
     @mock.patch("cloudinit.net.read_sys_net")
@@ -2489,7 +2499,7 @@ class TestNetplanNetRendering(CiTestCase):
     def test_default_generation(self, mock_get_devicelist,
                                 mock_read_sys_net,
                                 mock_sys_dev_path,
-                                mock_clean_default):
+                                mock_clean_default, m_get_cmdline):
         tmp_dir = self.tmp_dir()
         _setup_test(tmp_dir, mock_get_devicelist,
                     mock_read_sys_net, mock_sys_dev_path)
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index 7a203ce2..5a14479a 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -24,6 +24,7 @@ except ImportError:
 
 
 BASH = util.which('bash')
+BOGUS_COMMAND = 'this-is-not-expected-to-be-a-program-name'
 
 
 class FakeSelinux(object):
@@ -742,6 +743,8 @@ class TestReadSeeded(helpers.TestCase):
 
 class TestSubp(helpers.CiTestCase):
     with_logs = True
+    allowed_subp = [BASH, 'cat', helpers.CiTestCase.SUBP_SHELL_TRUE,
+                    BOGUS_COMMAND, sys.executable]
 
     stdin2err = [BASH, '-c', 'cat >&2']
     stdin2out = ['cat']
@@ -749,7 +752,6 @@ class TestSubp(helpers.CiTestCase):
     utf8_valid = b'start \xc3\xa9 end'
     utf8_valid_2 = b'd\xc3\xa9j\xc8\xa7'
     printenv = [BASH, '-c', 'for n in "$@"; do echo "$n=${!n}"; done', '--']
-    bogus_command = 'this-is-not-expected-to-be-a-program-name'
 
     def printf_cmd(self, *args):
         # bash's printf supports \xaa.  So does /usr/bin/printf
@@ -848,9 +850,10 @@ class TestSubp(helpers.CiTestCase):
         util.write_file(noshebang, 'true\n')
 
         os.chmod(noshebang, os.stat(noshebang).st_mode | stat.S_IEXEC)
-        self.assertRaisesRegex(util.ProcessExecutionError,
-                               r'Missing #! in script\?',
-                               util.subp, (noshebang,))
+        with self.allow_subp([noshebang]):
+            self.assertRaisesRegex(util.ProcessExecutionError,
+                                   r'Missing #! in script\?',
+                                   util.subp, (noshebang,))
 
     def test_subp_combined_stderr_stdout(self):
         """Providing combine_capture as True redirects stderr to stdout."""
@@ -868,14 +871,14 @@ class TestSubp(helpers.CiTestCase):
     def test_exception_has_out_err_are_bytes_if_decode_false(self):
         """Raised exc should have stderr, stdout as bytes if no decode."""
         with self.assertRaises(util.ProcessExecutionError) as cm:
-            util.subp([self.bogus_command], decode=False)
+            util.subp([BOGUS_COMMAND], decode=False)
         self.assertTrue(isinstance(cm.exception.stdout, bytes))
         self.assertTrue(isinstance(cm.exception.stderr, bytes))
 
     def test_exception_has_out_err_are_bytes_if_decode_true(self):
         """Raised exc should have stderr, stdout as string if no decode."""
         with self.assertRaises(util.ProcessExecutionError) as cm:
-            util.subp([self.bogus_command], decode=True)
+            util.subp([BOGUS_COMMAND], decode=True)
         self.assertTrue(isinstance(cm.exception.stdout, six.string_types))
         self.assertTrue(isinstance(cm.exception.stderr, six.string_types))
 
@@ -925,10 +928,10 @@ class TestSubp(helpers.CiTestCase):
             logs.append(log)
 
         with self.assertRaises(util.ProcessExecutionError):
-            util.subp([self.bogus_command], status_cb=status_cb)
+            util.subp([BOGUS_COMMAND], status_cb=status_cb)
 
         expected = [
-            'Begin run command: {cmd}\n'.format(cmd=self.bogus_command),
+            'Begin run command: {cmd}\n'.format(cmd=BOGUS_COMMAND),
             'ERROR: End run command: invalid command provided\n']
         self.assertEqual(expected, logs)
 
@@ -940,13 +943,13 @@ class TestSubp(helpers.CiTestCase):
             logs.append(log)
 
         with self.assertRaises(util.ProcessExecutionError):
-            util.subp(['ls', '/I/dont/exist'], status_cb=status_cb)
-        util.subp(['ls'], status_cb=status_cb)
+            util.subp([BASH, '-c', 'exit 2'], status_cb=status_cb)
+        util.subp([BASH, '-c', 'exit 0'], status_cb=status_cb)
 
         expected = [
-            'Begin run command: ls /I/dont/exist\n',
+            'Begin run command: %s -c exit 2\n' % BASH,
             'ERROR: End run command: exit(2)\n',
-            'Begin run command: ls\n',
+            'Begin run command: %s -c exit 0\n' % BASH,
             'End run command: exit(0)\n']
         self.assertEqual(expected, logs)
 
-- 
cgit v1.2.3


From c7555762f3a30190ce7726b4d013bc3e83c7e4b6 Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Tue, 11 Sep 2018 17:31:46 +0000
Subject: user-data: jinja template to render instance-data.json in
 cloud-config

Allow users to provide '## template: jinja' as the first line or their
#cloud-config or custom script user-data parts. When this header exists,
the cloud-config or script will be rendered as a jinja template.

All instance metadata keys and values present in
/run/cloud-init/instance-data.json will be available as jinja variables
for the template. This means any cloud-config module or script can
reference any standardized instance data in templates and scripts.

Additionally, any standardized instance-data.json keys scoped below a
'<v#>' key will be promoted as a top-level key for ease of reference in
templates. This means that '{{ local_hostname }}' is the same as using the
latest '{{ v#.local_hostname }}'.

Since instance-data is written to /run/cloud-init/instance-data.json, make
sure it is persisted across reboots when the cached datasource opject is
reloaded.

LP: #1791781
---
 bash_completion/cloud-init                         |   2 +
 cloudinit/cmd/devel/__init__.py                    |  25 ++
 cloudinit/cmd/devel/parser.py                      |   5 +-
 cloudinit/cmd/devel/render.py                      |  90 ++++++
 cloudinit/cmd/devel/tests/test_render.py           | 101 +++++++
 cloudinit/cmd/main.py                              |  16 +-
 cloudinit/handlers/__init__.py                     |  11 +-
 cloudinit/handlers/boot_hook.py                    |  12 +-
 cloudinit/handlers/cloud_config.py                 |  15 +-
 cloudinit/handlers/jinja_template.py               | 137 +++++++++
 cloudinit/handlers/shell_script.py                 |   9 +-
 cloudinit/handlers/upstart_job.py                  |   9 +-
 cloudinit/helpers.py                               |   4 +
 cloudinit/log.py                                   |  12 +-
 cloudinit/sources/__init__.py                      |  47 ++-
 cloudinit/sources/tests/test_init.py               |  75 ++++-
 cloudinit/stages.py                                |  22 +-
 cloudinit/templater.py                             |  28 +-
 cloudinit/tests/helpers.py                         |   9 +
 doc/rtd/topics/capabilities.rst                    |  15 +-
 doc/rtd/topics/datasources.rst                     |  47 +++
 doc/rtd/topics/format.rst                          |  21 +-
 tests/cloud_tests/testcases/base.py                |   8 +-
 tests/unittests/test_builtin_handlers.py           | 324 +++++++++++++++++++--
 .../test_handler/test_handler_etc_hosts.py         |   1 +
 tests/unittests/test_handler/test_handler_ntp.py   |   1 +
 tests/unittests/test_templating.py                 |  23 ++
 27 files changed, 959 insertions(+), 110 deletions(-)
 create mode 100755 cloudinit/cmd/devel/render.py
 create mode 100644 cloudinit/cmd/devel/tests/test_render.py
 create mode 100644 cloudinit/handlers/jinja_template.py

(limited to 'cloudinit/sources')

diff --git a/bash_completion/cloud-init b/bash_completion/cloud-init
index f38164b0..b3a5ced3 100644
--- a/bash_completion/cloud-init
+++ b/bash_completion/cloud-init
@@ -62,6 +62,8 @@ _cloudinit_complete()
                 net-convert)
                     COMPREPLY=($(compgen -W "--help --network-data --kind --directory --output-kind" -- $cur_word))
                     ;;
+                render)
+                    COMPREPLY=($(compgen -W "--help --instance-data --debug" -- $cur_word))
                 schema)
                     COMPREPLY=($(compgen -W "--help --config-file --doc --annotate" -- $cur_word))
                     ;;
diff --git a/cloudinit/cmd/devel/__init__.py b/cloudinit/cmd/devel/__init__.py
index e69de29b..3ae28b69 100644
--- a/cloudinit/cmd/devel/__init__.py
+++ b/cloudinit/cmd/devel/__init__.py
@@ -0,0 +1,25 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Common cloud-init devel commandline utility functions."""
+
+
+import logging
+
+from cloudinit import log
+from cloudinit.stages import Init
+
+
+def addLogHandlerCLI(logger, log_level):
+    """Add a commandline logging handler to emit messages to stderr."""
+    formatter = logging.Formatter('%(levelname)s: %(message)s')
+    log.setupBasicLogging(log_level, formatter=formatter)
+    return logger
+
+
+def read_cfg_paths():
+    """Return a Paths object based on the system configuration on disk."""
+    init = Init(ds_deps=[])
+    init.read_cfg()
+    return init.paths
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/devel/parser.py b/cloudinit/cmd/devel/parser.py
index 40a4b019..99a234ce 100644
--- a/cloudinit/cmd/devel/parser.py
+++ b/cloudinit/cmd/devel/parser.py
@@ -8,6 +8,7 @@ import argparse
 from cloudinit.config import schema
 
 from . import net_convert
+from . import render
 
 
 def get_parser(parser=None):
@@ -22,7 +23,9 @@ def get_parser(parser=None):
         ('schema', 'Validate cloud-config files for document schema',
          schema.get_parser, schema.handle_schema_args),
         (net_convert.NAME, net_convert.__doc__,
-         net_convert.get_parser, net_convert.handle_args)
+         net_convert.get_parser, net_convert.handle_args),
+        (render.NAME, render.__doc__,
+         render.get_parser, render.handle_args)
     ]
     for (subcmd, helpmsg, get_parser, handler) in subcmds:
         parser = subparsers.add_parser(subcmd, help=helpmsg)
diff --git a/cloudinit/cmd/devel/render.py b/cloudinit/cmd/devel/render.py
new file mode 100755
index 00000000..e85933db
--- /dev/null
+++ b/cloudinit/cmd/devel/render.py
@@ -0,0 +1,90 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Debug jinja template rendering of user-data."""
+
+import argparse
+import os
+import sys
+
+from cloudinit.handlers.jinja_template import render_jinja_payload_from_file
+from cloudinit import log
+from cloudinit.sources import INSTANCE_JSON_FILE
+from cloudinit import util
+from . import addLogHandlerCLI, read_cfg_paths
+
+NAME = 'render'
+DEFAULT_INSTANCE_DATA = '/run/cloud-init/instance-data.json'
+
+LOG = log.getLogger(NAME)
+
+
+def get_parser(parser=None):
+    """Build or extend and arg parser for jinja render utility.
+
+    @param parser: Optional existing ArgumentParser instance representing the
+        subcommand which will be extended to support the args of this utility.
+
+    @returns: ArgumentParser with proper argument configuration.
+    """
+    if not parser:
+        parser = argparse.ArgumentParser(prog=NAME, description=__doc__)
+    parser.add_argument(
+        'user_data', type=str, help='Path to the user-data file to render')
+    parser.add_argument(
+        '-i', '--instance-data', type=str,
+        help=('Optional path to instance-data.json file. Defaults to'
+              ' /run/cloud-init/instance-data.json'))
+    parser.add_argument('-d', '--debug', action='store_true', default=False,
+                        help='Add verbose messages during template render')
+    return parser
+
+
+def handle_args(name, args):
+    """Render the provided user-data template file using instance-data values.
+
+    Also setup CLI log handlers to report to stderr since this is a development
+    utility which should be run by a human on the CLI.
+
+    @return 0 on success, 1 on failure.
+    """
+    addLogHandlerCLI(LOG, log.DEBUG if args.debug else log.WARNING)
+    if not args.instance_data:
+        paths = read_cfg_paths()
+        instance_data_fn = os.path.join(
+            paths.run_dir, INSTANCE_JSON_FILE)
+    else:
+        instance_data_fn = args.instance_data
+    try:
+        with open(instance_data_fn) as stream:
+            instance_data = stream.read()
+        instance_data = util.load_json(instance_data)
+    except IOError:
+        LOG.error('Missing instance-data.json file: %s', instance_data_fn)
+        return 1
+    try:
+        with open(args.user_data) as stream:
+            user_data = stream.read()
+    except IOError:
+        LOG.error('Missing user-data file: %s', args.user_data)
+        return 1
+    rendered_payload = render_jinja_payload_from_file(
+        payload=user_data, payload_fn=args.user_data,
+        instance_data_file=instance_data_fn,
+        debug=True if args.debug else False)
+    if not rendered_payload:
+        LOG.error('Unable to render user-data file: %s', args.user_data)
+        return 1
+    sys.stdout.write(rendered_payload)
+    return 0
+
+
+def main():
+    args = get_parser().parse_args()
+    return(handle_args(NAME, args))
+
+
+if __name__ == '__main__':
+    sys.exit(main())
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/devel/tests/test_render.py b/cloudinit/cmd/devel/tests/test_render.py
new file mode 100644
index 00000000..fc5d2c0d
--- /dev/null
+++ b/cloudinit/cmd/devel/tests/test_render.py
@@ -0,0 +1,101 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from six import StringIO
+import os
+
+from collections import namedtuple
+from cloudinit.cmd.devel import render
+from cloudinit.helpers import Paths
+from cloudinit.sources import INSTANCE_JSON_FILE
+from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJinja
+from cloudinit.util import ensure_dir, write_file
+
+
+class TestRender(CiTestCase):
+
+    with_logs = True
+
+    args = namedtuple('renderargs', 'user_data instance_data debug')
+
+    def setUp(self):
+        super(TestRender, self).setUp()
+        self.tmp = self.tmp_dir()
+
+    def test_handle_args_error_on_missing_user_data(self):
+        """When user_data file path does not exist, log an error."""
+        absent_file = self.tmp_path('user-data', dir=self.tmp)
+        instance_data = self.tmp_path('instance-data', dir=self.tmp)
+        write_file(instance_data, '{}')
+        args = self.args(
+            user_data=absent_file, instance_data=instance_data, debug=False)
+        with mock.patch('sys.stderr', new_callable=StringIO):
+            self.assertEqual(1, render.handle_args('anyname', args))
+        self.assertIn(
+            'Missing user-data file: %s' % absent_file,
+            self.logs.getvalue())
+
+    def test_handle_args_error_on_missing_instance_data(self):
+        """When instance_data file path does not exist, log an error."""
+        user_data = self.tmp_path('user-data', dir=self.tmp)
+        absent_file = self.tmp_path('instance-data', dir=self.tmp)
+        args = self.args(
+            user_data=user_data, instance_data=absent_file, debug=False)
+        with mock.patch('sys.stderr', new_callable=StringIO):
+            self.assertEqual(1, render.handle_args('anyname', args))
+        self.assertIn(
+            'Missing instance-data.json file: %s' % absent_file,
+            self.logs.getvalue())
+
+    def test_handle_args_defaults_instance_data(self):
+        """When no instance_data argument, default to configured run_dir."""
+        user_data = self.tmp_path('user-data', dir=self.tmp)
+        run_dir = self.tmp_path('run_dir', dir=self.tmp)
+        ensure_dir(run_dir)
+        paths = Paths({'run_dir': run_dir})
+        self.add_patch('cloudinit.cmd.devel.render.read_cfg_paths', 'm_paths')
+        self.m_paths.return_value = paths
+        args = self.args(
+            user_data=user_data, instance_data=None, debug=False)
+        with mock.patch('sys.stderr', new_callable=StringIO):
+            self.assertEqual(1, render.handle_args('anyname', args))
+        json_file = os.path.join(run_dir, INSTANCE_JSON_FILE)
+        self.assertIn(
+            'Missing instance-data.json file: %s' % json_file,
+            self.logs.getvalue())
+
+    @skipUnlessJinja()
+    def test_handle_args_renders_instance_data_vars_in_template(self):
+        """If user_data file is a jinja template render instance-data vars."""
+        user_data = self.tmp_path('user-data', dir=self.tmp)
+        write_file(user_data, '##template: jinja\nrendering: {{ my_var }}')
+        instance_data = self.tmp_path('instance-data', dir=self.tmp)
+        write_file(instance_data, '{"my-var": "jinja worked"}')
+        args = self.args(
+            user_data=user_data, instance_data=instance_data, debug=True)
+        with mock.patch('sys.stderr', new_callable=StringIO) as m_console_err:
+            with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+                self.assertEqual(0, render.handle_args('anyname', args))
+        self.assertIn(
+            'DEBUG: Converted jinja variables\n{', self.logs.getvalue())
+        self.assertIn(
+            'DEBUG: Converted jinja variables\n{', m_console_err.getvalue())
+        self.assertEqual('rendering: jinja worked', m_stdout.getvalue())
+
+    @skipUnlessJinja()
+    def test_handle_args_warns_and_gives_up_on_invalid_jinja_operation(self):
+        """If user_data file has invalid jinja operations log warnings."""
+        user_data = self.tmp_path('user-data', dir=self.tmp)
+        write_file(user_data, '##template: jinja\nrendering: {{ my-var }}')
+        instance_data = self.tmp_path('instance-data', dir=self.tmp)
+        write_file(instance_data, '{"my-var": "jinja worked"}')
+        args = self.args(
+            user_data=user_data, instance_data=instance_data, debug=True)
+        with mock.patch('sys.stderr', new_callable=StringIO):
+            self.assertEqual(1, render.handle_args('anyname', args))
+        self.assertIn(
+            'WARNING: Ignoring jinja template for %s: Undefined jinja'
+            ' variable: "my-var". Jinja tried subtraction. Perhaps you meant'
+            ' "my_var"?' % user_data,
+            self.logs.getvalue())
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index 4ea4fe7f..0eee583c 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -348,6 +348,7 @@ def main_init(name, args):
             LOG.debug("[%s] barreling on in force mode without datasource",
                       mode)
 
+    _maybe_persist_instance_data(init)
     # Stage 6
     iid = init.instancify()
     LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s",
@@ -490,6 +491,7 @@ def main_modules(action_name, args):
         print_exc(msg)
         if not args.force:
             return [(msg)]
+    _maybe_persist_instance_data(init)
     # Stage 3
     mods = stages.Modules(init, extract_fns(args), reporter=args.reporter)
     # Stage 4
@@ -541,6 +543,7 @@ def main_single(name, args):
                    " likely bad things to come!"))
         if not args.force:
             return 1
+    _maybe_persist_instance_data(init)
     # Stage 3
     mods = stages.Modules(init, extract_fns(args), reporter=args.reporter)
     mod_args = args.module_args
@@ -688,6 +691,15 @@ def status_wrapper(name, args, data_d=None, link_d=None):
     return len(v1[mode]['errors'])
 
 
+def _maybe_persist_instance_data(init):
+    """Write instance-data.json file if absent and datasource is restored."""
+    if init.ds_restored:
+        instance_data_file = os.path.join(
+            init.paths.run_dir, sources.INSTANCE_JSON_FILE)
+        if not os.path.exists(instance_data_file):
+            init.datasource.persist_instance_data()
+
+
 def _maybe_set_hostname(init, stage, retry_stage):
     """Call set-hostname if metadata, vendordata or userdata provides it.
 
@@ -887,6 +899,8 @@ def main(sysv_args=None):
 if __name__ == '__main__':
     if 'TZ' not in os.environ:
         os.environ['TZ'] = ":/etc/localtime"
-    main(sys.argv)
+    return_value = main(sys.argv)
+    if return_value:
+        sys.exit(return_value)
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py
index c3576c04..0db75af9 100644
--- a/cloudinit/handlers/__init__.py
+++ b/cloudinit/handlers/__init__.py
@@ -41,7 +41,7 @@ PART_HANDLER_FN_TMPL = 'part-handler-%03d'
 # For parts without filenames
 PART_FN_TPL = 'part-%03d'
 
-# Different file beginnings to there content type
+# Different file beginnings to their content type
 INCLUSION_TYPES_MAP = {
     '#include': 'text/x-include-url',
     '#include-once': 'text/x-include-once-url',
@@ -52,6 +52,7 @@ INCLUSION_TYPES_MAP = {
     '#cloud-boothook': 'text/cloud-boothook',
     '#cloud-config-archive': 'text/cloud-config-archive',
     '#cloud-config-jsonp': 'text/cloud-config-jsonp',
+    '## template: jinja': 'text/jinja2',
 }
 
 # Sorted longest first
@@ -69,9 +70,13 @@ class Handler(object):
     def __repr__(self):
         return "%s: [%s]" % (type_utils.obj_name(self), self.list_types())
 
-    @abc.abstractmethod
     def list_types(self):
-        raise NotImplementedError()
+        # Each subclass must define the supported content prefixes it handles.
+        if not hasattr(self, 'prefixes'):
+            raise NotImplementedError('Missing prefixes subclass attribute')
+        else:
+            return [INCLUSION_TYPES_MAP[prefix]
+                    for prefix in getattr(self, 'prefixes')]
 
     @abc.abstractmethod
     def handle_part(self, *args, **kwargs):
diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py
index 057b4dbc..dca50a49 100644
--- a/cloudinit/handlers/boot_hook.py
+++ b/cloudinit/handlers/boot_hook.py
@@ -17,10 +17,13 @@ from cloudinit import util
 from cloudinit.settings import (PER_ALWAYS)
 
 LOG = logging.getLogger(__name__)
-BOOTHOOK_PREFIX = "#cloud-boothook"
 
 
 class BootHookPartHandler(handlers.Handler):
+
+    # The content prefixes this handler understands.
+    prefixes = ['#cloud-boothook']
+
     def __init__(self, paths, datasource, **_kwargs):
         handlers.Handler.__init__(self, PER_ALWAYS)
         self.boothook_dir = paths.get_ipath("boothooks")
@@ -28,16 +31,11 @@ class BootHookPartHandler(handlers.Handler):
         if datasource:
             self.instance_id = datasource.get_instance_id()
 
-    def list_types(self):
-        return [
-            handlers.type_from_starts_with(BOOTHOOK_PREFIX),
-        ]
-
     def _write_part(self, payload, filename):
         filename = util.clean_filename(filename)
         filepath = os.path.join(self.boothook_dir, filename)
         contents = util.strip_prefix_suffix(util.dos2unix(payload),
-                                            prefix=BOOTHOOK_PREFIX)
+                                            prefix=self.prefixes[0])
         util.write_file(filepath, contents.lstrip(), 0o700)
         return filepath
 
diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py
index 178a5b9b..99bf0e61 100644
--- a/cloudinit/handlers/cloud_config.py
+++ b/cloudinit/handlers/cloud_config.py
@@ -42,14 +42,12 @@ DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()')
 CLOUD_PREFIX = "#cloud-config"
 JSONP_PREFIX = "#cloud-config-jsonp"
 
-# The file header -> content types this module will handle.
-CC_TYPES = {
-    JSONP_PREFIX: handlers.type_from_starts_with(JSONP_PREFIX),
-    CLOUD_PREFIX: handlers.type_from_starts_with(CLOUD_PREFIX),
-}
-
 
 class CloudConfigPartHandler(handlers.Handler):
+
+    # The content prefixes this handler understands.
+    prefixes = [CLOUD_PREFIX, JSONP_PREFIX]
+
     def __init__(self, paths, **_kwargs):
         handlers.Handler.__init__(self, PER_ALWAYS, version=3)
         self.cloud_buf = None
@@ -58,9 +56,6 @@ class CloudConfigPartHandler(handlers.Handler):
             self.cloud_fn = paths.get_ipath(_kwargs["cloud_config_path"])
         self.file_names = []
 
-    def list_types(self):
-        return list(CC_TYPES.values())
-
     def _write_cloud_config(self):
         if not self.cloud_fn:
             return
@@ -138,7 +133,7 @@ class CloudConfigPartHandler(handlers.Handler):
             # First time through, merge with an empty dict...
             if self.cloud_buf is None or not self.file_names:
                 self.cloud_buf = {}
-            if ctype == CC_TYPES[JSONP_PREFIX]:
+            if ctype == handlers.INCLUSION_TYPES_MAP[JSONP_PREFIX]:
                 self._merge_patch(payload)
             else:
                 self._merge_part(payload, headers)
diff --git a/cloudinit/handlers/jinja_template.py b/cloudinit/handlers/jinja_template.py
new file mode 100644
index 00000000..3fa4097e
--- /dev/null
+++ b/cloudinit/handlers/jinja_template.py
@@ -0,0 +1,137 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import os
+import re
+
+try:
+    from jinja2.exceptions import UndefinedError as JUndefinedError
+except ImportError:
+    # No jinja2 dependency
+    JUndefinedError = Exception
+
+from cloudinit import handlers
+from cloudinit import log as logging
+from cloudinit.sources import INSTANCE_JSON_FILE
+from cloudinit.templater import render_string, MISSING_JINJA_PREFIX
+from cloudinit.util import b64d, load_file, load_json, json_dumps
+
+from cloudinit.settings import PER_ALWAYS
+
+LOG = logging.getLogger(__name__)
+
+
+class JinjaTemplatePartHandler(handlers.Handler):
+
+    prefixes = ['## template: jinja']
+
+    def __init__(self, paths, **_kwargs):
+        handlers.Handler.__init__(self, PER_ALWAYS, version=3)
+        self.paths = paths
+        self.sub_handlers = {}
+        for handler in _kwargs.get('sub_handlers', []):
+            for ctype in handler.list_types():
+                self.sub_handlers[ctype] = handler
+
+    def handle_part(self, data, ctype, filename, payload, frequency, headers):
+        if ctype in handlers.CONTENT_SIGNALS:
+            return
+        jinja_json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE)
+        rendered_payload = render_jinja_payload_from_file(
+            payload, filename, jinja_json_file)
+        if not rendered_payload:
+            return
+        subtype = handlers.type_from_starts_with(rendered_payload)
+        sub_handler = self.sub_handlers.get(subtype)
+        if not sub_handler:
+            LOG.warning(
+                'Ignoring jinja template for %s. Could not find supported'
+                ' sub-handler for type %s', filename, subtype)
+            return
+        if sub_handler.handler_version == 3:
+            sub_handler.handle_part(
+                data, ctype, filename, rendered_payload, frequency, headers)
+        elif sub_handler.handler_version == 2:
+            sub_handler.handle_part(
+                data, ctype, filename, rendered_payload, frequency)
+
+
+def render_jinja_payload_from_file(
+        payload, payload_fn, instance_data_file, debug=False):
+    """Render a jinja template payload sourcing variables from jinja_vars_path.
+
+    @param payload: String of jinja template content. Should begin with
+        ## template: jinja\n.
+    @param payload_fn: String representing the filename from which the payload
+        was read used in error reporting. Generally in part-handling this is
+        'part-##'.
+    @param instance_data_file: A path to a json file containing variables that
+        will be used as jinja template variables.
+
+    @return: A string of jinja-rendered content with the jinja header removed.
+        Returns None on error.
+    """
+    instance_data = {}
+    rendered_payload = None
+    if not os.path.exists(instance_data_file):
+        raise RuntimeError(
+            'Cannot render jinja template vars. Instance data not yet'
+            ' present at %s' % instance_data_file)
+    instance_data = load_json(load_file(instance_data_file))
+    rendered_payload = render_jinja_payload(
+        payload, payload_fn, instance_data, debug)
+    if not rendered_payload:
+        return None
+    return rendered_payload
+
+
+def render_jinja_payload(payload, payload_fn, instance_data, debug=False):
+    instance_jinja_vars = convert_jinja_instance_data(
+        instance_data,
+        decode_paths=instance_data.get('base64-encoded-keys', []))
+    if debug:
+        LOG.debug('Converted jinja variables\n%s',
+                  json_dumps(instance_jinja_vars))
+    try:
+        rendered_payload = render_string(payload, instance_jinja_vars)
+    except (TypeError, JUndefinedError) as e:
+        LOG.warning(
+            'Ignoring jinja template for %s: %s', payload_fn, str(e))
+        return None
+    warnings = [
+        "'%s'" % var.replace(MISSING_JINJA_PREFIX, '')
+        for var in re.findall(
+            r'%s[^\s]+' % MISSING_JINJA_PREFIX, rendered_payload)]
+    if warnings:
+        LOG.warning(
+            "Could not render jinja template variables in file '%s': %s",
+            payload_fn, ', '.join(warnings))
+    return rendered_payload
+
+
+def convert_jinja_instance_data(data, prefix='', sep='/', decode_paths=()):
+    """Process instance-data.json dict for use in jinja templates.
+
+    Replace hyphens with underscores for jinja templates and decode any
+    base64_encoded_keys.
+    """
+    result = {}
+    decode_paths = [path.replace('-', '_') for path in decode_paths]
+    for key, value in sorted(data.items()):
+        if '-' in key:
+            # Standardize keys for use in #cloud-config/shell templates
+            key = key.replace('-', '_')
+        key_path = '{0}{1}{2}'.format(prefix, sep, key) if prefix else key
+        if key_path in decode_paths:
+            value = b64d(value)
+        if isinstance(value, dict):
+            result[key] = convert_jinja_instance_data(
+                value, key_path, sep=sep, decode_paths=decode_paths)
+            if re.match(r'v\d+', key):
+                # Copy values to top-level aliases
+                for subkey, subvalue in result[key].items():
+                    result[subkey] = subvalue
+        else:
+            result[key] = value
+    return result
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py
index e4945a23..214714bc 100644
--- a/cloudinit/handlers/shell_script.py
+++ b/cloudinit/handlers/shell_script.py
@@ -17,21 +17,18 @@ from cloudinit import util
 from cloudinit.settings import (PER_ALWAYS)
 
 LOG = logging.getLogger(__name__)
-SHELL_PREFIX = "#!"
 
 
 class ShellScriptPartHandler(handlers.Handler):
+
+    prefixes = ['#!']
+
     def __init__(self, paths, **_kwargs):
         handlers.Handler.__init__(self, PER_ALWAYS)
         self.script_dir = paths.get_ipath_cur('scripts')
         if 'script_path' in _kwargs:
             self.script_dir = paths.get_ipath_cur(_kwargs['script_path'])
 
-    def list_types(self):
-        return [
-            handlers.type_from_starts_with(SHELL_PREFIX),
-        ]
-
     def handle_part(self, data, ctype, filename, payload, frequency):
         if ctype in handlers.CONTENT_SIGNALS:
             # TODO(harlowja): maybe delete existing things here
diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py
index dc338769..83fb0724 100644
--- a/cloudinit/handlers/upstart_job.py
+++ b/cloudinit/handlers/upstart_job.py
@@ -18,19 +18,16 @@ from cloudinit import util
 from cloudinit.settings import (PER_INSTANCE)
 
 LOG = logging.getLogger(__name__)
-UPSTART_PREFIX = "#upstart-job"
 
 
 class UpstartJobPartHandler(handlers.Handler):
+
+    prefixes = ['#upstart-job']
+
     def __init__(self, paths, **_kwargs):
         handlers.Handler.__init__(self, PER_INSTANCE)
         self.upstart_dir = paths.upstart_conf_d
 
-    def list_types(self):
-        return [
-            handlers.type_from_starts_with(UPSTART_PREFIX),
-        ]
-
     def handle_part(self, data, ctype, filename, payload, frequency):
         if ctype in handlers.CONTENT_SIGNALS:
             return
diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 1979cd96..3cc1fb19 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -449,4 +449,8 @@ class DefaultingConfigParser(RawConfigParser):
             contents = '\n'.join([header, contents, ''])
         return contents
 
+
+def identity(object):
+    return object
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/log.py b/cloudinit/log.py
index 1d75c9ff..5ae312ba 100644
--- a/cloudinit/log.py
+++ b/cloudinit/log.py
@@ -38,10 +38,18 @@ DEF_CON_FORMAT = '%(asctime)s - %(filename)s[%(levelname)s]: %(message)s'
 logging.Formatter.converter = time.gmtime
 
 
-def setupBasicLogging(level=DEBUG):
+def setupBasicLogging(level=DEBUG, formatter=None):
+    if not formatter:
+        formatter = logging.Formatter(DEF_CON_FORMAT)
     root = logging.getLogger()
+    for handler in root.handlers:
+        if hasattr(handler, 'stream') and hasattr(handler.stream, 'name'):
+            if handler.stream.name == '<stderr>':
+                handler.setLevel(level)
+                return
+    # Didn't have an existing stderr handler; create a new handler
     console = logging.StreamHandler(sys.stderr)
-    console.setFormatter(logging.Formatter(DEF_CON_FORMAT))
+    console.setFormatter(formatter)
     console.setLevel(level)
     root.addHandler(console)
     root.setLevel(level)
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 41fde9ba..a775f1a8 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -58,22 +58,27 @@ class InvalidMetaDataException(Exception):
     pass
 
 
-def process_base64_metadata(metadata, key_path=''):
-    """Strip ci-b64 prefix and return metadata with base64-encoded-keys set."""
+def process_instance_metadata(metadata, key_path=''):
+    """Process all instance metadata cleaning it up for persisting as json.
+
+    Strip ci-b64 prefix and catalog any 'base64_encoded_keys' as a list
+
+    @return Dict copy of processed metadata.
+    """
     md_copy = copy.deepcopy(metadata)
-    md_copy['base64-encoded-keys'] = []
+    md_copy['base64_encoded_keys'] = []
     for key, val in metadata.items():
         if key_path:
             sub_key_path = key_path + '/' + key
         else:
             sub_key_path = key
         if isinstance(val, str) and val.startswith('ci-b64:'):
-            md_copy['base64-encoded-keys'].append(sub_key_path)
+            md_copy['base64_encoded_keys'].append(sub_key_path)
             md_copy[key] = val.replace('ci-b64:', '')
         if isinstance(val, dict):
-            return_val = process_base64_metadata(val, sub_key_path)
-            md_copy['base64-encoded-keys'].extend(
-                return_val.pop('base64-encoded-keys'))
+            return_val = process_instance_metadata(val, sub_key_path)
+            md_copy['base64_encoded_keys'].extend(
+                return_val.pop('base64_encoded_keys'))
             md_copy[key] = return_val
     return md_copy
 
@@ -180,15 +185,24 @@ class DataSource(object):
         """
         self._dirty_cache = True
         return_value = self._get_data()
-        json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE)
         if not return_value:
             return return_value
+        self.persist_instance_data()
+        return return_value
+
+    def persist_instance_data(self):
+        """Process and write INSTANCE_JSON_FILE with all instance metadata.
 
+        Replace any hyphens with underscores in key names for use in template
+        processing.
+
+        @return True on successful write, False otherwise.
+        """
         instance_data = {
             'ds': {
-                'meta-data': self.metadata,
-                'user-data': self.get_userdata_raw(),
-                'vendor-data': self.get_vendordata_raw()}}
+                'meta_data': self.metadata,
+                'user_data': self.get_userdata_raw(),
+                'vendor_data': self.get_vendordata_raw()}}
         if hasattr(self, 'network_json'):
             network_json = getattr(self, 'network_json')
             if network_json != UNSET:
@@ -202,16 +216,17 @@ class DataSource(object):
         try:
             # Process content base64encoding unserializable values
             content = util.json_dumps(instance_data)
-            # Strip base64: prefix and return base64-encoded-keys
-            processed_data = process_base64_metadata(json.loads(content))
+            # Strip base64: prefix and set base64_encoded_keys list.
+            processed_data = process_instance_metadata(json.loads(content))
         except TypeError as e:
             LOG.warning('Error persisting instance-data.json: %s', str(e))
-            return return_value
+            return False
         except UnicodeDecodeError as e:
             LOG.warning('Error persisting instance-data.json: %s', str(e))
-            return return_value
+            return False
+        json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE)
         write_json(json_file, processed_data, mode=0o600)
-        return return_value
+        return True
 
     def _get_data(self):
         """Walk metadata sources, process crawled data and save attributes."""
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index 9e939c1e..8299af23 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -20,10 +20,12 @@ class DataSourceTestSubclassNet(DataSource):
     dsname = 'MyTestSubclass'
     url_max_wait = 55
 
-    def __init__(self, sys_cfg, distro, paths, custom_userdata=None):
+    def __init__(self, sys_cfg, distro, paths, custom_userdata=None,
+                 get_data_retval=True):
         super(DataSourceTestSubclassNet, self).__init__(
             sys_cfg, distro, paths)
         self._custom_userdata = custom_userdata
+        self._get_data_retval = get_data_retval
 
     def _get_cloud_name(self):
         return 'SubclassCloudName'
@@ -37,7 +39,7 @@ class DataSourceTestSubclassNet(DataSource):
         else:
             self.userdata_raw = 'userdata_raw'
         self.vendordata_raw = 'vendordata_raw'
-        return True
+        return self._get_data_retval
 
 
 class InvalidDataSourceTestSubclassNet(DataSource):
@@ -264,7 +266,18 @@ class TestDataSource(CiTestCase):
                 self.assertEqual('fqdnhostname.domain.com',
                                  datasource.get_hostname(fqdn=True))
 
-    def test_get_data_write_json_instance_data(self):
+    def test_get_data_does_not_write_instance_data_on_failure(self):
+        """get_data does not write INSTANCE_JSON_FILE on get_data False."""
+        tmp = self.tmp_dir()
+        datasource = DataSourceTestSubclassNet(
+            self.sys_cfg, self.distro, Paths({'run_dir': tmp}),
+            get_data_retval=False)
+        self.assertFalse(datasource.get_data())
+        json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
+        self.assertFalse(
+            os.path.exists(json_file), 'Found unexpected file %s' % json_file)
+
+    def test_get_data_writes_json_instance_data_on_success(self):
         """get_data writes INSTANCE_JSON_FILE to run_dir as readonly root."""
         tmp = self.tmp_dir()
         datasource = DataSourceTestSubclassNet(
@@ -273,7 +286,7 @@ class TestDataSource(CiTestCase):
         json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
         content = util.load_file(json_file)
         expected = {
-            'base64-encoded-keys': [],
+            'base64_encoded_keys': [],
             'v1': {
                 'availability-zone': 'myaz',
                 'cloud-name': 'subclasscloudname',
@@ -281,11 +294,12 @@ class TestDataSource(CiTestCase):
                 'local-hostname': 'test-subclass-hostname',
                 'region': 'myregion'},
             'ds': {
-                'meta-data': {'availability_zone': 'myaz',
+                'meta_data': {'availability_zone': 'myaz',
                               'local-hostname': 'test-subclass-hostname',
                               'region': 'myregion'},
-                'user-data': 'userdata_raw',
-                'vendor-data': 'vendordata_raw'}}
+                'user_data': 'userdata_raw',
+                'vendor_data': 'vendordata_raw'}}
+        self.maxDiff = None
         self.assertEqual(expected, util.load_json(content))
         file_stat = os.stat(json_file)
         self.assertEqual(0o600, stat.S_IMODE(file_stat.st_mode))
@@ -296,7 +310,7 @@ class TestDataSource(CiTestCase):
         datasource = DataSourceTestSubclassNet(
             self.sys_cfg, self.distro, Paths({'run_dir': tmp}),
             custom_userdata={'key1': 'val1', 'key2': {'key2.1': self.paths}})
-        self.assertTrue(datasource.get_data())
+        datasource.get_data()
         json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
         content = util.load_file(json_file)
         expected_userdata = {
@@ -306,7 +320,40 @@ class TestDataSource(CiTestCase):
                           " 'cloudinit.helpers.Paths'>"}}
         instance_json = util.load_json(content)
         self.assertEqual(
-            expected_userdata, instance_json['ds']['user-data'])
+            expected_userdata, instance_json['ds']['user_data'])
+
+    def test_persist_instance_data_writes_ec2_metadata_when_set(self):
+        """When ec2_metadata class attribute is set, persist to json."""
+        tmp = self.tmp_dir()
+        datasource = DataSourceTestSubclassNet(
+            self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
+        datasource.ec2_metadata = UNSET
+        datasource.get_data()
+        json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
+        instance_data = util.load_json(util.load_file(json_file))
+        self.assertNotIn('ec2_metadata', instance_data['ds'])
+        datasource.ec2_metadata = {'ec2stuff': 'is good'}
+        datasource.persist_instance_data()
+        instance_data = util.load_json(util.load_file(json_file))
+        self.assertEqual(
+            {'ec2stuff': 'is good'},
+            instance_data['ds']['ec2_metadata'])
+
+    def test_persist_instance_data_writes_network_json_when_set(self):
+        """When network_data.json class attribute is set, persist to json."""
+        tmp = self.tmp_dir()
+        datasource = DataSourceTestSubclassNet(
+            self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
+        datasource.get_data()
+        json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
+        instance_data = util.load_json(util.load_file(json_file))
+        self.assertNotIn('network_json', instance_data['ds'])
+        datasource.network_json = {'network_json': 'is good'}
+        datasource.persist_instance_data()
+        instance_data = util.load_json(util.load_file(json_file))
+        self.assertEqual(
+            {'network_json': 'is good'},
+            instance_data['ds']['network_json'])
 
     @skipIf(not six.PY3, "json serialization on <= py2.7 handles bytes")
     def test_get_data_base64encodes_unserializable_bytes(self):
@@ -320,11 +367,11 @@ class TestDataSource(CiTestCase):
         content = util.load_file(json_file)
         instance_json = util.load_json(content)
         self.assertEqual(
-            ['ds/user-data/key2/key2.1'],
-            instance_json['base64-encoded-keys'])
+            ['ds/user_data/key2/key2.1'],
+            instance_json['base64_encoded_keys'])
         self.assertEqual(
             {'key1': 'val1', 'key2': {'key2.1': 'EjM='}},
-            instance_json['ds']['user-data'])
+            instance_json['ds']['user_data'])
 
     @skipIf(not six.PY2, "json serialization on <= py2.7 handles bytes")
     def test_get_data_handles_bytes_values(self):
@@ -337,10 +384,10 @@ class TestDataSource(CiTestCase):
         json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
         content = util.load_file(json_file)
         instance_json = util.load_json(content)
-        self.assertEqual([], instance_json['base64-encoded-keys'])
+        self.assertEqual([], instance_json['base64_encoded_keys'])
         self.assertEqual(
             {'key1': 'val1', 'key2': {'key2.1': '\x123'}},
-            instance_json['ds']['user-data'])
+            instance_json['ds']['user_data'])
 
     @skipIf(not six.PY2, "Only python2 hits UnicodeDecodeErrors on non-utf8")
     def test_non_utf8_encoding_logs_warning(self):
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 8874d405..ef5c6996 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -17,10 +17,11 @@ from cloudinit.settings import (
 from cloudinit import handlers
 
 # Default handlers (used if not overridden)
-from cloudinit.handlers import boot_hook as bh_part
-from cloudinit.handlers import cloud_config as cc_part
-from cloudinit.handlers import shell_script as ss_part
-from cloudinit.handlers import upstart_job as up_part
+from cloudinit.handlers.boot_hook import BootHookPartHandler
+from cloudinit.handlers.cloud_config import CloudConfigPartHandler
+from cloudinit.handlers.jinja_template import JinjaTemplatePartHandler
+from cloudinit.handlers.shell_script import ShellScriptPartHandler
+from cloudinit.handlers.upstart_job import UpstartJobPartHandler
 
 from cloudinit.event import EventType
 
@@ -413,12 +414,17 @@ class Init(object):
             'datasource': self.datasource,
         })
         # TODO(harlowja) Hmmm, should we dynamically import these??
+        cloudconfig_handler = CloudConfigPartHandler(**opts)
+        shellscript_handler = ShellScriptPartHandler(**opts)
         def_handlers = [
-            cc_part.CloudConfigPartHandler(**opts),
-            ss_part.ShellScriptPartHandler(**opts),
-            bh_part.BootHookPartHandler(**opts),
-            up_part.UpstartJobPartHandler(**opts),
+            cloudconfig_handler,
+            shellscript_handler,
+            BootHookPartHandler(**opts),
+            UpstartJobPartHandler(**opts),
         ]
+        opts.update(
+            {'sub_handlers': [cloudconfig_handler, shellscript_handler]})
+        def_handlers.append(JinjaTemplatePartHandler(**opts))
         return def_handlers
 
     def _default_userdata_handlers(self):
diff --git a/cloudinit/templater.py b/cloudinit/templater.py
index 7e7acb86..b668674b 100644
--- a/cloudinit/templater.py
+++ b/cloudinit/templater.py
@@ -13,6 +13,7 @@
 import collections
 import re
 
+
 try:
     from Cheetah.Template import Template as CTemplate
     CHEETAH_AVAILABLE = True
@@ -20,23 +21,44 @@ except (ImportError, AttributeError):
     CHEETAH_AVAILABLE = False
 
 try:
-    import jinja2
+    from jinja2.runtime import implements_to_string
     from jinja2 import Template as JTemplate
+    from jinja2 import DebugUndefined as JUndefined
     JINJA_AVAILABLE = True
 except (ImportError, AttributeError):
+    from cloudinit.helpers import identity
+    implements_to_string = identity
     JINJA_AVAILABLE = False
+    JUndefined = object
 
 from cloudinit import log as logging
 from cloudinit import type_utils as tu
 from cloudinit import util
 
+
 LOG = logging.getLogger(__name__)
 TYPE_MATCHER = re.compile(r"##\s*template:(.*)", re.I)
 BASIC_MATCHER = re.compile(r'\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)')
+MISSING_JINJA_PREFIX = u'CI_MISSING_JINJA_VAR/'
+
+
+@implements_to_string   # Needed for python2.7. Otherwise cached super.__str__
+class UndefinedJinjaVariable(JUndefined):
+    """Class used to represent any undefined jinja template varible."""
+
+    def __str__(self):
+        return u'%s%s' % (MISSING_JINJA_PREFIX, self._undefined_name)
+
+    def __sub__(self, other):
+        other = str(other).replace(MISSING_JINJA_PREFIX, '')
+        raise TypeError(
+            'Undefined jinja variable: "{this}-{other}". Jinja tried'
+            ' subtraction. Perhaps you meant "{this}_{other}"?'.format(
+                this=self._undefined_name, other=other))
 
 
 def basic_render(content, params):
-    """This does simple replacement of bash variable like templates.
+    """This does sumple replacement of bash variable like templates.
 
     It identifies patterns like ${a} or $a and can also identify patterns like
     ${a.b} or $a.b which will look for a key 'b' in the dictionary rooted
@@ -82,7 +104,7 @@ def detect_template(text):
         # keep_trailing_newline is in jinja2 2.7+, not 2.6
         add = "\n" if content.endswith("\n") else ""
         return JTemplate(content,
-                         undefined=jinja2.StrictUndefined,
+                         undefined=UndefinedJinjaVariable,
                          trim_blocks=True).render(**params) + add
 
     if text.find("\n") != -1:
diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py
index 42f56c27..2eb7b0cd 100644
--- a/cloudinit/tests/helpers.py
+++ b/cloudinit/tests/helpers.py
@@ -32,6 +32,7 @@ from cloudinit import cloud
 from cloudinit import distros
 from cloudinit import helpers as ch
 from cloudinit.sources import DataSourceNone
+from cloudinit.templater import JINJA_AVAILABLE
 from cloudinit import util
 
 _real_subp = util.subp
@@ -518,6 +519,14 @@ def skipUnlessJsonSchema():
         _missing_jsonschema_dep, "No python-jsonschema dependency present.")
 
 
+def skipUnlessJinja():
+    return skipIf(not JINJA_AVAILABLE, "No jinja dependency present.")
+
+
+def skipIfJinja():
+    return skipIf(JINJA_AVAILABLE, "Jinja dependency present.")
+
+
 # older versions of mock do not have the useful 'assert_not_called'
 if not hasattr(mock.Mock, 'assert_not_called'):
     def __mock_assert_not_called(mmock):
diff --git a/doc/rtd/topics/capabilities.rst b/doc/rtd/topics/capabilities.rst
index 3e2c9e31..2d8e2538 100644
--- a/doc/rtd/topics/capabilities.rst
+++ b/doc/rtd/topics/capabilities.rst
@@ -16,13 +16,15 @@ User configurability
 
 `Cloud-init`_ 's behavior can be configured via user-data.
 
-    User-data can be given by the user at instance launch time.
+    User-data can be given by the user at instance launch time. See
+    :ref:`user_data_formats` for acceptable user-data content.
+    
 
 This is done via the ``--user-data`` or ``--user-data-file`` argument to
 ec2-run-instances for example.
 
-* Check your local clients documentation for how to provide a `user-data`
-  string or `user-data` file for usage by cloud-init on instance creation.
+* Check your local client's documentation for how to provide a `user-data`
+  string or `user-data` file to cloud-init on instance creation.
 
 
 Feature detection
@@ -166,6 +168,13 @@ likely be promoted to top-level subcommands when stable.
    validation is work in progress and supports a subset of cloud-config
    modules.
 
+ * ``cloud-init devel render``: Use cloud-init's jinja template render to
+   process  **#cloud-config** or **custom-scripts**, injecting any variables
+   from ``/run/cloud-init/instance-data.json``. It accepts a user-data file
+   containing  the jinja template header ``## template: jinja`` and renders
+   that content with any instance-data.json variables present.
+
+
 .. _cli_clean:
 
 cloud-init clean
diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst
index 83034589..14432e65 100644
--- a/doc/rtd/topics/datasources.rst
+++ b/doc/rtd/topics/datasources.rst
@@ -18,6 +18,8 @@ single way to access the different cloud systems methods to provide this data
 through the typical usage of subclasses.
 
 
+.. _instance_metadata:
+
 instance-data
 -------------
 For reference, cloud-init stores all the metadata, vendordata and userdata
@@ -110,6 +112,51 @@ Below is an instance-data.json example from an OpenStack instance:
    }
   }
 
+ 
+As of cloud-init v. 18.4, any values present in
+``/run/cloud-init/instance-data.json`` can be used in cloud-init user data
+scripts or cloud config data. This allows consumers to use cloud-init's
+vendor-neutral, standardized metadata keys as well as datasource-specific
+content for any scripts or cloud-config modules they are using.
+
+To use instance-data.json values in scripts and **#config-config** files the
+user-data will need to contain the following header as the first line **## template: jinja**. Cloud-init will source all variables defined in
+``/run/cloud-init/instance-data.json`` and allow scripts or cloud-config files 
+to reference those paths. Below are two examples::
+
+ * Cloud config calling home with the ec2 public hostname and avaliability-zone
+    ```
+    ## template: jinja
+    #cloud-config
+    runcmd:
+        - echo 'EC2 public hostname allocated to instance: {{ ds.meta_data.public_hostname }}' > /tmp/instance_metadata
+        - echo 'EC2 avaiability zone: {{ v1.availability_zone }}' >> /tmp/instance_metadata 
+        - curl -X POST -d '{"hostname": "{{ds.meta_data.public_hostname }}", "availability-zone": "{{ v1.availability_zone }}"}'  https://example.com.com
+    ```
+
+ * Custom user script performing different operations based on region
+    ```
+    ## template: jinja
+    #!/bin/bash
+    {% if v1.region == 'us-east-2' -%}
+    echo 'Installing custom proxies for {{ v1.region }}
+    sudo apt-get install my-xtra-fast-stack
+    {%- endif %}
+    ...
+
+    ```
+
+.. note::
+  Trying to reference jinja variables that don't exist in
+  instance-data.json will result in warnings in ``/var/log/cloud-init.log``
+  and the following string in your rendered user-data:
+  ``CI_MISSING_JINJA_VAR/<your_varname>``.
+  
+.. note::
+  To save time designing your user-data for a specific cloud's
+  instance-data.json, use the 'render' cloud-init command on an
+  instance booted on your favorite cloud. See :ref:`cli_devel` for more
+  information.
 
 
 Datasource API
diff --git a/doc/rtd/topics/format.rst b/doc/rtd/topics/format.rst
index 1b0ff366..15234d21 100644
--- a/doc/rtd/topics/format.rst
+++ b/doc/rtd/topics/format.rst
@@ -1,6 +1,8 @@
-*******
-Formats
-*******
+.. _user_data_formats:
+
+*****************
+User-Data Formats
+*****************
 
 User data that will be acted upon by cloud-init must be in one of the following types.
 
@@ -65,6 +67,11 @@ Typically used by those who just want to execute a shell script.
 
 Begins with: ``#!`` or ``Content-Type: text/x-shellscript`` when using a MIME archive.
 
+.. note::
+   New in cloud-init v. 18.4: User-data scripts can also render cloud instance
+   metadata variables using jinja templating. See
+   :ref:`instance_metadata` for more information.
+
 Example
 -------
 
@@ -103,12 +110,18 @@ These things include:
 - certain ssh keys should be imported
 - *and many more...*
 
-**Note:** The file must be valid yaml syntax.
+.. note::
+   This file must be valid yaml syntax.
 
 See the :ref:`yaml_examples` section for a commented set of examples of supported cloud config formats.
 
 Begins with: ``#cloud-config`` or ``Content-Type: text/cloud-config`` when using a MIME archive.
 
+.. note::
+   New in cloud-init v. 18.4: Cloud config dta can also render cloud instance
+   metadata variables using jinja templating. See
+   :ref:`instance_metadata` for more information.
+
 Upstart Job
 ===========
 
diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py
index 696db8dd..27458271 100644
--- a/tests/cloud_tests/testcases/base.py
+++ b/tests/cloud_tests/testcases/base.py
@@ -168,7 +168,7 @@ class CloudTestCase(unittest.TestCase):
                 ' OS: %s not bionic or newer' % self.os_name)
         instance_data = json.loads(out)
         self.assertEqual(
-            ['ds/user-data'], instance_data['base64-encoded-keys'])
+            ['ds/user_data'], instance_data['base64_encoded_keys'])
         ds = instance_data.get('ds', {})
         v1_data = instance_data.get('v1', {})
         metadata = ds.get('meta-data', {})
@@ -214,8 +214,8 @@ class CloudTestCase(unittest.TestCase):
         instance_data = json.loads(out)
         v1_data = instance_data.get('v1', {})
         self.assertEqual(
-            ['ds/user-data', 'ds/vendor-data'],
-            sorted(instance_data['base64-encoded-keys']))
+            ['ds/user_data', 'ds/vendor_data'],
+            sorted(instance_data['base64_encoded_keys']))
         self.assertEqual('nocloud', v1_data['cloud-name'])
         self.assertIsNone(
             v1_data['availability-zone'],
@@ -249,7 +249,7 @@ class CloudTestCase(unittest.TestCase):
         instance_data = json.loads(out)
         v1_data = instance_data.get('v1', {})
         self.assertEqual(
-            ['ds/user-data'], instance_data['base64-encoded-keys'])
+            ['ds/user_data'], instance_data['base64_encoded_keys'])
         self.assertEqual('nocloud', v1_data['cloud-name'])
         self.assertIsNone(
             v1_data['availability-zone'],
diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py
index 9751ed95..abe820e1 100644
--- a/tests/unittests/test_builtin_handlers.py
+++ b/tests/unittests/test_builtin_handlers.py
@@ -2,27 +2,34 @@
 
 """Tests of the built-in user data handlers."""
 
+import copy
 import os
 import shutil
 import tempfile
+from textwrap import dedent
 
-try:
-    from unittest import mock
-except ImportError:
-    import mock
 
-from cloudinit.tests import helpers as test_helpers
+from cloudinit.tests.helpers import (
+    FilesystemMockingTestCase, CiTestCase, mock, skipUnlessJinja)
 
 from cloudinit import handlers
 from cloudinit import helpers
 from cloudinit import util
 
-from cloudinit.handlers import upstart_job
+from cloudinit.handlers.cloud_config import CloudConfigPartHandler
+from cloudinit.handlers.jinja_template import (
+    JinjaTemplatePartHandler, convert_jinja_instance_data,
+    render_jinja_payload)
+from cloudinit.handlers.shell_script import ShellScriptPartHandler
+from cloudinit.handlers.upstart_job import UpstartJobPartHandler
 
 from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE)
 
 
-class TestBuiltins(test_helpers.FilesystemMockingTestCase):
+class TestUpstartJobPartHandler(FilesystemMockingTestCase):
+
+    mpath = 'cloudinit.handlers.upstart_job.'
+
     def test_upstart_frequency_no_out(self):
         c_root = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, c_root)
@@ -32,14 +39,13 @@ class TestBuiltins(test_helpers.FilesystemMockingTestCase):
             'cloud_dir': c_root,
             'upstart_dir': up_root,
         })
-        freq = PER_ALWAYS
-        h = upstart_job.UpstartJobPartHandler(paths)
+        h = UpstartJobPartHandler(paths)
         # No files should be written out when
         # the frequency is ! per-instance
         h.handle_part('', handlers.CONTENT_START,
                       None, None, None)
         h.handle_part('blah', 'text/upstart-job',
-                      'test.conf', 'blah', freq)
+                      'test.conf', 'blah', frequency=PER_ALWAYS)
         h.handle_part('', handlers.CONTENT_END,
                       None, None, None)
         self.assertEqual(0, len(os.listdir(up_root)))
@@ -48,7 +54,6 @@ class TestBuiltins(test_helpers.FilesystemMockingTestCase):
         # files should be written out when frequency is ! per-instance
         new_root = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, new_root)
-        freq = PER_INSTANCE
 
         self.patchOS(new_root)
         self.patchUtils(new_root)
@@ -56,22 +61,297 @@ class TestBuiltins(test_helpers.FilesystemMockingTestCase):
             'upstart_dir': "/etc/upstart",
         })
 
-        upstart_job.SUITABLE_UPSTART = True
         util.ensure_dir("/run")
         util.ensure_dir("/etc/upstart")
 
-        with mock.patch.object(util, 'subp') as mockobj:
-            h = upstart_job.UpstartJobPartHandler(paths)
-            h.handle_part('', handlers.CONTENT_START,
-                          None, None, None)
-            h.handle_part('blah', 'text/upstart-job',
-                          'test.conf', 'blah', freq)
-            h.handle_part('', handlers.CONTENT_END,
-                          None, None, None)
+        with mock.patch(self.mpath + 'SUITABLE_UPSTART', return_value=True):
+            with mock.patch.object(util, 'subp') as m_subp:
+                h = UpstartJobPartHandler(paths)
+                h.handle_part('', handlers.CONTENT_START,
+                              None, None, None)
+                h.handle_part('blah', 'text/upstart-job',
+                              'test.conf', 'blah', frequency=PER_INSTANCE)
+                h.handle_part('', handlers.CONTENT_END,
+                              None, None, None)
 
-            self.assertEqual(len(os.listdir('/etc/upstart')), 1)
+        self.assertEqual(len(os.listdir('/etc/upstart')), 1)
 
-        mockobj.assert_called_once_with(
+        m_subp.assert_called_once_with(
             ['initctl', 'reload-configuration'], capture=False)
 
+
+class TestJinjaTemplatePartHandler(CiTestCase):
+
+    with_logs = True
+
+    mpath = 'cloudinit.handlers.jinja_template.'
+
+    def setUp(self):
+        super(TestJinjaTemplatePartHandler, self).setUp()
+        self.tmp = self.tmp_dir()
+        self.run_dir = os.path.join(self.tmp, 'run_dir')
+        util.ensure_dir(self.run_dir)
+        self.paths = helpers.Paths({
+            'cloud_dir': self.tmp, 'run_dir': self.run_dir})
+
+    def test_jinja_template_part_handler_defaults(self):
+        """On init, paths are saved and subhandler types are empty."""
+        h = JinjaTemplatePartHandler(self.paths)
+        self.assertEqual(['## template: jinja'], h.prefixes)
+        self.assertEqual(3, h.handler_version)
+        self.assertEqual(self.paths, h.paths)
+        self.assertEqual({}, h.sub_handlers)
+
+    def test_jinja_template_part_handler_looks_up_sub_handler_types(self):
+        """When sub_handlers are passed, init lists types of subhandlers."""
+        script_handler = ShellScriptPartHandler(self.paths)
+        cloudconfig_handler = CloudConfigPartHandler(self.paths)
+        h = JinjaTemplatePartHandler(
+            self.paths, sub_handlers=[script_handler, cloudconfig_handler])
+        self.assertItemsEqual(
+            ['text/cloud-config', 'text/cloud-config-jsonp',
+             'text/x-shellscript'],
+            h.sub_handlers)
+
+    def test_jinja_template_part_handler_looks_up_subhandler_types(self):
+        """When sub_handlers are passed, init lists types of subhandlers."""
+        script_handler = ShellScriptPartHandler(self.paths)
+        cloudconfig_handler = CloudConfigPartHandler(self.paths)
+        h = JinjaTemplatePartHandler(
+            self.paths, sub_handlers=[script_handler, cloudconfig_handler])
+        self.assertItemsEqual(
+            ['text/cloud-config', 'text/cloud-config-jsonp',
+             'text/x-shellscript'],
+            h.sub_handlers)
+
+    def test_jinja_template_handle_noop_on_content_signals(self):
+        """Perform no part handling when content type is CONTENT_SIGNALS."""
+        script_handler = ShellScriptPartHandler(self.paths)
+
+        h = JinjaTemplatePartHandler(
+            self.paths, sub_handlers=[script_handler])
+        with mock.patch.object(script_handler, 'handle_part') as m_handle_part:
+            h.handle_part(
+                data='data', ctype=handlers.CONTENT_START, filename='part-1',
+                payload='## template: jinja\n#!/bin/bash\necho himom',
+                frequency='freq', headers='headers')
+        m_handle_part.assert_not_called()
+
+    @skipUnlessJinja()
+    def test_jinja_template_handle_subhandler_v2_with_clean_payload(self):
+        """Call version 2 subhandler.handle_part with stripped payload."""
+        script_handler = ShellScriptPartHandler(self.paths)
+        self.assertEqual(2, script_handler.handler_version)
+
+        # Create required instance-data.json file
+        instance_json = os.path.join(self.run_dir, 'instance-data.json')
+        instance_data = {'topkey': 'echo himom'}
+        util.write_file(instance_json, util.json_dumps(instance_data))
+        h = JinjaTemplatePartHandler(
+            self.paths, sub_handlers=[script_handler])
+        with mock.patch.object(script_handler, 'handle_part') as m_part:
+            # ctype with leading '!' not in handlers.CONTENT_SIGNALS
+            h.handle_part(
+                data='data', ctype="!" + handlers.CONTENT_START,
+                filename='part01',
+                payload='## template: jinja   \t \n#!/bin/bash\n{{ topkey }}',
+                frequency='freq', headers='headers')
+        m_part.assert_called_once_with(
+            'data', '!__begin__', 'part01', '#!/bin/bash\necho himom', 'freq')
+
+    @skipUnlessJinja()
+    def test_jinja_template_handle_subhandler_v3_with_clean_payload(self):
+        """Call version 3 subhandler.handle_part with stripped payload."""
+        cloudcfg_handler = CloudConfigPartHandler(self.paths)
+        self.assertEqual(3, cloudcfg_handler.handler_version)
+
+        # Create required instance-data.json file
+        instance_json = os.path.join(self.run_dir, 'instance-data.json')
+        instance_data = {'topkey': {'sub': 'runcmd: [echo hi]'}}
+        util.write_file(instance_json, util.json_dumps(instance_data))
+        h = JinjaTemplatePartHandler(
+            self.paths, sub_handlers=[cloudcfg_handler])
+        with mock.patch.object(cloudcfg_handler, 'handle_part') as m_part:
+            # ctype with leading '!' not in handlers.CONTENT_SIGNALS
+            h.handle_part(
+                data='data', ctype="!" + handlers.CONTENT_END,
+                filename='part01',
+                payload='## template: jinja\n#cloud-config\n{{ topkey.sub }}',
+                frequency='freq', headers='headers')
+        m_part.assert_called_once_with(
+            'data', '!__end__', 'part01', '#cloud-config\nruncmd: [echo hi]',
+            'freq', 'headers')
+
+    def test_jinja_template_handle_errors_on_missing_instance_data_json(self):
+        """If instance-data is absent, raise an error from handle_part."""
+        script_handler = ShellScriptPartHandler(self.paths)
+        h = JinjaTemplatePartHandler(
+            self.paths, sub_handlers=[script_handler])
+        with self.assertRaises(RuntimeError) as context_manager:
+            h.handle_part(
+                data='data', ctype="!" + handlers.CONTENT_START,
+                filename='part01',
+                payload='## template: jinja  \n#!/bin/bash\necho himom',
+                frequency='freq', headers='headers')
+        script_file = os.path.join(script_handler.script_dir, 'part01')
+        self.assertEqual(
+            'Cannot render jinja template vars. Instance data not yet present'
+            ' at {}/instance-data.json'.format(
+                self.run_dir), str(context_manager.exception))
+        self.assertFalse(
+            os.path.exists(script_file),
+            'Unexpected file created %s' % script_file)
+
+    @skipUnlessJinja()
+    def test_jinja_template_handle_renders_jinja_content(self):
+        """When present, render jinja variables from instance-data.json."""
+        script_handler = ShellScriptPartHandler(self.paths)
+        instance_json = os.path.join(self.run_dir, 'instance-data.json')
+        instance_data = {'topkey': {'subkey': 'echo himom'}}
+        util.write_file(instance_json, util.json_dumps(instance_data))
+        h = JinjaTemplatePartHandler(
+            self.paths, sub_handlers=[script_handler])
+        h.handle_part(
+            data='data', ctype="!" + handlers.CONTENT_START,
+            filename='part01',
+            payload=(
+                '## template: jinja  \n'
+                '#!/bin/bash\n'
+                '{{ topkey.subkey|default("nosubkey") }}'),
+            frequency='freq', headers='headers')
+        script_file = os.path.join(script_handler.script_dir, 'part01')
+        self.assertNotIn(
+            'Instance data not yet present at {}/instance-data.json'.format(
+                self.run_dir),
+            self.logs.getvalue())
+        self.assertEqual(
+            '#!/bin/bash\necho himom', util.load_file(script_file))
+
+    @skipUnlessJinja()
+    def test_jinja_template_handle_renders_jinja_content_missing_keys(self):
+        """When specified jinja variable is undefined, log a warning."""
+        script_handler = ShellScriptPartHandler(self.paths)
+        instance_json = os.path.join(self.run_dir, 'instance-data.json')
+        instance_data = {'topkey': {'subkey': 'echo himom'}}
+        util.write_file(instance_json, util.json_dumps(instance_data))
+        h = JinjaTemplatePartHandler(
+            self.paths, sub_handlers=[script_handler])
+        h.handle_part(
+            data='data', ctype="!" + handlers.CONTENT_START,
+            filename='part01',
+            payload='## template: jinja  \n#!/bin/bash\n{{ goodtry }}',
+            frequency='freq', headers='headers')
+        script_file = os.path.join(script_handler.script_dir, 'part01')
+        self.assertTrue(
+            os.path.exists(script_file),
+            'Missing expected file %s' % script_file)
+        self.assertIn(
+            "WARNING: Could not render jinja template variables in file"
+            " 'part01': 'goodtry'\n",
+            self.logs.getvalue())
+
+
+class TestConvertJinjaInstanceData(CiTestCase):
+
+    def test_convert_instance_data_hyphens_to_underscores(self):
+        """Replace hyphenated keys with underscores in instance-data."""
+        data = {'hyphenated-key': 'hyphenated-val',
+                'underscore_delim_key': 'underscore_delimited_val'}
+        expected_data = {'hyphenated_key': 'hyphenated-val',
+                         'underscore_delim_key': 'underscore_delimited_val'}
+        self.assertEqual(
+            expected_data,
+            convert_jinja_instance_data(data=data))
+
+    def test_convert_instance_data_promotes_versioned_keys_to_top_level(self):
+        """Any versioned keys are promoted as top-level keys
+
+        This provides any cloud-init standardized keys up at a top-level to
+        allow ease of reference for users. Intsead of v1.availability_zone,
+        the name availability_zone can be used in templates.
+        """
+        data = {'ds': {'dskey1': 1, 'dskey2': 2},
+                'v1': {'v1key1': 'v1.1'},
+                'v2': {'v2key1': 'v2.1'}}
+        expected_data = copy.deepcopy(data)
+        expected_data.update({'v1key1': 'v1.1', 'v2key1': 'v2.1'})
+
+        converted_data = convert_jinja_instance_data(data=data)
+        self.assertItemsEqual(
+            ['ds', 'v1', 'v2', 'v1key1', 'v2key1'], converted_data.keys())
+        self.assertEqual(
+            expected_data,
+            converted_data)
+
+    def test_convert_instance_data_most_recent_version_of_promoted_keys(self):
+        """The most-recent versioned key value is promoted to top-level."""
+        data = {'v1': {'key1': 'old v1 key1', 'key2': 'old v1 key2'},
+                'v2': {'key1': 'newer v2 key1', 'key3': 'newer v2 key3'},
+                'v3': {'key1': 'newest v3 key1'}}
+        expected_data = copy.deepcopy(data)
+        expected_data.update(
+            {'key1': 'newest v3 key1', 'key2': 'old v1 key2',
+             'key3': 'newer v2 key3'})
+
+        converted_data = convert_jinja_instance_data(data=data)
+        self.assertEqual(
+            expected_data,
+            converted_data)
+
+    def test_convert_instance_data_decodes_decode_paths(self):
+        """Any decode_paths provided are decoded by convert_instance_data."""
+        data = {'key1': {'subkey1': 'aGkgbW9t'}, 'key2': 'aGkgZGFk'}
+        expected_data = copy.deepcopy(data)
+        expected_data['key1']['subkey1'] = 'hi mom'
+
+        converted_data = convert_jinja_instance_data(
+            data=data, decode_paths=('key1/subkey1',))
+        self.assertEqual(
+            expected_data,
+            converted_data)
+
+
+class TestRenderJinjaPayload(CiTestCase):
+
+    with_logs = True
+
+    @skipUnlessJinja()
+    def test_render_jinja_payload_logs_jinja_vars_on_debug(self):
+        """When debug is True, log jinja varables available."""
+        payload = (
+            '## template: jinja\n#!/bin/sh\necho hi from {{ v1.hostname }}')
+        instance_data = {'v1': {'hostname': 'foo'}, 'instance-id': 'iid'}
+        expected_log = dedent("""\
+            DEBUG: Converted jinja variables
+            {
+             "hostname": "foo",
+             "instance_id": "iid",
+             "v1": {
+              "hostname": "foo"
+             }
+            }
+            """)
+        self.assertEqual(
+            render_jinja_payload(
+                payload=payload, payload_fn='myfile',
+                instance_data=instance_data, debug=True),
+            '#!/bin/sh\necho hi from foo')
+        self.assertEqual(expected_log, self.logs.getvalue())
+
+    @skipUnlessJinja()
+    def test_render_jinja_payload_replaces_missing_variables_and_warns(self):
+        """Warn on missing jinja variables and replace the absent variable."""
+        payload = (
+            '## template: jinja\n#!/bin/sh\necho hi from {{ NOTHERE }}')
+        instance_data = {'v1': {'hostname': 'foo'}, 'instance-id': 'iid'}
+        self.assertEqual(
+            render_jinja_payload(
+                payload=payload, payload_fn='myfile',
+                instance_data=instance_data),
+            '#!/bin/sh\necho hi from CI_MISSING_JINJA_VAR/NOTHERE')
+        expected_log = (
+            'WARNING: Could not render jinja template variables in file'
+            " 'myfile': 'NOTHERE'")
+        self.assertIn(expected_log, self.logs.getvalue())
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_etc_hosts.py b/tests/unittests/test_handler/test_handler_etc_hosts.py
index ced05a8d..d854afcb 100644
--- a/tests/unittests/test_handler/test_handler_etc_hosts.py
+++ b/tests/unittests/test_handler/test_handler_etc_hosts.py
@@ -49,6 +49,7 @@ class TestHostsFile(t_help.FilesystemMockingTestCase):
         if '192.168.1.1\tblah.blah.us\tblah' not in contents:
             self.assertIsNone('Default etc/hosts content modified')
 
+    @t_help.skipUnlessJinja()
     def test_write_etc_hosts_suse_template(self):
         cfg = {
             'manage_etc_hosts': 'template',
diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
index 6fe3659d..0f22e579 100644
--- a/tests/unittests/test_handler/test_handler_ntp.py
+++ b/tests/unittests/test_handler/test_handler_ntp.py
@@ -3,6 +3,7 @@
 from cloudinit.config import cc_ntp
 from cloudinit.sources import DataSourceNone
 from cloudinit import (distros, helpers, cloud, util)
+
 from cloudinit.tests.helpers import (
     CiTestCase, FilesystemMockingTestCase, mock, skipUnlessJsonSchema)
 
diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py
index 20c87efa..c36e6eb0 100644
--- a/tests/unittests/test_templating.py
+++ b/tests/unittests/test_templating.py
@@ -21,6 +21,9 @@ except ImportError:
 
 
 class TestTemplates(test_helpers.CiTestCase):
+
+    with_logs = True
+
     jinja_utf8 = b'It\xe2\x80\x99s not ascii, {{name}}\n'
     jinja_utf8_rbob = b'It\xe2\x80\x99s not ascii, bob\n'.decode('utf-8')
 
@@ -124,6 +127,13 @@ $a,$b'''
                 self.add_header("jinja", self.jinja_utf8), {"name": "bob"}),
             self.jinja_utf8_rbob)
 
+    def test_jinja_nonascii_render_undefined_variables_to_default_py3(self):
+        """Test py3 jinja render_to_string with undefined variable default."""
+        self.assertEqual(
+            templater.render_string(
+                self.add_header("jinja", self.jinja_utf8), {}),
+            self.jinja_utf8_rbob.replace('bob', 'CI_MISSING_JINJA_VAR/name'))
+
     def test_jinja_nonascii_render_to_file(self):
         """Test jinja render_to_file of a filename with non-ascii content."""
         tmpl_fn = self.tmp_path("j-render-to-file.template")
@@ -144,5 +154,18 @@ $a,$b'''
         result = templater.render_from_file(tmpl_fn, {"name": "bob"})
         self.assertEqual(result, self.jinja_utf8_rbob)
 
+    @test_helpers.skipIfJinja()
+    def test_jinja_warns_on_missing_dep_and_uses_basic_renderer(self):
+        """Test jinja render_from_file will fallback to basic renderer."""
+        tmpl_fn = self.tmp_path("j-render-from-file.template")
+        write_file(tmpl_fn, omode="wb",
+                   content=self.add_header(
+                       "jinja", self.jinja_utf8).encode('utf-8'))
+        result = templater.render_from_file(tmpl_fn, {"name": "bob"})
+        self.assertEqual(result, self.jinja_utf8.decode())
+        self.assertIn(
+            'WARNING: Jinja not available as the selected renderer for desired'
+            ' template, reverting to the basic renderer.',
+            self.logs.getvalue())
 
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From c75c582ed1824dc3ff39cefdbddccfc2924b868c Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 12 Sep 2018 18:44:19 +0000
Subject: OpenStack: fix bug causing 'latest' version to be used from network.

Cloud-init was reading a list of versions from the OpenStack metadata
service (http://169.254.169.254/openstack/) and attempt to select the
newest known supported version.  The problem was that the list
of versions was not being decoded, so we were comparing a list of
bytes (found versions) to a list of strings (known versions).

LP: #1792157
---
 cloudinit/sources/helpers/openstack.py            |  2 +-
 tests/unittests/test_datasource/test_openstack.py | 30 +++++++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 8f9c1441..e3d372b6 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -439,7 +439,7 @@ class MetadataReader(BaseReader):
             return self._versions
         found = []
         version_path = self._path_join(self.base_path, "openstack")
-        content = self._path_read(version_path)
+        content = self._path_read(version_path, decode=True)
         for line in content.splitlines():
             line = line.strip()
             if not line:
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index 6e1e971b..ab5d3adc 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -555,4 +555,34 @@ class TestDetectOpenStack(test_helpers.CiTestCase):
         m_proc_env.assert_called_with(1)
 
 
+class TestMetadataReader(test_helpers.HttprettyTestCase):
+    """Test the MetadataReader."""
+    burl = 'http://169.254.169.254/'
+
+    def register(self, path, body=None, status=200):
+        hp.register_uri(
+            hp.GET, self.burl + "openstack" + path, status=status, body=body)
+
+    def register_versions(self, versions):
+        self.register("", '\n'.join(versions).encode('utf-8'))
+
+    def test__find_working_version(self):
+        """Test a working version ignores unsupported."""
+        unsup = "2016-11-09"
+        self.register_versions(
+            [openstack.OS_FOLSOM, openstack.OS_LIBERTY, unsup,
+             openstack.OS_LATEST])
+        self.assertEqual(
+            openstack.OS_LIBERTY,
+            openstack.MetadataReader(self.burl)._find_working_version())
+
+    def test__find_working_version_uses_latest(self):
+        """'latest' should be used if no supported versions."""
+        unsup1, unsup2 = ("2016-11-09", '2017-06-06')
+        self.register_versions([unsup1, unsup2, openstack.OS_LATEST])
+        self.assertEqual(
+            openstack.OS_LATEST,
+            openstack.MetadataReader(self.burl)._find_working_version())
+
+
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From 4361e0e42a34ebf9f49dde2356b7261715cf0cd5 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Thu, 13 Sep 2018 18:44:20 +0000
Subject: OpenStack: support reading of newer versions of metdata.

Mark as supported for reading some newer versions of openstack metadata:
 2016-06-30 : Newton one
 2016-10-06 : Newton two
 2017-02-22 : Ocata
 2018-08-27 : Rocky
---
 cloudinit/sources/helpers/openstack.py            | 16 ++++++
 tests/unittests/test_datasource/test_openstack.py | 66 +++++++++++++++++++++--
 2 files changed, 79 insertions(+), 3 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index e3d372b6..1e651440 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -38,17 +38,33 @@ KEY_COPIES = (
     ('local-hostname', 'hostname', False),
     ('instance-id', 'uuid', True),
 )
+
+# Versions and names taken from nova source nova/api/metadata/base.py
 OS_LATEST = 'latest'
 OS_FOLSOM = '2012-08-10'
 OS_GRIZZLY = '2013-04-04'
 OS_HAVANA = '2013-10-17'
 OS_LIBERTY = '2015-10-15'
+# NEWTON_ONE adds 'devices' to md (sriov-pf-passthrough-neutron-port-vlan)
+OS_NEWTON_ONE = '2016-06-30'
+# NEWTON_TWO adds vendor_data2.json (vendordata-reboot)
+OS_NEWTON_TWO = '2016-10-06'
+# OS_OCATA adds 'vif' field to devices (sriov-pf-passthrough-neutron-port-vlan)
+OS_OCATA = '2017-02-22'
+# OS_ROCKY adds a vf_trusted field to devices (sriov-trusted-vfs)
+OS_ROCKY = '2018-08-27'
+
+
 # keep this in chronological order. new supported versions go at the end.
 OS_VERSIONS = (
     OS_FOLSOM,
     OS_GRIZZLY,
     OS_HAVANA,
     OS_LIBERTY,
+    OS_NEWTON_ONE,
+    OS_NEWTON_TWO,
+    OS_OCATA,
+    OS_ROCKY,
 )
 
 PHYSICAL_TYPES = (
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index ab5d3adc..a731f1ed 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -12,7 +12,7 @@ import re
 from cloudinit.tests import helpers as test_helpers
 
 from six.moves.urllib.parse import urlparse
-from six import StringIO
+from six import StringIO, text_type
 
 from cloudinit import helpers
 from cloudinit import settings
@@ -558,13 +558,36 @@ class TestDetectOpenStack(test_helpers.CiTestCase):
 class TestMetadataReader(test_helpers.HttprettyTestCase):
     """Test the MetadataReader."""
     burl = 'http://169.254.169.254/'
+    md_base = {
+        'availability_zone': 'myaz1',
+        'hostname': 'sm-foo-test.novalocal',
+        "keys": [{"data": PUBKEY, "name": "brickies", "type": "ssh"}],
+        'launch_index': 0,
+        'name': 'sm-foo-test',
+        'public_keys': {'mykey': PUBKEY},
+        'project_id': '6a103f813b774b9fb15a4fcd36e1c056',
+        'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c'}
 
     def register(self, path, body=None, status=200):
+        content = (body if not isinstance(body, text_type)
+                   else body.encode('utf-8'))
         hp.register_uri(
-            hp.GET, self.burl + "openstack" + path, status=status, body=body)
+            hp.GET, self.burl + "openstack" + path, status=status,
+            body=content)
 
     def register_versions(self, versions):
-        self.register("", '\n'.join(versions).encode('utf-8'))
+        self.register("", '\n'.join(versions))
+        self.register("/", '\n'.join(versions))
+
+    def register_version(self, version, data):
+        content = '\n'.join(sorted(data.keys()))
+        self.register(version, content)
+        self.register(version + "/", content)
+        for path, content in data.items():
+            self.register("/%s/%s" % (version, path), content)
+            self.register("/%s/%s" % (version, path), content)
+        if 'user_data' not in data:
+            self.register("/%s/user_data" % version, "nodata", status=404)
 
     def test__find_working_version(self):
         """Test a working version ignores unsupported."""
@@ -584,5 +607,42 @@ class TestMetadataReader(test_helpers.HttprettyTestCase):
             openstack.OS_LATEST,
             openstack.MetadataReader(self.burl)._find_working_version())
 
+    def test_read_v2_os_ocata(self):
+        """Validate return value of read_v2 for os_ocata data."""
+        md = copy.deepcopy(self.md_base)
+        md['devices'] = []
+        network_data = {'links': [], 'networks': [], 'services': []}
+        vendor_data = {}
+        vendor_data2 = {"static": {}}
+
+        data = {
+            'meta_data.json': json.dumps(md),
+            'network_data.json': json.dumps(network_data),
+            'vendor_data.json': json.dumps(vendor_data),
+            'vendor_data2.json': json.dumps(vendor_data2),
+        }
+
+        self.register_versions([openstack.OS_OCATA, openstack.OS_LATEST])
+        self.register_version(openstack.OS_OCATA, data)
+
+        mock_read_ec2 = test_helpers.mock.MagicMock(
+            return_value={'instance-id': 'unused-ec2'})
+        expected_md = copy.deepcopy(md)
+        expected_md.update(
+            {'instance-id': md['uuid'], 'local-hostname': md['hostname']})
+        expected = {
+            'userdata': '',  # Annoying, no user-data results in empty string.
+            'version': 2,
+            'metadata': expected_md,
+            'vendordata': vendor_data,
+            'networkdata': network_data,
+            'ec2-metadata': mock_read_ec2.return_value,
+            'files': {},
+        }
+        reader = openstack.MetadataReader(self.burl)
+        reader._read_ec2_metadata = mock_read_ec2
+        self.assertEqual(expected, reader.read_v2())
+        self.assertEqual(1, mock_read_ec2.call_count)
+
 
 # vi: ts=4 expandtab
-- 
cgit v1.2.3


From 12066fc8f6518d294501aa126b5332ea90c4896c Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Thu, 13 Sep 2018 22:15:39 +0000
Subject: EphemeralIPv4Network: Be more explicit when adding default route.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On OpenStack based OVH public cloud, we got DHCP response with
  fixed-address 54.36.113.86;
  option subnet-mask 255.255.255.255;
  option routers 54.36.112.1;

The router clearly is not on the subnet.  So 'ip' would fail when
we tried to add the default route.
The solution here is to add an explicit route on that interface
to the router and then add the default route.

Also add 'bgpovs' to the list of 'physical' types for OpenStack
network configuration.  That type is used on OVH public cloud.

LP: #1792415
---
 cloudinit/net/__init__.py              |  7 +++++++
 cloudinit/net/tests/test_init.py       | 11 ++++++++---
 cloudinit/sources/helpers/openstack.py |  1 +
 3 files changed, 16 insertions(+), 3 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 3ffde52c..5e87bcad 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -697,6 +697,13 @@ class EphemeralIPv4Network(object):
                 'Skip ephemeral route setup. %s already has default route: %s',
                 self.interface, out.strip())
             return
+        util.subp(
+            ['ip', '-4', 'route', 'add', self.router, 'dev', self.interface,
+             'src', self.ip], capture=True)
+        self.cleanup_cmds.insert(
+            0,
+            ['ip', '-4', 'route', 'del', self.router, 'dev', self.interface,
+             'src', self.ip])
         util.subp(
             ['ip', '-4', 'route', 'add', 'default', 'via', self.router,
              'dev', self.interface], capture=True)
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
index 8b444f14..58e0a591 100644
--- a/cloudinit/net/tests/test_init.py
+++ b/cloudinit/net/tests/test_init.py
@@ -515,12 +515,17 @@ class TestEphemeralIPV4Network(CiTestCase):
                 capture=True),
             mock.call(
                 ['ip', 'route', 'show', '0.0.0.0/0'], capture=True),
+            mock.call(['ip', '-4', 'route', 'add', '192.168.2.1',
+                       'dev', 'eth0', 'src', '192.168.2.2'], capture=True),
             mock.call(
                 ['ip', '-4', 'route', 'add', 'default', 'via',
                  '192.168.2.1', 'dev', 'eth0'], capture=True)]
-        expected_teardown_calls = [mock.call(
-            ['ip', '-4', 'route', 'del', 'default', 'dev', 'eth0'],
-            capture=True)]
+        expected_teardown_calls = [
+            mock.call(['ip', '-4', 'route', 'del', 'default', 'dev', 'eth0'],
+                      capture=True),
+            mock.call(['ip', '-4', 'route', 'del', '192.168.2.1',
+                       'dev', 'eth0', 'src', '192.168.2.2'], capture=True),
+        ]
 
         with net.EphemeralIPv4Network(**params):
             self.assertEqual(expected_setup_calls, m_subp.call_args_list)
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 1e651440..d6f39a1e 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -69,6 +69,7 @@ OS_VERSIONS = (
 
 PHYSICAL_TYPES = (
     None,
+    'bgpovs',  # not present in OpenStack upstream but used on OVH cloud.
     'bridge',
     'dvs',
     'ethernet',
-- 
cgit v1.2.3


From c6cfed7fb58be76a3f95963a5de469aa03542cd3 Mon Sep 17 00:00:00 2001
From: Fabian Wiesel <fabian.wiesel@sap.com>
Date: Fri, 14 Sep 2018 19:33:01 +0000
Subject: OpenStack: Support setting mac address on bond.

Fix a bug where setting of mac address on a bond device was
ignored when provided in OpenStack network_config.json.

LP: #1682064
---
 cloudinit/sources/helpers/openstack.py              | 2 ++
 tests/unittests/test_datasource/test_configdrive.py | 6 +++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index d6f39a1e..76a6e310 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -604,6 +604,8 @@ def convert_net_json(network_json=None, known_macs=None):
             cfg.update({'type': 'physical', 'mac_address': link_mac_addr})
         elif link['type'] in ['bond']:
             params = {}
+            if link_mac_addr:
+                params['mac_address'] = link_mac_addr
             for k, v in link.items():
                 if k == 'bond_links':
                     continue
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index b0abfc5f..231619c9 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -136,6 +136,7 @@ NETWORK_DATA_3 = {
     ]
 }
 
+BOND_MAC = "fa:16:3e:b3:72:36"
 NETWORK_DATA_BOND = {
     "services": [
         {"type": "dns", "address": "1.1.1.191"},
@@ -163,7 +164,7 @@ NETWORK_DATA_BOND = {
         {"bond_links": ["eth0", "eth1"],
          "bond_miimon": 100, "bond_mode": "4",
          "bond_xmit_hash_policy": "layer3+4",
-         "ethernet_mac_address": "0c:c4:7a:34:6e:3c",
+         "ethernet_mac_address": BOND_MAC,
          "id": "bond0", "type": "bond"},
         {"ethernet_mac_address": "fa:16:3e:b3:72:30",
          "id": "vlan2", "type": "vlan", "vlan_id": 602,
@@ -691,6 +692,9 @@ class TestConvertNetworkData(CiTestCase):
         self.assertIn("auto oeth0", eni_rendering)
         self.assertIn("auto oeth1", eni_rendering)
         self.assertIn("auto bond0", eni_rendering)
+        # The bond should have the given mac address
+        pos = eni_rendering.find("auto bond0")
+        self.assertIn(BOND_MAC, eni_rendering[pos:])
 
     def test_vlan(self):
         # light testing of vlan config conversion and eni rendering
-- 
cgit v1.2.3


From c3680475f9c97028d75a8f1bb58139ddd47d00ed Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Thu, 20 Sep 2018 09:08:37 +0000
Subject: Remove dead-code _write_network distro implementations.

Any distro that has a '_write_nework_config' method should no
longer get their _write_network called at all.  So lets drop
that code and raise a RuntimeError any time we got there.

Replace the one caller of 'apply_network' (legacy openstack path)
with a call to apply_network_config after converting the ENI to
network config.
---
 cloudinit/distros/__init__.py                  |   7 +-
 cloudinit/distros/debian.py                    |   5 --
 cloudinit/distros/opensuse.py                  |  47 -----------
 cloudinit/distros/rhel.py                      |  49 -----------
 cloudinit/sources/DataSourceConfigDrive.py     |   2 +-
 tests/unittests/test_distros/test_netconfig.py | 111 -------------------------
 6 files changed, 4 insertions(+), 217 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index b8a48e85..ac150bef 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -74,11 +74,10 @@ class Distro(object):
     def install_packages(self, pkglist):
         raise NotImplementedError()
 
-    @abc.abstractmethod
     def _write_network(self, settings):
-        # In the future use the http://fedorahosted.org/netcf/
-        # to write this blob out in a distro format
-        raise NotImplementedError()
+        raise RuntimeError(
+            "Legacy function '_write_network' was called in distro '%s'.\n"
+            "_write_network_config needs implementation.\n" % self.name)
 
     def _write_network_config(self, settings):
         raise NotImplementedError()
diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py
index 33cc0bf1..d517fb88 100644
--- a/cloudinit/distros/debian.py
+++ b/cloudinit/distros/debian.py
@@ -109,11 +109,6 @@ class Distro(distros.Distro):
         self.update_package_sources()
         self.package_command('install', pkgs=pkglist)
 
-    def _write_network(self, settings):
-        # this is a legacy method, it will always write eni
-        util.write_file(self.network_conf_fn["eni"], settings)
-        return ['all']
-
     def _write_network_config(self, netconfig):
         _maybe_remove_legacy_eth0()
         return self._supported_write_network_config(netconfig)
diff --git a/cloudinit/distros/opensuse.py b/cloudinit/distros/opensuse.py
index 1fe896aa..1bfe0478 100644
--- a/cloudinit/distros/opensuse.py
+++ b/cloudinit/distros/opensuse.py
@@ -16,7 +16,6 @@ from cloudinit import helpers
 from cloudinit import log as logging
 from cloudinit import util
 
-from cloudinit.distros import net_util
 from cloudinit.distros import rhel_util as rhutil
 from cloudinit.settings import PER_INSTANCE
 
@@ -172,52 +171,6 @@ class Distro(distros.Distro):
             conf.set_hostname(hostname)
             util.write_file(out_fn, str(conf), 0o644)
 
-    def _write_network(self, settings):
-        # Convert debian settings to ifcfg format
-        entries = net_util.translate_network(settings)
-        LOG.debug("Translated ubuntu style network settings %s into %s",
-                  settings, entries)
-        # Make the intermediate format as the suse format...
-        nameservers = []
-        searchservers = []
-        dev_names = entries.keys()
-        for (dev, info) in entries.items():
-            net_fn = self.network_script_tpl % (dev)
-            route_fn = self.route_conf_tpl % (dev)
-            mode = None
-            if info.get('auto', None):
-                mode = 'auto'
-            else:
-                mode = 'manual'
-            bootproto = info.get('bootproto', None)
-            gateway = info.get('gateway', None)
-            net_cfg = {
-                'BOOTPROTO': bootproto,
-                'BROADCAST': info.get('broadcast'),
-                'GATEWAY': gateway,
-                'IPADDR': info.get('address'),
-                'LLADDR': info.get('hwaddress'),
-                'NETMASK': info.get('netmask'),
-                'STARTMODE': mode,
-                'USERCONTROL': 'no'
-            }
-            if dev != 'lo':
-                net_cfg['ETHTOOL_OPTIONS'] = ''
-            else:
-                net_cfg['FIREWALL'] = 'no'
-            rhutil.update_sysconfig_file(net_fn, net_cfg, True)
-            if gateway and bootproto == 'static':
-                default_route = 'default    %s' % gateway
-                util.write_file(route_fn, default_route, 0o644)
-            if 'dns-nameservers' in info:
-                nameservers.extend(info['dns-nameservers'])
-            if 'dns-search' in info:
-                searchservers.extend(info['dns-search'])
-        if nameservers or searchservers:
-            rhutil.update_resolve_conf_file(self.resolve_conf_fn,
-                                            nameservers, searchservers)
-        return dev_names
-
     def _write_network_config(self, netconfig):
         return self._supported_write_network_config(netconfig)
 
diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py
index ff513438..f55d96f7 100644
--- a/cloudinit/distros/rhel.py
+++ b/cloudinit/distros/rhel.py
@@ -13,7 +13,6 @@ from cloudinit import helpers
 from cloudinit import log as logging
 from cloudinit import util
 
-from cloudinit.distros import net_util
 from cloudinit.distros import rhel_util
 from cloudinit.settings import PER_INSTANCE
 
@@ -65,54 +64,6 @@ class Distro(distros.Distro):
     def _write_network_config(self, netconfig):
         return self._supported_write_network_config(netconfig)
 
-    def _write_network(self, settings):
-        # TODO(harlowja) fix this... since this is the ubuntu format
-        entries = net_util.translate_network(settings)
-        LOG.debug("Translated ubuntu style network settings %s into %s",
-                  settings, entries)
-        # Make the intermediate format as the rhel format...
-        nameservers = []
-        searchservers = []
-        dev_names = entries.keys()
-        use_ipv6 = False
-        for (dev, info) in entries.items():
-            net_fn = self.network_script_tpl % (dev)
-            net_cfg = {
-                'DEVICE': dev,
-                'NETMASK': info.get('netmask'),
-                'IPADDR': info.get('address'),
-                'BOOTPROTO': info.get('bootproto'),
-                'GATEWAY': info.get('gateway'),
-                'BROADCAST': info.get('broadcast'),
-                'MACADDR': info.get('hwaddress'),
-                'ONBOOT': _make_sysconfig_bool(info.get('auto')),
-            }
-            if info.get('inet6'):
-                use_ipv6 = True
-                net_cfg.update({
-                    'IPV6INIT': _make_sysconfig_bool(True),
-                    'IPV6ADDR': info.get('ipv6').get('address'),
-                    'IPV6_DEFAULTGW': info.get('ipv6').get('gateway'),
-                })
-            rhel_util.update_sysconfig_file(net_fn, net_cfg)
-            if 'dns-nameservers' in info:
-                nameservers.extend(info['dns-nameservers'])
-            if 'dns-search' in info:
-                searchservers.extend(info['dns-search'])
-        if nameservers or searchservers:
-            rhel_util.update_resolve_conf_file(self.resolve_conf_fn,
-                                               nameservers, searchservers)
-        if dev_names:
-            net_cfg = {
-                'NETWORKING': _make_sysconfig_bool(True),
-            }
-            # If IPv6 interface present, enable ipv6 networking
-            if use_ipv6:
-                net_cfg['NETWORKING_IPV6'] = _make_sysconfig_bool(True)
-                net_cfg['IPV6_AUTOCONF'] = _make_sysconfig_bool(False)
-            rhel_util.update_sysconfig_file(self.network_conf_fn, net_cfg)
-        return dev_names
-
     def apply_locale(self, locale, out_fn=None):
         if self.uses_systemd():
             if not out_fn:
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 4cb28977..664dc4b7 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -196,7 +196,7 @@ def on_first_boot(data, distro=None, network=True):
         net_conf = data.get("network_config", '')
         if net_conf and distro:
             LOG.warning("Updating network interfaces from config drive")
-            distro.apply_network(net_conf)
+            distro.apply_network_config(eni.convert_eni_data(net_conf))
     write_injected_files(data.get('files'))
 
 
diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py
index 9bbff451..6e339355 100644
--- a/tests/unittests/test_distros/test_netconfig.py
+++ b/tests/unittests/test_distros/test_netconfig.py
@@ -367,16 +367,6 @@ class TestNetCfgDistroUbuntuEni(TestNetCfgDistroBase):
             self.assertEqual(expected, results[cfgpath])
             self.assertEqual(0o644, get_mode(cfgpath, tmpd))
 
-    def test_simple_write_ub(self):
-        expected_cfgs = {
-            self.eni_path(): BASE_NET_CFG,
-        }
-
-        # ub_distro.apply_network(BASE_NET_CFG, False)
-        self._apply_and_verify_eni(self.distro.apply_network,
-                                   BASE_NET_CFG,
-                                   expected_cfgs=expected_cfgs.copy())
-
     def test_apply_network_config_eni_ub(self):
         expected_cfgs = {
             self.eni_path(): V1_NET_CFG_OUTPUT,
@@ -467,35 +457,6 @@ class TestNetCfgDistroRedhat(TestNetCfgDistroBase):
             self.assertCfgEquals(expected, results[cfgpath])
             self.assertEqual(0o644, get_mode(cfgpath, tmpd))
 
-    def test_simple_write_rh(self):
-        expected_cfgs = {
-            self.ifcfg_path('lo'): dedent("""\
-                DEVICE="lo"
-                ONBOOT=yes
-                """),
-            self.ifcfg_path('eth0'): dedent("""\
-                DEVICE="eth0"
-                BOOTPROTO="static"
-                NETMASK="255.255.255.0"
-                IPADDR="192.168.1.5"
-                ONBOOT=yes
-                GATEWAY="192.168.1.254"
-                BROADCAST="192.168.1.0"
-                """),
-            self.ifcfg_path('eth1'): dedent("""\
-                DEVICE="eth1"
-                BOOTPROTO="dhcp"
-                ONBOOT=yes
-                """),
-            self.control_path(): dedent("""\
-                NETWORKING=yes
-                """),
-        }
-        # rh_distro.apply_network(BASE_NET_CFG, False)
-        self._apply_and_verify(self.distro.apply_network,
-                               BASE_NET_CFG,
-                               expected_cfgs=expected_cfgs.copy())
-
     def test_apply_network_config_rh(self):
         expected_cfgs = {
             self.ifcfg_path('eth0'): dedent("""\
@@ -527,47 +488,6 @@ class TestNetCfgDistroRedhat(TestNetCfgDistroBase):
                                V1_NET_CFG,
                                expected_cfgs=expected_cfgs.copy())
 
-    def test_write_ipv6_rhel(self):
-        expected_cfgs = {
-            self.ifcfg_path('lo'): dedent("""\
-                DEVICE="lo"
-                ONBOOT=yes
-                """),
-            self.ifcfg_path('eth0'): dedent("""\
-                DEVICE="eth0"
-                BOOTPROTO="static"
-                NETMASK="255.255.255.0"
-                IPADDR="192.168.1.5"
-                ONBOOT=yes
-                GATEWAY="192.168.1.254"
-                BROADCAST="192.168.1.0"
-                IPV6INIT=yes
-                IPV6ADDR="2607:f0d0:1002:0011::2"
-                IPV6_DEFAULTGW="2607:f0d0:1002:0011::1"
-                """),
-            self.ifcfg_path('eth1'): dedent("""\
-                DEVICE="eth1"
-                BOOTPROTO="static"
-                NETMASK="255.255.255.0"
-                IPADDR="192.168.1.6"
-                ONBOOT=no
-                GATEWAY="192.168.1.254"
-                BROADCAST="192.168.1.0"
-                IPV6INIT=yes
-                IPV6ADDR="2607:f0d0:1002:0011::3"
-                IPV6_DEFAULTGW="2607:f0d0:1002:0011::1"
-                """),
-            self.control_path(): dedent("""\
-                NETWORKING=yes
-                NETWORKING_IPV6=yes
-                IPV6_AUTOCONF=no
-                """),
-        }
-        # rh_distro.apply_network(BASE_NET_CFG_IPV6, False)
-        self._apply_and_verify(self.distro.apply_network,
-                               BASE_NET_CFG_IPV6,
-                               expected_cfgs=expected_cfgs.copy())
-
     def test_apply_network_config_ipv6_rh(self):
         expected_cfgs = {
             self.ifcfg_path('eth0'): dedent("""\
@@ -627,37 +547,6 @@ class TestNetCfgDistroOpensuse(TestNetCfgDistroBase):
             self.assertCfgEquals(expected, results[cfgpath])
             self.assertEqual(0o644, get_mode(cfgpath, tmpd))
 
-    def test_simple_write_opensuse(self):
-        """Opensuse network rendering writes appropriate sysconfig files."""
-        expected_cfgs = {
-            self.ifcfg_path('lo'): dedent('''
-                STARTMODE="auto"
-                USERCONTROL="no"
-                FIREWALL="no"
-                '''),
-            self.ifcfg_path('eth0'): dedent('''
-                BOOTPROTO="static"
-                BROADCAST="192.168.1.0"
-                GATEWAY="192.168.1.254"
-                IPADDR="192.168.1.5"
-                NETMASK="255.255.255.0"
-                STARTMODE="auto"
-                USERCONTROL="no"
-                ETHTOOL_OPTIONS=""
-                '''),
-            self.ifcfg_path('eth1'): dedent('''
-                BOOTPROTO="dhcp"
-                STARTMODE="auto"
-                USERCONTROL="no"
-                ETHTOOL_OPTIONS=""
-                ''')
-        }
-
-        # distro.apply_network(BASE_NET_CFG, False)
-        self._apply_and_verify(self.distro.apply_network,
-                               BASE_NET_CFG,
-                               expected_cfgs=expected_cfgs.copy())
-
     def test_apply_network_config_opensuse(self):
         """Opensuse uses apply_network_config and renders sysconfig"""
         expected_cfgs = {
-- 
cgit v1.2.3


From fc4b966ba928b30b1c586407e752e0b51b1031e8 Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Tue, 25 Sep 2018 21:59:16 +0000
Subject: cli: add cloud-init query subcommand to query instance metadata
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Cloud-init caches any cloud metadata crawled during boot in the file
/run/cloud-init/instance-data.json. Cloud-init also standardizes some of
that metadata across all clouds. The command 'cloud-init query' surfaces a
simple CLI to query or format any cached instance metadata so that scripts
or end-users do not have to write tools to crawl metadata themselves.

Since 'cloud-init query' is runnable by non-root users, redact any
sensitive data from instance-data.json and provide a root-readable
unredacted instance-data-sensitive.json. Datasources can now define a
sensitive_metadata_keys tuple which will redact any matching keys
which could contain passwords or credentials from instance-data.json.

Also add the following standardized 'v1' instance-data.json keys:
  - user_data: The base64encoded user-data provided at instance launch
  - vendor_data: Any vendor_data provided to the instance at launch
  - underscore_delimited versions of existing hyphenated keys:
    instance_id, local_hostname, availability_zone, cloud_name
---
 bash_completion/cloud-init           |   4 +-
 cloudinit/cmd/devel/render.py        |   7 +-
 cloudinit/cmd/main.py                |  10 ++
 cloudinit/cmd/query.py               | 155 ++++++++++++++++++
 cloudinit/cmd/tests/test_query.py    | 193 +++++++++++++++++++++++
 cloudinit/helpers.py                 |   4 +
 cloudinit/sources/__init__.py        |  76 +++++++--
 cloudinit/sources/tests/test_init.py | 130 ++++++++++++---
 doc/rtd/index.rst                    |   1 +
 doc/rtd/topics/capabilities.rst      | 105 ++++++++++---
 doc/rtd/topics/datasources.rst       | 148 +----------------
 doc/rtd/topics/instancedata.rst      | 297 +++++++++++++++++++++++++++++++++++
 integration-requirements.txt         |   3 +-
 tests/cloud_tests/testcases/base.py  |  52 +++---
 14 files changed, 952 insertions(+), 233 deletions(-)
 create mode 100644 cloudinit/cmd/query.py
 create mode 100644 cloudinit/cmd/tests/test_query.py
 create mode 100644 doc/rtd/topics/instancedata.rst

(limited to 'cloudinit/sources')

diff --git a/bash_completion/cloud-init b/bash_completion/cloud-init
index 6d01bf3a..8c25032f 100644
--- a/bash_completion/cloud-init
+++ b/bash_completion/cloud-init
@@ -10,7 +10,7 @@ _cloudinit_complete()
     cur_word="${COMP_WORDS[COMP_CWORD]}"
     prev_word="${COMP_WORDS[COMP_CWORD-1]}"
 
-    subcmds="analyze clean collect-logs devel dhclient-hook features init modules single status"
+    subcmds="analyze clean collect-logs devel dhclient-hook features init modules query single status"
     base_params="--help --file --version --debug --force"
     case ${COMP_CWORD} in
         1)
@@ -40,6 +40,8 @@ _cloudinit_complete()
                     COMPREPLY=($(compgen -W "--help --mode" -- $cur_word))
                     ;;
 
+                query)
+                    COMPREPLY=($(compgen -W "--all --help --instance-data --list-keys --user-data --vendor-data --debug" -- $cur_word));;
                 single)
                     COMPREPLY=($(compgen -W "--help --name --frequency --report" -- $cur_word))
                     ;;
diff --git a/cloudinit/cmd/devel/render.py b/cloudinit/cmd/devel/render.py
index e85933db..2ba6b681 100755
--- a/cloudinit/cmd/devel/render.py
+++ b/cloudinit/cmd/devel/render.py
@@ -9,7 +9,6 @@ import sys
 from cloudinit.handlers.jinja_template import render_jinja_payload_from_file
 from cloudinit import log
 from cloudinit.sources import INSTANCE_JSON_FILE
-from cloudinit import util
 from . import addLogHandlerCLI, read_cfg_paths
 
 NAME = 'render'
@@ -54,11 +53,7 @@ def handle_args(name, args):
             paths.run_dir, INSTANCE_JSON_FILE)
     else:
         instance_data_fn = args.instance_data
-    try:
-        with open(instance_data_fn) as stream:
-            instance_data = stream.read()
-        instance_data = util.load_json(instance_data)
-    except IOError:
+    if not os.path.exists(instance_data_fn):
         LOG.error('Missing instance-data.json file: %s', instance_data_fn)
         return 1
     try:
diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index 0eee583c..5a437020 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -791,6 +791,10 @@ def main(sysv_args=None):
                                      ' pass to this module'))
     parser_single.set_defaults(action=('single', main_single))
 
+    parser_query = subparsers.add_parser(
+        'query',
+        help='Query standardized instance metadata from the command line.')
+
     parser_dhclient = subparsers.add_parser('dhclient-hook',
                                             help=('run the dhclient hook'
                                                   'to record network info'))
@@ -842,6 +846,12 @@ def main(sysv_args=None):
             clean_parser(parser_clean)
             parser_clean.set_defaults(
                 action=('clean', handle_clean_args))
+        elif sysv_args[0] == 'query':
+            from cloudinit.cmd.query import (
+                get_parser as query_parser, handle_args as handle_query_args)
+            query_parser(parser_query)
+            parser_query.set_defaults(
+                action=('render', handle_query_args))
         elif sysv_args[0] == 'status':
             from cloudinit.cmd.status import (
                 get_parser as status_parser, handle_status_args)
diff --git a/cloudinit/cmd/query.py b/cloudinit/cmd/query.py
new file mode 100644
index 00000000..7d2d4fe4
--- /dev/null
+++ b/cloudinit/cmd/query.py
@@ -0,0 +1,155 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Query standardized instance metadata from the command line."""
+
+import argparse
+import os
+import six
+import sys
+
+from cloudinit.handlers.jinja_template import (
+    convert_jinja_instance_data, render_jinja_payload)
+from cloudinit.cmd.devel import addLogHandlerCLI, read_cfg_paths
+from cloudinit import log
+from cloudinit.sources import (
+    INSTANCE_JSON_FILE, INSTANCE_JSON_SENSITIVE_FILE, REDACT_SENSITIVE_VALUE)
+from cloudinit import util
+
+NAME = 'query'
+LOG = log.getLogger(NAME)
+
+
+def get_parser(parser=None):
+    """Build or extend an arg parser for query utility.
+
+    @param parser: Optional existing ArgumentParser instance representing the
+        query subcommand which will be extended to support the args of
+        this utility.
+
+    @returns: ArgumentParser with proper argument configuration.
+    """
+    if not parser:
+        parser = argparse.ArgumentParser(
+            prog=NAME, description='Query cloud-init instance data')
+    parser.add_argument(
+        '-d', '--debug', action='store_true', default=False,
+        help='Add verbose messages during template render')
+    parser.add_argument(
+        '-i', '--instance-data', type=str,
+        help=('Path to instance-data.json file. Default is /run/cloud-init/%s'
+              % INSTANCE_JSON_FILE))
+    parser.add_argument(
+        '-l', '--list-keys', action='store_true', default=False,
+        help=('List query keys available at the provided instance-data'
+              ' <varname>.'))
+    parser.add_argument(
+        '-u', '--user-data', type=str,
+        help=('Path to user-data file. Default is'
+              ' /var/lib/cloud/instance/user-data.txt'))
+    parser.add_argument(
+        '-v', '--vendor-data', type=str,
+        help=('Path to vendor-data file. Default is'
+              ' /var/lib/cloud/instance/vendor-data.txt'))
+    parser.add_argument(
+        'varname', type=str, nargs='?',
+        help=('A dot-delimited instance data variable to query from'
+              ' instance-data query. For example: v2.local_hostname'))
+    parser.add_argument(
+        '-a', '--all', action='store_true', default=False, dest='dump_all',
+        help='Dump all available instance-data')
+    parser.add_argument(
+        '-f', '--format', type=str, dest='format',
+        help=('Optionally specify a custom output format string. Any'
+              ' instance-data variable can be specified between double-curly'
+              ' braces. For example -f "{{ v2.cloud_name }}"'))
+    return parser
+
+
+def handle_args(name, args):
+    """Handle calls to 'cloud-init query' as a subcommand."""
+    paths = None
+    addLogHandlerCLI(LOG, log.DEBUG if args.debug else log.WARNING)
+    if not any([args.list_keys, args.varname, args.format, args.dump_all]):
+        LOG.error(
+            'Expected one of the options: --all, --format,'
+            ' --list-keys or varname')
+        get_parser().print_help()
+        return 1
+
+    uid = os.getuid()
+    if not all([args.instance_data, args.user_data, args.vendor_data]):
+        paths = read_cfg_paths()
+    if not args.instance_data:
+        if uid == 0:
+            default_json_fn = INSTANCE_JSON_SENSITIVE_FILE
+        else:
+            default_json_fn = INSTANCE_JSON_FILE  # World readable
+        instance_data_fn = os.path.join(paths.run_dir, default_json_fn)
+    else:
+        instance_data_fn = args.instance_data
+    if not args.user_data:
+        user_data_fn = os.path.join(paths.instance_link, 'user-data.txt')
+    else:
+        user_data_fn = args.user_data
+    if not args.vendor_data:
+        vendor_data_fn = os.path.join(paths.instance_link, 'vendor-data.txt')
+    else:
+        vendor_data_fn = args.vendor_data
+
+    try:
+        instance_json = util.load_file(instance_data_fn)
+    except IOError:
+        LOG.error('Missing instance-data.json file: %s', instance_data_fn)
+        return 1
+
+    instance_data = util.load_json(instance_json)
+    if uid != 0:
+        instance_data['userdata'] = (
+            '<%s> file:%s' % (REDACT_SENSITIVE_VALUE, user_data_fn))
+        instance_data['vendordata'] = (
+            '<%s> file:%s' % (REDACT_SENSITIVE_VALUE, vendor_data_fn))
+    else:
+        instance_data['userdata'] = util.load_file(user_data_fn)
+        instance_data['vendordata'] = util.load_file(vendor_data_fn)
+    if args.format:
+        payload = '## template: jinja\n{fmt}'.format(fmt=args.format)
+        rendered_payload = render_jinja_payload(
+            payload=payload, payload_fn='query commandline',
+            instance_data=instance_data,
+            debug=True if args.debug else False)
+        if rendered_payload:
+            print(rendered_payload)
+            return 0
+        return 1
+
+    response = convert_jinja_instance_data(instance_data)
+    if args.varname:
+        try:
+            for var in args.varname.split('.'):
+                response = response[var]
+        except KeyError:
+            LOG.error('Undefined instance-data key %s', args.varname)
+            return 1
+        if args.list_keys:
+            if not isinstance(response, dict):
+                LOG.error("--list-keys provided but '%s' is not a dict", var)
+                return 1
+            response = '\n'.join(sorted(response.keys()))
+    elif args.list_keys:
+        response = '\n'.join(sorted(response.keys()))
+    if not isinstance(response, six.string_types):
+        response = util.json_dumps(response)
+    print(response)
+    return 0
+
+
+def main():
+    """Tool to query specific instance-data values."""
+    parser = get_parser()
+    sys.exit(handle_args(NAME, parser.parse_args()))
+
+
+if __name__ == '__main__':
+    main()
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/tests/test_query.py b/cloudinit/cmd/tests/test_query.py
new file mode 100644
index 00000000..fb87c6ab
--- /dev/null
+++ b/cloudinit/cmd/tests/test_query.py
@@ -0,0 +1,193 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from six import StringIO
+from textwrap import dedent
+import os
+
+from collections import namedtuple
+from cloudinit.cmd import query
+from cloudinit.helpers import Paths
+from cloudinit.sources import REDACT_SENSITIVE_VALUE, INSTANCE_JSON_FILE
+from cloudinit.tests.helpers import CiTestCase, mock
+from cloudinit.util import ensure_dir, write_file
+
+
+class TestQuery(CiTestCase):
+
+    with_logs = True
+
+    args = namedtuple(
+        'queryargs',
+        ('debug dump_all format instance_data list_keys user_data vendor_data'
+         ' varname'))
+
+    def setUp(self):
+        super(TestQuery, self).setUp()
+        self.tmp = self.tmp_dir()
+        self.instance_data = self.tmp_path('instance-data', dir=self.tmp)
+
+    def test_handle_args_error_on_missing_param(self):
+        """Error when missing required parameters and print usage."""
+        args = self.args(
+            debug=False, dump_all=False, format=None, instance_data=None,
+            list_keys=False, user_data=None, vendor_data=None, varname=None)
+        with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
+            with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+                self.assertEqual(1, query.handle_args('anyname', args))
+        expected_error = (
+            'ERROR: Expected one of the options: --all, --format, --list-keys'
+            ' or varname\n')
+        self.assertIn(expected_error, self.logs.getvalue())
+        self.assertIn('usage: query', m_stdout.getvalue())
+        self.assertIn(expected_error, m_stderr.getvalue())
+
+    def test_handle_args_error_on_missing_instance_data(self):
+        """When instance_data file path does not exist, log an error."""
+        absent_fn = self.tmp_path('absent', dir=self.tmp)
+        args = self.args(
+            debug=False, dump_all=True, format=None, instance_data=absent_fn,
+            list_keys=False, user_data='ud', vendor_data='vd', varname=None)
+        with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
+            self.assertEqual(1, query.handle_args('anyname', args))
+        self.assertIn(
+            'ERROR: Missing instance-data.json file: %s' % absent_fn,
+            self.logs.getvalue())
+        self.assertIn(
+            'ERROR: Missing instance-data.json file: %s' % absent_fn,
+            m_stderr.getvalue())
+
+    def test_handle_args_defaults_instance_data(self):
+        """When no instance_data argument, default to configured run_dir."""
+        args = self.args(
+            debug=False, dump_all=True, format=None, instance_data=None,
+            list_keys=False, user_data=None, vendor_data=None, varname=None)
+        run_dir = self.tmp_path('run_dir', dir=self.tmp)
+        ensure_dir(run_dir)
+        paths = Paths({'run_dir': run_dir})
+        self.add_patch('cloudinit.cmd.query.read_cfg_paths', 'm_paths')
+        self.m_paths.return_value = paths
+        with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
+            self.assertEqual(1, query.handle_args('anyname', args))
+        json_file = os.path.join(run_dir, INSTANCE_JSON_FILE)
+        self.assertIn(
+            'ERROR: Missing instance-data.json file: %s' % json_file,
+            self.logs.getvalue())
+        self.assertIn(
+            'ERROR: Missing instance-data.json file: %s' % json_file,
+            m_stderr.getvalue())
+
+    def test_handle_args_dumps_all_instance_data(self):
+        """When --all is specified query will dump all instance data vars."""
+        write_file(self.instance_data, '{"my-var": "it worked"}')
+        args = self.args(
+            debug=False, dump_all=True, format=None,
+            instance_data=self.instance_data, list_keys=False,
+            user_data='ud', vendor_data='vd', varname=None)
+        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+            self.assertEqual(0, query.handle_args('anyname', args))
+        self.assertEqual(
+            '{\n "my_var": "it worked",\n "userdata": "<%s> file:ud",\n'
+            ' "vendordata": "<%s> file:vd"\n}\n' % (
+                REDACT_SENSITIVE_VALUE, REDACT_SENSITIVE_VALUE),
+            m_stdout.getvalue())
+
+    def test_handle_args_returns_top_level_varname(self):
+        """When the argument varname is passed, report its value."""
+        write_file(self.instance_data, '{"my-var": "it worked"}')
+        args = self.args(
+            debug=False, dump_all=True, format=None,
+            instance_data=self.instance_data, list_keys=False,
+            user_data='ud', vendor_data='vd', varname='my_var')
+        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+            self.assertEqual(0, query.handle_args('anyname', args))
+        self.assertEqual('it worked\n', m_stdout.getvalue())
+
+    def test_handle_args_returns_nested_varname(self):
+        """If user_data file is a jinja template render instance-data vars."""
+        write_file(self.instance_data,
+                   '{"v1": {"key-2": "value-2"}, "my-var": "it worked"}')
+        args = self.args(
+            debug=False, dump_all=False, format=None,
+            instance_data=self.instance_data, user_data='ud', vendor_data='vd',
+            list_keys=False, varname='v1.key_2')
+        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+            self.assertEqual(0, query.handle_args('anyname', args))
+        self.assertEqual('value-2\n', m_stdout.getvalue())
+
+    def test_handle_args_returns_standardized_vars_to_top_level_aliases(self):
+        """Any standardized vars under v# are promoted as top-level aliases."""
+        write_file(
+            self.instance_data,
+            '{"v1": {"v1_1": "val1.1"}, "v2": {"v2_2": "val2.2"},'
+            ' "top": "gun"}')
+        expected = dedent("""\
+            {
+             "top": "gun",
+             "userdata": "<redacted for non-root user> file:ud",
+             "v1": {
+              "v1_1": "val1.1"
+             },
+             "v1_1": "val1.1",
+             "v2": {
+              "v2_2": "val2.2"
+             },
+             "v2_2": "val2.2",
+             "vendordata": "<redacted for non-root user> file:vd"
+            }
+        """)
+        args = self.args(
+            debug=False, dump_all=True, format=None,
+            instance_data=self.instance_data, user_data='ud', vendor_data='vd',
+            list_keys=False, varname=None)
+        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+            self.assertEqual(0, query.handle_args('anyname', args))
+        self.assertEqual(expected, m_stdout.getvalue())
+
+    def test_handle_args_list_keys_sorts_top_level_keys_when_no_varname(self):
+        """Sort all top-level keys when only --list-keys provided."""
+        write_file(
+            self.instance_data,
+            '{"v1": {"v1_1": "val1.1"}, "v2": {"v2_2": "val2.2"},'
+            ' "top": "gun"}')
+        expected = 'top\nuserdata\nv1\nv1_1\nv2\nv2_2\nvendordata\n'
+        args = self.args(
+            debug=False, dump_all=False, format=None,
+            instance_data=self.instance_data, list_keys=True, user_data='ud',
+            vendor_data='vd', varname=None)
+        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+            self.assertEqual(0, query.handle_args('anyname', args))
+        self.assertEqual(expected, m_stdout.getvalue())
+
+    def test_handle_args_list_keys_sorts_nested_keys_when_varname(self):
+        """Sort all nested keys of varname object when --list-keys provided."""
+        write_file(
+            self.instance_data,
+            '{"v1": {"v1_1": "val1.1", "v1_2": "val1.2"}, "v2":' +
+            ' {"v2_2": "val2.2"}, "top": "gun"}')
+        expected = 'v1_1\nv1_2\n'
+        args = self.args(
+            debug=False, dump_all=False, format=None,
+            instance_data=self.instance_data, list_keys=True,
+            user_data='ud', vendor_data='vd', varname='v1')
+        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+            self.assertEqual(0, query.handle_args('anyname', args))
+        self.assertEqual(expected, m_stdout.getvalue())
+
+    def test_handle_args_list_keys_errors_when_varname_is_not_a_dict(self):
+        """Raise an error when --list-keys and varname specify a non-list."""
+        write_file(
+            self.instance_data,
+            '{"v1": {"v1_1": "val1.1", "v1_2": "val1.2"}, "v2": ' +
+            '{"v2_2": "val2.2"}, "top": "gun"}')
+        expected_error = "ERROR: --list-keys provided but 'top' is not a dict"
+        args = self.args(
+            debug=False, dump_all=False, format=None,
+            instance_data=self.instance_data, list_keys=True, user_data='ud',
+            vendor_data='vd',  varname='top')
+        with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
+            with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+                self.assertEqual(1, query.handle_args('anyname', args))
+        self.assertEqual('', m_stdout.getvalue())
+        self.assertIn(expected_error, m_stderr.getvalue())
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 3cc1fb19..dcd2645e 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -239,6 +239,10 @@ class ConfigMerger(object):
             if cc_fn and os.path.isfile(cc_fn):
                 try:
                     i_cfgs.append(util.read_conf(cc_fn))
+                except PermissionError:
+                    LOG.debug(
+                        'Skipped loading cloud-config from %s due to'
+                        ' non-root.', cc_fn)
                 except Exception:
                     util.logexc(LOG, 'Failed loading of cloud-config from %s',
                                 cc_fn)
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index a775f1a8..730e8174 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -38,8 +38,12 @@ DEP_FILESYSTEM = "FILESYSTEM"
 DEP_NETWORK = "NETWORK"
 DS_PREFIX = 'DataSource'
 
-# File in which instance meta-data, user-data and vendor-data is written
+# File in which public available instance meta-data is written
+# security-sensitive key values are redacted from this world-readable file
 INSTANCE_JSON_FILE = 'instance-data.json'
+# security-sensitive key values are present in this root-readable file
+INSTANCE_JSON_SENSITIVE_FILE = 'instance-data-sensitive.json'
+REDACT_SENSITIVE_VALUE = 'redacted for non-root user'
 
 # Key which can be provide a cloud's official product name to cloud-init
 METADATA_CLOUD_NAME_KEY = 'cloud-name'
@@ -58,7 +62,7 @@ class InvalidMetaDataException(Exception):
     pass
 
 
-def process_instance_metadata(metadata, key_path=''):
+def process_instance_metadata(metadata, key_path='', sensitive_keys=()):
     """Process all instance metadata cleaning it up for persisting as json.
 
     Strip ci-b64 prefix and catalog any 'base64_encoded_keys' as a list
@@ -67,22 +71,46 @@ def process_instance_metadata(metadata, key_path=''):
     """
     md_copy = copy.deepcopy(metadata)
     md_copy['base64_encoded_keys'] = []
+    md_copy['sensitive_keys'] = []
     for key, val in metadata.items():
         if key_path:
             sub_key_path = key_path + '/' + key
         else:
             sub_key_path = key
+        if key in sensitive_keys or sub_key_path in sensitive_keys:
+            md_copy['sensitive_keys'].append(sub_key_path)
         if isinstance(val, str) and val.startswith('ci-b64:'):
             md_copy['base64_encoded_keys'].append(sub_key_path)
             md_copy[key] = val.replace('ci-b64:', '')
         if isinstance(val, dict):
-            return_val = process_instance_metadata(val, sub_key_path)
+            return_val = process_instance_metadata(
+                val, sub_key_path, sensitive_keys)
             md_copy['base64_encoded_keys'].extend(
                 return_val.pop('base64_encoded_keys'))
+            md_copy['sensitive_keys'].extend(
+                return_val.pop('sensitive_keys'))
             md_copy[key] = return_val
     return md_copy
 
 
+def redact_sensitive_keys(metadata, redact_value=REDACT_SENSITIVE_VALUE):
+    """Redact any sensitive keys from to provided metadata dictionary.
+
+    Replace any keys values listed in 'sensitive_keys' with redact_value.
+    """
+    if not metadata.get('sensitive_keys', []):
+        return metadata
+    md_copy = copy.deepcopy(metadata)
+    for key_path in metadata.get('sensitive_keys'):
+        path_parts = key_path.split('/')
+        obj = md_copy
+        for path in path_parts:
+            if isinstance(obj[path], dict) and path != path_parts[-1]:
+                obj = obj[path]
+        obj[path] = redact_value
+    return md_copy
+
+
 URLParams = namedtuple(
     'URLParms', ['max_wait_seconds', 'timeout_seconds', 'num_retries'])
 
@@ -127,6 +155,10 @@ class DataSource(object):
 
     _dirty_cache = False
 
+    # N-tuple of keypaths or keynames redact from instance-data.json for
+    # non-root users
+    sensitive_metadata_keys = ('security-credentials',)
+
     def __init__(self, sys_cfg, distro, paths, ud_proc=None):
         self.sys_cfg = sys_cfg
         self.distro = distro
@@ -152,12 +184,24 @@ class DataSource(object):
 
     def _get_standardized_metadata(self):
         """Return a dictionary of standardized metadata keys."""
-        return {'v1': {
-            'local-hostname': self.get_hostname(),
-            'instance-id': self.get_instance_id(),
-            'cloud-name': self.cloud_name,
-            'region': self.region,
-            'availability-zone': self.availability_zone}}
+        local_hostname = self.get_hostname()
+        instance_id = self.get_instance_id()
+        availability_zone = self.availability_zone
+        cloud_name = self.cloud_name
+        # When adding new standard keys prefer underscore-delimited instead
+        # of hyphen-delimted to support simple variable references in jinja
+        # templates.
+        return {
+            'v1': {
+                'availability-zone': availability_zone,
+                'availability_zone': availability_zone,
+                'cloud-name': cloud_name,
+                'cloud_name': cloud_name,
+                'instance-id': instance_id,
+                'instance_id': instance_id,
+                'local-hostname': local_hostname,
+                'local_hostname': local_hostname,
+                'region': self.region}}
 
     def clear_cached_attrs(self, attr_defaults=()):
         """Reset any cached metadata attributes to datasource defaults.
@@ -200,9 +244,7 @@ class DataSource(object):
         """
         instance_data = {
             'ds': {
-                'meta_data': self.metadata,
-                'user_data': self.get_userdata_raw(),
-                'vendor_data': self.get_vendordata_raw()}}
+                'meta_data': self.metadata}}
         if hasattr(self, 'network_json'):
             network_json = getattr(self, 'network_json')
             if network_json != UNSET:
@@ -217,7 +259,9 @@ class DataSource(object):
             # Process content base64encoding unserializable values
             content = util.json_dumps(instance_data)
             # Strip base64: prefix and set base64_encoded_keys list.
-            processed_data = process_instance_metadata(json.loads(content))
+            processed_data = process_instance_metadata(
+                json.loads(content),
+                sensitive_keys=self.sensitive_metadata_keys)
         except TypeError as e:
             LOG.warning('Error persisting instance-data.json: %s', str(e))
             return False
@@ -225,7 +269,11 @@ class DataSource(object):
             LOG.warning('Error persisting instance-data.json: %s', str(e))
             return False
         json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE)
-        write_json(json_file, processed_data, mode=0o600)
+        write_json(json_file, processed_data)  # World readable
+        json_sensitive_file = os.path.join(self.paths.run_dir,
+                                           INSTANCE_JSON_SENSITIVE_FILE)
+        write_json(json_sensitive_file,
+                   redact_sensitive_keys(processed_data), mode=0o600)
         return True
 
     def _get_data(self):
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index 8299af23..6b965750 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -1,5 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
+import copy
 import inspect
 import os
 import six
@@ -9,7 +10,8 @@ from cloudinit.event import EventType
 from cloudinit.helpers import Paths
 from cloudinit import importer
 from cloudinit.sources import (
-    INSTANCE_JSON_FILE, DataSource, UNSET)
+    INSTANCE_JSON_FILE, INSTANCE_JSON_SENSITIVE_FILE, REDACT_SENSITIVE_VALUE,
+    UNSET, DataSource, redact_sensitive_keys)
 from cloudinit.tests.helpers import CiTestCase, skipIf, mock
 from cloudinit.user_data import UserDataProcessor
 from cloudinit import util
@@ -20,20 +22,24 @@ class DataSourceTestSubclassNet(DataSource):
     dsname = 'MyTestSubclass'
     url_max_wait = 55
 
-    def __init__(self, sys_cfg, distro, paths, custom_userdata=None,
-                 get_data_retval=True):
+    def __init__(self, sys_cfg, distro, paths, custom_metadata=None,
+                 custom_userdata=None, get_data_retval=True):
         super(DataSourceTestSubclassNet, self).__init__(
             sys_cfg, distro, paths)
         self._custom_userdata = custom_userdata
+        self._custom_metadata = custom_metadata
         self._get_data_retval = get_data_retval
 
     def _get_cloud_name(self):
         return 'SubclassCloudName'
 
     def _get_data(self):
-        self.metadata = {'availability_zone': 'myaz',
-                         'local-hostname': 'test-subclass-hostname',
-                         'region': 'myregion'}
+        if self._custom_metadata:
+            self.metadata = self._custom_metadata
+        else:
+            self.metadata = {'availability_zone': 'myaz',
+                             'local-hostname': 'test-subclass-hostname',
+                             'region': 'myregion'}
         if self._custom_userdata:
             self.userdata_raw = self._custom_userdata
         else:
@@ -278,7 +284,7 @@ class TestDataSource(CiTestCase):
             os.path.exists(json_file), 'Found unexpected file %s' % json_file)
 
     def test_get_data_writes_json_instance_data_on_success(self):
-        """get_data writes INSTANCE_JSON_FILE to run_dir as readonly root."""
+        """get_data writes INSTANCE_JSON_FILE to run_dir as world readable."""
         tmp = self.tmp_dir()
         datasource = DataSourceTestSubclassNet(
             self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
@@ -287,40 +293,90 @@ class TestDataSource(CiTestCase):
         content = util.load_file(json_file)
         expected = {
             'base64_encoded_keys': [],
+            'sensitive_keys': [],
             'v1': {
                 'availability-zone': 'myaz',
+                'availability_zone': 'myaz',
                 'cloud-name': 'subclasscloudname',
+                'cloud_name': 'subclasscloudname',
                 'instance-id': 'iid-datasource',
+                'instance_id': 'iid-datasource',
                 'local-hostname': 'test-subclass-hostname',
+                'local_hostname': 'test-subclass-hostname',
                 'region': 'myregion'},
             'ds': {
                 'meta_data': {'availability_zone': 'myaz',
                               'local-hostname': 'test-subclass-hostname',
-                              'region': 'myregion'},
-                'user_data': 'userdata_raw',
-                'vendor_data': 'vendordata_raw'}}
-        self.maxDiff = None
+                              'region': 'myregion'}}}
         self.assertEqual(expected, util.load_json(content))
         file_stat = os.stat(json_file)
+        self.assertEqual(0o644, stat.S_IMODE(file_stat.st_mode))
+        self.assertEqual(expected, util.load_json(content))
+
+    def test_get_data_writes_json_instance_data_sensitive(self):
+        """get_data writes INSTANCE_JSON_SENSITIVE_FILE as readonly root."""
+        tmp = self.tmp_dir()
+        datasource = DataSourceTestSubclassNet(
+            self.sys_cfg, self.distro, Paths({'run_dir': tmp}),
+            custom_metadata={
+                'availability_zone': 'myaz',
+                'local-hostname': 'test-subclass-hostname',
+                'region': 'myregion',
+                'some': {'security-credentials': {
+                    'cred1': 'sekret', 'cred2': 'othersekret'}}})
+        self.assertEqual(
+            ('security-credentials',), datasource.sensitive_metadata_keys)
+        datasource.get_data()
+        json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
+        sensitive_json_file = self.tmp_path(INSTANCE_JSON_SENSITIVE_FILE, tmp)
+        redacted = util.load_json(util.load_file(json_file))
+        self.assertEqual(
+            {'cred1': 'sekret', 'cred2': 'othersekret'},
+            redacted['ds']['meta_data']['some']['security-credentials'])
+        content = util.load_file(sensitive_json_file)
+        expected = {
+            'base64_encoded_keys': [],
+            'sensitive_keys': ['ds/meta_data/some/security-credentials'],
+            'v1': {
+                'availability-zone': 'myaz',
+                'availability_zone': 'myaz',
+                'cloud-name': 'subclasscloudname',
+                'cloud_name': 'subclasscloudname',
+                'instance-id': 'iid-datasource',
+                'instance_id': 'iid-datasource',
+                'local-hostname': 'test-subclass-hostname',
+                'local_hostname': 'test-subclass-hostname',
+                'region': 'myregion'},
+            'ds': {
+                'meta_data': {
+                    'availability_zone': 'myaz',
+                    'local-hostname': 'test-subclass-hostname',
+                    'region': 'myregion',
+                    'some': {'security-credentials': REDACT_SENSITIVE_VALUE}}}
+        }
+        self.maxDiff = None
+        self.assertEqual(expected, util.load_json(content))
+        file_stat = os.stat(sensitive_json_file)
         self.assertEqual(0o600, stat.S_IMODE(file_stat.st_mode))
+        self.assertEqual(expected, util.load_json(content))
 
     def test_get_data_handles_redacted_unserializable_content(self):
         """get_data warns unserializable content in INSTANCE_JSON_FILE."""
         tmp = self.tmp_dir()
         datasource = DataSourceTestSubclassNet(
             self.sys_cfg, self.distro, Paths({'run_dir': tmp}),
-            custom_userdata={'key1': 'val1', 'key2': {'key2.1': self.paths}})
+            custom_metadata={'key1': 'val1', 'key2': {'key2.1': self.paths}})
         datasource.get_data()
         json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
         content = util.load_file(json_file)
-        expected_userdata = {
+        expected_metadata = {
             'key1': 'val1',
             'key2': {
                 'key2.1': "Warning: redacted unserializable type <class"
                           " 'cloudinit.helpers.Paths'>"}}
         instance_json = util.load_json(content)
         self.assertEqual(
-            expected_userdata, instance_json['ds']['user_data'])
+            expected_metadata, instance_json['ds']['meta_data'])
 
     def test_persist_instance_data_writes_ec2_metadata_when_set(self):
         """When ec2_metadata class attribute is set, persist to json."""
@@ -361,17 +417,17 @@ class TestDataSource(CiTestCase):
         tmp = self.tmp_dir()
         datasource = DataSourceTestSubclassNet(
             self.sys_cfg, self.distro, Paths({'run_dir': tmp}),
-            custom_userdata={'key1': 'val1', 'key2': {'key2.1': b'\x123'}})
+            custom_metadata={'key1': 'val1', 'key2': {'key2.1': b'\x123'}})
         self.assertTrue(datasource.get_data())
         json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
         content = util.load_file(json_file)
         instance_json = util.load_json(content)
-        self.assertEqual(
-            ['ds/user_data/key2/key2.1'],
+        self.assertItemsEqual(
+            ['ds/meta_data/key2/key2.1'],
             instance_json['base64_encoded_keys'])
         self.assertEqual(
             {'key1': 'val1', 'key2': {'key2.1': 'EjM='}},
-            instance_json['ds']['user_data'])
+            instance_json['ds']['meta_data'])
 
     @skipIf(not six.PY2, "json serialization on <= py2.7 handles bytes")
     def test_get_data_handles_bytes_values(self):
@@ -379,7 +435,7 @@ class TestDataSource(CiTestCase):
         tmp = self.tmp_dir()
         datasource = DataSourceTestSubclassNet(
             self.sys_cfg, self.distro, Paths({'run_dir': tmp}),
-            custom_userdata={'key1': 'val1', 'key2': {'key2.1': b'\x123'}})
+            custom_metadata={'key1': 'val1', 'key2': {'key2.1': b'\x123'}})
         self.assertTrue(datasource.get_data())
         json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
         content = util.load_file(json_file)
@@ -387,7 +443,7 @@ class TestDataSource(CiTestCase):
         self.assertEqual([], instance_json['base64_encoded_keys'])
         self.assertEqual(
             {'key1': 'val1', 'key2': {'key2.1': '\x123'}},
-            instance_json['ds']['user_data'])
+            instance_json['ds']['meta_data'])
 
     @skipIf(not six.PY2, "Only python2 hits UnicodeDecodeErrors on non-utf8")
     def test_non_utf8_encoding_logs_warning(self):
@@ -395,7 +451,7 @@ class TestDataSource(CiTestCase):
         tmp = self.tmp_dir()
         datasource = DataSourceTestSubclassNet(
             self.sys_cfg, self.distro, Paths({'run_dir': tmp}),
-            custom_userdata={'key1': 'val1', 'key2': {'key2.1': b'ab\xaadef'}})
+            custom_metadata={'key1': 'val1', 'key2': {'key2.1': b'ab\xaadef'}})
         self.assertTrue(datasource.get_data())
         json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
         self.assertFalse(os.path.exists(json_file))
@@ -509,4 +565,36 @@ class TestDataSource(CiTestCase):
             self.logs.getvalue())
 
 
+class TestRedactSensitiveData(CiTestCase):
+
+    def test_redact_sensitive_data_noop_when_no_sensitive_keys_present(self):
+        """When sensitive_keys is absent or empty from metadata do nothing."""
+        md = {'my': 'data'}
+        self.assertEqual(
+            md, redact_sensitive_keys(md, redact_value='redacted'))
+        md['sensitive_keys'] = []
+        self.assertEqual(
+            md, redact_sensitive_keys(md, redact_value='redacted'))
+
+    def test_redact_sensitive_data_redacts_exact_match_name(self):
+        """Only exact matched sensitive_keys are redacted from metadata."""
+        md = {'sensitive_keys': ['md/secure'],
+              'md': {'secure': 's3kr1t', 'insecure': 'publik'}}
+        secure_md = copy.deepcopy(md)
+        secure_md['md']['secure'] = 'redacted'
+        self.assertEqual(
+            secure_md,
+            redact_sensitive_keys(md, redact_value='redacted'))
+
+    def test_redact_sensitive_data_does_redacts_with_default_string(self):
+        """When redact_value is absent, REDACT_SENSITIVE_VALUE is used."""
+        md = {'sensitive_keys': ['md/secure'],
+              'md': {'secure': 's3kr1t', 'insecure': 'publik'}}
+        secure_md = copy.deepcopy(md)
+        secure_md['md']['secure'] = 'redacted for non-root user'
+        self.assertEqual(
+            secure_md,
+            redact_sensitive_keys(md))
+
+
 # vi: ts=4 expandtab
diff --git a/doc/rtd/index.rst b/doc/rtd/index.rst
index de67f361..20a99a30 100644
--- a/doc/rtd/index.rst
+++ b/doc/rtd/index.rst
@@ -31,6 +31,7 @@ initialization of a cloud instance.
    topics/capabilities.rst
    topics/availability.rst
    topics/format.rst
+   topics/instancedata.rst
    topics/dir_layout.rst
    topics/examples.rst
    topics/boot.rst
diff --git a/doc/rtd/topics/capabilities.rst b/doc/rtd/topics/capabilities.rst
index 2d8e2538..0d8b8947 100644
--- a/doc/rtd/topics/capabilities.rst
+++ b/doc/rtd/topics/capabilities.rst
@@ -18,7 +18,7 @@ User configurability
 
     User-data can be given by the user at instance launch time. See
     :ref:`user_data_formats` for acceptable user-data content.
-    
+
 
 This is done via the ``--user-data`` or ``--user-data-file`` argument to
 ec2-run-instances for example.
@@ -53,10 +53,9 @@ system:
 
   % cloud-init --help
   usage: cloud-init [-h] [--version] [--file FILES]
-
                     [--debug] [--force]
-                    {init,modules,single,dhclient-hook,features,analyze,devel,collect-logs,clean,status}
-                    ...
+                    {init,modules,single,query,dhclient-hook,features,analyze,devel,collect-logs,clean,status}
+                                                         ...
 
   optional arguments:
     -h, --help            show this help message and exit
@@ -68,17 +67,19 @@ system:
                           your own risk)
 
   Subcommands:
-    {init,modules,single,dhclient-hook,features,analyze,devel,collect-logs,clean,status}
+    {init,modules,single,query,dhclient-hook,features,analyze,devel,collect-logs,clean,status}
       init                initializes cloud-init and performs initial modules
       modules             activates modules using a given configuration key
       single              run a single module
+      query               Query instance metadata from the command line
       dhclient-hook       run the dhclient hookto record network info
       features            list defined features
       analyze             Devel tool: Analyze cloud-init logs and data
       devel               Run development tools
       collect-logs        Collect and tar all cloud-init debug info
-      clean               Remove logs and artifacts so cloud-init can re-run.
-      status              Report cloud-init status or wait on completion.
+      clean               Remove logs and artifacts so cloud-init can re-run
+      status              Report cloud-init status or wait on completion
+
 
 CLI Subcommand details
 ======================
@@ -104,8 +105,8 @@ cloud-init status
 Report whether cloud-init is running, done, disabled or errored. Exits
 non-zero if an error is detected in cloud-init.
 
- * **--long**: Detailed status information.
- * **--wait**: Block until cloud-init completes.
+* **--long**: Detailed status information.
+* **--wait**: Block until cloud-init completes.
 
 .. code-block:: shell-session
 
@@ -143,6 +144,68 @@ Logs collected are:
  * journalctl output
  * /var/lib/cloud/instance/user-data.txt
 
+.. _cli_query:
+
+cloud-init query
+------------------
+Query standardized cloud instance metadata crawled by cloud-init and stored
+in ``/run/cloud-init/instance-data.json``. This is a convenience command-line
+interface to reference any cached configuration metadata that cloud-init
+crawls when booting the instance. See :ref:`instance_metadata` for more info.
+
+* **--all**: Dump all available instance data as json which can be queried.
+* **--instance-data**: Optional path to a different instance-data.json file to
+  source for queries.
+* **--list-keys**: List available query keys from cached instance data.
+
+.. code-block:: shell-session
+
+  # List all top-level query keys available (includes standardized aliases)
+  % cloud-init query --list-keys
+  availability_zone
+  base64_encoded_keys
+  cloud_name
+  ds
+  instance_id
+  local_hostname
+  region
+  v1
+
+* **<varname>**: A dot-delimited variable path into the instance-data.json
+   object.
+
+.. code-block:: shell-session
+
+  # Query cloud-init standardized metadata on any cloud
+  % cloud-init query v1.cloud_name
+  aws  # or openstack, azure, gce etc.
+
+  # Any standardized instance-data under a <v#> key is aliased as a top-level
+  # key for convenience.
+  % cloud-init query cloud_name
+  aws  # or openstack, azure, gce etc.
+
+  # Query datasource-specific metadata on EC2
+  % cloud-init query ds.meta_data.public_ipv4
+
+* **--format** A string that will use jinja-template syntax to render a string
+   replacing
+
+.. code-block:: shell-session
+
+  # Generate a custom hostname fqdn based on instance-id, cloud and region
+  % cloud-init query --format 'custom-{{instance_id}}.{{region}}.{{v1.cloud_name}}.com'
+  custom-i-0e91f69987f37ec74.us-east-2.aws.com
+
+
+.. note::
+  The standardized instance data keys under **v#** are guaranteed not to change
+  behavior or format. If using top-level convenience aliases for any
+  standardized instance data keys, the most value (highest **v#**) of that key
+  name is what is reported as the top-level value. So these aliases act as a
+  'latest'.
+
+
 .. _cli_analyze:
 
 cloud-init analyze
@@ -150,10 +213,10 @@ cloud-init analyze
 Get detailed reports of where cloud-init spends most of its time. See
 :ref:`boot_time_analysis` for more info.
 
- * **blame** Report ordered by most costly operations.
- * **dump** Machine-readable JSON dump of all cloud-init tracked events.
- * **show** show time-ordered report of the cost of operations during each
-   boot stage.
+* **blame** Report ordered by most costly operations.
+* **dump** Machine-readable JSON dump of all cloud-init tracked events.
+* **show** show time-ordered report of the cost of operations during each
+  boot stage.
 
 .. _cli_devel:
 
@@ -182,8 +245,8 @@ cloud-init clean
 Remove cloud-init artifacts from /var/lib/cloud and optionally reboot the
 machine to so cloud-init re-runs all stages as it did on first boot.
 
- * **--logs**: Optionally remove /var/log/cloud-init*log files.
- * **--reboot**: Reboot the system after removing artifacts.
+* **--logs**: Optionally remove /var/log/cloud-init*log files.
+* **--reboot**: Reboot the system after removing artifacts.
 
 .. _cli_init:
 
@@ -195,7 +258,7 @@ Can be run on the commandline, but is generally gated to run only once
 due to semaphores in **/var/lib/cloud/instance/sem/** and
 **/var/lib/cloud/sem**.
 
- * **--local**: Run *init-local* stage instead of *init*.
+* **--local**: Run *init-local* stage instead of *init*.
 
 .. _cli_modules:
 
@@ -210,8 +273,8 @@ declared to run in various boot stages in the file
 commandline, but each module is gated to run only once due to semaphores
 in ``/var/lib/cloud/``.
 
- * **--mode (init|config|final)**: Run *modules:init*, *modules:config* or
-   *modules:final* cloud-init stages. See :ref:`boot_stages` for more info.
+* **--mode (init|config|final)**: Run *modules:init*, *modules:config* or
+  *modules:final* cloud-init stages. See :ref:`boot_stages` for more info.
 
 .. _cli_single:
 
@@ -221,9 +284,9 @@ Attempt to run a single named cloud config module.  The following example
 re-runs the cc_set_hostname module ignoring the module default frequency
 of once-per-instance:
 
- * **--name**: The cloud-config module name to run
- * **--frequency**: Optionally override the declared module frequency
-   with one of (always|once-per-instance|once)
+* **--name**: The cloud-config module name to run
+* **--frequency**: Optionally override the declared module frequency
+  with one of (always|once-per-instance|once)
 
 .. code-block:: shell-session
 
diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst
index 14432e65..e34f145c 100644
--- a/doc/rtd/topics/datasources.rst
+++ b/doc/rtd/topics/datasources.rst
@@ -17,146 +17,10 @@ own way) internally a datasource abstract class was created to allow for a
 single way to access the different cloud systems methods to provide this data
 through the typical usage of subclasses.
 
-
-.. _instance_metadata:
-
-instance-data
--------------
-For reference, cloud-init stores all the metadata, vendordata and userdata
-provided by a cloud in a json blob at ``/run/cloud-init/instance-data.json``.
-While the json contains datasource-specific keys and names, cloud-init will
-maintain a minimal set of standardized keys that will remain stable on any
-cloud. Standardized instance-data keys will be present under a "v1" key.
-Any datasource metadata cloud-init consumes will all be present under the
-"ds" key.
-
-Below is an instance-data.json example from an OpenStack instance:
-
-.. sourcecode:: json
-
-  {
-   "base64-encoded-keys": [
-    "ds/meta-data/random_seed",
-    "ds/user-data"
-   ],
-   "ds": {
-    "ec2_metadata": {
-     "ami-id": "ami-0000032f",
-     "ami-launch-index": "0",
-     "ami-manifest-path": "FIXME",
-     "block-device-mapping": {
-      "ami": "vda",
-      "ephemeral0": "/dev/vdb",
-      "root": "/dev/vda"
-     },
-     "hostname": "xenial-test.novalocal",
-     "instance-action": "none",
-     "instance-id": "i-0006e030",
-     "instance-type": "m1.small",
-     "local-hostname": "xenial-test.novalocal",
-     "local-ipv4": "10.5.0.6",
-     "placement": {
-      "availability-zone": "None"
-     },
-     "public-hostname": "xenial-test.novalocal",
-     "public-ipv4": "10.245.162.145",
-     "reservation-id": "r-fxm623oa",
-     "security-groups": "default"
-    },
-    "meta-data": {
-     "availability_zone": null,
-     "devices": [],
-     "hostname": "xenial-test.novalocal",
-     "instance-id": "3e39d278-0644-4728-9479-678f9212d8f0",
-     "launch_index": 0,
-     "local-hostname": "xenial-test.novalocal",
-     "name": "xenial-test",
-     "project_id": "e0eb2d2538814...",
-     "random_seed": "A6yPN...",
-     "uuid": "3e39d278-0644-4728-9479-678f92..."
-    },
-    "network_json": {
-     "links": [
-      {
-       "ethernet_mac_address": "fa:16:3e:7d:74:9b",
-       "id": "tap9ca524d5-6e",
-       "mtu": 8958,
-       "type": "ovs",
-       "vif_id": "9ca524d5-6e5a-4809-936a-6901..."
-      }
-     ],
-     "networks": [
-      {
-       "id": "network0",
-       "link": "tap9ca524d5-6e",
-       "network_id": "c6adfc18-9753-42eb-b3ea-18b57e6b837f",
-       "type": "ipv4_dhcp"
-      }
-     ],
-     "services": [
-      {
-       "address": "10.10.160.2",
-       "type": "dns"
-      }
-     ]
-    },
-    "user-data": "I2Nsb3VkLWNvbmZpZ...",
-    "vendor-data": null
-   },
-   "v1": {
-    "availability-zone": null,
-    "cloud-name": "openstack",
-    "instance-id": "3e39d278-0644-4728-9479-678f9212d8f0",
-    "local-hostname": "xenial-test",
-    "region": null
-   }
-  }
-
- 
-As of cloud-init v. 18.4, any values present in
-``/run/cloud-init/instance-data.json`` can be used in cloud-init user data
-scripts or cloud config data. This allows consumers to use cloud-init's
-vendor-neutral, standardized metadata keys as well as datasource-specific
-content for any scripts or cloud-config modules they are using.
-
-To use instance-data.json values in scripts and **#config-config** files the
-user-data will need to contain the following header as the first line **## template: jinja**. Cloud-init will source all variables defined in
-``/run/cloud-init/instance-data.json`` and allow scripts or cloud-config files 
-to reference those paths. Below are two examples::
-
- * Cloud config calling home with the ec2 public hostname and avaliability-zone
-    ```
-    ## template: jinja
-    #cloud-config
-    runcmd:
-        - echo 'EC2 public hostname allocated to instance: {{ ds.meta_data.public_hostname }}' > /tmp/instance_metadata
-        - echo 'EC2 avaiability zone: {{ v1.availability_zone }}' >> /tmp/instance_metadata 
-        - curl -X POST -d '{"hostname": "{{ds.meta_data.public_hostname }}", "availability-zone": "{{ v1.availability_zone }}"}'  https://example.com.com
-    ```
-
- * Custom user script performing different operations based on region
-    ```
-    ## template: jinja
-    #!/bin/bash
-    {% if v1.region == 'us-east-2' -%}
-    echo 'Installing custom proxies for {{ v1.region }}
-    sudo apt-get install my-xtra-fast-stack
-    {%- endif %}
-    ...
-
-    ```
-
-.. note::
-  Trying to reference jinja variables that don't exist in
-  instance-data.json will result in warnings in ``/var/log/cloud-init.log``
-  and the following string in your rendered user-data:
-  ``CI_MISSING_JINJA_VAR/<your_varname>``.
-  
-.. note::
-  To save time designing your user-data for a specific cloud's
-  instance-data.json, use the 'render' cloud-init command on an
-  instance booted on your favorite cloud. See :ref:`cli_devel` for more
-  information.
+Any metadata processed by cloud-init's datasources is persisted as
+``/run/cloud0-init/instance-data.json``. Cloud-init provides tooling
+to quickly introspect some of that data. See :ref:`instance_metadata` for
+more information.
 
 
 Datasource API
@@ -196,14 +60,14 @@ The current interface that a datasource object must provide is the following:
     # or does not exist)
     def device_name_to_device(self, name)
 
-    # gets the locale string this instance should be applying 
+    # gets the locale string this instance should be applying
     # which typically used to adjust the instances locale settings files
     def get_locale(self)
 
     @property
     def availability_zone(self)
 
-    # gets the instance id that was assigned to this instance by the 
+    # gets the instance id that was assigned to this instance by the
     # cloud provider or when said instance id does not exist in the backing
     # metadata this will return 'iid-datasource'
     def get_instance_id(self)
diff --git a/doc/rtd/topics/instancedata.rst b/doc/rtd/topics/instancedata.rst
new file mode 100644
index 00000000..634e1807
--- /dev/null
+++ b/doc/rtd/topics/instancedata.rst
@@ -0,0 +1,297 @@
+.. _instance_metadata:
+
+*****************
+Instance Metadata
+*****************
+
+What is a instance data?
+========================
+
+Instance data is the collection of all configuration data that cloud-init
+processes to configure the instance. This configuration typically
+comes from any number of sources:
+
+* cloud-provided metadata services (aka metadata)
+* custom config-drive attached to the instance
+* cloud-config seed files in the booted cloud image or distribution
+* vendordata provided from files or cloud metadata services
+* userdata provided at instance creation
+
+Each cloud provider presents unique configuration metadata in different
+formats to the instance. Cloud-init provides a cache of any crawled metadata
+as well as a versioned set of standardized instance data keys which it makes
+available on all platforms.
+
+Cloud-init produces a simple json object in
+``/run/cloud-init/instance-data.json`` which represents standardized and
+versioned representation of the metadata it consumes during initial boot. The
+intent is to provide the following benefits to users or scripts on any system
+deployed with cloud-init:
+
+* simple static object to query to obtain a instance's metadata
+* speed: avoid costly network transactions for metadata that is already cached
+  on the filesytem
+* reduce need to recrawl metadata services for static metadata that is already
+  cached
+* leverage cloud-init's best practices for crawling cloud-metadata services
+* avoid rolling unique metadata crawlers on each cloud platform to get
+  metadata configuration values
+
+Cloud-init stores any instance data processed in the following files:
+
+* ``/run/cloud-init/instance-data.json``: world-readable json containing
+  standardized keys, sensitive keys redacted
+* ``/run/cloud-init/instance-data-sensitive.json``: root-readable unredacted
+  json blob
+* ``/var/lib/cloud/instance/user-data.txt``: root-readable sensitive raw
+  userdata
+* ``/var/lib/cloud/instance/vendor-data.txt``: root-readable sensitive raw
+  vendordata
+
+Cloud-init redacts any security sensitive content from instance-data.json,
+stores ``/run/cloud-init/instance-data.json`` as a world-readable json file.
+Because user-data and vendor-data can contain passwords both of these files
+are readonly for *root* as well. The *root* user can also read
+``/run/cloud-init/instance-data-sensitive.json`` which is all instance data
+from instance-data.json as well as unredacted sensitive content.
+
+
+Format of instance-data.json
+============================
+
+The instance-data.json and instance-data-sensitive.json files are well-formed
+JSON and record the set of keys and values for any metadata processed by
+cloud-init. Cloud-init standardizes the format for this content so that it
+can be generalized across different cloud platforms.
+
+There are three basic top-level keys:
+
+* **base64_encoded_keys**: A list of forward-slash delimited key paths into
+  the instance-data.json object whose value is base64encoded for json
+  compatibility. Values at these paths should be decoded to get the original
+  value.
+
+* **sensitive_keys**: A list of forward-slash delimited key paths into
+  the instance-data.json object whose value is considered by the datasource as
+  'security sensitive'. Only the keys listed here will be redacted from
+  instance-data.json for non-root users.
+
+* **ds**: Datasource-specific metadata crawled for the specific cloud
+  platform. It should closely represent the structure of the cloud metadata
+  crawled. The structure of content and details provided are entirely
+  cloud-dependent. Mileage will vary depending on what the cloud exposes.
+  The content exposed under the 'ds' key is currently **experimental** and
+  expected to change slightly in the upcoming cloud-init release.
+
+* **v1**: Standardized cloud-init metadata keys, these keys are guaranteed to
+  exist on all cloud platforms. They will also retain their current behavior
+  and format and will be carried forward even if cloud-init introduces a new
+  version of standardized keys with **v2**.
+
+The standardized keys present:
+
++----------------------+-----------------------------------------------+---------------------------+
+|  Key path            | Description                                   | Examples                  |
++======================+===============================================+===========================+
+| v1.cloud_name        | The name of the cloud provided by metadata    | aws, openstack, azure,    |
+|                      | key 'cloud-name' or the cloud-init datasource | configdrive, nocloud,     |
+|                      | name which was discovered.                    | ovf, etc.                 |
++----------------------+-----------------------------------------------+---------------------------+
+| v1.instance_id       | Unique instance_id allocated by the cloud     | i-<somehash>              |
++----------------------+-----------------------------------------------+---------------------------+
+| v1.local_hostname    | The internal or local hostname of the system  | ip-10-41-41-70,           |
+|                      |                                               | <user-provided-hostname>  |
++----------------------+-----------------------------------------------+---------------------------+
+| v1.region            | The physical region/datacenter in which the   | us-east-2                 |
+|                      | instance is deployed                          |                           |
++----------------------+-----------------------------------------------+---------------------------+
+| v1.availability_zone | The physical availability zone in which the   | us-east-2b, nova, null    |
+|                      | instance is deployed                          |                           |
++----------------------+-----------------------------------------------+---------------------------+
+
+
+Below is an example of ``/run/cloud-init/instance_data.json`` on an EC2
+instance:
+
+.. sourcecode:: json
+
+  {
+   "base64_encoded_keys": [],
+   "sensitive_keys": [],
+   "ds": {
+    "meta_data": {
+     "ami-id": "ami-014e1416b628b0cbf",
+     "ami-launch-index": "0",
+     "ami-manifest-path": "(unknown)",
+     "block-device-mapping": {
+      "ami": "/dev/sda1",
+      "ephemeral0": "sdb",
+      "ephemeral1": "sdc",
+      "root": "/dev/sda1"
+     },
+     "hostname": "ip-10-41-41-70.us-east-2.compute.internal",
+     "instance-action": "none",
+     "instance-id": "i-04fa31cfc55aa7976",
+     "instance-type": "t2.micro",
+     "local-hostname": "ip-10-41-41-70.us-east-2.compute.internal",
+     "local-ipv4": "10.41.41.70",
+     "mac": "06:b6:92:dd:9d:24",
+     "metrics": {
+      "vhostmd": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+     },
+     "network": {
+      "interfaces": {
+       "macs": {
+	"06:b6:92:dd:9d:24": {
+	 "device-number": "0",
+	 "interface-id": "eni-08c0c9fdb99b6e6f4",
+	 "ipv4-associations": {
+	  "18.224.22.43": "10.41.41.70"
+	 },
+	 "local-hostname": "ip-10-41-41-70.us-east-2.compute.internal",
+	 "local-ipv4s": "10.41.41.70",
+	 "mac": "06:b6:92:dd:9d:24",
+	 "owner-id": "437526006925",
+	 "public-hostname": "ec2-18-224-22-43.us-east-2.compute.amazonaws.com",
+	 "public-ipv4s": "18.224.22.43",
+	 "security-group-ids": "sg-828247e9",
+	 "security-groups": "Cloud-init integration test secgroup",
+	 "subnet-id": "subnet-282f3053",
+	 "subnet-ipv4-cidr-block": "10.41.41.0/24",
+	 "subnet-ipv6-cidr-blocks": "2600:1f16:b80:ad00::/64",
+	 "vpc-id": "vpc-252ef24d",
+	 "vpc-ipv4-cidr-block": "10.41.0.0/16",
+	 "vpc-ipv4-cidr-blocks": "10.41.0.0/16",
+	 "vpc-ipv6-cidr-blocks": "2600:1f16:b80:ad00::/56"
+	}
+       }
+      }
+     },
+     "placement": {
+      "availability-zone": "us-east-2b"
+     },
+     "profile": "default-hvm",
+     "public-hostname": "ec2-18-224-22-43.us-east-2.compute.amazonaws.com",
+     "public-ipv4": "18.224.22.43",
+     "public-keys": {
+      "cloud-init-integration": [
+       "ssh-rsa
+  AAAAB3NzaC1yc2EAAAADAQABAAABAQDSL7uWGj8cgWyIOaspgKdVy0cKJ+UTjfv7jBOjG2H/GN8bJVXy72XAvnhM0dUM+CCs8FOf0YlPX+Frvz2hKInrmRhZVwRSL129PasD12MlI3l44u6IwS1o/W86Q+tkQYEljtqDOo0a+cOsaZkvUNzUyEXUwz/lmYa6G4hMKZH4NBj7nbAAF96wsMCoyNwbWryBnDYUr6wMbjRR1J9Pw7Xh7WRC73wy4Va2YuOgbD3V/5ZrFPLbWZW/7TFXVrql04QVbyei4aiFR5n//GvoqwQDNe58LmbzX/xvxyKJYdny2zXmdAhMxbrpFQsfpkJ9E/H5w0yOdSvnWbUoG5xNGoOB
+  cloud-init-integration"
+      ]
+     },
+     "reservation-id": "r-06ab75e9346f54333",
+     "security-groups": "Cloud-init integration test secgroup",
+     "services": {
+      "domain": "amazonaws.com",
+      "partition": "aws"
+     }
+    }
+   },
+   "v1": {
+    "availability-zone": "us-east-2b",
+    "availability_zone": "us-east-2b",
+    "cloud-name": "aws",
+    "cloud_name": "aws",
+    "instance-id": "i-04fa31cfc55aa7976",
+    "instance_id": "i-04fa31cfc55aa7976",
+    "local-hostname": "ip-10-41-41-70",
+    "local_hostname": "ip-10-41-41-70",
+    "region": "us-east-2"
+   }
+  }
+
+
+Using instance-data
+===================
+
+As of cloud-init v. 18.4, any variables present in
+``/run/cloud-init/instance-data.json`` can be used in:
+
+* User-data scripts
+* Cloud config data
+* Command line interface via **cloud-init query** or
+  **cloud-init devel render**
+
+Many clouds allow users to provide user-data to an instance at
+the time the instance is launched. Cloud-init supports a number of
+:ref:`user_data_formats`.
+
+Both user-data scripts and **#cloud-config** data support jinja template
+rendering.
+When the first line of the provided user-data begins with,
+**## template: jinja** cloud-init will use jinja to render that file.
+Any instance-data-sensitive.json variables are surfaced as dot-delimited
+jinja template variables because cloud-config modules are run as 'root'
+user.
+
+
+Below are some examples of providing these types of user-data:
+
+* Cloud config calling home with the ec2 public hostname and avaliability-zone
+
+.. code-block:: shell-session
+
+  ## template: jinja
+  #cloud-config
+  runcmd:
+      - echo 'EC2 public hostname allocated to instance: {{
+        ds.meta_data.public_hostname }}' > /tmp/instance_metadata
+      - echo 'EC2 avaiability zone: {{ v1.availability_zone }}' >>
+        /tmp/instance_metadata
+      - curl -X POST -d '{"hostname": "{{ds.meta_data.public_hostname }}",
+        "availability-zone": "{{ v1.availability_zone }}"}'
+        https://example.com
+
+* Custom user-data script performing different operations based on region
+
+.. code-block:: shell-session
+
+   ## template: jinja
+   #!/bin/bash
+   {% if v1.region == 'us-east-2' -%}
+   echo 'Installing custom proxies for {{ v1.region }}
+   sudo apt-get install my-xtra-fast-stack
+   {%- endif %}
+   ...
+
+.. note::
+  Trying to reference jinja variables that don't exist in
+  instance-data.json will result in warnings in ``/var/log/cloud-init.log``
+  and the following string in your rendered user-data:
+  ``CI_MISSING_JINJA_VAR/<your_varname>``.
+
+Cloud-init also surfaces a commandline tool **cloud-init query** which can
+assist developers or scripts with obtaining instance metadata easily. See
+:ref:`cli_query` for more information.
+
+To cut down on keystrokes on the command line, cloud-init also provides
+top-level key aliases for any standardized ``v#`` keys present. The preceding
+``v1`` is not required of ``v1.var_name`` These aliases will represent the
+value of the highest versioned standard key. For example, ``cloud_name``
+value will be ``v2.cloud_name`` if both ``v1`` and ``v2`` keys are present in
+instance-data.json.
+The **query** command also publishes ``userdata`` and ``vendordata`` keys to
+the root user which will contain the decoded user and vendor data provided to
+this instance. Non-root users referencing userdata or vendordata keys will
+see only redacted values.
+
+.. code-block:: shell-session
+
+ # List all top-level instance-data keys available
+ % cloud-init query --list-keys
+
+ # Find your EC2 ami-id
+ % cloud-init query ds.metadata.ami_id
+
+ # Format your cloud_name and region using jinja template syntax
+ % cloud-init query --format 'cloud: {{ v1.cloud_name }} myregion: {{
+ % v1.region }}'
+
+.. note::
+  To save time designing a user-data template for a specific cloud's
+  instance-data.json, use the 'render' cloud-init command on an
+  instance booted on your favorite cloud. See :ref:`cli_devel` for more
+  information.
+
+.. vi: textwidth=78
diff --git a/integration-requirements.txt b/integration-requirements.txt
index f80cb942..880d9886 100644
--- a/integration-requirements.txt
+++ b/integration-requirements.txt
@@ -5,16 +5,17 @@
 # the packages/pkg-deps.json file as well.
 #
 
+unittest2
 # ec2 backend
 boto3==1.5.9
 
 # ssh communication
 paramiko==2.4.1
 
+
 # lxd backend
 # 04/03/2018: enables use of lxd 3.0
 git+https://github.com/lxc/pylxd.git@4b8ab1802f9aee4eb29cf7b119dae0aa47150779
 
-
 # finds latest image information
 git+https://git.launchpad.net/simplestreams
diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py
index 27458271..c5457968 100644
--- a/tests/cloud_tests/testcases/base.py
+++ b/tests/cloud_tests/testcases/base.py
@@ -5,15 +5,15 @@
 import crypt
 import json
 import re
-import unittest
+import unittest2
 
 
 from cloudinit import util as c_util
 
-SkipTest = unittest.SkipTest
+SkipTest = unittest2.SkipTest
 
 
-class CloudTestCase(unittest.TestCase):
+class CloudTestCase(unittest2.TestCase):
     """Base test class for verifiers."""
 
     # data gets populated in get_suite.setUpClass
@@ -167,8 +167,9 @@ class CloudTestCase(unittest.TestCase):
                 'Skipping instance-data.json test.'
                 ' OS: %s not bionic or newer' % self.os_name)
         instance_data = json.loads(out)
-        self.assertEqual(
-            ['ds/user_data'], instance_data['base64_encoded_keys'])
+        self.assertItemsEqual(
+            [],
+            instance_data['base64_encoded_keys'])
         ds = instance_data.get('ds', {})
         v1_data = instance_data.get('v1', {})
         metadata = ds.get('meta-data', {})
@@ -187,10 +188,10 @@ class CloudTestCase(unittest.TestCase):
             metadata.get('placement', {}).get('availability-zone'),
             'Could not determine EC2 Availability zone placement')
         self.assertIsNotNone(
-            v1_data['availability-zone'], 'expected ec2 availability-zone')
-        self.assertEqual('aws', v1_data['cloud-name'])
-        self.assertIn('i-', v1_data['instance-id'])
-        self.assertIn('ip-', v1_data['local-hostname'])
+            v1_data['availability_zone'], 'expected ec2 availability_zone')
+        self.assertEqual('aws', v1_data['cloud_name'])
+        self.assertIn('i-', v1_data['instance_id'])
+        self.assertIn('ip-', v1_data['local_hostname'])
         self.assertIsNotNone(v1_data['region'], 'expected ec2 region')
 
     def test_instance_data_json_lxd(self):
@@ -213,16 +214,14 @@ class CloudTestCase(unittest.TestCase):
                 ' OS: %s not bionic or newer' % self.os_name)
         instance_data = json.loads(out)
         v1_data = instance_data.get('v1', {})
-        self.assertEqual(
-            ['ds/user_data', 'ds/vendor_data'],
-            sorted(instance_data['base64_encoded_keys']))
-        self.assertEqual('nocloud', v1_data['cloud-name'])
+        self.assertItemsEqual([], sorted(instance_data['base64_encoded_keys']))
+        self.assertEqual('nocloud', v1_data['cloud_name'])
         self.assertIsNone(
-            v1_data['availability-zone'],
-            'found unexpected lxd availability-zone %s' %
-            v1_data['availability-zone'])
-        self.assertIn('cloud-test', v1_data['instance-id'])
-        self.assertIn('cloud-test', v1_data['local-hostname'])
+            v1_data['availability_zone'],
+            'found unexpected lxd availability_zone %s' %
+            v1_data['availability_zone'])
+        self.assertIn('cloud-test', v1_data['instance_id'])
+        self.assertIn('cloud-test', v1_data['local_hostname'])
         self.assertIsNone(
             v1_data['region'],
             'found unexpected lxd region %s' % v1_data['region'])
@@ -248,18 +247,17 @@ class CloudTestCase(unittest.TestCase):
                 ' OS: %s not bionic or newer' % self.os_name)
         instance_data = json.loads(out)
         v1_data = instance_data.get('v1', {})
-        self.assertEqual(
-            ['ds/user_data'], instance_data['base64_encoded_keys'])
-        self.assertEqual('nocloud', v1_data['cloud-name'])
+        self.assertItemsEqual([], instance_data['base64_encoded_keys'])
+        self.assertEqual('nocloud', v1_data['cloud_name'])
         self.assertIsNone(
-            v1_data['availability-zone'],
-            'found unexpected kvm availability-zone %s' %
-            v1_data['availability-zone'])
+            v1_data['availability_zone'],
+            'found unexpected kvm availability_zone %s' %
+            v1_data['availability_zone'])
         self.assertIsNotNone(
             re.match(r'[\da-f]{8}(-[\da-f]{4}){3}-[\da-f]{12}',
-                     v1_data['instance-id']),
-            'kvm instance-id is not a UUID: %s' % v1_data['instance-id'])
-        self.assertIn('ubuntu', v1_data['local-hostname'])
+                     v1_data['instance_id']),
+            'kvm instance_id is not a UUID: %s' % v1_data['instance_id'])
+        self.assertIn('ubuntu', v1_data['local_hostname'])
         self.assertIsNone(
             v1_data['region'],
             'found unexpected lxd region %s' % v1_data['region'])
-- 
cgit v1.2.3


From e7b0e5f72e134779cfe22cd07b09a42c22d2bfe1 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Wed, 26 Sep 2018 17:59:07 +0000
Subject: Add support for Infiniband network interfaces (IPoIB).

OpenStack ironic references Infiniband interfaces via a 6 byte 'MAC
address' formed from bytes 13-15 and 18-20 of interface's hardware
address. This address is used as the ethernet_mac_address of Infiniband
links in network_data.json in configdrives generated by OpenStack nova.
We can use this address to map links in network_data.json to their
corresponding interface names.

When generating interface configuration files, we need to use the
interface's full hardware address as the HWADDR, rather than the 6 byte
MAC address provided by network_data.json.

This change allows IB interfaces to be referenced in this dual mode - by
MAC address and hardware address, depending on the context.

Support TYPE=InfiniBand for sysconfig configuration of IB interfaces.
---
 cloudinit/net/__init__.py              | 38 ++++++++++++++++
 cloudinit/net/network_state.py         |  4 ++
 cloudinit/net/sysconfig.py             | 14 ++++++
 cloudinit/sources/helpers/openstack.py | 11 +++++
 tests/unittests/test_net.py            | 81 +++++++++++++++++++++++++++++++++-
 5 files changed, 147 insertions(+), 1 deletion(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 5e87bcad..f83d3681 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -569,6 +569,20 @@ def get_interface_mac(ifname):
     return read_sys_net_safe(ifname, path)
 
 
+def get_ib_interface_hwaddr(ifname, ethernet_format):
+    """Returns the string value of an Infiniband interface's hardware
+    address. If ethernet_format is True, an Ethernet MAC-style 6 byte
+    representation of the address will be returned.
+    """
+    # Type 32 is Infiniband.
+    if read_sys_net_safe(ifname, 'type') == '32':
+        mac = get_interface_mac(ifname)
+        if mac and ethernet_format:
+            # Use bytes 13-15 and 18-20 of the hardware address.
+            mac = mac[36:-14] + mac[51:]
+        return mac
+
+
 def get_interfaces_by_mac():
     """Build a dictionary of tuples {mac: name}.
 
@@ -580,6 +594,15 @@ def get_interfaces_by_mac():
                 "duplicate mac found! both '%s' and '%s' have mac '%s'" %
                 (name, ret[mac], mac))
         ret[mac] = name
+        # Try to get an Infiniband hardware address (in 6 byte Ethernet format)
+        # for the interface.
+        ib_mac = get_ib_interface_hwaddr(name, True)
+        if ib_mac:
+            if ib_mac in ret:
+                raise RuntimeError(
+                    "duplicate mac found! both '%s' and '%s' have mac '%s'" %
+                    (name, ret[ib_mac], ib_mac))
+            ret[ib_mac] = name
     return ret
 
 
@@ -607,6 +630,21 @@ def get_interfaces():
     return ret
 
 
+def get_ib_hwaddrs_by_interface():
+    """Build a dictionary mapping Infiniband interface names to their hardware
+    address."""
+    ret = {}
+    for name, _, _, _ in get_interfaces():
+        ib_mac = get_ib_interface_hwaddr(name, False)
+        if ib_mac:
+            if ib_mac in ret:
+                raise RuntimeError(
+                    "duplicate mac found! both '%s' and '%s' have mac '%s'" %
+                    (name, ret[ib_mac], ib_mac))
+            ret[name] = ib_mac
+    return ret
+
+
 class EphemeralIPv4Network(object):
     """Context manager which sets up temporary static network configuration.
 
diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
index 72c803eb..f76e508a 100644
--- a/cloudinit/net/network_state.py
+++ b/cloudinit/net/network_state.py
@@ -483,6 +483,10 @@ class NetworkStateInterpreter(object):
 
         interfaces.update({iface['name']: iface})
 
+    @ensure_command_keys(['name'])
+    def handle_infiniband(self, command):
+        self.handle_physical(command)
+
     @ensure_command_keys(['address'])
     def handle_nameserver(self, command):
         dns = self._network_state.get('dns')
diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py
index 66e970e0..9c16d3a7 100644
--- a/cloudinit/net/sysconfig.py
+++ b/cloudinit/net/sysconfig.py
@@ -174,6 +174,7 @@ class NetInterface(ConfigMap):
         'ethernet': 'Ethernet',
         'bond': 'Bond',
         'bridge': 'Bridge',
+        'infiniband': 'InfiniBand',
     }
 
     def __init__(self, iface_name, base_sysconf_dir, templates,
@@ -568,6 +569,18 @@ class Renderer(renderer.Renderer):
             cls._render_subnets(iface_cfg, iface_subnets)
             cls._render_subnet_routes(iface_cfg, route_cfg, iface_subnets)
 
+    @classmethod
+    def _render_ib_interfaces(cls, network_state, iface_contents):
+        ib_filter = renderer.filter_by_type('infiniband')
+        for iface in network_state.iter_interfaces(ib_filter):
+            iface_name = iface['name']
+            iface_cfg = iface_contents[iface_name]
+            iface_cfg.kind = 'infiniband'
+            iface_subnets = iface.get("subnets", [])
+            route_cfg = iface_cfg.routes
+            cls._render_subnets(iface_cfg, iface_subnets)
+            cls._render_subnet_routes(iface_cfg, route_cfg, iface_subnets)
+
     @classmethod
     def _render_sysconfig(cls, base_sysconf_dir, network_state,
                           templates=None):
@@ -586,6 +599,7 @@ class Renderer(renderer.Renderer):
         cls._render_bond_interfaces(network_state, iface_contents)
         cls._render_vlan_interfaces(network_state, iface_contents)
         cls._render_bridge_interfaces(network_state, iface_contents)
+        cls._render_ib_interfaces(network_state, iface_contents)
         contents = {}
         for iface_name, iface_cfg in iface_contents.items():
             if iface_cfg or iface_cfg.children:
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 76a6e310..9c29ceac 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -675,6 +675,17 @@ def convert_net_json(network_json=None, known_macs=None):
             else:
                 cfg[key] = fmt % link_id_info[target]['name']
 
+    # Infiniband interfaces may be referenced in network_data.json by a 6 byte
+    # Ethernet MAC-style address, and we use that address to look up the
+    # interface name above. Now ensure that the hardware address is set to the
+    # full 20 byte address.
+    ib_known_hwaddrs = net.get_ib_hwaddrs_by_interface()
+    if ib_known_hwaddrs:
+        for cfg in config:
+            if cfg['name'] in ib_known_hwaddrs:
+                cfg['mac_address'] = ib_known_hwaddrs[cfg['name']]
+                cfg['type'] = 'infiniband'
+
     for service in services:
         cfg = service
         cfg.update({'type': 'nameserver'})
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index f3165da9..5d9c7d92 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -3260,11 +3260,15 @@ class TestGetInterfacesByMac(CiTestCase):
     def _se_interface_has_own_mac(self, name):
         return name in self.data['own_macs']
 
+    def _se_get_ib_interface_hwaddr(self, name, ethernet_format):
+        ib_hwaddr = self.data.get('ib_hwaddr', {})
+        return ib_hwaddr.get(name, {}).get(ethernet_format)
+
     def _mock_setup(self):
         self.data = copy.deepcopy(self._data)
         self.data['devices'] = set(list(self.data['macs'].keys()))
         mocks = ('get_devicelist', 'get_interface_mac', 'is_bridge',
-                 'interface_has_own_mac', 'is_vlan')
+                 'interface_has_own_mac', 'is_vlan', 'get_ib_interface_hwaddr')
         self.mocks = {}
         for n in mocks:
             m = mock.patch('cloudinit.net.' + n,
@@ -3338,6 +3342,20 @@ class TestGetInterfacesByMac(CiTestCase):
         ret = net.get_interfaces_by_mac()
         self.assertEqual('lo', ret[empty_mac])
 
+    def test_ib(self):
+        ib_addr = '80:00:00:28:fe:80:00:00:00:00:00:00:00:11:22:03:00:33:44:56'
+        ib_addr_eth_format = '00:11:22:33:44:56'
+        self._mock_setup()
+        self.data['devices'] = ['enp0s1', 'ib0']
+        self.data['own_macs'].append('ib0')
+        self.data['macs']['ib0'] = ib_addr
+        self.data['ib_hwaddr'] = {'ib0': {True: ib_addr_eth_format,
+                                          False: ib_addr}}
+        result = net.get_interfaces_by_mac()
+        expected = {'aa:aa:aa:aa:aa:01': 'enp0s1',
+                    ib_addr_eth_format: 'ib0', ib_addr: 'ib0'}
+        self.assertEqual(expected, result)
+
 
 class TestInterfacesSorting(CiTestCase):
 
@@ -3352,6 +3370,67 @@ class TestInterfacesSorting(CiTestCase):
             ['enp0s3', 'enp0s8', 'enp0s13', 'enp1s2', 'enp2s0', 'enp2s3'])
 
 
+class TestGetIBHwaddrsByInterface(CiTestCase):
+
+    _ib_addr = '80:00:00:28:fe:80:00:00:00:00:00:00:00:11:22:03:00:33:44:56'
+    _ib_addr_eth_format = '00:11:22:33:44:56'
+    _data = {'devices': ['enp0s1', 'enp0s2', 'bond1', 'bridge1',
+                         'bridge1-nic', 'tun0', 'ib0'],
+             'bonds': ['bond1'],
+             'bridges': ['bridge1'],
+             'own_macs': ['enp0s1', 'enp0s2', 'bridge1-nic', 'bridge1', 'ib0'],
+             'macs': {'enp0s1': 'aa:aa:aa:aa:aa:01',
+                      'enp0s2': 'aa:aa:aa:aa:aa:02',
+                      'bond1': 'aa:aa:aa:aa:aa:01',
+                      'bridge1': 'aa:aa:aa:aa:aa:03',
+                      'bridge1-nic': 'aa:aa:aa:aa:aa:03',
+                      'tun0': None,
+                      'ib0': _ib_addr},
+             'ib_hwaddr': {'ib0': {True: _ib_addr_eth_format,
+                                   False: _ib_addr}}}
+    data = {}
+
+    def _mock_setup(self):
+        self.data = copy.deepcopy(self._data)
+        mocks = ('get_devicelist', 'get_interface_mac', 'is_bridge',
+                 'interface_has_own_mac', 'get_ib_interface_hwaddr')
+        self.mocks = {}
+        for n in mocks:
+            m = mock.patch('cloudinit.net.' + n,
+                           side_effect=getattr(self, '_se_' + n))
+            self.addCleanup(m.stop)
+            self.mocks[n] = m.start()
+
+    def _se_get_devicelist(self):
+        return self.data['devices']
+
+    def _se_get_interface_mac(self, name):
+        return self.data['macs'][name]
+
+    def _se_is_bridge(self, name):
+        return name in self.data['bridges']
+
+    def _se_interface_has_own_mac(self, name):
+        return name in self.data['own_macs']
+
+    def _se_get_ib_interface_hwaddr(self, name, ethernet_format):
+        ib_hwaddr = self.data.get('ib_hwaddr', {})
+        return ib_hwaddr.get(name, {}).get(ethernet_format)
+
+    def test_ethernet(self):
+        self._mock_setup()
+        self.data['devices'].remove('ib0')
+        result = net.get_ib_hwaddrs_by_interface()
+        expected = {}
+        self.assertEqual(expected, result)
+
+    def test_ib(self):
+        self._mock_setup()
+        result = net.get_ib_hwaddrs_by_interface()
+        expected = {'ib0': self._ib_addr}
+        self.assertEqual(expected, result)
+
+
 def _gzip_data(data):
     with io.BytesIO() as iobuf:
         gzfp = gzip.GzipFile(mode="wb", fileobj=iobuf)
-- 
cgit v1.2.3


From fd873fa83fb5ca9c6d5f22a7bec6a308448c77ff Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Wed, 26 Sep 2018 21:18:24 +0000
Subject: docs: surface experimental doc in instance-data.json

---
 cloudinit/sources/__init__.py        | 9 +++++++--
 cloudinit/sources/tests/test_init.py | 6 ++++--
 2 files changed, 11 insertions(+), 4 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 730e8174..5ac98826 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -38,6 +38,11 @@ DEP_FILESYSTEM = "FILESYSTEM"
 DEP_NETWORK = "NETWORK"
 DS_PREFIX = 'DataSource'
 
+EXPERIMENTAL_TEXT = (
+    "EXPERIMENTAL: The structure and format of content scoped under the 'ds'"
+    " key may change in subsequent releases of cloud-init.")
+
+
 # File in which public available instance meta-data is written
 # security-sensitive key values are redacted from this world-readable file
 INSTANCE_JSON_FILE = 'instance-data.json'
@@ -243,8 +248,8 @@ class DataSource(object):
         @return True on successful write, False otherwise.
         """
         instance_data = {
-            'ds': {
-                'meta_data': self.metadata}}
+            'ds': {'_doc': EXPERIMENTAL_TEXT,
+                   'meta_data': self.metadata}}
         if hasattr(self, 'network_json'):
             network_json = getattr(self, 'network_json')
             if network_json != UNSET:
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index 6b965750..8082019e 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -10,8 +10,8 @@ from cloudinit.event import EventType
 from cloudinit.helpers import Paths
 from cloudinit import importer
 from cloudinit.sources import (
-    INSTANCE_JSON_FILE, INSTANCE_JSON_SENSITIVE_FILE, REDACT_SENSITIVE_VALUE,
-    UNSET, DataSource, redact_sensitive_keys)
+    EXPERIMENTAL_TEXT, INSTANCE_JSON_FILE, INSTANCE_JSON_SENSITIVE_FILE,
+    REDACT_SENSITIVE_VALUE, UNSET, DataSource, redact_sensitive_keys)
 from cloudinit.tests.helpers import CiTestCase, skipIf, mock
 from cloudinit.user_data import UserDataProcessor
 from cloudinit import util
@@ -305,6 +305,7 @@ class TestDataSource(CiTestCase):
                 'local_hostname': 'test-subclass-hostname',
                 'region': 'myregion'},
             'ds': {
+                '_doc': EXPERIMENTAL_TEXT,
                 'meta_data': {'availability_zone': 'myaz',
                               'local-hostname': 'test-subclass-hostname',
                               'region': 'myregion'}}}
@@ -348,6 +349,7 @@ class TestDataSource(CiTestCase):
                 'local_hostname': 'test-subclass-hostname',
                 'region': 'myregion'},
             'ds': {
+                '_doc': EXPERIMENTAL_TEXT,
                 'meta_data': {
                     'availability_zone': 'myaz',
                     'local-hostname': 'test-subclass-hostname',
-- 
cgit v1.2.3