From 49b562c94838f4e2c0aa0df01a5247d06bc8a561 Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Thu, 12 Apr 2018 15:51:45 -0600
Subject: tools: Fix make-tarball cli tool usage for development

This tool is used to assist during the creation of ubuntu packages for
release testing. Address the following on the command-line:
 * --help option now print usage
 * Add --orig-tarball which creates named output file
   cloud-init_<release-version>.orig.tar.gz
 * drop unused --verbose option
---
 tools/make-tarball | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

(limited to 'tools')

diff --git a/tools/make-tarball b/tools/make-tarball
index 3197689f..8d540139 100755
--- a/tools/make-tarball
+++ b/tools/make-tarball
@@ -13,22 +13,28 @@ Usage: ${0##*/} [revision]
     create a tarball of revision (default HEAD)
 
     options:
-      -o | --output FILE   write to file
+      -h | --help		print usage
+      -o | --output FILE	write to file
+           --orig-tarball	Write file cloud-init_<version>.orig.tar.gz
+           --long		Use git describe --long for versioning
 EOF
 }
 
 short_opts="ho:v"
-long_opts="help,output:,long,verbose"
+long_opts="help,output:,orig-tarball,long"
 getopt_out=$(getopt --name "${0##*/}" \
     --options "${short_opts}" --long "${long_opts}" -- "$@") &&
     eval set -- "${getopt_out}" || { Usage 1>&2; exit 1; }
 
 long_opt=""
+orig_opt=""
 while [ $# -ne 0 ]; do
     cur=$1; next=$2
     case "$cur" in
+        -h|--help) Usage; exit 0;;
         -o|--output) output=$next; shift;;
            --long) long_opt="--long";;
+           --orig-tarball) orig_opt=".orig";;
         --) shift; break;;
     esac
     shift;
@@ -39,7 +45,10 @@ version=$(git describe --abbrev=8 "--match=[0-9]*" ${long_opt} $rev)
 
 archive_base="cloud-init-$version"
 if [ -z "$output" ]; then
-    output="$archive_base.tar.gz"
+    if [ ! -z "$orig_opt" ]; then
+        archive_base="cloud-init_$version"
+    fi
+    output="$archive_base$orig_opt.tar.gz"
 fi
 
 # when building an archiving from HEAD, ensure that there aren't any
-- 
cgit v1.2.3


From 6ef92c98c3d2b127b05d6708337efc8a81e00071 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Thu, 26 Apr 2018 16:24:24 -0500
Subject: IBMCloud: recognize provisioning environment during debug boots.

When images are deployed from template in a production environment
the artifacts of the provisioning stage (provisioningConfiguration.cfg)
that cloud-init referenced are cleaned up.  However, when provisioned
in "debug" mode (internal to IBM) the artifacts are left.

This changes the 'is_ibm_provisioning' implementations in both
ds-identify and in the IBM datasource to identify the provisioning
stage more correctly.  The change is to consider provisioning only
if the provisioing file existed and there was no log file or
the log file was older than this boot.

LP: #1767166
---
 cloudinit/sources/DataSourceIBMCloud.py          | 42 +++++++++-----
 cloudinit/tests/helpers.py                       | 13 ++++-
 tests/unittests/test_datasource/test_ibmcloud.py | 50 ++++++++++++++++
 tests/unittests/test_ds_identify.py              | 72 +++++++++++++++++++++---
 tools/ds-identify                                | 21 ++++++-
 5 files changed, 175 insertions(+), 23 deletions(-)

(limited to 'tools')

diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py
index cfa724bf..01106ec0 100644
--- a/cloudinit/sources/DataSourceIBMCloud.py
+++ b/cloudinit/sources/DataSourceIBMCloud.py
@@ -8,17 +8,11 @@ There are 2 different api exposed launch methods.
  * template: This is the legacy method of launching instances.
    When booting from an image template, the system boots first into
    a "provisioning" mode.  There, host <-> guest mechanisms are utilized
-   to execute code in the guest and provision it.
+   to execute code in the guest and configure it.  The configuration
+   includes configuring the system network and possibly installing
+   packages and other software stack.
 
-   Cloud-init will disable itself when it detects that it is in the
-   provisioning mode.  It detects this by the presence of
-   a file '/root/provisioningConfiguration.cfg'.
-
-   When provided with user-data, the "first boot" will contain a
-   ConfigDrive-like disk labeled with 'METADATA'.  If there is no user-data
-   provided, then there is no data-source.
-
-   Cloud-init never does any network configuration in this mode.
+   After the provisioning is finished, the system reboots.
 
  * os_code: Essentially "launch by OS Code" (Operating System Code).
    This is a more modern approach.  There is no specific "provisioning" boot.
@@ -200,8 +194,30 @@ def _is_xen():
     return os.path.exists("/proc/xen")
 
 
-def _is_ibm_provisioning():
-    return os.path.exists("/root/provisioningConfiguration.cfg")
+def _is_ibm_provisioning(
+        prov_cfg="/root/provisioningConfiguration.cfg",
+        inst_log="/root/swinstall.log",
+        boot_ref="/proc/1/environ"):
+    """Return boolean indicating if this boot is ibm provisioning boot."""
+    if os.path.exists(prov_cfg):
+        msg = "config '%s' exists." % prov_cfg
+        result = True
+        if os.path.exists(inst_log):
+            if os.path.exists(boot_ref):
+                result = (os.stat(inst_log).st_mtime >
+                          os.stat(boot_ref).st_mtime)
+                msg += (" log '%s' from %s boot." %
+                        (inst_log, "current" if result else "previous"))
+            else:
+                msg += (" log '%s' existed, but no reference file '%s'." %
+                        (inst_log, boot_ref))
+                result = False
+        else:
+            msg += " log '%s' did not exist." % inst_log
+    else:
+        result, msg = (False, "config '%s' did not exist." % prov_cfg)
+    LOG.debug("ibm_provisioning=%s: %s", result, msg)
+    return result
 
 
 def get_ibm_platform():
@@ -251,7 +267,7 @@ def get_ibm_platform():
         else:
             return (Platforms.TEMPLATE_LIVE_METADATA, metadata_path)
     elif _is_ibm_provisioning():
-            return (Platforms.TEMPLATE_PROVISIONING_NODATA, None)
+        return (Platforms.TEMPLATE_PROVISIONING_NODATA, None)
     return not_found
 
 
diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py
index 4999f1f6..117a9cfe 100644
--- a/cloudinit/tests/helpers.py
+++ b/cloudinit/tests/helpers.py
@@ -8,6 +8,7 @@ import os
 import shutil
 import sys
 import tempfile
+import time
 import unittest
 
 import mock
@@ -263,7 +264,8 @@ class FilesystemMockingTestCase(ResourceUsingTestCase):
             os.path: [('isfile', 1), ('exists', 1),
                       ('islink', 1), ('isdir', 1), ('lexists', 1)],
             os: [('listdir', 1), ('mkdir', 1),
-                 ('lstat', 1), ('symlink', 2)]
+                 ('lstat', 1), ('symlink', 2),
+                 ('stat', 1)]
         }
 
         if hasattr(os, 'scandir'):
@@ -349,6 +351,15 @@ def populate_dir(path, files):
     return ret
 
 
+def populate_dir_with_ts(path, data):
+    """data is {'file': ('contents', mtime)}.  mtime relative to now."""
+    populate_dir(path, dict((k, v[0]) for k, v in data.items()))
+    btime = time.time()
+    for fpath, (_contents, mtime) in data.items():
+        ts = btime + mtime if mtime else btime
+        os.utime(os.path.sep.join((path, fpath)), (ts, ts))
+
+
 def dir2dict(startdir, prefix=None):
     flist = {}
     if prefix is None:
diff --git a/tests/unittests/test_datasource/test_ibmcloud.py b/tests/unittests/test_datasource/test_ibmcloud.py
index 621cfe49..e639ae47 100644
--- a/tests/unittests/test_datasource/test_ibmcloud.py
+++ b/tests/unittests/test_datasource/test_ibmcloud.py
@@ -259,4 +259,54 @@ class TestReadMD(test_helpers.CiTestCase):
                          ret['metadata'])
 
 
+class TestIsIBMProvisioning(test_helpers.FilesystemMockingTestCase):
+    """Test the _is_ibm_provisioning method."""
+    inst_log = "/root/swinstall.log"
+    prov_cfg = "/root/provisioningConfiguration.cfg"
+    boot_ref = "/proc/1/environ"
+    with_logs = True
+
+    def _call_with_root(self, rootd):
+        self.reRoot(rootd)
+        return ibm._is_ibm_provisioning()
+
+    def test_no_config(self):
+        """No provisioning config means not provisioning."""
+        self.assertFalse(self._call_with_root(self.tmp_dir()))
+
+    def test_config_only(self):
+        """A provisioning config without a log means provisioning."""
+        rootd = self.tmp_dir()
+        test_helpers.populate_dir(rootd, {self.prov_cfg: "key=value"})
+        self.assertTrue(self._call_with_root(rootd))
+
+    def test_config_with_old_log(self):
+        """A config with a log from previous boot is not provisioning."""
+        rootd = self.tmp_dir()
+        data = {self.prov_cfg: ("key=value\nkey2=val2\n", -10),
+                self.inst_log: ("log data\n", -30),
+                self.boot_ref: ("PWD=/", 0)}
+        test_helpers.populate_dir_with_ts(rootd, data)
+        self.assertFalse(self._call_with_root(rootd=rootd))
+        self.assertIn("from previous boot", self.logs.getvalue())
+
+    def test_config_with_new_log(self):
+        """A config with a log from this boot is provisioning."""
+        rootd = self.tmp_dir()
+        data = {self.prov_cfg: ("key=value\nkey2=val2\n", -10),
+                self.inst_log: ("log data\n", 30),
+                self.boot_ref: ("PWD=/", 0)}
+        test_helpers.populate_dir_with_ts(rootd, data)
+        self.assertTrue(self._call_with_root(rootd=rootd))
+        self.assertIn("from current boot", self.logs.getvalue())
+
+    def test_config_and_log_no_reference(self):
+        """If the config and log existed, but no reference, assume not."""
+        rootd = self.tmp_dir()
+        test_helpers.populate_dir(
+            rootd, {self.prov_cfg: "key=value", self.inst_log: "log data\n"})
+        self.assertFalse(self._call_with_root(rootd=rootd))
+        self.assertIn("no reference file", self.logs.getvalue())
+
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 53643989..ad7fe41e 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -1,5 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
+from collections import namedtuple
 import copy
 import os
 from uuid import uuid4
@@ -7,7 +8,7 @@ from uuid import uuid4
 from cloudinit import safeyaml
 from cloudinit import util
 from cloudinit.tests.helpers import (
-    CiTestCase, dir2dict, populate_dir)
+    CiTestCase, dir2dict, populate_dir, populate_dir_with_ts)
 
 from cloudinit.sources import DataSourceIBMCloud as dsibm
 
@@ -66,7 +67,6 @@ P_SYS_VENDOR = "sys/class/dmi/id/sys_vendor"
 P_SEED_DIR = "var/lib/cloud/seed"
 P_DSID_CFG = "etc/cloud/ds-identify.cfg"
 
-IBM_PROVISIONING_CHECK_PATH = "/root/provisioningConfiguration.cfg"
 IBM_CONFIG_UUID = "9796-932E"
 
 MOCK_VIRT_IS_KVM = {'name': 'detect_virt', 'RET': 'kvm', 'ret': 0}
@@ -74,11 +74,17 @@ MOCK_VIRT_IS_VMWARE = {'name': 'detect_virt', 'RET': 'vmware', 'ret': 0}
 MOCK_VIRT_IS_XEN = {'name': 'detect_virt', 'RET': 'xen', 'ret': 0}
 MOCK_UNAME_IS_PPC64 = {'name': 'uname', 'out': UNAME_PPC64EL, 'ret': 0}
 
+shell_true = 0
+shell_false = 1
 
-class TestDsIdentify(CiTestCase):
+CallReturn = namedtuple('CallReturn',
+                        ['rc', 'stdout', 'stderr', 'cfg', 'files'])
+
+
+class DsIdentifyBase(CiTestCase):
     dsid_path = os.path.realpath('tools/ds-identify')
 
-    def call(self, rootd=None, mocks=None, args=None, files=None,
+    def call(self, rootd=None, mocks=None, func="main", args=None, files=None,
              policy_dmi=DI_DEFAULT_POLICY,
              policy_no_dmi=DI_DEFAULT_POLICY_NO_DMI,
              ec2_strict_id=DI_EC2_STRICT_ID_DEFAULT):
@@ -135,7 +141,7 @@ class TestDsIdentify(CiTestCase):
                 mocklines.append(write_mock(d))
 
         endlines = [
-            'main %s' % ' '.join(['"%s"' % s for s in args])
+            func + ' ' + ' '.join(['"%s"' % s for s in args])
         ]
 
         with open(wrap, "w") as fp:
@@ -159,7 +165,7 @@ class TestDsIdentify(CiTestCase):
                 cfg = {"_INVALID_YAML": contents,
                        "_EXCEPTION": str(e)}
 
-        return rc, out, err, cfg, dir2dict(rootd)
+        return CallReturn(rc, out, err, cfg, dir2dict(rootd))
 
     def _call_via_dict(self, data, rootd=None, **kwargs):
         # return output of self.call with a dict input like VALID_CFG[item]
@@ -190,6 +196,8 @@ class TestDsIdentify(CiTestCase):
                 _print_run_output(rc, out, err, cfg, files)
         return rc, out, err, cfg, files
 
+
+class TestDsIdentify(DsIdentifyBase):
     def test_wb_print_variables(self):
         """_print_info reports an array of discovered variables to stderr."""
         data = VALID_CFG['Azure-dmi-detection']
@@ -250,7 +258,10 @@ class TestDsIdentify(CiTestCase):
         Template provisioning with user-data has METADATA disk,
         datasource should return not found."""
         data = copy.deepcopy(VALID_CFG['IBMCloud-metadata'])
-        data['files'] = {IBM_PROVISIONING_CHECK_PATH: 'xxx'}
+        # change the 'is_ibm_provisioning' mock to return 1 (false)
+        isprov_m = [m for m in data['mocks']
+                    if m["name"] == "is_ibm_provisioning"][0]
+        isprov_m['ret'] = shell_true
         return self._check_via_dict(data, RC_NOT_FOUND)
 
     def test_ibmcloud_template_userdata(self):
@@ -265,7 +276,8 @@ class TestDsIdentify(CiTestCase):
 
         no disks attached.  Datasource should return not found."""
         data = copy.deepcopy(VALID_CFG['IBMCloud-nodisks'])
-        data['files'] = {IBM_PROVISIONING_CHECK_PATH: 'xxx'}
+        data['mocks'].append(
+            {'name': 'is_ibm_provisioning', 'ret': shell_true})
         return self._check_via_dict(data, RC_NOT_FOUND)
 
     def test_ibmcloud_template_no_userdata(self):
@@ -446,6 +458,47 @@ class TestDsIdentify(CiTestCase):
         self._test_ds_found('Hetzner')
 
 
+class TestIsIBMProvisioning(DsIdentifyBase):
+    """Test the is_ibm_provisioning method in ds-identify."""
+
+    inst_log = "/root/swinstall.log"
+    prov_cfg = "/root/provisioningConfiguration.cfg"
+    boot_ref = "/proc/1/environ"
+    funcname = "is_ibm_provisioning"
+
+    def test_no_config(self):
+        """No provisioning config means not provisioning."""
+        ret = self.call(files={}, func=self.funcname)
+        self.assertEqual(shell_false, ret.rc)
+
+    def test_config_only(self):
+        """A provisioning config without a log means provisioning."""
+        ret = self.call(files={self.prov_cfg: "key=value"}, func=self.funcname)
+        self.assertEqual(shell_true, ret.rc)
+
+    def test_config_with_old_log(self):
+        """A config with a log from previous boot is not provisioning."""
+        rootd = self.tmp_dir()
+        data = {self.prov_cfg: ("key=value\nkey2=val2\n", -10),
+                self.inst_log: ("log data\n", -30),
+                self.boot_ref: ("PWD=/", 0)}
+        populate_dir_with_ts(rootd, data)
+        ret = self.call(rootd=rootd, func=self.funcname)
+        self.assertEqual(shell_false, ret.rc)
+        self.assertIn("from previous boot", ret.stderr)
+
+    def test_config_with_new_log(self):
+        """A config with a log from this boot is provisioning."""
+        rootd = self.tmp_dir()
+        data = {self.prov_cfg: ("key=value\nkey2=val2\n", -10),
+                self.inst_log: ("log data\n", 30),
+                self.boot_ref: ("PWD=/", 0)}
+        populate_dir_with_ts(rootd, data)
+        ret = self.call(rootd=rootd, func=self.funcname)
+        self.assertEqual(shell_true, ret.rc)
+        self.assertIn("from current boot", ret.stderr)
+
+
 def blkid_out(disks=None):
     """Convert a list of disk dictionaries into blkid content."""
     if disks is None:
@@ -639,6 +692,7 @@ VALID_CFG = {
         'ds': 'IBMCloud',
         'mocks': [
             MOCK_VIRT_IS_XEN,
+            {'name': 'is_ibm_provisioning', 'ret': shell_false},
             {'name': 'blkid', 'ret': 0,
              'out': blkid_out(
                  [{'DEVNAME': 'xvda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()},
@@ -652,6 +706,7 @@ VALID_CFG = {
         'ds': 'IBMCloud',
         'mocks': [
             MOCK_VIRT_IS_XEN,
+            {'name': 'is_ibm_provisioning', 'ret': shell_false},
             {'name': 'blkid', 'ret': 0,
              'out': blkid_out(
                  [{'DEVNAME': 'xvda1', 'TYPE': 'ext3', 'PARTUUID': uuid4(),
@@ -669,6 +724,7 @@ VALID_CFG = {
         'ds': 'IBMCloud',
         'mocks': [
             MOCK_VIRT_IS_XEN,
+            {'name': 'is_ibm_provisioning', 'ret': shell_false},
             {'name': 'blkid', 'ret': 0,
              'out': blkid_out(
                  [{'DEVNAME': 'xvda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()},
diff --git a/tools/ds-identify b/tools/ds-identify
index 9a2db5c4..7fff5d1e 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -125,6 +125,7 @@ DI_ON_NOTFOUND=""
 DI_EC2_STRICT_ID_DEFAULT="true"
 
 _IS_IBM_CLOUD=""
+_IS_IBM_PROVISIONING=""
 
 error() {
     set -- "ERROR:" "$@";
@@ -1006,7 +1007,25 @@ dscheck_Hetzner() {
 }
 
 is_ibm_provisioning() {
-    [ -f "${PATH_ROOT}/root/provisioningConfiguration.cfg" ]
+    local pcfg="${PATH_ROOT}/root/provisioningConfiguration.cfg"
+    local logf="${PATH_ROOT}/root/swinstall.log"
+    local is_prov=false msg="config '$pcfg' did not exist."
+    if [ -f "$pcfg" ]; then
+        msg="config '$pcfg' exists."
+        is_prov=true
+        if [ -f "$logf" ]; then
+            if [ "$logf" -nt "$PATH_PROC_1_ENVIRON" ]; then
+                msg="$msg log '$logf' from current boot."
+            else
+                is_prov=false
+                msg="$msg log '$logf' from previous boot."
+            fi
+        else
+            msg="$msg log '$logf' did not exist."
+        fi
+    fi
+    debug 2 "ibm_provisioning=$is_prov: $msg"
+    [ "$is_prov" = "true" ]
 }
 
 is_ibm_cloud() {
-- 
cgit v1.2.3


From 11172924a48a47a7231d19d9cefe628dfddda8bf Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Mon, 30 Apr 2018 13:21:51 -0600
Subject: IBMCloud: Disable config-drive and nocloud only if IBMCloud is
 enabled.

Ubuntu images on IBMCloud for 16.04 have some seed data in
/var/lib/cloud/data/seed/nocloud-net.  In order to have systems with
IBMCloud enabled, we modified ds-identify detection to skip that seed
if the system was on IBMCloud.  That change did not consider the
fact that IBMCloud might not be in the datasource list.

There was similar logic in the ConfigDrive datasource in ds-identify
and the datasource itself.

Config drive is now updated to only check and avoid IBMCloud if IBMCloud
is enabled.  The check in ds-identify for nocloud was dropped.  If a
user provides a nocloud seed on IBMCloud, then that can be used.

This means that systems running Xenial will continue to get their
old datasources.

LP: #1766401
---
 cloudinit/sources/DataSourceConfigDrive.py | 11 +++--
 tests/unittests/test_ds_identify.py        | 77 +++++++++++++++++++++++++++---
 tools/ds-identify                          | 17 +++++--
 3 files changed, 91 insertions(+), 14 deletions(-)

(limited to 'tools')

diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index c7b5fe5f..121cf215 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -69,7 +69,8 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
                 util.logexc(LOG, "Failed reading config drive from %s", sdir)
 
         if not found:
-            for dev in find_candidate_devs():
+            dslist = self.sys_cfg.get('datasource_list')
+            for dev in find_candidate_devs(dslist=dslist):
                 try:
                     # Set mtype if freebsd and turn off sync
                     if dev.startswith("/dev/cd"):
@@ -211,7 +212,7 @@ def write_injected_files(files):
                 util.logexc(LOG, "Failed writing file: %s", filename)
 
 
-def find_candidate_devs(probe_optical=True):
+def find_candidate_devs(probe_optical=True, dslist=None):
     """Return a list of devices that may contain the config drive.
 
     The returned list is sorted by search order where the first item has
@@ -227,6 +228,9 @@ def find_candidate_devs(probe_optical=True):
         * either vfat or iso9660 formated
         * labeled with 'config-2' or 'CONFIG-2'
     """
+    if dslist is None:
+        dslist = []
+
     # query optical drive to get it in blkid cache for 2.6 kernels
     if probe_optical:
         for device in OPTICAL_DEVICES:
@@ -257,7 +261,8 @@ def find_candidate_devs(probe_optical=True):
     devices = [d for d in candidates
                if d in by_label or not util.is_partition(d)]
 
-    if devices:
+    LOG.debug("devices=%s dslist=%s", devices, dslist)
+    if devices and "IBMCloud" in dslist:
         # IBMCloud uses config-2 label, but limited to a single UUID.
         ibm_platform, ibm_path = get_ibm_platform()
         if ibm_path in devices:
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index ad7fe41e..4d8a4360 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -184,17 +184,18 @@ class DsIdentifyBase(CiTestCase):
             data, RC_FOUND, dslist=[data.get('ds'), DS_NONE])
 
     def _check_via_dict(self, data, rc, dslist=None, **kwargs):
-        found_rc, out, err, cfg, files = self._call_via_dict(data, **kwargs)
+        ret = self._call_via_dict(data, **kwargs)
         good = False
         try:
-            self.assertEqual(rc, found_rc)
+            self.assertEqual(rc, ret.rc)
             if dslist is not None:
-                self.assertEqual(dslist, cfg['datasource_list'])
+                self.assertEqual(dslist, ret.cfg['datasource_list'])
             good = True
         finally:
             if not good:
-                _print_run_output(rc, out, err, cfg, files)
-        return rc, out, err, cfg, files
+                _print_run_output(ret.rc, ret.stdout, ret.stderr, ret.cfg,
+                                  ret.files)
+        return ret
 
 
 class TestDsIdentify(DsIdentifyBase):
@@ -245,13 +246,40 @@ class TestDsIdentify(DsIdentifyBase):
     def test_config_drive(self):
         """ConfigDrive datasource has a disk with LABEL=config-2."""
         self._test_ds_found('ConfigDrive')
-        return
 
     def test_config_drive_upper(self):
         """ConfigDrive datasource has a disk with LABEL=CONFIG-2."""
         self._test_ds_found('ConfigDriveUpper')
         return
 
+    def test_config_drive_seed(self):
+        """Config Drive seed directory."""
+        self._test_ds_found('ConfigDrive-seed')
+
+    def test_config_drive_interacts_with_ibmcloud_config_disk(self):
+        """Verify ConfigDrive interaction with IBMCloud.
+
+        If ConfigDrive is enabled and not IBMCloud, then ConfigDrive
+        should claim the ibmcloud 'config-2' disk.
+        If IBMCloud is enabled, then ConfigDrive should skip."""
+        data = copy.deepcopy(VALID_CFG['IBMCloud-config-2'])
+        files = data.get('files', {})
+        if not files:
+            data['files'] = files
+        cfgpath = 'etc/cloud/cloud.cfg.d/99_networklayer_common.cfg'
+
+        # with list including IBMCloud, config drive should be not found.
+        files[cfgpath] = 'datasource_list: [ ConfigDrive, IBMCloud ]\n'
+        ret = self._check_via_dict(data, shell_true)
+        self.assertEqual(
+            ret.cfg.get('datasource_list'), ['IBMCloud', 'None'])
+
+        # But if IBMCloud is not enabled, config drive should claim this.
+        files[cfgpath] = 'datasource_list: [ ConfigDrive, NoCloud ]\n'
+        ret = self._check_via_dict(data, shell_true)
+        self.assertEqual(
+            ret.cfg.get('datasource_list'), ['ConfigDrive', 'None'])
+
     def test_ibmcloud_template_userdata_in_provisioning(self):
         """Template provisioned with user-data during provisioning stage.
 
@@ -307,6 +335,37 @@ class TestDsIdentify(DsIdentifyBase):
         self._check_via_dict(
             data, rc=RC_FOUND, dslist=['ConfigDrive', DS_NONE])
 
+    def test_ibmcloud_with_nocloud_seed(self):
+        """NoCloud seed should be preferred over IBMCloud.
+
+        A nocloud seed should be preferred over IBMCloud even if enabled.
+        Ubuntu 16.04 images have <vlc>/seed/nocloud-net. LP: #1766401."""
+        data = copy.deepcopy(VALID_CFG['IBMCloud-config-2'])
+        files = data.get('files', {})
+        if not files:
+            data['files'] = files
+        files.update(VALID_CFG['NoCloud-seed']['files'])
+        ret = self._check_via_dict(data, shell_true)
+        self.assertEqual(
+            ['NoCloud', 'IBMCloud', 'None'],
+            ret.cfg.get('datasource_list'))
+
+    def test_ibmcloud_with_configdrive_seed(self):
+        """ConfigDrive seed should be preferred over IBMCloud.
+
+        A ConfigDrive seed should be preferred over IBMCloud even if enabled.
+        Ubuntu 16.04 images have a fstab entry that mounts the
+        METADATA disk into <vlc>/seed/config_drive. LP: ##1766401."""
+        data = copy.deepcopy(VALID_CFG['IBMCloud-config-2'])
+        files = data.get('files', {})
+        if not files:
+            data['files'] = files
+        files.update(VALID_CFG['ConfigDrive-seed']['files'])
+        ret = self._check_via_dict(data, shell_true)
+        self.assertEqual(
+            ['ConfigDrive', 'IBMCloud', 'None'],
+            ret.cfg.get('datasource_list'))
+
     def test_policy_disabled(self):
         """A Builtin policy of 'disabled' should return not found.
 
@@ -684,6 +743,12 @@ VALID_CFG = {
              },
         ],
     },
+    'ConfigDrive-seed': {
+        'ds': 'ConfigDrive',
+        'files': {
+            os.path.join(P_SEED_DIR, 'config_drive', 'openstack',
+                         'latest', 'meta_data.json'): 'md\n'},
+    },
     'Hetzner': {
         'ds': 'Hetzner',
         'files': {P_SYS_VENDOR: 'Hetzner\n'},
diff --git a/tools/ds-identify b/tools/ds-identify
index 7fff5d1e..9f0d96f7 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -601,7 +601,6 @@ dscheck_NoCloud() {
         *\ ds=nocloud*) return ${DS_FOUND};;
     esac
 
-    is_ibm_cloud && return ${DS_NOT_FOUND}
     for d in nocloud nocloud-net; do
         check_seed_dir "$d" meta-data user-data && return ${DS_FOUND}
         check_writable_seed_dir "$d" meta-data user-data && return ${DS_FOUND}
@@ -612,11 +611,12 @@ dscheck_NoCloud() {
     return ${DS_NOT_FOUND}
 }
 
+is_ds_enabled() {
+    local name="$1" pad=" ${DI_DSLIST} "
+    [ "${pad#* $name }" != "${pad}" ]
+}
+
 check_configdrive_v2() {
-    is_ibm_cloud && return ${DS_NOT_FOUND}
-    if has_fs_with_label CONFIG-2 config-2; then
-        return ${DS_FOUND}
-    fi
     # look in /config-drive <vlc>/seed/config_drive for a directory
     # openstack/YYYY-MM-DD format with a file meta_data.json
     local d=""
@@ -631,6 +631,13 @@ check_configdrive_v2() {
         debug 1 "config drive seeded directory had only 'latest'"
         return ${DS_FOUND}
     fi
+
+    is_ds_enabled "IBMCloud"
+    debug 1 "is_ds_enabled returned $?: $DI_DSLIST"
+    is_ds_enabled "IBMCloud" && is_ibm_cloud && return ${DS_NOT_FOUND}
+    if has_fs_with_label CONFIG-2 config-2; then
+        return ${DS_FOUND}
+    fi
     return ${DS_NOT_FOUND}
 }
 
-- 
cgit v1.2.3


From 4c1af5c7eb8db67f51f35130e13157a735256d2b Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Tue, 8 May 2018 13:59:33 -0400
Subject: ds-identify: make shellcheck 0.4.6 happy with ds-identify.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This fixes warnings reported by shellcheck at 0.4.6.
The complaints that we are ignoring globally (top of the file) are:
 2015: Note that A && B || C is not if-then-else. C may run if A is true.
 2039: In POSIX sh, 'local' is undefined.
 2162: read without -r will mangle backslashes.
 2166: Prefer [ p ] && [ q ] as [ p -a q ] is not well defined.

Most of the complaints were just noise, but a few unused variables
were reported and fixed.

Related shellcheck issues opened:
 - https://github.com/koalaman/shellcheck/issues/1191
 - https://github.com/koalaman/shellcheck/issues/1192
 - https://github.com/koalaman/shellcheck/issues/1193
 - https://github.com/koalaman/shellcheck/issues/1194
---
 tools/ds-identify | 50 ++++++++++++++++++++++++++++++++------------------
 1 file changed, 32 insertions(+), 18 deletions(-)

(limited to 'tools')

diff --git a/tools/ds-identify b/tools/ds-identify
index 9f0d96f7..a062e4d7 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -1,4 +1,5 @@
 #!/bin/sh
+# shellcheck disable=2015,2039,2162,2166
 #
 # ds-identify is configured via /etc/cloud/ds-identify.cfg
 # or on the kernel command line. It takes primarily 2 inputs:
@@ -125,7 +126,6 @@ DI_ON_NOTFOUND=""
 DI_EC2_STRICT_ID_DEFAULT="true"
 
 _IS_IBM_CLOUD=""
-_IS_IBM_PROVISIONING=""
 
 error() {
     set -- "ERROR:" "$@";
@@ -211,7 +211,9 @@ read_fs_info() {
     # 'set --' will collapse multiple consecutive entries in IFS for
     # whitespace characters (\n, tab, " ") so we cannot rely on getting
     # empty lines in "$@" below.
-    IFS="$CR"; set -- $out; IFS="$oifs"
+
+    # shellcheck disable=2086
+    { IFS="$CR"; set -- $out; IFS="$oifs"; }
 
     for line in "$@"; do
         case "${line}" in
@@ -311,6 +313,7 @@ read_dmi_product_serial() {
     DI_DMI_PRODUCT_SERIAL="$_RET"
 }
 
+# shellcheck disable=2034
 read_uname_info() {
     # run uname, and parse output.
     # uname is tricky to parse as it outputs always in a given order
@@ -330,6 +333,7 @@ read_uname_info() {
             return $ret
         }
     fi
+    # shellcheck disable=2086
     set -- $out
     DI_UNAME_KERNEL_NAME="$1"
     DI_UNAME_NODENAME="$2"
@@ -357,7 +361,8 @@ parse_yaml_array() {
     # the fix was to quote the open bracket (val=${val#"["}) (LP: #1689648)
     val=${val#"["}
     val=${val%"]"}
-    IFS=","; set -- $val; IFS="$oifs"
+    # shellcheck disable=2086
+    { IFS=","; set -- $val; IFS="$oifs"; }
     for tok in "$@"; do
         trim "$tok"
         unquote "$_RET"
@@ -393,7 +398,7 @@ read_datasource_list() {
     fi
     if [ -z "$dslist" ]; then
         dslist=${DI_DSLIST_DEFAULT}
-        debug 1 "no datasource_list found, using default:" $dslist
+        debug 1 "no datasource_list found, using default: $dslist"
     fi
     DI_DSLIST=$dslist
     return 0
@@ -404,7 +409,8 @@ read_pid1_product_name() {
     cached "${DI_PID_1_PRODUCT_NAME}" && return
     [ -r "${PATH_PROC_1_ENVIRON}" ] || return
     out=$(tr '\0' '\n' <"${PATH_PROC_1_ENVIRON}")
-    IFS="$CR"; set -- $out; IFS="$oifs"
+    # shellcheck disable=2086
+    { IFS="$CR"; set -- $out; IFS="$oifs"; }
     for tok in "$@"; do
         key=${tok%%=*}
         [ "$key" != "$tok" ] || continue
@@ -471,6 +477,7 @@ nocase_equal() {
     [ "$1" = "$2" ] && return 0
 
     local delim="-delim-"
+    # shellcheck disable=2018,2019
     out=$(echo "$1${delim}$2" | tr A-Z a-z)
     [ "${out#*${delim}}" = "${out%${delim}*}" ]
 }
@@ -547,11 +554,13 @@ check_config() {
     else
         files="$*"
     fi
-    set +f; set -- $files; set -f;
+    # shellcheck disable=2086
+    { set +f; set -- $files; set -f; }
     if [ "$1" = "$files" -a ! -f "$1" ]; then
         return 1
     fi
     local fname="" line="" ret="" found=0 found_fn=""
+    # shellcheck disable=2094
     for fname in "$@"; do
         [ -f "$fname" ] || continue
         while read line; do
@@ -794,7 +803,7 @@ ec2_read_strict_setting() {
     # 3. look for the key 'strict_id' (datasource/Ec2/strict_id)
     # only in cloud.cfg or cloud.cfg.d/EC2.cfg (case insensitive)
     local cfg="${PATH_ETC_CI_CFG}" cfg_d="${PATH_ETC_CI_CFG_D}"
-    if check_config strict_id $cfg "$cfg_d/*[Ee][Cc]2*.cfg"; then
+    if check_config strict_id "$cfg" "$cfg_d/*[Ee][Cc]2*.cfg"; then
         debug 2 "${_RET_fname} set strict_id to $_RET"
         return 0
     fi
@@ -1001,7 +1010,7 @@ dscheck_Scaleway() {
         *\ scaleway\ *) return ${DS_FOUND};;
     esac
 
-    if [ -f ${PATH_ROOT}/var/run/scaleway ]; then
+    if [ -f "${PATH_ROOT}/var/run/scaleway" ]; then
         return ${DS_FOUND}
     fi
 
@@ -1156,6 +1165,7 @@ found() {
 }
 
 trim() {
+    # shellcheck disable=2048,2086
     set -- $*
     _RET="$*"
 }
@@ -1176,7 +1186,7 @@ _read_config() {
     # if no parameters are set, modifies _rc scoped environment vars.
     # if keyname is provided, then returns found value of that key.
     local keyname="${1:-_unset}"
-    local line="" hash="#" ckey="" key="" val=""
+    local line="" hash="#" key="" val=""
     while read line; do
         line=${line%%${hash}*}
         key="${line%%:*}"
@@ -1254,7 +1264,8 @@ parse_policy() {
 
     local mode="" report="" found="" maybe="" notfound=""
     local oifs="$IFS" tok="" val=""
-    IFS=","; set -- $policy; IFS="$oifs"
+    # shellcheck disable=2086
+    { IFS=","; set -- $policy; IFS="$oifs"; }
     for tok in "$@"; do
         val=${tok#*=}
         case "$tok" in
@@ -1321,15 +1332,15 @@ manual_clean_and_existing() {
 }
 
 read_uptime() {
-    local up idle
+    local up _
     _RET="${UNAVAILABLE}"
-    [ -f "$PATH_PROC_UPTIME" ] &&
-        read up idle < "$PATH_PROC_UPTIME" && _RET="$up"
+    [ -f "$PATH_PROC_UPTIME" ] && read up _ < "$PATH_PROC_UPTIME" &&
+        _RET="$up"
     return
 }
 
 _main() {
-    local dscheck="" ret_dis=1 ret_en=0
+    local dscheck_fn="" ret_dis=1 ret_en=0
 
     read_uptime
     debug 1 "[up ${_RET}s]" "ds-identify $*"
@@ -1364,8 +1375,9 @@ _main() {
         return
     fi
 
-    # if there is only a single entry in $DI_DSLIST
+    # shellcheck disable=2086
     set -- $DI_DSLIST
+    # if there is only a single entry in $DI_DSLIST
     if [ $# -eq 1 ] || [ $# -eq 2 -a "$2" = "None" ] ; then
         debug 1 "single entry in datasource_list ($DI_DSLIST) use that."
         found "$@"
@@ -1398,6 +1410,7 @@ _main() {
     done
 
     debug 2 "found=${found# } maybe=${maybe# }"
+    # shellcheck disable=2086
     set -- $found
     if [ $# -ne 0 ]; then
         if [ $# -eq 1 ]; then
@@ -1413,6 +1426,7 @@ _main() {
         return
     fi
 
+    # shellcheck disable=2086
     set -- $maybe
     if [ $# -ne 0 -a "${DI_ON_MAYBE}" != "none" ]; then
         debug 1 "$# datasources returned maybe: $*"
@@ -1441,7 +1455,7 @@ _main() {
         *) error "Unexpected result";;
     esac
     debug 1 "$msg"
-    return $ret
+    return "$ret"
 }
 
 main() {
@@ -1452,7 +1466,7 @@ main() {
         if read ret < "$PATH_RUN_DI_RESULT"; then
             if [ "$ret" = "0" ] || [ "$ret" = "1" ]; then
                 debug 2 "used cached result $ret. pass --force to re-run."
-                return $ret;
+                return "$ret";
             fi
             debug 1 "previous run returned unexpected '$ret'. Re-running."
         else
@@ -1464,7 +1478,7 @@ main() {
     echo "$ret" > "$PATH_RUN_DI_RESULT"
     read_uptime
     debug 1 "[up ${_RET}s]" "returning $ret"
-    return $ret
+    return "$ret"
 }
 
 noop() {
-- 
cgit v1.2.3


From bde30070ec5f20aeb4d48cee8cf6c49b900ee311 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 9 May 2018 20:08:53 -0600
Subject: ds-identify: Remove dupe call to is_ds_enabled, improve debug
 message.

We had two calls to is_ds_enabled, and the debug message looked
something like this:
  is_ds_enabled returned 1: ConfigDrive NoCloud
Now instead we have just one call, and the debug message like:
  is_ds_enabled(IBMCloud) = true
---
 tools/ds-identify | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'tools')

diff --git a/tools/ds-identify b/tools/ds-identify
index a062e4d7..435a5bcb 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -641,9 +641,11 @@ check_configdrive_v2() {
         return ${DS_FOUND}
     fi
 
-    is_ds_enabled "IBMCloud"
-    debug 1 "is_ds_enabled returned $?: $DI_DSLIST"
-    is_ds_enabled "IBMCloud" && is_ibm_cloud && return ${DS_NOT_FOUND}
+    local ibm_enabled=false
+    is_ds_enabled "IBMCloud" && ibm_enabled=true
+    debug 1 "is_ds_enabled(IBMCloud) = $ibm_enabled."
+    [ "$ibm_enabled" = "true" ] && is_ibm_cloud && return ${DS_NOT_FOUND}
+
     if has_fs_with_label CONFIG-2 config-2; then
         return ${DS_FOUND}
     fi
-- 
cgit v1.2.3


From 0d7ee5592621d09699d079945ffd6febf16669b2 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Tue, 15 May 2018 13:38:20 -0400
Subject: ds-identify: recognize container-other as a container, test SmartOS.

In playing with a SmartOS container I found that ds-identify did
not identify the container there as a container.  Systemd-detect-virt
identifies it as 'container-other'.

Also here are tests for ds-identify for the SmartOS platform
identification, and some indentation fixes in ds-identify.
---
 cloudinit/sources/DataSourceSmartOS.py |  4 +--
 tests/unittests/test_ds_identify.py    | 51 ++++++++++++++++++++++++++++++++--
 tools/ds-identify                      | 12 ++++----
 3 files changed, 57 insertions(+), 10 deletions(-)

(limited to 'tools')

diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 4ea00eb1..fcb46b14 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -745,7 +745,7 @@ def get_smartos_environ(uname_version=None, product_name=None):
     # report 'BrandZ virtual linux' as the kernel version
     if uname_version is None:
         uname_version = uname[3]
-    if uname_version.lower() == 'brandz virtual linux':
+    if uname_version == 'BrandZ virtual linux':
         return SMARTOS_ENV_LX_BRAND
 
     if product_name is None:
@@ -753,7 +753,7 @@ def get_smartos_environ(uname_version=None, product_name=None):
     else:
         system_type = product_name
 
-    if system_type and 'smartdc' in system_type.lower():
+    if system_type and system_type.startswith('SmartDC'):
         return SMARTOS_ENV_KVM
 
     return None
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 4d8a4360..7f12be59 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -10,7 +10,8 @@ from cloudinit import util
 from cloudinit.tests.helpers import (
     CiTestCase, dir2dict, populate_dir, populate_dir_with_ts)
 
-from cloudinit.sources import DataSourceIBMCloud as dsibm
+from cloudinit.sources import DataSourceIBMCloud as ds_ibm
+from cloudinit.sources import DataSourceSmartOS as ds_smartos
 
 UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu "
                "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux")
@@ -69,8 +70,12 @@ P_DSID_CFG = "etc/cloud/ds-identify.cfg"
 
 IBM_CONFIG_UUID = "9796-932E"
 
+MOCK_VIRT_IS_CONTAINER_OTHER = {'name': 'detect_virt',
+                                'RET': 'container-other', 'ret': 0}
 MOCK_VIRT_IS_KVM = {'name': 'detect_virt', 'RET': 'kvm', 'ret': 0}
 MOCK_VIRT_IS_VMWARE = {'name': 'detect_virt', 'RET': 'vmware', 'ret': 0}
+# currenty' SmartOS hypervisor "bhyve" is unknown by systemd-detect-virt.
+MOCK_VIRT_IS_VM_OTHER = {'name': 'detect_virt', 'RET': 'vm-other', 'ret': 0}
 MOCK_VIRT_IS_XEN = {'name': 'detect_virt', 'RET': 'xen', 'ret': 0}
 MOCK_UNAME_IS_PPC64 = {'name': 'uname', 'out': UNAME_PPC64EL, 'ret': 0}
 
@@ -330,7 +335,7 @@ class TestDsIdentify(DsIdentifyBase):
                 break
         if not offset:
             raise ValueError("Expected to find 'blkid' mock, but did not.")
-        data['mocks'][offset]['out'] = d['out'].replace(dsibm.IBM_CONFIG_UUID,
+        data['mocks'][offset]['out'] = d['out'].replace(ds_ibm.IBM_CONFIG_UUID,
                                                         "DEAD-BEEF")
         self._check_via_dict(
             data, rc=RC_FOUND, dslist=['ConfigDrive', DS_NONE])
@@ -516,6 +521,20 @@ class TestDsIdentify(DsIdentifyBase):
         """Hetzner cloud is identified in sys_vendor."""
         self._test_ds_found('Hetzner')
 
+    def test_smartos_bhyve(self):
+        """SmartOS cloud identified by SmartDC in dmi."""
+        self._test_ds_found('SmartOS-bhyve')
+
+    def test_smartos_lxbrand(self):
+        """SmartOS cloud identified on lxbrand container."""
+        self._test_ds_found('SmartOS-lxbrand')
+
+    def test_smartos_lxbrand_requires_socket(self):
+        """SmartOS cloud should not be identified if no socket file."""
+        mycfg = copy.deepcopy(VALID_CFG['SmartOS-lxbrand'])
+        del mycfg['files'][ds_smartos.METADATA_SOCKFILE]
+        self._check_via_dict(mycfg, rc=RC_NOT_FOUND, policy_dmi="disabled")
+
 
 class TestIsIBMProvisioning(DsIdentifyBase):
     """Test the is_ibm_provisioning method in ds-identify."""
@@ -777,7 +796,7 @@ VALID_CFG = {
                  [{'DEVNAME': 'xvda1', 'TYPE': 'ext3', 'PARTUUID': uuid4(),
                    'UUID': uuid4(), 'LABEL': 'cloudimg-bootfs'},
                   {'DEVNAME': 'xvdb', 'TYPE': 'vfat', 'LABEL': 'config-2',
-                   'UUID': dsibm.IBM_CONFIG_UUID},
+                   'UUID': ds_ibm.IBM_CONFIG_UUID},
                   {'DEVNAME': 'xvda2', 'TYPE': 'ext4',
                    'LABEL': 'cloudimg-rootfs', 'PARTUUID': uuid4(),
                    'UUID': uuid4()},
@@ -798,6 +817,32 @@ VALID_CFG = {
              },
         ],
     },
+    'SmartOS-bhyve': {
+        'ds': 'SmartOS',
+        'mocks': [
+            MOCK_VIRT_IS_VM_OTHER,
+            {'name': 'blkid', 'ret': 0,
+             'out': blkid_out(
+                 [{'DEVNAME': 'vda1', 'TYPE': 'ext4',
+                   'PARTUUID': '49ec635a-01'},
+                  {'DEVNAME': 'vda2', 'TYPE': 'swap',
+                   'LABEL': 'cloudimg-swap', 'PARTUUID': '49ec635a-02'}]),
+             },
+        ],
+        'files': {P_PRODUCT_NAME: 'SmartDC HVM\n'},
+    },
+    'SmartOS-lxbrand': {
+        'ds': 'SmartOS',
+        'mocks': [
+            MOCK_VIRT_IS_CONTAINER_OTHER,
+            {'name': 'uname', 'ret': 0,
+             'out': ("Linux d43da87a-daca-60e8-e6d4-d2ed372662a3 4.3.0 "
+                     "BrandZ virtual linux x86_64 GNU/Linux")},
+            {'name': 'blkid', 'ret': 2, 'out': ''},
+        ],
+        'files': {ds_smartos.METADATA_SOCKFILE: 'would be a socket\n'},
+    }
+
 }
 
 # vi: ts=4 expandtab
diff --git a/tools/ds-identify b/tools/ds-identify
index 435a5bcb..dc7ac3a7 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -261,7 +261,7 @@ read_virt() {
 
 is_container() {
     case "${DI_VIRT}" in
-        lxc|lxc-libvirt|systemd-nspawn|docker|rkt) return 0;;
+        container-other|lxc|lxc-libvirt|systemd-nspawn|docker|rkt) return 0;;
         *) return 1;;
     esac
 }
@@ -990,12 +990,14 @@ dscheck_SmartOS() {
     # joyent cloud has two virt types: kvm and container
     # on kvm, product name on joyent public cloud shows 'SmartDC HVM'
     # on the container platform, uname's version has: BrandZ virtual linux
+    # for container, we also verify that the socketfile exists to protect
+    # against embedded containers (lxd running on brandz)
     local smartdc_kver="BrandZ virtual linux"
+    local metadata_sockfile="${PATH_ROOT}/native/.zonecontrol/metadata.sock"
     dmi_product_name_matches "SmartDC*" && return $DS_FOUND
-    if [ "${DI_UNAME_KERNEL_VERSION}" = "${smartdc_kver}" ] &&
-       [ "${DI_VIRT}" = "container-other" ]; then
-       return ${DS_FOUND}
-    fi
+    [ "${DI_UNAME_KERNEL_VERSION}" = "${smartdc_kver}" ] &&
+        [ -e "${metadata_sockfile}" ] &&
+        return ${DS_FOUND}
     return ${DS_NOT_FOUND}
 }
 
-- 
cgit v1.2.3


From b4ae0e1fb8a48a83ea325cf032eb1acb196ee31c Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Tue, 22 May 2018 10:07:59 -0400
Subject: ds-identify: ensure that we have certain tokens in PATH.

SuSE builds were not getting a PATH set in generator's environment.
This may seem like mis-configuration on the system, but caused ds-identify
to fail to find blkid (or any other program).

The change here just ensures that we get /sbin /usr/sbin /bin /usr/bin
into the PATH when main is run.

LP: #1771382
---
 tests/unittests/test_ds_identify.py | 23 ++++++++++++++++++++++-
 tools/ds-identify                   | 11 +++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)

(limited to 'tools')

diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 7f12be59..64d9f9f8 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -175,7 +175,9 @@ class DsIdentifyBase(CiTestCase):
     def _call_via_dict(self, data, rootd=None, **kwargs):
         # return output of self.call with a dict input like VALID_CFG[item]
         xwargs = {'rootd': rootd}
-        for k in ('mocks', 'args', 'policy_dmi', 'policy_no_dmi', 'files'):
+        passthrough = ('mocks', 'func', 'args', 'policy_dmi',
+                       'policy_no_dmi', 'files')
+        for k in passthrough:
             if k in data:
                 xwargs[k] = data[k]
             if k in kwargs:
@@ -535,6 +537,25 @@ class TestDsIdentify(DsIdentifyBase):
         del mycfg['files'][ds_smartos.METADATA_SOCKFILE]
         self._check_via_dict(mycfg, rc=RC_NOT_FOUND, policy_dmi="disabled")
 
+    def test_path_env_gets_set_from_main(self):
+        """PATH environment should always have some tokens when main is run.
+
+        We explicitly call main as we want to ensure it updates PATH."""
+        cust = copy.deepcopy(VALID_CFG['NoCloud'])
+        rootd = self.tmp_dir()
+        mpp = 'main-printpath'
+        pre = "MYPATH="
+        cust['files'][mpp] = (
+            'PATH="/mycust/path"; main; r=$?; echo ' + pre + '$PATH; exit $r;')
+        ret = self._check_via_dict(
+            cust, RC_FOUND,
+            func=".", args=[os.path.join(rootd, mpp)], rootd=rootd)
+        line = [l for l in ret.stdout.splitlines() if l.startswith(pre)][0]
+        toks = line.replace(pre, "").split(":")
+        expected = ["/sbin", "/bin", "/usr/sbin", "/usr/bin", "/mycust/path"]
+        self.assertEqual(expected, [p for p in expected if p in toks],
+                         "path did not have expected tokens")
+
 
 class TestIsIBMProvisioning(DsIdentifyBase):
     """Test the is_ibm_provisioning method in ds-identify."""
diff --git a/tools/ds-identify b/tools/ds-identify
index dc7ac3a7..ce0477a5 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -187,6 +187,16 @@ block_dev_with_label() {
     return 0
 }
 
+ensure_sane_path() {
+    local t
+    for t in /sbin /usr/sbin /bin /usr/bin; do
+        case ":$PATH:" in
+            *:$t:*|*:$t/:*) continue;;
+        esac
+        PATH="${PATH:+${PATH}:}$t"
+    done
+}
+
 read_fs_info() {
     cached "${DI_BLKID_OUTPUT}" && return 0
     # do not rely on links in /dev/disk which might not be present yet.
@@ -1464,6 +1474,7 @@ _main() {
 
 main() {
     local ret=""
+    ensure_sane_path
     [ -d "$PATH_RUN_CI" ] || mkdir -p "$PATH_RUN_CI"
     if [ "${1:+$1}" != "--force" ] && [ -f "$PATH_RUN_CI_CFG" ] &&
         [ -f "$PATH_RUN_DI_RESULT" ]; then
-- 
cgit v1.2.3


From a821cde1722b18ec54279a5fbfb5da96032ef95e Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 23 May 2018 12:31:48 -0400
Subject: tools/run-container: replace tools/run-centos with more generic.

tools/run-container is like tools/run-centos, but currently supports
the following images from lxc-images
   opensuse/42.3
   centos/6
   centos/7
   ubuntu/16.04
   debian/10
   debian/sid

Also here is to make installation via zypper in tools/read-dependencies
not prompt user.
---
 tools/read-dependencies |   8 +-
 tools/run-centos        | 340 +++-------------------------
 tools/run-container     | 590 ++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 626 insertions(+), 312 deletions(-)
 create mode 100755 tools/run-container

(limited to 'tools')

diff --git a/tools/read-dependencies b/tools/read-dependencies
index 421f470a..b4656e69 100755
--- a/tools/read-dependencies
+++ b/tools/read-dependencies
@@ -51,6 +51,10 @@ MAYBE_RELIABLE_YUM_INSTALL = [
     """,
     'reliable-yum-install']
 
+ZYPPER_INSTALL = [
+    'zypper', '--non-interactive', '--gpg-auto-import-keys', 'install',
+    '--auto-agree-with-licenses']
+
 DRY_DISTRO_INSTALL_PKG_CMD = {
     'centos': ['yum', 'install', '--assumeyes'],
     'redhat': ['yum', 'install', '--assumeyes'],
@@ -61,8 +65,8 @@ DISTRO_INSTALL_PKG_CMD = {
     'redhat': MAYBE_RELIABLE_YUM_INSTALL,
     'debian': ['apt', 'install', '-y'],
     'ubuntu': ['apt', 'install', '-y'],
-    'opensuse': ['zypper', 'install'],
-    'suse': ['zypper', 'install']
+    'opensuse': ZYPPER_INSTALL,
+    'suse': ZYPPER_INSTALL,
 }
 
 
diff --git a/tools/run-centos b/tools/run-centos
index cb241ee5..4506b20d 100755
--- a/tools/run-centos
+++ b/tools/run-centos
@@ -1,18 +1,17 @@
 #!/bin/bash
 # This file is part of cloud-init. See LICENSE file for license information.
 
-set -u
-
-VERBOSITY=0
-TEMP_D=""
-KEEP=false
-CONTAINER=""
-
-error() { echo "$@" 1>&2; }
-fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
-errorrc() { local r=$?; error "$@" "ret=$r"; return $r; }
+deprecated() {
+cat <<EOF
+             ================ DEPRECATED ================
+             | run-centos is deprecated. Please replace |
+             | your usage with tools/run-container .    |
+             ================ DEPRECATED ================
+EOF
+}
 
 Usage() {
+    deprecated
     cat <<EOF
 Usage: ${0##*/} [ options ] version
 
@@ -33,320 +32,41 @@ Usage: ${0##*/} [ options ] version
 
     Example:
       * ${0##*/} --rpm --srpm --unittest 6
+EOF
+    deprecated
 EOF
 }
 
 bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; return 1; }
-cleanup() {
-    if [ -n "$CONTAINER" -a "$KEEP" = "false" ]; then
-        delete_container "$CONTAINER"
-    fi
-    [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
-}
-
-debug() {
-    local level=${1}; shift;
-    [ "${level}" -gt "${VERBOSITY}" ] && return
-    error "${@}"
-}
-
-
-inside_as() {
-    # inside_as(container_name, user, cmd[, args])
-    # executes cmd with args inside container as user in users home dir.
-    local name="$1" user="$2"
-    shift 2
-    if [ "$user" = "root" ]; then
-        inside "$name" "$@"
-        return
-    fi
-    local stuffed="" b64=""
-    stuffed=$(getopt --shell sh --options "" -- -- "$@")
-    stuffed=${stuffed# -- }
-    b64=$(printf "%s\n" "$stuffed" | base64 --wrap=0)
-    inside "$name" su "$user" -c \
-        'cd; eval set -- "$(echo '$b64' | base64 --decode)" && exec "$@"'
-}
-
-inside_as_cd() {
-    local name="$1" user="$2" dir="$3"
-    shift 3
-    inside_as "$name" "$user" sh -c 'cd "$0" && exec "$@"' "$dir" "$@"
-}
-
-inside() {
-    local name="$1"
-    shift
-    lxc exec "$name" -- "$@"
-}
-
-inject_cloud_init(){
-    # take current cloud-init git dir and put it inside $name at
-    # ~$user/cloud-init.
-    local name="$1" user="$2" dirty="$3"
-    local changes="" top_d="" dname="cloud-init" pstat=""
-    local gitdir="" commitish=""
-    gitdir=$(git rev-parse --git-dir) || {
-        errorrc "Failed to get git dir in $PWD";
-        return
-    }
-    local t=${gitdir%/*}
-    case "$t" in
-        */worktrees) 
-            if [ -f "${t%worktrees}/config" ]; then
-                gitdir="${t%worktrees}"
-            fi
-    esac
-
-    # attempt to get branch name.
-    commitish=$(git rev-parse --abbrev-ref HEAD) || {
-        errorrc "Failed git rev-parse --abbrev-ref HEAD"
-        return
-    }
-    if [ "$commitish" = "HEAD" ]; then
-        # detached head
-        commitish=$(git rev-parse HEAD) || {
-            errorrc "failed git rev-parse HEAD"
-            return
-        }
-    fi
-
-    local local_changes=false
-    if ! git diff --quiet "$commitish"; then
-        # there are local changes not committed.
-        local_changes=true
-        if [ "$dirty" = "false" ]; then
-            error "WARNING: You had uncommitted changes.  Those changes will "
-            error "be put into 'local-changes.diff' inside the container. "
-            error "To test these changes you must pass --dirty."
-        fi
-    fi
-
-    debug 1 "collecting ${gitdir} ($dname) into user $user in $name."
-    tar -C "${gitdir}" -cpf - . |
-        inside_as "$name" "$user" sh -ec '
-            dname=$1
-            commitish=$2
-            rm -Rf "$dname"
-            mkdir -p $dname/.git
-            cd $dname/.git
-            tar -xpf -
-            cd ..
-            git config core.bare false
-            out=$(git checkout $commitish 2>&1) ||
-                { echo "failed git checkout $commitish: $out" 1>&2; exit 1; }
-            out=$(git checkout . 2>&1) ||
-                { echo "failed git checkout .: $out" 1>&2; exit 1; }
-            ' extract "$dname" "$commitish"
-    [ "${PIPESTATUS[*]}" = "0 0" ] || {
-        error "Failed to push tarball of '$gitdir' into $name" \
-            " for user $user (dname=$dname)"
-        return 1
-    }
 
-    echo "local_changes=$local_changes dirty=$dirty"
-    if [ "$local_changes" = "true" ]; then
-        git diff "$commitish" |
-            inside_as "$name" "$user" sh -exc '
-                cd "$1"
-                if [ "$2" = "true" ]; then
-                    git apply
-                else
-                    cat > local-changes.diff
-                fi
-                ' insert_changes "$dname" "$dirty"
-        [ "${PIPESTATUS[*]}" = "0 0" ] || {
-            error "Failed to apply local changes."
-            return 1
-        }
-    fi
-
-    return 0
-}
-
-prep() {
-    # we need some very basic things not present in the container.
-    #  - git
-    #  - tar (CentOS 6 lxc container does not have it)
-    #  - python-argparse (or python3)
-    local needed="" pair="" pkg="" cmd="" needed=""
-    for pair in tar:tar git:git; do
-        pkg=${pair#*:}
-        cmd=${pair%%:*}
-        command -v $cmd >/dev/null 2>&1 || needed="${needed} $pkg"
-    done
-    if ! command -v python3; then
-        python -c "import argparse" >/dev/null 2>&1 ||
-            needed="${needed} python-argparse"
-    fi
-    needed=${needed# }
-    if [ -z "$needed" ]; then
-        error "No prep packages needed"
-        return 0
+main() {
+    if [ "$1" = "-h" -o "$1" == "--help" ]; then
+        Usage 1>&2;
+        exit 0;
     fi
-    error "Installing prep packages: ${needed}"
-    set -- $needed
-    local n max r
-    n=0; max=10;
-    bcmd="yum install --downloadonly --assumeyes --setopt=keepcache=1"
-    while n=$(($n+1)); do
-       error ":: running $bcmd $* [$n/$max]"
-       $bcmd "$@"
-       r=$?
-       [ $r -eq 0 ] && break
-       [ $n -ge $max ] && { error "gave up on $bcmd"; exit $r; }
-       nap=$(($n*5))
-       error ":: failed [$r] ($n/$max). sleeping $nap."
-       sleep $nap
-    done
-    error ":: running yum install --cacheonly --assumeyes $*"
-    yum install --cacheonly --assumeyes "$@"
-}
-
-start_container() {
-    local src="$1" name="$2"
-    debug 1 "starting container $name from '$src'"
-    lxc launch "$src" "$name" || {
-        errorrc "Failed to start container '$name' from '$src'";
+    local pt="" mydir=$(dirname "$0")
+    local run_container="$mydir/run-container"
+    if [ ! -x "$run_container" ]; then
+        bad_Usage "Could not find run-container."
         return
-    }
-    CONTAINER=$name
-
-    local out="" ret=""
-    debug 1 "waiting for networking"
-    out=$(inside "$name" sh -c '
-        i=0
-        while [ $i -lt 60 ]; do
-            getent hosts mirrorlist.centos.org && exit 0
-            sleep 2
-        done' 2>&1)
-    ret=$?
-    if [ $ret -ne 0 ]; then
-        error "Waiting for network in container '$name' failed. [$ret]"
-        error "$out"
-        return $ret
-    fi
-
-    if [ ! -z "${http_proxy-}" ]; then
-        debug 1 "configuring proxy ${http_proxy}"
-        inside "$name" sh -c "echo proxy=$http_proxy >> /etc/yum.conf"
-        inside "$name" sed -i s/enabled=1/enabled=0/ /etc/yum/pluginconf.d/fastestmirror.conf
     fi
-}
-
-delete_container() {
-    debug 1 "removing container $1 [--keep to keep]"
-    lxc delete --force "$1"
-}
-
-main() {
-    local short_opts="ahkrsuv"
-    local long_opts="artifact,dirty,help,keep,rpm,srpm,unittest,verbose"
-    local getopt_out=""
-    getopt_out=$(getopt --name "${0##*/}" \
-        --options "${short_opts}" --long "${long_opts}" -- "$@") &&
-        eval set -- "${getopt_out}" ||
-        { bad_Usage; return; }
-
-    local cur="" next=""
-    local artifact="" keep="" rpm="" srpm="" unittest="" version=""
-    local dirty=false
-
+    
+    pt=( "$run_container" )
     while [ $# -ne 0 ]; do
         cur="${1:-}"; next="${2:-}";
         case "$cur" in
-            -a|--artifact) artifact=1;;
-               --dirty) dirty=true;;
-            -h|--help) Usage ; exit 0;;
-            -k|--keep) KEEP=true;;
-            -r|--rpm) rpm=1;;
-            -s|--srpm) srpm=1;;
-            -u|--unittest) unittest=1;;
-            -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));;
-            --) shift; break;;
+            -r|--rpm) cur="--package";;
+            -s|--srpm) cur="--source-package";;
+            -a|--artifact) cur="--artifacts=.";;
+            6|7) cur="centos/$cur";;
         esac
+        pt[${#pt[@]}]="$cur"
         shift;
     done
-
-    [ $# -eq 1 ] || { bad_Usage "ERROR: Must provide version!"; return; }
-    version="$1"
-    case "$version" in
-        6|7) :;;
-        *) error "Expected version of 6 or 7, not '$version'"; return;;
-    esac
-
-    TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") ||
-        fail "failed to make tempdir"
-    trap cleanup EXIT
-
-    # program starts here
-    local uuid="" name="" user="ci-test" cdir=""
-    cdir="/home/$user/cloud-init"
-    uuid=$(uuidgen -t) || { error "no uuidgen"; return 1; }
-    name="cloud-init-centos-${uuid%%-*}"
-
-    start_container "images:centos/$version" "$name"
-
-    # prep the container (install very basic dependencies)
-    inside "$name" bash -s prep <"$0" ||
-        { errorrc "Failed to prep container $name"; return; }
-
-    # add the user
-    inside "$name" useradd "$user"
-
-    debug 1 "inserting cloud-init"
-    inject_cloud_init "$name" "$user" "$dirty" || {
-        errorrc "FAIL: injecting cloud-init into $name failed."
-        return
-    }
-
-    inside_as_cd "$name" root "$cdir" \
-        ./tools/read-dependencies --distro=centos --test-distro || {
-        errorrc "FAIL: failed to install dependencies with read-dependencies"
-        return
-    }
-
-    local errors=0
-    inside_as_cd "$name" "$user" "$cdir" \
-        sh -ec "git status" ||
-            { errorrc "git checkout failed."; errors=$(($errors+1)); }
-
-    if [ -n "$unittest" ]; then
-        debug 1 "running unit tests."
-        inside_as_cd "$name" "$user" "$cdir" \
-            nosetests tests/unittests cloudinit ||
-            { errorrc "nosetests failed."; errors=$(($errors+1)); }
-    fi
-
-    if [ -n "$srpm" ]; then
-        debug 1 "building srpm."
-        inside_as_cd "$name" "$user" "$cdir" ./packages/brpm --srpm ||
-            { errorrc "brpm --srpm."; errors=$(($errors+1)); }
-    fi
-
-    if [ -n "$rpm" ]; then
-        debug 1 "building rpm."
-        inside_as_cd "$name" "$user" "$cdir" ./packages/brpm ||
-            { errorrc "brpm failed."; errors=$(($errors+1)); }
-    fi
-
-    if [ -n "$artifact" ]; then
-        for built_rpm in $(inside "$name" sh -c "echo $cdir/*.rpm"); do
-            lxc file pull "$name/$built_rpm" .
-        done
-    fi
-
-    if [ "$errors" != "0" ]; then
-        error "there were $errors errors."
-        return 1
-    fi
-    return 0
+    deprecated
+    exec "${pt[@]}"
 }
 
-if [ "${1:-}" = "prep" ]; then
-    shift
-    prep "$@"
-else
-    main "$@"
-fi
+main "$@"
+
 # vi: ts=4 expandtab
diff --git a/tools/run-container b/tools/run-container
new file mode 100755
index 00000000..499e85b0
--- /dev/null
+++ b/tools/run-container
@@ -0,0 +1,590 @@
+#!/bin/bash
+# This file is part of cloud-init. See LICENSE file for license information.
+#
+# shellcheck disable=2015,2016,2039,2162,2166
+
+set -u
+
+VERBOSITY=0
+KEEP=false
+CONTAINER=""
+DEFAULT_WAIT_MAX=30
+
+error() { echo "$@" 1>&2; }
+fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
+errorrc() { local r=$?; error "$@" "ret=$r"; return $r; }
+
+Usage() {
+    cat <<EOF
+Usage: ${0##*/} [ options ] [images:]image-ref
+
+    This utility can makes it easier to run tests, build rpm and source rpm
+        generation inside a LXC of the specified version of CentOS.
+
+    To see images available, run 'lxc image list images:'
+    Example input:
+       centos/7
+       opensuse/42.3
+       debian/10
+
+    options:
+      -a | --artifacts DIR   copy build artifacts out to DIR.
+                             by default artifacts are not copied out.
+           --dirty           apply local changes before running tests.
+                             If not provided, a clean checkout of branch is
+                             tested.  Inside container, changes are in
+                             local-changes.diff.
+      -k | --keep            keep container after tests
+           --pyexe V         python version to use.  Default=auto.
+                             Should be name of an executable.
+                             ('python2' or 'python3')
+      -p | --package         build a binary package (.deb or .rpm)
+      -s | --source-package  build source package (debuild -S or srpm)
+      -u | --unittest        run unit tests
+
+    Example:
+      * ${0##*/} --package --source-package --unittest centos/6
+EOF
+}
+
+bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; return 1; }
+cleanup() {
+    if [ -n "$CONTAINER" ]; then
+        if [ "$KEEP" = "true" ]; then
+            error "not deleting container '$CONTAINER' due to --keep"
+        else
+            delete_container "$CONTAINER"
+        fi
+    fi
+}
+
+debug() {
+    local level=${1}; shift;
+    [ "${level}" -gt "${VERBOSITY}" ] && return
+    error "${@}"
+}
+
+
+inside_as() {
+    # inside_as(container_name, user, cmd[, args])
+    # executes cmd with args inside container as user in users home dir.
+    local name="$1" user="$2"
+    shift 2
+    if [ "$user" = "root" ]; then
+        inside "$name" "$@"
+        return
+    fi
+    local stuffed="" b64=""
+    stuffed=$(getopt --shell sh --options "" -- -- "$@")
+    stuffed=${stuffed# -- }
+    b64=$(printf "%s\n" "$stuffed" | base64 --wrap=0)
+    inside "$name" su "$user" -c \
+        'cd; eval set -- "$(echo '"$b64"' | base64 --decode)" && exec "$@"';
+}
+
+inside_as_cd() {
+    local name="$1" user="$2" dir="$3"
+    shift 3
+    inside_as "$name" "$user" sh -c 'cd "$0" && exec "$@"' "$dir" "$@"
+}
+
+inside() {
+    local name="$1"
+    shift
+    lxc exec "$name" -- "$@"
+}
+
+inject_cloud_init(){
+    # take current cloud-init git dir and put it inside $name at
+    # ~$user/cloud-init.
+    local name="$1" user="$2" dirty="$3"
+    local dname="cloud-init" gitdir="" commitish=""
+    gitdir=$(git rev-parse --git-dir) || {
+        errorrc "Failed to get git dir in $PWD";
+        return
+    }
+    local t=${gitdir%/*}
+    case "$t" in
+        */worktrees) 
+            if [ -f "${t%worktrees}/config" ]; then
+                gitdir="${t%worktrees}"
+            fi
+    esac
+
+    # attempt to get branch name.
+    commitish=$(git rev-parse --abbrev-ref HEAD) || {
+        errorrc "Failed git rev-parse --abbrev-ref HEAD"
+        return
+    }
+    if [ "$commitish" = "HEAD" ]; then
+        # detached head
+        commitish=$(git rev-parse HEAD) || {
+            errorrc "failed git rev-parse HEAD"
+            return
+        }
+    fi
+
+    local local_changes=false
+    if ! git diff --quiet "$commitish"; then
+        # there are local changes not committed.
+        local_changes=true
+        if [ "$dirty" = "false" ]; then
+            error "WARNING: You had uncommitted changes.  Those changes will "
+            error "be put into 'local-changes.diff' inside the container. "
+            error "To test these changes you must pass --dirty."
+        fi
+    fi
+
+    debug 1 "collecting ${gitdir} ($dname) into user $user in $name."
+    tar -C "${gitdir}" -cpf - . |
+        inside_as "$name" "$user" sh -ec '
+            dname=$1
+            commitish=$2
+            rm -Rf "$dname"
+            mkdir -p $dname/.git
+            cd $dname/.git
+            tar -xpf -
+            cd ..
+            git config core.bare false
+            out=$(git checkout $commitish 2>&1) ||
+                { echo "failed git checkout $commitish: $out" 1>&2; exit 1; }
+            out=$(git checkout . 2>&1) ||
+                { echo "failed git checkout .: $out" 1>&2; exit 1; }
+            ' extract "$dname" "$commitish"
+    [ "${PIPESTATUS[*]}" = "0 0" ] || {
+        error "Failed to push tarball of '$gitdir' into $name" \
+            " for user $user (dname=$dname)"
+        return 1
+    }
+
+    echo "local_changes=$local_changes dirty=$dirty"
+    if [ "$local_changes" = "true" ]; then
+        git diff "$commitish" |
+            inside_as "$name" "$user" sh -exc '
+                cd "$1"
+                if [ "$2" = "true" ]; then
+                    git apply
+                else
+                    cat > local-changes.diff
+                fi
+                ' insert_changes "$dname" "$dirty"
+        [ "${PIPESTATUS[*]}" = "0 0" ] || {
+            error "Failed to apply local changes."
+            return 1
+        }
+    fi
+
+    return 0
+}
+
+get_os_info_in() {
+    # prep the container (install very basic dependencies)
+    [ -n "${OS_VERSION:-}" -a -n "${OS_NAME:-}" ] && return 0
+    data=$(run_self_inside "$name" os_info) ||
+        { errorrc "Failed to get os-info in container $name"; return; }
+    eval "$data" && [ -n "${OS_VERSION:-}" -a -n "${OS_NAME:-}" ] || return
+    debug 1 "determined $name is $OS_NAME/$OS_VERSION"
+}
+
+os_info() {
+    get_os_info || return
+    echo "OS_NAME=$OS_NAME"
+    echo "OS_VERSION=$OS_VERSION"
+}
+
+get_os_info() {
+    # run inside container, set OS_NAME, OS_VERSION
+    # example OS_NAME are centos, debian, opensuse
+    [ -n "${OS_NAME:-}" -a -n "${OS_VERSION:-}" ] && return 0
+    if [ -f /etc/os-release ]; then
+        OS_NAME=$(sh -c '. /etc/os-release; echo $ID')
+        OS_VERSION=$(sh -c '. /etc/os-release; echo $VERSION_ID')
+        if [ -z "$OS_VERSION" ]; then
+            local pname=""
+            pname=$(sh -c '. /etc/os-release; echo $PRETTY_NAME')
+            case "$pname" in
+                *buster*) OS_VERSION=10;;
+                *sid*) OS_VERSION="sid";;
+            esac
+        fi
+    elif [ -f /etc/centos-release ]; then
+        local line=""
+        read line < /etc/centos-release
+        case "$line" in
+            CentOS\ *\ 6.*) OS_VERSION="6"; OS_NAME="centos";;
+        esac
+    fi
+    [ -n "${OS_NAME:-}" -a -n "${OS_VERSION:-}" ] ||
+        { error "Unable to determine OS_NAME/OS_VERSION"; return 1; }
+}
+
+yum_install() {
+    local n=0 max=10 ret
+    bcmd="yum install --downloadonly --assumeyes --setopt=keepcache=1"
+    while n=$((n+1)); do
+       error ":: running $bcmd $* [$n/$max]"
+       $bcmd "$@"
+       ret=$?
+       [ $ret -eq 0 ] && break
+       [ $n -ge $max ] && { error "gave up on $bcmd"; exit $ret; }
+       nap=$((n*5))
+       error ":: failed [$ret] ($n/$max). sleeping $nap."
+       sleep $nap
+    done
+    error ":: running yum install --cacheonly --assumeyes $*"
+    yum install --cacheonly --assumeyes "$@"
+}
+
+zypper_install() {
+    local pkgs="$*"
+    set -- zypper --non-interactive --gpg-auto-import-keys install \
+        --auto-agree-with-licenses "$@"
+    debug 1 ":: installing $pkgs with zypper: $*"
+    "$@"
+}
+
+apt_install() {
+    apt-get update -q && apt-get install --no-install-recommends "$@"
+}
+
+install_packages() {
+    get_os_info || return
+    case "$OS_NAME" in
+        centos) yum_install "$@";;
+        opensuse) zypper_install "$@";;
+        debian|ubuntu) apt_install "$@";;
+        *) error "Do not know how to install packages on ${OS_NAME}";
+           return 1;;
+    esac
+}
+
+prep() {
+    # we need some very basic things not present in the container.
+    #  - git
+    #  - tar (CentOS 6 lxc container does not have it)
+    #  - python-argparse (or python3)
+    local needed="" pair="" pkg="" cmd="" needed=""
+    local pairs="tar:tar git:git"
+    local pyexe="$1"
+    get_os_info
+    local py2pkg="python2" py3pkg="python3"
+    case "$OS_NAME" in
+        opensuse)
+            py2pkg="python-base"
+            py3pkg="python3-base";;
+    esac
+
+    case "$pyexe" in
+        python2) pairs="$pairs python2:$py2pkg";;
+        python3) pairs="$pairs python3:$py3pkg";;
+    esac
+
+    for pair in $pairs; do
+        pkg=${pair#*:}
+        cmd=${pair%%:*}
+        command -v "$cmd" >/dev/null 2>&1 || needed="${needed} $pkg"
+    done
+    if [ "$OS_NAME" = "centos" -a "$pyexe" = "python2" ]; then
+        python -c "import argparse" >/dev/null 2>&1 ||
+            needed="${needed} python-argparse"
+    fi
+    needed=${needed# }
+    if [ -z "$needed" ]; then
+        error "No prep packages needed"
+        return 0
+    fi
+    error "Installing prep packages: ${needed}"
+    # shellcheck disable=SC2086
+    set -- $needed
+    install_packages "$@"
+}
+
+nose() {
+    local pyexe="$1" cmd=""
+    shift
+    get_os_info
+    if [ "$OS_NAME/$OS_VERSION" = "centos/6" ]; then
+        cmd="nosetests"
+    else
+        cmd="$pyexe -m nose"
+    fi
+    ${cmd} "$@"
+}
+
+is_done_cloudinit() {
+    [ -e "/run/cloud-init/result.json" ]
+    _RET=""
+}
+
+is_done_systemd() {
+    local s="" num="$1"
+    s=$(systemctl is-system-running 2>&1);
+    _RET="$? $s"
+    case "$s" in
+        initializing|starting) return 1;;
+        *[Ff]ailed*connect*bus*)
+            # warn if not the first run.
+            [ "$num" -lt 5 ] ||
+                error "Failed to connect to systemd bus [${_RET%% *}]";
+            return 1;;
+    esac
+    return 0
+}
+
+is_done_other() {
+    local out=""
+    out=$(getent hosts ubuntu.com 2>&1)
+    return
+}
+
+wait_inside() {
+    local name="$1" max="${2:-${DEFAULT_WAIT_MAX}}" debug=${3:-0}
+    local i=0 check="is_done_other";
+    if [ -e /run/systemd ]; then
+        check=is_done_systemd
+    elif [ -x /usr/bin/cloud-init ]; then
+        check=is_done_cloudinit
+    fi
+    [ "$debug" != "0" ] && debug 1 "check=$check"
+    while ! $check $i && i=$((i+1)); do
+        [ "$i" -ge "$max" ] && exit 1
+        [ "$debug" = "0" ] || echo -n .
+        sleep 1
+    done
+    if [ "$debug" != "0" ]; then
+        read up _ </proc/uptime
+        debug 1 "[$name ${i:+done after $i }up=$up${_RET:+ ${_RET}}]"
+    fi
+}
+
+wait_for_boot() {
+    local name="$1"
+    local out="" ret="" wtime=$DEFAULT_WAIT_MAX
+    get_os_info_in "$name"
+    [ "$OS_NAME" = "debian" ] && wtime=300 &&
+        debug 1 "on debian we wait for ${wtime}s"
+    debug 1 "waiting for boot of $name"
+    run_self_inside "$name" wait_inside "$name" "$wtime" "$VERBOSITY" ||
+        { errorrc "wait inside $name failed."; return; }
+
+    if [ ! -z "${http_proxy-}" ]; then
+        if [ "$OS_NAME" = "centos" ]; then
+            debug 1 "configuring proxy ${http_proxy}"
+            inside "$name" sh -c "echo proxy=$http_proxy >> /etc/yum.conf"
+            inside "$name" sed -i s/enabled=1/enabled=0/ \
+                /etc/yum/pluginconf.d/fastestmirror.conf
+        else
+            debug 1 "do not know how to configure proxy on $OS_NAME"
+        fi
+    fi
+}
+
+start_container() {
+    local src="$1" name="$2"
+    debug 1 "starting container $name from '$src'"
+    lxc launch "$src" "$name" || {
+        errorrc "Failed to start container '$name' from '$src'";
+        return
+    }
+    CONTAINER=$name
+    wait_for_boot "$name"
+}
+
+delete_container() {
+    debug 1 "removing container $1 [--keep to keep]"
+    lxc delete --force "$1"
+}
+
+run_self_inside() {
+    # run_self_inside(container, args)
+    local name="$1"
+    shift
+    inside "$name" bash -s "$@" <"$0"
+}
+
+run_self_inside_as_cd() {
+    local name="$1" user="$2" dir="$3"
+    shift 3
+    inside_as_cd "$name" "$user" "$dir" bash -s "$@" <"$0"
+}
+
+main() {
+    local short_opts="a:hknpsuv"
+    local long_opts="artifacts:,dirty,help,keep,name:,pyexe:,package,source-package,unittest,verbose"
+    local getopt_out=""
+    getopt_out=$(getopt --name "${0##*/}" \
+        --options "${short_opts}" --long "${long_opts}" -- "$@") &&
+        eval set -- "${getopt_out}" ||
+        { bad_Usage; return; }
+
+    local cur="" next=""
+    local package="" source_package="" unittest="" name=""
+    local dirty=false pyexe="auto" artifact_d="."
+
+    while [ $# -ne 0 ]; do
+        cur="${1:-}"; next="${2:-}";
+        case "$cur" in
+            -a|--artifacts) artifact_d="$next";;
+               --dirty) dirty=true;;
+            -h|--help) Usage ; exit 0;;
+            -k|--keep) KEEP=true;;
+            -n|--name) name="$next"; shift;;
+               --pyexe) pyexe=$next; shift;;
+            -p|--package) package=1;;
+            -s|--source-package) source_package=1;;
+            -u|--unittest) unittest=1;;
+            -v|--verbose) VERBOSITY=$((VERBOSITY+1));;
+            --) shift; break;;
+        esac
+        shift;
+    done
+
+    [ $# -eq 1 ] || { bad_Usage "Expected 1 arg, got $# ($*)"; return; }
+    local img_ref_in="$1"
+    case "${img_ref_in}" in
+        *:*) img_ref="${img_ref_in}";;
+        *) img_ref="images:${img_ref_in}";;
+    esac
+
+    # program starts here
+    local out="" user="ci-test" cdir="" home=""
+    home="/home/$user"
+    cdir="$home/cloud-init"
+    if [ -z "$name" ]; then
+        if out=$(petname 2>&1); then
+            name="ci-${out}"
+        elif out=$(uuidgen -t 2>&1); then
+            name="ci-${out%%-*}"
+        else
+            error "Must provide name or have petname or uuidgen"
+            return 1
+        fi
+    fi
+
+    trap cleanup EXIT
+
+    start_container "$img_ref" "$name" ||
+        { errorrc "Failed to start container for $img_ref"; return; }
+
+    get_os_info_in "$name" ||
+        { errorrc "failed to get os_info in $name"; return; }
+
+    if [ "$pyexe" = "auto" ]; then
+        case "$OS_NAME/$OS_VERSION" in
+            centos/*|opensuse/*) pyexe=python2;;
+            *) pyexe=python3;;
+        esac
+        debug 1 "set pyexe=$pyexe for $OS_NAME/$OS_VERSION"
+    fi
+
+    # prep the container (install very basic dependencies)
+    run_self_inside "$name" prep "$pyexe" ||
+        { errorrc "Failed to prep container $name"; return; }
+
+    # add the user
+    inside "$name" useradd "$user" --create-home "--home-dir=$home" ||
+        { errorrc "Failed to add user '$user' in '$name'"; return 1; }
+
+    debug 1 "inserting cloud-init"
+    inject_cloud_init "$name" "$user" "$dirty" || {
+        errorrc "FAIL: injecting cloud-init into $name failed."
+        return
+    }
+
+    inside_as_cd "$name" root "$cdir" \
+        $pyexe ./tools/read-dependencies "--distro=${OS_NAME}" \
+            --test-distro || {
+        errorrc "FAIL: failed to install dependencies with read-dependencies"
+        return
+    }
+
+    local errors=( )
+    inside_as_cd "$name" "$user" "$cdir" git status || {
+        errorrc "git checkout failed."
+        errors[${#errors[@]}]="git checkout";
+    }
+
+    if [ -n "$unittest" ]; then
+        debug 1 "running unit tests."
+        run_self_inside_as_cd "$name" "$user" "$cdir" nose "$pyexe" \
+            tests/unittests cloudinit/ || {
+                errorrc "nosetests failed.";
+                errors[${#errors[@]}]="nosetests"
+            }
+    fi
+
+    local build_pkg="" build_srcpkg="" pkg_ext="" distflag=""
+    case "$OS_NAME" in
+        centos) distflag="--distro=redhat";;
+        opensuse) distflag="--distro=suse";;
+    esac
+
+    case "$OS_NAME" in
+        debian|ubuntu)
+            build_pkg="./packages/bddeb -d" 
+            build_srcpkg="./packages/bddeb -S -d"
+            pkg_ext=".deb";;
+        centos|opensuse)
+            build_pkg="./packages/brpm $distflag"
+            build_srcpkg="./packages/brpm $distflag --srpm"
+            pkg_ext=".rpm";;
+    esac
+    if [ -n "$source_package" ]; then
+        [ -n "$build_pkg" ] || {
+            error "Unknown package command for $OS_NAME"
+            return 1
+        }
+        debug 1 "building source package with $build_srcpkg."
+        # shellcheck disable=SC2086
+        inside_as_cd "$name" "$user" "$cdir" $pyexe $build_srcpkg || {
+            errorrc "failed: $build_srcpkg";
+            errors[${#errors[@]}]="source package"
+        }
+    fi
+
+    if [ -n "$package" ]; then
+        [ -n "$build_srcpkg" ] || {
+            error "Unknown build source command for $OS_NAME"
+            return 1
+        }
+        debug 1 "building binary package with $build_pkg."
+        inside_as_cd "$name" "$user" "$cdir" $pyexe $build_pkg || {
+            errorrc "failed: $build_pkg";
+            errors[${#errors[@]}]="binary package"
+        }
+    fi
+
+    if [ -n "$artifact_d" ]; then
+        local art=""
+        artifact_d="${artifact_d%/}/"
+        [ -d "${artifact_d}" ] || mkdir -p "$artifact_d" || {
+            errorrc "failed to create artifact dir '$artifact_d'"
+            return
+        }
+
+        for art in $(inside "$name" sh -c "echo $cdir/*${pkg_ext}"); do
+            lxc file pull "$name/$art" "$artifact_d" || {
+                errorrc "Failed to pull '$name/$art' to ${artifact_d}"
+                errors[${#errors[@]}]="artifact copy: $art"
+            }
+            debug 1 "wrote ${artifact_d}${art##*/}"
+        done
+    fi
+
+    if [ "${#errors[@]}" != "0" ]; then
+        local e=""
+        error "there were ${#errors[@]} errors."
+        for e in "${errors[@]}"; do
+            error "  $e"
+        done
+        return 1
+    fi
+    return 0
+}
+
+case "${1:-}" in
+    prep|os_info|wait_inside|nose) _n=$1; shift; "$_n" "$@";;
+    *) main "$@";;
+esac
+
+# vi: ts=4 expandtab
-- 
cgit v1.2.3


From c42a926ae730994f66fe87c264b65f6e4dca69a1 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Thu, 28 Jun 2018 15:03:56 -0400
Subject: tools: Fix run-container when neither source or binary package
 requested.

If run-container was called without --package or --binary-package, then
it would still try to copy out artifacts and would fail doing so as
there were no artifacts to collect.

Also fix a bug when only --source-package without --package.
---
 tools/run-container | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

(limited to 'tools')

diff --git a/tools/run-container b/tools/run-container
index 499e85b0..6dedb757 100755
--- a/tools/run-container
+++ b/tools/run-container
@@ -418,7 +418,7 @@ main() {
         { bad_Usage; return; }
 
     local cur="" next=""
-    local package="" source_package="" unittest="" name=""
+    local package=false srcpackage=false unittest="" name=""
     local dirty=false pyexe="auto" artifact_d="."
 
     while [ $# -ne 0 ]; do
@@ -430,8 +430,8 @@ main() {
             -k|--keep) KEEP=true;;
             -n|--name) name="$next"; shift;;
                --pyexe) pyexe=$next; shift;;
-            -p|--package) package=1;;
-            -s|--source-package) source_package=1;;
+            -p|--package) package=true;;
+            -s|--source-package) srcpackage=true;;
             -u|--unittest) unittest=1;;
             -v|--verbose) VERBOSITY=$((VERBOSITY+1));;
             --) shift; break;;
@@ -529,8 +529,8 @@ main() {
             build_srcpkg="./packages/brpm $distflag --srpm"
             pkg_ext=".rpm";;
     esac
-    if [ -n "$source_package" ]; then
-        [ -n "$build_pkg" ] || {
+    if [ "$srcpackage" = "true" ]; then
+        [ -n "$build_srcpkg" ] || {
             error "Unknown package command for $OS_NAME"
             return 1
         }
@@ -542,19 +542,21 @@ main() {
         }
     fi
 
-    if [ -n "$package" ]; then
-        [ -n "$build_srcpkg" ] || {
+    if [ "$package" = "true" ]; then
+        [ -n "$build_pkg" ] || {
             error "Unknown build source command for $OS_NAME"
             return 1
         }
         debug 1 "building binary package with $build_pkg."
+        # shellcheck disable=SC2086
         inside_as_cd "$name" "$user" "$cdir" $pyexe $build_pkg || {
             errorrc "failed: $build_pkg";
             errors[${#errors[@]}]="binary package"
         }
     fi
 
-    if [ -n "$artifact_d" ]; then
+    if [ -n "$artifact_d" ] &&
+        [ "$package" = "true" -o "$srcpackage" = "true" ]; then
         local art=""
         artifact_d="${artifact_d%/}/"
         [ -d "${artifact_d}" ] || mkdir -p "$artifact_d" || {
-- 
cgit v1.2.3


From b07e491d2b69a25ab6dc5e56f6ba8b84d54867a9 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Tue, 17 Jul 2018 17:17:34 +0000
Subject: tools: add '--debug' to tools/net-convert.py

In order to see some of the WARNING messages added by bug 1774666
I wanted logging output of tools/net-convert.  This does:

a.) add '--debug' and make it print the network state and read yaml only
    if --debug is provided.
b.) set up basic logging so warnings goes to console by default and
    debug goes to console if --debug is provided.
---
 tools/net-convert.py | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

(limited to 'tools')

diff --git a/tools/net-convert.py b/tools/net-convert.py
index 68559cbf..d1a4a646 100755
--- a/tools/net-convert.py
+++ b/tools/net-convert.py
@@ -4,11 +4,13 @@
 import argparse
 import json
 import os
+import sys
 import yaml
 
 from cloudinit.sources.helpers import openstack
 
 from cloudinit.net import eni
+from cloudinit import log
 from cloudinit.net import netplan
 from cloudinit.net import network_state
 from cloudinit.net import sysconfig
@@ -29,14 +31,23 @@ def main():
                         metavar="name,mac",
                         action='append',
                         help="interface name to mac mapping")
+    parser.add_argument("--debug", action='store_true',
+                        help='enable debug logging to stderr.')
     parser.add_argument("--output-kind", "-ok",
                         choices=['eni', 'netplan', 'sysconfig'],
                         required=True)
     args = parser.parse_args()
 
+    if not args.directory.endswith("/"):
+        args.directory += "/"
+
     if not os.path.isdir(args.directory):
         os.makedirs(args.directory)
 
+    if args.debug:
+        log.setupBasicLogging(level=log.DEBUG)
+    else:
+        log.setupBasicLogging(level=log.WARN)
     if args.mac:
         known_macs = {}
         for item in args.mac:
@@ -53,8 +64,10 @@ def main():
         pre_ns = yaml.load(net_data)
         if 'network' in pre_ns:
             pre_ns = pre_ns.get('network')
-        print("Input YAML")
-        print(yaml.dump(pre_ns, default_flow_style=False, indent=4))
+        if args.debug:
+            sys.stderr.write('\n'.join(
+                ["Input YAML",
+                 yaml.dump(pre_ns, default_flow_style=False, indent=4), ""]))
         ns = network_state.parse_net_config_data(pre_ns)
     else:
         pre_ns = openstack.convert_net_json(
@@ -65,8 +78,10 @@ def main():
         raise RuntimeError("No valid network_state object created from"
                            "input data")
 
-    print("\nInternal State")
-    print(yaml.dump(ns, default_flow_style=False, indent=4))
+    if args.debug:
+        sys.stderr.write('\n'.join([
+            "", "Internal State",
+            yaml.dump(ns, default_flow_style=False, indent=4), ""]))
     if args.output_kind == "eni":
         r_cls = eni.Renderer
     elif args.output_kind == "netplan":
@@ -75,6 +90,11 @@ def main():
         r_cls = sysconfig.Renderer
 
     r = r_cls()
+    sys.stderr.write(''.join([
+        "Read input format '%s' from '%s'.\n" % (
+            args.kind, args.network_data.name),
+        "Wrote output format '%s' to '%s'\n" % (
+            args.output_kind, args.directory)]) + "\n")
     r.render_network_state(network_state=ns, target=args.directory)
 
 
-- 
cgit v1.2.3


From 538af8e64facb7df55c759d9ca82a0fb3a6ce325 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Fri, 3 Aug 2018 22:29:40 +0000
Subject: Use typeset or local in profile.d scripts.

Bash and most other "bourne-like" shells allow declaring function
local variables via 'local'.  ksh does not.  Instead of using 'local'
always, use 'typeset' when the KSH_VERSION variable is present in
environment.

LP: #1784713
---
 tools/Z99-cloud-locale-test.sh  | 13 ++++++++-----
 tools/Z99-cloudinit-warnings.sh |  8 +++++---
 2 files changed, 13 insertions(+), 8 deletions(-)

(limited to 'tools')

diff --git a/tools/Z99-cloud-locale-test.sh b/tools/Z99-cloud-locale-test.sh
index 4978d87e..9ee44bd2 100644
--- a/tools/Z99-cloud-locale-test.sh
+++ b/tools/Z99-cloud-locale-test.sh
@@ -11,8 +11,11 @@
 #  of how to fix them.
 
 locale_warn() {
-    local bad_names="" bad_lcs="" key="" val="" var="" vars="" bad_kv=""
-    local w1 w2 w3 w4 remain
+    command -v local >/dev/null && local _local="local" ||
+        typeset _local="typeset"
+
+    $_local bad_names="" bad_lcs="" key="" val="" var="" vars="" bad_kv=""
+    $_local w1 w2 w3 w4 remain
 
     # if shell is zsh, act like sh only for this function (-L).
     # The behavior change will not permenently affect user's shell.
@@ -53,8 +56,8 @@ locale_warn() {
     printf " This can affect your user experience significantly, including the\n"
     printf " ability to manage packages. You may install the locales by running:\n\n"
 
-    local bad invalid="" to_gen="" sfile="/usr/share/i18n/SUPPORTED"
-    local pkgs=""
+    $_local bad invalid="" to_gen="" sfile="/usr/share/i18n/SUPPORTED"
+    $_local local pkgs=""
     if [ -e "$sfile" ]; then
         for bad in ${bad_lcs}; do
             grep -q -i "${bad}" "$sfile" &&
@@ -67,7 +70,7 @@ locale_warn() {
     fi
     to_gen=${to_gen# }
 
-    local pkgs=""
+    $_local pkgs=""
     for bad in ${to_gen}; do
         pkgs="${pkgs} language-pack-${bad%%_*}"
     done
diff --git a/tools/Z99-cloudinit-warnings.sh b/tools/Z99-cloudinit-warnings.sh
index 1d413374..cb8b4638 100644
--- a/tools/Z99-cloudinit-warnings.sh
+++ b/tools/Z99-cloudinit-warnings.sh
@@ -4,9 +4,11 @@
 # Purpose: show user warnings on login.
 
 cloud_init_warnings() {
-    local warning="" idir="/var/lib/cloud/instance" n=0
-    local warndir="$idir/warnings"
-    local ufile="$HOME/.cloud-warnings.skip" sfile="$warndir/.skip"
+    command -v local >/dev/null && local _local="local" ||
+        typeset _local="typeset"
+    $_local warning="" idir="/var/lib/cloud/instance" n=0
+    $_local warndir="$idir/warnings"
+    $_local ufile="$HOME/.cloud-warnings.skip" sfile="$warndir/.skip"
     [ -d "$warndir" ] || return 0
     [ ! -f "$ufile" ] || return 0
     [ ! -f "$sfile" ] || return 0
-- 
cgit v1.2.3


From a6f95c72259f2890e4a9f9f11166310812173c68 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Mon, 6 Aug 2018 16:50:51 +0000
Subject: tools: Add 'net-convert' subcommand command to 'cloud-init devel'.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Move the tools/net-convert.py to be exposed as part of 'cloud-init devel'
subcommands.
It can now be called like:
  $ cloud-init devel net-convert
Or, if you just have checked out source (and no cli executable):

  $ python3 -m cloudinit.cmd.devel.net_convert
or
  $ python3 -m cloudinit.cmd.main devel net-convert
---
 bash_completion/cloud-init         |   7 ++-
 cloudinit/cmd/devel/net_convert.py | 115 +++++++++++++++++++++++++++++++++++++
 cloudinit/cmd/devel/parser.py      |  20 ++++---
 tests/unittests/test_cli.py        |   3 +-
 tools/net-convert.py               | 104 ---------------------------------
 5 files changed, 134 insertions(+), 115 deletions(-)
 create mode 100755 cloudinit/cmd/devel/net_convert.py
 delete mode 100755 tools/net-convert.py

(limited to 'tools')

diff --git a/bash_completion/cloud-init b/bash_completion/cloud-init
index 581432c8..f38164b0 100644
--- a/bash_completion/cloud-init
+++ b/bash_completion/cloud-init
@@ -28,7 +28,7 @@ _cloudinit_complete()
                     COMPREPLY=($(compgen -W "--help --tarfile --include-userdata" -- $cur_word))
                     ;;
                 devel)
-                    COMPREPLY=($(compgen -W "--help schema" -- $cur_word))
+                    COMPREPLY=($(compgen -W "--help schema net-convert" -- $cur_word))
                     ;;
                 dhclient-hook|features)
                     COMPREPLY=($(compgen -W "--help" -- $cur_word))
@@ -59,6 +59,9 @@ _cloudinit_complete()
                 --frequency)
                     COMPREPLY=($(compgen -W "--help instance always once" -- $cur_word))
                     ;;
+                net-convert)
+                    COMPREPLY=($(compgen -W "--help --network-data --kind --directory --output-kind" -- $cur_word))
+                    ;;
                 schema)
                     COMPREPLY=($(compgen -W "--help --config-file --doc --annotate" -- $cur_word))
                     ;;
@@ -74,4 +77,4 @@ _cloudinit_complete()
 }
 complete -F _cloudinit_complete cloud-init
 
-# vi: syntax=bash expandtab
+# vi: syntax=sh expandtab
diff --git a/cloudinit/cmd/devel/net_convert.py b/cloudinit/cmd/devel/net_convert.py
new file mode 100755
index 00000000..1ec08a3c
--- /dev/null
+++ b/cloudinit/cmd/devel/net_convert.py
@@ -0,0 +1,115 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Debug network config format conversions."""
+import argparse
+import json
+import os
+import sys
+import yaml
+
+from cloudinit.sources.helpers import openstack
+
+from cloudinit.net import eni, netplan, network_state, sysconfig
+from cloudinit import log
+
+NAME = 'net-convert'
+
+
+def get_parser(parser=None):
+    """Build or extend and arg parser for net-convert utility.
+
+    @param parser: Optional existing ArgumentParser instance representing the
+        subcommand which will be extended to support the args of this utility.
+
+    @returns: ArgumentParser with proper argument configuration.
+    """
+    if not parser:
+        parser = argparse.ArgumentParser(prog=NAME, description=__doc__)
+    parser.add_argument("-p", "--network-data", type=open,
+                        metavar="PATH", required=True)
+    parser.add_argument("-k", "--kind",
+                        choices=['eni', 'network_data.json', 'yaml'],
+                        required=True)
+    parser.add_argument("-d", "--directory",
+                        metavar="PATH",
+                        help="directory to place output in",
+                        required=True)
+    parser.add_argument("-m", "--mac",
+                        metavar="name,mac",
+                        action='append',
+                        help="interface name to mac mapping")
+    parser.add_argument("--debug", action='store_true',
+                        help='enable debug logging to stderr.')
+    parser.add_argument("-O", "--output-kind",
+                        choices=['eni', 'netplan', 'sysconfig'],
+                        required=True)
+    return parser
+
+
+def handle_args(name, args):
+    if not args.directory.endswith("/"):
+        args.directory += "/"
+
+    if not os.path.isdir(args.directory):
+        os.makedirs(args.directory)
+
+    if args.debug:
+        log.setupBasicLogging(level=log.DEBUG)
+    else:
+        log.setupBasicLogging(level=log.WARN)
+    if args.mac:
+        known_macs = {}
+        for item in args.mac:
+            iface_name, iface_mac = item.split(",", 1)
+            known_macs[iface_mac] = iface_name
+    else:
+        known_macs = None
+
+    net_data = args.network_data.read()
+    if args.kind == "eni":
+        pre_ns = eni.convert_eni_data(net_data)
+        ns = network_state.parse_net_config_data(pre_ns)
+    elif args.kind == "yaml":
+        pre_ns = yaml.load(net_data)
+        if 'network' in pre_ns:
+            pre_ns = pre_ns.get('network')
+        if args.debug:
+            sys.stderr.write('\n'.join(
+                ["Input YAML",
+                 yaml.dump(pre_ns, default_flow_style=False, indent=4), ""]))
+        ns = network_state.parse_net_config_data(pre_ns)
+    else:
+        pre_ns = openstack.convert_net_json(
+            json.loads(net_data), known_macs=known_macs)
+        ns = network_state.parse_net_config_data(pre_ns)
+
+    if not ns:
+        raise RuntimeError("No valid network_state object created from"
+                           "input data")
+
+    if args.debug:
+        sys.stderr.write('\n'.join([
+            "", "Internal State",
+            yaml.dump(ns, default_flow_style=False, indent=4), ""]))
+    if args.output_kind == "eni":
+        r_cls = eni.Renderer
+    elif args.output_kind == "netplan":
+        r_cls = netplan.Renderer
+    else:
+        r_cls = sysconfig.Renderer
+
+    r = r_cls()
+    sys.stderr.write(''.join([
+        "Read input format '%s' from '%s'.\n" % (
+            args.kind, args.network_data.name),
+        "Wrote output format '%s' to '%s'\n" % (
+            args.output_kind, args.directory)]) + "\n")
+    r.render_network_state(network_state=ns, target=args.directory)
+
+
+if __name__ == '__main__':
+    args = get_parser().parse_args()
+    handle_args(NAME, args)
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/devel/parser.py b/cloudinit/cmd/devel/parser.py
index acacc4ed..40a4b019 100644
--- a/cloudinit/cmd/devel/parser.py
+++ b/cloudinit/cmd/devel/parser.py
@@ -5,8 +5,9 @@
 """Define 'devel' subcommand argument parsers to include in cloud-init cmd."""
 
 import argparse
-from cloudinit.config.schema import (
-    get_parser as schema_parser, handle_schema_args)
+from cloudinit.config import schema
+
+from . import net_convert
 
 
 def get_parser(parser=None):
@@ -17,10 +18,15 @@ def get_parser(parser=None):
     subparsers = parser.add_subparsers(title='Subcommands', dest='subcommand')
     subparsers.required = True
 
-    parser_schema = subparsers.add_parser(
-        'schema', help='Validate cloud-config files or document schema')
-    # Construct schema subcommand parser
-    schema_parser(parser_schema)
-    parser_schema.set_defaults(action=('schema', handle_schema_args))
+    subcmds = [
+        ('schema', 'Validate cloud-config files for document schema',
+         schema.get_parser, schema.handle_schema_args),
+        (net_convert.NAME, net_convert.__doc__,
+         net_convert.get_parser, net_convert.handle_args)
+    ]
+    for (subcmd, helpmsg, get_parser, handler) in subcmds:
+        parser = subparsers.add_parser(subcmd, help=helpmsg)
+        get_parser(parser)
+        parser.set_defaults(action=(subcmd, handler))
 
     return parser
diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py
index 0c0f427a..199d69b0 100644
--- a/tests/unittests/test_cli.py
+++ b/tests/unittests/test_cli.py
@@ -208,8 +208,7 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
         for subcommand in expected_subcommands:
             self.assertIn(subcommand, error)
 
-    @mock.patch('cloudinit.config.schema.handle_schema_args')
-    def test_wb_devel_schema_subcommand_parser(self, m_schema):
+    def test_wb_devel_schema_subcommand_parser(self):
         """The subcommand cloud-init schema calls the correct subparser."""
         exit_code = self._call_main(['cloud-init', 'devel', 'schema'])
         self.assertEqual(1, exit_code)
diff --git a/tools/net-convert.py b/tools/net-convert.py
deleted file mode 100755
index d1a4a646..00000000
--- a/tools/net-convert.py
+++ /dev/null
@@ -1,104 +0,0 @@
-#!/usr/bin/python3
-# This file is part of cloud-init. See LICENSE file for license information.
-
-import argparse
-import json
-import os
-import sys
-import yaml
-
-from cloudinit.sources.helpers import openstack
-
-from cloudinit.net import eni
-from cloudinit import log
-from cloudinit.net import netplan
-from cloudinit.net import network_state
-from cloudinit.net import sysconfig
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--network-data", "-p", type=open,
-                        metavar="PATH", required=True)
-    parser.add_argument("--kind", "-k",
-                        choices=['eni', 'network_data.json', 'yaml'],
-                        required=True)
-    parser.add_argument("-d", "--directory",
-                        metavar="PATH",
-                        help="directory to place output in",
-                        required=True)
-    parser.add_argument("-m", "--mac",
-                        metavar="name,mac",
-                        action='append',
-                        help="interface name to mac mapping")
-    parser.add_argument("--debug", action='store_true',
-                        help='enable debug logging to stderr.')
-    parser.add_argument("--output-kind", "-ok",
-                        choices=['eni', 'netplan', 'sysconfig'],
-                        required=True)
-    args = parser.parse_args()
-
-    if not args.directory.endswith("/"):
-        args.directory += "/"
-
-    if not os.path.isdir(args.directory):
-        os.makedirs(args.directory)
-
-    if args.debug:
-        log.setupBasicLogging(level=log.DEBUG)
-    else:
-        log.setupBasicLogging(level=log.WARN)
-    if args.mac:
-        known_macs = {}
-        for item in args.mac:
-            iface_name, iface_mac = item.split(",", 1)
-            known_macs[iface_mac] = iface_name
-    else:
-        known_macs = None
-
-    net_data = args.network_data.read()
-    if args.kind == "eni":
-        pre_ns = eni.convert_eni_data(net_data)
-        ns = network_state.parse_net_config_data(pre_ns)
-    elif args.kind == "yaml":
-        pre_ns = yaml.load(net_data)
-        if 'network' in pre_ns:
-            pre_ns = pre_ns.get('network')
-        if args.debug:
-            sys.stderr.write('\n'.join(
-                ["Input YAML",
-                 yaml.dump(pre_ns, default_flow_style=False, indent=4), ""]))
-        ns = network_state.parse_net_config_data(pre_ns)
-    else:
-        pre_ns = openstack.convert_net_json(
-            json.loads(net_data), known_macs=known_macs)
-        ns = network_state.parse_net_config_data(pre_ns)
-
-    if not ns:
-        raise RuntimeError("No valid network_state object created from"
-                           "input data")
-
-    if args.debug:
-        sys.stderr.write('\n'.join([
-            "", "Internal State",
-            yaml.dump(ns, default_flow_style=False, indent=4), ""]))
-    if args.output_kind == "eni":
-        r_cls = eni.Renderer
-    elif args.output_kind == "netplan":
-        r_cls = netplan.Renderer
-    else:
-        r_cls = sysconfig.Renderer
-
-    r = r_cls()
-    sys.stderr.write(''.join([
-        "Read input format '%s' from '%s'.\n" % (
-            args.kind, args.network_data.name),
-        "Wrote output format '%s' to '%s'\n" % (
-            args.output_kind, args.directory)]) + "\n")
-    r.render_network_state(network_state=ns, target=args.directory)
-
-
-if __name__ == '__main__':
-    main()
-
-# vi: ts=4 expandtab
-- 
cgit v1.2.3


From aaffd59431fe05932a66016db941fe197c4e7620 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Fri, 17 Aug 2018 20:25:31 +0000
Subject: Add datasource Oracle Compute Infrastructure (OCI).

This adds a Oracle specific datasource that functions with OCI.
It is a simplified version of the OpenStack metadata server
with support for vendor-data.

It does not support the OCI-C (classic) platform.

Also here is a move of BrokenMetadata to common 'sources'
as this was the third occurrence of that class.
---
 .pylintrc                                         |   3 +-
 cloudinit/apport.py                               |   1 +
 cloudinit/settings.py                             |   1 +
 cloudinit/sources/DataSourceIBMCloud.py           |  13 +-
 cloudinit/sources/DataSourceOpenStack.py          |  12 +-
 cloudinit/sources/DataSourceOracle.py             | 233 +++++++++++++++
 cloudinit/sources/__init__.py                     |   4 +
 cloudinit/sources/helpers/openstack.py            |   6 +-
 cloudinit/sources/tests/test_oracle.py            | 331 ++++++++++++++++++++++
 doc/rtd/topics/datasources.rst                    |   1 +
 doc/rtd/topics/datasources/oracle.rst             |  26 ++
 tests/unittests/test_datasource/test_common.py    |   2 +
 tests/unittests/test_datasource/test_openstack.py |  13 +-
 tests/unittests/test_ds_identify.py               |  19 ++
 tools/ds-identify                                 |   8 +-
 15 files changed, 650 insertions(+), 23 deletions(-)
 create mode 100644 cloudinit/sources/DataSourceOracle.py
 create mode 100644 cloudinit/sources/tests/test_oracle.py
 create mode 100644 doc/rtd/topics/datasources/oracle.rst

(limited to 'tools')

diff --git a/.pylintrc b/.pylintrc
index 3bfa0c81..e376b48b 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -61,7 +61,8 @@ ignored-modules=
 # List of class names for which member attributes should not be checked (useful
 # for classes with dynamically set attributes). This supports the use of
 # qualified names.
-ignored-classes=optparse.Values,thread._local
+# argparse.Namespace from https://github.com/PyCQA/pylint/issues/2413
+ignored-classes=argparse.Namespace,optparse.Values,thread._local
 
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
diff --git a/cloudinit/apport.py b/cloudinit/apport.py
index 130ff269..22cb7fde 100644
--- a/cloudinit/apport.py
+++ b/cloudinit/apport.py
@@ -30,6 +30,7 @@ KNOWN_CLOUD_NAMES = [
     'NoCloud',
     'OpenNebula',
     'OpenStack',
+    'Oracle',
     'OVF',
     'OpenTelekomCloud',
     'Scaleway',
diff --git a/cloudinit/settings.py b/cloudinit/settings.py
index dde5749d..ea367cb7 100644
--- a/cloudinit/settings.py
+++ b/cloudinit/settings.py
@@ -38,6 +38,7 @@ CFG_BUILTIN = {
         'Scaleway',
         'Hetzner',
         'IBMCloud',
+        'Oracle',
         # At the end to act as a 'catch' when none of the above work...
         'None',
     ],
diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py
index 01106ec0..a5358148 100644
--- a/cloudinit/sources/DataSourceIBMCloud.py
+++ b/cloudinit/sources/DataSourceIBMCloud.py
@@ -295,7 +295,7 @@ def read_md():
             results = metadata_from_dir(path)
         else:
             results = util.mount_cb(path, metadata_from_dir)
-    except BrokenMetadata as e:
+    except sources.BrokenMetadata as e:
         raise RuntimeError(
             "Failed reading IBM config disk (platform=%s path=%s): %s" %
             (platform, path, e))
@@ -304,10 +304,6 @@ def read_md():
     return ret
 
 
-class BrokenMetadata(IOError):
-    pass
-
-
 def metadata_from_dir(source_dir):
     """Walk source_dir extracting standardized metadata.
 
@@ -352,12 +348,13 @@ def metadata_from_dir(source_dir):
             try:
                 data = transl(raw)
             except Exception as e:
-                raise BrokenMetadata("Failed decoding %s: %s" % (path, e))
+                raise sources.BrokenMetadata(
+                    "Failed decoding %s: %s" % (path, e))
 
         results[name] = data
 
     if results.get('metadata_raw') is None:
-        raise BrokenMetadata(
+        raise sources.BrokenMetadata(
             "%s missing required file 'meta_data.json'" % source_dir)
 
     results['metadata'] = {}
@@ -368,7 +365,7 @@ def metadata_from_dir(source_dir):
         try:
             md['random_seed'] = base64.b64decode(md_raw['random_seed'])
         except (ValueError, TypeError) as e:
-            raise BrokenMetadata(
+            raise sources.BrokenMetadata(
                 "Badly formatted metadata random_seed entry: %s" % e)
 
     renames = (
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index b9ade90d..4a015240 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -13,6 +13,7 @@ from cloudinit import url_helper
 from cloudinit import util
 
 from cloudinit.sources.helpers import openstack
+from cloudinit.sources import DataSourceOracle as oracle
 
 LOG = logging.getLogger(__name__)
 
@@ -28,8 +29,7 @@ DMI_PRODUCT_NOVA = 'OpenStack Nova'
 DMI_PRODUCT_COMPUTE = 'OpenStack Compute'
 VALID_DMI_PRODUCT_NAMES = [DMI_PRODUCT_NOVA, DMI_PRODUCT_COMPUTE]
 DMI_ASSET_TAG_OPENTELEKOM = 'OpenTelekomCloud'
-DMI_ASSET_TAG_ORACLE_CLOUD = 'OracleCloud.com'
-VALID_DMI_ASSET_TAGS = [DMI_ASSET_TAG_OPENTELEKOM, DMI_ASSET_TAG_ORACLE_CLOUD]
+VALID_DMI_ASSET_TAGS = [DMI_ASSET_TAG_OPENTELEKOM]
 
 
 class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
@@ -122,8 +122,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
             False when unable to contact metadata service or when metadata
             format is invalid or disabled.
         """
-        if not detect_openstack():
+        oracle_considered = 'Oracle' in self.sys_cfg.get('datasource_list')
+        if not detect_openstack(accept_oracle=not oracle_considered):
             return False
+
         if self.perform_dhcp_setup:  # Setup networking in init-local stage.
             try:
                 with EphemeralDHCPv4(self.fallback_interface):
@@ -215,7 +217,7 @@ def read_metadata_service(base_url, ssl_details=None,
     return reader.read_v2()
 
 
-def detect_openstack():
+def detect_openstack(accept_oracle=False):
     """Return True when a potential OpenStack platform is detected."""
     if not util.is_x86():
         return True  # Non-Intel cpus don't properly report dmi product names
@@ -224,6 +226,8 @@ def detect_openstack():
         return True
     elif util.read_dmi_data('chassis-asset-tag') in VALID_DMI_ASSET_TAGS:
         return True
+    elif accept_oracle and oracle._is_platform_viable():
+        return True
     elif util.get_proc_env(1).get('product_name') == DMI_PRODUCT_NOVA:
         return True
     return False
diff --git a/cloudinit/sources/DataSourceOracle.py b/cloudinit/sources/DataSourceOracle.py
new file mode 100644
index 00000000..fab39af3
--- /dev/null
+++ b/cloudinit/sources/DataSourceOracle.py
@@ -0,0 +1,233 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+"""Datasource for Oracle (OCI/Oracle Cloud Infrastructure)
+
+OCI provides a OpenStack like metadata service which provides only
+'2013-10-17' and 'latest' versions..
+
+Notes:
+ * This datasource does not support the OCI-Classic. OCI-Classic
+   provides an EC2 lookalike metadata service.
+ * The uuid provided in DMI data is not the same as the meta-data provided
+   instance-id, but has an equivalent lifespan.
+ * We do need to support upgrade from an instance that cloud-init
+   identified as OpenStack.
+ * Both bare-metal and vms use iscsi root
+ * Both bare-metal and vms provide chassis-asset-tag of OracleCloud.com
+"""
+
+from cloudinit.url_helper import combine_url, readurl, UrlError
+from cloudinit.net import dhcp
+from cloudinit import net
+from cloudinit import sources
+from cloudinit import util
+from cloudinit.net import cmdline
+from cloudinit import log as logging
+
+import json
+import re
+
+LOG = logging.getLogger(__name__)
+
+CHASSIS_ASSET_TAG = "OracleCloud.com"
+METADATA_ENDPOINT = "http://169.254.169.254/openstack/"
+
+
+class DataSourceOracle(sources.DataSource):
+
+    dsname = 'Oracle'
+    system_uuid = None
+    vendordata_pure = None
+    _network_config = sources.UNSET
+
+    def _is_platform_viable(self):
+        """Check platform environment to report if this datasource may run."""
+        return _is_platform_viable()
+
+    def _get_data(self):
+        if not self._is_platform_viable():
+            return False
+
+        # network may be configured if iscsi root.  If that is the case
+        # then read_kernel_cmdline_config will return non-None.
+        if _is_iscsi_root():
+            data = self.crawl_metadata()
+        else:
+            with dhcp.EphemeralDHCPv4(net.find_fallback_nic()):
+                data = self.crawl_metadata()
+
+        self._crawled_metadata = data
+        vdata = data['2013-10-17']
+
+        self.userdata_raw = vdata.get('user_data')
+        self.system_uuid = vdata['system_uuid']
+
+        vd = vdata.get('vendor_data')
+        if vd:
+            self.vendordata_pure = vd
+            try:
+                self.vendordata_raw = sources.convert_vendordata(vd)
+            except ValueError as e:
+                LOG.warning("Invalid content in vendor-data: %s", e)
+                self.vendordata_raw = None
+
+        mdcopies = ('public_keys',)
+        md = dict([(k, vdata['meta_data'].get(k))
+                   for k in mdcopies if k in vdata['meta_data']])
+
+        mdtrans = (
+            # oracle meta_data.json name, cloudinit.datasource.metadata name
+            ('availability_zone', 'availability-zone'),
+            ('hostname', 'local-hostname'),
+            ('launch_index', 'launch-index'),
+            ('uuid', 'instance-id'),
+        )
+        for dsname, ciname in mdtrans:
+            if dsname in vdata['meta_data']:
+                md[ciname] = vdata['meta_data'][dsname]
+
+        self.metadata = md
+        return True
+
+    def crawl_metadata(self):
+        return read_metadata()
+
+    def check_instance_id(self, sys_cfg):
+        """quickly check (local only) if self.instance_id is still valid
+
+        On Oracle, the dmi-provided system uuid differs from the instance-id
+        but has the same life-span."""
+        return sources.instance_id_matches_system_uuid(self.system_uuid)
+
+    def get_public_ssh_keys(self):
+        return sources.normalize_pubkey_data(self.metadata.get('public_keys'))
+
+    @property
+    def network_config(self):
+        """Network config is read from initramfs provided files
+        If none is present, then we fall back to fallback configuration.
+
+        One thing to note here is that this method is not currently
+        considered at all if there is is kernel/initramfs provided
+        data.  In that case, stages considers that the cmdline data
+        overrides datasource provided data and does not consult here.
+
+        We nonetheless return cmdline provided config if present
+        and fallback to generate fallback."""
+        if self._network_config == sources.UNSET:
+            cmdline_cfg = cmdline.read_kernel_cmdline_config()
+            if cmdline_cfg:
+                self._network_config = cmdline_cfg
+            else:
+                self._network_config = self.distro.generate_fallback_config()
+        return self._network_config
+
+
+def _read_system_uuid():
+    sys_uuid = util.read_dmi_data('system-uuid')
+    return None if sys_uuid is None else sys_uuid.lower()
+
+
+def _is_platform_viable():
+    asset_tag = util.read_dmi_data('chassis-asset-tag')
+    return asset_tag == CHASSIS_ASSET_TAG
+
+
+def _is_iscsi_root():
+    return bool(cmdline.read_kernel_cmdline_config())
+
+
+def _load_index(content):
+    """Return a list entries parsed from content.
+
+    OpenStack's metadata service returns a newline delimited list
+    of items.  Oracle's implementation has html formatted list of links.
+    The parser here just grabs targets from <a href="target">
+    and throws away "../".
+
+    Oracle has accepted that to be buggy and may fix in the future
+    to instead return a '\n' delimited plain text list.  This function
+    will continue to work if that change is made."""
+    if not content.lower().startswith("<html>"):
+        return content.splitlines()
+    items = re.findall(
+        r'href="(?P<target>[^"]*)"', content, re.MULTILINE | re.IGNORECASE)
+    return [i for i in items if not i.startswith(".")]
+
+
+def read_metadata(endpoint_base=METADATA_ENDPOINT, sys_uuid=None,
+                  version='2013-10-17'):
+    """Read metadata, return a dictionary.
+
+    Each path listed in the index will be represented in the dictionary.
+    If the path ends in .json, then the content will be decoded and
+    populated into the dictionary.
+
+    The system uuid (/sys/class/dmi/id/product_uuid) is also populated.
+    Example: given paths = ('user_data', 'meta_data.json')
+    This would return:
+      {version: {'user_data': b'blob', 'meta_data': json.loads(blob.decode())
+                 'system_uuid': '3b54f2e0-3ab2-458d-b770-af9926eee3b2'}}
+    """
+    endpoint = combine_url(endpoint_base, version) + "/"
+    if sys_uuid is None:
+        sys_uuid = _read_system_uuid()
+    if not sys_uuid:
+        raise sources.BrokenMetadata("Failed to read system uuid.")
+
+    try:
+        resp = readurl(endpoint)
+        if not resp.ok():
+            raise sources.BrokenMetadata(
+                "Bad response from %s: %s" % (endpoint, resp.code))
+    except UrlError as e:
+        raise sources.BrokenMetadata(
+            "Failed to read index at %s: %s" % (endpoint, e))
+
+    entries = _load_index(resp.contents.decode('utf-8'))
+    LOG.debug("index url %s contained: %s", endpoint, entries)
+
+    # meta_data.json is required.
+    mdj = 'meta_data.json'
+    if mdj not in entries:
+        raise sources.BrokenMetadata(
+            "Required field '%s' missing in index at %s" % (mdj, endpoint))
+
+    ret = {'system_uuid': sys_uuid}
+    for path in entries:
+        response = readurl(combine_url(endpoint, path))
+        if path.endswith(".json"):
+            ret[path.rpartition(".")[0]] = (
+                json.loads(response.contents.decode('utf-8')))
+        else:
+            ret[path] = response.contents
+
+    return {version: ret}
+
+
+# Used to match classes to dependencies
+datasources = [
+    (DataSourceOracle, (sources.DEP_FILESYSTEM,)),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
+
+
+if __name__ == "__main__":
+    import argparse
+    import os
+
+    parser = argparse.ArgumentParser(description='Query Oracle Cloud Metadata')
+    parser.add_argument("--endpoint", metavar="URL",
+                        help="The url of the metadata service.",
+                        default=METADATA_ENDPOINT)
+    args = parser.parse_args()
+    sys_uuid = "uuid-not-available-not-root" if os.geteuid() != 0 else None
+
+    data = read_metadata(endpoint_base=args.endpoint, sys_uuid=sys_uuid)
+    data['is_platform_viable'] = _is_platform_viable()
+    print(util.json_dumps(data))
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 06e613f8..41fde9ba 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -671,6 +671,10 @@ def convert_vendordata(data, recurse=True):
     raise ValueError("Unknown data type for vendordata: %s" % type(data))
 
 
+class BrokenMetadata(IOError):
+    pass
+
+
 # 'depends' is a list of dependencies (DEP_FILESYSTEM)
 # ds_list is a list of 2 item lists
 # ds_list = [
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index a4cf0667..8f9c1441 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -21,6 +21,8 @@ from cloudinit import sources
 from cloudinit import url_helper
 from cloudinit import util
 
+from cloudinit.sources import BrokenMetadata
+
 # See https://docs.openstack.org/user-guide/cli-config-drive.html
 
 LOG = logging.getLogger(__name__)
@@ -68,10 +70,6 @@ class NonReadable(IOError):
     pass
 
 
-class BrokenMetadata(IOError):
-    pass
-
-
 class SourceMixin(object):
     def _ec2_name_to_device(self, name):
         if not self.ec2_metadata:
diff --git a/cloudinit/sources/tests/test_oracle.py b/cloudinit/sources/tests/test_oracle.py
new file mode 100644
index 00000000..7599126c
--- /dev/null
+++ b/cloudinit/sources/tests/test_oracle.py
@@ -0,0 +1,331 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit.sources import DataSourceOracle as oracle
+from cloudinit.sources import BrokenMetadata
+from cloudinit import helpers
+
+from cloudinit.tests import helpers as test_helpers
+
+from textwrap import dedent
+import argparse
+import httpretty
+import json
+import mock
+import os
+import six
+import uuid
+
+DS_PATH = "cloudinit.sources.DataSourceOracle"
+MD_VER = "2013-10-17"
+
+
+class TestDataSourceOracle(test_helpers.CiTestCase):
+    """Test datasource DataSourceOracle."""
+
+    ds_class = oracle.DataSourceOracle
+
+    my_uuid = str(uuid.uuid4())
+    my_md = {"uuid": "ocid1.instance.oc1.phx.abyhqlj",
+             "name": "ci-vm1", "availability_zone": "phx-ad-3",
+             "hostname": "ci-vm1hostname",
+             "launch_index": 0, "files": [],
+             "public_keys": {"0": "ssh-rsa AAAAB3N...== user@host"},
+             "meta": {}}
+
+    def _patch_instance(self, inst, patches):
+        """Patch an instance of a class 'inst'.
+        for each name, kwargs in patches:
+             inst.name = mock.Mock(**kwargs)
+        returns a namespace object that has
+             namespace.name = mock.Mock(**kwargs)
+        Do not bother with cleanup as instance is assumed transient."""
+        mocks = argparse.Namespace()
+        for name, kwargs in patches.items():
+            imock = mock.Mock(name=name, spec=getattr(inst, name), **kwargs)
+            setattr(mocks, name, imock)
+            setattr(inst, name, imock)
+        return mocks
+
+    def _get_ds(self, sys_cfg=None, distro=None, paths=None, ud_proc=None,
+                patches=None):
+        if sys_cfg is None:
+            sys_cfg = {}
+        if patches is None:
+            patches = {}
+        if paths is None:
+            tmpd = self.tmp_dir()
+            dirs = {'cloud_dir': self.tmp_path('cloud_dir', tmpd),
+                    'run_dir': self.tmp_path('run_dir')}
+            for d in dirs.values():
+                os.mkdir(d)
+            paths = helpers.Paths(dirs)
+
+        ds = self.ds_class(sys_cfg=sys_cfg, distro=distro,
+                           paths=paths, ud_proc=ud_proc)
+
+        return ds, self._patch_instance(ds, patches)
+
+    def test_platform_not_viable_returns_false(self):
+        ds, mocks = self._get_ds(
+            patches={'_is_platform_viable': {'return_value': False}})
+        self.assertFalse(ds._get_data())
+        mocks._is_platform_viable.assert_called_once_with()
+
+    @mock.patch(DS_PATH + "._is_iscsi_root", return_value=True)
+    def test_without_userdata(self, m_is_iscsi_root):
+        """If no user-data is provided, it should not be in return dict."""
+        ds, mocks = self._get_ds(patches={
+            '_is_platform_viable': {'return_value': True},
+            'crawl_metadata': {
+                'return_value': {
+                    MD_VER: {'system_uuid': self.my_uuid,
+                             'meta_data': self.my_md}}}})
+        self.assertTrue(ds._get_data())
+        mocks._is_platform_viable.assert_called_once_with()
+        mocks.crawl_metadata.assert_called_once_with()
+        self.assertEqual(self.my_uuid, ds.system_uuid)
+        self.assertEqual(self.my_md['availability_zone'], ds.availability_zone)
+        self.assertIn(self.my_md["public_keys"]["0"], ds.get_public_ssh_keys())
+        self.assertEqual(self.my_md['uuid'], ds.get_instance_id())
+        self.assertIsNone(ds.userdata_raw)
+
+    @mock.patch(DS_PATH + "._is_iscsi_root", return_value=True)
+    def test_with_vendordata(self, m_is_iscsi_root):
+        """Test with vendor data."""
+        vd = {'cloud-init': '#cloud-config\nkey: value'}
+        ds, mocks = self._get_ds(patches={
+            '_is_platform_viable': {'return_value': True},
+            'crawl_metadata': {
+                'return_value': {
+                    MD_VER: {'system_uuid': self.my_uuid,
+                             'meta_data': self.my_md,
+                             'vendor_data': vd}}}})
+        self.assertTrue(ds._get_data())
+        mocks._is_platform_viable.assert_called_once_with()
+        mocks.crawl_metadata.assert_called_once_with()
+        self.assertEqual(vd, ds.vendordata_pure)
+        self.assertEqual(vd['cloud-init'], ds.vendordata_raw)
+
+    @mock.patch(DS_PATH + "._is_iscsi_root", return_value=True)
+    def test_with_userdata(self, m_is_iscsi_root):
+        """Ensure user-data is populated if present and is binary."""
+        my_userdata = b'abcdefg'
+        ds, mocks = self._get_ds(patches={
+            '_is_platform_viable': {'return_value': True},
+            'crawl_metadata': {
+                'return_value': {
+                    MD_VER: {'system_uuid': self.my_uuid,
+                             'meta_data': self.my_md,
+                             'user_data': my_userdata}}}})
+        self.assertTrue(ds._get_data())
+        mocks._is_platform_viable.assert_called_once_with()
+        mocks.crawl_metadata.assert_called_once_with()
+        self.assertEqual(self.my_uuid, ds.system_uuid)
+        self.assertIn(self.my_md["public_keys"]["0"], ds.get_public_ssh_keys())
+        self.assertEqual(self.my_md['uuid'], ds.get_instance_id())
+        self.assertEqual(my_userdata, ds.userdata_raw)
+
+    @mock.patch(DS_PATH + ".cmdline.read_kernel_cmdline_config")
+    @mock.patch(DS_PATH + "._is_iscsi_root", return_value=True)
+    def test_network_cmdline(self, m_is_iscsi_root, m_cmdline_config):
+        """network_config should read kernel cmdline."""
+        distro = mock.MagicMock()
+        ds, _ = self._get_ds(distro=distro, patches={
+            '_is_platform_viable': {'return_value': True},
+            'crawl_metadata': {
+                'return_value': {
+                    MD_VER: {'system_uuid': self.my_uuid,
+                             'meta_data': self.my_md}}}})
+        ncfg = {'version': 1, 'config': [{'a': 'b'}]}
+        m_cmdline_config.return_value = ncfg
+        self.assertTrue(ds._get_data())
+        self.assertEqual(ncfg, ds.network_config)
+        m_cmdline_config.assert_called_once_with()
+        self.assertFalse(distro.generate_fallback_config.called)
+
+    @mock.patch(DS_PATH + ".cmdline.read_kernel_cmdline_config")
+    @mock.patch(DS_PATH + "._is_iscsi_root", return_value=True)
+    def test_network_fallback(self, m_is_iscsi_root, m_cmdline_config):
+        """test that fallback network is generated if no kernel cmdline."""
+        distro = mock.MagicMock()
+        ds, _ = self._get_ds(distro=distro, patches={
+            '_is_platform_viable': {'return_value': True},
+            'crawl_metadata': {
+                'return_value': {
+                    MD_VER: {'system_uuid': self.my_uuid,
+                             'meta_data': self.my_md}}}})
+        ncfg = {'version': 1, 'config': [{'a': 'b'}]}
+        m_cmdline_config.return_value = None
+        self.assertTrue(ds._get_data())
+        ncfg = {'version': 1, 'config': [{'distro1': 'value'}]}
+        distro.generate_fallback_config.return_value = ncfg
+        self.assertEqual(ncfg, ds.network_config)
+        m_cmdline_config.assert_called_once_with()
+        distro.generate_fallback_config.assert_called_once_with()
+        self.assertEqual(1, m_cmdline_config.call_count)
+
+        # test that the result got cached, and the methods not re-called.
+        self.assertEqual(ncfg, ds.network_config)
+        self.assertEqual(1, m_cmdline_config.call_count)
+
+
+@mock.patch(DS_PATH + "._read_system_uuid", return_value=str(uuid.uuid4()))
+class TestReadMetaData(test_helpers.HttprettyTestCase):
+    """Test the read_metadata which interacts with http metadata service."""
+
+    mdurl = oracle.METADATA_ENDPOINT
+    my_md = {"uuid": "ocid1.instance.oc1.phx.abyhqlj",
+             "name": "ci-vm1", "availability_zone": "phx-ad-3",
+             "hostname": "ci-vm1hostname",
+             "launch_index": 0, "files": [],
+             "public_keys": {"0": "ssh-rsa AAAAB3N...== user@host"},
+             "meta": {}}
+
+    def populate_md(self, data):
+        """call httppretty.register_url for each item dict 'data',
+           including valid indexes. Text values converted to bytes."""
+        httpretty.register_uri(
+            httpretty.GET, self.mdurl + MD_VER + "/",
+            '\n'.join(data.keys()).encode('utf-8'))
+        for k, v in data.items():
+            httpretty.register_uri(
+                httpretty.GET, self.mdurl + MD_VER + "/" + k,
+                v if not isinstance(v, six.text_type) else v.encode('utf-8'))
+
+    def test_broken_no_sys_uuid(self, m_read_system_uuid):
+        """Datasource requires ability to read system_uuid and true return."""
+        m_read_system_uuid.return_value = None
+        self.assertRaises(BrokenMetadata, oracle.read_metadata)
+
+    def test_broken_no_metadata_json(self, m_read_system_uuid):
+        """Datasource requires meta_data.json."""
+        httpretty.register_uri(
+            httpretty.GET, self.mdurl + MD_VER + "/",
+            '\n'.join(['user_data']).encode('utf-8'))
+        with self.assertRaises(BrokenMetadata) as cm:
+            oracle.read_metadata()
+        self.assertIn("Required field 'meta_data.json' missing",
+                      str(cm.exception))
+
+    def test_with_userdata(self, m_read_system_uuid):
+        data = {'user_data': b'#!/bin/sh\necho hi world\n',
+                'meta_data.json': json.dumps(self.my_md)}
+        self.populate_md(data)
+        result = oracle.read_metadata()[MD_VER]
+        self.assertEqual(data['user_data'], result['user_data'])
+        self.assertEqual(self.my_md, result['meta_data'])
+
+    def test_without_userdata(self, m_read_system_uuid):
+        data = {'meta_data.json': json.dumps(self.my_md)}
+        self.populate_md(data)
+        result = oracle.read_metadata()[MD_VER]
+        self.assertNotIn('user_data', result)
+        self.assertEqual(self.my_md, result['meta_data'])
+
+    def test_unknown_fields_included(self, m_read_system_uuid):
+        """Unknown fields listed in index should be included.
+        And those ending in .json should be decoded."""
+        some_data = {'key1': 'data1', 'subk1': {'subd1': 'subv'}}
+        some_vendor_data = {'cloud-init': 'foo'}
+        data = {'meta_data.json': json.dumps(self.my_md),
+                'some_data.json': json.dumps(some_data),
+                'vendor_data.json': json.dumps(some_vendor_data),
+                'other_blob': b'this is blob'}
+        self.populate_md(data)
+        result = oracle.read_metadata()[MD_VER]
+        self.assertNotIn('user_data', result)
+        self.assertEqual(self.my_md, result['meta_data'])
+        self.assertEqual(some_data, result['some_data'])
+        self.assertEqual(some_vendor_data, result['vendor_data'])
+        self.assertEqual(data['other_blob'], result['other_blob'])
+
+
+class TestIsPlatformViable(test_helpers.CiTestCase):
+    @mock.patch(DS_PATH + ".util.read_dmi_data",
+                return_value=oracle.CHASSIS_ASSET_TAG)
+    def test_expected_viable(self, m_read_dmi_data):
+        """System with known chassis tag is viable."""
+        self.assertTrue(oracle._is_platform_viable())
+        m_read_dmi_data.assert_has_calls([mock.call('chassis-asset-tag')])
+
+    @mock.patch(DS_PATH + ".util.read_dmi_data", return_value=None)
+    def test_expected_not_viable_dmi_data_none(self, m_read_dmi_data):
+        """System without known chassis tag is not viable."""
+        self.assertFalse(oracle._is_platform_viable())
+        m_read_dmi_data.assert_has_calls([mock.call('chassis-asset-tag')])
+
+    @mock.patch(DS_PATH + ".util.read_dmi_data", return_value="LetsGoCubs")
+    def test_expected_not_viable_other(self, m_read_dmi_data):
+        """System with unnown chassis tag is not viable."""
+        self.assertFalse(oracle._is_platform_viable())
+        m_read_dmi_data.assert_has_calls([mock.call('chassis-asset-tag')])
+
+
+class TestLoadIndex(test_helpers.CiTestCase):
+    """_load_index handles parsing of an index into a proper list.
+    The tests here guarantee correct parsing of html version or
+    a fixed version.  See the function docstring for more doc."""
+
+    _known_html_api_versions = dedent("""\
+        <html>
+        <head><title>Index of /openstack/</title></head>
+        <body bgcolor="white">
+        <h1>Index of /openstack/</h1><hr><pre><a href="../">../</a>
+        <a href="2013-10-17/">2013-10-17/</a>   27-Jun-2018 12:22  -
+        <a href="latest/">latest/</a>           27-Jun-2018 12:22  -
+        </pre><hr></body>
+        </html>""")
+
+    _known_html_contents = dedent("""\
+        <html>
+        <head><title>Index of /openstack/2013-10-17/</title></head>
+        <body bgcolor="white">
+        <h1>Index of /openstack/2013-10-17/</h1><hr><pre><a href="../">../</a>
+        <a href="meta_data.json">meta_data.json</a>  27-Jun-2018 12:22  679
+        <a href="user_data">user_data</a>            27-Jun-2018 12:22  146
+        </pre><hr></body>
+        </html>""")
+
+    def test_parse_html(self):
+        """Test parsing of lower case html."""
+        self.assertEqual(
+            ['2013-10-17/', 'latest/'],
+            oracle._load_index(self._known_html_api_versions))
+        self.assertEqual(
+            ['meta_data.json', 'user_data'],
+            oracle._load_index(self._known_html_contents))
+
+    def test_parse_html_upper(self):
+        """Test parsing of upper case html, although known content is lower."""
+        def _toupper(data):
+            return data.replace("<a", "<A").replace("html>", "HTML>")
+
+        self.assertEqual(
+            ['2013-10-17/', 'latest/'],
+            oracle._load_index(_toupper(self._known_html_api_versions)))
+        self.assertEqual(
+            ['meta_data.json', 'user_data'],
+            oracle._load_index(_toupper(self._known_html_contents)))
+
+    def test_parse_newline_list_with_endl(self):
+        """Test parsing of newline separated list with ending newline."""
+        self.assertEqual(
+            ['2013-10-17/', 'latest/'],
+            oracle._load_index("\n".join(["2013-10-17/", "latest/", ""])))
+        self.assertEqual(
+            ['meta_data.json', 'user_data'],
+            oracle._load_index("\n".join(["meta_data.json", "user_data", ""])))
+
+    def test_parse_newline_list_without_endl(self):
+        """Test parsing of newline separated list with no ending newline.
+
+        Actual openstack implementation does not include trailing newline."""
+        self.assertEqual(
+            ['2013-10-17/', 'latest/'],
+            oracle._load_index("\n".join(["2013-10-17/", "latest/"])))
+        self.assertEqual(
+            ['meta_data.json', 'user_data'],
+            oracle._load_index("\n".join(["meta_data.json", "user_data"])))
+
+
+# vi: ts=4 expandtab
diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst
index 30e57d85..83034589 100644
--- a/doc/rtd/topics/datasources.rst
+++ b/doc/rtd/topics/datasources.rst
@@ -189,6 +189,7 @@ Follow for more information.
    datasources/nocloud.rst
    datasources/opennebula.rst
    datasources/openstack.rst
+   datasources/oracle.rst
    datasources/ovf.rst
    datasources/smartos.rst
    datasources/fallback.rst
diff --git a/doc/rtd/topics/datasources/oracle.rst b/doc/rtd/topics/datasources/oracle.rst
new file mode 100644
index 00000000..f2383cee
--- /dev/null
+++ b/doc/rtd/topics/datasources/oracle.rst
@@ -0,0 +1,26 @@
+.. _datasource_oracle:
+
+Oracle
+======
+
+This datasource reads metadata, vendor-data and user-data from
+`Oracle Compute Infrastructure`_ (OCI).
+
+Oracle Platform
+---------------
+OCI provides bare metal and virtual machines.  In both cases, 
+the platform identifies itself via DMI data in the chassis asset tag
+with the string 'OracleCloud.com'.
+
+Oracle's platform provides a metadata service that mimics the 2013-10-17
+version of OpenStack metadata service.  Initially support for Oracle
+was done via the OpenStack datasource.
+
+Cloud-init has a specific datasource for Oracle in order to:
+ a. allow and support future growth of the OCI platform.
+ b. address small differences between OpenStack and Oracle metadata
+    implementation.
+
+
+.. _Oracle Compute Infrastructure: https://cloud.oracle.com/
+.. vi: textwidth=78
diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
index 1a5a3db2..6b01a4ea 100644
--- a/tests/unittests/test_datasource/test_common.py
+++ b/tests/unittests/test_datasource/test_common.py
@@ -20,6 +20,7 @@ from cloudinit.sources import (
     DataSourceNoCloud as NoCloud,
     DataSourceOpenNebula as OpenNebula,
     DataSourceOpenStack as OpenStack,
+    DataSourceOracle as Oracle,
     DataSourceOVF as OVF,
     DataSourceScaleway as Scaleway,
     DataSourceSmartOS as SmartOS,
@@ -37,6 +38,7 @@ DEFAULT_LOCAL = [
     IBMCloud.DataSourceIBMCloud,
     NoCloud.DataSourceNoCloud,
     OpenNebula.DataSourceOpenNebula,
+    Oracle.DataSourceOracle,
     OVF.DataSourceOVF,
     SmartOS.DataSourceSmartOS,
     Ec2.DataSourceEc2Local,
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index d862f4bc..6e1e971b 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -16,7 +16,7 @@ from six import StringIO
 
 from cloudinit import helpers
 from cloudinit import settings
-from cloudinit.sources import convert_vendordata, UNSET
+from cloudinit.sources import BrokenMetadata, convert_vendordata, UNSET
 from cloudinit.sources import DataSourceOpenStack as ds
 from cloudinit.sources.helpers import openstack
 from cloudinit import util
@@ -186,7 +186,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('meta_data.json'):
                 os_files[k] = json.dumps(os_meta)
         _register_uris(self.VERSION, {}, {}, os_files)
-        self.assertRaises(openstack.BrokenMetadata, _read_metadata_service)
+        self.assertRaises(BrokenMetadata, _read_metadata_service)
 
     def test_userdata_empty(self):
         os_files = copy.deepcopy(OS_FILES)
@@ -217,7 +217,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('vendor_data.json'):
                 os_files[k] = '{'  # some invalid json
         _register_uris(self.VERSION, {}, {}, os_files)
-        self.assertRaises(openstack.BrokenMetadata, _read_metadata_service)
+        self.assertRaises(BrokenMetadata, _read_metadata_service)
 
     def test_metadata_invalid(self):
         os_files = copy.deepcopy(OS_FILES)
@@ -225,7 +225,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('meta_data.json'):
                 os_files[k] = '{'  # some invalid json
         _register_uris(self.VERSION, {}, {}, os_files)
-        self.assertRaises(openstack.BrokenMetadata, _read_metadata_service)
+        self.assertRaises(BrokenMetadata, _read_metadata_service)
 
     @test_helpers.mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
     def test_datasource(self, m_dhcp):
@@ -525,8 +525,11 @@ class TestDetectOpenStack(test_helpers.CiTestCase):
 
         m_dmi.side_effect = fake_dmi_read
         self.assertTrue(
-            ds.detect_openstack(),
+            ds.detect_openstack(accept_oracle=True),
             'Expected detect_openstack == True on OracleCloud.com')
+        self.assertFalse(
+            ds.detect_openstack(accept_oracle=False),
+            'Expected detect_openstack == False.')
 
     @test_helpers.mock.patch(MOCK_PATH + 'util.get_proc_env')
     @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data')
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 64d9f9f8..e08e7908 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -12,6 +12,7 @@ from cloudinit.tests.helpers import (
 
 from cloudinit.sources import DataSourceIBMCloud as ds_ibm
 from cloudinit.sources import DataSourceSmartOS as ds_smartos
+from cloudinit.sources import DataSourceOracle as ds_oracle
 
 UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu "
                "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux")
@@ -598,6 +599,18 @@ class TestIsIBMProvisioning(DsIdentifyBase):
         self.assertIn("from current boot", ret.stderr)
 
 
+class TestOracle(DsIdentifyBase):
+    def test_found_by_chassis(self):
+        """Simple positive test of Oracle by chassis id."""
+        self._test_ds_found('Oracle')
+
+    def test_not_found(self):
+        """Simple negative test of Oracle."""
+        mycfg = copy.deepcopy(VALID_CFG['Oracle'])
+        mycfg['files'][P_CHASSIS_ASSET_TAG] = "Not Oracle"
+        self._check_via_dict(mycfg, rc=RC_NOT_FOUND)
+
+
 def blkid_out(disks=None):
     """Convert a list of disk dictionaries into blkid content."""
     if disks is None:
@@ -838,6 +851,12 @@ VALID_CFG = {
              },
         ],
     },
+    'Oracle': {
+        'ds': 'Oracle',
+        'files': {
+            P_CHASSIS_ASSET_TAG: ds_oracle.CHASSIS_ASSET_TAG + '\n',
+        }
+    },
     'SmartOS-bhyve': {
         'ds': 'SmartOS',
         'mocks': [
diff --git a/tools/ds-identify b/tools/ds-identify
index ce0477a5..fcc60149 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -116,7 +116,7 @@ DI_DSNAME=""
 # be searched if there is no setting found in config.
 DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \
 CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \
-OVF SmartOS Scaleway Hetzner IBMCloud"
+OVF SmartOS Scaleway Hetzner IBMCloud Oracle"
 DI_DSLIST=""
 DI_MODE=""
 DI_ON_FOUND=""
@@ -1036,6 +1036,12 @@ dscheck_Hetzner() {
     return ${DS_NOT_FOUND}
 }
 
+dscheck_Oracle() {
+    local asset_tag="OracleCloud.com"
+    dmi_chassis_asset_tag_matches "${asset_tag}" && return ${DS_FOUND}
+    return ${DS_NOT_FOUND}
+}
+
 is_ibm_provisioning() {
     local pcfg="${PATH_ROOT}/root/provisioningConfiguration.cfg"
     local logf="${PATH_ROOT}/root/swinstall.log"
-- 
cgit v1.2.3


From 8d9d4c8477732c5cc559eb10ddacef4598f93591 Mon Sep 17 00:00:00 2001
From: Joshua Powers <josh.powers@canonical.com>
Date: Fri, 31 Aug 2018 22:11:49 +0000
Subject: read-version: enhance error message

The error message when read-vesion is not very useful and does not help
the end-user know how to overcome the issue. This adds a short message
explaining that the user does not have the latest upstream tags and how
to get those tags.
---
 tools/read-version | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'tools')

diff --git a/tools/read-version b/tools/read-version
index 3ea9e66e..e69c2ce0 100755
--- a/tools/read-version
+++ b/tools/read-version
@@ -76,6 +76,12 @@ if is_gitdir(_tdir) and which("git"):
     if not version.startswith(src_version):
         sys.stderr.write("git describe version (%s) differs from "
                          "cloudinit.version (%s)\n" % (version, src_version))
+        sys.stderr.write(
+            "Please get the latest upstream tags.\n"
+            "As an example, this can be done with the following:\n"
+            "$ git remote add upstream https://git.launchpad.net/cloud-init\n"
+            "$ git fetch upstream --tags\n"
+        )
         sys.exit(1)
 
     version_long = tiny_p(cmd + ["--long"]).strip()
-- 
cgit v1.2.3


From bb60f61bfcd92465f4076e8d8abbb486cf6bbcf1 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Fri, 14 Sep 2018 21:12:17 +0000
Subject: ds-identify: doc string cleanup.

There was a typo in the doc string at the top of ds-identify
(disable -> disabled).  That is fixed here as well as adding some
better examples on content in /etc/cloud/ds-identify.cfg.
---
 tools/ds-identify | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

(limited to 'tools')

diff --git a/tools/ds-identify b/tools/ds-identify
index fcc60149..5afe5aa1 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -2,16 +2,24 @@
 # shellcheck disable=2015,2039,2162,2166
 #
 # ds-identify is configured via /etc/cloud/ds-identify.cfg
-# or on the kernel command line. It takes primarily 2 inputs:
+# or on the kernel command line. It takes the following inputs:
+#
 # datasource: can specify the datasource that should be used.
-#     kernel command line option: ci.datasource=<dsname>
+#   kernel command line option: ci.datasource=<dsname> or ci.ds=<dsname>
+#   example line in /etc/cloud/ds-identify.cfg:
+#      datasource: Ec2
 #
 # policy: a string that indicates how ds-identify should operate.
-#     kernel command line option: ci.di.policy=<policy>
+#
 #   The format is:
 #        <mode>,found=value,maybe=value,notfound=value
 #   default setting is:
-#     search,found=all,maybe=all,notfound=disable
+#     search,found=all,maybe=all,notfound=disabled
+#
+#   kernel command line option: ci.di.policy=<policy>
+#   example line in /etc/cloud/ds-identify.cfg:
+#      policy: search,found=all,maybe=none,notfound=disabled
+#
 #
 #   Mode:
 #     disabled: disable cloud-init
-- 
cgit v1.2.3


From 0b0378dd07f16d45c16e5750b6815b22a771860d Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Tue, 25 Sep 2018 18:03:35 +0000
Subject: tools/tox-venv: update for new features.

This update to tox-venv allows you to do:
 ./tools/tox-venv py3 - tests/unittests/test_util.py
---
 tools/tox-venv | 189 ++++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 166 insertions(+), 23 deletions(-)

(limited to 'tools')

diff --git a/tools/tox-venv b/tools/tox-venv
index 76ed5076..a5d21625 100755
--- a/tools/tox-venv
+++ b/tools/tox-venv
@@ -1,42 +1,185 @@
 #!/bin/sh
+# https://gist.github.com/smoser/2d4100a6a5d230ca937f
 
+CR='
+'
 error() { echo "$@" 1>&2; }
 fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
+get_env_dirs() {
+    # read 'tox --showconfig'. return list of
+    #  envname:dir
+    local key="" equal="" val="" curenv="" out=""
+    while read key equal val; do
+        case "$key" in
+           "[testenv:"*)
+                curenv=${key#*:};
+                curenv=${curenv%%"]"*};
+                continue;;
+        esac
+        if [ "${key#*=}" != "$key" ]; then
+            # older tox shows key=value or key=   value
+            # newer tox shows: key =    value
+            key=${key%%=*}
+            val=${equal}
+        fi
+        [ "$key" = "envdir" ] || continue
+        out="${out:+${out}${CR}}${curenv}:$val"
+    done
+    echo "$out"
+}
+
+load_config() {
+    local tox_ini="$1" out="" envs=""
+    if [ "$tox_ini" = "${CACHED_ENVS_INI}" ]; then
+        _RET="$CACHED_ENVS"
+        return
+    fi
+    out=$(tox -c "$tox_ini" --showconfig) || return 1
+    envs=$(echo "$out" | get_env_dirs) || return 1
+    CACHED_ENVS="$envs"
+    CACHED_ENVS_INI="$tox_ini"
+    _RET="$envs"
+}
+
+list_environments() {
+    local tox_ini="$1" prefix="  " out="" envs="" oifs="$IFS"
+    load_config "$tox_ini" || return 1
+    envs="${_RET}"
+    IFS="$CR"
+    for d in ${envs}; do
+        env=${d%%:*}
+        dir=${d#*:}
+        [ -f "$dir/bin/activate" ] && s="*" || s=""
+        echo "${prefix}$env$s";
+    done
+    IFS="$oifs"
+}
+
+get_command() {
+    local tox_ini="$1" env="$2" out=""
+    shift 2
+    out=$(
+        sed -e ':x; /\\$/ { N; s/\\\n[ ]*//; tx };' "${tox_ini}" |
+        gawk '
+        $1 ~ /^\[testenv.*\]/ {
+            name=$1;
+            sub("\\[", "", name); sub(".*:", "", name);
+            sub("].*", "", name);
+            curenv=name; };
+        $1 == "basepython" && (name == "testenv" || name == n) { python=$3 }
+        $1 == "commands" && (name == "testenv" || name == n) {
+            sub("commands = ", ""); cmd = $0; };
+        END {
+            sub("{envpython}", python, cmd);
+            sub("{toxinidir}", toxinidir, cmd);
+            if (inargs == "") replacement = "\\1"
+            else replacement = inargs
+            cmd = gensub(/{posargs:?([^}]*)}/, replacement, "global", cmd)
+            print(cmd);
+            }' n="$env" toxinidir="$(dirname $tox_ini)" inargs="$*")
+    if [ -z "$out" ]; then
+        error "Failed to find command for $env in $tox_ini"
+        return 1
+    fi
+    echo "$out"
+}
+
+get_env_dir() {
+    local tox_ini="$1" env="$2" oifs="$IFS" t="" d="" envs=""
+    if [ "${TOX_VENV_SHORTCUT:-1}" != "0" ]; then
+        local stox_d="${tox_ini%/*}/.tox/${env}"
+        if [ -e "${stox_d}/bin/activate" ]; then
+            _RET="${stox_d}"
+            return
+        fi
+    fi
+    load_config "$tox_ini" && envs="$_RET" || return 1
+    IFS="$CR"
+    for t in $envs; do
+        [ "$env" = "${t%%:*}" ] && d="${t#*:}" && break
+    done
+    IFS=${oifs}
+    [ -n "$d" ] || return 1
+    _RET="$d"
+}
+
 Usage() {
-   cat <<EOF
-Usage: ${0##*/} tox-environment [command [args]]
+    local tox_ini="$1"
+    cat <<EOF
+Usage: ${0##*/} [--no-create] tox-environment [command [args]]
    run command with provided arguments in the provided tox environment
-   command defaults to \${SHELL:-/bin/sh}.
+   command defaults to 'cmd' (see below).
+
+   run with '--list' to show available environments
 
-   invoke with '--list' to show available environments
+   if 'command' above is literal 'cmd' or '-', then the 'command' will
+   be read from tox.ini.  This allows you to do:
+      tox-venv py27 - tests/some/sub/dir
+   and have the 'command' read correctly and have that execute:
+      python -m nose tests/some/sub/dir
 EOF
-}
-list_toxes() {
-   local td="$1" pre="$2" d=""
-   ( cd "$tox_d" &&
-     for d in *; do [ -f "$d/bin/activate" ] && echo "${pre}$d"; done)
+
+    if [ -f "$tox_ini" ]; then
+        local oini=${tox_ini}
+        [ "${tox_ini}" -ef "$PWD/tox.ini" ] && oini="./tox.ini"
+        echo
+        echo "environments in $oini"
+        list_environments "$tox_ini"
+    fi
 }
 
-[ $# -eq 0 ] && { Usage 1>&2; exit 1; }
-[ "$1" = "-h" -o "$1" = "--help" ] && { Usage; exit 0; }
+if [ -f tox.ini ]; then
+    tox_ini="$PWD/tox.ini"
+else
+    tox_ini="${0%/*}/../tox.ini"
+fi
 
-env="$1"
-shift
-tox_d="${0%/*}/../.tox"
-activate="$tox_d/$env/bin/activate"
+[ $# -eq 0 ] && { Usage "$tox_ini" 1>&2; exit 1; }
+[ "$1" = "-h" -o "$1" = "--help" ] && { Usage "$tox_ini"; exit 0; }
 
+[ -f "$tox_ini" ] || fail "$tox_ini: did not find tox.ini"
 
-[ -d "$tox_d" ] || fail "$tox_d: not a dir. maybe run 'tox'?"
+if [ "$1" = "-l" -o "$1" = "--list" ]; then
+    list_environments "$tox_ini"
+    exit
+fi
 
-[ "$env" = "-l" -o "$env" = "--list" ] && { list_toxes ; exit ; }
+nocreate="false"
+if [ "$1" = "--no-create" ]; then
+    nocreate="true"
+    shift
+fi
 
-if [ ! -f "$activate" ]; then
-   error "$env: not a valid tox environment?"
-   error "try one of:"
-   list_toxes "$tox_d" "  "
-   fail
+env="$1"
+shift
+[ "$1" = "--" ] && shift
+get_env_dir "$tox_ini" "$env" && activate="$_RET/bin/activate" || activate=""
+
+if [ -z "$activate" -o ! -f "$activate" ]; then
+    if $nocreate; then
+        fail "tox env '$env' did not exist, and no-create specified"
+    elif [ -n "$activate" ]; then
+        error "attempting to create $env:"
+        error "    tox -c $tox_ini --recreate --notest -e $env"
+        tox -c "$tox_ini" --recreate --notest -e "$env" ||
+            fail "failed creation of env $env"
+    else
+        error "$env: not a valid tox environment?"
+        error "found tox_ini=$tox_ini"
+        error "try one of:"
+        list_environments "$tox_ini" 1>&2
+        fail
+    fi
 fi
 . "$activate"
 
-[ "$#" -gt 0 ] || set -- ${SHELL:-/bin/bash}
+[ $# -eq 0 ] && set -- cmd
+if [ "$1" = "cmd" -o "$1" = "-" ]; then
+   shift
+   out=$(get_command "$tox_ini" "$env" "$@") || exit
+   eval set -- "$out"
+fi
+echo "inside tox:$env running: $*" 1>&2
 debian_chroot="tox:$env" exec "$@"
+
+# vi: ts=4 expandtab
-- 
cgit v1.2.3