IBMCloud: Initial IBM Cloud datasource.

This adds a specific IBM Cloud datasource.
IBM Cloud is identified by:
 a.) running on xen
 b.) one of a LABEL=METADATA disk or a LABEL=config-2 disk with
     UUID=9796-932E

The datasource contains its own config-drive reader that reads
only the currently supported portion of config-drive needed for
ibm cloud.

During the provisioning boot, cloud-init is disabled.

See the docstring in DataSourceIBMCloud.py for more more information.

2 files changed, 363 insertions, 3 deletions

diff --git a/tests/unittests/test_datasource/test_ibmcloud.py b/tests/unittests/test_datasource/test_ibmcloud.py
new file mode 100644
index 00000000..621cfe49
--- /dev/null
+++ b/tests/unittests/test_datasource/test_ibmcloud.py
@@ -0,0 +1,262 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit.sources import DataSourceIBMCloud as ibm
+from cloudinit.tests import helpers as test_helpers
+
+import base64
+import copy
+import json
+import mock
+from textwrap import dedent
+
+D_PATH = "cloudinit.sources.DataSourceIBMCloud."
+
+
+class TestIBMCloud(test_helpers.CiTestCase):
+    """Test the datasource."""
+    def setUp(self):
+        super(TestIBMCloud, self).setUp()
+        pass
+
+
+@mock.patch(D_PATH + "_is_xen", return_value=True)
+@mock.patch(D_PATH + "_is_ibm_provisioning")
+@mock.patch(D_PATH + "util.blkid")
+class TestGetIBMPlatform(test_helpers.CiTestCase):
+    """Test the get_ibm_platform helper."""
+
+    blkid_base = {
+        "/dev/xvda1": {
+            "DEVNAME": "/dev/xvda1", "LABEL": "cloudimg-bootfs",
+            "TYPE": "ext3"},
+        "/dev/xvda2": {
+            "DEVNAME": "/dev/xvda2", "LABEL": "cloudimg-rootfs",
+            "TYPE": "ext4"},
+    }
+
+    blkid_metadata_disk = {
+        "/dev/xvdh1": {
+            "DEVNAME": "/dev/xvdh1", "LABEL": "METADATA", "TYPE": "vfat",
+            "SEC_TYPE": "msdos", "UUID": "681B-8C5D",
+            "PARTUUID": "3d631e09-01"},
+    }
+
+    blkid_oscode_disk = {
+        "/dev/xvdh": {
+            "DEVNAME": "/dev/xvdh", "LABEL": "config-2", "TYPE": "vfat",
+            "SEC_TYPE": "msdos", "UUID": ibm.IBM_CONFIG_UUID}
+    }
+
+    def setUp(self):
+        self.blkid_metadata = copy.deepcopy(self.blkid_base)
+        self.blkid_metadata.update(copy.deepcopy(self.blkid_metadata_disk))
+
+        self.blkid_oscode = copy.deepcopy(self.blkid_base)
+        self.blkid_oscode.update(copy.deepcopy(self.blkid_oscode_disk))
+
+    def test_id_template_live_metadata(self, m_blkid, m_is_prov, _m_xen):
+        """identify TEMPLATE_LIVE_METADATA."""
+        m_blkid.return_value = self.blkid_metadata
+        m_is_prov.return_value = False
+        self.assertEqual(
+            (ibm.Platforms.TEMPLATE_LIVE_METADATA, "/dev/xvdh1"),
+            ibm.get_ibm_platform())
+
+    def test_id_template_prov_metadata(self, m_blkid, m_is_prov, _m_xen):
+        """identify TEMPLATE_PROVISIONING_METADATA."""
+        m_blkid.return_value = self.blkid_metadata
+        m_is_prov.return_value = True
+        self.assertEqual(
+            (ibm.Platforms.TEMPLATE_PROVISIONING_METADATA, "/dev/xvdh1"),
+            ibm.get_ibm_platform())
+
+    def test_id_template_prov_nodata(self, m_blkid, m_is_prov, _m_xen):
+        """identify TEMPLATE_PROVISIONING_NODATA."""
+        m_blkid.return_value = self.blkid_base
+        m_is_prov.return_value = True
+        self.assertEqual(
+            (ibm.Platforms.TEMPLATE_PROVISIONING_NODATA, None),
+            ibm.get_ibm_platform())
+
+    def test_id_os_code(self, m_blkid, m_is_prov, _m_xen):
+        """Identify OS_CODE."""
+        m_blkid.return_value = self.blkid_oscode
+        m_is_prov.return_value = False
+        self.assertEqual((ibm.Platforms.OS_CODE, "/dev/xvdh"),
+                         ibm.get_ibm_platform())
+
+    def test_id_os_code_must_match_uuid(self, m_blkid, m_is_prov, _m_xen):
+        """Test against false positive on openstack with non-ibm UUID."""
+        blkid = self.blkid_oscode
+        blkid["/dev/xvdh"]["UUID"] = "9999-9999"
+        m_blkid.return_value = blkid
+        m_is_prov.return_value = False
+        self.assertEqual((None, None), ibm.get_ibm_platform())
+
+
+@mock.patch(D_PATH + "_read_system_uuid", return_value=None)
+@mock.patch(D_PATH + "get_ibm_platform")
+class TestReadMD(test_helpers.CiTestCase):
+    """Test the read_datasource helper."""
+
+    template_md = {
+        "files": [],
+        "network_config": {"content_path": "/content/interfaces"},
+        "hostname": "ci-fond-ram",
+        "name": "ci-fond-ram",
+        "domain": "testing.ci.cloud-init.org",
+        "meta": {"dsmode": "net"},
+        "uuid": "8e636730-9f5d-c4a5-327c-d7123c46e82f",
+        "public_keys": {"1091307": "ssh-rsa AAAAB3NzaC1...Hw== ci-pubkey"},
+    }
+
+    oscode_md = {
+        "hostname": "ci-grand-gannet.testing.ci.cloud-init.org",
+        "name": "ci-grand-gannet",
+        "uuid": "2f266908-8e6c-4818-9b5c-42e9cc66a785",
+        "random_seed": "bm90LXJhbmRvbQo=",
+        "crypt_key": "ssh-rsa AAAAB3NzaC1yc2..n6z/",
+        "configuration_token": "eyJhbGciOi..M3ZA",
+        "public_keys": {"1091307": "ssh-rsa AAAAB3N..Hw== ci-pubkey"},
+    }
+
+    content_interfaces = dedent("""\
+        auto lo
+        iface lo inet loopback
+
+        auto eth0
+        allow-hotplug eth0
+        iface eth0 inet static
+        address 10.82.43.5
+        netmask 255.255.255.192
+        """)
+
+    userdata = b"#!/bin/sh\necho hi mom\n"
+    # meta.js file gets json encoded userdata as a list.
+    meta_js = '["#!/bin/sh\necho hi mom\n"]'
+    vendor_data = {
+        "cloud-init": "#!/bin/bash\necho 'root:$6$5ab01p1m1' | chpasswd -e"}
+
+    network_data = {
+        "links": [
+            {"id": "interface_29402281", "name": "eth0", "mtu": None,
+             "type": "phy", "ethernet_mac_address": "06:00:f1:bd:da:25"},
+            {"id": "interface_29402279", "name": "eth1", "mtu": None,
+             "type": "phy", "ethernet_mac_address": "06:98:5e:d0:7f:86"}
+        ],
+        "networks": [
+            {"id": "network_109887563", "link": "interface_29402281",
+             "type": "ipv4", "ip_address": "10.82.43.2",
+             "netmask": "255.255.255.192",
+             "routes": [
+                 {"network": "10.0.0.0", "netmask": "255.0.0.0",
+                  "gateway": "10.82.43.1"},
+                 {"network": "161.26.0.0", "netmask": "255.255.0.0",
+                  "gateway": "10.82.43.1"}]},
+            {"id": "network_109887551", "link": "interface_29402279",
+             "type": "ipv4", "ip_address": "108.168.194.252",
+             "netmask": "255.255.255.248",
+             "routes": [
+                 {"network": "0.0.0.0", "netmask": "0.0.0.0",
+                  "gateway": "108.168.194.249"}]}
+        ],
+        "services": [
+            {"type": "dns", "address": "10.0.80.11"},
+            {"type": "dns", "address": "10.0.80.12"}
+        ],
+    }
+
+    sysuuid = '7f79ebf5-d791-43c3-a723-854e8389d59f'
+
+    def _get_expected_metadata(self, os_md):
+        """return expected 'metadata' for data loaded from meta_data.json."""
+        os_md = copy.deepcopy(os_md)
+        renames = (
+            ('hostname', 'local-hostname'),
+            ('uuid', 'instance-id'),
+            ('public_keys', 'public-keys'))
+        ret = {}
+        for osname, mdname in renames:
+            if osname in os_md:
+                ret[mdname] = os_md[osname]
+        if 'random_seed' in os_md:
+            ret['random_seed'] = base64.b64decode(os_md['random_seed'])
+
+        return ret
+
+    def test_provisioning_md(self, m_platform, m_sysuuid):
+        """Provisioning env with a metadata disk should return None."""
+        m_platform.return_value = (
+            ibm.Platforms.TEMPLATE_PROVISIONING_METADATA, "/dev/xvdh")
+        self.assertIsNone(ibm.read_md())
+
+    def test_provisioning_no_metadata(self, m_platform, m_sysuuid):
+        """Provisioning env with no metadata disk should return None."""
+        m_platform.return_value = (
+            ibm.Platforms.TEMPLATE_PROVISIONING_NODATA, None)
+        self.assertIsNone(ibm.read_md())
+
+    def test_provisioning_not_ibm(self, m_platform, m_sysuuid):
+        """Provisioning env but not identified as IBM should return None."""
+        m_platform.return_value = (None, None)
+        self.assertIsNone(ibm.read_md())
+
+    def test_template_live(self, m_platform, m_sysuuid):
+        """Template live environment should be identified."""
+        tmpdir = self.tmp_dir()
+        m_platform.return_value = (
+            ibm.Platforms.TEMPLATE_LIVE_METADATA, tmpdir)
+        m_sysuuid.return_value = self.sysuuid
+
+        test_helpers.populate_dir(tmpdir, {
+            'openstack/latest/meta_data.json': json.dumps(self.template_md),
+            'openstack/latest/user_data': self.userdata,
+            'openstack/content/interfaces': self.content_interfaces,
+            'meta.js': self.meta_js})
+
+        ret = ibm.read_md()
+        self.assertEqual(ibm.Platforms.TEMPLATE_LIVE_METADATA,
+                         ret['platform'])
+        self.assertEqual(tmpdir, ret['source'])
+        self.assertEqual(self.userdata, ret['userdata'])
+        self.assertEqual(self._get_expected_metadata(self.template_md),
+                         ret['metadata'])
+        self.assertEqual(self.sysuuid, ret['system-uuid'])
+
+    def test_os_code_live(self, m_platform, m_sysuuid):
+        """Verify an os_code metadata path."""
+        tmpdir = self.tmp_dir()
+        m_platform.return_value = (ibm.Platforms.OS_CODE, tmpdir)
+        netdata = json.dumps(self.network_data)
+        test_helpers.populate_dir(tmpdir, {
+            'openstack/latest/meta_data.json': json.dumps(self.oscode_md),
+            'openstack/latest/user_data': self.userdata,
+            'openstack/latest/vendor_data.json': json.dumps(self.vendor_data),
+            'openstack/latest/network_data.json': netdata,
+        })
+
+        ret = ibm.read_md()
+        self.assertEqual(ibm.Platforms.OS_CODE, ret['platform'])
+        self.assertEqual(tmpdir, ret['source'])
+        self.assertEqual(self.userdata, ret['userdata'])
+        self.assertEqual(self._get_expected_metadata(self.oscode_md),
+                         ret['metadata'])
+
+    def test_os_code_live_no_userdata(self, m_platform, m_sysuuid):
+        """Verify os_code without user-data."""
+        tmpdir = self.tmp_dir()
+        m_platform.return_value = (ibm.Platforms.OS_CODE, tmpdir)
+        test_helpers.populate_dir(tmpdir, {
+            'openstack/latest/meta_data.json': json.dumps(self.oscode_md),
+            'openstack/latest/vendor_data.json': json.dumps(self.vendor_data),
+        })
+
+        ret = ibm.read_md()
+        self.assertEqual(ibm.Platforms.OS_CODE, ret['platform'])
+        self.assertEqual(tmpdir, ret['source'])
+        self.assertIsNone(ret['userdata'])
+        self.assertEqual(self._get_expected_metadata(self.oscode_md),
+                         ret['metadata'])
+
+
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 85999b7a..53643989 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -9,6 +9,8 @@ from cloudinit import util
 from cloudinit.tests.helpers import (
     CiTestCase, dir2dict, populate_dir)
 
+from cloudinit.sources import DataSourceIBMCloud as dsibm
+
 UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu "
                "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux")
 UNAME_PPC64EL = ("Linux diamond 4.4.0-83-generic #106-Ubuntu SMP "
@@ -37,8 +39,8 @@ BLKID_UEFI_UBUNTU = [
 
 POLICY_FOUND_ONLY = "search,found=all,maybe=none,notfound=disabled"
 POLICY_FOUND_OR_MAYBE = "search,found=all,maybe=all,notfound=disabled"
-DI_DEFAULT_POLICY = "search,found=all,maybe=all,notfound=enabled"
-DI_DEFAULT_POLICY_NO_DMI = "search,found=all,maybe=all,notfound=disabled"
+DI_DEFAULT_POLICY = "search,found=all,maybe=all,notfound=disabled"
+DI_DEFAULT_POLICY_NO_DMI = "search,found=all,maybe=all,notfound=enabled"
 DI_EC2_STRICT_ID_DEFAULT = "true"
 OVF_MATCH_STRING = 'http://schemas.dmtf.org/ovf/environment/1'
 
@@ -64,6 +66,9 @@ P_SYS_VENDOR = "sys/class/dmi/id/sys_vendor"
 P_SEED_DIR = "var/lib/cloud/seed"
 P_DSID_CFG = "etc/cloud/ds-identify.cfg"
 
+IBM_PROVISIONING_CHECK_PATH = "/root/provisioningConfiguration.cfg"
+IBM_CONFIG_UUID = "9796-932E"
+
 MOCK_VIRT_IS_KVM = {'name': 'detect_virt', 'RET': 'kvm', 'ret': 0}
 MOCK_VIRT_IS_VMWARE = {'name': 'detect_virt', 'RET': 'vmware', 'ret': 0}
 MOCK_VIRT_IS_XEN = {'name': 'detect_virt', 'RET': 'xen', 'ret': 0}
@@ -239,6 +244,57 @@ class TestDsIdentify(CiTestCase):
         self._test_ds_found('ConfigDriveUpper')
         return
 
+    def test_ibmcloud_template_userdata_in_provisioning(self):
+        """Template provisioned with user-data during provisioning stage.
+
+        Template provisioning with user-data has METADATA disk,
+        datasource should return not found."""
+        data = copy.deepcopy(VALID_CFG['IBMCloud-metadata'])
+        data['files'] = {IBM_PROVISIONING_CHECK_PATH: 'xxx'}
+        return self._check_via_dict(data, RC_NOT_FOUND)
+
+    def test_ibmcloud_template_userdata(self):
+        """Template provisioned with user-data first boot.
+
+        Template provisioning with user-data has METADATA disk.
+        datasource should return found."""
+        self._test_ds_found('IBMCloud-metadata')
+
+    def test_ibmcloud_template_no_userdata_in_provisioning(self):
+        """Template provisioned with no user-data during provisioning.
+
+        no disks attached.  Datasource should return not found."""
+        data = copy.deepcopy(VALID_CFG['IBMCloud-nodisks'])
+        data['files'] = {IBM_PROVISIONING_CHECK_PATH: 'xxx'}
+        return self._check_via_dict(data, RC_NOT_FOUND)
+
+    def test_ibmcloud_template_no_userdata(self):
+        """Template provisioned with no user-data first boot.
+
+        no disks attached.  Datasource should return found."""
+        self._check_via_dict(VALID_CFG['IBMCloud-nodisks'], RC_NOT_FOUND)
+
+    def test_ibmcloud_os_code(self):
+        """Launched by os code always has config-2 disk."""
+        self._test_ds_found('IBMCloud-config-2')
+
+    def test_ibmcloud_os_code_different_uuid(self):
+        """IBM cloud config-2 disks must be explicit match on UUID.
+
+        If the UUID is not 9796-932E then we actually expect ConfigDrive."""
+        data = copy.deepcopy(VALID_CFG['IBMCloud-config-2'])
+        offset = None
+        for m, d in enumerate(data['mocks']):
+            if d.get('name') == "blkid":
+                offset = m
+                break
+        if not offset:
+            raise ValueError("Expected to find 'blkid' mock, but did not.")
+        data['mocks'][offset]['out'] = d['out'].replace(dsibm.IBM_CONFIG_UUID,
+                                                        "DEAD-BEEF")
+        self._check_via_dict(
+            data, rc=RC_FOUND, dslist=['ConfigDrive', DS_NONE])
+
     def test_policy_disabled(self):
         """A Builtin policy of 'disabled' should return not found.
 
@@ -452,7 +508,7 @@ VALID_CFG = {
     },
     'Ec2-xen': {
         'ds': 'Ec2',
-        'mocks': [{'name': 'detect_virt', 'RET': 'xen', 'ret': 0}],
+        'mocks': [MOCK_VIRT_IS_XEN],
         'files': {
             'sys/hypervisor/uuid': 'ec2c6e2f-5fac-4fc7-9c82-74127ec14bbb\n'
         },
@@ -579,6 +635,48 @@ VALID_CFG = {
         'ds': 'Hetzner',
         'files': {P_SYS_VENDOR: 'Hetzner\n'},
     },
+    'IBMCloud-metadata': {
+        'ds': 'IBMCloud',
+        'mocks': [
+            MOCK_VIRT_IS_XEN,
+            {'name': 'blkid', 'ret': 0,
+             'out': blkid_out(
+                 [{'DEVNAME': 'xvda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()},
+                  {'DEVNAME': 'xvda2', 'TYPE': 'ext4',
+                   'LABEL': 'cloudimg-rootfs', 'PARTUUID': uuid4()},
+                  {'DEVNAME': 'xvdb', 'TYPE': 'vfat', 'LABEL': 'METADATA'}]),
+             },
+        ],
+    },
+    'IBMCloud-config-2': {
+        'ds': 'IBMCloud',
+        'mocks': [
+            MOCK_VIRT_IS_XEN,
+            {'name': 'blkid', 'ret': 0,
+             'out': blkid_out(
+                 [{'DEVNAME': 'xvda1', 'TYPE': 'ext3', 'PARTUUID': uuid4(),
+                   'UUID': uuid4(), 'LABEL': 'cloudimg-bootfs'},
+                  {'DEVNAME': 'xvdb', 'TYPE': 'vfat', 'LABEL': 'config-2',
+                   'UUID': dsibm.IBM_CONFIG_UUID},
+                  {'DEVNAME': 'xvda2', 'TYPE': 'ext4',
+                   'LABEL': 'cloudimg-rootfs', 'PARTUUID': uuid4(),
+                   'UUID': uuid4()},
+                  ]),
+             },
+        ],
+    },
+    'IBMCloud-nodisks': {
+        'ds': 'IBMCloud',
+        'mocks': [
+            MOCK_VIRT_IS_XEN,
+            {'name': 'blkid', 'ret': 0,
+             'out': blkid_out(
+                 [{'DEVNAME': 'xvda1', 'TYPE': 'vfat', 'PARTUUID': uuid4()},
+                  {'DEVNAME': 'xvda2', 'TYPE': 'ext4',
+                   'LABEL': 'cloudimg-rootfs', 'PARTUUID': uuid4()}]),
+             },
+        ],
+    },
 }
 
 # vi: ts=4 expandtab