add support for the config-drive-v2 datasource

config-drive-v2 was implemented in openstack at 
https://review.openstack.org/#/c/11184/ .  This adds support to
cloud-init for reading that.

LP: #1037567

3 files changed, 433 insertions, 71 deletions

diff --git a/ChangeLog b/ChangeLog
index 0dae0ca5..b28566e5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,5 @@
 0.7.0:
+ - add support for config-drive-v2 (LP:#1037567)
  - support creating users, including the default user.
    [Ben Howard] (LP: #1028503)
  - add apt_reboot_if_required to reboot if an upgrade or package installation
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 850b281c..b8154367 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -30,88 +30,119 @@ LOG = logging.getLogger(__name__)
 # Various defaults/constants...
 DEFAULT_IID = "iid-dsconfigdrive"
 DEFAULT_MODE = 'pass'
-CFG_DRIVE_FILES = [
+CFG_DRIVE_FILES_V1 = [
     "etc/network/interfaces",
     "root/.ssh/authorized_keys",
     "meta.js",
 ]
 DEFAULT_METADATA = {
     "instance-id": DEFAULT_IID,
-    "dsmode": DEFAULT_MODE,
 }
-CFG_DRIVE_DEV_ENV = 'CLOUD_INIT_CONFIG_DRIVE_DEVICE'
+VALID_DSMODES = ("local", "net", "pass", "disabled")
 
 
 class DataSourceConfigDrive(sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
-        self.seed = None
-        self.cfg = {}
+        self.source = None
         self.dsmode = 'local'
         self.seed_dir = os.path.join(paths.seed_dir, 'config_drive')
+        self.version = None
 
     def __str__(self):
-        mstr = "%s [%s]" % (util.obj_name(self), self.dsmode)
-        mstr += "[seed=%s]" % (self.seed)
+        mstr = "%s [%s,ver=%s]" % (util.obj_name(self), self.dsmode,
+                                   self.version)
+        mstr += "[source=%s]" % (self.source)
         return mstr
 
     def get_data(self):
         found = None
         md = {}
-        ud = ""
 
+        results = {}
         if os.path.isdir(self.seed_dir):
             try:
-                (md, ud) = read_config_drive_dir(self.seed_dir)
+                results = read_config_drive_dir(self.seed_dir)
                 found = self.seed_dir
             except NonConfigDriveDir:
                 util.logexc(LOG, "Failed reading config drive from %s",
                             self.seed_dir)
         if not found:
-            dev = find_cfg_drive_device()
-            if dev:
+            devlist = find_candidate_devs()
+            for dev in devlist:
                 try:
-                    (md, ud) = util.mount_cb(dev, read_config_drive_dir)
+                    results = util.mount_cb(dev, read_config_drive_dir)
                     found = dev
+                    break
                 except (NonConfigDriveDir, util.MountFailedError):
                     pass
+                except BrokenConfigDriveDir:
+                    util.logexc(LOG, "broken config drive: %s", dev)
 
         if not found:
             return False
 
-        if 'dsconfig' in md:
-            self.cfg = md['dscfg']
-
+        md = results['metadata']
         md = util.mergedict(md, DEFAULT_METADATA)
 
-        # Update interfaces and ifup only on the local datasource
-        # this way the DataSourceConfigDriveNet doesn't do it also.
-        if 'network-interfaces' in md and self.dsmode == "local":
+        user_dsmode = results.get('dsmode', None)
+        if user_dsmode not in VALID_DSMODES + (None,):
+            LOG.warn("user specified invalid mode: %s" % user_dsmode)
+            user_dsmode = None
+
+        dsmode = get_ds_mode(cfgdrv_ver=results['cfgdrive_ver'],
+                             ds_cfg=self.ds_cfg.get('dsmode'),
+                             user=user_dsmode)
+
+        if dsmode == "disabled":
+            # most likely user specified
+            return False
+
+        # TODO(smoser): fix this, its dirty.
+        # we want to do some things (writing files and network config)
+        # only on first boot, and even then, we want to do so in the
+        # local datasource (so they happen earlier) even if the configured
+        # dsmode is 'net' or 'pass'. To do this, we check the previous
+        # instance-id
+        prev_iid = get_previous_iid(self.paths)
+        cur_iid = md['instance-id']
+
+        if ('network_config' in results and self.dsmode == "local" and
+            prev_iid != cur_iid):
             LOG.debug("Updating network interfaces from config drive (%s)",
-                     md['dsmode'])
-            self.distro.apply_network(md['network-interfaces'])
+                      dsmode)
+            self.distro.apply_network(results['network_config'])
 
-        self.seed = found
-        self.metadata = md
-        self.userdata_raw = ud
+        # file writing occurs in local mode (to be as early as possible)
+        if self.dsmode == "local" and prev_iid != cur_iid and results['files']:
+            LOG.debug("writing injected files")
+            try:
+                write_files(results['files'])
+            except:
+                util.logexc(LOG, "Failed writing files")
+
+        # dsmode != self.dsmode here if:
+        #  * dsmode = "pass",  pass means it should only copy files and then
+        #    pass to another datasource
+        #  * dsmode = "net" and self.dsmode = "local"
+        #    so that user boothooks would be applied with network, the
+        #    local datasource just gets out of the way, and lets the net claim
+        if dsmode != self.dsmode:
+            LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode)
+            return False
 
-        if md['dsmode'] == self.dsmode:
-            return True
+        self.source = found
+        self.metadata = md
+        self.userdata_raw = results.get('userdata')
+        self.version = results['cfgdrive_ver']
 
-        LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode'])
-        return False
+        return True
 
     def get_public_ssh_keys(self):
         if not 'public-keys' in self.metadata:
             return []
         return self.metadata['public-keys']
 
-    # The data sources' config_obj is a cloud-config formated
-    # object that came to it from ways other than cloud-config
-    # because cloud-config content would be handled elsewhere
-    def get_config_obj(self):
-        return self.cfg
-
 
 class DataSourceConfigDriveNet(DataSourceConfigDrive):
     def __init__(self, sys_cfg, distro, paths):
@@ -123,48 +154,146 @@ class NonConfigDriveDir(Exception):
     pass
 
 
-def find_cfg_drive_device():
-    """Get the config drive device.  Return a string like '/dev/vdb'
-       or None (if there is no non-root device attached). This does not
-       check the contents, only reports that if there *were* a config_drive
-       attached, it would be this device.
-       Note: per config_drive documentation, this is
-       "associated as the last available disk on the instance"
-    """
+class BrokenConfigDriveDir(Exception):
+    pass
 
-    # This seems to be for debugging??
-    if CFG_DRIVE_DEV_ENV in os.environ:
-        return os.environ[CFG_DRIVE_DEV_ENV]
 
-    # We are looking for a raw block device (sda, not sda1) with a vfat
-    # filesystem on it....
-    letters = "abcdefghijklmnopqrstuvwxyz"
-    devs = util.find_devs_with("TYPE=vfat")
+def find_candidate_devs():
+    """Return a list of devices that may contain the config drive.
 
-    # Filter out anything not ending in a letter (ignore partitions)
-    devs = [f for f in devs if f[-1] in letters]
+    The returned list is sorted by search order where the first item has
+    should be searched first (highest priority)
+
+    config drive v1:
+       Per documentation, this is "associated as the last available disk on the
+       instance", and should be VFAT.
+       Currently, we do not restrict search list to "last available disk"
+
+    config drive v2:
+       Disk should be:
+        * either vfat or iso9660 formated
+        * labeled with 'config-2'
+    """
 
-    # Sort them in reverse so "last" device is first
-    devs.sort(reverse=True)
+    by_fstype = (util.find_devs_with("TYPE=vfat") +
+                 util.find_devs_with("TYPE=iso9660"))
+    by_label = util.find_devs_with("LABEL=config-2")
 
-    if devs:
-        return devs[0]
+    # give preference to "last available disk" (vdb over vda)
+    # note, this is not a perfect rendition of that.
+    by_fstype.sort(reverse=True)
+    by_label.sort(reverse=True)
 
-    return None
+    # combine list of items by putting by-label items first
+    # followed by fstype items, but with dupes removed
+    combined = (by_label + [d for d in by_fstype if d not in by_label])
+
+    # We are looking for block device (sda, not sda1), ignore partitions
+    combined = [d for d in combined if d[-1] not in "0123456789"]
+
+    return combined
 
 
 def read_config_drive_dir(source_dir):
+    last_e = NonConfigDriveDir("Not found")
+    for finder in (read_config_drive_dir_v2, read_config_drive_dir_v1):
+        try:
+            data = finder(source_dir)
+            return data
+        except NonConfigDriveDir as exc:
+            last_e = exc
+    raise last_e
+
+
+def read_config_drive_dir_v2(source_dir, version="2012-08-10"):
+
+    if (not os.path.isdir(os.path.join(source_dir, "openstack", version)) and
+        os.path.isdir(os.path.join(source_dir, "openstack", "latest"))):
+        LOG.warn("version '%s' not available, attempting to use 'latest'" %
+                 version)
+        version = "latest"
+
+    datafiles = (
+        ('metadata',
+         "openstack/%s/meta_data.json" % version, True, json.loads),
+        ('userdata', "openstack/%s/user_data" % version, False, None),
+        ('ec2-metadata', "ec2/latest/metadata.json", False, json.loads),
+    )
+
+    results = {'userdata': None}
+    for (name, path, required, process) in datafiles:
+        fpath = os.path.join(source_dir, path)
+        data = None
+        found = False
+        if os.path.isfile(fpath):
+            try:
+                with open(fpath) as fp:
+                    data = fp.read()
+            except Exception as exc:
+                raise BrokenConfigDriveDir("failed to read: %s" % fpath)
+            found = True
+        elif required:
+            raise NonConfigDriveDir("missing mandatory %s" % fpath)
+
+        if found and process:
+            try:
+                data = process(data)
+            except Exception as exc:
+                raise BrokenConfigDriveDir("failed to process: %s" % fpath)
+
+        if found:
+            results[name] = data
+
+    # instance-id is 'uuid' for openstack. just copy it to instance-id.
+    if 'instance-id' not in results['metadata']:
+        try:
+            results['metadata']['instance-id'] = results['metadata']['uuid']
+        except KeyError:
+            raise BrokenConfigDriveDir("No uuid entry in metadata")
+
+    def read_content_path(item):
+        # do not use os.path.join here, as content_path starts with /
+        cpath = os.path.sep.join((source_dir, "openstack",
+                                  "./%s" % item['content_path']))
+        with open(cpath) as fp:
+            return(fp.read())
+
+    files = {}
+    try:
+        for item in results['metadata'].get('files', {}):
+            files[item['path']] = read_content_path(item)
+
+        # the 'network_config' item in metadata is a content pointer
+        # to the network config that should be applied.
+        # in folsom, it is just a '/etc/network/interfaces' file.
+        item = results['metadata'].get("network_config", None)
+        if item:
+            results['network_config'] = read_content_path(item)
+    except Exception as exc:
+        raise BrokenConfigDriveDir("failed to read file %s: %s" % (item, exc))
+
+    # to openstack, user can specify meta ('nova boot --meta=key=value') and
+    # those will appear under metadata['meta'].
+    # if they specify 'dsmode' they're indicating the mode that they intend
+    # for this datasource to operate in.
+    try:
+        results['dsmode'] = results['metadata']['meta']['dsmode']
+    except KeyError:
+        pass
+
+    results['files'] = files
+    results['cfgdrive_ver'] = 2
+    return results
+
+
+def read_config_drive_dir_v1(source_dir):
     """
-    read_config_drive_dir(source_dir):
-       read source_dir, and return a tuple with metadata dict and user-data
-       string populated.  If not a valid dir, raise a NonConfigDriveDir
+    read source_dir, and return a tuple with metadata dict, user-data,
+    files and version (1).  If not a valid dir, raise a NonConfigDriveDir
     """
 
-    # TODO(harlowja): fix this for other operating systems...
-    # Ie: this is where https://fedorahosted.org/netcf/ or similar should
-    # be hooked in... (or could be)
     found = {}
-    for af in CFG_DRIVE_FILES:
+    for af in CFG_DRIVE_FILES_V1:
         fn = os.path.join(source_dir, af)
         if os.path.isfile(fn):
             found[af] = fn
@@ -173,11 +302,10 @@ def read_config_drive_dir(source_dir):
         raise NonConfigDriveDir("%s: %s" % (source_dir, "no files found"))
 
     md = {}
-    ud = ""
     keydata = ""
     if "etc/network/interfaces" in found:
         fn = found["etc/network/interfaces"]
-        md['network-interfaces'] = util.load_file(fn)
+        md['network_config'] = util.load_file(fn)
 
     if "root/.ssh/authorized_keys" in found:
         fn = found["root/.ssh/authorized_keys"]
@@ -197,21 +325,77 @@ def read_config_drive_dir(source_dir):
                 (source_dir, "invalid json in meta.js", e))
         md['meta_js'] = content
 
-    # Key data override??
+    # keydata in meta_js is preferred over "injected"
     keydata = meta_js.get('public-keys', keydata)
     if keydata:
         lines = keydata.splitlines()
         md['public-keys'] = [l for l in lines
             if len(l) and not l.startswith("#")]
 
-    for copy in ('dsmode', 'instance-id', 'dscfg'):
-        if copy in meta_js:
-            md[copy] = meta_js[copy]
+    # config-drive-v1 has no way for openstack to provide the instance-id
+    # so we copy that into metadata from the user input
+    if 'instance-id' in meta_js:
+        md['instance-id'] = meta_js['instance-id']
+
+    results = {'cfgdrive_ver': 1, 'metadata': md}
+
+    # allow the user to specify 'dsmode' in a meta tag
+    if 'dsmode' in meta_js:
+        results['dsmode'] = meta_js['dsmode']
+
+    # config-drive-v1 has no way of specifying user-data, so the user has
+    # to cheat and stuff it in a meta tag also.
+    results['userdata'] = meta_js.get('user-data')
 
-    if 'user-data' in meta_js:
-        ud = meta_js['user-data']
+    # this implementation does not support files
+    # (other than network/interfaces and authorized_keys)
+    results['files'] = []
 
-    return (md, ud)
+    return results
+
+
+def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None):
+    """Determine what mode should be used.
+    valid values are 'pass', 'disabled', 'local', 'net'
+    """
+    # user passed data trumps everything
+    if user is not None:
+        return user
+
+    if ds_cfg is not None:
+        return ds_cfg
+
+    # at config-drive version 1, the default behavior was pass.  That
+    # meant to not use use it as primary data source, but expect a ec2 metadata
+    # source. for version 2, we default to 'net', which means
+    # the DataSourceConfigDriveNet, would be used.
+    #
+    # this could change in the future.  If there was definitive metadata
+    # that indicated presense of an openstack metadata service, then
+    # we could change to 'pass' by default also. The motivation for that
+    # would be 'cloud-init query' as the web service could be more dynamic
+    if cfgdrv_ver == 1:
+        return "pass"
+    return "net"
+
+
+def get_previous_iid(paths):
+    # interestingly, for this purpose the "previous" instance-id is the current
+    # instance-id.  cloud-init hasn't moved them over yet as this datasource
+    # hasn't declared itself found.
+    fname = os.path.join(paths.get_cpath('data'), 'instance-id')
+    try:
+        with open(fname) as fp:
+            return fp.read()
+    except IOError:
+        return None
+
+
+def write_files(files):
+    for (name, content) in files.iteritems():
+        if name[0] != os.sep:
+            name = os.sep + name
+        util.write_file(name, content, mode=0660)
 
 
 # Used to match classes to dependencies
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
new file mode 100644
index 00000000..55573114
--- /dev/null
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -0,0 +1,177 @@
+from copy import copy
+import json
+import os
+import os.path
+import shutil
+import tempfile
+from unittest import TestCase
+
+from cloudinit.sources import DataSourceConfigDrive as ds
+from cloudinit import util
+
+
+PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n'
+EC2_META = {
+    'ami-id': 'ami-00000001',
+    'ami-launch-index': 0,
+    'ami-manifest-path': 'FIXME',
+    'block-device-mapping': {
+        'ami': 'sda1',
+        'ephemeral0': 'sda2',
+        'root': '/dev/sda1',
+        'swap': 'sda3'},
+    'hostname': 'sm-foo-test.novalocal',
+    'instance-action': 'none',
+    'instance-id': 'i-00000001',
+    'instance-type': 'm1.tiny',
+    'local-hostname': 'sm-foo-test.novalocal',
+    'local-ipv4': None,
+    'placement': {'availability-zone': 'nova'},
+    'public-hostname': 'sm-foo-test.novalocal',
+    'public-ipv4': '',
+    'public-keys': {'0': {'openssh-key': PUBKEY}},
+    'reservation-id': 'r-iru5qm4m',
+    'security-groups': ['default']
+}
+USER_DATA = '#!/bin/sh\necho This is user data\n'
+OSTACK_META = {
+    'availability_zone': 'nova',
+    'files': [{'content_path': '/content/0000', 'path': '/etc/foo.cfg'},
+              {'content_path': '/content/0001', 'path': '/etc/bar/bar.cfg'}],
+    'hostname': 'sm-foo-test.novalocal',
+    'meta': {'dsmode': 'local', 'my-meta': 'my-value'},
+    'name': 'sm-foo-test',
+    'public_keys': {'mykey': PUBKEY},
+    'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c'}
+
+CONTENT_0 = 'This is contents of /etc/foo.cfg\n'
+CONTENT_1 = '# this is /etc/bar/bar.cfg\n'
+
+CFG_DRIVE_FILES_V2 = {
+  'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META),
+  'ec2/2009-04-04/user-data': USER_DATA,
+  'ec2/latest/meta-data.json': json.dumps(EC2_META),
+  'ec2/latest/user-data': USER_DATA,
+  'openstack/2012-08-10/meta_data.json': json.dumps(OSTACK_META),
+  'openstack/2012-08-10/user_data': USER_DATA,
+  'openstack/content/0000': CONTENT_0,
+  'openstack/content/0001': CONTENT_1,
+  'openstack/latest/meta_data.json': json.dumps(OSTACK_META),
+  'openstack/latest/user_data': USER_DATA}
+
+
+class TestConfigDriveDataSource(TestCase):
+
+    def setUp(self):
+        super(TestConfigDriveDataSource, self).setUp()
+        self.tmp = tempfile.mkdtemp()
+
+    def tearDown(self):
+        try:
+            shutil.rmtree(self.tmp)
+        except OSError:
+            pass
+
+    def test_dir_valid(self):
+        """Verify a dir is read as such."""
+
+        populate_dir(self.tmp, CFG_DRIVE_FILES_V2)
+
+        found = ds.read_config_drive_dir(self.tmp)
+
+        expected_md = copy(OSTACK_META)
+        expected_md['instance-id'] = expected_md['uuid']
+
+        self.assertEqual(USER_DATA, found['userdata'])
+        self.assertEqual(expected_md, found['metadata'])
+        self.assertEqual(found['files']['/etc/foo.cfg'], CONTENT_0)
+        self.assertEqual(found['files']['/etc/bar/bar.cfg'], CONTENT_1)
+
+    def test_seed_dir_valid_extra(self):
+        """Verify extra files do not affect datasource validity."""
+
+        data = copy(CFG_DRIVE_FILES_V2)
+        data["myfoofile.txt"] = "myfoocontent"
+        data["openstack/latest/random-file.txt"] = "random-content"
+
+        populate_dir(self.tmp, data)
+
+        found = ds.read_config_drive_dir(self.tmp)
+
+        expected_md = copy(OSTACK_META)
+        expected_md['instance-id'] = expected_md['uuid']
+
+        self.assertEqual(expected_md, found['metadata'])
+
+    def test_seed_dir_bad_json_metadata(self):
+        """Verify that bad json in metadata raises BrokenConfigDriveDir."""
+        data = copy(CFG_DRIVE_FILES_V2)
+
+        data["openstack/2012-08-10/meta_data.json"] = "non-json garbage {}"
+        data["openstack/latest/meta_data.json"] = "non-json garbage {}"
+
+        populate_dir(self.tmp, data)
+
+        self.assertRaises(ds.BrokenConfigDriveDir,
+                          ds.read_config_drive_dir, self.tmp)
+
+    def test_seed_dir_no_configdrive(self):
+        """Verify that no metadata raises NonConfigDriveDir."""
+
+        my_d = os.path.join(self.tmp, "non-configdrive")
+        data = copy(CFG_DRIVE_FILES_V2)
+        data["myfoofile.txt"] = "myfoocontent"
+        data["openstack/latest/random-file.txt"] = "random-content"
+        data["content/foo"] = "foocontent"
+
+        self.assertRaises(ds.NonConfigDriveDir,
+                          ds.read_config_drive_dir, my_d)
+
+    def test_seed_dir_missing(self):
+        """Verify that missing seed_dir raises NonConfigDriveDir."""
+        my_d = os.path.join(self.tmp, "nonexistantdirectory")
+        self.assertRaises(ds.NonConfigDriveDir,
+                          ds.read_config_drive_dir, my_d)
+
+    def test_find_candidates(self):
+        devs_with_answers = {
+            "TYPE=vfat": [],
+            "TYPE=iso9660": ["/dev/vdb"],
+            "LABEL=config-2": ["/dev/vdb"],
+        }
+
+        def my_devs_with(criteria):
+            return devs_with_answers[criteria]
+
+        try:
+            orig_find_devs_with = util.find_devs_with
+            util.find_devs_with = my_devs_with
+
+            self.assertEqual(["/dev/vdb"], ds.find_candidate_devs())
+
+            # add a vfat item
+            # zdd reverse sorts after vdb, but config-2 label is preferred
+            devs_with_answers['TYPE=vfat'] = ["/dev/zdd"]
+            self.assertEqual(["/dev/vdb", "/dev/zdd"],
+                             ds.find_candidate_devs())
+
+            # verify that partitions are not considered
+            devs_with_answers = {"TYPE=vfat": ["/dev/sda1"],
+                "TYPE=iso9660": [], "LABEL=config-2": ["/dev/vdb3"]}
+            self.assertEqual([], ds.find_candidate_devs())
+
+        finally:
+            util.find_devs_with = orig_find_devs_with
+
+
+def populate_dir(seed_dir, files):
+    for (name, content) in files.iteritems():
+        path = os.path.join(seed_dir, name)
+        dirname = os.path.dirname(path)
+        if not os.path.isdir(dirname):
+            os.makedirs(dirname)
+        with open(path, "w") as fp:
+            fp.write(content)
+            fp.close()
+
+# vi: ts=4 expandtab