diff options
| -rw-r--r-- | cloudinit/sources/DataSourceConfigDrive.py | 232 | ||||
| -rw-r--r-- | tests/unittests/test_datasource/test_configdrive.py | 74 |
2 files changed, 212 insertions, 94 deletions
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 1b10bc1e..b68a97bb 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -37,16 +37,14 @@ CFG_DRIVE_FILES_V1 = [ ] DEFAULT_METADATA = { "instance-id": DEFAULT_IID, - "dsmode": DEFAULT_MODE, } -CFG_DRIVE_DEV_ENV = 'CLOUD_INIT_CONFIG_DRIVE_DEVICE' +VALID_DSMODES = ("local", "net", "pass", "disabled") class DataSourceConfigDrive(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.seed = None - self.cfg = {} + self.source = None self.dsmode = 'local' self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') self.version = None @@ -54,7 +52,7 @@ class DataSourceConfigDrive(sources.DataSource): def __str__(self): mstr = "%s [%s,ver=%s]" % (util.obj_name(self), self.dsmode, self.version) - mstr += "[seed=%s]" % (self.seed) + mstr += "[source=%s]" % (self.source) return mstr def get_data(self): @@ -70,18 +68,7 @@ class DataSourceConfigDrive(sources.DataSource): util.logexc(LOG, "Failed reading config drive from %s", self.seed_dir) if not found: - fslist = util.find_devs_with("TYPE=vfat") - fslist.extend(util.find_devs_with("TYPE=iso9660")) - - label_list = util.find_devs_with("LABEL=config-2") - devlist = list(set(fslist) & set(label_list)) - - dev = find_cfg_drive_device() - if dev not in devlist: - devlist.append(dev) - - devlist.sort(reverse=True) - + devlist = find_candidate_devs() for dev in devlist: try: results = util.mount_cb(dev, read_config_drive_dir) @@ -89,43 +76,73 @@ class DataSourceConfigDrive(sources.DataSource): break except (NonConfigDriveDir, util.MountFailedError): pass + except BrokenConfigDriveDir: + util.logexc(LOG, "broken config drive: %s", dev) if not found: return False - if 'dsconfig' in md: - self.cfg = md['dscfg'] - + md = results['metadata'] md = util.mergedict(md, DEFAULT_METADATA) - # Update interfaces and ifup only on the local datasource - # this way the DataSourceConfigDriveNet doesn't do it also. - if 'network-interfaces' in md and self.dsmode == "local": + user_dsmode = results.get('dsmode', None) + if user_dsmode not in VALID_DSMODES + (None,): + LOG.warn("user specified invalid mode: %s" % user_dsmode) + user_dsmode = None + + dsmode = get_ds_mode(cfgdrv_ver=results['cfgdrive_ver'], + ds_cfg=self.ds_cfg.get('dsmode'), + user=user_dsmode) + + if dsmode == "disabled": + # most likely user specified + return False + + # TODO(smoser): fix this, its dirty. + # we want to do some things (writing files and network config) + # only on first boot, and even then, we want to do so in the + # local datasource (so they happen earlier) even if the configured + # dsmode is 'net' or 'pass'. To do this, we check the previous + # instance-id + prev_iid = get_previous_iid(self.paths) + cur_iid = md['instance-id'] + + if ('network_config' in results and self.dsmode == "local" and + prev_iid != cur_iid): LOG.debug("Updating network interfaces from config drive (%s)", - md['dsmode']) - self.distro.apply_network(md['network-interfaces']) + dsmode) + self.distro.apply_network(results['network_config']) - self.seed = found - self.metadata = results['metadata'] - self.userdata_raw = results.get('userdata') + # file writing occurs in local mode (to be as early as possible) + if self.dsmode == "local" and prev_iid != cur_iid and results['files']: + LOG.debug("writing injected files") + try: + write_files(results['files']) + except: + util.logexc(LOG, "Failed writing files") + + # dsmode != self.dsmode here if: + # * dsmode = "pass", pass means it should only copy files and then + # pass to another datasource + # * dsmode = "net" and self.dsmode = "local" + # so that user boothooks would be applied with network, the + # local datasource just gets out of the way, and lets the net claim + if dsmode != self.dsmode: + LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) + return False - if md['dsmode'] == self.dsmode: - return True + self.source = found + self.metadata = md + self.userdata_raw = results.get('userdata') + self.version = results['cfgdrive_ver'] - LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode']) - return False + return True def get_public_ssh_keys(self): if not 'public-keys' in self.metadata: return [] return self.metadata['public-keys'] - # The data sources' config_obj is a cloud-config formated - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return self.cfg - class DataSourceConfigDriveNet(DataSourceConfigDrive): def __init__(self, sys_cfg, distro, paths): @@ -141,34 +158,40 @@ class BrokenConfigDriveDir(Exception): pass -def find_cfg_drive_device(): - """Get the config drive device. Return a string like '/dev/vdb' - or None (if there is no non-root device attached). This does not - check the contents, only reports that if there *were* a config_drive - attached, it would be this device. - Note: per config_drive documentation, this is - "associated as the last available disk on the instance" - """ +def find_candidate_devs(): + """Return a list of devices that may contain the config drive. - # This seems to be for debugging?? - if CFG_DRIVE_DEV_ENV in os.environ: - return os.environ[CFG_DRIVE_DEV_ENV] + The returned list is sorted by search order where the first item has + should be searched first (highest priority) - # We are looking for a raw block device (sda, not sda1) with a vfat - # filesystem on it.... - letters = "abcdefghijklmnopqrstuvwxyz" - devs = util.find_devs_with("TYPE=vfat") + config drive v1: + Per documentation, this is "associated as the last available disk on the + instance", and should be VFAT. + Currently, we do not restrict search list to "last available disk" - # Filter out anything not ending in a letter (ignore partitions) - devs = [f for f in devs if f[-1] in letters] + config drive v2: + Disk should be: + * either vfat or iso9660 formated + * labeled with 'config-2' + """ + + by_fstype = (util.find_devs_with("TYPE=vfat") + + util.find_devs_with("TYPE=iso9660")) + by_label = util.find_devs_with("LABEL=config-2") - # Sort them in reverse so "last" device is first - devs.sort(reverse=True) + # give preference to "last available disk" (vdb over vda) + # note, this is not a perfect rendition of that. + by_fstype.sort(reverse=True) + by_label.sort(reverse=True) - if devs: - return devs[0] + # combine list of items by putting by-label items first + # followed by fstype items, but with dupes removed + combined = (by_label + [d for d in by_fstype if d not in by_label]) - return None + # We are looking for block device (sda, not sda1), ignore partitions + combined = [d for d in combined if d[-1] not in "0123456789"] + + return combined def read_config_drive_dir(source_dir): @@ -190,7 +213,7 @@ def read_config_drive_dir_v2(source_dir, version="latest"): ('ec2-metadata', "ec2/latest/metadata.json", False, json.loads), ) - results = {} + results = {'userdata': None} for (name, path, required, process) in datafiles: fpath = os.path.join(source_dir, path) data = None @@ -226,12 +249,24 @@ def read_config_drive_dir_v2(source_dir, version="latest"): for item in results['metadata'].get('files', {}): files[item['path']] = read_content_path(item) + # the 'network_config' item in metadata is a content pointer + # to the network config that should be applied. + # in folsom, it is just a '/etc/network/interfaces' file. item = results['metadata'].get("network_config", None) if item: results['network_config'] = read_content_path(item) except Exception as exc: raise BrokenConfigDriveDir("failed to read file %s: %s" % (item, exc)) + # to openstack, user can specify meta ('nova boot --meta=key=value') and + # those will appear under metadata['meta']. + # if they specify 'dsmode' they're indicating the mode that they intend + # for this datasource to operate in. + try: + results['dsmode'] = results['metadata']['meta']['dsmode'] + except KeyError: + pass + results['files'] = files results['cfgdrive_ver'] = 2 return results @@ -243,9 +278,6 @@ def read_config_drive_dir_v1(source_dir): files and version (1). If not a valid dir, raise a NonConfigDriveDir """ - # TODO(harlowja): fix this for other operating systems... - # Ie: this is where https://fedorahosted.org/netcf/ or similar should - # be hooked in... (or could be) found = {} for af in CFG_DRIVE_FILES_V1: fn = os.path.join(source_dir, af) @@ -256,11 +288,10 @@ def read_config_drive_dir_v1(source_dir): raise NonConfigDriveDir("%s: %s" % (source_dir, "no files found")) md = {} - ud = "" keydata = "" if "etc/network/interfaces" in found: fn = found["etc/network/interfaces"] - md['network-interfaces'] = util.load_file(fn) + md['network_config'] = util.load_file(fn) if "root/.ssh/authorized_keys" in found: fn = found["root/.ssh/authorized_keys"] @@ -280,22 +311,75 @@ def read_config_drive_dir_v1(source_dir): (source_dir, "invalid json in meta.js", e)) md['meta_js'] = content - # Key data override?? + # keydata in meta_js is preferred over "injected" keydata = meta_js.get('public-keys', keydata) if keydata: lines = keydata.splitlines() md['public-keys'] = [l for l in lines if len(l) and not l.startswith("#")] - for copy in ('dsmode', 'instance-id', 'dscfg'): - if copy in meta_js: - md[copy] = meta_js[copy] + # config-drive-v1 has no way for openstack to provide the instance-id + # so we copy that into metadata from the user input + if 'instance-id' in meta_js: + md['instance-id'] = meta_js['instance-id'] + + results = {'cfgdrive_ver': 1, 'metadata': md} + + # allow the user to specify 'dsmode' in a meta tag + if 'dsmode' in meta_js: + results['dsmode'] = meta_js['dsmode'] + + # config-drive-v1 has no way of specifying user-data, so the user has + # to cheat and stuff it in a meta tag also. + results['userdata'] = meta_js.get('user-data') + + # this implementation does not support files + # (other than network/interfaces and authorized_keys) + results['files'] = [] + + return results + + +def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): + """Determine what mode should be used. + valid values are 'pass', 'disabled', 'local', 'net' + """ + # user passed data trumps everything + if user is not None: + return user + + if ds_cfg is not None: + return ds_cfg + + # at config-drive version 1, the default behavior was pass. That + # meant to not use use it as primary data source, but expect a ec2 metadata + # source. for version 2, we default to 'net', which means + # the DataSourceConfigDriveNet, would be used. + # + # this could change in the future. If there was definitive metadata + # that indicated presense of an openstack metadata service, then + # we could change to 'pass' by default also. The motivation for that + # would be 'cloud-init query' as the web service could be more dynamic + if cfgdrv_ver == 1: + return "pass" + return "net" + + +def get_previous_iid(paths): + fname = os.path.join(paths.get_cpath('data'), + 'previous-instance-id') + try: + with open(fname) as fp: + return fp.read() + except IOError: + return None - if 'user-data' in meta_js: - ud = meta_js['user-data'] - # metadata, user-data, 'files', 1 - return {'metadata': md, 'userdata': ud, 'files': [], 'cfgdrive_ver': 1} +def write_files(files): + for (name, content) in files.iteritems(): + if name[0] != os.sep: + name = os.sep + name + util.write_file(name, content, mode=0660) # Used to match classes to dependencies diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index c5d678f9..50b97ef8 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -2,9 +2,12 @@ from copy import copy import json import os import os.path +import shutil +import tempfile +from unittest import TestCase -from cloudinit.sources import DataSourceConfigDrive -from mocker import MockerTestCase +from cloudinit.sources import DataSourceConfigDrive as ds +from cloudinit import util PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' @@ -57,20 +60,24 @@ CFG_DRIVE_FILES_V2 = { 'openstack/latest/user_data': USER_DATA} -class TestConfigDriveDataSource(MockerTestCase): +class TestConfigDriveDataSource(TestCase): def setUp(self): super(TestConfigDriveDataSource, self).setUp() - # Make a temp directoy for tests to use. - self.tmp = self.makeDir() + self.tmp = tempfile.mkdtemp() + + def tearDown(self): + try: + shutil.rmtree(self.tmp) + except OSError: + pass def test_dir_valid(self): """Verify a dir is read as such.""" - my_d = os.path.join(self.tmp, "valid") - populate_dir(my_d, CFG_DRIVE_FILES_V2) + populate_dir(self.tmp, CFG_DRIVE_FILES_V2) - found = DataSourceConfigDrive.read_config_drive_dir(my_d) + found = ds.read_config_drive_dir(self.tmp) self.assertEqual(USER_DATA, found['userdata']) self.assertEqual(OSTACK_META, found['metadata']) @@ -80,28 +87,26 @@ class TestConfigDriveDataSource(MockerTestCase): def test_seed_dir_valid_extra(self): """Verify extra files do not affect datasource validity.""" - my_d = os.path.join(self.tmp, "valid_extra") data = copy(CFG_DRIVE_FILES_V2) data["myfoofile.txt"] = "myfoocontent" data["openstack/latest/random-file.txt"] = "random-content" - populate_dir(my_d, data) + populate_dir(self.tmp, data) - found = DataSourceConfigDrive.read_config_drive_dir(my_d) + found = ds.read_config_drive_dir(self.tmp) self.assertEqual(OSTACK_META, found['metadata']) def test_seed_dir_bad_json_metadata(self): """Verify that bad json in metadata raises BrokenConfigDriveDir.""" - my_d = os.path.join(self.tmp, "bad-json-metadata") data = copy(CFG_DRIVE_FILES_V2) data["openstack/2012-08-10/meta_data.json"] = "non-json garbage {}" data["openstack/latest/meta_data.json"] = "non-json garbage {}" - populate_dir(my_d, data) + populate_dir(self.tmp, data) - self.assertRaises(DataSourceConfigDrive.BrokenConfigDriveDir, - DataSourceConfigDrive.read_config_drive_dir, my_d) + self.assertRaises(ds.BrokenConfigDriveDir, + ds.read_config_drive_dir, self.tmp) def test_seed_dir_no_configdrive(self): """Verify that no metadata raises NonConfigDriveDir.""" @@ -112,18 +117,47 @@ class TestConfigDriveDataSource(MockerTestCase): data["openstack/latest/random-file.txt"] = "random-content" data["content/foo"] = "foocontent" - self.assertRaises(DataSourceConfigDrive.NonConfigDriveDir, - DataSourceConfigDrive.read_config_drive_dir, my_d) + self.assertRaises(ds.NonConfigDriveDir, + ds.read_config_drive_dir, my_d) def test_seed_dir_missing(self): """Verify that missing seed_dir raises NonConfigDriveDir.""" my_d = os.path.join(self.tmp, "nonexistantdirectory") - self.assertRaises(DataSourceConfigDrive.NonConfigDriveDir, - DataSourceConfigDrive.read_config_drive_dir, my_d) + self.assertRaises(ds.NonConfigDriveDir, + ds.read_config_drive_dir, my_d) + + def test_find_candidates(self): + devs_with_answers = { + "TYPE=vfat": [], + "TYPE=iso9660": ["/dev/vdb"], + "LABEL=config-2": ["/dev/vdb"], + } + + def my_devs_with(criteria): + return devs_with_answers[criteria] + + try: + orig_find_devs_with = util.find_devs_with + util.find_devs_with = my_devs_with + + self.assertEqual(["/dev/vdb"], ds.find_candidate_devs()) + + # add a vfat item + # zdd reverse sorts after vdb, but config-2 label is preferred + devs_with_answers['TYPE=vfat'] = ["/dev/zdd"] + self.assertEqual(["/dev/vdb", "/dev/zdd"], + ds.find_candidate_devs()) + + # verify that partitions are not considered + devs_with_answers = {"TYPE=vfat": ["/dev/sda1"], + "TYPE=iso9660": [], "LABEL=config-2": ["/dev/vdb3"]} + self.assertEqual([], ds.find_candidate_devs()) + + finally: + util.find_devs_with = orig_find_devs_with def populate_dir(seed_dir, files): - os.mkdir(seed_dir) for (name, content) in files.iteritems(): path = os.path.join(seed_dir, name) dirname = os.path.dirname(path) |