From f3b25e68ac6d28cdacf7408c91da6e9a215ad1e6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 4 Feb 2013 22:23:42 -0500 Subject: make config of nocloud datasource able to specify meta-data and user-data LP: #1115833 --- cloudinit/sources/DataSourceNoCloud.py | 72 ++++++++++++++++++------------- doc/examples/cloud-config-datasources.txt | 11 +++++ 2 files changed, 52 insertions(+), 31 deletions(-) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index bed500a2..d8484437 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -77,37 +77,47 @@ class DataSourceNoCloud(sources.DataSource): found.append("ds_config") md["seedfrom"] = self.ds_cfg['seedfrom'] - fslist = util.find_devs_with("TYPE=vfat") - fslist.extend(util.find_devs_with("TYPE=iso9660")) - - label_list = util.find_devs_with("LABEL=cidata") - devlist = list(set(fslist) & set(label_list)) - devlist.sort(reverse=True) - - for dev in devlist: - try: - LOG.debug("Attempting to use data from %s", dev) - - (newmd, newud) = util.mount_cb(dev, util.read_seeded) - md = util.mergedict(newmd, md) - ud = newud - - # For seed from a device, the default mode is 'net'. - # that is more likely to be what is desired. - # If they want dsmode of local, then they must - # specify that. - if 'dsmode' not in md: - md['dsmode'] = "net" - - LOG.debug("Using data from %s", dev) - found.append(dev) - break - except OSError as e: - if e.errno != errno.ENOENT: - raise - except util.MountFailedError: - util.logexc(LOG, ("Failed to mount %s" - " when looking for data"), dev) + # if ds_cfg has 'user-data' and 'meta-data' + if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg: + if self.ds_cfg['user-data']: + ud = util.mergedict(md, self.ds_cfg['user-data']) + if self.ds_cfg['meta-data'] is not False: + md = util.mergedict(md, self.ds_cfg['meta-data']) + if 'ds_config' not in found: + found.append("ds_config") + + if self.ds_cfg.get('fs_label', "cidata"): + fslist = util.find_devs_with("TYPE=vfat") + fslist.extend(util.find_devs_with("TYPE=iso9660")) + + label = self.ds_cfg.get('fs_label') + label_list = util.find_devs_with("LABEL=%s" % label) + devlist = list(set(fslist) & set(label_list)) + devlist.sort(reverse=True) + + for dev in devlist: + try: + LOG.debug("Attempting to use data from %s", dev) + + (newmd, newud) = util.mount_cb(dev, util.read_seeded) + md = util.mergedict(newmd, md) + ud = newud + + # For seed from a device, the default mode is 'net'. + # that is more likely to be what is desired. If they want + # dsmode of local, then they must specify that. + if 'dsmode' not in md: + md['dsmode'] = "net" + + LOG.debug("Using data from %s", dev) + found.append(dev) + break + except OSError as e: + if e.errno != errno.ENOENT: + raise + except util.MountFailedError: + util.logexc(LOG, ("Failed to mount %s" + " when looking for data"), dev) # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index d10dde05..fc8c22d4 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -31,3 +31,14 @@ datasource: # /user-data and /meta-data # seedfrom: http://my.example.com/i-abcde seedfrom: None + + # fs_label: the label on filesystems to be searched for NoCloud source + fs_label: cidata + + # these are optional, but allow you to basically provide a datasource + # right here + user-data: | + # This is the user-data verbatum + meta-data: + instance-id: i-87018aed + local-hostname: myhost.internal -- cgit v1.2.3 From bf0db8d0793c9c18871cbbafbbad9c127b0bd8ee Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 7 Feb 2013 08:33:48 -0500 Subject: user-data doesn't merge in from meta-data (typo) --- cloudinit/sources/DataSourceNoCloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index d8484437..5ccd6b99 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -80,7 +80,7 @@ class DataSourceNoCloud(sources.DataSource): # if ds_cfg has 'user-data' and 'meta-data' if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg: if self.ds_cfg['user-data']: - ud = util.mergedict(md, self.ds_cfg['user-data']) + ud = self.ds_cfg['user-data'] if self.ds_cfg['meta-data'] is not False: md = util.mergedict(md, self.ds_cfg['meta-data']) if 'ds_config' not in found: -- cgit v1.2.3 From 88a369c5324e74a2d1bb8dd0bdf8fdc9a95393c8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 7 Feb 2013 09:12:36 -0500 Subject: add test_nocloud unit tests, fix one issue found --- cloudinit/sources/DataSourceNoCloud.py | 2 + tests/unittests/test_datasource/test_nocloud.py | 56 +++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 tests/unittests/test_datasource/test_nocloud.py diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 5ccd6b99..097bbc52 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -205,6 +205,8 @@ def parse_cmdline_data(ds_id, fill, cmdline=None): # short2long mapping to save cmdline typing s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"} for item in kvpairs: + if item == "": + continue try: (k, v) = item.split("=", 1) except: diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py new file mode 100644 index 00000000..850d3214 --- /dev/null +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -0,0 +1,56 @@ +from cloudinit.sources import DataSourceNoCloud + +from mocker import MockerTestCase + + +class TestNoCloudDataSource(MockerTestCase): + + def setUp(self): + super(TestNoCloudDataSource, self).setUp() + + def test_parse_cmdline_data_valid(self): + parse = DataSourceNoCloud.parse_cmdline_data + + ds_id = "ds=nocloud" + pairs = ( + ("root=/dev/sda1 %(ds_id)s", {}), + ("%(ds_id)s; root=/dev/foo", {}), + ("%(ds_id)s", {}), + ("%(ds_id)s;", {}), + ("%(ds_id)s;s=SEED", {'seedfrom': 'SEED'}), + ("%(ds_id)s;seedfrom=SEED;local-hostname=xhost", + {'seedfrom': 'SEED', 'local-hostname': 'xhost'}), + ("%(ds_id)s;h=xhost", + {'local-hostname': 'xhost'}), + ("%(ds_id)s;h=xhost;i=IID", + {'local-hostname': 'xhost', 'instance-id': 'IID'}), + ) + + for (fmt, expected) in pairs: + fill = {} + cmdline = fmt % {'ds_id': ds_id} + ret = parse(ds_id=ds_id, fill=fill, cmdline=cmdline) + self.assertEqual(expected, fill) + self.assertTrue(ret) + + def test_parse_cmdline_data_none(self): + parse = DataSourceNoCloud.parse_cmdline_data + + ds_id = "ds=foo" + cmdlines = ( + "root=/dev/sda1 ro", + "console=/dev/ttyS0 root=/dev/foo", + "", + "ds=foocloud", + "ds=foo-net", + "ds=nocloud;s=SEED", + ) + + for cmdline in cmdlines: + fill = {} + ret = parse(ds_id=ds_id, fill=fill, cmdline=cmdline) + self.assertEqual(fill, {}) + self.assertFalse(ret) + + +# vi: ts=4 expandtab -- cgit v1.2.3 From a0f882fdd306ef7b3b90c53d5622eb5a6878cb81 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 7 Feb 2013 12:08:30 -0500 Subject: more test cases for nocloud including one for config seed --- tests/unittests/helpers.py | 7 ++ tests/unittests/test_datasource/test_maas.py | 8 +- tests/unittests/test_datasource/test_nocloud.py | 113 ++++++++++++++++++++++-- 3 files changed, 115 insertions(+), 13 deletions(-) diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 4258a29d..91a50e18 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -182,3 +182,10 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): trap_func = retarget_many_wrapper(new_root, 1, func) setattr(mod, f, trap_func) self.patched_funcs.append((mod, f, func)) + +def populate_dir(path, files): + os.makedirs(path) + for (name, content) in files.iteritems(): + with open(os.path.join(path, name), "w") as fp: + fp.write(content) + fp.close() diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 85e6add0..b56fea82 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -3,6 +3,7 @@ import os from cloudinit.sources import DataSourceMAAS from cloudinit import url_helper +from tests.unittests.helpers import populate_dir from mocker import MockerTestCase @@ -137,11 +138,4 @@ class TestMAASDataSource(MockerTestCase): pass -def populate_dir(seed_dir, files): - os.mkdir(seed_dir) - for (name, content) in files.iteritems(): - with open(os.path.join(seed_dir, name), "w") as fp: - fp.write(content) - fp.close() - # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 850d3214..28e0a472 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -1,16 +1,106 @@ +from cloudinit import helpers +from tests.unittests.helpers import populate_dir from cloudinit.sources import DataSourceNoCloud +from cloudinit import util from mocker import MockerTestCase +import os +import yaml class TestNoCloudDataSource(MockerTestCase): def setUp(self): + self.tmp = self.makeDir() + self.paths = helpers.Paths({'cloud_dir': self.tmp}) + + self.cmdline = "root=TESTCMDLINE" + + self.unapply = [] + self.apply_patches([(util, 'get_cmdline', self._getcmdline)]) super(TestNoCloudDataSource, self).setUp() - def test_parse_cmdline_data_valid(self): - parse = DataSourceNoCloud.parse_cmdline_data + def tearDown(self): + apply_patches([i for i in reversed(self.unapply)]) + super(TestNoCloudDataSource, self).setUp() + + def apply_patches(self, patches): + ret = apply_patches(patches) + self.unapply += ret + + def _getcmdline(self): + return self.cmdline + + def test_nocloud_seed_dir(self): + md = {'instance-id': 'IID', 'dsmode': 'local'} + ud = "USER_DATA_HERE" + populate_dir(os.path.join(self.paths.seed_dir, "nocloud"), + {'user-data': ud, 'meta-data': yaml.safe_dump(md)}) + + sys_cfg = { + 'datasource': {'NoCloud': {'fs_label': None}} + } + + ds = DataSourceNoCloud.DataSourceNoCloud + + dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) + ret = dsrc.get_data() + self.assertEqual(dsrc.userdata_raw, ud) + self.assertEqual(dsrc.metadata, md) + self.assertTrue(ret) + + def test_fs_label(self): + #find_devs_with should not be called ff fs_label is None + ds = DataSourceNoCloud.DataSourceNoCloud + + class PsuedoException(Exception): + pass + + def my_find_devs_with(*args, **kwargs): + _f = (args, kwargs) + raise PsuedoException + + self.apply_patches([(util, 'find_devs_with', my_find_devs_with)]) + # by default, NoCloud should search for filesystems by label + sys_cfg = {'datasource': {'NoCloud': {}}} + dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) + self.assertRaises(PsuedoException, dsrc.get_data) + + # but disabling searching should just end up with None found + sys_cfg = {'datasource': {'NoCloud': {'fs_label': None}}} + dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) + ret = dsrc.get_data() + self.assertFalse(ret) + + def test_no_datasource_expected(self): + #no source should be found if no cmdline, config, and fs_label=None + sys_cfg = {'datasource': {'NoCloud': {'fs_label': None}}} + + ds = DataSourceNoCloud.DataSourceNoCloud + dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) + self.assertFalse(dsrc.get_data()) + + def test_seed_in_config(self): + ds = DataSourceNoCloud.DataSourceNoCloud + + data = { + 'fs_label': None, + 'meta-data': {'instance-id': 'IID'}, + 'user-data': "USER_DATA_RAW", + } + + sys_cfg = {'datasource': {'NoCloud': data}} + dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) + ret = dsrc.get_data() + self.assertEqual(dsrc.userdata_raw, "USER_DATA_RAW") + self.assertEqual(dsrc.metadata.get('instance-id'), 'IID') + self.assertTrue(ret) + + +class TestParseCommandLineData(MockerTestCase): + + def test_parse_cmdline_data_valid(self): ds_id = "ds=nocloud" pairs = ( ("root=/dev/sda1 %(ds_id)s", {}), @@ -29,13 +119,12 @@ class TestNoCloudDataSource(MockerTestCase): for (fmt, expected) in pairs: fill = {} cmdline = fmt % {'ds_id': ds_id} - ret = parse(ds_id=ds_id, fill=fill, cmdline=cmdline) + ret = DataSourceNoCloud.parse_cmdline_data(ds_id=ds_id, fill=fill, + cmdline=cmdline) self.assertEqual(expected, fill) self.assertTrue(ret) def test_parse_cmdline_data_none(self): - parse = DataSourceNoCloud.parse_cmdline_data - ds_id = "ds=foo" cmdlines = ( "root=/dev/sda1 ro", @@ -48,9 +137,21 @@ class TestNoCloudDataSource(MockerTestCase): for cmdline in cmdlines: fill = {} - ret = parse(ds_id=ds_id, fill=fill, cmdline=cmdline) + ret = DataSourceNoCloud.parse_cmdline_data(ds_id=ds_id, fill=fill, + cmdline=cmdline) self.assertEqual(fill, {}) self.assertFalse(ret) +def apply_patches(patches): + ret = [] + for (ref, name, replace) in patches: + if replace is None: + continue + orig = getattr(ref, name) + setattr(ref, name, replace) + ret.append((ref, name, orig)) + return ret + + # vi: ts=4 expandtab -- cgit v1.2.3