diff options
| -rw-r--r-- | ChangeLog | 2 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceNoCloud.py | 92 | ||||
| -rw-r--r-- | cloudinit/util.py | 31 | ||||
| -rw-r--r-- | tests/unittests/helpers.py | 3 | ||||
| -rw-r--r-- | tests/unittests/test_datasource/test_nocloud.py | 35 | ||||
| -rw-r--r-- | tests/unittests/test_pathprefix2dict.py | 40 |
6 files changed, 166 insertions, 37 deletions
@@ -13,7 +13,7 @@ redirect cloud-init stderr and stdout /var/log/cloud-init-output.log. - drop support for resizing partitions with parted entirely (LP: #1212492). This was broken as it was anyway. - - add support for vendordata. + - add support for vendordata in SmartOS and NoCloud datasources. - drop dependency on boto for crawling ec2 metadata service. - add 'Requires' on sudo (for OpenNebula datasource) in rpm specs, and 'Recommends' in the debian/control.in [Vlastimil Holer] diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 4ef92a56..cbaac29f 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -50,40 +50,47 @@ class DataSourceNoCloud(sources.DataSource): } found = [] - md = {} - ud = "" + mydata = {'meta-data': {}, 'user-data': "", 'vendor-data': ""} try: # Parse the kernel command line, getting data passed in + md = {} if parse_cmdline_data(self.cmdline_id, md): found.append("cmdline") + mydata.update(md) except: util.logexc(LOG, "Unable to parse command line data") return False # Check to see if the seed dir has data. - seedret = {} - if util.read_optional_seed(seedret, base=self.seed_dir + "/"): - md = util.mergemanydict([md, seedret['meta-data']]) - ud = seedret['user-data'] + pp2d_kwargs = {'required': ['user-data', 'meta-data'], + 'optional': ['vendor-data']} + + try: + seeded = util.pathprefix2dict(self.seed_dir, **pp2d_kwargs) found.append(self.seed_dir) - LOG.debug("Using seeded cache data from %s", self.seed_dir) + LOG.debug("Using seeded data from %s", self.seed_dir) + except ValueError as e: + pass + + if self.seed_dir in found: + mydata = _merge_new_seed(mydata, seeded) # If the datasource config had a 'seedfrom' entry, then that takes # precedence over a 'seedfrom' that was found in a filesystem # but not over external media - if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']: - found.append("ds_config") - md["seedfrom"] = self.ds_cfg['seedfrom'] + if self.ds_cfg.get('seedfrom'): + found.append("ds_config_seedfrom") + mydata['meta-data']["seedfrom"] = self.ds_cfg['seedfrom'] - # if ds_cfg has 'user-data' and 'meta-data' + # fields appropriately named can also just come from the datasource + # config (ie, 'user-data', 'meta-data', 'vendor-data' there) if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg: - if self.ds_cfg['user-data']: - ud = self.ds_cfg['user-data'] - if self.ds_cfg['meta-data'] is not False: - md = util.mergemanydict([md, self.ds_cfg['meta-data']]) - if 'ds_config' not in found: - found.append("ds_config") + mydata = _merge_new_seed(mydata, self.ds_cfg) + found.append("ds_config") + + def _pp2d_callback(mp, data): + util.pathprefix2dict(mp, **data) label = self.ds_cfg.get('fs_label', "cidata") if label is not None: @@ -102,15 +109,21 @@ class DataSourceNoCloud(sources.DataSource): try: LOG.debug("Attempting to use data from %s", dev) - (newmd, newud) = util.mount_cb(dev, util.read_seeded) - md = util.mergemanydict([newmd, md]) - ud = newud + try: + seeded = util.mount_cb(dev, _pp2d_callback) + except ValueError as e: + if dev in label_list: + LOG.warn("device %s with label=%s not a" + "valid seed.", dev, label) + continue + + mydata = _merge_new_seed(mydata, seeded) # For seed from a device, the default mode is 'net'. # that is more likely to be what is desired. If they want # dsmode of local, then they must specify that. - if 'dsmode' not in md: - md['dsmode'] = "net" + if 'dsmode' not in mydata['meta-data']: + mydata['meta-data'] = "net" LOG.debug("Using data from %s", dev) found.append(dev) @@ -133,8 +146,8 @@ class DataSourceNoCloud(sources.DataSource): # attempt to seed the userdata / metadata from its value # its primarily value is in allowing the user to type less # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg - if "seedfrom" in md: - seedfrom = md["seedfrom"] + if "seedfrom" in mydata['meta-data']: + seedfrom = mydata['meta-data']["seedfrom"] seedfound = False for proto in self.supported_seed_starts: if seedfrom.startswith(proto): @@ -144,7 +157,7 @@ class DataSourceNoCloud(sources.DataSource): LOG.debug("Seed from %s not supported by %s", seedfrom, self) return False - if 'network-interfaces' in md: + if 'network-interfaces' in mydata['meta-data']: seeded_interfaces = self.dsmode # This could throw errors, but the user told us to do it @@ -153,25 +166,30 @@ class DataSourceNoCloud(sources.DataSource): LOG.debug("Using seeded cache data from %s", seedfrom) # Values in the command line override those from the seed - md = util.mergemanydict([md, md_seed]) + mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], + md_seed]) + mydata['user-data'] = ud found.append(seedfrom) # Now that we have exhausted any other places merge in the defaults - md = util.mergemanydict([md, defaults]) + mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], + defaults]) # Update the network-interfaces if metadata had 'network-interfaces' # entry and this is the local datasource, or 'seedfrom' was used # and the source of the seed was self.dsmode # ('local' for NoCloud, 'net' for NoCloudNet') - if ('network-interfaces' in md and + if ('network-interfaces' in mydata['meta-data'] and (self.dsmode in ("local", seeded_interfaces))): LOG.debug("Updating network interfaces from %s", self) - self.distro.apply_network(md['network-interfaces']) + self.distro.apply_network( + mydata['meta-data']['network-interfaces']) - if md['dsmode'] == self.dsmode: + if mydata['meta-data']['dsmode'] == self.dsmode: self.seed = ",".join(found) - self.metadata = md - self.userdata_raw = ud + self.metadata = mydata['meta-data'] + self.userdata_raw = mydata['user-data'] + self.vendordata = mydata['vendor-data'] return True LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode']) @@ -222,6 +240,16 @@ def parse_cmdline_data(ds_id, fill, cmdline=None): return True +def _merge_new_seed(cur, seeded): + ret = cur.copy() + ret['meta-data'] = util.mergemanydict([cur['meta-data'], + util.load_yaml(seeded['meta-data'])]) + ret['user-data'] = seeded['user-data'] + if 'vendor-data' in seeded: + ret['vendor-data'] = seeded['vendor-data'] + return ret + + class DataSourceNoCloudNet(DataSourceNoCloud): def __init__(self, sys_cfg, distro, paths): DataSourceNoCloud.__init__(self, sys_cfg, distro, paths) diff --git a/cloudinit/util.py b/cloudinit/util.py index 61bcdeb1..08cdd8c8 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -369,11 +369,11 @@ def is_ipv4(instr): return False try: - toks = [x for x in toks if (int(x) < 256 and int(x) >= 0)] + toks = [x for x in toks if int(x) < 256 and int(x) >= 0] except: return False - return (len(toks) == 4) + return len(toks) == 4 def get_cfg_option_bool(yobj, key, default=False): @@ -972,7 +972,7 @@ def gethostbyaddr(ip): def is_resolvable_url(url): """determine if this url is resolvable (existing or ip).""" - return (is_resolvable(urlparse.urlparse(url).hostname)) + return is_resolvable(urlparse.urlparse(url).hostname) def search_for_mirror(candidates): @@ -1889,3 +1889,28 @@ def expand_dotted_devname(dotted): return toks else: return (dotted, None) + + +def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep): + # return a dictionary populated with keys in 'required' and 'optional' + # by reading files in prefix + delim + entry + if required is None: + required = [] + if optional is None: + optional = [] + + missing = [] + ret = {} + for f in required + optional: + try: + ret[f] = load_file(base + delim + f, quiet=False) + except IOError as e: + if e.errno != errno.ENOENT: + raise + if f in required: + missing.append(f) + + if len(missing): + raise ValueError("Missing required files: %s", ','.join(missing)) + + return ret diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index c0da0983..5b4f4208 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -187,7 +187,8 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): def populate_dir(path, files): - os.makedirs(path) + if not os.path.exists(path): + os.makedirs(path) for (name, content) in files.iteritems(): with open(os.path.join(path, name), "w") as fp: fp.write(content) diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 7328b240..af575a10 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -97,6 +97,41 @@ class TestNoCloudDataSource(MockerTestCase): self.assertEqual(dsrc.metadata.get('instance-id'), 'IID') self.assertTrue(ret) + def test_nocloud_seed_with_vendordata(self): + md = {'instance-id': 'IID', 'dsmode': 'local'} + ud = "USER_DATA_HERE" + vd = "THIS IS MY VENDOR_DATA" + + populate_dir(os.path.join(self.paths.seed_dir, "nocloud"), + {'user-data': ud, 'meta-data': yaml.safe_dump(md), + 'vendor-data': vd}) + + sys_cfg = { + 'datasource': {'NoCloud': {'fs_label': None}} + } + + ds = DataSourceNoCloud.DataSourceNoCloud + + dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) + ret = dsrc.get_data() + self.assertEqual(dsrc.userdata_raw, ud) + self.assertEqual(dsrc.metadata, md) + self.assertEqual(dsrc.vendordata, vd) + self.assertTrue(ret) + + def test_nocloud_no_vendordata(self): + populate_dir(os.path.join(self.paths.seed_dir, "nocloud"), + {'user-data': "ud", 'meta-data': "instance-id: IID\n"}) + + sys_cfg = {'datasource': {'NoCloud': {'fs_label': None}}} + + ds = DataSourceNoCloud.DataSourceNoCloud + + dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths) + ret = dsrc.get_data() + self.assertEqual(dsrc.userdata_raw, "ud") + self.assertFalse(dsrc.vendordata) + self.assertTrue(ret) class TestParseCommandLineData(MockerTestCase): diff --git a/tests/unittests/test_pathprefix2dict.py b/tests/unittests/test_pathprefix2dict.py new file mode 100644 index 00000000..c68c263c --- /dev/null +++ b/tests/unittests/test_pathprefix2dict.py @@ -0,0 +1,40 @@ +from cloudinit import util + +from mocker import MockerTestCase +from tests.unittests.helpers import populate_dir + + +class TestPathPrefix2Dict(MockerTestCase): + + def setUp(self): + self.tmp = self.makeDir() + + def test_required_only(self): + dirdata = {'f1': 'f1content', 'f2': 'f2content'} + populate_dir(self.tmp, dirdata) + + ret = util.pathprefix2dict(self.tmp, required=['f1', 'f2']) + self.assertEqual(dirdata, ret) + + def test_required_missing(self): + dirdata = {'f1': 'f1content'} + populate_dir(self.tmp, dirdata) + kwargs = {'required': ['f1', 'f2']} + self.assertRaises(ValueError, util.pathprefix2dict, self.tmp, **kwargs) + + def test_no_required_and_optional(self): + dirdata = {'f1': 'f1c', 'f2': 'f2c'} + populate_dir(self.tmp, dirdata) + + ret = util.pathprefix2dict(self.tmp, required=None, + optional=['f1', 'f2']) + self.assertEqual(dirdata, ret) + + def test_required_and_optional(self): + dirdata = {'f1': 'f1c', 'f2': 'f2c'} + populate_dir(self.tmp, dirdata) + + ret = util.pathprefix2dict(self.tmp, required=['f1'], optional=['f2']) + self.assertEqual(dirdata, ret) + +# vi: ts=4 expandtab |