Remerged with trunk

4 files changed, 162 insertions, 42 deletions

diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index c45a1119..1d30fe08 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -246,10 +246,13 @@ def find_candidate_devs(probe_optical=True):
 
     # combine list of items by putting by-label items first
     # followed by fstype items, but with dupes removed
-    combined = (by_label + [d for d in by_fstype if d not in by_label])
+    candidates = (by_label + [d for d in by_fstype if d not in by_label])
 
-    # we are looking for block device (sda, not sda1), ignore partitions
-    return [d for d in combined if not util.is_partition(d)]
+    # We are looking for a block device or partition with necessary label or
+    # an unpartitioned block device (ex sda, not sda1)
+    devices = [d for d in candidates
+               if d in by_label or not util.is_partition(d)]
+    return devices
 
 
 # Used to match classes to dependencies
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 4ef92a56..cbaac29f 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -50,40 +50,47 @@ class DataSourceNoCloud(sources.DataSource):
         }
 
         found = []
-        md = {}
-        ud = ""
+        mydata = {'meta-data': {}, 'user-data': "", 'vendor-data': ""}
 
         try:
             # Parse the kernel command line, getting data passed in
+            md = {}
             if parse_cmdline_data(self.cmdline_id, md):
                 found.append("cmdline")
+            mydata.update(md)
         except:
             util.logexc(LOG, "Unable to parse command line data")
             return False
 
         # Check to see if the seed dir has data.
-        seedret = {}
-        if util.read_optional_seed(seedret, base=self.seed_dir + "/"):
-            md = util.mergemanydict([md, seedret['meta-data']])
-            ud = seedret['user-data']
+        pp2d_kwargs = {'required': ['user-data', 'meta-data'],
+                       'optional': ['vendor-data']}
+
+        try:
+            seeded = util.pathprefix2dict(self.seed_dir, **pp2d_kwargs)
             found.append(self.seed_dir)
-            LOG.debug("Using seeded cache data from %s", self.seed_dir)
+            LOG.debug("Using seeded data from %s", self.seed_dir)
+        except ValueError as e:
+            pass
+
+        if self.seed_dir in found:
+            mydata = _merge_new_seed(mydata, seeded)
 
         # If the datasource config had a 'seedfrom' entry, then that takes
         # precedence over a 'seedfrom' that was found in a filesystem
         # but not over external media
-        if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']:
-            found.append("ds_config")
-            md["seedfrom"] = self.ds_cfg['seedfrom']
+        if self.ds_cfg.get('seedfrom'):
+            found.append("ds_config_seedfrom")
+            mydata['meta-data']["seedfrom"] = self.ds_cfg['seedfrom']
 
-        # if ds_cfg has 'user-data' and 'meta-data'
+        # fields appropriately named can also just come from the datasource
+        # config (ie, 'user-data', 'meta-data', 'vendor-data' there)
         if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg:
-            if self.ds_cfg['user-data']:
-                ud = self.ds_cfg['user-data']
-            if self.ds_cfg['meta-data'] is not False:
-                md = util.mergemanydict([md, self.ds_cfg['meta-data']])
-            if 'ds_config' not in found:
-                found.append("ds_config")
+            mydata = _merge_new_seed(mydata, self.ds_cfg)
+            found.append("ds_config")
+
+        def _pp2d_callback(mp, data):
+            util.pathprefix2dict(mp, **data)
 
         label = self.ds_cfg.get('fs_label', "cidata")
         if label is not None:
@@ -102,15 +109,21 @@ class DataSourceNoCloud(sources.DataSource):
                 try:
                     LOG.debug("Attempting to use data from %s", dev)
 
-                    (newmd, newud) = util.mount_cb(dev, util.read_seeded)
-                    md = util.mergemanydict([newmd, md])
-                    ud = newud
+                    try:
+                        seeded = util.mount_cb(dev, _pp2d_callback)
+                    except ValueError as e:
+                        if dev in label_list:
+                            LOG.warn("device %s with label=%s not a"
+                                     "valid seed.", dev, label)
+                        continue
+
+                    mydata = _merge_new_seed(mydata, seeded)
 
                     # For seed from a device, the default mode is 'net'.
                     # that is more likely to be what is desired.  If they want
                     # dsmode of local, then they must specify that.
-                    if 'dsmode' not in md:
-                        md['dsmode'] = "net"
+                    if 'dsmode' not in mydata['meta-data']:
+                        mydata['meta-data'] = "net"
 
                     LOG.debug("Using data from %s", dev)
                     found.append(dev)
@@ -133,8 +146,8 @@ class DataSourceNoCloud(sources.DataSource):
         # attempt to seed the userdata / metadata from its value
         # its primarily value is in allowing the user to type less
         # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg
-        if "seedfrom" in md:
-            seedfrom = md["seedfrom"]
+        if "seedfrom" in mydata['meta-data']:
+            seedfrom = mydata['meta-data']["seedfrom"]
             seedfound = False
             for proto in self.supported_seed_starts:
                 if seedfrom.startswith(proto):
@@ -144,7 +157,7 @@ class DataSourceNoCloud(sources.DataSource):
                 LOG.debug("Seed from %s not supported by %s", seedfrom, self)
                 return False
 
-            if 'network-interfaces' in md:
+            if 'network-interfaces' in mydata['meta-data']:
                 seeded_interfaces = self.dsmode
 
             # This could throw errors, but the user told us to do it
@@ -153,25 +166,30 @@ class DataSourceNoCloud(sources.DataSource):
             LOG.debug("Using seeded cache data from %s", seedfrom)
 
             # Values in the command line override those from the seed
-            md = util.mergemanydict([md, md_seed])
+            mydata['meta-data'] = util.mergemanydict([mydata['meta-data'],
+                                                      md_seed])
+            mydata['user-data'] = ud
             found.append(seedfrom)
 
         # Now that we have exhausted any other places merge in the defaults
-        md = util.mergemanydict([md, defaults])
+        mydata['meta-data'] = util.mergemanydict([mydata['meta-data'],
+                                                  defaults])
 
         # Update the network-interfaces if metadata had 'network-interfaces'
         # entry and this is the local datasource, or 'seedfrom' was used
         # and the source of the seed was self.dsmode
         # ('local' for NoCloud, 'net' for NoCloudNet')
-        if ('network-interfaces' in md and
+        if ('network-interfaces' in mydata['meta-data'] and
             (self.dsmode in ("local", seeded_interfaces))):
             LOG.debug("Updating network interfaces from %s", self)
-            self.distro.apply_network(md['network-interfaces'])
+            self.distro.apply_network(
+                mydata['meta-data']['network-interfaces'])
 
-        if md['dsmode'] == self.dsmode:
+        if mydata['meta-data']['dsmode'] == self.dsmode:
             self.seed = ",".join(found)
-            self.metadata = md
-            self.userdata_raw = ud
+            self.metadata = mydata['meta-data']
+            self.userdata_raw = mydata['user-data']
+            self.vendordata = mydata['vendor-data']
             return True
 
         LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode'])
@@ -222,6 +240,16 @@ def parse_cmdline_data(ds_id, fill, cmdline=None):
     return True
 
 
+def _merge_new_seed(cur, seeded):
+    ret = cur.copy()
+    ret['meta-data'] = util.mergemanydict([cur['meta-data'],
+                                          util.load_yaml(seeded['meta-data'])])
+    ret['user-data'] = seeded['user-data']
+    if 'vendor-data' in seeded:
+        ret['vendor-data'] = seeded['vendor-data']
+    return ret
+
+
 class DataSourceNoCloudNet(DataSourceNoCloud):
     def __init__(self, sys_cfg, distro, paths):
         DataSourceNoCloud.__init__(self, sys_cfg, distro, paths)
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 6593ce6e..140c7814 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -25,7 +25,9 @@
 #        requests on the console. For example, to get the hostname, you
 #        would send "GET hostname" on /dev/ttyS1.
 #
-
+#   Certain behavior is defined by the DataDictionary
+#       http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html
+#       Comments with "@datadictionary" are snippets of the definition
 
 import base64
 from cloudinit import log as logging
@@ -43,10 +45,11 @@ SMARTOS_ATTRIB_MAP = {
     'local-hostname': ('hostname', True),
     'public-keys': ('root_authorized_keys', True),
     'user-script': ('user-script', False),
-    'user-data': ('user-data', False),
+    'legacy-user-data': ('user-data', False),
+    'user-data': ('cloud-init:user-data', False),
     'iptables_disable': ('iptables_disable', True),
     'motd_sys_info': ('motd_sys_info', True),
-    'availability_zone': ('datacenter_name', True),
+    'availability_zone': ('sdc:datacenter_name', True),
     'vendordata': ('sdc:operator-script', False),
 }
 
@@ -71,7 +74,11 @@ BUILTIN_DS_CONFIG = {
     'seed_timeout': 60,
     'no_base64_decode': ['root_authorized_keys',
                          'motd_sys_info',
-                         'iptables_disable'],
+                         'iptables_disable',
+                         'user-data',
+                         'user-script',
+                         'sdc:datacenter_name',
+                        ],
     'base64_keys': [],
     'base64_all': False,
     'disk_aliases': {'ephemeral0': '/dev/vdb'},
@@ -88,6 +95,11 @@ BUILTIN_CLOUD_CONFIG = {
                   'device': 'ephemeral0'}],
 }
 
+# @datadictionary: this is legacy path for placing files from metadata
+#   per the SmartOS location. It is not preferable, but is done for
+#   legacy reasons
+LEGACY_USER_D = "/var/db"
+
 
 class DataSourceSmartOS(sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
@@ -107,6 +119,9 @@ class DataSourceSmartOS(sources.DataSource):
         self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode')
         self.b64_keys = self.ds_cfg.get('base64_keys')
         self.b64_all = self.ds_cfg.get('base64_all')
+        self.script_base_d = os.path.join(self.paths.get_cpath("scripts"))
+        self.user_script_d = os.path.join(self.paths.get_cpath("scripts"),
+                                          'per-boot')
 
     def __str__(self):
         root = sources.DataSource.__str__(self)
@@ -144,14 +159,32 @@ class DataSourceSmartOS(sources.DataSource):
             smartos_noun, strip = attribute
             md[ci_noun] = self.query(smartos_noun, strip=strip)
 
+        # @datadictionary: This key may contain a program that is written
+        # to a file in the filesystem of the guest on each boot and then
+        # executed. It may be of any format that would be considered
+        # executable in the guest instance.
+        u_script = md.get('user-script')
+        u_script_f = "%s/99_user_script" % self.user_script_d
+        u_script_l = "%s/user-script" % LEGACY_USER_D
+        write_boot_content(u_script, u_script_f, link=u_script_l, shebang=True,
+                           mode=0700)
+
+        # @datadictionary:  This key has no defined format, but its value
+        # is written to the file /var/db/mdata-user-data on each boot prior
+        # to the phase that runs user-script. This file is not to be executed.
+        # This allows a configuration file of some kind to be injected into
+        # the machine to be consumed by the user-script when it runs.
+        u_data = md.get('legacy-user-data')
+        u_data_f = "%s/mdata-user-data" % LEGACY_USER_D
+        write_boot_content(u_data, u_data_f)
+
+        # Handle the cloud-init regular meta
         if not md['local-hostname']:
             md['local-hostname'] = system_uuid
 
         ud = None
         if md['user-data']:
             ud = md['user-data']
-        elif md['user-script']:
-            ud = md['user-script']
 
         self.metadata = util.mergemanydict([md, self.metadata])
         self.userdata_raw = ud
@@ -279,6 +312,62 @@ def dmi_data():
     return (sys_uuid.lower().strip(), sys_type.strip())
 
 
+def write_boot_content(content, content_f, link=None, shebang=False,
+                       mode=0400):
+    """
+    Write the content to content_f. Under the following rules:
+        1. If no content, remove the file
+        2. Write the content
+        3. If executable and no file magic, add it
+        4. If there is a link, create it
+
+    @param content: what to write
+    @param content_f: the file name
+    @param backup_d: the directory to save the backup at
+    @param link: if defined, location to create a symlink to
+    @param shebang: if no file magic, set shebang
+    @param mode: file mode
+
+    Becuase of the way that Cloud-init executes scripts (no shell),
+    a script will fail to execute if does not have a magic bit (shebang) set
+    for the file. If shebang=True, then the script will be checked for a magic
+    bit and to the SmartOS default of assuming that bash.
+    """
+
+    if not content and os.path.exists(content_f):
+        os.unlink(content_f)
+    if link and os.path.islink(link):
+        os.unlink(link)
+    if not content:
+        return
+
+    util.write_file(content_f, content, mode=mode)
+
+    if shebang and not content.startswith("#!"):
+        try:
+            cmd = ["file", "--brief", "--mime-type", content_f]
+            (f_type, _err) = util.subp(cmd)
+            LOG.debug("script %s mime type is %s", content_f, f_type)
+            if f_type.strip() == "text/plain":
+                new_content = "\n".join(["#!/bin/bash", content])
+                util.write_file(content_f, new_content, mode=mode)
+                LOG.debug("added shebang to file %s", content_f)
+
+        except Exception as e:
+            util.logexc(LOG, ("Failed to identify script type for %s" %
+                             content_f, e))
+
+    if link:
+        try:
+            if os.path.islink(link):
+                os.unlink(link)
+            if content and os.path.exists(content_f):
+                util.ensure_dir(os.path.dirname(link))
+                os.symlink(content_f, link)
+        except IOError as e:
+            util.logexc(LOG, "failed establishing content link", e)
+
+
 # Used to match classes to dependencies
 datasources = [
     (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 4b3bf62f..fef4d460 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -129,7 +129,7 @@ class DataSource(object):
         # when the kernel named them 'vda' or 'xvda'
         # we want to return the correct value for what will actually
         # exist in this instance
-        mappings = {"sd": ("vd", "xvd")}
+        mappings = {"sd": ("vd", "xvd", "vtb")}
         for (nfrom, tlist) in mappings.iteritems():
             if not short_name.startswith(nfrom):
                 continue