merge from trunk

14 files changed, 219 insertions, 32 deletions

diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py
index e92cba4a..b24294e4 100644
--- a/cloudinit/config/cc_final_message.py
+++ b/cloudinit/config/cc_final_message.py
@@ -53,6 +53,7 @@ def handle(_name, cfg, cloud, log, args):
             'version': cver,
             'datasource': str(cloud.datasource),
         }
+        subs.update(dict([(k.upper(), v) for k, v in subs.items()]))
         util.multi_log("%s\n" % (templater.render_string(msg_in, subs)),
                        console=False, stderr=True, log=log)
     except Exception:
diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py
index 561c5abd..8f99e887 100644
--- a/cloudinit/config/cc_power_state_change.py
+++ b/cloudinit/config/cc_power_state_change.py
@@ -22,7 +22,6 @@ from cloudinit import util
 import errno
 import os
 import re
-import signal
 import subprocess
 import time
 
diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py
index 22a31f29..49a6b3e8 100644
--- a/cloudinit/config/cc_seed_random.py
+++ b/cloudinit/config/cc_seed_random.py
@@ -1,8 +1,11 @@
 # vi: ts=4 expandtab
 #
 #    Copyright (C) 2013 Yahoo! Inc.
+#    Copyright (C) 2014 Canonical, Ltd
 #
 #    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#    Author: Dustin Kirkland <kirkland@ubuntu.com>
+#    Author: Scott Moser <scott.moser@canonical.com>
 #
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License version 3, as
@@ -17,12 +20,15 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import base64
+import os
 from StringIO import StringIO
 
 from cloudinit.settings import PER_INSTANCE
+from cloudinit import log as logging
 from cloudinit import util
 
 frequency = PER_INSTANCE
+LOG = logging.getLogger(__name__)
 
 
 def _decode(data, encoding=None):
@@ -38,24 +44,50 @@ def _decode(data, encoding=None):
         raise IOError("Unknown random_seed encoding: %s" % (encoding))
 
 
-def handle(name, cfg, cloud, log, _args):
-    if not cfg or "random_seed" not in cfg:
-        log.debug(("Skipping module named %s, "
-                   "no 'random_seed' configuration found"), name)
+def handle_random_seed_command(command, required, env=None):
+    if not command and required:
+        raise ValueError("no command found but required=true")
+    elif not command:
+        LOG.debug("no command provided")
         return
 
-    my_cfg = cfg['random_seed']
-    seed_path = my_cfg.get('file', '/dev/urandom')
+    cmd = command[0]
+    if not util.which(cmd):
+        if required:
+            raise ValueError("command '%s' not found but required=true", cmd)
+        else:
+            LOG.debug("command '%s' not found for seed_command", cmd)
+            return
+    util.subp(command, env=env, capture=False)
+
+
+def handle(name, cfg, cloud, log, _args):
+    mycfg = cfg.get('random_seed', {})
+    seed_path = mycfg.get('file', '/dev/urandom')
+    seed_data = mycfg.get('data', '')
+
     seed_buf = StringIO()
-    seed_buf.write(_decode(my_cfg.get('data', ''),
-                           encoding=my_cfg.get('encoding')))
+    if seed_data:
+        seed_buf.write(_decode(seed_data, encoding=mycfg.get('encoding')))
 
+    # 'random_seed' is set up by Azure datasource, and comes already in
+    # openstack meta_data.json
     metadata = cloud.datasource.metadata
     if metadata and 'random_seed' in metadata:
         seed_buf.write(metadata['random_seed'])
 
     seed_data = seed_buf.getvalue()
     if len(seed_data):
-        log.debug("%s: adding %s bytes of random seed entrophy to %s", name,
+        log.debug("%s: adding %s bytes of random seed entropy to %s", name,
                   len(seed_data), seed_path)
         util.append_file(seed_path, seed_data)
+
+    command = mycfg.get('command', ['pollinate', '-q'])
+    req = mycfg.get('command_required', False)
+    try:
+        env = os.environ.copy()
+        env['RANDOM_SEED_FILE'] = seed_path
+        handle_random_seed_command(command=command, required=req, env=env)
+    except ValueError as e:
+        log.warn("handling random command [%s] failed: %s", command, e)
+        raise e
diff --git a/cloudinit/cs_utils.py b/cloudinit/cs_utils.py
index 4e53c31a..dcf56431 100644
--- a/cloudinit/cs_utils.py
+++ b/cloudinit/cs_utils.py
@@ -35,6 +35,10 @@ import platform
 
 import serial
 
+# these high timeouts are necessary as read may read a lot of data.
+READ_TIMEOUT = 60
+WRITE_TIMEOUT = 10
+
 SERIAL_PORT = '/dev/ttyS1'
 if platform.system() == 'Windows':
     SERIAL_PORT = 'COM2'
@@ -76,7 +80,9 @@ class CepkoResult(object):
         self.result = self._marshal(self.raw_result)
 
     def _execute(self):
-        connection = serial.Serial(SERIAL_PORT)
+        connection = serial.Serial(port=SERIAL_PORT,
+                                   timeout=READ_TIMEOUT,
+                                   writeTimeout=WRITE_TIMEOUT)
         connection.write(self.request)
         return connection.readline().strip('\x04\n')
 
diff --git a/cloudinit/importer.py b/cloudinit/importer.py
index a094141a..a1929137 100644
--- a/cloudinit/importer.py
+++ b/cloudinit/importer.py
@@ -45,8 +45,6 @@ def find_module(base_name, search_paths, required_attrs=None):
         real_path.append(base_name)
         full_path = '.'.join(real_path)
         real_paths.append(full_path)
-    LOG.debug("Looking for modules %s that have attributes %s",
-              real_paths, required_attrs)
     for full_path in real_paths:
         mod = None
         try:
@@ -62,6 +60,4 @@ def find_module(base_name, search_paths, required_attrs=None):
                 found_attrs += 1
         if found_attrs == len(required_attrs):
             found_places.append(full_path)
-    LOG.debug("Found %s with attributes %s in %s", base_name,
-              required_attrs, found_places)
     return found_places
diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py
index 0978b2c6..650b42a9 100644
--- a/cloudinit/mergers/__init__.py
+++ b/cloudinit/mergers/__init__.py
@@ -55,9 +55,6 @@ class UnknownMerger(object):
         if not meth:
             meth = self._handle_unknown
             args.insert(0, method_name)
-        LOG.debug("Merging '%s' into '%s' using method '%s' of '%s'",
-                  type_name, type_utils.obj_name(merge_with),
-                  meth.__name__, self)
         return meth(*args)
 
 
@@ -84,8 +81,6 @@ class LookupMerger(UnknownMerger):
                 # First one that has that method/attr gets to be
                 # the one that will be called
                 meth = getattr(merger, meth_wanted)
-                LOG.debug(("Merging using located merger '%s'"
-                            " since it had method '%s'"), merger, meth_wanted)
                 break
         if not meth:
             return UnknownMerger._handle_unknown(self, meth_wanted,
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index c7331da5..bd75e6d8 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -18,12 +18,14 @@
 
 import base64
 import crypt
+import fnmatch
 import os
 import os.path
 import time
 from xml.dom import minidom
 
 from cloudinit import log as logging
+from cloudinit.settings import PER_ALWAYS
 from cloudinit import sources
 from cloudinit import util
 
@@ -53,14 +55,15 @@ BUILTIN_CLOUD_CONFIG = {
     'disk_setup': {
         'ephemeral0': {'table_type': 'mbr',
                        'layout': True,
-                       'overwrite': False}
-         },
+                       'overwrite': False},
+        },
     'fs_setup': [{'filesystem': 'ext4',
                   'device': 'ephemeral0.1',
-                  'replace_fs': 'ntfs'}]
+                  'replace_fs': 'ntfs'}],
 }
 
 DS_CFG_PATH = ['datasource', DS_NAME]
+DEF_EPHEMERAL_LABEL = 'Temporary Storage'
 
 
 class DataSourceAzureNet(sources.DataSource):
@@ -189,8 +192,17 @@ class DataSourceAzureNet(sources.DataSource):
                 LOG.warn("failed to get instance id in %s: %s", shcfgxml, e)
 
         pubkeys = pubkeys_from_crt_files(fp_files)
-
         self.metadata['public-keys'] = pubkeys
+
+        found_ephemeral = find_ephemeral_disk()
+        if found_ephemeral:
+            self.ds_cfg['disk_aliases']['ephemeral0'] = found_ephemeral
+            LOG.debug("using detected ephemeral0 of %s", found_ephemeral)
+
+        cc_modules_override = support_new_ephemeral(self.sys_cfg)
+        if cc_modules_override:
+            self.cfg['cloud_config_modules'] = cc_modules_override
+
         return True
 
     def device_name_to_device(self, name):
@@ -200,6 +212,92 @@ class DataSourceAzureNet(sources.DataSource):
         return self.cfg
 
 
+def count_files(mp):
+    return len(fnmatch.filter(os.listdir(mp), '*[!cdrom]*'))
+
+
+def find_ephemeral_part():
+    """
+    Locate the default ephmeral0.1 device. This will be the first device
+    that has a LABEL of DEF_EPHEMERAL_LABEL and is a NTFS device. If Azure
+    gets more ephemeral devices, this logic will only identify the first
+    such device.
+    """
+    c_label_devs = util.find_devs_with("LABEL=%s" % DEF_EPHEMERAL_LABEL)
+    c_fstype_devs = util.find_devs_with("TYPE=ntfs")
+    for dev in c_label_devs:
+        if dev in c_fstype_devs:
+            return dev
+    return None
+
+
+def find_ephemeral_disk():
+    """
+    Get the ephemeral disk.
+    """
+    part_dev = find_ephemeral_part()
+    if part_dev and str(part_dev[-1]).isdigit():
+        return part_dev[:-1]
+    elif part_dev:
+        return part_dev
+    return None
+
+
+def support_new_ephemeral(cfg):
+    """
+    Windows Azure makes ephemeral devices ephemeral to boot; a ephemeral device
+    may be presented as a fresh device, or not.
+
+    Since the knowledge of when a disk is supposed to be plowed under is
+    specific to Windows Azure, the logic resides here in the datasource. When a
+    new ephemeral device is detected, cloud-init overrides the default
+    frequency for both disk-setup and mounts for the current boot only.
+    """
+    device = find_ephemeral_part()
+    if not device:
+        LOG.debug("no default fabric formated ephemeral0.1 found")
+        return None
+    LOG.debug("fabric formated ephemeral0.1 device at %s", device)
+
+    file_count = 0
+    try:
+        file_count = util.mount_cb(device, count_files)
+    except:
+        return None
+    LOG.debug("fabric prepared ephmeral0.1 has %s files on it", file_count)
+
+    if file_count >= 1:
+        LOG.debug("fabric prepared ephemeral0.1 will be preserved")
+        return None
+    else:
+        # if device was already mounted, then we need to unmount it
+        # race conditions could allow for a check-then-unmount
+        # to have a false positive. so just unmount and then check.
+        try:
+            util.subp(['umount', device])
+        except util.ProcessExecutionError as e:
+            if device in util.mounts():
+                LOG.warn("Failed to unmount %s, will not reformat.", device)
+                LOG.debug("Failed umount: %s", e)
+                return None
+
+    LOG.debug("cloud-init will format ephemeral0.1 this boot.")
+    LOG.debug("setting disk_setup and mounts modules 'always' for this boot")
+
+    cc_modules = cfg.get('cloud_config_modules')
+    if not cc_modules:
+        return None
+
+    mod_list = []
+    for mod in cc_modules:
+        if mod in ("disk_setup", "mounts"):
+            mod_list.append([mod, PER_ALWAYS])
+            LOG.debug("set module '%s' to 'always' for this boot", mod)
+        else:
+            mod_list.append(mod)
+    return mod_list
+
+
 def handle_set_hostname(enabled, hostname, cfg):
     if not util.is_true(enabled):
         return
diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py
index dad37119..707cd0ce 100644
--- a/cloudinit/sources/DataSourceCloudSigma.py
+++ b/cloudinit/sources/DataSourceCloudSigma.py
@@ -15,10 +15,13 @@
 #
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+from base64 import b64decode
+import os
 import re
 
 from cloudinit import log as logging
 from cloudinit import sources
+from cloudinit import util
 from cloudinit.cs_utils import Cepko
 
 LOG = logging.getLogger(__name__)
@@ -39,12 +42,40 @@ class DataSourceCloudSigma(sources.DataSource):
         self.ssh_public_key = ''
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
 
+    def is_running_in_cloudsigma(self):
+        """
+        Uses dmidecode to detect if this instance of cloud-init is running
+        in the CloudSigma's infrastructure.
+        """
+        uname_arch = os.uname()[4]
+        if uname_arch.startswith("arm") or uname_arch == "aarch64":
+            # Disabling because dmidecode in CMD_DMI_SYSTEM crashes kvm process
+            LOG.debug("Disabling CloudSigma datasource on arm (LP: #1243287)")
+            return False
+
+        dmidecode_path = util.which('dmidecode')
+        if not dmidecode_path:
+            return False
+
+        LOG.debug("Determining hypervisor product name via dmidecode")
+        try:
+            cmd = [dmidecode_path, "--string", "system-product-name"]
+            system_product_name, _ = util.subp(cmd)
+            return 'cloudsigma' in system_product_name.lower()
+        except:
+            LOG.warn("Failed to get hypervisor product name via dmidecode")
+
+        return False
+
     def get_data(self):
         """
         Metadata is the whole server context and /meta/cloud-config is used
         as userdata.
         """
         dsmode = None
+        if not self.is_running_in_cloudsigma():
+            return False
+
         try:
             server_context = self.cepko.all().result
             server_meta = server_context['meta']
@@ -61,7 +92,13 @@ class DataSourceCloudSigma(sources.DataSource):
         if dsmode == "disabled" or dsmode != self.dsmode:
             return False
 
+        base64_fields = server_meta.get('base64_fields', '').split(',')
         self.userdata_raw = server_meta.get('cloudinit-user-data', "")
+        if 'cloudinit-user-data' in base64_fields:
+            self.userdata_raw = b64decode(self.userdata_raw)
+        if 'cloudinit' in server_context.get('vendor_data', {}):
+            self.vendordata_raw = server_context["vendor_data"]["cloudinit"]
+
         self.metadata = server_context
         self.ssh_public_key = server_meta['ssh_public_key']
 
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 8dc96ab6..a315aae0 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -57,7 +57,7 @@ class DataSourceNoCloud(sources.DataSource):
             md = {}
             if parse_cmdline_data(self.cmdline_id, md):
                 found.append("cmdline")
-            mydata.update(md)
+            mydata['meta-data'].update(md)
         except:
             util.logexc(LOG, "Unable to parse command line data")
             return False
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index b0464cbb..34557f8b 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -4,11 +4,13 @@
 #    Copyright (C) 2012 Yahoo! Inc.
 #    Copyright (C) 2012-2013 CERIT Scientific Cloud
 #    Copyright (C) 2012-2013 OpenNebula.org
+#    Copyright (C) 2014 Consejo Superior de Investigaciones Cientificas
 #
 #    Author: Scott Moser <scott.moser@canonical.com>
 #    Author: Joshua Harlow <harlowja@yahoo-inc.com>
 #    Author: Vlastimil Holer <xholer@mail.muni.cz>
 #    Author: Javier Fontan <jfontan@opennebula.org>
+#    Author: Enol Fernandez <enolfc@ifca.unican.es>
 #
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License version 3, as
@@ -22,6 +24,7 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import base64
 import os
 import pwd
 import re
@@ -417,6 +420,16 @@ def read_context_disk_dir(source_dir, asuser=None):
     elif "USERDATA" in context:
         results['userdata'] = context["USERDATA"]
 
+    # b64decode user data if necessary (default)
+    if 'userdata' in results:
+        encoding = context.get('USERDATA_ENCODING',
+                               context.get('USER_DATA_ENCODING'))
+        if encoding == "base64":
+            try:
+                results['userdata'] = base64.b64decode(results['userdata'])
+            except TypeError:
+                LOG.warn("Failed base64 decoding of userdata")
+
     # generate static /etc/network/interfaces
     # only if there are any required context variables
     # http://opennebula.org/documentation:rel3.8:cong#network_configuration
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 7c1eb09a..65ec0339 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -170,8 +170,9 @@ class DataSourceSmartOS(sources.DataSource):
         md = {}
         ud = ""
 
-        if not os.path.exists(self.seed):
-            LOG.debug("Host does not appear to be on SmartOS")
+        if not device_exists(self.seed):
+            LOG.debug("No serial device '%s' found for SmartOS datasource",
+                      self.seed)
             return False
 
         uname_arch = os.uname()[4]
@@ -274,6 +275,11 @@ class DataSourceSmartOS(sources.DataSource):
                           b64=b64)
 
 
+def device_exists(device):
+    """Symplistic method to determine if the device exists or not"""
+    return os.path.exists(device)
+
+
 def get_serial(seed_device, seed_timeout):
     """This is replaced in unit testing, allowing us to replace
         serial.Serial with a mocked class.
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 58349ffc..9e071fc4 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -397,8 +397,8 @@ class Init(object):
                     mod = handlers.fixup_handler(mod)
                     types = c_handlers.register(mod)
                     if types:
-                        LOG.debug("Added custom handler for %s from %s",
-                                  types, fname)
+                        LOG.debug("Added custom handler for %s [%s] from %s",
+                                  types, mod, fname)
                 except Exception:
                     util.logexc(LOG, "Failed to register handler from %s",
                                 fname)
@@ -644,6 +644,8 @@ class Modules(object):
                     freq = mod.frequency
                 if not freq in FREQUENCIES:
                     freq = PER_INSTANCE
+                LOG.debug("Running module %s (%s) with frequency %s",
+                          name, mod, freq)
 
                 # Use the configs logger and not our own
                 # TODO(harlowja): possibly check the module
@@ -657,7 +659,7 @@ class Modules(object):
                 run_name = "config-%s" % (name)
                 cc.run(run_name, mod.handle, func_args, freq=freq)
             except Exception as e:
-                util.logexc(LOG, "Running %s (%s) failed", name, mod)
+                util.logexc(LOG, "Running module %s (%s) failed", name, mod)
                 failures.append((name, e))
         return (which_ran, failures)
 
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 87b0c853..06039ee2 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -1395,8 +1395,10 @@ def get_builtin_cfg():
     return obj_copy.deepcopy(CFG_BUILTIN)
 
 
-def sym_link(source, link):
+def sym_link(source, link, force=False):
     LOG.debug("Creating symbolic link from %r => %r", link, source)
+    if force and os.path.exists(link):
+        del_file(link)
     os.symlink(source, link)
 
 
diff --git a/cloudinit/version.py b/cloudinit/version.py
index 3db57235..edb651a9 100644
--- a/cloudinit/version.py
+++ b/cloudinit/version.py
@@ -20,7 +20,7 @@ from distutils import version as vr
 
 
 def version():
-    return vr.StrictVersion("0.7.5")
+    return vr.StrictVersion("0.7.6")
 
 
 def version_string():