merge from trunk

11 files changed, 284 insertions, 47 deletions

diff --git a/ChangeLog b/ChangeLog
index 56f6ea68..a53a6b6f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -15,6 +15,8 @@
    which also reads from uptime. uptime is useful as clock may change during
    boot due to ntp.
  - prefer growpart resizer to 'parted resizepart' (LP: #1212492)
+ - support random data seed from config drive or azure, and a module
+   'seed_random' to read that and write it to /dev/urandom.
  - add OpenNebula Datasource [Vlastimil Holer]
 0.7.2:
  - add a debian watch file
diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py
new file mode 100644
index 00000000..22a31f29
--- /dev/null
+++ b/cloudinit/config/cc_seed_random.py
@@ -0,0 +1,61 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2013 Yahoo! Inc.
+#
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import base64
+from StringIO import StringIO
+
+from cloudinit.settings import PER_INSTANCE
+from cloudinit import util
+
+frequency = PER_INSTANCE
+
+
+def _decode(data, encoding=None):
+    if not data:
+        return ''
+    if not encoding or encoding.lower() in ['raw']:
+        return data
+    elif encoding.lower() in ['base64', 'b64']:
+        return base64.b64decode(data)
+    elif encoding.lower() in ['gzip', 'gz']:
+        return util.decomp_gzip(data, quiet=False)
+    else:
+        raise IOError("Unknown random_seed encoding: %s" % (encoding))
+
+
+def handle(name, cfg, cloud, log, _args):
+    if not cfg or "random_seed" not in cfg:
+        log.debug(("Skipping module named %s, "
+                   "no 'random_seed' configuration found"), name)
+        return
+
+    my_cfg = cfg['random_seed']
+    seed_path = my_cfg.get('file', '/dev/urandom')
+    seed_buf = StringIO()
+    seed_buf.write(_decode(my_cfg.get('data', ''),
+                           encoding=my_cfg.get('encoding')))
+
+    metadata = cloud.datasource.metadata
+    if metadata and 'random_seed' in metadata:
+        seed_buf.write(metadata['random_seed'])
+
+    seed_data = seed_buf.getvalue()
+    if len(seed_data):
+        log.debug("%s: adding %s bytes of random seed entrophy to %s", name,
+                  len(seed_data), seed_path)
+        util.append_file(seed_path, seed_data)
diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 1c46efde..e5eac6a7 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -292,11 +292,16 @@ class ContentHandlers(object):
     def is_registered(self, content_type):
         return content_type in self.registered
 
-    def register(self, mod, initialized=False):
+    def register(self, mod, initialized=False, overwrite=True):
         types = set()
         for t in mod.list_types():
+            if overwrite:
+                types.add(t)
+            else:
+                if not self.is_registered(t):
+                    types.add(t)
+        for t in types:
             self.registered[t] = mod
-            types.add(t)
         if initialized and mod not in self.initialized:
             self.initialized.append(mod)
         return types
@@ -310,15 +315,6 @@ class ContentHandlers(object):
     def iteritems(self):
         return self.registered.iteritems()
 
-    def register_defaults(self, defs):
-        registered = set()
-        for mod in defs:
-            for t in mod.list_types():
-                if not self.is_registered(t):
-                    self.registered[t] = mod
-                    registered.add(t)
-        return registered
-
 
 class Paths(object):
     def __init__(self, path_cfgs, ds=None):
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 66d7728b..a77c3d9a 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -106,6 +106,11 @@ class DataSourceAzureNet(sources.DataSource):
         if found == ddir:
             LOG.debug("using files cached in %s", ddir)
 
+        # azure / hyper-v provides random data here
+        seed = util.load_file("/sys/firmware/acpi/tables/OEM0", quiet=True)
+        if seed:
+            self.metadata['random_seed'] = seed
+
         # now update ds_cfg to reflect contents pass in config
         usercfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {})
         self.ds_cfg = util.mergemanydict([usercfg, self.ds_cfg])
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 835f2a9a..4f437244 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -18,6 +18,7 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import base64
 import json
 import os
 
@@ -41,6 +42,25 @@ DEFAULT_METADATA = {
 VALID_DSMODES = ("local", "net", "pass", "disabled")
 
 
+class ConfigDriveHelper(object):
+    def __init__(self, distro):
+        self.distro = distro
+
+    def on_first_boot(self, data):
+        if not data:
+            data = {}
+        if 'network_config' in data:
+            LOG.debug("Updating network interfaces from config drive")
+            self.distro.apply_network(data['network_config'])
+        files = data.get('files')
+        if files:
+            LOG.debug("Writing %s injected files", len(files))
+            try:
+                write_files(files)
+            except IOError:
+                util.logexc(LOG, "Failed writing files")
+
+
 class DataSourceConfigDrive(sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
@@ -49,6 +69,7 @@ class DataSourceConfigDrive(sources.DataSource):
         self.seed_dir = os.path.join(paths.seed_dir, 'config_drive')
         self.version = None
         self.ec2_metadata = None
+        self.helper = ConfigDriveHelper(distro)
 
     def __str__(self):
         root = sources.DataSource.__str__(self)
@@ -187,20 +208,8 @@ class DataSourceConfigDrive(sources.DataSource):
         # instance-id
         prev_iid = get_previous_iid(self.paths)
         cur_iid = md['instance-id']
-
-        if ('network_config' in results and self.dsmode == "local" and
-            prev_iid != cur_iid):
-            LOG.debug("Updating network interfaces from config drive (%s)",
-                      dsmode)
-            self.distro.apply_network(results['network_config'])
-
-        # file writing occurs in local mode (to be as early as possible)
-        if self.dsmode == "local" and prev_iid != cur_iid and results['files']:
-            LOG.debug("writing injected files")
-            try:
-                write_files(results['files'])
-            except:
-                util.logexc(LOG, "Failed writing files")
+        if prev_iid != cur_iid and self.dsmode == "local":
+            self.helper.on_first_boot(results)
 
         # dsmode != self.dsmode here if:
         #  * dsmode = "pass",  pass means it should only copy files and then
@@ -338,6 +347,13 @@ def read_config_drive_dir_v2(source_dir, version="2012-08-10"):
         except KeyError:
             raise BrokenConfigDriveDir("No uuid entry in metadata")
 
+    if 'random_seed' in results['metadata']:
+        random_seed = results['metadata']['random_seed']
+        try:
+            results['metadata']['random_seed'] = base64.b64decode(random_seed)
+        except (ValueError, TypeError) as exc:
+            raise BrokenConfigDriveDir("Badly formatted random_seed: %s" % exc)
+
     def read_content_path(item):
         # do not use os.path.join here, as content_path starts with /
         cpath = os.path.sep.join((source_dir, "openstack",
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 3e49e8c5..07c55802 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -375,7 +375,9 @@ class Init(object):
                     mod = importer.import_module(mod_locs[0])
                     mod = handlers.fixup_handler(mod)
                     types = c_handlers.register(mod)
-                    LOG.debug("Added handler for %s from %s", types, fname)
+                    if types:
+                        LOG.debug("Added custom handler for %s from %s",
+                                  types, fname)
                 except Exception:
                     util.logexc(LOG, "Failed to register handler from %s",
                                 fname)
@@ -386,10 +388,10 @@ class Init(object):
         # Register any other handlers that come from the default set. This
         # is done after the cloud-dir handlers so that the cdir modules can
         # take over the default user-data handler content-types.
-        def_handlers = self._default_userdata_handlers()
-        applied_def_handlers = c_handlers.register_defaults(def_handlers)
-        if applied_def_handlers:
-            LOG.debug("Registered default handlers: %s", applied_def_handlers)
+        for mod in self._default_userdata_handlers():
+            types = c_handlers.register(mod, overwrite=False)
+            if types:
+                LOG.debug("Added default handler for %s from %s", types, mod)
 
         # Form our cloud interface
         data = self.cloudify()
diff --git a/cloudinit/util.py b/cloudinit/util.py
index e1c51f31..d50d3e18 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -1798,15 +1798,19 @@ def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False):
         ret = func(*args, **kwargs)
     finally:
         delta = time.time() - start
+        udelta = None
         if ustart is not None:
             try:
                 udelta = float(uptime()) - ustart
             except ValueError:
-                udelta = "N/A"
+                pass
 
         tmsg = " took %0.3f seconds" % delta
         if get_uptime:
-            tmsg += "(%0.2f)" % udelta
+            if isinstance(udelta, (float)):
+                tmsg += " (%0.2f)" % udelta
+            else:
+                tmsg += " (N/A)"
         try:
             logfunc(msg + tmsg)
         except:
diff --git a/config/cloud.cfg b/config/cloud.cfg
index b61b8a7d..cce1f376 100644
--- a/config/cloud.cfg
+++ b/config/cloud.cfg
@@ -24,6 +24,7 @@ preserve_hostname: false
 # The modules that run in the 'init' stage
 cloud_init_modules:
  - migrator
+ - seed_random
  - bootcmd
  - write-files
  - growpart
diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py
new file mode 100644
index 00000000..1b0675bb
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_seed_random.py
@@ -0,0 +1,150 @@
+            #    Copyright (C) 2013 Hewlett-Packard Development Company, L.P.
+#
+#    Author: Juerg Haefliger <juerg.haefliger@hp.com>
+#
+#    Based on test_handler_set_hostname.py
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from cloudinit.config import cc_seed_random
+
+import base64
+import tempfile
+import gzip
+
+from StringIO import StringIO
+
+from cloudinit import cloud
+from cloudinit import distros
+from cloudinit import helpers
+from cloudinit import util
+
+from cloudinit.sources import DataSourceNone
+
+from tests.unittests import helpers as t_help
+
+import logging
+
+LOG = logging.getLogger(__name__)
+
+
+class TestRandomSeed(t_help.TestCase):
+    def setUp(self):
+        super(TestRandomSeed, self).setUp()
+        self._seed_file = tempfile.mktemp()
+
+    def tearDown(self):
+        util.del_file(self._seed_file)
+
+    def _compress(self, text):
+        contents = StringIO()
+        gz_fh = gzip.GzipFile(mode='wb', fileobj=contents)
+        gz_fh.write(text)
+        gz_fh.close()
+        return contents.getvalue()
+
+    def _get_cloud(self, distro, metadata=None):
+        paths = helpers.Paths({})
+        cls = distros.fetch(distro)
+        ubuntu_distro = cls(distro, {}, paths)
+        ds = DataSourceNone.DataSourceNone({}, ubuntu_distro, paths)
+        if metadata:
+            ds.metadata = metadata
+        return cloud.Cloud(ds, paths, {}, ubuntu_distro, None)
+
+    def test_append_random(self):
+        cfg = {
+            'random_seed': {
+                'file': self._seed_file,
+                'data': 'tiny-tim-was-here',
+            }
+        }
+        cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, [])
+        contents = util.load_file(self._seed_file)
+        self.assertEquals("tiny-tim-was-here", contents)
+
+    def test_append_random_unknown_encoding(self):
+        data = self._compress("tiny-toe")
+        cfg = {
+            'random_seed': {
+                'file': self._seed_file,
+                'data': data,
+                'encoding': 'special_encoding',
+            }
+        }
+        self.assertRaises(IOError, cc_seed_random.handle, 'test', cfg,
+                          self._get_cloud('ubuntu'), LOG, [])
+
+    def test_append_random_gzip(self):
+        data = self._compress("tiny-toe")
+        cfg = {
+            'random_seed': {
+                'file': self._seed_file,
+                'data': data,
+                'encoding': 'gzip',
+            }
+        }
+        cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, [])
+        contents = util.load_file(self._seed_file)
+        self.assertEquals("tiny-toe", contents)
+
+    def test_append_random_gz(self):
+        data = self._compress("big-toe")
+        cfg = {
+            'random_seed': {
+                'file': self._seed_file,
+                'data': data,
+                'encoding': 'gz',
+            }
+        }
+        cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, [])
+        contents = util.load_file(self._seed_file)
+        self.assertEquals("big-toe", contents)
+
+    def test_append_random_base64(self):
+        data = base64.b64encode('bubbles')
+        cfg = {
+            'random_seed': {
+                'file': self._seed_file,
+                'data': data,
+                'encoding': 'base64',
+            }
+        }
+        cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, [])
+        contents = util.load_file(self._seed_file)
+        self.assertEquals("bubbles", contents)
+
+    def test_append_random_b64(self):
+        data = base64.b64encode('kit-kat')
+        cfg = {
+            'random_seed': {
+                'file': self._seed_file,
+                'data': data,
+                'encoding': 'b64',
+            }
+        }
+        cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, [])
+        contents = util.load_file(self._seed_file)
+        self.assertEquals("kit-kat", contents)
+
+    def test_append_random_metadata(self):
+        cfg = {
+            'random_seed': {
+                'file': self._seed_file,
+                'data': 'tiny-tim-was-here',
+            }
+        }
+        c = self._get_cloud('ubuntu', {'random_seed': '-so-was-josh'})
+        cc_seed_random.handle('test', cfg, c, LOG, [])
+        contents = util.load_file(self._seed_file)
+        self.assertEquals('tiny-tim-was-here-so-was-josh', contents)
diff --git a/tools/read-dependencies b/tools/read-dependencies
index cadb09a8..87db5d83 100755
--- a/tools/read-dependencies
+++ b/tools/read-dependencies
@@ -12,20 +12,20 @@ find_root() {
    [ $? -eq 0 -a -f "${topd}/setup.py" ] || return
    ROOT_DIR="$topd"
 }
+fail() { echo "$0:" "$@" 1>&2; exit 1; }
 
 if ! find_root; then
-    echo "Unable to locate 'setup.py' file that should" \
-          "exist in the cloud-init root directory." 1>&2
-    exit 1;
+    fail "Unable to locate 'setup.py' file that should " \
+         "exist in the cloud-init root directory."
 fi
 
 REQUIRES="$ROOT_DIR/Requires"
 
 if [ ! -e "$REQUIRES" ]; then
-    echo "Unable to find 'Requires' file located at $REQUIRES"
-    exit 1
+    fail "Unable to find 'Requires' file located at '$REQUIRES'"
 fi
 
-# Filter out comments and empty liens
-DEPS=$(grep -Pv "^\s*#" "$REQUIRES" | grep -Pv '^\s*$')
+# Filter out comments and empty lines
+DEPS=$(sed -n -e 's,#.*,,' -e '/./p' "$REQUIRES") ||
+   fail "failed to read deps from '${REQUIRES}'"
 echo "$DEPS" | sort -d -f
diff --git a/tools/read-version b/tools/read-version
index c76b24a9..c37228f8 100755
--- a/tools/read-version
+++ b/tools/read-version
@@ -12,20 +12,20 @@ find_root() {
    [ $? -eq 0 -a -f "${topd}/setup.py" ] || return
    ROOT_DIR="$topd"
 }
+fail() { echo "$0:" "$@" 1>&2; exit 1; }
 
 if ! find_root; then
-    echo "Unable to locate 'setup.py' file that should" \
-          "exist in the cloud-init root directory." 1>&2
-    exit 1;
+    fail "Unable to locate 'setup.py' file that should " \
+         "exist in the cloud-init root directory."
 fi
 
 CHNG_LOG="$ROOT_DIR/ChangeLog"
 
-if [ ! -e "$CHNG_LOG" ]
-then
-    echo "Unable to find 'ChangeLog' file located at $CHNG_LOG"
-    exit 1
+if [ ! -e "$CHNG_LOG" ]; then
+    fail "Unable to find 'ChangeLog' file located at '$CHNG_LOG'"
 fi
 
-VERSION=$(grep -P "\d+.\d+.\d+:" "$CHNG_LOG"  | cut -f1 -d ":" | head -n 1)
+VERSION=$(sed -n '/^[0-9]\+[.][0-9]\+[.][0-9]\+:/ {s/://; p; :a;n; ba}' \
+          "$CHNG_LOG") ||
+   fail "failed to get version from '$CHNG_LOG'"
 echo "$VERSION"