From 6b37d05a4848e00d34c34b4b23a74d9187f29bba Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Mon, 11 Jun 2012 17:09:38 -0700
Subject: All main stage helpers go here, ie semaphores, handler registrys,
 runners...

---
 cloudinit/helpers.py | 204 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 204 insertions(+)
 create mode 100644 cloudinit/helpers.py

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
new file mode 100644
index 00000000..cdb8a07e
--- /dev/null
+++ b/cloudinit/helpers.py
@@ -0,0 +1,204 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2012 Canonical Ltd.
+#    Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
+#    Copyright (C) 2012 Yahoo! Inc.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#    Author: Juerg Haefliger <juerg.haefliger@hp.com>
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from time import time
+
+import contextlib
+import os
+
+from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE)
+
+from cloudinit import log as logging
+from cloudinit import util
+
+from cloudinit.user_data import boot_hook as bh_part
+from cloudinit.user_data import cloud_config as cc_part
+from cloudinit.user_data import shell_script as ss_part
+from cloudinit.user_data import upstart_job as up_part
+
+LOG = logging.getLogger(__name__)
+
+
+class DummySemaphores(object):
+    def __init__(self):
+        pass
+
+    @contextlib.contextmanager
+    def lock(self, _name, _freq, _clear_on_fail):
+        yield True
+
+    def has_run(self, _name, _freq):
+        return False
+
+
+class Semaphores(object):
+    def __init__(self, sem_path):
+        self.sem_path = sem_path
+
+    @contextlib.contextmanager
+    def lock(self, name, freq, clear_on_fail):
+        try:
+            yield self._acquire(name, freq)
+        except:
+            if clear_on_fail:
+                self.clear(name, freq)
+            raise
+
+    def clear(self, name, freq):
+        sem_file = self._get_path(name, freq)
+        try:
+            util.del_file(sem_file)
+        except (IOError, OSError):
+            return False
+        return True
+
+    def _acquire(self, name, freq):
+        if self.has_run(name, freq):
+            return None
+        # This is a race condition since nothing atomic is happening
+        # here, but this should be ok due to the nature of when
+        # and where cloud-init runs... (file writing is not a lock..)
+        sem_file = self._get_path(name, freq)
+        contents = "%s: %s\n" % (os.getpid(), time())
+        try:
+            util.write_file(sem_file, contents)
+        except (IOError, OSError):
+            return None
+        return sem_file
+
+    def has_run(self, name, freq):
+        if freq == PER_ALWAYS:
+            return False
+        sem_file = self._get_path(name, freq)
+        if os.path.exists(sem_file):
+            return True
+        return False
+
+    def _get_path(self, name, freq):
+        sem_path = self.sem_path
+        if not freq or freq == PER_INSTANCE:
+            return os.path.join(sem_path, name)
+        else:
+            return os.path.join(sem_path, "%s.%s" % (name, freq))
+
+
+class Runners(object):
+    def __init__(self, paths):
+        self.paths = paths
+        self.sems = {}
+
+    def _get_sem(self, freq):
+        if freq == PER_ALWAYS or not freq:
+            return None
+        sem_path = None
+        if freq == PER_INSTANCE:
+            sem_path = self.paths.get_ipath("sem")
+        elif freq == PER_ONCE:
+            sem_path = self.paths.get_cpath("sem")
+        if not sem_path:
+            return None
+        if sem_path not in self.sems:
+            self.sems[sem_path] = Semaphores(sem_path)
+        return self.sems[sem_path]
+
+    def run(self, name, functor, args, freq=None, clear_on_fail=False):
+        sem = self._get_sem(freq)
+        if not sem:
+            sem = DummySemaphores()
+        if not args:
+            args = []
+        if sem.has_run(name, freq):
+            LOG.info("%s already ran (freq=%s)", name, freq)
+            return None
+        with sem.lock(name, freq, clear_on_fail) as lk:
+            if not lk:
+                raise RuntimeError("Failed to acquire lock on %s" % name)
+            else:
+                LOG.debug("Running %s with args %s using lock %s",
+                          functor, args, lk)
+                return functor(*args)
+
+
+class ContentHandlers(object):
+
+    def __init__(self, paths):
+        self.paths = paths
+        self.registered = {}
+
+    def __contains__(self, item):
+        return self.is_registered(item)
+
+    def __getitem__(self, key):
+        return self._get_handler(key)
+
+    def is_registered(self, content_type):
+        return content_type in self.registered
+
+    def register(self, mod):
+        types = set()
+        for t in mod.list_types():
+            self.registered[t] = mod
+            types.add(t)
+        return types
+
+    def _get_handler(self, content_type):
+        return self.registered[content_type]
+
+    def items(self):
+        return self.registered.items()
+
+    def iteritems(self):
+        return self.registered.iteritems()
+
+    def _get_default_handlers(self):
+        def_handlers = []
+
+        cc_path = self.paths.get_ipath("cloud_config")
+        if cc_path:
+            cc_h = cc_part.CloudConfigPartHandler(cc_path)
+            def_handlers.append(cc_h)
+
+        sc_path = self.paths.get_ipath_cur('scripts')
+        if sc_path:
+            ss_h = ss_part.ShellScriptPartHandler(sc_path)
+            def_handlers.append(ss_h)
+
+        bh_path = self.paths.get_ipath("boothooks")
+        if bh_path:
+            bh_h = bh_part.BootHookPartHandler(bh_path)
+            def_handlers.append(bh_h)
+
+        upstart_pth = self.paths.upstart_conf_d
+        if upstart_pth:
+            up_h = up_part.UpstartJobPartHandler(upstart_pth)
+            def_handlers.append(up_h)
+
+        return def_handlers
+
+    def register_defaults(self):
+        registered = set()
+        for mod in self._get_default_handlers():
+            for t in mod.list_types():
+                if not self.is_registered(t):
+                    self.registered[t] = mod
+                    registered.add(t)
+        return registered
-- 
cgit v1.2.3


From 830c54fec367f852b2f4b0caa71849aa621b5377 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Mon, 11 Jun 2012 17:11:46 -0700
Subject: Move paths to here, since it also qualifies.

---
 cloudinit/helpers.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index cdb8a07e..0bd13c78 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -202,3 +202,66 @@ class ContentHandlers(object):
                     self.registered[t] = mod
                     registered.add(t)
         return registered
+
+
+class Paths(object):
+    def __init__(self, sys_info):
+        self.cloud_dir = sys_info.get('cloud_dir', '/var/lib/cloud')
+        self.instance_link = os.path.join(self.cloud_dir, 'instance')
+        self.boot_finished = os.path.join(self.instance_link, "boot-finished")
+        self.upstart_conf_d = sys_info.get('upstart_dir')
+        template_dir = sys_info.get('templates_dir', '/etc/cloud/templates/')
+        self.template_tpl = os.path.join(template_dir, '%s.tmpl')
+        self.seed_dir = os.path.join(self.cloud_dir, 'seed')
+        self.lookups = {
+           "handlers": "handlers",
+           "scripts": "scripts",
+           "sem": "sem",
+           "boothooks": "boothooks",
+           "userdata_raw": "user-data.txt",
+           "userdata": "user-data.txt.i",
+           "obj_pkl": "obj.pkl",
+           "cloud_config": "cloud-config.txt",
+           "data": "data",
+        }
+        # Set when a datasource becomes active
+        self.datasource = None
+
+    # get_ipath_cur: get the current instance path for an item
+    def get_ipath_cur(self, name=None):
+        ipath = self.instance_link
+        add_on = self.lookups.get(name)
+        if add_on:
+            ipath = os.path.join(ipath, add_on)
+        return ipath
+
+    # get_cpath : get the "clouddir" (/var/lib/cloud/<name>)
+    # for a name in dirmap
+    def get_cpath(self, name=None):
+        cpath = self.cloud_dir
+        add_on = self.lookups.get(name)
+        if add_on:
+            cpath = os.path.join(cpath, add_on)
+        return cpath
+
+    def _get_ipath(self, name=None):
+        if not self.datasource:
+            return None
+        iid = self.datasource.get_instance_id()
+        if iid is None:
+            return None
+        ipath = os.path.join(self.cloud_dir, 'instances', iid)
+        add_on = self.lookups.get(name)
+        if add_on:
+            ipath = os.path.join(ipath, add_on)
+        return ipath
+
+    # (/var/lib/cloud/instances/<instance>/<name>)
+    def get_ipath(self, name=None):
+        ipath = self._get_ipath(name)
+        if not ipath:
+            LOG.warn(("No per instance semaphores available, "
+                              "is there an datasource/iid set?"))
+            return None
+        else:
+            return ipath
-- 
cgit v1.2.3


From 00399cd672e14b7e6953954eaf126f0ccfe35ecd Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Mon, 11 Jun 2012 18:01:59 -0700
Subject: Fix log statement.

---
 cloudinit/helpers.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 0bd13c78..16548001 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -260,8 +260,8 @@ class Paths(object):
     def get_ipath(self, name=None):
         ipath = self._get_ipath(name)
         if not ipath:
-            LOG.warn(("No per instance semaphores available, "
-                              "is there an datasource/iid set?"))
+            LOG.warn(("No per instance data available, "
+                      "is there an datasource/iid set?"))
             return None
         else:
             return ipath
-- 
cgit v1.2.3


From 1f120980a3217289f4bad02b813053664732e4bf Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Mon, 11 Jun 2012 20:11:44 -0700
Subject: Rename sem class to file sem class, add clear_all methods, add empty
 methods to dummy sem class.

---
 cloudinit/helpers.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 16548001..e5f33a26 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -49,8 +49,14 @@ class DummySemaphores(object):
     def has_run(self, _name, _freq):
         return False
 
+    def clear(self, _name, _freq):
+        return True
+
+    def clear_all(self):
+        pass
+
 
-class Semaphores(object):
+class FileSemaphores(object):
     def __init__(self, sem_path):
         self.sem_path = sem_path
 
@@ -71,6 +77,12 @@ class Semaphores(object):
             return False
         return True
 
+    def clear_all(self):
+        try:
+            util.del_dir(self.sem_path)
+        except (IOError, OSError):
+            pass
+
     def _acquire(self, name, freq):
         if self.has_run(name, freq):
             return None
@@ -117,7 +129,7 @@ class Runners(object):
         if not sem_path:
             return None
         if sem_path not in self.sems:
-            self.sems[sem_path] = Semaphores(sem_path)
+            self.sems[sem_path] = FileSemaphores(sem_path)
         return self.sems[sem_path]
 
     def run(self, name, functor, args, freq=None, clear_on_fail=False):
-- 
cgit v1.2.3


From 52d8a9617da4cbdb167d8cf4d470967192d17001 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Fri, 15 Jun 2012 17:51:27 -0700
Subject: 1. Handle returning values from handlers better. 2. Throw lock
 specific exception when semaphores can't be obtained 3. Make the paths
 constructor config variable named better to its actual usage.

---
 cloudinit/helpers.py | 39 ++++++++++++++++++++++++++-------------
 1 file changed, 26 insertions(+), 13 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index e5f33a26..2ecda3e9 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -38,12 +38,16 @@ from cloudinit.user_data import upstart_job as up_part
 LOG = logging.getLogger(__name__)
 
 
+class LockFailure(Exception):
+    pass
+
+
 class DummySemaphores(object):
     def __init__(self):
         pass
 
     @contextlib.contextmanager
-    def lock(self, _name, _freq, _clear_on_fail):
+    def lock(self, _name, _freq, _clear_on_fail=False):
         yield True
 
     def has_run(self, _name, _freq):
@@ -61,7 +65,7 @@ class FileSemaphores(object):
         self.sem_path = sem_path
 
     @contextlib.contextmanager
-    def lock(self, name, freq, clear_on_fail):
+    def lock(self, name, freq, clear_on_fail=False):
         try:
             yield self._acquire(name, freq)
         except:
@@ -73,15 +77,17 @@ class FileSemaphores(object):
         sem_file = self._get_path(name, freq)
         try:
             util.del_file(sem_file)
-        except (IOError, OSError):
+        except (IOError, OSError) as e:
+            util.logexc(LOG, "Failed deleting semaphore %s", sem_file)
             return False
         return True
 
     def clear_all(self):
         try:
             util.del_dir(self.sem_path)
-        except (IOError, OSError):
-            pass
+        except (IOError, OSError) as e:
+            LOG.debug("Failed deleting semaphore directory %s due to %s",
+                      self.sem_path, e)
 
     def _acquire(self, name, freq):
         if self.has_run(name, freq):
@@ -93,7 +99,8 @@ class FileSemaphores(object):
         contents = "%s: %s\n" % (os.getpid(), time())
         try:
             util.write_file(sem_file, contents)
-        except (IOError, OSError):
+        except (IOError, OSError) as e:
+            util.logexc(LOG, "Failed writing semaphore file %s", sem_file)
             return None
         return sem_file
 
@@ -143,11 +150,14 @@ class Runners(object):
             return None
         with sem.lock(name, freq, clear_on_fail) as lk:
             if not lk:
-                raise RuntimeError("Failed to acquire lock on %s" % name)
+                raise LockFailure("Failed to acquire lock for %s" % name)
             else:
                 LOG.debug("Running %s with args %s using lock %s",
                           functor, args, lk)
-                return functor(*args)
+                if isinstance(args, (dict)):
+                    return functor(**args)
+                else:
+                    return functor(*args)
 
 
 class ContentHandlers(object):
@@ -217,12 +227,12 @@ class ContentHandlers(object):
 
 
 class Paths(object):
-    def __init__(self, sys_info):
-        self.cloud_dir = sys_info.get('cloud_dir', '/var/lib/cloud')
+    def __init__(self, path_cfgs, ds=None):
+        self.cloud_dir = path_cfgs.get('cloud_dir', '/var/lib/cloud')
         self.instance_link = os.path.join(self.cloud_dir, 'instance')
         self.boot_finished = os.path.join(self.instance_link, "boot-finished")
-        self.upstart_conf_d = sys_info.get('upstart_dir')
-        template_dir = sys_info.get('templates_dir', '/etc/cloud/templates/')
+        self.upstart_conf_d = path_cfgs.get('upstart_dir')
+        template_dir = path_cfgs.get('templates_dir', '/etc/cloud/templates/')
         self.template_tpl = os.path.join(template_dir, '%s.tmpl')
         self.seed_dir = os.path.join(self.cloud_dir, 'seed')
         self.lookups = {
@@ -237,7 +247,7 @@ class Paths(object):
            "data": "data",
         }
         # Set when a datasource becomes active
-        self.datasource = None
+        self.datasource = ds
 
     # get_ipath_cur: get the current instance path for an item
     def get_ipath_cur(self, name=None):
@@ -256,6 +266,8 @@ class Paths(object):
             cpath = os.path.join(cpath, add_on)
         return cpath
 
+    # get_ipath : get the instance path for a name in pathmap
+    # (/var/lib/cloud/instances/<instance>/<name>)
     def _get_ipath(self, name=None):
         if not self.datasource:
             return None
@@ -268,6 +280,7 @@ class Paths(object):
             ipath = os.path.join(ipath, add_on)
         return ipath
 
+    # get_ipath : get the instance path for a name in pathmap
     # (/var/lib/cloud/instances/<instance>/<name>)
     def get_ipath(self, name=None):
         ipath = self._get_ipath(name)
-- 
cgit v1.2.3


From 450261a1fcf1f8929a2f7a25c2c278ba40689289 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Fri, 15 Jun 2012 21:33:55 -0700
Subject: Fixups to ensure that pylint does not find anything major wrong.

---
 cloudinit/cloud.py                              |  2 -
 cloudinit/distros/__init__.py                   | 21 +++---
 cloudinit/distros/ubuntu.py                     | 23 +++---
 cloudinit/handlers/boot_hook.py                 | 14 ++--
 cloudinit/handlers/cloud_config.py              |  4 +-
 cloudinit/handlers/shell_script.py              |  5 +-
 cloudinit/handlers/upstart_job.py               |  7 +-
 cloudinit/helpers.py                            | 41 ++---------
 cloudinit/log.py                                |  2 +-
 cloudinit/sources/DataSourceCloudStack.py       |  2 +-
 cloudinit/sources/DataSourceConfigDrive.py      |  4 +-
 cloudinit/sources/DataSourceMAAS.py             | 14 ++--
 cloudinit/sources/DataSourceNoCloud.py          |  3 +-
 cloudinit/sources/DataSourceOVF.py              | 12 +--
 cloudinit/sources/__init__.py                   | 23 +++---
 cloudinit/stages.py                             | 98 ++++++++++++++++---------
 cloudinit/transforms/__init__.py                |  2 +-
 cloudinit/transforms/cc_apt_update_upgrade.py   | 12 ++-
 cloudinit/transforms/cc_bootcmd.py              |  8 +-
 cloudinit/transforms/cc_byobu.py                |  2 +-
 cloudinit/transforms/cc_ca_certs.py             | 13 +++-
 cloudinit/transforms/cc_chef.py                 |  3 +-
 cloudinit/transforms/cc_disable_ec2_metadata.py |  3 +-
 cloudinit/transforms/cc_final_message.py        |  4 +-
 cloudinit/transforms/cc_foo.py                  |  6 +-
 cloudinit/transforms/cc_grub_dpkg.py            |  4 +-
 cloudinit/transforms/cc_keys_to_console.py      | 23 ++++--
 cloudinit/transforms/cc_landscape.py            |  3 +-
 cloudinit/transforms/cc_locale.py               |  2 +-
 cloudinit/transforms/cc_mcollective.py          |  9 +--
 cloudinit/transforms/cc_mounts.py               | 15 ++--
 cloudinit/transforms/cc_phone_home.py           | 36 ++++++---
 cloudinit/transforms/cc_puppet.py               | 17 +++--
 cloudinit/transforms/cc_resizefs.py             | 38 ++++++----
 cloudinit/transforms/cc_rightscale_userdata.py  | 16 ++--
 cloudinit/transforms/cc_rsyslog.py              | 14 ++--
 cloudinit/transforms/cc_runcmd.py               |  9 ++-
 cloudinit/transforms/cc_salt_minion.py          |  7 +-
 cloudinit/transforms/cc_scripts_per_boot.py     |  5 +-
 cloudinit/transforms/cc_scripts_per_instance.py |  5 +-
 cloudinit/transforms/cc_scripts_per_once.py     |  5 +-
 cloudinit/transforms/cc_scripts_user.py         |  9 ++-
 cloudinit/transforms/cc_set_hostname.py         |  2 +-
 cloudinit/transforms/cc_set_passwords.py        | 10 ++-
 cloudinit/transforms/cc_ssh.py                  | 22 +++---
 cloudinit/transforms/cc_ssh_import_id.py        |  8 +-
 cloudinit/transforms/cc_timezone.py             |  8 +-
 cloudinit/transforms/cc_update_etc_hosts.py     | 22 ++++--
 cloudinit/transforms/cc_update_hostname.py      |  4 +-
 cloudinit/transforms/cc_welcome.py              |  6 +-
 cloudinit/user_data.py                          | 13 ++--
 cloudinit/util.py                               | 48 ++++++------
 52 files changed, 388 insertions(+), 300 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py
index b2dfc749..8372d123 100644
--- a/cloudinit/cloud.py
+++ b/cloudinit/cloud.py
@@ -23,8 +23,6 @@
 import copy
 import os
 
-from cloudinit import distros
-from cloudinit import helpers
 from cloudinit import log as logging
 
 LOG = logging.getLogger(__name__)
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 90607668..fd4c70c1 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -20,29 +20,32 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from StringIO import StringIO
+
 import abc
 import copy
 
 from cloudinit import importer
+from cloudinit import log as logging
 from cloudinit import util
 
-from StringIO import StringIO
-
 # TODO: Make this via config??
 IFACE_ACTIONS = {
     'up': ['ifup', '--all'],
     'down': ['ifdown', '--all'],
 }
 
+LOG = logging.getLogger(__name__)
+
 
 class Distro(object):
 
     __metaclass__ = abc.ABCMeta
 
-    def __init__(self, cfg, runner):
+    def __init__(self, name, cfg, runner):
         self._runner = runner
-        self._cfg = util.get_cfg_by_path(cfg, ('system_info', ), {})
-        self.name = self._cfg.pop("distro", 'generic')
+        self._cfg = cfg
+        self.name = name
 
     @abc.abstractmethod
     def install_packages(self, pkglist):
@@ -135,10 +138,9 @@ class Distro(object):
                      action, cmd)
             (_out, err) = util.subp(cmd)
             if len(err):
-                LOG.warn("Running %s resulted in stderr output: %s",
-                         IF_UP_CMD, err)
+                LOG.warn("Running %s resulted in stderr output: %s", cmd, err)
             return True
-        except util.ProcessExecutionError as exc:
+        except util.ProcessExecutionError:
             util.logexc(LOG, "Running %s failed", cmd)
             return False
 
@@ -152,7 +154,8 @@ def fetch(distro_name, mods=(__name__, )):
         except RuntimeError:
             pass
     if not mod:
-        raise RuntimeError("No distribution found for distro %s" % (distro_name))
+        raise RuntimeError("No distribution found for distro %s"
+                           % (distro_name))
     distro_cls = getattr(mod, 'Distro')
     return distro_cls
     
diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py
index 6b0aff47..9252a1c4 100644
--- a/cloudinit/distros/ubuntu.py
+++ b/cloudinit/distros/ubuntu.py
@@ -20,17 +20,13 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from StringIO import StringIO
-
 import os
-import socket
 
 from cloudinit import distros
 from cloudinit import log as logging
-from cloudinit import templater
 from cloudinit import util
 
-from cloudinit.settings import (PER_INSTANCE)
+from cloudinit.settings import PER_INSTANCE
 
 LOG = logging.getLogger(__name__)
 
@@ -65,9 +61,11 @@ class Distro(distros.Distro):
             try:
                 util.write_file(fn, "%s\n" % hostname, 0644)
             except:
-                util.logexc(LOG, "Failed to write hostname %s to %s", hostname, fn)
-        if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev:
-             LOG.debug(("%s differs from /etc/hostname."
+                util.logexc(LOG, "Failed to write hostname %s to %s",
+                            hostname, fn)
+        if (hostname_in_etc and hostname_prev and
+            hostname_in_etc != hostname_prev):
+            LOG.debug(("%s differs from /etc/hostname."
                         " Assuming user maintained hostname."), prev_file)
         if "/etc/hostname" in update_files:
             LOG.debug("Setting hostname to %s", hostname)
@@ -91,7 +89,8 @@ class Distro(distros.Distro):
     def set_timezone(self, tz):
         tz_file = os.path.join("/usr/share/zoneinfo", tz)
         if not os.path.isfile(tz_file):
-            raise Exception("Invalid timezone %s, no file found at %s" % (tz, tz_file))
+            raise Exception(("Invalid timezone %s,"
+                             " no file found at %s") % (tz, tz_file))
         tz_contents = "%s\n" % tz
         util.write_file("/etc/timezone", tz_contents)
         # TODO, this should be in a rhel distro subclass??
@@ -101,9 +100,6 @@ class Distro(distros.Distro):
         # This ensures that the correct tz will be used for the system
         util.copy(tz_file, "/etc/localtime")
 
-    def name(self):
-        return "ubuntu"
-
     # apt_get top level command (install, update...), and args to pass it
     def _apt_get(self, tlc, args=None):
         e = os.environ.copy()
@@ -116,4 +112,5 @@ class Distro(distros.Distro):
         util.subp(cmd, env=e, capture=False)
 
     def _update_package_sources(self):
-        self.runner.run("update-sources", self._apt_get, ["update"], freq=PER_INSTANCE)
\ No newline at end of file
+        self._runner.run("update-sources", self._apt_get,
+                         ["update"], freq=PER_INSTANCE)
\ No newline at end of file
diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py
index c75aeb72..b3aab366 100644
--- a/cloudinit/handlers/boot_hook.py
+++ b/cloudinit/handlers/boot_hook.py
@@ -32,9 +32,9 @@ LOG = logging.getLogger(__name__)
 
 
 class BootHookPartHandler(ud.PartHandler):
-    def __init__(self, boothook_dir, instance_id):
+    def __init__(self, paths, instance_id, **_kwargs):
         ud.PartHandler.__init__(self, PER_ALWAYS)
-        self.boothook_dir = boothook_dir
+        self.boothook_dir = paths.get_ipath("boothooks")
         self.instance_id = instance_id
 
     def list_types(self):
@@ -54,13 +54,15 @@ class BootHookPartHandler(ud.PartHandler):
             start = len(prefix) + 1
 
         filepath = os.path.join(self.boothook_dir, filename)
-        util.write_file(filepath, payload[start:], 0700)
+        contents = payload[start:]
+        util.write_file(filepath, contents, 0700)
         try:
             env = os.environ.copy()
-            env['INSTANCE_ID'] = str(self.instance_id)
+            if self.instance_id:
+                env['INSTANCE_ID'] = str(self.instance_id)
             util.subp([filepath], env=env)
-        except util.ProcessExecutionError as e:
+        except util.ProcessExecutionError:
             util.logexc(LOG, "Boothooks script %s execution error", filepath)
-        except Exception as e:
+        except Exception:
             util.logexc(LOG, ("Boothooks unknown "
                               "error when running %s"), filepath)
diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py
index f0e88eeb..12d1bd96 100644
--- a/cloudinit/handlers/cloud_config.py
+++ b/cloudinit/handlers/cloud_config.py
@@ -30,10 +30,10 @@ LOG = logging.getLogger(__name__)
 
 
 class CloudConfigPartHandler(ud.PartHandler):
-    def __init__(self, cloud_fn):
+    def __init__(self, paths, **_kwargs):
         ud.PartHandler.__init__(self, PER_ALWAYS)
         self.cloud_buf = []
-        self.cloud_fn = cloud_fn
+        self.cloud_fn = paths.get_ipath("cloud_config")
 
     def list_types(self):
         return [
diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py
index 564e4623..f6e2ef16 100644
--- a/cloudinit/handlers/shell_script.py
+++ b/cloudinit/handlers/shell_script.py
@@ -32,10 +32,9 @@ LOG = logging.getLogger(__name__)
 
 
 class ShellScriptPartHandler(ud.PartHandler):
-
-    def __init__(self, script_dir):
+    def __init__(self, paths, **_kwargs):
         ud.PartHandler.__init__(self, PER_ALWAYS)
-        self.script_dir = script_dir
+        self.script_dir = paths.get_ipath_cur('scripts')
 
     def list_types(self):
         return [
diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py
index 568a644a..059a4851 100644
--- a/cloudinit/handlers/upstart_job.py
+++ b/cloudinit/handlers/upstart_job.py
@@ -33,9 +33,9 @@ LOG = logging.getLogger(__name__)
 
 
 class UpstartJobPartHandler(ud.PartHandler):
-    def __init__(self, upstart_dir):
+    def __init__(self, paths, **_kwargs):
         ud.PartHandler.__init__(self, PER_INSTANCE)
-        self.upstart_dir = upstart_dir
+        self.upstart_dir = paths.upstart_conf_d
 
     def list_types(self):
         return [
@@ -46,6 +46,9 @@ class UpstartJobPartHandler(ud.PartHandler):
         if ctype in ud.CONTENT_SIGNALS:
             return
 
+        if not self.upstart_dir:
+            return
+
         filename = util.clean_filename(filename)
         (_name, ext) = os.path.splitext(filename)
         if not ext:
diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 2ecda3e9..c276a54c 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -30,11 +30,6 @@ from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE)
 from cloudinit import log as logging
 from cloudinit import util
 
-from cloudinit.user_data import boot_hook as bh_part
-from cloudinit.user_data import cloud_config as cc_part
-from cloudinit.user_data import shell_script as ss_part
-from cloudinit.user_data import upstart_job as up_part
-
 LOG = logging.getLogger(__name__)
 
 
@@ -77,7 +72,7 @@ class FileSemaphores(object):
         sem_file = self._get_path(name, freq)
         try:
             util.del_file(sem_file)
-        except (IOError, OSError) as e:
+        except (IOError, OSError):
             util.logexc(LOG, "Failed deleting semaphore %s", sem_file)
             return False
         return True
@@ -99,7 +94,7 @@ class FileSemaphores(object):
         contents = "%s: %s\n" % (os.getpid(), time())
         try:
             util.write_file(sem_file, contents)
-        except (IOError, OSError) as e:
+        except (IOError, OSError):
             util.logexc(LOG, "Failed writing semaphore file %s", sem_file)
             return None
         return sem_file
@@ -162,9 +157,10 @@ class Runners(object):
 
 class ContentHandlers(object):
 
-    def __init__(self, paths):
+    def __init__(self, paths, iid=None):
         self.paths = paths
         self.registered = {}
+        self.iid = iid
 
     def __contains__(self, item):
         return self.is_registered(item)
@@ -191,34 +187,9 @@ class ContentHandlers(object):
     def iteritems(self):
         return self.registered.iteritems()
 
-    def _get_default_handlers(self):
-        def_handlers = []
-
-        cc_path = self.paths.get_ipath("cloud_config")
-        if cc_path:
-            cc_h = cc_part.CloudConfigPartHandler(cc_path)
-            def_handlers.append(cc_h)
-
-        sc_path = self.paths.get_ipath_cur('scripts')
-        if sc_path:
-            ss_h = ss_part.ShellScriptPartHandler(sc_path)
-            def_handlers.append(ss_h)
-
-        bh_path = self.paths.get_ipath("boothooks")
-        if bh_path:
-            bh_h = bh_part.BootHookPartHandler(bh_path)
-            def_handlers.append(bh_h)
-
-        upstart_pth = self.paths.upstart_conf_d
-        if upstart_pth:
-            up_h = up_part.UpstartJobPartHandler(upstart_pth)
-            def_handlers.append(up_h)
-
-        return def_handlers
-
-    def register_defaults(self):
+    def register_defaults(self, defs):
         registered = set()
-        for mod in self._get_default_handlers():
+        for mod in defs:
             for t in mod.list_types():
                 if not self.is_registered(t):
                     self.registered[t] = mod
diff --git a/cloudinit/log.py b/cloudinit/log.py
index c247eb9e..5fcb77ef 100644
--- a/cloudinit/log.py
+++ b/cloudinit/log.py
@@ -56,7 +56,7 @@ def setupBasicLogging():
         cfile = logging.FileHandler('/var/log/cloud-init.log')
         cfile.setFormatter(logging.Formatter(DEF_CON_FORMAT))
         cfile.setLevel(DEBUG)
-        root.addHandle(cfile)
+        root.addHandler(cfile)
     except (IOError, OSError):
         # Likely that u can't write to that file...
         # Make console now have DEBUG??
diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index 791df68f..27217e65 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -79,7 +79,7 @@ class DataSourceCloudStack(sources.DataSource):
             tot_time = (time.time() - start)
             LOG.debug("Crawl of metadata service took %s", int(tot_time))
             return True
-        except Exception as e:
+        except Exception:
             util.logexc(LOG, ('Failed fetching from metadata '
                               'service %s'), self.metadata_address)
             return False
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 176b62b0..7450572f 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -51,8 +51,8 @@ class DataSourceConfigDrive(sources.DataSource):
         self.seed_dir = os.path.join(paths.seed_dir, 'config_drive')
 
     def __str__(self):
-        mstr = "%s[%s]" % (util.obj_name(self), self.dsmode)
-        mstr += " [seed=%s]" % (self.seed)
+        mstr = "%s [%s]" % (util.obj_name(self), self.dsmode)
+        mstr += "[seed=%s]" % (self.seed)
         return mstr
 
     def get_data(self):
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index 27196265..9e639649 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -18,9 +18,9 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import os
 import errno
 import oauth.oauth as oauth
+import os
 import time
 import urllib2
 
@@ -48,7 +48,7 @@ class DataSourceMAAS(sources.DataSource):
         self.seed_dir = os.path.join(paths.seed_dir, 'maas')
 
     def __str__(self):
-        return "%s[%s]" % (util.obj_name(self), self.base_url)
+        return "%s [%s]" % (util.obj_name(self), self.base_url)
 
     def get_data(self):
         mcfg = self.ds_cfg
@@ -122,9 +122,10 @@ class DataSourceMAAS(sources.DataSource):
 
         starttime = time.time()
         check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION)
-        url = util.wait_for_url(urls=[check_url], max_wait=max_wait,
-                                timeout=timeout, status_cb=LOG.warn,
-                                headers_cb=self.md_headers)
+        urls = [check_url]
+        url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
+                                 timeout=timeout, status_cb=LOG.warn,
+                                 headers_cb=self.md_headers)
 
         if url:
             LOG.info("Using metadata source: '%s'", url)
@@ -185,7 +186,8 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
             headers = {}
         try:
             (resp, sc) = uhelp.readurl(url, headers=headers, timeout=timeout)
-            md[name] = resp
+            if uhelp.ok_http_code(sc):
+                md[name] = resp
         except urllib2.HTTPError as e:
             if e.code != 404:
                 raise
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 84d0f99d..2b016d1c 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -106,7 +106,8 @@ class DataSourceNoCloud(sources.DataSource):
                 if e.errno != errno.ENOENT:
                     raise
             except util.MountFailedError:
-                util.logexc(LOG, "Failed to mount %s when looking for seed", dev)
+                util.logexc(LOG, ("Failed to mount %s"
+                                  " when looking for data"), dev)
 
         # There was no indication on kernel cmdline or data
         # in the seeddir suggesting this handler should be used.
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index bb0f46c2..258d8d03 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -21,10 +21,10 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from xml.dom import minidom
+
 import base64
 import os
 import re
-import tempfile
 
 from cloudinit import log as logging
 from cloudinit import sources
@@ -51,7 +51,7 @@ class DataSourceOVF(sources.DataSource):
         ud = ""
 
         defaults = {
-            "instance-id": "iid-dsovf"
+            "instance-id": "iid-dsovf",
         }
 
         (seedfile, contents) = get_ovf_env(self.paths.seed_dir)
@@ -198,7 +198,7 @@ def transport_iso9660(require_iso=True):
     for dev in devs:
         fullp = os.path.join("/dev/", dev)
 
-        if (fullp in mounted or
+        if (fullp in mounts or
             not cdmatch.match(dev) or os.path.isdir(fullp)):
             continue
 
@@ -210,7 +210,8 @@ def transport_iso9660(require_iso=True):
             continue
 
         try:
-            (fname, contents) = utils.mount_cb(fullp, get_ovf_env, mtype="iso9660")
+            (fname, contents) = util.mount_cb(fullp,
+                                               get_ovf_env, mtype="iso9660")
         except util.MountFailedError:
             util.logexc(LOG, "Failed mounting %s", fullp)
             continue
@@ -265,7 +266,8 @@ def get_properties(contents):
         raise XmlError("No 'PropertySection's")
 
     props = {}
-    propElems = find_child(propSections[0], lambda n: n.localName == "Property")
+    propElems = find_child(propSections[0],
+                            (lambda n: n.localName == "Property"))
 
     for elem in propElems:
         key = elem.attributes.getNamedItemNS(envNsURI, "key").value
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 08669f5d..beb0f3d7 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -22,10 +22,9 @@
 
 from cloudinit import importer
 from cloudinit import log as logging
+from cloudinit import user_data as ud
 from cloudinit import util
 
-from cloudinit.user_data import processor as ud_proc
-
 DEP_FILESYSTEM = "FILESYSTEM"
 DEP_NETWORK = "NETWORK"
 DS_PREFIX = 'DataSource'
@@ -42,7 +41,6 @@ class DataSource(object):
         self.sys_cfg = sys_cfg
         self.distro = distro
         self.paths = paths
-        self.userdata_proc = ud_proc.UserDataProcessor(paths)
         self.userdata = None
         self.metadata = None
         self.userdata_raw = None
@@ -55,7 +53,7 @@ class DataSource(object):
     def get_userdata(self):
         if self.userdata is None:
             raw_data = self.get_userdata_raw()
-            self.userdata = self.userdata_proc.process(raw_data)
+            self.userdata = ud.UserDataProcessor(self.paths).process(raw_data)
         return self.userdata
 
     def get_userdata_raw(self):
@@ -73,7 +71,7 @@ class DataSource(object):
         if not self.metadata or 'public-keys' not in self.metadata:
             return keys
 
-        if isinstance(self.metadata['public-keys'], (str)):
+        if isinstance(self.metadata['public-keys'], (basestring, str)):
             return str(self.metadata['public-keys']).splitlines()
 
         if isinstance(self.metadata['public-keys'], (list, set)):
@@ -84,11 +82,12 @@ class DataSource(object):
                 # lp:506332 uec metadata service responds with
                 # data that makes boto populate a string for 'klist' rather
                 # than a list.
-                if isinstance(klist, (str)):
+                if isinstance(klist, (str, basestring)):
                     klist = [klist]
                 if isinstance(klist, (list, set)):
                     for pkey in klist:
-                        # there is an empty string at the end of the keylist, trim it
+                        # There is an empty string at
+                        # the end of the keylist, trim it
                         if pkey:
                             keys.append(pkey)
 
@@ -159,13 +158,14 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list):
     ds_list = list_sources(cfg_list, ds_deps, pkg_list)
     ds_names = [util.obj_name(f) for f in ds_list]
     LOG.info("Searching for data source in: %s", ds_names)
+
     for cls in ds_list:
         ds = util.obj_name(cls)
         try:
             s = cls(distro, sys_cfg, paths)
             if s.get_data():
                 return (s, ds)
-        except Exception as e:
+        except Exception:
             util.logexc(LOG, "Getting data from %s failed", ds)
 
     msg = "Did not find any data source, searched classes: %s" % (ds_names)
@@ -178,7 +178,8 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list):
 # return an ordered list of classes that match
 def list_sources(cfg_list, depends, pkg_list):
     src_list = []
-    LOG.info("Looking for for data source in: %s, %s that match %s", cfg_list, pkg_list, depends)
+    LOG.info(("Looking for for data source in: %s,"
+              " %s that matches %s"), cfg_list, pkg_list, depends)
     for ds_coll in cfg_list:
         ds_name = str(ds_coll)
         if not ds_name.startswith(DS_PREFIX):
@@ -201,8 +202,8 @@ def list_sources(cfg_list, depends, pkg_list):
             if not cls_matches:
                 continue
             src_list.extend(cls_matches)
-            LOG.debug("Found a match for data source %s in %s with matches %s", 
-                        ds_name, mod, cls_matches)
+            LOG.debug(("Found a match for data source %s"
+                       " in %s with matches %s"), ds_name, mod, cls_matches)
             break
     return src_list
 
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 2615d59f..b9076881 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -31,19 +31,23 @@ try:
 except ImportError:
     ConfigObj = None
 
-from cloudinit.settings import (PER_INSTANCE, FREQUENCIES)
 from cloudinit.settings import (OLD_CLOUD_CONFIG)
+from cloudinit.settings import (PER_INSTANCE, FREQUENCIES)
+
+from cloudinit.handlers import boot_hook as bh_part
+from cloudinit.handlers import cloud_config as cc_part
+from cloudinit.handlers import shell_script as ss_part
+from cloudinit.handlers import upstart_job as up_part
 
 from cloudinit import cloud
 from cloudinit import distros
-from cloudinit import modules
 from cloudinit import helpers
 from cloudinit import importer
 from cloudinit import log as logging
 from cloudinit import sources
-from cloudinit import util
-
+from cloudinit import transforms
 from cloudinit import user_data as ud
+from cloudinit import util
 
 LOG = logging.getLogger(__name__)
 
@@ -73,12 +77,19 @@ class Init(object):
     def distro(self):
         if not self._distro:
             d_cfg = util.get_cfg_by_path(self.cfg, ('system_info'), {})
+            # Ensure its a dictionary
+            if not isinstance(d_cfg, (dict)):
+                d_cfg = {}
             # Ensure not modified indirectly
             d_cfg = copy.deepcopy(d_cfg)
+            # Remove this since its path config, not distro config
             d_cfg.pop('paths', None)
-            distro_cls = distros.fetch(sys_cfg.pop('distro', 'ubuntu'))
+            # Try to find the right class to use
+            distro_name = d_cfg.pop('distro', 'ubuntu')
+            distro_cls = distros.fetch(distro_name)
             LOG.debug("Using distro class %s", distro_cls)
-            distro = distro_cls(d_cfg, helpers.Runners(self.paths))
+            distro = distro_cls(distro_name, d_cfg,
+                                helpers.Runners(self.paths))
             self._distro = distro
         return self._distro
 
@@ -93,7 +104,8 @@ class Init(object):
     @property
     def paths(self):
         if not self._paths:
-            path_info = util.get_cfg_by_path(self.cfg, ('system_info', 'paths'), {})
+            path_info = util.get_cfg_by_path(self.cfg,
+                                            ('system_info', 'paths'), {})
             # Ensure not modified indirectly
             path_info = copy.deepcopy(path_info)
             self._paths = helpers.Paths(path_info, self.datasource)
@@ -156,7 +168,7 @@ class Init(object):
             # by using the instance link, if purge_cache was called
             # the file wont exist
             return pickle.loads(util.load_file(pickled_fn))
-        except Exception as e:
+        except Exception:
             util.logexc(LOG, "Failed loading pickled datasource from %s",
                         pickled_fn)
             return None
@@ -166,7 +178,7 @@ class Init(object):
         try:
             contents = pickle.dumps(self.datasource)
             util.write_file(pickled_fn, contents, mode=0400)
-        except Exception as e:
+        except Exception:
             util.logexc(LOG, "Failed pickling datasource to %s", pickled_fn)
             return False
 
@@ -192,7 +204,8 @@ class Init(object):
             # (which will affect user-data handlers down the line...)
             sys_cfg = copy.deepcopy(self.cfg)
             ds_deps = copy.deepcopy(self.ds_deps)
-            (ds, dsname) = sources.find_source(sys_cfg, self.distro, self.paths,
+            (ds, dsname) = sources.find_source(sys_cfg, self.distro,
+                                               self.paths,
                                                ds_deps, cfg_list, pkg_list)
             LOG.debug("Loaded datasource %s - %s", dsname, ds)
         self.datasource = ds
@@ -270,6 +283,20 @@ class Init(object):
         processed_ud = "%s" % (self.datasource.get_userdata())
         util.write_file(self.paths.get_ipath('userdata'), processed_ud, 0600)
 
+    def _default_userdata_handlers(self):
+        opts = {
+            'paths': self.paths,
+            'instance_id': self.datasource.get_instance_id(),
+        }
+        # TODO Hmmm, should we dynamically import these??
+        def_handlers = [
+            cc_part.CloudConfigPartHandler(**opts),
+            ss_part.ShellScriptPartHandler(**opts),
+            bh_part.BootHookPartHandler(**opts),
+            up_part.UpstartJobPartHandler(**opts),
+        ]
+        return def_handlers
+
     def consume(self, frequency=PER_INSTANCE):
         cdir = self.paths.get_cpath("handlers")
         idir = self.paths.get_ipath("handlers")
@@ -279,8 +306,11 @@ class Init(object):
         sys.path.insert(0, cdir)
         sys.path.insert(0, idir)
 
+        # Ensure datasource fetched before activation (just incase)
+        ud_obj = self.datasource.get_userdata()
+
         # This keeps track of all the active handlers
-        c_handlers = helpers.ContentHandlers(self.paths)
+        c_handlers = helpers.ContentHandlers(paths=self.paths)
 
         # Add handlers in cdir
         potential_handlers = util.find_modules(cdir)
@@ -292,13 +322,10 @@ class Init(object):
             except:
                 util.logexc(LOG, "Failed to register handler from %s", fname)
 
-        def_handlers = c_handlers.register_defaults()
-        if def_handlers:
-            LOG.debug("Registered default handlers for %s", def_handlers)
-
-
-        # Ensure userdata fetched before activation (just incase)
-        ud_obj = self.datasource.get_userdata()
+        def_handlers = self._default_userdata_handlers()
+        applied_def_handlers = c_handlers.register_defaults(def_handlers)
+        if applied_def_handlers:
+            LOG.debug("Registered default handlers: %s", applied_def_handlers)
 
         # Form our cloud interface
         data = self.cloudify()
@@ -334,11 +361,11 @@ class Init(object):
 
 
 class Transforms(object):
-    def __init__(self, cloudobj, cfgfile=None):
-        self.datasource = cloudobj.datasource
+    def __init__(self, init, cfgfile=None):
+        self.datasource = init.fetch()
         self.cfgfile = cfgfile
-        self.basecfg = copy.deepcopy(cloudobj.cfg)
-        self.cloudobj = cloudobj
+        self.basecfg = copy.deepcopy(init.cfg)
+        self.init = init
         # Created on first use
         self._cachedcfg = None
 
@@ -409,25 +436,28 @@ class Transforms(object):
                                  (item, util.obj_name(item)))
         return module_list
 
-    def _transforms_modules(self, raw_mods):
+    def _fixup_transforms(self, raw_mods):
         mostly_mods = []
         for raw_mod in raw_mods:
             raw_name = raw_mod['mod']
             freq = raw_mod.get('freq')
             run_args = raw_mod.get('args') or []
-            mod_name = modules.form_module_name(raw_name)
+            mod_name = transforms.form_module_name(raw_name)
             if not mod_name:
                 continue
             if freq and freq not in FREQUENCIES:
-                LOG.warn("Config specified module %s has an unknown frequency %s", raw_name, freq)
+                LOG.warn(("Config specified transform %s"
+                          " has an unknown frequency %s"), raw_name, freq)
                 # Reset it so when ran it will get set to a known value
                 freq = None
-            mod = modules.fixup_module(importer.import_module(mod_name))
+            mod = transforms.fixup_module(importer.import_module(mod_name))
             mostly_mods.append([mod, raw_name, freq, run_args])
         return mostly_mods
 
     def _run_transforms(self, mostly_mods):
         failures = []
+        d_name = self.init.distro.name
+        c_cloud = self.init.cloudify()
         for (mod, name, freq, args) in mostly_mods:
             try:
                 # Try the modules frequency, otherwise fallback to a known one
@@ -436,17 +466,17 @@ class Transforms(object):
                 if not freq in FREQUENCIES:
                     freq = PER_INSTANCE
                 worked_distros = mod.distros
-                if worked_distros and self.cloud.distro.name() not in worked_distros:
-                    LOG.warn(("Module %s is verified on %s distros"
+                if (worked_distros and d_name not in worked_distros):
+                    LOG.warn(("Transform %s is verified on %s distros"
                               " but not on %s distro. It may or may not work"
-                              " correctly."), name, worked_distros,
-                              self.cloud.distro.name())
+                              " correctly."), name, worked_distros, d_name)
                 # Deep copy the config so that modules can't alter it
+                # Use the transforms logger and not our own
                 func_args = [name, copy.deepcopy(self.cfg),
-                             self.cloudobj, LOG, args]
-                # This name will affect the semphapore name created
+                             c_cloud, transforms.LOG, args]
+                # This name will affect the semaphore name created
                 run_name = "config-%s" % (name)
-                self.cloudobj.run(run_name, mod.handle, func_args, freq=freq)
+                c_cloud.run(run_name, mod.handle, func_args, freq=freq)
             except Exception as e:
                 util.logexc(LOG, "Running %s failed", mod)
                 failures.append((name, e))
@@ -454,5 +484,5 @@ class Transforms(object):
 
     def run(self, name):
         raw_mods = self._read_transforms(name)
-        mostly_mods = self._transforms_modules(raw_mods)
+        mostly_mods = self._fixup_transforms(raw_mods)
         return self._run_transforms(mostly_mods)
diff --git a/cloudinit/transforms/__init__.py b/cloudinit/transforms/__init__.py
index 8275b375..40affc4b 100644
--- a/cloudinit/transforms/__init__.py
+++ b/cloudinit/transforms/__init__.py
@@ -44,7 +44,7 @@ def fixup_module(mod, def_freq=PER_INSTANCE):
     else:
         freq = mod.frequency
         if freq and freq not in FREQUENCIES:
-            LOG.warn("Module %s has an unknown frequency %s", mod, freq)
+            LOG.warn("Transform %s has an unknown frequency %s", mod, freq)
     if not hasattr(mod, 'handle'):
         def empty_handle(_name, _cfg, _cloud, _log, _args):
             pass
diff --git a/cloudinit/transforms/cc_apt_update_upgrade.py b/cloudinit/transforms/cc_apt_update_upgrade.py
index c4a543ed..a4e058c6 100644
--- a/cloudinit/transforms/cc_apt_update_upgrade.py
+++ b/cloudinit/transforms/cc_apt_update_upgrade.py
@@ -71,7 +71,7 @@ def handle(_name, cfg, cloud, log, _args):
         except:
             util.logexc(log, "Failed to run debconf-set-selections")
 
-    pkglist = util.get_cfg_option_list_or_str(cfg, 'packages', [])
+    pkglist = util.get_cfg_option_list(cfg, 'packages', [])
 
     errors = []
     if update or len(pkglist) or upgrade:
@@ -96,7 +96,9 @@ def handle(_name, cfg, cloud, log, _args):
             errors.append(e)
 
     if len(errors):
-        raise errors[0]
+        log.warn("%s failed with exceptions, re-raising the last one",
+                 len(errors))
+        raise errors[-1]
 
 
 def mirror2lists_fileprefix(mirror):
@@ -186,7 +188,8 @@ def add_sources(srclist, template_params=None):
         try:
             util.write_file(ent['filename'], source + "\n", omode="ab")
         except:
-            errorlist.append([source, "failed write to file %s" % ent['filename']])
+            errorlist.append([source,
+                             "failed write to file %s" % ent['filename']])
 
     return errorlist
 
@@ -219,9 +222,10 @@ def find_apt_mirror(cloud, cfg):
             doms.extend((".localdomain", "",))
 
             mirror_list = []
+            distro = cloud.distro.name
             mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro)
             for post in doms:
-                mirror_list.append(mirrorfmt % post)
+                mirror_list.append(mirrorfmt % (post))
 
             mirror = util.search_for_mirror(mirror_list)
 
diff --git a/cloudinit/transforms/cc_bootcmd.py b/cloudinit/transforms/cc_bootcmd.py
index a2efad32..80afb5e7 100644
--- a/cloudinit/transforms/cc_bootcmd.py
+++ b/cloudinit/transforms/cc_bootcmd.py
@@ -30,7 +30,8 @@ frequency = PER_ALWAYS
 def handle(name, cfg, cloud, log, _args):
 
     if "bootcmd" not in cfg:
-        log.debug("Skipping module named %s,  no 'bootcomd' key in configuration", name)
+        log.debug(("Skipping transform named %s,"
+                   " no 'bootcomd' key in configuration"), name)
         return
 
     with tempfile.NamedTemporaryFile(suffix=".sh") as tmpf:
@@ -39,7 +40,7 @@ def handle(name, cfg, cloud, log, _args):
             tmpf.write(content)
             tmpf.flush()
         except:
-            log.warn("Failed to shellify bootcmd")
+            util.logexc(log, "Failed to shellify bootcmd")
             raise
 
         try:
@@ -48,5 +49,6 @@ def handle(name, cfg, cloud, log, _args):
             cmd = ['/bin/sh', tmpf.name]
             util.subp(cmd, env=env, capture=False)
         except:
-            log.warn("Failed to run commands from bootcmd")
+            util.logexc(log,
+                        ("Failed to run bootcmd transform %s"), name)
             raise
diff --git a/cloudinit/transforms/cc_byobu.py b/cloudinit/transforms/cc_byobu.py
index 38586174..741aa934 100644
--- a/cloudinit/transforms/cc_byobu.py
+++ b/cloudinit/transforms/cc_byobu.py
@@ -30,7 +30,7 @@ def handle(name, cfg, _cloud, log, args):
         value = util.get_cfg_option_str(cfg, "byobu_by_default", "")
 
     if not value:
-        log.debug("Skipping module named %s, no 'byobu' values found", name)
+        log.debug("Skipping transform named %s, no 'byobu' values found", name)
         return
 
     if value == "user" or value == "system":
diff --git a/cloudinit/transforms/cc_ca_certs.py b/cloudinit/transforms/cc_ca_certs.py
index 8ca9a200..e0802bfe 100644
--- a/cloudinit/transforms/cc_ca_certs.py
+++ b/cloudinit/transforms/cc_ca_certs.py
@@ -23,6 +23,8 @@ CA_CERT_FILENAME = "cloud-init-ca-certs.crt"
 CA_CERT_CONFIG = "/etc/ca-certificates.conf"
 CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/"
 
+distros = ['ubuntu']
+
 
 def update_ca_certs():
     """
@@ -70,22 +72,25 @@ def handle(name, cfg, _cloud, log, _args):
     """
     # If there isn't a ca-certs section in the configuration don't do anything
     if "ca-certs" not in cfg:
-        log.debug("Skipping module named %s, no 'ca-certs' key in configuration", name)
+        log.debug(("Skipping transform named %s,"
+                   " no 'ca-certs' key in configuration"), name)
         return
+
     ca_cert_cfg = cfg['ca-certs']
 
     # If there is a remove-defaults option set to true, remove the system
     # default trusted CA certs first.
     if ca_cert_cfg.get("remove-defaults", False):
-        log.debug("removing default certificates")
+        log.debug("Removing default certificates")
         remove_default_ca_certs()
 
     # If we are given any new trusted CA certs to add, add them.
     if "trusted" in ca_cert_cfg:
-        trusted_certs = util.get_cfg_option_list_or_str(ca_cert_cfg, "trusted")
+        trusted_certs = util.get_cfg_option_list(ca_cert_cfg, "trusted")
         if trusted_certs:
-            log.debug("adding %d certificates" % len(trusted_certs))
+            log.debug("Adding %d certificates" % len(trusted_certs))
             add_ca_certs(trusted_certs)
 
     # Update the system with the new cert configuration.
+    log.debug("Updating certificates")
     update_ca_certs()
diff --git a/cloudinit/transforms/cc_chef.py b/cloudinit/transforms/cc_chef.py
index 12c2f539..473e5f8b 100644
--- a/cloudinit/transforms/cc_chef.py
+++ b/cloudinit/transforms/cc_chef.py
@@ -31,7 +31,8 @@ def handle(name, cfg, cloud, log, _args):
 
     # If there isn't a chef key in the configuration don't do anything
     if 'chef' not in cfg:
-        log.debug("Skipping module named %s, no 'chef' key in configuration", name)
+        log.debug(("Skipping transform named %s,"
+                  " no 'chef' key in configuration"), name)
         return
     chef_cfg = cfg['chef']
 
diff --git a/cloudinit/transforms/cc_disable_ec2_metadata.py b/cloudinit/transforms/cc_disable_ec2_metadata.py
index 4d2a7f55..3c0dd57b 100644
--- a/cloudinit/transforms/cc_disable_ec2_metadata.py
+++ b/cloudinit/transforms/cc_disable_ec2_metadata.py
@@ -28,5 +28,6 @@ reject_cmd = ['route', 'add', '-host', '169.254.169.254', 'reject']
 
 
 def handle(_name, cfg, _cloud, _log, _args):
-    if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False):
+    disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False)
+    if disabled:
         util.subp(reject_cmd)
diff --git a/cloudinit/transforms/cc_final_message.py b/cloudinit/transforms/cc_final_message.py
index dc4ae34c..c257b6d0 100644
--- a/cloudinit/transforms/cc_final_message.py
+++ b/cloudinit/transforms/cc_final_message.py
@@ -32,7 +32,7 @@ final_message_def = ("Cloud-init v. {{version}} finished at {{timestamp}}."
                      " Up {{uptime}} seconds.")
 
 
-def handle(name, cfg, cloud, log, args):
+def handle(_name, cfg, cloud, log, args):
 
     msg_in = None
     if len(args) != 0:
@@ -60,7 +60,7 @@ def handle(name, cfg, cloud, log, args):
         # Use stdout, stderr or the logger??
         content = templater.render_string(msg_in, subs)
         sys.stderr.write("%s\n" % (content))
-    except Exception as e:
+    except Exception:
         util.logexc(log, "Failed to render final message template")
 
     boot_fin_fn = cloud.paths.boot_finished
diff --git a/cloudinit/transforms/cc_foo.py b/cloudinit/transforms/cc_foo.py
index 8007f981..99135704 100644
--- a/cloudinit/transforms/cc_foo.py
+++ b/cloudinit/transforms/cc_foo.py
@@ -45,8 +45,8 @@ from cloudinit.settings import PER_INSTANCE
 #    informational purposes. If non existent all distros are assumed and 
 #    no warning occurs.
 
-frequency = settings.PER_INSTANCE
+frequency = PER_INSTANCE
 
 
-def handle(name, _cfg, _cloud, _log, _args):
-    print("Hi from %s" % (name))
+def handle(name, _cfg, _cloud, log, _args):
+    log.debug("Hi from transform %s", name)
diff --git a/cloudinit/transforms/cc_grub_dpkg.py b/cloudinit/transforms/cc_grub_dpkg.py
index c048d5cc..02f05ce3 100644
--- a/cloudinit/transforms/cc_grub_dpkg.py
+++ b/cloudinit/transforms/cc_grub_dpkg.py
@@ -54,9 +54,9 @@ def handle(_name, cfg, _cloud, log, _args):
     # now idevs and idevs_empty are set to determined values
     # or, those set by user
 
-    dconf_sel = ("grub-pc grub-pc/install_devices string %s\n"
+    dconf_sel = (("grub-pc grub-pc/install_devices string %s\n"
                  "grub-pc grub-pc/install_devices_empty boolean %s\n") %
-                (idevs, idevs_empty)
+                 (idevs, idevs_empty))
 
     log.debug("Setting grub debconf-set-selections with '%s','%s'" %
         (idevs, idevs_empty))
diff --git a/cloudinit/transforms/cc_keys_to_console.py b/cloudinit/transforms/cc_keys_to_console.py
index 2f2a5297..e974375f 100644
--- a/cloudinit/transforms/cc_keys_to_console.py
+++ b/cloudinit/transforms/cc_keys_to_console.py
@@ -18,23 +18,34 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
+
 from cloudinit.settings import PER_INSTANCE
 from cloudinit import util
 
 frequency = PER_INSTANCE
 
+# This is a tool that cloud init provides
+helper_tool = '/usr/lib/cloud-init/write-ssh-key-fingerprints'
+
 
-def handle(_name, cfg, _cloud, log, _args):
-    cmd = ['/usr/lib/cloud-init/write-ssh-key-fingerprints']
-    fp_blacklist = util.get_cfg_option_list_or_str(cfg,
+def handle(name, cfg, _cloud, log, _args):
+    if not os.path.exists(helper_tool):
+        log.warn(("Unable to activate transform %s,"
+                  " helper tool not found at %s"), name, helper_tool)
+        return
+
+    fp_blacklist = util.get_cfg_option_list(cfg,
         "ssh_fp_console_blacklist", [])
-    key_blacklist = util.get_cfg_option_list_or_str(cfg,
+    key_blacklist = util.get_cfg_option_list(cfg,
         "ssh_key_console_blacklist", ["ssh-dss"])
+
     try:
+        cmd = [helper_tool]
         cmd.append(','.join(fp_blacklist))
         cmd.append(','.join(key_blacklist))
-        (stdout, stderr) = util.subp(cmd)
+        (stdout, _stderr) = util.subp(cmd)
         util.write_file('/dev/console', stdout)
     except:
-        log.warn("Writing keys to console failed!")
+        log.warn("Writing keys to /dev/console failed!")
         raise
diff --git a/cloudinit/transforms/cc_landscape.py b/cloudinit/transforms/cc_landscape.py
index 48491992..19948d0e 100644
--- a/cloudinit/transforms/cc_landscape.py
+++ b/cloudinit/transforms/cc_landscape.py
@@ -55,7 +55,8 @@ def handle(name, cfg, _cloud, log, _args):
     /etc/landscape/client.conf
     """
     if not ConfigObj:
-        log.warn("'ConfigObj' support not enabled, running %s disabled", name)
+        log.warn(("'ConfigObj' support not available,"
+                  " running transform %s disabled"), name)
         return
 
     ls_cloudcfg = cfg.get("landscape", {})
diff --git a/cloudinit/transforms/cc_locale.py b/cloudinit/transforms/cc_locale.py
index 3fb4c5d9..7f273123 100644
--- a/cloudinit/transforms/cc_locale.py
+++ b/cloudinit/transforms/cc_locale.py
@@ -49,7 +49,7 @@ def handle(name, cfg, cloud, log, args):
                                              "/etc/default/locale")
 
     if not locale:
-        log.debug(("Skipping module named %s, "
+        log.debug(("Skipping transform named %s, "
                    "no 'locale' configuration found"), name)
         return
 
diff --git a/cloudinit/transforms/cc_mcollective.py b/cloudinit/transforms/cc_mcollective.py
index aeeda9d2..5464fe8c 100644
--- a/cloudinit/transforms/cc_mcollective.py
+++ b/cloudinit/transforms/cc_mcollective.py
@@ -19,13 +19,10 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from ConfigParser import ConfigParser
 from StringIO import StringIO
 
-import os
-
+from cloudinit import cfg as config
 from cloudinit import util
-from cloudinit import cfg
 
 pubcert_file = "/etc/mcollective/ssl/server-public.pem"
 pricert_file = "/etc/mcollective/ssl/server-private.pem"
@@ -35,7 +32,7 @@ def handle(name, cfg, cloud, log, _args):
 
     # If there isn't a mcollective key in the configuration don't do anything
     if 'mcollective' not in cfg:
-        log.debug(("Skipping module named %s, "
+        log.debug(("Skipping transform named %s, "
                    "no 'mcollective' key in configuration"), name)
         return
 
@@ -47,7 +44,7 @@ def handle(name, cfg, cloud, log, _args):
     # ... and then update the mcollective configuration
     if 'conf' in mcollective_cfg:
         # Create object for reading server.cfg values
-        mcollective_config = cfg.DefaultingConfigParser()
+        mcollective_config = config.DefaultingConfigParser()
         # Read server.cfg values from original file in order to be able to mix
         # the rest up
         old_contents = util.load_file('/etc/mcollective/server.cfg')
diff --git a/cloudinit/transforms/cc_mounts.py b/cloudinit/transforms/cc_mounts.py
index babcbda1..44182b87 100644
--- a/cloudinit/transforms/cc_mounts.py
+++ b/cloudinit/transforms/cc_mounts.py
@@ -20,7 +20,6 @@
 
 from string import whitespace  # pylint: disable=W0402
 
-import os
 import re
 
 from cloudinit import util
@@ -28,7 +27,7 @@ from cloudinit import util
 # shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1
 shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$"
 shortname = re.compile(shortname_filter)
-ws = re.compile("[%s]+" % whitespace)
+ws = re.compile("[%s]+" % (whitespace))
 
 
 def is_mdname(name):
@@ -65,13 +64,14 @@ def handle(_name, cfg, cloud, log, _args):
             continue
 
         startname = str(cfgmnt[i][0])
-        LOG.debug("Attempting to determine the real name of %s", startname)
+        log.debug("Attempting to determine the real name of %s", startname)
 
         # workaround, allow user to specify 'ephemeral'
         # rather than more ec2 correct 'ephemeral0'
         if startname == "ephemeral":
             cfgmnt[i][0] = "ephemeral0"
-            log.debug("Adjusted mount option %s name from ephemeral to ephemeral0", (i + 1))
+            log.debug(("Adjusted mount option %s "
+                       "name from ephemeral to ephemeral0"), (i + 1))
 
         if is_mdname(startname):
             newname = cloud.device_name_to_device(startname)
@@ -136,7 +136,8 @@ def handle(_name, cfg, cloud, log, _args):
                 break
 
         if cfgmnt_has:
-            log.debug("Not including %s, already previously included", startname)
+            log.debug(("Not including %s, already"
+                       " previously included"), startname)
             continue
         cfgmnt.append(defmnt)
 
@@ -159,7 +160,7 @@ def handle(_name, cfg, cloud, log, _args):
     dirs = []
     for line in actlist:
         # write 'comment' in the fs_mntops, entry,  claiming this
-        line[3] = "%s,comment=cloudconfig" % line[3]
+        line[3] = "%s,%s" % (line[3], comment)
         if line[2] == "swap":
             needswap = True
         if line[1].startswith("/"):
@@ -168,7 +169,7 @@ def handle(_name, cfg, cloud, log, _args):
 
     fstab_lines = []
     fstab = util.load_file("/etc/fstab")
-    for line in fstab.read().splitlines():
+    for line in fstab.splitlines():
         try:
             toks = ws.split(line)
             if toks[3].find(comment) != -1:
diff --git a/cloudinit/transforms/cc_phone_home.py b/cloudinit/transforms/cc_phone_home.py
index 36af6dfa..98ff2b85 100644
--- a/cloudinit/transforms/cc_phone_home.py
+++ b/cloudinit/transforms/cc_phone_home.py
@@ -24,9 +24,8 @@ from cloudinit import util
 
 from cloudinit.settings import PER_INSTANCE
 
-from time import sleep
-
 frequency = PER_INSTANCE
+
 post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa',
                  'instance_id', 'hostname']
 
@@ -49,7 +48,7 @@ def handle(name, cfg, cloud, log, args):
         ph_cfg = cfg['phone_home']
 
     if 'url' not in ph_cfg:
-        log.warn(("Skipping module named %s, "
+        log.warn(("Skipping transform named %s, "
                   "no 'url' found in 'phone_home' configuration"), name)
         return
 
@@ -60,7 +59,8 @@ def handle(name, cfg, cloud, log, args):
         tries = int(tries)
     except:
         tries = 10
-        util.logexc(log, "Configuration entry 'tries' is not an integer, using %s", tries)
+        util.logexc(log, ("Configuration entry 'tries'"
+                          " is not an integer, using %s instead"), tries)
 
     if post_list == "all":
         post_list = post_list_all
@@ -75,23 +75,37 @@ def handle(name, cfg, cloud, log, args):
         'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub',
     }
 
-    for n, path in pubkeys.iteritems():
+    for (n, path) in pubkeys.iteritems():
         try:
             all_keys[n] = util.load_file(path)
         except:
-            util.logexc(log, "%s: failed to open, can not phone home that data", path)
+            util.logexc(log, ("%s: failed to open, can not"
+                              " phone home that data"), path)
 
     submit_keys = {}
     for k in post_list:
         if k in all_keys:
             submit_keys[k] = all_keys[k]
         else:
-            submit_keys[k] = "N/A"
-            log.warn("Requested key %s from 'post' configuration list not available", k)
+            submit_keys[k] = None
+            log.warn(("Requested key %s from 'post'"
+                      " configuration list not available"), k)
 
-    url = templater.render_string(url, {'INSTANCE_ID': all_keys['instance_id']})
+    # Get them read to be posted
+    real_submit_keys = {}
+    for (k, v) in submit_keys.iteritems():
+        if v is None:
+            real_submit_keys[k] = 'N/A'
+        else:
+            real_submit_keys[k] = str(v)
 
+    # Incase the url is parameterized
+    url_params = {
+        'INSTANCE_ID': all_keys['instance_id'],
+    }
+    url = templater.render_string(url, url_params)
     try:
-        uhelp.readurl(url, data=submit_keys, retries=tries, sec_between=3)
+        uhelp.readurl(url, data=real_submit_keys, retries=tries, sec_between=3)
     except:
-        util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries)
+        util.logexc(log, ("Failed to post phone home data to"
+                          " %s in %s tries"), url, tries)
diff --git a/cloudinit/transforms/cc_puppet.py b/cloudinit/transforms/cc_puppet.py
index 0a21a929..76cc9732 100644
--- a/cloudinit/transforms/cc_puppet.py
+++ b/cloudinit/transforms/cc_puppet.py
@@ -24,31 +24,32 @@ import os
 import pwd
 import socket
 
+from cloudinit import cfg as config
 from cloudinit import util
-from cloudinit import cfg
 
 
 def handle(name, cfg, cloud, log, _args):
     # If there isn't a puppet key in the configuration don't do anything
     if 'puppet' not in cfg:
-        log.debug(("Skipping module named %s,"
+        log.debug(("Skipping transform named %s,"
                    " no 'puppet' configuration found"), name)
         return
 
     puppet_cfg = cfg['puppet']
 
     # Start by installing the puppet package ...
-    cloud.distro.install_packages(("puppet",))
+    cloud.distro.install_packages(["puppet"])
 
     # ... and then update the puppet configuration
     if 'conf' in puppet_cfg:
         # Add all sections from the conf object to puppet.conf
         contents = util.load_file('/etc/puppet/puppet.conf')
         # Create object for reading puppet.conf values
-        puppet_config = cfg.DefaultingConfigParser()
+        puppet_config = config.DefaultingConfigParser()
         # Read puppet.conf values from original file in order to be able to
         # mix the rest up. First clean them up (TODO is this really needed??)
-        cleaned_contents = '\n'.join([i.lstrip() for i in contents.splitlines()])
+        cleaned_lines = [i.lstrip() for i in contents.splitlines()]
+        cleaned_contents = '\n'.join(cleaned_lines)
         puppet_config.readfp(StringIO(cleaned_contents),
                              filename='/etc/puppet/puppet.conf')
         for (cfg_name, cfg) in puppet_cfg['conf'].iteritems():
@@ -81,7 +82,8 @@ def handle(name, cfg, cloud, log, _args):
                     puppet_config.set(cfg_name, o, v)
             # We got all our config as wanted we'll rename
             # the previous puppet.conf and create our new one
-            util.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old')
+            util.rename('/etc/puppet/puppet.conf',
+                        '/etc/puppet/puppet.conf.old')
             contents = puppet_config.stringify()
             util.write_file('/etc/puppet/puppet.conf', contents)
 
@@ -91,7 +93,8 @@ def handle(name, cfg, cloud, log, _args):
                   '-e', 's/^START=.*/START=yes/',
                   '/etc/default/puppet'], capture=False)
     elif os.path.exists('/bin/systemctl'):
-        util.subp(['/bin/systemctl', 'enable', 'puppet.service'], capture=False)
+        util.subp(['/bin/systemctl', 'enable', 'puppet.service'],
+                  capture=False)
     elif os.path.exists('/sbin/chkconfig'):
         util.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False)
     else:
diff --git a/cloudinit/transforms/cc_resizefs.py b/cloudinit/transforms/cc_resizefs.py
index daaf4da9..fe012417 100644
--- a/cloudinit/transforms/cc_resizefs.py
+++ b/cloudinit/transforms/cc_resizefs.py
@@ -18,11 +18,8 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import errno
 import os
 import stat
-import sys
-import tempfile
 import time
 
 from cloudinit import util
@@ -46,15 +43,18 @@ def nodeify_path(devpth, where, log):
         if util.is_container():
             log.debug("Inside container, ignoring mknod failure in resizefs")
             return
-        log.warn("Failed to make device node to resize %s at %s", where, devpth)
+        log.warn("Failed to make device node to resize %s at %s",
+                 where, devpth)
         raise
 
 
 def get_fs_type(st_dev, path, log):
     try:
-        fs_type = util.find_devs_with(tag='TYPE', oformat='value',
+        dev_entries = util.find_devs_with(tag='TYPE', oformat='value',
                                          no_cache=True, path=path)
-        return fs_type
+        if not dev_entries:
+            return None
+        return dev_entries[0].strip()
     except util.ProcessExecutionError:
         util.logexc(log, ("Failed to get filesystem type"
                           " of maj=%s, min=%s for path %s"),
@@ -69,12 +69,16 @@ def handle(name, cfg, _cloud, log, args):
         resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True)
 
     if not util.translate_bool(resize_root):
-        log.debug("Skipping module named %s,  resizing disabled", name)
+        log.debug("Skipping transform named %s, resizing disabled", name)
         return
 
     # TODO is the directory ok to be used??
     resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run")
     util.ensure_dir(resize_root_d)
+
+    # TODO: allow what is to be resized to
+    # be configurable??
+    resize_what = "/"
     with util.SilentTemporaryFile(prefix="cloudinit.resizefs.",
                                   dir=resize_root_d, delete=True) as tfh:
         devpth = tfh.name
@@ -86,23 +90,25 @@ def handle(name, cfg, _cloud, log, args):
         # auto deletion
         tfh.unlink_now()
     
-        # TODO: allow what is to be resized to
-        # be configurable??
-        st_dev = nodeify_path(devpth, "/", log)
-        fs_type = get_fs_type(st_dev, devpath, log)
+        st_dev = nodeify_path(devpth, resize_what, log)
+        fs_type = get_fs_type(st_dev, devpth, log)
+        if not fs_type:
+            log.warn("Could not determine filesystem type of %s", resize_what)
+            return
     
         resizer = None
-        fstype_lc = fstype.lower()
+        fstype_lc = fs_type.lower()
         for (pfix, root_cmd) in resize_fs_prefixes_cmds:
             if fstype_lc.startswith(pfix):
                 resizer = root_cmd
                 break
 
         if not resizer:
-            log.warn("Not resizing unknown filesystem type %s", fs_type)
+            log.warn("Not resizing unknown filesystem type %s for %s",
+                     fs_type, resize_what)
             return
 
-        log.debug("Resizing using %s", resizer)
+        log.debug("Resizing %s (%s) using %s", resize_what, fs_type, resizer)
         resize_cmd = [resizer, devpth]
 
         if resize_root == "noblock":
@@ -125,8 +131,8 @@ def do_resize(resize_cmd, log):
     start = time.time()
     try:
         util.subp(resize_cmd)
-    except util.ProcessExecutionError as e:
-        util.logexc(log, "Failed to resize filesystem (using %s)", resize_cmd)
+    except util.ProcessExecutionError:
+        util.logexc(log, "Failed to resize filesystem (cmd=%s)", resize_cmd)
         raise
     tot_time = int(time.time() - start)
     log.debug("Resizing took %s seconds", tot_time)
diff --git a/cloudinit/transforms/cc_rightscale_userdata.py b/cloudinit/transforms/cc_rightscale_userdata.py
index cde11b54..40d76c89 100644
--- a/cloudinit/transforms/cc_rightscale_userdata.py
+++ b/cloudinit/transforms/cc_rightscale_userdata.py
@@ -53,16 +53,19 @@ def handle(name, _cfg, cloud, log, _args):
     try:
         ud = cloud.get_userdata_raw()
     except:
-        log.warn("Failed to get raw userdata in module %s", name)
+        log.warn("Failed to get raw userdata in transform %s", name)
         return
 
     try:
         mdict = parse_qs(ud)
         if not mdict or not my_hookname in mdict:
-            log.debug("Skipping module %s, did not find %s in parsed raw userdata", name, my_hookname)
+            log.debug(("Skipping transform %s, "
+                       "did not find %s in parsed"
+                       " raw userdata"), name, my_hookname)
             return
     except:
-        log.warn("Failed to parse query string %s into a dictionary", ud)
+        util.logexc(log, ("Failed to parse query string %s"
+                           " into a dictionary"), ud)
         raise
 
     wrote_fns = []
@@ -83,7 +86,8 @@ def handle(name, _cfg, cloud, log, _args):
                 wrote_fns.append(fname)
         except Exception as e:
             captured_excps.append(e)
-            util.logexc(log, "%s failed to read %s and write %s", my_name, url, fname)
+            util.logexc(log, "%s failed to read %s and write %s",
+                        my_name, url, fname)
 
     if wrote_fns:
         log.debug("Wrote out rightscale userdata to %s files", len(wrote_fns))
@@ -93,6 +97,6 @@ def handle(name, _cfg, cloud, log, _args):
         log.debug("%s urls were skipped or failed", skipped)
 
     if captured_excps:
-        log.warn("%s failed with exceptions, re-raising the last one", len(captured_excps))
+        log.warn("%s failed with exceptions, re-raising the last one",
+                 len(captured_excps))
         raise captured_excps[-1]
-
diff --git a/cloudinit/transforms/cc_rsyslog.py b/cloudinit/transforms/cc_rsyslog.py
index ccbe68ff..71b74711 100644
--- a/cloudinit/transforms/cc_rsyslog.py
+++ b/cloudinit/transforms/cc_rsyslog.py
@@ -36,7 +36,8 @@ def handle(name, cfg, cloud, log, _args):
 
     # process 'rsyslog'
     if not 'rsyslog' in cfg:
-        log.debug("Skipping module named %s, no 'rsyslog' key in configuration", name)
+        log.debug(("Skipping transform named %s,"
+                   " no 'rsyslog' key in configuration"), name)
         return
 
     def_dir = cfg.get('rsyslog_dir', DEF_DIR)
@@ -62,15 +63,16 @@ def handle(name, cfg, cloud, log, _args):
         if not filename.startswith("/"):
             filename = os.path.join(def_dir, filename)
 
+        # Truncate filename first time you see it
         omode = "ab"
-        # truncate filename first time you see it
         if filename not in files:
             omode = "wb"
             files.append(filename)
 
         try:
-            util.write_file(filename, content + "\n", omode=omode)
-        except Exception as e:
+            contents = "%s\n" % (content)
+            util.write_file(filename, contents, omode=omode)
+        except Exception:
             util.logexc(log, "Failed to write to %s", filename)
 
     # Attempt to restart syslogd
@@ -87,8 +89,8 @@ def handle(name, cfg, cloud, log, _args):
         log.debug("Restarting rsyslog")
         util.subp(['service', 'rsyslog', 'restart'])
         restarted = True
-    except Exception as e:
-        util.logexc("Failed restarting rsyslog")
+    except Exception:
+        util.logexc(log, "Failed restarting rsyslog")
 
     if restarted:
         # This only needs to run if we *actually* restarted
diff --git a/cloudinit/transforms/cc_runcmd.py b/cloudinit/transforms/cc_runcmd.py
index 19c0e721..31a254a5 100644
--- a/cloudinit/transforms/cc_runcmd.py
+++ b/cloudinit/transforms/cc_runcmd.py
@@ -25,13 +25,14 @@ from cloudinit import util
 
 def handle(name, cfg, cloud, log, _args):
     if "runcmd" not in cfg:
-        log.debug("Skipping module named %s, no 'runcmd' key in configuration", name)
+        log.debug(("Skipping transform named %s,"
+                   " no 'runcmd' key in configuration"), name)
         return
 
-    outfile = os.path.join(cloud.get_ipath('scripts'), "runcmd")
+    out_fn = os.path.join(cloud.get_ipath('scripts'), "runcmd")
     cmd = cfg["runcmd"]
     try:
         content = util.shellify(cmd)
-        util.write_file(outfile, content, 0700)
+        util.write_file(out_fn, content, 0700)
     except:
-        util.logexc(log, "Failed to shellify %s into file %s", cmd, outfile)
+        util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn)
diff --git a/cloudinit/transforms/cc_salt_minion.py b/cloudinit/transforms/cc_salt_minion.py
index 47cbc194..d05d2a1e 100644
--- a/cloudinit/transforms/cc_salt_minion.py
+++ b/cloudinit/transforms/cc_salt_minion.py
@@ -21,16 +21,17 @@ from cloudinit import util
 # Note: see http://saltstack.org/topics/installation/
 
 
-def handle(name, cfg, cloud, _log, _args):
+def handle(name, cfg, cloud, log, _args):
     # If there isn't a salt key in the configuration don't do anything
     if 'salt_minion' not in cfg:
-        log.debug("Skipping module named %s, no 'salt_minion' key in configuration", name)
+        log.debug(("Skipping transform named %s,"
+                   " no 'salt_minion' key in configuration"), name)
         return
 
     salt_cfg = cfg['salt_minion']
 
     # Start by installing the salt package ...
-    cloud.distro.install_packages(("salt",))
+    cloud.distro.install_packages(["salt"])
     
     # Ensure we can configure files at the right dir
     config_dir = salt_cfg.get("config_dir", '/etc/salt')
diff --git a/cloudinit/transforms/cc_scripts_per_boot.py b/cloudinit/transforms/cc_scripts_per_boot.py
index bcdf4400..364e1d02 100644
--- a/cloudinit/transforms/cc_scripts_per_boot.py
+++ b/cloudinit/transforms/cc_scripts_per_boot.py
@@ -29,12 +29,13 @@ frequency = PER_ALWAYS
 script_subdir = 'per-boot'
 
 
-def handle(_name, _cfg, cloud, log, _args):
+def handle(name, _cfg, cloud, log, _args):
     # Comes from the following:
     # https://forums.aws.amazon.com/thread.jspa?threadID=96918
     runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir)
     try:
         util.runparts(runparts_path)
     except:
-        log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path)
+        log.warn("Failed to run transform %s (%s in %s)",
+                 name, script_subdir, runparts_path)
         raise
diff --git a/cloudinit/transforms/cc_scripts_per_instance.py b/cloudinit/transforms/cc_scripts_per_instance.py
index 8d6609a1..d75ab47d 100644
--- a/cloudinit/transforms/cc_scripts_per_instance.py
+++ b/cloudinit/transforms/cc_scripts_per_instance.py
@@ -29,12 +29,13 @@ frequency = PER_INSTANCE
 script_subdir = 'per-instance'
 
 
-def handle(_name, _cfg, cloud, log, _args):
+def handle(name, _cfg, cloud, log, _args):
     # Comes from the following:
     # https://forums.aws.amazon.com/thread.jspa?threadID=96918
     runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir)
     try:
         util.runparts(runparts_path)
     except:
-        log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path)
+        log.warn("Failed to run transform %s (%s in %s)",
+                 name, script_subdir, runparts_path)
         raise
diff --git a/cloudinit/transforms/cc_scripts_per_once.py b/cloudinit/transforms/cc_scripts_per_once.py
index dbcec05d..80f8c325 100644
--- a/cloudinit/transforms/cc_scripts_per_once.py
+++ b/cloudinit/transforms/cc_scripts_per_once.py
@@ -29,12 +29,13 @@ frequency = PER_ONCE
 script_subdir = 'per-once'
 
 
-def handle(_name, _cfg, cloud, log, _args):
+def handle(name, _cfg, cloud, log, _args):
     # Comes from the following:
     # https://forums.aws.amazon.com/thread.jspa?threadID=96918
     runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir)
     try:
         util.runparts(runparts_path)
     except:
-        log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path)
+        log.warn("Failed to run transform %s (%s in %s)",
+                 name, script_subdir, runparts_path)
         raise
diff --git a/cloudinit/transforms/cc_scripts_user.py b/cloudinit/transforms/cc_scripts_user.py
index 1e438ee6..f4fe3a2a 100644
--- a/cloudinit/transforms/cc_scripts_user.py
+++ b/cloudinit/transforms/cc_scripts_user.py
@@ -26,14 +26,17 @@ from cloudinit.settings import PER_INSTANCE
 
 frequency = PER_INSTANCE
 
+script_subdir = 'scripts'
 
-def handle(_name, _cfg, cloud, log, _args):
+
+def handle(name, _cfg, cloud, log, _args):
     # This is written to by the user data handlers
     # Ie, any custom shell scripts that come down
     # go here...
-    runparts_path = os.path.join(cloud.get_ipath_cur(), "scripts")
+    runparts_path = os.path.join(cloud.get_ipath_cur(), script_subdir)
     try:
         util.runparts(runparts_path)
     except:
-        log.warn("Failed to run-parts(%s) in %s", "user-data", runparts_path)
+        log.warn("Failed to run transform %s (%s in %s)",
+                 name, script_subdir, runparts_path)
         raise
diff --git a/cloudinit/transforms/cc_set_hostname.py b/cloudinit/transforms/cc_set_hostname.py
index fa2b59c2..3ac8a8fa 100644
--- a/cloudinit/transforms/cc_set_hostname.py
+++ b/cloudinit/transforms/cc_set_hostname.py
@@ -24,7 +24,7 @@ from cloudinit import util
 def handle(name, cfg, cloud, log, _args):
     if util.get_cfg_option_bool(cfg, "preserve_hostname", False):
         log.debug(("Configuration option 'preserve_hostname' is set,"
-                    " not setting the hostname in %s"), name)
+                    " not setting the hostname in transform %s"), name)
         return
 
     (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud)
diff --git a/cloudinit/transforms/cc_set_passwords.py b/cloudinit/transforms/cc_set_passwords.py
index 4f2cdb97..c0cc4e84 100644
--- a/cloudinit/transforms/cc_set_passwords.py
+++ b/cloudinit/transforms/cc_set_passwords.py
@@ -22,7 +22,7 @@ import sys
 
 from cloudinit import util
 
-from string import letters, digits
+from string import letters, digits  # pylint: disable=W0402
 
 # We are removing certain 'painful' letters/numbers
 pw_set = (letters.translate(None, 'loLOI') +
@@ -71,11 +71,13 @@ def handle(_name, cfg, cloud, log, args):
             util.subp(['chpasswd'], ch_in)
         except Exception as e:
             errors.append(e)
-            util.logexc(log, "Failed to set passwords with chpasswd for %s", users)
+            util.logexc(log,
+                        "Failed to set passwords with chpasswd for %s", users)
 
         if len(randlist):
-            sys.stderr.write("%s\n%s\n" % ("Set the following 'random' passwords\n",
-                '\n'.join(randlist)))
+            blurb = ("Set the following 'random' passwords\n",
+                     '\n'.join(randlist))
+            sys.stderr.write("%s\n%s\n" % blurb)
 
         if expire:
             expired_users = []
diff --git a/cloudinit/transforms/cc_ssh.py b/cloudinit/transforms/cc_ssh.py
index db6848d9..3c2b3622 100644
--- a/cloudinit/transforms/cc_ssh.py
+++ b/cloudinit/transforms/cc_ssh.py
@@ -65,8 +65,7 @@ def handle(_name, cfg, cloud, log, _args):
                 tgt_fn = key2file[key][0]
                 tgt_perms = key2file[key][1]
                 util.write_file(tgt_fn, val, tgt_perms)
-    
-        cmd = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"'
+
         for priv, pub in priv2pub.iteritems():
             if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']:
                 continue
@@ -78,11 +77,15 @@ def handle(_name, cfg, cloud, log, _args):
                     util.subp(cmd, capture=False)
                 log.debug("Generated a key for %s from %s", pair[0], pair[1])
             except:
-                util.logexc(log, "Failed generated a key for %s from %s", pair[0], pair[1])
+                util.logexc(log, ("Failed generated a key"
+                                  " for %s from %s"), pair[0], pair[1])
     else:
         # if not, generate them
-        for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', generate_keys):
-            keyfile = '/etc/ssh/ssh_host_%s_key' % keytype
+        genkeys = util.get_cfg_option_list(cfg,
+                                            'ssh_genkeytypes', 
+                                            generate_keys)
+        for keytype in genkeys:
+            keyfile = '/etc/ssh/ssh_host_%s_key' % (keytype)
             if not os.path.exists(keyfile):
                 cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile]
                 try:
@@ -90,26 +93,27 @@ def handle(_name, cfg, cloud, log, _args):
                     with util.SeLinuxGuard("/etc/ssh", recursive=True):
                         util.subp(cmd, capture=False)
                 except:
-                    util.logexc(log, "Failed generating key type %s to file %s", keytype, keyfile)
+                    util.logexc(log, ("Failed generating key type"
+                                      " %s to file %s"), keytype, keyfile)
 
     try:
         user = util.get_cfg_option_str(cfg, 'user')
         disable_root = util.get_cfg_option_bool(cfg, "disable_root", True)
         disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts",
-            DISABLE_ROOT_OPTS)
+                                                    DISABLE_ROOT_OPTS)
 
         keys = cloud.get_public_ssh_keys() or []
         if "ssh_authorized_keys" in cfg:
             cfgkeys = cfg["ssh_authorized_keys"]
             keys.extend(cfgkeys)
 
-        apply_credentials(keys, user, disable_root, disable_root_opts, log)
+        apply_credentials(keys, user, disable_root, disable_root_opts)
     except:
         util.logexc(log, "Applying ssh credentials failed!")
 
 
 def apply_credentials(keys, user, disable_root,
-                      disable_root_opts=DISABLE_ROOT_OPTS, log=None):
+                      disable_root_opts=DISABLE_ROOT_OPTS):
 
     keys = set(keys)
     if user:
diff --git a/cloudinit/transforms/cc_ssh_import_id.py b/cloudinit/transforms/cc_ssh_import_id.py
index 019413d4..d57e4665 100644
--- a/cloudinit/transforms/cc_ssh_import_id.py
+++ b/cloudinit/transforms/cc_ssh_import_id.py
@@ -33,10 +33,14 @@ def handle(name, cfg, _cloud, log, args):
             ids = args[1:]
     else:
         user = util.get_cfg_option_str(cfg, "user", "ubuntu")
-        ids = util.get_cfg_option_list_or_str(cfg, "ssh_import_id", [])
+        ids = util.get_cfg_option_list(cfg, "ssh_import_id", [])
 
     if len(ids) == 0:
-        log.debug("Skipping module named %s, no ids found to import", name)
+        log.debug("Skipping transform named %s, no ids found to import", name)
+        return
+
+    if not user:
+        log.debug("Skipping transform named %s, no user found to import", name)
         return
 
     cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids
diff --git a/cloudinit/transforms/cc_timezone.py b/cloudinit/transforms/cc_timezone.py
index 02cbf2dc..747c436c 100644
--- a/cloudinit/transforms/cc_timezone.py
+++ b/cloudinit/transforms/cc_timezone.py
@@ -18,20 +18,22 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from cloudinit import util
+
 from cloudinit.settings import PER_INSTANCE
 
 frequency = PER_INSTANCE
 
 
-def handle(_name, cfg, cloud, log, args):
+def handle(name, cfg, cloud, log, args):
     if len(args) != 0:
         timezone = args[0]
     else:
         timezone = util.get_cfg_option_str(cfg, "timezone", False)
 
     if not timezone:
-        log.debug("Skipping module named %s, no 'timezone' specified", name)
+        log.debug("Skipping transform named %s, no 'timezone' specified", name)
         return
-    
+
     # Let the distro handle settings its timezone
     cloud.distro.set_timezone(timezone)
diff --git a/cloudinit/transforms/cc_update_etc_hosts.py b/cloudinit/transforms/cc_update_etc_hosts.py
index 361097a6..d0e56183 100644
--- a/cloudinit/transforms/cc_update_etc_hosts.py
+++ b/cloudinit/transforms/cc_update_etc_hosts.py
@@ -30,22 +30,30 @@ def handle(name, cfg, cloud, log, _args):
     manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False)
     if util.translate_bool(manage_hosts, addons=['template']):
         (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud)
-        # Render from template file
         if not hostname:
-            log.warn("Option 'manage_etc_hosts' was set, but no hostname was found")
+            log.warn(("Option 'manage_etc_hosts' was set,"
+                     " but no hostname was found"))
             return
-        tpl_fn_name = cloud.get_template_filename("hosts.%s" % (cloud.distro.name()))
+
+        # Render from a template file
+        distro_n = cloud.distro.name
+        tpl_fn_name = cloud.get_template_filename("hosts.%s" % (distro_n))
         if not tpl_fn_name:
-            raise Exception("No hosts template could be found for distro %s" % (cloud.distro.name()))
+            raise Exception(("No hosts template could be"
+                             " found for distro %s") % (distro_n))
+
         templater.render_to_file(tpl_fn_name, '/etc/hosts',
                                 {'hostname': hostname, 'fqdn': fqdn})
+
     elif manage_hosts == "localhost":
-        log.debug("Managing localhost in /etc/hosts")
         (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud)
         if not hostname:
-            log.warn("Option 'manage_etc_hosts' was set, but no hostname was found")
+            log.warn(("Option 'manage_etc_hosts' was set,"
+                     " but no hostname was found"))
             return
+
+        log.debug("Managing localhost in /etc/hosts")
         cloud.distro.update_etc_hosts(hostname, fqdn)
     else:
         log.debug(("Configuration option 'manage_etc_hosts' is not set,"
-                    " not managing /etc/hosts in %s"), name)
+                    " not managing /etc/hosts in transform %s"), name)
diff --git a/cloudinit/transforms/cc_update_hostname.py b/cloudinit/transforms/cc_update_hostname.py
index 439bdcb3..58444fab 100644
--- a/cloudinit/transforms/cc_update_hostname.py
+++ b/cloudinit/transforms/cc_update_hostname.py
@@ -18,6 +18,8 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
+
 from cloudinit import util
 from cloudinit.settings import PER_ALWAYS
 
@@ -27,7 +29,7 @@ frequency = PER_ALWAYS
 def handle(name, cfg, cloud, log, _args):
     if util.get_cfg_option_bool(cfg, "preserve_hostname", False):
         log.debug(("Configuration option 'preserve_hostname' is set,"
-                    " not updating the hostname in %s"), name)
+                    " not updating the hostname in transform %s"), name)
         return
 
     (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud)
diff --git a/cloudinit/transforms/cc_welcome.py b/cloudinit/transforms/cc_welcome.py
index 0db71125..04691d21 100644
--- a/cloudinit/transforms/cc_welcome.py
+++ b/cloudinit/transforms/cc_welcome.py
@@ -35,9 +35,9 @@ welcome_message_def = ("Cloud-init v. {{version}} starting stage {{stage}} at "
 frequency = PER_ALWAYS
 
 
-def handle(name, cfg, cloud, log, args):
+def handle(_name, cfg, cloud, log, args):
 
-    welcome_msg = util.get_cfg_option_str(cfg, "welcome_msg"):
+    welcome_msg = util.get_cfg_option_str(cfg, "welcome_msg")
     if not welcome_msg:
         tpl_fn = cloud.get_template_filename("welcome_msg")
         if tpl_fn:
@@ -54,7 +54,7 @@ def handle(name, cfg, cloud, log, args):
         'stage': stage,
         'version': version.version_string(),
         'uptime': util.uptime(),
-        'timestamp', util.time_rfc2822(),
+        'timestamp': util.time_rfc2822(),
     }
     try:
         contents = templater.render_string(welcome_msg, tpl_params)
diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index 64fc2734..9915b8b0 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -22,14 +22,15 @@
 
 
 import os
-import glob
 
 import email
-
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
 from email.mime.base import MIMEBase
 
 from cloudinit import importer
 from cloudinit import log as logging
+from cloudinit import url_helper
 from cloudinit import util
 
 from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, FREQUENCIES)
@@ -86,7 +87,7 @@ class UserDataProcessor(object):
         self.paths = paths
 
     def process(self, blob):
-        base_msg = ud.convert_string(blob)
+        base_msg = convert_string(blob)
         process_msg = MIMEMultipart()
         self._process_msg(base_msg, process_msg)
         return process_msg
@@ -105,7 +106,7 @@ class UserDataProcessor(object):
                 ctype_orig = UNDEF_TYPE
     
             if ctype_orig in TYPE_NEEDED:
-                ctype = ud.type_from_starts_with(payload)
+                ctype = type_from_starts_with(payload)
     
             if ctype is None:
                 ctype = ctype_orig
@@ -158,7 +159,7 @@ class UserDataProcessor(object):
                 if not url_helper.ok_http_code(st):
                     content = ''
 
-            new_msg = ud.convert_string(content)
+            new_msg = convert_string(content)
             self._process_msg(new_msg, append_msg)
 
     def _explode_archive(self, archive, append_msg):
@@ -179,7 +180,7 @@ class UserDataProcessor(object):
             content = ent.get('content', '')
             mtype = ent.get('type')
             if not mtype:
-                mtype = ud.type_from_starts_with(content, ARCHIVE_UNDEF_TYPE)
+                mtype = type_from_starts_with(content, ARCHIVE_UNDEF_TYPE)
 
             maintype, subtype = mtype.split('/', 1)
             if maintype == "text":
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 7259d933..1f884df8 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -22,8 +22,8 @@
 
 from StringIO import StringIO
 
+import copy as obj_copy
 import contextlib
-import copy
 import errno
 import glob
 import grp
@@ -35,12 +35,11 @@ import pwd
 import random
 import shutil
 import socket
-import string
+import string # pylint: disable=W0402
 import subprocess
 import sys
 import tempfile
 import time
-import traceback
 import types
 import urlparse
 
@@ -171,7 +170,8 @@ def fork_cb(child_cb, *args):
             child_cb(*args)
             os._exit(0)  # pylint: disable=W0212
         except:
-            logexc(LOG, "Failed forking and calling callback %s", obj_name(child_cb))
+            logexc(LOG, ("Failed forking and"
+                         " calling callback %s"), obj_name(child_cb))
             os._exit(1)  # pylint: disable=W0212
     else:
         LOG.debug("Forked child %s who will run callback %s",
@@ -549,10 +549,11 @@ def load_yaml(blob, default=None, allowed=(dict,)):
         converted = yaml.load(blob)
         if not isinstance(converted, allowed):
             # Yes this will just be caught, but thats ok for now...
-            raise TypeError("Yaml load allows %s root types, but got %s instead" %
+            raise TypeError(("Yaml load allows %s root types,"
+                             " but got %s instead") %
                             (allowed, obj_name(converted)))
         loaded = converted
-    except (yaml.YAMLError, TypeError, ValueError) as exc:
+    except (yaml.YAMLError, TypeError, ValueError):
         logexc(LOG, "Failed loading yaml blob")
     return loaded
 
@@ -833,15 +834,12 @@ def find_devs_with(criteria=None, oformat='device',
         options.append(path)
     cmd = blk_id_cmd + options
     (out, _err) = subp(cmd)
-    if path:
-        return out.strip()
-    else:
-        entries = []
-        for line in out.splitlines():
-            line = line.strip()
-            if line:
-                entries.append(line)
-        return entries
+    entries = []
+    for line in out.splitlines():
+        line = line.strip()
+        if line:
+            entries.append(line)
+    return entries
 
 
 def load_file(fname, read_cb=None, quiet=False):
@@ -1109,7 +1107,7 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None):
 
 def get_builtin_cfg():
     # Deep copy so that others can't modify
-    return copy.deepcopy(CFG_BUILTIN)
+    return obj_copy.deepcopy(CFG_BUILTIN)
 
 
 def sym_link(source, link):
@@ -1140,16 +1138,14 @@ def time_rfc2822():
 
 
 def uptime():
+    uptime_str = '??'
     try:
-        uptimef = load_file("/proc/uptime").strip()
-        if not uptimef:
-            uptime = 'na'
-        else:
-            uptime = uptimef.split()[0]
+        contents = load_file("/proc/uptime").strip()
+        if contents:
+            uptime_str = contents.split()[0]
     except:
         logexc(LOG, "Unable to read uptime from /proc/uptime")
-        uptime = '??'
-    return uptime
+    return uptime_str
 
 
 def ensure_file(path):
@@ -1261,7 +1257,8 @@ def shellify(cmdlist, add_header=True):
             content = "%s%s\n" % (content, args)
         else:
             raise RuntimeError(("Unable to shellify type %s"
-                                " which is not a list or string") % (obj_name(args)))
+                                " which is not a list or string")
+                               % (obj_name(args)))
     LOG.debug("Shellified %s to %s", cmdlist, content)
     return content
 
@@ -1275,8 +1272,7 @@ def is_container():
         try:
             # try to run a helper program. if it returns true/zero
             # then we're inside a container. otherwise, no
-            cmd = [helper]
-            subp(cmd, allowed_rc=[0])
+            subp([helper])
             return True
         except (IOError, OSError):
             pass
-- 
cgit v1.2.3


From f0c2776f4e48c6f88958c8ec0d5c8f72ab33cd3a Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Sat, 16 Jun 2012 08:54:20 -0700
Subject: 1. Remove content handler iid param, not used. 2. Ensure if iid is
 found that it is converted to a string before path joining.

---
 cloudinit/helpers.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index c276a54c..5e47794b 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -80,16 +80,17 @@ class FileSemaphores(object):
     def clear_all(self):
         try:
             util.del_dir(self.sem_path)
-        except (IOError, OSError) as e:
-            LOG.debug("Failed deleting semaphore directory %s due to %s",
-                      self.sem_path, e)
+        except (IOError, OSError):
+            util.logexc(LOG, "Failed deleting semaphore directory %s", 
+                        self.sem_path)
 
     def _acquire(self, name, freq):
+        # Check again if its been already gotten
         if self.has_run(name, freq):
             return None
         # This is a race condition since nothing atomic is happening
         # here, but this should be ok due to the nature of when
-        # and where cloud-init runs... (file writing is not a lock..)
+        # and where cloud-init runs... (file writing is not a lock...)
         sem_file = self._get_path(name, freq)
         contents = "%s: %s\n" % (os.getpid(), time())
         try:
@@ -100,9 +101,11 @@ class FileSemaphores(object):
         return sem_file
 
     def has_run(self, name, freq):
-        if freq == PER_ALWAYS:
+        if not freq or freq == PER_ALWAYS:
             return False
         sem_file = self._get_path(name, freq)
+        # This isn't really a good atomic check
+        # but it suffices for where and when cloudinit runs
         if os.path.exists(sem_file):
             return True
         return False
@@ -157,10 +160,9 @@ class Runners(object):
 
 class ContentHandlers(object):
 
-    def __init__(self, paths, iid=None):
+    def __init__(self, paths):
         self.paths = paths
         self.registered = {}
-        self.iid = iid
 
     def __contains__(self, item):
         return self.is_registered(item)
@@ -245,7 +247,7 @@ class Paths(object):
         iid = self.datasource.get_instance_id()
         if iid is None:
             return None
-        ipath = os.path.join(self.cloud_dir, 'instances', iid)
+        ipath = os.path.join(self.cloud_dir, 'instances', str(iid))
         add_on = self.lookups.get(name)
         if add_on:
             ipath = os.path.join(ipath, add_on)
-- 
cgit v1.2.3


From 1e6455899af5532314d7244bbc660a0e34a84eff Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Sat, 16 Jun 2012 09:51:45 -0700
Subject: Content handlers does not need the paths variable anymore so removed
 that.

---
 cloudinit/helpers.py | 3 +--
 cloudinit/stages.py  | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 5e47794b..76faa1d5 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -160,8 +160,7 @@ class Runners(object):
 
 class ContentHandlers(object):
 
-    def __init__(self, paths):
-        self.paths = paths
+    def __init__(self):
         self.registered = {}
 
     def __contains__(self, item):
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 9c772243..5dc289ea 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -313,7 +313,7 @@ class Init(object):
         ud_obj = self.datasource.get_userdata()
 
         # This keeps track of all the active handlers
-        c_handlers = helpers.ContentHandlers(paths=self.paths)
+        c_handlers = helpers.ContentHandlers()
 
         # Add handlers in cdir
         potential_handlers = util.find_modules(cdir)
-- 
cgit v1.2.3


From 45c510457910b7d0225d3bc20f372d8710cc1006 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Sat, 16 Jun 2012 12:34:33 -0700
Subject: Add comment as to why we need to check for none/empty later when
 fetching an ipath.

---
 cloudinit/helpers.py | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 76faa1d5..e5c45632 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -128,6 +128,11 @@ class Runners(object):
             return None
         sem_path = None
         if freq == PER_INSTANCE:
+            # This may not exist,
+            # so thats why we still check for none
+            # below if say the paths object
+            # doesn't have a datasource that can
+            # provide this instance path...
             sem_path = self.paths.get_ipath("sem")
         elif freq == PER_ONCE:
             sem_path = self.paths.get_cpath("sem")
-- 
cgit v1.2.3


From 50ec3b78686ae812c44b22835bf1203c92f5e5e3 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Tue, 19 Jun 2012 11:04:42 -0700
Subject: 1. When running, return the function results as well as a boolean
 that stated if it ran.

---
 cloudinit/helpers.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index e5c45632..3fd819b3 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -150,7 +150,7 @@ class Runners(object):
             args = []
         if sem.has_run(name, freq):
             LOG.info("%s already ran (freq=%s)", name, freq)
-            return None
+            return (False, None)
         with sem.lock(name, freq, clear_on_fail) as lk:
             if not lk:
                 raise LockFailure("Failed to acquire lock for %s" % name)
@@ -158,9 +158,10 @@ class Runners(object):
                 LOG.debug("Running %s with args %s using lock %s",
                           functor, args, lk)
                 if isinstance(args, (dict)):
-                    return functor(**args)
+                    results = functor(**args)
                 else:
-                    return functor(*args)
+                    results = functor(*args)
+                return (True, results)
 
 
 class ContentHandlers(object):
-- 
cgit v1.2.3


From f71f1d0a80b79f99fbe3795d250c656efd2eace4 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Tue, 19 Jun 2012 16:13:29 -0700
Subject: Initial add of read and write roots, this should cover a large set of
 cases that use the path object.

1. This basically allows configuration to specify 'read_root' which will be used for read operations, right now just templates
   and a 'write_root' via config that will be used for non read operations (ie 'var/lib/cloud' operations where most of the writing now happens)
---
 cloudinit/helpers.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 3fd819b3..3938e7ee 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -206,11 +206,17 @@ class ContentHandlers(object):
 
 class Paths(object):
     def __init__(self, path_cfgs, ds=None):
-        self.cloud_dir = path_cfgs.get('cloud_dir', '/var/lib/cloud')
+        self.cfgs = path_cfgs
+        # Populate all the initial paths
+        self.cloud_dir = self.join_paths(False,
+                                         path_cfgs.get('cloud_dir',
+                                                       '/var/lib/cloud'))
         self.instance_link = os.path.join(self.cloud_dir, 'instance')
         self.boot_finished = os.path.join(self.instance_link, "boot-finished")
         self.upstart_conf_d = path_cfgs.get('upstart_dir')
-        template_dir = path_cfgs.get('templates_dir', '/etc/cloud/templates/')
+        template_dir = self.join_paths(True,
+                                       path_cfgs.get('templates_dir',
+                                                     '/etc/cloud/templates/'))
         self.template_tpl = os.path.join(template_dir, '%s.tmpl')
         self.seed_dir = os.path.join(self.cloud_dir, 'seed')
         self.lookups = {
@@ -227,6 +233,20 @@ class Paths(object):
         # Set when a datasource becomes active
         self.datasource = ds
 
+    # joins the paths but also appends a read 
+    # or write root if available
+    def join_paths(self, read_only, *paths):
+        if read_only:
+            root = self.cfgs.get('read_root', '/')
+        else:
+            root = self.cfgs.get('write_root', '/')
+        if not paths:
+            return root
+        joined = os.path.join(*paths)
+        if root:
+            joined = os.path.join(root, joined.lstrip("/"))
+        return joined
+
     # get_ipath_cur: get the current instance path for an item
     def get_ipath_cur(self, name=None):
         ipath = self.instance_link
-- 
cgit v1.2.3


From 43eb6d5aace53bef2116dde0796807befef1d8ff Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Tue, 19 Jun 2012 17:58:41 -0700
Subject: Make most of all the places use the paths join() function so that
 testing with non-real read/write paths is easier.

---
 cloudinit/distros/rhel.py                   |  6 ++++-
 cloudinit/distros/ubuntu.py                 |  7 ++++--
 cloudinit/helpers.py                        | 34 +++++++++++++++++++----------
 cloudinit/ssh_util.py                       | 10 ++++++---
 cloudinit/transforms/apt_pipelining.py      | 15 +++++--------
 cloudinit/transforms/apt_update_upgrade.py  | 21 +++++++++++-------
 cloudinit/transforms/ca_certs.py            | 23 ++++++++++---------
 cloudinit/transforms/chef.py                | 31 +++++++++++++++++---------
 cloudinit/transforms/keys_to_console.py     |  9 ++++----
 cloudinit/transforms/landscape.py           | 20 ++++++++++-------
 cloudinit/transforms/mcollective.py         | 25 +++++++++++----------
 cloudinit/transforms/mounts.py              |  6 ++---
 cloudinit/transforms/phone_home.py          |  2 +-
 cloudinit/transforms/puppet.py              | 34 +++++++++++++++++------------
 cloudinit/transforms/resizefs.py            |  8 +++----
 cloudinit/transforms/rightscale_userdata.py |  1 +
 cloudinit/transforms/rsyslog.py             |  3 ++-
 cloudinit/transforms/runcmd.py              |  2 +-
 cloudinit/transforms/salt_minion.py         |  2 ++
 cloudinit/transforms/set_passwords.py       |  4 ++--
 cloudinit/transforms/ssh.py                 | 13 ++++++-----
 21 files changed, 167 insertions(+), 109 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py
index e0ca5909..aef7f6f3 100644
--- a/cloudinit/distros/rhel.py
+++ b/cloudinit/distros/rhel.py
@@ -73,6 +73,7 @@ class Distro(distros.Distro):
             lines.insert(0, '# Created by cloud-init')
             contents = "\n".join(lines)
             net_fn = NETWORK_FN_TPL % (dev)
+            net_fn = self._paths.join(False, net_fn)
             util.write_file(net_fn, contents, 0644)
 
     def set_hostname(self, hostname):
@@ -104,6 +105,7 @@ class Distro(distros.Distro):
             new_contents.append("# Added by cloud-init")
             new_contents.append("HOSTNAME=%s" % (hostname))
         contents = "\n".join(new_contents)
+        out_fn = self._paths.join(False, out_fn)
         util.write_file(out_fn, contents, 0644)
 
     def update_hostname(self, hostname, prev_file):
@@ -143,6 +145,7 @@ class Distro(distros.Distro):
         return default
 
     def _read_conf(self, filename):
+        filename = self._paths.join(True, filename)
         contents = util.load_file(filename, quiet=True)
         conf_lines = []
         for line in contents.splitlines():
@@ -194,7 +197,8 @@ class Distro(distros.Distro):
             new_contents.append("# Added by cloud-init")
             new_contents.append('ZONE="%s"' % (tz))
         tz_contents = "\n".join(new_contents)
-        util.write_file("/etc/sysconfig/clock", tz_contents)
+        tz_fn = self._paths.join(False, "/etc/sysconfig/clock")
+        util.write_file(tz_fn, tz_contents)
         # This ensures that the correct tz will be used for the system
         util.copy(tz_file, "/etc/localtime")
 
diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py
index 24724d83..94565b14 100644
--- a/cloudinit/distros/ubuntu.py
+++ b/cloudinit/distros/ubuntu.py
@@ -46,7 +46,8 @@ class Distro(distros.Distro):
         self.package_command('install', pkglist)
 
     def _write_network(self, settings):
-        util.write_file("/etc/network/interfaces", settings)
+        n_fn = self._paths.join(False, "/etc/network/interfaces")
+        util.write_file(n_fn, settings)
 
     def set_hostname(self, hostname):
         self._write_hostname(hostname, "/etc/hostname")
@@ -84,6 +85,7 @@ class Distro(distros.Distro):
             util.subp(['hostname', hostname])
 
     def _read_hostname(self, filename, default=None):
+        filename = self._paths.join(True, filename)
         contents = util.load_file(filename, quiet=True)
         for line in contents.splitlines():
             c_pos = line.find("#")
@@ -105,7 +107,8 @@ class Distro(distros.Distro):
             raise Exception(("Invalid timezone %s,"
                              " no file found at %s") % (tz, tz_file))
         tz_contents = "%s\n" % tz
-        util.write_file("/etc/timezone", tz_contents)
+        tz_fn = self._paths.join(False, "/etc/timezone")
+        util.write_file(tz_fn, tz_contents)
         util.copy(tz_file, "/etc/localtime")
 
     def package_command(self, command, args=None):
diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 3938e7ee..9f55a984 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -208,17 +208,18 @@ class Paths(object):
     def __init__(self, path_cfgs, ds=None):
         self.cfgs = path_cfgs
         # Populate all the initial paths
-        self.cloud_dir = self.join_paths(False,
-                                         path_cfgs.get('cloud_dir',
-                                                       '/var/lib/cloud'))
+        self.cloud_dir = self.join(False,
+                                   path_cfgs.get('cloud_dir', 
+                                                 '/var/lib/cloud'))
         self.instance_link = os.path.join(self.cloud_dir, 'instance')
         self.boot_finished = os.path.join(self.instance_link, "boot-finished")
         self.upstart_conf_d = path_cfgs.get('upstart_dir')
-        template_dir = self.join_paths(True,
-                                       path_cfgs.get('templates_dir',
-                                                     '/etc/cloud/templates/'))
-        self.template_tpl = os.path.join(template_dir, '%s.tmpl')
+        if self.upstart_conf_d:
+            self.upstart_conf_d = self.join(False, self.upstart_conf_d)
         self.seed_dir = os.path.join(self.cloud_dir, 'seed')
+        # This one isn't joined, since it should just be read-only
+        template_dir = path_cfgs.get('templates_dir', '/etc/cloud/templates/')
+        self.template_tpl = os.path.join(template_dir, '%s.tmpl')
         self.lookups = {
            "handlers": "handlers",
            "scripts": "scripts",
@@ -235,16 +236,25 @@ class Paths(object):
 
     # joins the paths but also appends a read 
     # or write root if available
-    def join_paths(self, read_only, *paths):
+    def join(self, read_only, *paths):
         if read_only:
-            root = self.cfgs.get('read_root', '/')
+            root = self.cfgs.get('read_root')
         else:
-            root = self.cfgs.get('write_root', '/')
+            root = self.cfgs.get('write_root')
         if not paths:
             return root
-        joined = os.path.join(*paths)
+        if len(paths) > 1:
+            joined = os.path.join(*paths)
+        else:
+            joined = paths[0]
         if root:
-            joined = os.path.join(root, joined.lstrip("/"))
+            pre_joined = joined
+            # Need to remove any starting '/' since this
+            # will confuse os.path.join
+            joined = joined.lstrip("/")
+            joined = os.path.join(root, joined)
+            LOG.debug("Translated %s to adjusted path %s (%s)",
+                      pre_joined, joined, read_only)
         return joined
 
     # get_ipath_cur: get the current instance path for an item
diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py
index 13adbb09..96143d32 100644
--- a/cloudinit/ssh_util.py
+++ b/cloudinit/ssh_util.py
@@ -210,9 +210,12 @@ def update_authorized_keys(fname, keys):
     return '\n'.join(lines)
 
 
-def setup_user_keys(keys, user, key_prefix, sshd_config_fn=DEF_SSHD_CFG):
+def setup_user_keys(keys, user, key_prefix, paths):
+    
+    # Make sure the users .ssh dir is setup accordingly
     pwent = pwd.getpwnam(user)
     ssh_dir = os.path.join(pwent.pw_dir, '.ssh')
+    ssh_dir = paths.join(False, ssh_dir)
     if not os.path.exists(ssh_dir):
         util.ensure_dir(ssh_dir, mode=0700)
         util.chownbyid(ssh_dir, pwent.pw_uid, pwent.pw_gid)
@@ -223,6 +226,7 @@ def setup_user_keys(keys, user, key_prefix, sshd_config_fn=DEF_SSHD_CFG):
     for k in keys:
         key_entries.append(parser.parse(str(k), def_opt=key_prefix))
 
+    sshd_conf_fn = paths.join(True, DEF_SSHD_CFG)
     with util.SeLinuxGuard(ssh_dir, recursive=True):
         try:
             # AuthorizedKeysFile may contain tokens
@@ -230,7 +234,7 @@ def setup_user_keys(keys, user, key_prefix, sshd_config_fn=DEF_SSHD_CFG):
             # The following tokens are defined: %% is replaced by a literal
             # '%', %h is replaced by the home directory of the user being
             # authenticated and %u is replaced by the username of that user.
-            ssh_cfg = parse_ssh_config(sshd_config_fn)
+            ssh_cfg = parse_ssh_config(sshd_conf_fn)
             akeys = ssh_cfg.get("authorizedkeysfile", '')
             akeys = akeys.strip()
             if not akeys:
@@ -247,7 +251,7 @@ def setup_user_keys(keys, user, key_prefix, sshd_config_fn=DEF_SSHD_CFG):
                               " in ssh config"
                               " from %s, using 'AuthorizedKeysFile' file"
                               " %s instead"),
-                        sshd_config_fn, authorized_keys)
+                        sshd_conf_fn, authorized_keys)
 
         content = update_authorized_keys(authorized_keys, key_entries)
         util.ensure_dir(os.path.dirname(authorized_keys), mode=0700)
diff --git a/cloudinit/transforms/apt_pipelining.py b/cloudinit/transforms/apt_pipelining.py
index d8e574b9..f460becb 100644
--- a/cloudinit/transforms/apt_pipelining.py
+++ b/cloudinit/transforms/apt_pipelining.py
@@ -23,7 +23,7 @@ frequency = PER_INSTANCE
 
 distros = ['ubuntu', 'debian']
 
-default_file = "/etc/apt/apt.conf.d/90cloud-init-pipelining"
+DEFAULT_FILE = "/etc/apt/apt.conf.d/90cloud-init-pipelining"
 
 # Acquire::http::Pipeline-Depth can be a value
 # from 0 to 5 indicating how many outstanding requests APT should send.
@@ -31,30 +31,27 @@ default_file = "/etc/apt/apt.conf.d/90cloud-init-pipelining"
 # on TCP connections - otherwise data corruption will occur.
 
 
-def handle(_name, cfg, _cloud, log, _args):
+def handle(_name, cfg, cloud, log, _args):
 
     apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False)
     apt_pipe_value_s = str(apt_pipe_value).lower().strip()
 
     if apt_pipe_value_s == "false":
-        write_apt_snippet("0", log)
-
+        write_apt_snippet(cloud, "0", log, DEFAULT_FILE)
     elif apt_pipe_value_s in ("none", "unchanged", "os"):
         return
-
     elif apt_pipe_value_s in [str(b) for b in xrange(0, 6)]:
-        write_apt_snippet(apt_pipe_value_s, log)
-
+        write_apt_snippet(cloud, apt_pipe_value_s, log, DEFAULT_FILE)
     else:
         log.warn("Invalid option for apt_pipeling: %s", apt_pipe_value)
 
 
-def write_apt_snippet(setting, log, f_name=default_file):
+def write_apt_snippet(cloud, setting, log, f_name):
     """ Writes f_name with apt pipeline depth 'setting' """
 
     file_contents = ("//Written by cloud-init per 'apt_pipelining'\n"
                      'Acquire::http::Pipeline-Depth "%s";\n') % (setting)
 
-    util.write_file(f_name, file_contents)
+    util.write_file(cloud.paths.join(False, f_name), file_contents)
 
     log.debug("Wrote %s with apt pipeline depth setting %s", f_name, setting)
diff --git a/cloudinit/transforms/apt_update_upgrade.py b/cloudinit/transforms/apt_update_upgrade.py
index d49d8bd2..29bbc1ae 100644
--- a/cloudinit/transforms/apt_update_upgrade.py
+++ b/cloudinit/transforms/apt_update_upgrade.py
@@ -26,6 +26,8 @@ from cloudinit import util
 
 distros = ['ubuntu', 'debian']
 
+PROXY_TPL = "Acquire::HTTP::Proxy \"%s\";\n"
+
 
 def handle(_name, cfg, cloud, log, _args):
     update = util.get_cfg_option_bool(cfg, 'apt_update', False)
@@ -44,22 +46,23 @@ def handle(_name, cfg, cloud, log, _args):
                                           "archive.ubuntu.com/ubuntu")
         rename_apt_lists(old_mir, mirror)
 
-    # set up proxy
+    # Set up any apt proxy
     proxy = cfg.get("apt_proxy", None)
     proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy"
     if proxy:
         try:
-            # See http://linux.die.net/man/5/apt.conf
-            contents = "Acquire::HTTP::Proxy \"%s\";\n"
-            util.write_file(proxy_filename, contents % (proxy))
+            # See man 'apt.conf'
+            contents = PROXY_TPL % (proxy)
+            util.write_file(cloud.paths.join(False, proxy_filename),
+                            contents)
         except Exception as e:
             util.logexc(log, "Failed to write proxy to %s", proxy_filename)
     elif os.path.isfile(proxy_filename):
         util.del_file(proxy_filename)
 
-    # process 'apt_sources'
+    # Process 'apt_sources'
     if 'apt_sources' in cfg:
-        errors = add_sources(cfg['apt_sources'],
+        errors = add_sources(cloud, cfg['apt_sources'],
                              {'MIRROR': mirror, 'RELEASE': release})
         for e in errors:
             log.warn("Source Error: %s", ':'.join(e))
@@ -138,7 +141,7 @@ def generate_sources_list(codename, mirror, cloud, log):
         log.warn("No template found, not rendering /etc/apt/sources.list")
 
 
-def add_sources(srclist, template_params=None):
+def add_sources(cloud, srclist, template_params=None):
     """
     add entries in /etc/apt/sources.list.d for each abbreviated
     sources.list entry in 'srclist'.  When rendering template, also
@@ -187,7 +190,9 @@ def add_sources(srclist, template_params=None):
                 errorlist.append([source, "failed add key"])
 
         try:
-            util.write_file(ent['filename'], "%s\n" % (source), omode="ab")
+            contents = "%s\n" % (source)
+            util.write_file(cloud.paths.join(False, ent['filename']),
+                            contents, omode="ab")
         except:
             errorlist.append([source,
                              "failed write to file %s" % ent['filename']])
diff --git a/cloudinit/transforms/ca_certs.py b/cloudinit/transforms/ca_certs.py
index e0802bfe..56c41561 100644
--- a/cloudinit/transforms/ca_certs.py
+++ b/cloudinit/transforms/ca_certs.py
@@ -33,7 +33,7 @@ def update_ca_certs():
     util.subp(["update-ca-certificates"])
 
 
-def add_ca_certs(certs):
+def add_ca_certs(cloud, certs):
     """
     Adds certificates to the system. To actually apply the new certificates
     you must also call L{update_ca_certs}.
@@ -41,26 +41,29 @@ def add_ca_certs(certs):
     @param certs: A list of certificate strings.
     """
     if certs:
-        cert_file_contents = "\n".join(certs)
+        # First ensure they are strings...
+        cert_file_contents = "\n".join([str(c) for c in certs])
         cert_file_fullpath = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME)
+        cert_file_fullpath = cloud.paths.join(False, cert_file_fullpath)
         util.write_file(cert_file_fullpath, cert_file_contents, mode=0644)
         # Append cert filename to CA_CERT_CONFIG file.
-        util.write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="ab")
+        util.write_file(cloud.paths.join(False, CA_CERT_CONFIG),
+                        "\n%s" % CA_CERT_FILENAME, omode="ab")
 
 
-def remove_default_ca_certs():
+def remove_default_ca_certs(cloud):
     """
     Removes all default trusted CA certificates from the system. To actually
     apply the change you must also call L{update_ca_certs}.
     """
-    util.delete_dir_contents(CA_CERT_PATH)
-    util.delete_dir_contents(CA_CERT_SYSTEM_PATH)
-    util.write_file(CA_CERT_CONFIG, "", mode=0644)
+    util.delete_dir_contents(cloud.paths.join(False, CA_CERT_PATH))
+    util.delete_dir_contents(cloud.paths.join(False, CA_CERT_SYSTEM_PATH))
+    util.write_file(cloud.paths.join(False, CA_CERT_CONFIG), "", mode=0644)
     debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no"
     util.subp(('debconf-set-selections', '-'), debconf_sel)
 
 
-def handle(name, cfg, _cloud, log, _args):
+def handle(name, cfg, cloud, log, _args):
     """
     Call to handle ca-cert sections in cloud-config file.
 
@@ -82,14 +85,14 @@ def handle(name, cfg, _cloud, log, _args):
     # default trusted CA certs first.
     if ca_cert_cfg.get("remove-defaults", False):
         log.debug("Removing default certificates")
-        remove_default_ca_certs()
+        remove_default_ca_certs(cloud)
 
     # If we are given any new trusted CA certs to add, add them.
     if "trusted" in ca_cert_cfg:
         trusted_certs = util.get_cfg_option_list(ca_cert_cfg, "trusted")
         if trusted_certs:
             log.debug("Adding %d certificates" % len(trusted_certs))
-            add_ca_certs(trusted_certs)
+            add_ca_certs(cloud, trusted_certs)
 
     # Update the system with the new cert configuration.
     log.debug("Updating certificates")
diff --git a/cloudinit/transforms/chef.py b/cloudinit/transforms/chef.py
index 31bfb85f..4e8ef346 100644
--- a/cloudinit/transforms/chef.py
+++ b/cloudinit/transforms/chef.py
@@ -36,30 +36,40 @@ def handle(name, cfg, cloud, log, _args):
         return
     chef_cfg = cfg['chef']
 
-    # ensure the chef directories we use exist
-    util.ensure_dirs(['/etc/chef', '/var/log/chef', '/var/lib/chef',
-                     '/var/cache/chef', '/var/backups/chef', '/var/run/chef'])
+    # Ensure the chef directories we use exist
+    c_dirs = [
+        '/etc/chef', 
+        '/var/log/chef', 
+        '/var/lib/chef', 
+        '/var/cache/chef', 
+        '/var/backups/chef', 
+        '/var/run/chef',
+    ]
+    for d in c_dirs:
+        util.ensure_dir(cloud.paths.join(False, d))
 
-    # set the validation key based on the presence of either 'validation_key'
+    # Set the validation key based on the presence of either 'validation_key'
     # or 'validation_cert'. In the case where both exist, 'validation_key'
     # takes precedence
     for key in ('validation_key', 'validation_cert'):
         if key in chef_cfg and chef_cfg[key]:
-            util.write_file('/etc/chef/validation.pem', chef_cfg[key])
+            v_fn = cloud.paths.join(False, '/etc/chef/validation.pem')
+            util.write_file(v_fn, chef_cfg[key])
             break
 
-    # create the chef config from template
+    # Create the chef config from template
     template_fn = cloud.get_template_filename('chef_client.rb')
     if template_fn:
+        iid = str(cloud.datasource.get_instance_id())
         params = {
             'server_url': chef_cfg['server_url'],
-            'node_name': util.get_cfg_option_str(chef_cfg, 'node_name',
-                                    cloud.datasource.get_instance_id()),
+            'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', iid),
             'environment': util.get_cfg_option_str(chef_cfg, 'environment',
                                                    '_default'),
             'validation_name': chef_cfg['validation_name']
         }
-        templater.render_to_file(template_fn, '/etc/chef/client.rb', params)
+        out_fn = cloud.paths.join(False, '/etc/chef/client.rb')
+        templater.render_to_file(template_fn, out_fn, params)
     else:
         log.warn("No template found, not rendering to /etc/chef/client.rb")
 
@@ -71,7 +81,8 @@ def handle(name, cfg, cloud, log, _args):
         initial_attributes = chef_cfg['initial_attributes']
         for k in list(initial_attributes.keys()):
             initial_json[k] = initial_attributes[k]
-    util.write_file('/etc/chef/firstboot.json', json.dumps(initial_json))
+    firstboot_fn = cloud.paths.join(False, '/etc/chef/firstboot.json')
+    util.write_file(firstboot_fn, json.dumps(initial_json))
 
     # If chef is not installed, we install chef based on 'install_type'
     if not os.path.isfile('/usr/bin/chef-client'):
diff --git a/cloudinit/transforms/keys_to_console.py b/cloudinit/transforms/keys_to_console.py
index e974375f..40758198 100644
--- a/cloudinit/transforms/keys_to_console.py
+++ b/cloudinit/transforms/keys_to_console.py
@@ -29,23 +29,24 @@ frequency = PER_INSTANCE
 helper_tool = '/usr/lib/cloud-init/write-ssh-key-fingerprints'
 
 
-def handle(name, cfg, _cloud, log, _args):
+def handle(name, cfg, cloud, log, _args):
     if not os.path.exists(helper_tool):
         log.warn(("Unable to activate transform %s,"
                   " helper tool not found at %s"), name, helper_tool)
         return
 
     fp_blacklist = util.get_cfg_option_list(cfg,
-        "ssh_fp_console_blacklist", [])
+                                            "ssh_fp_console_blacklist", [])
     key_blacklist = util.get_cfg_option_list(cfg,
-        "ssh_key_console_blacklist", ["ssh-dss"])
+                                             "ssh_key_console_blacklist",
+                                              ["ssh-dss"])
 
     try:
         cmd = [helper_tool]
         cmd.append(','.join(fp_blacklist))
         cmd.append(','.join(key_blacklist))
         (stdout, _stderr) = util.subp(cmd)
-        util.write_file('/dev/console', stdout)
+        util.write_file(cloud.paths.join(False, '/dev/console'), stdout)
     except:
         log.warn("Writing keys to /dev/console failed!")
         raise
diff --git a/cloudinit/transforms/landscape.py b/cloudinit/transforms/landscape.py
index 19948d0e..29ce41b9 100644
--- a/cloudinit/transforms/landscape.py
+++ b/cloudinit/transforms/landscape.py
@@ -33,12 +33,12 @@ from cloudinit.settings import PER_INSTANCE
 
 frequency = PER_INSTANCE
 
-lsc_client_cfg_file = "/etc/landscape/client.conf"
+LSC_CLIENT_CFG_FILE = "/etc/landscape/client.conf"
 
 distros = ['ubuntu']
 
 # defaults taken from stock client.conf in landscape-client 11.07.1.1-0ubuntu2
-lsc_builtincfg = {
+LSC_BUILTIN_CFG = {
   'client': {
     'log_level': "info",
     'url': "https://landscape.canonical.com/message-system",
@@ -48,7 +48,7 @@ lsc_builtincfg = {
 }
 
 
-def handle(name, cfg, _cloud, log, _args):
+def handle(name, cfg, cloud, log, _args):
     """
     Basically turn a top level 'landscape' entry with a 'client' dict
     and render it to ConfigObj format under '[client]' section in
@@ -66,15 +66,19 @@ def handle(name, cfg, _cloud, log, _args):
                          " but not a dictionary type,"
                          " is a %s instead"), util.obj_name(ls_cloudcfg))
 
-    merged = merge_together([lsc_builtincfg, lsc_client_cfg_file, ls_cloudcfg])
+    lsc_client_fn = cloud.paths.join(True, LSC_CLIENT_CFG_FILE)
+    merged = merge_together([LSC_BUILTIN_CFG, lsc_client_fn, ls_cloudcfg])
 
-    if not os.path.isdir(os.path.dirname(lsc_client_cfg_file)):
-        util.ensure_dir(os.path.dirname(lsc_client_cfg_file))
+    lsc_dir = cloud.paths.join(False, os.path.dirname(lsc_client_fn))
+    if not os.path.isdir(lsc_dir):
+        util.ensure_dir(lsc_dir)
 
     contents = StringIO()
     merged.write(contents)
-    util.write_file(lsc_client_cfg_file, contents.getvalue())
-    log.debug("Wrote landscape config file to %s", lsc_client_cfg_file)
+    contents.flush()
+
+    util.write_file(lsc_client_fn, contents.getvalue())
+    log.debug("Wrote landscape config file to %s", lsc_client_fn)
 
 
 def merge_together(objs):
diff --git a/cloudinit/transforms/mcollective.py b/cloudinit/transforms/mcollective.py
index 5464fe8c..9754d6b8 100644
--- a/cloudinit/transforms/mcollective.py
+++ b/cloudinit/transforms/mcollective.py
@@ -24,8 +24,8 @@ from StringIO import StringIO
 from cloudinit import cfg as config
 from cloudinit import util
 
-pubcert_file = "/etc/mcollective/ssl/server-public.pem"
-pricert_file = "/etc/mcollective/ssl/server-private.pem"
+PUBCERT_FILE = "/etc/mcollective/ssl/server-public.pem"
+PRICERT_FILE = "/etc/mcollective/ssl/server-private.pem"
 
 
 def handle(name, cfg, cloud, log, _args):
@@ -47,7 +47,8 @@ def handle(name, cfg, cloud, log, _args):
         mcollective_config = config.DefaultingConfigParser()
         # Read server.cfg values from original file in order to be able to mix
         # the rest up
-        old_contents = util.load_file('/etc/mcollective/server.cfg')
+        server_cfg_fn = cloud.paths.join(True, '/etc/mcollective/server.cfg')
+        old_contents = util.load_file(server_cfg_fn)
         # It doesn't contain any sections so just add one temporarily
         # Use a hash id based off the contents,
         # just incase of conflicts... (try to not have any...)
@@ -61,17 +62,19 @@ def handle(name, cfg, cloud, log, _args):
             section_head = section_tpl % (attempts)
         sectioned_contents = "%s\n%s" % (section_head, old_contents)
         mcollective_config.readfp(StringIO(sectioned_contents),
-                                  filename='/etc/mcollective/server.cfg')
+                                  filename=server_cfg_fn)
         for (cfg_name, cfg) in mcollective_cfg['conf'].iteritems():
             if cfg_name == 'public-cert':
-                util.write_file(pubcert_file, cfg, mode=0644)
+                pubcert_fn = cloud.paths.join(True, PUBCERT_FILE)
+                util.write_file(pubcert_fn, cfg, mode=0644)
                 mcollective_config.set(cfg_name,
-                    'plugin.ssl_server_public', pubcert_file)
+                    'plugin.ssl_server_public', pubcert_fn)
                 mcollective_config.set(cfg_name, 'securityprovider', 'ssl')
             elif cfg_name == 'private-cert':
-                util.write_file(pricert_file, cfg, mode=0600)
+                pricert_fn = cloud.paths.join(True, PRICERT_FILE)
+                util.write_file(pricert_fn, cfg, mode=0600)
                 mcollective_config.set(cfg_name,
-                    'plugin.ssl_server_private', pricert_file)
+                    'plugin.ssl_server_private', pricert_fn)
                 mcollective_config.set(cfg_name, 'securityprovider', 'ssl')
             else:
                 # Iterate throug the config items, we'll use ConfigParser.set
@@ -80,15 +83,15 @@ def handle(name, cfg, cloud, log, _args):
                     mcollective_config.set(cfg_name, o, v)
         # We got all our config as wanted we'll rename
         # the previous server.cfg and create our new one
-        util.rename('/etc/mcollective/server.cfg',
-                    '/etc/mcollective/server.cfg.old')
+        old_fn = "%s.old" % (server_cfg_fn)
+        util.rename(server_cfg_fn, old_fn)
         # Now we got the whole file, write to disk except the section 
         # we added so that config parser won't error out when trying to read.
         # Note below, that we've just used ConfigParser because it generally
         # works.  Below, we remove the initial 'nullsection' header.
         contents = mcollective_config.stringify()
         contents = contents.replace("%s\n" % (section_head), "")
-        util.write_file('/etc/mcollective/server.cfg', contents, mode=0644)
+        util.write_file(server_cfg_fn, contents, mode=0644)
 
     # Start mcollective
     util.subp(['service', 'mcollective', 'start'], capture=False)
diff --git a/cloudinit/transforms/mounts.py b/cloudinit/transforms/mounts.py
index 44182b87..700fbc44 100644
--- a/cloudinit/transforms/mounts.py
+++ b/cloudinit/transforms/mounts.py
@@ -168,7 +168,7 @@ def handle(_name, cfg, cloud, log, _args):
         cc_lines.append('\t'.join(line))
 
     fstab_lines = []
-    fstab = util.load_file("/etc/fstab")
+    fstab = util.load_file(cloud.paths.join(True, "/etc/fstab"))
     for line in fstab.splitlines():
         try:
             toks = ws.split(line)
@@ -180,7 +180,7 @@ def handle(_name, cfg, cloud, log, _args):
 
     fstab_lines.extend(cc_lines)
     contents = "%s\n" % ('\n'.join(fstab_lines))
-    util.write_file("/etc/fstab", contents)
+    util.write_file(cloud.paths.join(False, "/etc/fstab"), contents)
 
     if needswap:
         try:
@@ -190,7 +190,7 @@ def handle(_name, cfg, cloud, log, _args):
 
     for d in dirs:
         try:
-            util.ensure_dir(d)
+            util.ensure_dir(cloud.paths.join(False, d))
         except:
             util.logexc(log, "Failed to make '%s' config-mount", d)
 
diff --git a/cloudinit/transforms/phone_home.py b/cloudinit/transforms/phone_home.py
index 98ff2b85..a8752527 100644
--- a/cloudinit/transforms/phone_home.py
+++ b/cloudinit/transforms/phone_home.py
@@ -77,7 +77,7 @@ def handle(name, cfg, cloud, log, args):
 
     for (n, path) in pubkeys.iteritems():
         try:
-            all_keys[n] = util.load_file(path)
+            all_keys[n] = util.load_file(cloud.paths.join(True, path))
         except:
             util.logexc(log, ("%s: failed to open, can not"
                               " phone home that data"), path)
diff --git a/cloudinit/transforms/puppet.py b/cloudinit/transforms/puppet.py
index 76cc9732..d55118ea 100644
--- a/cloudinit/transforms/puppet.py
+++ b/cloudinit/transforms/puppet.py
@@ -43,7 +43,8 @@ def handle(name, cfg, cloud, log, _args):
     # ... and then update the puppet configuration
     if 'conf' in puppet_cfg:
         # Add all sections from the conf object to puppet.conf
-        contents = util.load_file('/etc/puppet/puppet.conf')
+        puppet_conf_fn = cloud.paths.join(False, '/etc/puppet/puppet.conf')
+        contents = util.load_file(puppet_conf_fn)
         # Create object for reading puppet.conf values
         puppet_config = config.DefaultingConfigParser()
         # Read puppet.conf values from original file in order to be able to
@@ -51,21 +52,27 @@ def handle(name, cfg, cloud, log, _args):
         cleaned_lines = [i.lstrip() for i in contents.splitlines()]
         cleaned_contents = '\n'.join(cleaned_lines)
         puppet_config.readfp(StringIO(cleaned_contents),
-                             filename='/etc/puppet/puppet.conf')
+                             filename=puppet_conf_fn)
         for (cfg_name, cfg) in puppet_cfg['conf'].iteritems():
-            # ca_cert configuration is a special case
-            # Dump the puppetmaster ca certificate in the correct place
+            # Cert configuration is a special case
+            # Dump the puppet master ca certificate in the correct place
             if cfg_name == 'ca_cert':
                 # Puppet ssl sub-directory isn't created yet
                 # Create it with the proper permissions and ownership
-                util.ensure_dir('/var/lib/puppet/ssl', 0771)
-                util.chownbyid('/var/lib/puppet/ssl',
+                pp_ssl_dir = cloud.paths.join(False, '/var/lib/puppet/ssl')
+                util.ensure_dir(pp_ssl_dir, 0771)
+                util.chownbyid(pp_ssl_dir,
                                pwd.getpwnam('puppet').pw_uid, 0)
-                util.ensure_dir('/var/lib/puppet/ssl/certs/')
-                util.chownbyid('/var/lib/puppet/ssl/certs/',
+                pp_ssl_certs = cloud.paths.join(False, 
+                                                '/var/lib/puppet/ssl/certs/')
+                util.ensure_dir(pp_ssl_certs)
+                util.chownbyid(pp_ssl_certs,
                                pwd.getpwnam('puppet').pw_uid, 0)
-                util.write_file('/var/lib/puppet/ssl/certs/ca.pem', cfg)
-                util.chownbyid('/var/lib/puppet/ssl/certs/ca.pem',
+                pp_ssl_ca_certs = cloud.paths.join(False,
+                                                   ('/var/lib/puppet/'
+                                                    'ssl/certs/ca.pem'))
+                util.write_file(pp_ssl_ca_certs, cfg)
+                util.chownbyid(pp_ssl_ca_certs,
                                pwd.getpwnam('puppet').pw_uid, 0)
             else:
                 # Iterate throug the config items, we'll use ConfigParser.set
@@ -82,10 +89,9 @@ def handle(name, cfg, cloud, log, _args):
                     puppet_config.set(cfg_name, o, v)
             # We got all our config as wanted we'll rename
             # the previous puppet.conf and create our new one
-            util.rename('/etc/puppet/puppet.conf',
-                        '/etc/puppet/puppet.conf.old')
-            contents = puppet_config.stringify()
-            util.write_file('/etc/puppet/puppet.conf', contents)
+            puppet_conf_old_fn = "%s.old" % (puppet_conf_fn)
+            util.rename(puppet_conf_fn, puppet_conf_old_fn)
+            util.write_file(puppet_conf_fn, puppet_config.stringify())
 
     # Set puppet to automatically start
     if os.path.exists('/etc/default/puppet'):
diff --git a/cloudinit/transforms/resizefs.py b/cloudinit/transforms/resizefs.py
index fe012417..fd2bb9e1 100644
--- a/cloudinit/transforms/resizefs.py
+++ b/cloudinit/transforms/resizefs.py
@@ -62,7 +62,7 @@ def get_fs_type(st_dev, path, log):
         raise
 
 
-def handle(name, cfg, _cloud, log, args):
+def handle(name, cfg, cloud, log, args):
     if len(args) != 0:
         resize_root = args[0]
     else:
@@ -74,11 +74,11 @@ def handle(name, cfg, _cloud, log, args):
 
     # TODO is the directory ok to be used??
     resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run")
+    resize_root_d = cloud.paths.join(False, resize_root_d)
     util.ensure_dir(resize_root_d)
 
-    # TODO: allow what is to be resized to
-    # be configurable??
-    resize_what = "/"
+    # TODO: allow what is to be resized to be configurable??
+    resize_what = cloud.paths.join(False)
     with util.SilentTemporaryFile(prefix="cloudinit.resizefs.",
                                   dir=resize_root_d, delete=True) as tfh:
         devpth = tfh.name
diff --git a/cloudinit/transforms/rightscale_userdata.py b/cloudinit/transforms/rightscale_userdata.py
index 8dfd845f..dc06f9ec 100644
--- a/cloudinit/transforms/rightscale_userdata.py
+++ b/cloudinit/transforms/rightscale_userdata.py
@@ -78,6 +78,7 @@ def handle(name, _cfg, cloud, log, _args):
     urls = mdict[my_hookname]
     for (i, url) in enumerate(urls):
         fname = os.path.join(scripts_d, "rightscale-%02i" % (i))
+        fname = cloud.paths.join(False, fname)
         try:
             resp = uhelp.readurl(url)
             # Ensure its a valid http response (and something gotten)
diff --git a/cloudinit/transforms/rsyslog.py b/cloudinit/transforms/rsyslog.py
index 71b74711..f2c1de1e 100644
--- a/cloudinit/transforms/rsyslog.py
+++ b/cloudinit/transforms/rsyslog.py
@@ -71,7 +71,8 @@ def handle(name, cfg, cloud, log, _args):
 
         try:
             contents = "%s\n" % (content)
-            util.write_file(filename, contents, omode=omode)
+            util.write_file(cloud.paths.join(False, filename),
+                            contents, omode=omode)
         except Exception:
             util.logexc(log, "Failed to write to %s", filename)
 
diff --git a/cloudinit/transforms/runcmd.py b/cloudinit/transforms/runcmd.py
index 31a254a5..f121484b 100644
--- a/cloudinit/transforms/runcmd.py
+++ b/cloudinit/transforms/runcmd.py
@@ -33,6 +33,6 @@ def handle(name, cfg, cloud, log, _args):
     cmd = cfg["runcmd"]
     try:
         content = util.shellify(cmd)
-        util.write_file(out_fn, content, 0700)
+        util.write_file(cloud.paths.join(False, out_fn), content, 0700)
     except:
         util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn)
diff --git a/cloudinit/transforms/salt_minion.py b/cloudinit/transforms/salt_minion.py
index d05d2a1e..16f5286d 100644
--- a/cloudinit/transforms/salt_minion.py
+++ b/cloudinit/transforms/salt_minion.py
@@ -35,6 +35,7 @@ def handle(name, cfg, cloud, log, _args):
     
     # Ensure we can configure files at the right dir
     config_dir = salt_cfg.get("config_dir", '/etc/salt')
+    config_dir = cloud.paths.join(False, config_dir)
     util.ensure_dir(config_dir)
 
     # ... and then update the salt configuration
@@ -47,6 +48,7 @@ def handle(name, cfg, cloud, log, _args):
     # ... copy the key pair if specified
     if 'public_key' in salt_cfg and 'private_key' in salt_cfg:
         pki_dir = salt_cfg.get('pki_dir', '/etc/salt/pki')
+        pki_dir = cloud.paths.join(pki_dir)
         with util.umask(077):
             util.ensure_dir(pki_dir)
             pub_name = os.path.join(pki_dir, 'minion.pub')
diff --git a/cloudinit/transforms/set_passwords.py b/cloudinit/transforms/set_passwords.py
index c0cc4e84..e7049f22 100644
--- a/cloudinit/transforms/set_passwords.py
+++ b/cloudinit/transforms/set_passwords.py
@@ -130,8 +130,8 @@ def handle(_name, cfg, cloud, log, args):
             replaced_auth = True
             new_lines.append(replacement)
 
-        new_contents = "\n".join(new_lines)
-        util.write_file('/etc/ssh/sshd_config', new_contents)
+        util.write_file(cloud.paths.join(False, '/etc/ssh/sshd_config'),
+                        "\n".join(new_lines))
 
         try:
             cmd = ['service']
diff --git a/cloudinit/transforms/ssh.py b/cloudinit/transforms/ssh.py
index b1f2ce89..33d4bb54 100644
--- a/cloudinit/transforms/ssh.py
+++ b/cloudinit/transforms/ssh.py
@@ -64,7 +64,8 @@ def handle(_name, cfg, cloud, log, _args):
             if key in key2file:
                 tgt_fn = key2file[key][0]
                 tgt_perms = key2file[key][1]
-                util.write_file(tgt_fn, val, tgt_perms)
+                util.write_file(cloud.paths.join(False, tgt_fn),
+                                val, tgt_perms)
 
         for (priv, pub) in priv2pub.iteritems():
             if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']:
@@ -86,6 +87,7 @@ def handle(_name, cfg, cloud, log, _args):
                                             generate_keys)
         for keytype in genkeys:
             keyfile = '/etc/ssh/ssh_host_%s_key' % (keytype)
+            keyfile = cloud.paths.join(False, keyfile)
             if not os.path.exists(keyfile):
                 cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile]
                 try:
@@ -107,20 +109,21 @@ def handle(_name, cfg, cloud, log, _args):
             cfgkeys = cfg["ssh_authorized_keys"]
             keys.extend(cfgkeys)
 
-        apply_credentials(keys, user, disable_root, disable_root_opts)
+        apply_credentials(keys, user, cloud.paths,
+                          disable_root, disable_root_opts)
     except:
         util.logexc(log, "Applying ssh credentials failed!")
 
 
-def apply_credentials(keys, user, disable_root, disable_root_opts):
+def apply_credentials(keys, user, paths, disable_root, disable_root_opts):
 
     keys = set(keys)
     if user:
-        ssh_util.setup_user_keys(keys, user, '')
+        ssh_util.setup_user_keys(keys, user, '', paths)
 
     if disable_root and user:
         key_prefix = disable_root_opts.replace('$USER', user)
     else:
         key_prefix = ''
 
-    ssh_util.setup_user_keys(keys, 'root', key_prefix)
+    ssh_util.setup_user_keys(keys, 'root', key_prefix, paths)
-- 
cgit v1.2.3


From b334a1035b645d9e727bdeb03e1fe3729bfc4e1a Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Tue, 19 Jun 2012 18:43:18 -0700
Subject: More path joinings found + update in logging of when we translate a
 path

---
 cloudinit/helpers.py                       | 2 +-
 cloudinit/transforms/apt_update_upgrade.py | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 9f55a984..09686340 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -253,7 +253,7 @@ class Paths(object):
             # will confuse os.path.join
             joined = joined.lstrip("/")
             joined = os.path.join(root, joined)
-            LOG.debug("Translated %s to adjusted path %s (%s)",
+            LOG.debug("Translated %s to adjusted path %s (read-only=%s)",
                       pre_joined, joined, read_only)
         return joined
 
diff --git a/cloudinit/transforms/apt_update_upgrade.py b/cloudinit/transforms/apt_update_upgrade.py
index 29bbc1ae..f5b4b58f 100644
--- a/cloudinit/transforms/apt_update_upgrade.py
+++ b/cloudinit/transforms/apt_update_upgrade.py
@@ -124,6 +124,7 @@ def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"):
         return
     olen = len(oprefix)
     for filename in glob.glob("%s_*" % oprefix):
+        # TODO use the cloud.paths.join...
         util.rename(filename, "%s%s" % (nprefix, filename[olen:]))
 
 
@@ -136,7 +137,8 @@ def generate_sources_list(codename, mirror, cloud, log):
     template_fn = cloud.get_template_filename('sources.list')
     if template_fn:
         params = {'mirror': mirror, 'codename': codename}
-        templater.render_to_file(template_fn, '/etc/apt/sources.list', params)
+        out_fn = cloud.paths.join(False, '/etc/apt/sources.list')
+        templater.render_to_file(template_fn, out_fn, params)
     else:
         log.warn("No template found, not rendering /etc/apt/sources.list")
 
-- 
cgit v1.2.3


From baa139e0112f0afe27238ed681b44faef0c7132f Mon Sep 17 00:00:00 2001
From: harlowja <harlowja@virtualbox.rhel>
Date: Wed, 20 Jun 2012 00:31:45 -0700
Subject: Show the lock object name and the lock value - useful for debugging

---
 cloudinit/helpers.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 09686340..21d203db 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -155,8 +155,8 @@ class Runners(object):
             if not lk:
                 raise LockFailure("Failed to acquire lock for %s" % name)
             else:
-                LOG.debug("Running %s with args %s using lock %s",
-                          functor, args, lk)
+                LOG.debug("Running %s with args %s using lock %s (%s)",
+                          functor, args, util.obj_name(lk), lk)
                 if isinstance(args, (dict)):
                     results = functor(**args)
                 else:
-- 
cgit v1.2.3


From 6f347387d57049bec80c920757e327223889798a Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Wed, 20 Jun 2012 16:36:02 -0700
Subject: Move the configparser helper to here since it being in a cfg.py is
 confusing when there is also a directory named 'config'

---
 cloudinit/helpers.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 21d203db..45633e0f 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -23,8 +23,11 @@
 from time import time
 
 import contextlib
+import io
 import os
 
+from ConfigParser import (NoSectionError, NoOptionError, RawConfigParser)
+
 from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE)
 
 from cloudinit import log as logging
@@ -298,3 +301,63 @@ class Paths(object):
             return None
         else:
             return ipath
+
+
+# This config parser will not throw when sections don't exist
+# and you are setting values on those sections which is useful
+# when writing to new options that may not have corresponding
+# sections. Also it can default other values when doing gets
+# so that if those sections/options do not exist you will
+# get a default instead of an error. Another useful case where
+# you can avoid catching exceptions that you typically don't
+# care about...
+
+class DefaultingConfigParser(RawConfigParser):
+    DEF_INT = 0
+    DEF_FLOAT = 0.0
+    DEF_BOOLEAN = False
+    DEF_BASE = None
+
+    def get(self, section, option):
+        value = self.DEF_BASE
+        try:
+            value = RawConfigParser.get(self, section, option)
+        except NoSectionError:
+            pass
+        except NoOptionError:
+            pass
+        return value
+
+    def set(self, section, option, value):
+        if not self.has_section(section) and section.lower() != 'default':
+            self.add_section(section)
+        RawConfigParser.set(self, section, option, value)
+
+    def remove_option(self, section, option):
+        if self.has_option(section, option):
+            RawConfigParser.remove_option(self, section, option)
+
+    def getboolean(self, section, option):
+        if not self.has_option(section, option):
+            return self.DEF_BOOLEAN
+        return RawConfigParser.getboolean(self, section, option)
+
+    def getfloat(self, section, option):
+        if not self.has_option(section, option):
+            return self.DEF_FLOAT
+        return RawConfigParser.getfloat(self, section, option)
+
+    def getint(self, section, option):
+        if not self.has_option(section, option):
+            return self.DEF_INT
+        return RawConfigParser.getint(self, section, option)
+
+    def stringify(self, header=None):
+        contents = ''
+        with io.BytesIO() as outputstream:
+            self.write(outputstream)
+            outputstream.flush()
+            contents = outputstream.getvalue()
+            if header:
+                contents = "\n".join([header, contents])
+        return contents
-- 
cgit v1.2.3


From ec4bdc4fb8d8d3a8f8b4f498eb47eac740485ede Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Wed, 20 Jun 2012 17:13:55 -0700
Subject: Massive pylint + pep8 fixups!

---
 bin/cloud-init                              | 20 +++++++--------
 cloudinit/cloud.py                          |  3 ++-
 cloudinit/config/__init__.py                |  2 +-
 cloudinit/config/cc_chef.py                 | 14 +++++------
 cloudinit/config/cc_disable_ec2_metadata.py |  4 +--
 cloudinit/config/cc_final_message.py        |  8 +++---
 cloudinit/config/cc_foo.py                  |  8 +++---
 cloudinit/config/cc_keys_to_console.py      |  8 +++---
 cloudinit/config/cc_landscape.py            |  2 +-
 cloudinit/config/cc_mcollective.py          |  4 +--
 cloudinit/config/cc_mounts.py               | 13 +++++-----
 cloudinit/config/cc_phone_home.py           | 11 +++++---
 cloudinit/config/cc_puppet.py               |  2 +-
 cloudinit/config/cc_resizefs.py             | 10 ++++----
 cloudinit/config/cc_salt_minion.py          |  2 +-
 cloudinit/config/cc_scripts_per_boot.py     |  6 ++---
 cloudinit/config/cc_scripts_per_instance.py |  6 ++---
 cloudinit/config/cc_scripts_per_once.py     |  6 ++---
 cloudinit/config/cc_scripts_user.py         |  6 ++---
 cloudinit/config/cc_set_passwords.py        |  4 +--
 cloudinit/config/cc_ssh.py                  | 39 +++++++++++++++--------------
 cloudinit/distros/__init__.py               |  1 -
 cloudinit/distros/rhel.py                   | 14 +++++------
 cloudinit/distros/ubuntu.py                 |  6 ++---
 cloudinit/handlers/__init__.py              |  8 +++---
 cloudinit/helpers.py                        |  6 ++---
 cloudinit/log.py                            |  2 --
 cloudinit/settings.py                       |  2 +-
 cloudinit/sources/DataSourceCloudStack.py   |  2 +-
 cloudinit/sources/DataSourceConfigDrive.py  |  2 +-
 cloudinit/sources/DataSourceEc2.py          |  6 ++---
 cloudinit/sources/DataSourceMAAS.py         |  1 +
 cloudinit/sources/DataSourceNoCloud.py      |  2 +-
 cloudinit/ssh_util.py                       |  5 ++--
 cloudinit/stages.py                         |  6 ++---
 cloudinit/url_helper.py                     | 14 +++++------
 cloudinit/user_data.py                      | 28 +++++++++------------
 cloudinit/util.py                           | 37 ++++++++++++++-------------
 38 files changed, 159 insertions(+), 161 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/bin/cloud-init b/bin/cloud-init
index 032d5f39..c1788ef4 100755
--- a/bin/cloud-init
+++ b/bin/cloud-init
@@ -75,6 +75,7 @@ def welcome(action):
     sys.stderr.flush()
     LOG.info(welcome_msg)
 
+
 def extract_fns(args):
     # Files are already opened so lets just pass that along
     # since it would of broke if it couldn't have
@@ -329,11 +330,11 @@ def main_single(name, args):
 
 def main():
     parser = argparse.ArgumentParser()
-    
+
     # Top level args
-    parser.add_argument('--version', '-v', action='version', 
+    parser.add_argument('--version', '-v', action='version',
                         version='%(prog)s ' + (version.version_string()))
-    parser.add_argument('--file', '-f', action='append', 
+    parser.add_argument('--file', '-f', action='append',
                         dest='files',
                         help=('additional yaml configuration'
                               ' files to use'),
@@ -345,18 +346,18 @@ def main():
     subparsers = parser.add_subparsers()
 
     # Each action and its sub-options (if any)
-    parser_init = subparsers.add_parser('init', 
+    parser_init = subparsers.add_parser('init',
                                         help=('initializes cloud-init and'
                                               ' performs initial modules'))
     parser_init.add_argument("--local", '-l', action='store_true',
                              help="start in local mode (default: %(default)s)",
                              default=False)
-    # This is used so that we can know which action is selected + 
+    # This is used so that we can know which action is selected +
     # the functor to use to run this subcommand
     parser_init.set_defaults(action=('init', main_init))
 
     # These settings are used for the 'config' and 'final' stages
-    parser_mod = subparsers.add_parser('modules', 
+    parser_mod = subparsers.add_parser('modules',
                                       help=('activates modules '
                                             'using a given configuration key'))
     parser_mod.add_argument("--mode", '-m', action='store',
@@ -368,7 +369,7 @@ def main():
 
     # These settings are used when you want to query information
     # stored in the cloud-init data objects/directories/files
-    parser_query = subparsers.add_parser('query', 
+    parser_query = subparsers.add_parser('query',
                                          help=('query information stored '
                                                'in cloud-init'))
     parser_query.add_argument("--name", '-n', action="store",
@@ -378,7 +379,7 @@ def main():
     parser_query.set_defaults(action=('query', main_query))
 
     # This subcommand allows you to run a single module
-    parser_single = subparsers.add_parser('single', 
+    parser_single = subparsers.add_parser('single',
                                          help=('run a single module '))
     parser_single.set_defaults(action=('single', main_single))
     parser_single.add_argument("--name", '-n', action="store",
@@ -394,10 +395,10 @@ def main():
                                     ' pass to this module'))
     parser_single.set_defaults(action=('single', main_single))
 
-
     args = parser.parse_args()
 
     # Setup basic logging to start (until reinitialized)
+    # iff in debug mode...
     if args.debug:
         logging.setupBasicLogging()
 
@@ -407,4 +408,3 @@ def main():
 
 if __name__ == '__main__':
     sys.exit(main())
-
diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py
index 90679202..6cdcb76a 100644
--- a/cloudinit/cloud.py
+++ b/cloudinit/cloud.py
@@ -38,6 +38,7 @@ LOG = logging.getLogger(__name__)
 # as providing a backwards compatible object that can be maintained
 # while the stages/other objects can be worked on independently...
 
+
 class Cloud(object):
     def __init__(self, datasource, paths, cfg, distro, runners):
         self.datasource = datasource
@@ -71,7 +72,7 @@ class Cloud(object):
     # The rest of thes are just useful proxies
     def get_userdata(self):
         return self.datasource.get_userdata()
-    
+
     def get_instance_id(self):
         return self.datasource.get_instance_id()
 
diff --git a/cloudinit/config/__init__.py b/cloudinit/config/__init__.py
index 74e2f275..02e32462 100644
--- a/cloudinit/config/__init__.py
+++ b/cloudinit/config/__init__.py
@@ -25,7 +25,7 @@ from cloudinit import log as logging
 
 LOG = logging.getLogger(__name__)
 
-# This prefix is used to make it less 
+# This prefix is used to make it less
 # of a change that when importing
 # we will not find something else with the same
 # name in the lookup path...
diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py
index 4e8ef346..74af2a7e 100644
--- a/cloudinit/config/cc_chef.py
+++ b/cloudinit/config/cc_chef.py
@@ -24,7 +24,7 @@ import os
 from cloudinit import templater
 from cloudinit import util
 
-ruby_version_default = "1.8"
+RUBY_VERSION_DEFAULT = "1.8"
 
 
 def handle(name, cfg, cloud, log, _args):
@@ -38,11 +38,11 @@ def handle(name, cfg, cloud, log, _args):
 
     # Ensure the chef directories we use exist
     c_dirs = [
-        '/etc/chef', 
-        '/var/log/chef', 
-        '/var/lib/chef', 
-        '/var/cache/chef', 
-        '/var/backups/chef', 
+        '/etc/chef',
+        '/var/log/chef',
+        '/var/lib/chef',
+        '/var/cache/chef',
+        '/var/backups/chef',
         '/var/run/chef',
     ]
     for d in c_dirs:
@@ -92,7 +92,7 @@ def handle(name, cfg, cloud, log, _args):
             # this will install and run the chef-client from gems
             chef_version = util.get_cfg_option_str(chef_cfg, 'version', None)
             ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version',
-                                                   ruby_version_default)
+                                                   RUBY_VERSION_DEFAULT)
             install_chef_from_gems(cloud.distro, ruby_version, chef_version)
             # and finally, run chef-client
             log.debug('Running chef-client')
diff --git a/cloudinit/config/cc_disable_ec2_metadata.py b/cloudinit/config/cc_disable_ec2_metadata.py
index c7d26029..62cca7cc 100644
--- a/cloudinit/config/cc_disable_ec2_metadata.py
+++ b/cloudinit/config/cc_disable_ec2_metadata.py
@@ -24,13 +24,13 @@ from cloudinit.settings import PER_ALWAYS
 
 frequency = PER_ALWAYS
 
-reject_cmd = ['route', 'add', '-host', '169.254.169.254', 'reject']
+REJECT_CMD = ['route', 'add', '-host', '169.254.169.254', 'reject']
 
 
 def handle(name, cfg, _cloud, log, _args):
     disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False)
     if disabled:
-        util.subp(reject_cmd)
+        util.subp(REJECT_CMD)
     else:
         log.debug(("Skipping transform named %s,"
                    " disabling the ec2 route not enabled"), name)
diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py
index c257b6d0..fd59aa1e 100644
--- a/cloudinit/config/cc_final_message.py
+++ b/cloudinit/config/cc_final_message.py
@@ -28,7 +28,7 @@ from cloudinit.settings import PER_ALWAYS
 
 frequency = PER_ALWAYS
 
-final_message_def = ("Cloud-init v. {{version}} finished at {{timestamp}}."
+FINAL_MESSAGE_DEF = ("Cloud-init v. {{version}} finished at {{timestamp}}."
                      " Up {{uptime}} seconds.")
 
 
@@ -39,21 +39,21 @@ def handle(_name, cfg, cloud, log, args):
         msg_in = args[0]
     else:
         msg_in = util.get_cfg_option_str(cfg, "final_message")
-    
+
     if not msg_in:
         template_fn = cloud.get_template_filename('final_message')
         if template_fn:
             msg_in = util.load_file(template_fn)
 
     if not msg_in:
-        msg_in = final_message_def
+        msg_in = FINAL_MESSAGE_DEF
 
     uptime = util.uptime()
     ts = util.time_rfc2822()
     cver = version.version_string()
     try:
         subs = {
-            'uptime': uptime, 
+            'uptime': uptime,
             'timestamp': ts,
             'version': cver,
         }
diff --git a/cloudinit/config/cc_foo.py b/cloudinit/config/cc_foo.py
index 99135704..e81e7faa 100644
--- a/cloudinit/config/cc_foo.py
+++ b/cloudinit/config/cc_foo.py
@@ -30,19 +30,19 @@ from cloudinit.settings import PER_INSTANCE
 #    as well as any datasource provided configuration
 #    c) A cloud object that can be used to access various
 #    datasource and paths for the given distro and data provided
-#    by the various datasource instance types. 
+#    by the various datasource instance types.
 #    d) A argument list that may or may not be empty to this module.
 #    Typically those are from module configuration where the module
 #    is defined with some extra configuration that will eventually
 #    be translated from yaml into arguments to this module.
 # 2. A optional 'frequency' that defines how often this module should be ran.
-#    Typically one of PER_INSTANCE, PER_ALWAYS, PER_ONCE. If not 
-#    provided PER_INSTANCE will be assumed. 
+#    Typically one of PER_INSTANCE, PER_ALWAYS, PER_ONCE. If not
+#    provided PER_INSTANCE will be assumed.
 #    See settings.py for these constants.
 # 3. A optional 'distros' array/set/tuple that defines the known distros
 #    this module will work with (if not all of them). This is used to write
 #    a warning out if a module is being ran on a untested distribution for
-#    informational purposes. If non existent all distros are assumed and 
+#    informational purposes. If non existent all distros are assumed and
 #    no warning occurs.
 
 frequency = PER_INSTANCE
diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py
index 40758198..a8fb3ba7 100644
--- a/cloudinit/config/cc_keys_to_console.py
+++ b/cloudinit/config/cc_keys_to_console.py
@@ -26,13 +26,13 @@ from cloudinit import util
 frequency = PER_INSTANCE
 
 # This is a tool that cloud init provides
-helper_tool = '/usr/lib/cloud-init/write-ssh-key-fingerprints'
+HELPER_TOOL = '/usr/lib/cloud-init/write-ssh-key-fingerprints'
 
 
 def handle(name, cfg, cloud, log, _args):
-    if not os.path.exists(helper_tool):
+    if not os.path.exists(HELPER_TOOL):
         log.warn(("Unable to activate transform %s,"
-                  " helper tool not found at %s"), name, helper_tool)
+                  " helper tool not found at %s"), name, HELPER_TOOL)
         return
 
     fp_blacklist = util.get_cfg_option_list(cfg,
@@ -42,7 +42,7 @@ def handle(name, cfg, cloud, log, _args):
                                               ["ssh-dss"])
 
     try:
-        cmd = [helper_tool]
+        cmd = [HELPER_TOOL]
         cmd.append(','.join(fp_blacklist))
         cmd.append(','.join(key_blacklist))
         (stdout, _stderr) = util.subp(cmd)
diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py
index 29ce41b9..599276a7 100644
--- a/cloudinit/config/cc_landscape.py
+++ b/cloudinit/config/cc_landscape.py
@@ -62,7 +62,7 @@ def handle(name, cfg, cloud, log, _args):
     ls_cloudcfg = cfg.get("landscape", {})
 
     if not isinstance(ls_cloudcfg, dict):
-        raise Exception(("'landscape' key existed in config," 
+        raise Exception(("'landscape' key existed in config,"
                          " but not a dictionary type,"
                          " is a %s instead"), util.obj_name(ls_cloudcfg))
 
diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py
index 4cec6494..ba5e13ca 100644
--- a/cloudinit/config/cc_mcollective.py
+++ b/cloudinit/config/cc_mcollective.py
@@ -52,7 +52,7 @@ def handle(name, cfg, cloud, log, _args):
         # It doesn't contain any sections so just add one temporarily
         # Use a hash id based off the contents,
         # just incase of conflicts... (try to not have any...)
-        # This is so that an error won't occur when reading (and no 
+        # This is so that an error won't occur when reading (and no
         # sections exist in the file)
         section_tpl = "[nullsection_%s]"
         attempts = 0
@@ -85,7 +85,7 @@ def handle(name, cfg, cloud, log, _args):
         # the previous server.cfg and create our new one
         old_fn = "%s.old" % (server_cfg_fn)
         util.rename(server_cfg_fn, old_fn)
-        # Now we got the whole file, write to disk except the section 
+        # Now we got the whole file, write to disk except the section
         # we added so that config parser won't error out when trying to read.
         # Note below, that we've just used ConfigParser because it generally
         # works.  Below, we remove the initial 'nullsection' header.
diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py
index 700fbc44..ab097c2a 100644
--- a/cloudinit/config/cc_mounts.py
+++ b/cloudinit/config/cc_mounts.py
@@ -24,10 +24,10 @@ import re
 
 from cloudinit import util
 
-# shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1
-shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$"
-shortname = re.compile(shortname_filter)
-ws = re.compile("[%s]+" % (whitespace))
+# Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1
+SHORTNAME_FILTER = r"^[x]{0,1}[shv]d[a-z][0-9]*$"
+SHORTNAME = re.compile(SHORTNAME_FILTER)
+WS = re.compile("[%s]+" % (whitespace))
 
 
 def is_mdname(name):
@@ -55,7 +55,6 @@ def handle(_name, cfg, cloud, log, _args):
     if "mounts" in cfg:
         cfgmnt = cfg["mounts"]
 
-    
     for i in range(len(cfgmnt)):
         # skip something that wasn't a list
         if not isinstance(cfgmnt[i], list):
@@ -85,7 +84,7 @@ def handle(_name, cfg, cloud, log, _args):
                 cfgmnt[i][0] = renamed
                 log.debug("Mapped metadata name %s to %s", startname, renamed)
         else:
-            if shortname.match(startname):
+            if SHORTNAME.match(startname):
                 renamed = "/dev/%s" % startname
                 log.debug("Mapped shortname name %s to %s", startname, renamed)
                 cfgmnt[i][0] = renamed
@@ -171,7 +170,7 @@ def handle(_name, cfg, cloud, log, _args):
     fstab = util.load_file(cloud.paths.join(True, "/etc/fstab"))
     for line in fstab.splitlines():
         try:
-            toks = ws.split(line)
+            toks = WS.split(line)
             if toks[3].find(comment) != -1:
                 continue
         except:
diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py
index a8752527..dcb07b66 100644
--- a/cloudinit/config/cc_phone_home.py
+++ b/cloudinit/config/cc_phone_home.py
@@ -26,8 +26,13 @@ from cloudinit.settings import PER_INSTANCE
 
 frequency = PER_INSTANCE
 
-post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa',
-                 'instance_id', 'hostname']
+POST_LIST_ALL = [
+    'pub_key_dsa',
+    'pub_key_rsa',
+    'pub_key_ecdsa',
+    'instance_id',
+    'hostname'
+]
 
 
 # phone_home:
@@ -63,7 +68,7 @@ def handle(name, cfg, cloud, log, args):
                           " is not an integer, using %s instead"), tries)
 
     if post_list == "all":
-        post_list = post_list_all
+        post_list = POST_LIST_ALL
 
     all_keys = {}
     all_keys['instance_id'] = cloud.get_instance_id()
diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py
index 5fb88bf2..5154efba 100644
--- a/cloudinit/config/cc_puppet.py
+++ b/cloudinit/config/cc_puppet.py
@@ -63,7 +63,7 @@ def handle(name, cfg, cloud, log, _args):
                 util.ensure_dir(pp_ssl_dir, 0771)
                 util.chownbyid(pp_ssl_dir,
                                pwd.getpwnam('puppet').pw_uid, 0)
-                pp_ssl_certs = cloud.paths.join(False, 
+                pp_ssl_certs = cloud.paths.join(False,
                                                 '/var/lib/puppet/ssl/certs/')
                 util.ensure_dir(pp_ssl_certs)
                 util.chownbyid(pp_ssl_certs,
diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py
index 1690094a..c019989e 100644
--- a/cloudinit/config/cc_resizefs.py
+++ b/cloudinit/config/cc_resizefs.py
@@ -27,7 +27,7 @@ from cloudinit.settings import PER_ALWAYS
 
 frequency = PER_ALWAYS
 
-resize_fs_prefixes_cmds = [
+RESIZE_FS_PREFIXES_CMDS = [
     ('ext', 'resize2fs'),
     ('xfs', 'xfs_growfs'),
 ]
@@ -89,16 +89,16 @@ def handle(name, cfg, cloud, log, args):
         # occurs this temporary file will still benefit from
         # auto deletion
         tfh.unlink_now()
-    
+
         st_dev = nodeify_path(devpth, resize_what, log)
         fs_type = get_fs_type(st_dev, devpth, log)
         if not fs_type:
             log.warn("Could not determine filesystem type of %s", resize_what)
             return
-    
+
         resizer = None
         fstype_lc = fs_type.lower()
-        for (pfix, root_cmd) in resize_fs_prefixes_cmds:
+        for (pfix, root_cmd) in RESIZE_FS_PREFIXES_CMDS:
             if fstype_lc.startswith(pfix):
                 resizer = root_cmd
                 break
@@ -112,7 +112,7 @@ def handle(name, cfg, cloud, log, args):
         resize_cmd = [resizer, devpth]
 
         if resize_root == "noblock":
-            # Fork to a child that will run 
+            # Fork to a child that will run
             # the resize command
             util.fork_cb(do_resize, resize_cmd, log)
             # Don't delete the file now in the parent
diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py
index 16f5286d..986e6db6 100644
--- a/cloudinit/config/cc_salt_minion.py
+++ b/cloudinit/config/cc_salt_minion.py
@@ -32,7 +32,7 @@ def handle(name, cfg, cloud, log, _args):
 
     # Start by installing the salt package ...
     cloud.distro.install_packages(["salt"])
-    
+
     # Ensure we can configure files at the right dir
     config_dir = salt_cfg.get("config_dir", '/etc/salt')
     config_dir = cloud.paths.join(False, config_dir)
diff --git a/cloudinit/config/cc_scripts_per_boot.py b/cloudinit/config/cc_scripts_per_boot.py
index 364e1d02..d3c47442 100644
--- a/cloudinit/config/cc_scripts_per_boot.py
+++ b/cloudinit/config/cc_scripts_per_boot.py
@@ -26,16 +26,16 @@ from cloudinit.settings import PER_ALWAYS
 
 frequency = PER_ALWAYS
 
-script_subdir = 'per-boot'
+SCRIPT_SUBDIR = 'per-boot'
 
 
 def handle(name, _cfg, cloud, log, _args):
     # Comes from the following:
     # https://forums.aws.amazon.com/thread.jspa?threadID=96918
-    runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir)
+    runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR)
     try:
         util.runparts(runparts_path)
     except:
         log.warn("Failed to run transform %s (%s in %s)",
-                 name, script_subdir, runparts_path)
+                 name, SCRIPT_SUBDIR, runparts_path)
         raise
diff --git a/cloudinit/config/cc_scripts_per_instance.py b/cloudinit/config/cc_scripts_per_instance.py
index d75ab47d..8e428ac2 100644
--- a/cloudinit/config/cc_scripts_per_instance.py
+++ b/cloudinit/config/cc_scripts_per_instance.py
@@ -26,16 +26,16 @@ from cloudinit.settings import PER_INSTANCE
 
 frequency = PER_INSTANCE
 
-script_subdir = 'per-instance'
+SCRIPT_SUBDIR = 'per-instance'
 
 
 def handle(name, _cfg, cloud, log, _args):
     # Comes from the following:
     # https://forums.aws.amazon.com/thread.jspa?threadID=96918
-    runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir)
+    runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR)
     try:
         util.runparts(runparts_path)
     except:
         log.warn("Failed to run transform %s (%s in %s)",
-                 name, script_subdir, runparts_path)
+                 name, SCRIPT_SUBDIR, runparts_path)
         raise
diff --git a/cloudinit/config/cc_scripts_per_once.py b/cloudinit/config/cc_scripts_per_once.py
index 80f8c325..e7a29a44 100644
--- a/cloudinit/config/cc_scripts_per_once.py
+++ b/cloudinit/config/cc_scripts_per_once.py
@@ -26,16 +26,16 @@ from cloudinit.settings import PER_ONCE
 
 frequency = PER_ONCE
 
-script_subdir = 'per-once'
+SCRIPT_SUBDIR = 'per-once'
 
 
 def handle(name, _cfg, cloud, log, _args):
     # Comes from the following:
     # https://forums.aws.amazon.com/thread.jspa?threadID=96918
-    runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir)
+    runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR)
     try:
         util.runparts(runparts_path)
     except:
         log.warn("Failed to run transform %s (%s in %s)",
-                 name, script_subdir, runparts_path)
+                 name, SCRIPT_SUBDIR, runparts_path)
         raise
diff --git a/cloudinit/config/cc_scripts_user.py b/cloudinit/config/cc_scripts_user.py
index f4fe3a2a..1ff05aae 100644
--- a/cloudinit/config/cc_scripts_user.py
+++ b/cloudinit/config/cc_scripts_user.py
@@ -26,17 +26,17 @@ from cloudinit.settings import PER_INSTANCE
 
 frequency = PER_INSTANCE
 
-script_subdir = 'scripts'
+SCRIPT_SUBDIR = 'scripts'
 
 
 def handle(name, _cfg, cloud, log, _args):
     # This is written to by the user data handlers
     # Ie, any custom shell scripts that come down
     # go here...
-    runparts_path = os.path.join(cloud.get_ipath_cur(), script_subdir)
+    runparts_path = os.path.join(cloud.get_ipath_cur(), SCRIPT_SUBDIR)
     try:
         util.runparts(runparts_path)
     except:
         log.warn("Failed to run transform %s (%s in %s)",
-                 name, script_subdir, runparts_path)
+                 name, SCRIPT_SUBDIR, runparts_path)
         raise
diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py
index e7049f22..ce17f357 100644
--- a/cloudinit/config/cc_set_passwords.py
+++ b/cloudinit/config/cc_set_passwords.py
@@ -25,7 +25,7 @@ from cloudinit import util
 from string import letters, digits  # pylint: disable=W0402
 
 # We are removing certain 'painful' letters/numbers
-pw_set = (letters.translate(None, 'loLOI') +
+PW_SET = (letters.translate(None, 'loLOI') +
           digits.translate(None, '01'))
 
 
@@ -148,4 +148,4 @@ def handle(_name, cfg, cloud, log, args):
 
 
 def rand_user_password(pwlen=9):
-    return util.rand_str(pwlen, select_from=pw_set)
+    return util.rand_str(pwlen, select_from=PW_SET)
diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py
index e5e99560..4019ae90 100644
--- a/cloudinit/config/cc_ssh.py
+++ b/cloudinit/config/cc_ssh.py
@@ -24,11 +24,11 @@ import glob
 from cloudinit import util
 from cloudinit import ssh_util
 
-DISABLE_ROOT_OPTS = ( "no-port-forwarding,no-agent-forwarding," 
-"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " 
+DISABLE_ROOT_OPTS = ("no-port-forwarding,no-agent-forwarding,"
+"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" "
 "rather than the user \\\"root\\\".\';echo;sleep 10\"")
 
-key2file = {
+KEY_2_FILE = {
     "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600),
     "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644),
     "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600),
@@ -37,15 +37,17 @@ key2file = {
     "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644),
 }
 
-priv2pub = {
-    'rsa_private': 'rsa_public', 
+PRIV_2_PUB = {
+    'rsa_private': 'rsa_public',
     'dsa_private': 'dsa_public',
     'ecdsa_private': 'ecdsa_public',
 }
 
-key_gen_tpl = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"'
+KEY_GEN_TPL = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"'
 
-generate_keys = ['rsa', 'dsa', 'ecdsa']
+GENERATE_KEY_NAMES = ['rsa', 'dsa', 'ecdsa']
+
+KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key'
 
 
 def handle(_name, cfg, cloud, log, _args):
@@ -58,21 +60,21 @@ def handle(_name, cfg, cloud, log, _args):
                 util.del_file(f)
             except:
                 util.logexc(log, "Failed deleting key file %s", f)
-    
+
     if "ssh_keys" in cfg:
         # if there are keys in cloud-config, use them
         for (key, val) in cfg["ssh_keys"].iteritems():
-            if key in key2file:
-                tgt_fn = key2file[key][0]
-                tgt_perms = key2file[key][1]
+            if key in KEY_2_FILE:
+                tgt_fn = KEY_2_FILE[key][0]
+                tgt_perms = KEY_2_FILE[key][1]
                 util.write_file(cloud.paths.join(False, tgt_fn),
                                 val, tgt_perms)
 
-        for (priv, pub) in priv2pub.iteritems():
+        for (priv, pub) in PRIV_2_PUB.iteritems():
             if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']:
                 continue
-            pair = (key2file[priv][0], key2file[pub][0])
-            cmd = ['sh', '-xc', key_gen_tpl % pair]
+            pair = (KEY_2_FILE[priv][0], KEY_2_FILE[pub][0])
+            cmd = ['sh', '-xc', KEY_GEN_TPL % pair]
             try:
                 # TODO: Is this guard needed?
                 with util.SeLinuxGuard("/etc/ssh", recursive=True):
@@ -84,12 +86,11 @@ def handle(_name, cfg, cloud, log, _args):
     else:
         # if not, generate them
         genkeys = util.get_cfg_option_list(cfg,
-                                            'ssh_genkeytypes', 
-                                            generate_keys)
+                                           'ssh_genkeytypes',
+                                           GENERATE_KEY_NAMES)
         for keytype in genkeys:
-            keyfile = '/etc/ssh/ssh_host_%s_key' % (keytype)
-            keyfile = cloud.paths.join(False, keyfile)
-            util.ensure_dir(os.path.dirname(keyfile)) 
+            keyfile = cloud.paths.join(False, KEY_FILE_TPL % (keytype))
+            util.ensure_dir(os.path.dirname(keyfile))
             if not os.path.exists(keyfile):
                 cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile]
                 try:
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 45dd85ec..25a60c52 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -157,4 +157,3 @@ def fetch(distro_name, mods=(__name__, )):
                            % (distro_name))
     distro_cls = getattr(mod, 'Distro')
     return distro_cls
-    
diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py
index b67ae5b8..5cbefa6e 100644
--- a/cloudinit/distros/rhel.py
+++ b/cloudinit/distros/rhel.py
@@ -35,7 +35,7 @@ class Distro(distros.Distro):
 
     def __init__(self, name, cfg, paths):
         distros.Distro.__init__(self, name, cfg, paths)
-    
+
     def install_packages(self, pkglist):
         self.package_command('install', pkglist)
 
@@ -210,12 +210,12 @@ class Distro(distros.Distro):
     def package_command(self, command, args=None):
         cmd = ['yum']
         # If enabled, then yum will be tolerant of errors on the command line
-        # with regard to packages. 
-        # For example: if you request to install foo, bar and baz and baz is 
+        # with regard to packages.
+        # For example: if you request to install foo, bar and baz and baz is
         # installed; yum won't error out complaining that baz is already
-        # installed. 
+        # installed.
         cmd.append("-t")
-        # Determines whether or not yum prompts for confirmation 
+        # Determines whether or not yum prompts for confirmation
         # of critical actions. We don't want to prompt...
         cmd.append("-y")
         cmd.append(command)
@@ -223,8 +223,8 @@ class Distro(distros.Distro):
             cmd.extend(args)
         # Allow the output of this to flow outwards (ie not be captured)
         util.subp(cmd, capture=False)
-        
-        
+
+
 # This is a util function to translate a ubuntu /etc/network/interfaces 'blob'
 # to a rhel equiv. that can then be written to /etc/sysconfig/network-scripts/
 # TODO remove when we have python-netcf active...
diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py
index 5a1b572e..fd7b7b8d 100644
--- a/cloudinit/distros/ubuntu.py
+++ b/cloudinit/distros/ubuntu.py
@@ -36,11 +36,11 @@ class Distro(distros.Distro):
 
     def __init__(self, name, cfg, paths):
         distros.Distro.__init__(self, name, cfg, paths)
-        # This will be used to restrict certain 
+        # This will be used to restrict certain
         # calls from repeatly happening (when they
         # should only happen say once per instance...)
         self._runner = helpers.Runners(paths)
-    
+
     def install_packages(self, pkglist):
         self._update_package_sources()
         self.package_command('install', pkglist)
@@ -131,4 +131,4 @@ class Distro(distros.Distro):
 
     def _update_package_sources(self):
         self._runner.run("update-sources", self.package_command,
-                         ["update"], freq=PER_INSTANCE)
\ No newline at end of file
+                         ["update"], freq=PER_INSTANCE)
diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py
index c6f2119c..d52b1cba 100644
--- a/cloudinit/handlers/__init__.py
+++ b/cloudinit/handlers/__init__.py
@@ -104,7 +104,7 @@ def run_part(mod, data, ctype, filename, payload, frequency):
     except:
         mod_ver = 1
     try:
-        LOG.debug("Calling handler %s (%s, %s, %s) with frequency %s", 
+        LOG.debug("Calling handler %s (%s, %s, %s) with frequency %s",
                   mod, ctype, filename, mod_ver, frequency)
         if mod_ver >= 2:
             # Treat as v. 2 which does get a frequency
@@ -114,7 +114,7 @@ def run_part(mod, data, ctype, filename, payload, frequency):
             mod.handle_part(data, ctype, filename, payload)
     except:
         util.logexc(LOG, ("Failed calling handler %s (%s, %s, %s)"
-                         " with frequency %s"), 
+                         " with frequency %s"),
                     mod, ctype, filename,
                     mod_ver, frequency)
 
@@ -178,7 +178,7 @@ def walker_callback(pdata, ctype, filename, payload):
                  payload, pdata['frequency'])
 
 
-# Callback is a function that will be called with 
+# Callback is a function that will be called with
 # (data, content_type, filename, payload)
 def walk(msg, callback, data):
     partnum = 0
@@ -226,5 +226,3 @@ def type_from_starts_with(payload, default=None):
         if payload_lc.startswith(text):
             return INCLUSION_TYPES_MAP[text]
     return default
-
-
diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 45633e0f..4447d1ee 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -84,7 +84,7 @@ class FileSemaphores(object):
         try:
             util.del_dir(self.sem_path)
         except (IOError, OSError):
-            util.logexc(LOG, "Failed deleting semaphore directory %s", 
+            util.logexc(LOG, "Failed deleting semaphore directory %s",
                         self.sem_path)
 
     def _acquire(self, name, freq):
@@ -212,7 +212,7 @@ class Paths(object):
         self.cfgs = path_cfgs
         # Populate all the initial paths
         self.cloud_dir = self.join(False,
-                                   path_cfgs.get('cloud_dir', 
+                                   path_cfgs.get('cloud_dir',
                                                  '/var/lib/cloud'))
         self.instance_link = os.path.join(self.cloud_dir, 'instance')
         self.boot_finished = os.path.join(self.instance_link, "boot-finished")
@@ -237,7 +237,7 @@ class Paths(object):
         # Set when a datasource becomes active
         self.datasource = ds
 
-    # joins the paths but also appends a read 
+    # joins the paths but also appends a read
     # or write root if available
     def join(self, read_only, *paths):
         if read_only:
diff --git a/cloudinit/log.py b/cloudinit/log.py
index 478946f8..fc1428a2 100644
--- a/cloudinit/log.py
+++ b/cloudinit/log.py
@@ -20,7 +20,6 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
 import logging
 import logging.handlers
 import logging.config
@@ -53,7 +52,6 @@ def setupBasicLogging():
     root.setLevel(DEBUG)
 
 
-
 def setupLogging(cfg=None):
     # See if the config provides any logging conf...
     if not cfg:
diff --git a/cloudinit/settings.py b/cloudinit/settings.py
index 8a1eaeb3..fac9b862 100644
--- a/cloudinit/settings.py
+++ b/cloudinit/settings.py
@@ -47,7 +47,7 @@ CFG_BUILTIN = {
         'paths': {
             'cloud_dir': '/var/lib/cloud',
             'templates_dir': '/etc/cloud/templates/',
-        }, 
+        },
         'distro': 'ubuntu',
     },
 }
diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index b1817654..83c577e6 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -121,7 +121,7 @@ class DataSourceCloudStack(sources.DataSource):
                 None, self.metadata_address)
             self.metadata = boto_utils.get_instance_metadata(self.api_ver,
                 self.metadata_address)
-            LOG.debug("Crawl of metadata service took %s seconds", 
+            LOG.debug("Crawl of metadata service took %s seconds",
                       int(time.time() - start_time))
             return True
         except Exception:
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 5da1ffea..9905dad4 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -36,7 +36,7 @@ CFG_DRIVE_FILES = [
     "meta.js",
 ]
 DEFAULT_METADATA = {
-    "instance-id": DEFAULT_IID, 
+    "instance-id": DEFAULT_IID,
     "dsmode": DEFAULT_MODE,
 }
 CFG_DRIVE_DEV_ENV = 'CLOUD_INIT_CONFIG_DRIVE_DEVICE'
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 55447102..0598dfa2 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -38,7 +38,7 @@ DEF_MD_URL = "http://169.254.169.254"
 DEF_MD_VERSION = '2009-04-04'
 
 # Default metadata urls that will be used if none are provided
-# They will be checked for 'resolveability' and some of the 
+# They will be checked for 'resolveability' and some of the
 # following may be discarded if they do not resolve
 DEF_MD_URLS = [DEF_MD_URL, "http://instance-data:8773"]
 
@@ -69,7 +69,7 @@ class DataSourceEc2(sources.DataSource):
                 None, self.metadata_address)
             self.metadata = boto_utils.get_instance_metadata(self.api_ver,
                 self.metadata_address)
-            LOG.debug("Crawl of metadata service took %s seconds", 
+            LOG.debug("Crawl of metadata service took %s seconds",
                        int(time.time() - start_time))
             return True
         except Exception:
@@ -201,7 +201,7 @@ class DataSourceEc2(sources.DataSource):
             return None
 
         # Example:
-        # 'block-device-mapping': 
+        # 'block-device-mapping':
         # {'ami': '/dev/sda1',
         # 'ephemeral0': '/dev/sdb',
         # 'root': '/dev/sda1'}
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index bb8fbac1..104e7a54 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -251,6 +251,7 @@ datasources = [
   (DataSourceMAAS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
 ]
 
+
 # Return a list of data sources that match this set of dependencies
 def get_datasource_list(depends):
     return sources.list_from_depends(depends, datasources)
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 2b016d1c..8499a97c 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -154,7 +154,7 @@ class DataSourceNoCloud(sources.DataSource):
             (self.dsmode in ("local", seeded_interfaces))):
             LOG.info("Updating network interfaces from %s", self)
             self.distro.apply_network(md['network-interfaces'])
-            
+
         if md['dsmode'] == self.dsmode:
             self.seed = ",".join(found)
             self.metadata = md
diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py
index fef3d40f..45dd5535 100644
--- a/cloudinit/ssh_util.py
+++ b/cloudinit/ssh_util.py
@@ -103,10 +103,10 @@ class AuthKeyLineParser(object):
             elif curc == '"':
                 quoted = not quoted
             i = i + 1
-    
+
         options = ent[0:i]
         options_lst = []
-        
+
         # Now use a csv parser to pull the options
         # out of the above string that we just found an endpoint for.
         #
@@ -211,7 +211,6 @@ def update_authorized_keys(fname, keys):
 
 
 def setup_user_keys(keys, user, key_prefix, paths):
-    
     # Make sure the users .ssh dir is setup accordingly
     pwent = pwd.getpwnam(user)
     ssh_dir = os.path.join(pwent.pw_dir, '.ssh')
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index ae6e2de5..84a965c2 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -287,7 +287,7 @@ class Init(object):
 
     def cloudify(self):
         # Form the needed options to cloudify our members
-        return cloud.Cloud(self.datasource, 
+        return cloud.Cloud(self.datasource,
                            self.paths, self.cfg,
                            self.distro, helpers.Runners(self.paths))
 
@@ -318,7 +318,7 @@ class Init(object):
     def consume(self, frequency=PER_INSTANCE):
         cdir = self.paths.get_cpath("handlers")
         idir = self.paths.get_ipath("handlers")
-    
+
         # Add the path to the plugins dir to the top of our list for import
         # instance dir should be read before cloud-dir
         if cdir and cdir not in sys.path:
@@ -417,7 +417,7 @@ class Modules(object):
             except:
                 util.logexc(LOG, ("Failed loading of datasource"
                                   " config object from %s"), self.datasource)
-        
+
         if self.base_cfg:
             t_cfgs.append(self.base_cfg)
 
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 1c583eba..223278ce 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -47,11 +47,11 @@ class UrlResponse(object):
     @property
     def contents(self):
         return self._contents
-    
+
     @property
     def headers(self):
         return self._headers
-    
+
     def __str__(self):
         if not self.contents:
             return ''
@@ -66,7 +66,7 @@ class UrlResponse(object):
             return True
         else:
             return False
-    
+
 
 def readurl(url, data=None, timeout=None,
             retries=0, sec_between=1, headers=None):
@@ -89,8 +89,8 @@ def readurl(url, data=None, timeout=None,
 
     excepts = []
     LOG.info(("Attempting to open '%s' with %s attempts"
-                " (%s retries, timeout=%s) to be performed"), 
-             url, attempts, retries, timeout)
+              " (%s retries, timeout=%s) to be performed"),
+              url, attempts, retries, timeout)
     open_args = {}
     if timeout is not None:
         open_args['timeout'] = int(timeout)
@@ -112,7 +112,7 @@ def readurl(url, data=None, timeout=None,
             excepts.append(e)
         except urllib2.URLError as e:
             # This can be a message string or
-            # another exception instance 
+            # another exception instance
             # (socket.error for remote URLs, OSError for local URLs).
             if (isinstance(e.reason, (OSError)) and
                 e.reason.errno == errno.ENOENT):
@@ -128,7 +128,7 @@ def readurl(url, data=None, timeout=None,
 
     # Didn't work out
     LOG.warn("Failed reading from %s after %s attempts", url, attempts)
-    
+
     # It must of errored at least once for code
     # to get here so re-raise the last error
     LOG.debug("%s errors occured, re-raising the last one", len(excepts))
diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index b7902d44..4babb8e5 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -65,33 +65,33 @@ class UserDataProcessor(object):
             # multipart/* are just containers
             if part.get_content_maintype() == 'multipart':
                 continue
-    
+
             ctype = None
             ctype_orig = part.get_content_type()
             payload = part.get_payload(decode=True)
-    
+
             if not ctype_orig:
                 ctype_orig = UNDEF_TYPE
-    
+
             if ctype_orig in TYPE_NEEDED:
                 ctype = handlers.type_from_starts_with(payload)
-    
+
             if ctype is None:
                 ctype = ctype_orig
-    
+
             if ctype in INCLUDE_TYPES:
                 self._do_include(payload, append_msg)
                 continue
-    
+
             if ctype in ARCHIVE_TYPES:
                 self._explode_archive(payload, append_msg)
                 continue
-    
+
             if 'Content-Type' in base_msg:
                 base_msg.replace_header('Content-Type', ctype)
             else:
                 base_msg['Content-Type'] = ctype
-    
+
             self._attach_part(append_msg, part)
 
     def _get_include_once_filename(self, entry):
@@ -108,8 +108,8 @@ class UserDataProcessor(object):
             lc_line = line.lower()
             if lc_line.startswith("#include-once"):
                 line = line[len("#include-once"):].lstrip()
-                # Every following include will now 
-                # not be refetched.... but will be 
+                # Every following include will now
+                # not be refetched.... but will be
                 # re-read from a local urlcache (if it worked)
                 include_once_on = True
             elif lc_line.startswith("#include"):
@@ -190,10 +190,10 @@ class UserDataProcessor(object):
         """
         if ATTACHMENT_FIELD not in outer_msg:
             outer_msg[ATTACHMENT_FIELD] = '0'
-    
+
         if new_count is not None:
             outer_msg.replace_header(ATTACHMENT_FIELD, str(new_count))
-    
+
         fetched_count = 0
         try:
             fetched_count = int(outer_msg.get(ATTACHMENT_FIELD))
@@ -234,7 +234,3 @@ def convert_string(raw_data, headers=None):
         msg = MIMEBase(maintype, subtype, *headers)
         msg.set_payload(data)
     return msg
-
-
-
-
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 91d20a76..56c01fab 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -35,7 +35,7 @@ import pwd
 import random
 import shutil
 import socket
-import string # pylint: disable=W0402
+import string  # pylint: disable=W0402
 import subprocess
 import sys
 import tempfile
@@ -153,13 +153,15 @@ def SilentTemporaryFile(**kwargs):
     # file to unlink has been unlinked elsewhere..
     LOG.debug("Created temporary file %s", fh.name)
     fh.unlink = del_file
-    # Add a new method that will unlink 
+
+    # Add a new method that will unlink
     # right 'now' but still lets the exit
     # method attempt to remove it (which will
     # not throw due to our del file being quiet
     # about files that are not there)
     def unlink_now():
         fh.unlink(fh.name)
+
     setattr(fh, 'unlink_now', unlink_now)
     return fh
 
@@ -199,7 +201,7 @@ def is_false_str(val, addons=None):
 
 def translate_bool(val, addons=None):
     if not val:
-        # This handles empty lists and false and 
+        # This handles empty lists and false and
         # other things that python believes are false
         return False
     # If its already a boolean skip
@@ -214,7 +216,6 @@ def rand_str(strlen=32, select_from=None):
     return "".join([random.choice(select_from) for _x in range(0, strlen)])
 
 
-
 def read_conf(fname):
     try:
         return load_yaml(load_file(fname), default={})
@@ -275,7 +276,7 @@ def is_ipv4(instr):
 
 def merge_base_cfg(cfgfile, cfg_builtin=None):
     syscfg = read_conf_with_confd(cfgfile)
-    
+
     kern_contents = read_cc_from_cmdline()
     kerncfg = {}
     if kern_contents:
@@ -575,7 +576,7 @@ def load_yaml(blob, default=None, allowed=(dict,)):
     try:
         blob = str(blob)
         LOG.debug(("Attempting to load yaml from string "
-                 "of length %s with allowed root types %s"), 
+                 "of length %s with allowed root types %s"),
                  len(blob), allowed)
         converted = yaml.load(blob)
         if not isinstance(converted, allowed):
@@ -625,7 +626,7 @@ def read_conf_d(confd):
 
     # remove anything not ending in '.cfg'
     confs = [f for f in confs if f.endswith(".cfg")]
-    
+
     # remove anything not a file
     confs = [f for f in confs if os.path.isfile(os.path.join(confd, f))]
 
@@ -726,9 +727,9 @@ def get_fqdn_from_hosts(hostname, filename="/etc/hosts"):
     """
     For each host a single line should be present with
       the following information:
-    
-	     IP_address canonical_hostname [aliases...]
-    
+
+        IP_address canonical_hostname [aliases...]
+
       Fields of the entry are separated by any number of  blanks  and/or  tab
       characters.  Text  from	a "#" character until the end of the line is a
       comment, and is ignored.	 Host  names  may  contain  only  alphanumeric
@@ -747,7 +748,7 @@ def get_fqdn_from_hosts(hostname, filename="/etc/hosts"):
             if not line:
                 continue
 
-            # If there there is less than 3 entries 
+            # If there there is less than 3 entries
             # (IP_address, canonical_hostname, alias)
             # then ignore this line
             toks = line.split()
@@ -829,7 +830,7 @@ def close_stdin():
         os.dup2(fp.fileno(), sys.stdin.fileno())
 
 
-def find_devs_with(criteria=None, oformat='device', 
+def find_devs_with(criteria=None, oformat='device',
                     tag=None, no_cache=False, path=None):
     """
     find devices matching given criteria (via blkid)
@@ -841,23 +842,23 @@ def find_devs_with(criteria=None, oformat='device',
     blk_id_cmd = ['blkid']
     options = []
     if criteria:
-        # Search for block devices with tokens named NAME that 
+        # Search for block devices with tokens named NAME that
         # have the value 'value' and display any devices which are found.
         # Common values for NAME include  TYPE, LABEL, and UUID.
         # If there are no devices specified on the command line,
-        # all block devices will be searched; otherwise, 
+        # all block devices will be searched; otherwise,
         # only search the devices specified by the user.
         options.append("-t%s" % (criteria))
     if tag:
         # For each (specified) device, show only the tags that match tag.
         options.append("-s%s" % (tag))
     if no_cache:
-        # If you want to start with a clean cache 
-        # (i.e. don't report devices previously scanned 
+        # If you want to start with a clean cache
+        # (i.e. don't report devices previously scanned
         # but not necessarily available at this time), specify /dev/null.
         options.extend(["-c", "/dev/null"])
     if oformat:
-        # Display blkid's output using the specified format. 
+        # Display blkid's output using the specified format.
         # The format parameter may be:
         # full, value, list, device, udev, export
         options.append('-o%s' % (oformat))
@@ -1104,7 +1105,7 @@ def mounts():
                 (dev, mp, fstype, opts, _freq, _passno) = mpline.split()
             except:
                 continue
-            # If the name of the mount point contains spaces these 
+            # If the name of the mount point contains spaces these
             # can be escaped as '\040', so undo that..
             mp = mp.replace("\\040", " ")
             mounted[dev] = {
-- 
cgit v1.2.3


From d8f49316e20cd11885f2ee5cfb9ad0ac7cf336c6 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Wed, 20 Jun 2012 23:58:29 -0700
Subject: 1. Add a dummy lock object + a file lock object 2. Use this object
 when returning semaphores 3. Adjust the debug statement to show this object

---
 cloudinit/helpers.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 4447d1ee..b6974f3c 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -40,13 +40,17 @@ class LockFailure(Exception):
     pass
 
 
+class DummyLock(object):
+    pass
+
+
 class DummySemaphores(object):
     def __init__(self):
         pass
 
     @contextlib.contextmanager
     def lock(self, _name, _freq, _clear_on_fail=False):
-        yield True
+        yield DummyLock()
 
     def has_run(self, _name, _freq):
         return False
@@ -58,6 +62,11 @@ class DummySemaphores(object):
         pass
 
 
+class FileLock(object):
+    def __init__(self, fn):
+        self.fn = fn
+
+
 class FileSemaphores(object):
     def __init__(self, sem_path):
         self.sem_path = sem_path
@@ -101,7 +110,7 @@ class FileSemaphores(object):
         except (IOError, OSError):
             util.logexc(LOG, "Failed writing semaphore file %s", sem_file)
             return None
-        return sem_file
+        return FileLock(sem_file)
 
     def has_run(self, name, freq):
         if not freq or freq == PER_ALWAYS:
@@ -158,8 +167,8 @@ class Runners(object):
             if not lk:
                 raise LockFailure("Failed to acquire lock for %s" % name)
             else:
-                LOG.debug("Running %s with args %s using lock %s (%s)",
-                          functor, args, util.obj_name(lk), lk)
+                LOG.debug("Running %s with args %s using lock (%s)",
+                          functor, args, lk)
                 if isinstance(args, (dict)):
                     results = functor(**args)
                 else:
-- 
cgit v1.2.3


From f1cab0d88cbcfa7eaa698db7dcc252bb6543d6c0 Mon Sep 17 00:00:00 2001
From: harlowja <harlowja@virtualbox.rhel>
Date: Thu, 21 Jun 2012 08:38:12 -0700
Subject: 1. Move all info() logging methods to debug() 2. Adjust comment on
 sources list from depends 3. For the /etc/timezone 'writing', add a header
 that says created by cloud-init

---
 bin/cloud-init                             |  3 +--
 cloudinit/distros/__init__.py              |  6 +++---
 cloudinit/distros/ubuntu.py                |  6 +++++-
 cloudinit/helpers.py                       |  2 +-
 cloudinit/sources/DataSourceCloudStack.py  |  2 +-
 cloudinit/sources/DataSourceConfigDrive.py |  6 ++----
 cloudinit/sources/DataSourceEc2.py         |  2 +-
 cloudinit/sources/DataSourceMAAS.py        |  2 +-
 cloudinit/sources/DataSourceNoCloud.py     |  2 +-
 cloudinit/sources/__init__.py              | 22 +++++++++++-----------
 cloudinit/url_helper.py                    | 14 +++++++-------
 11 files changed, 34 insertions(+), 33 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/bin/cloud-init b/bin/cloud-init
index a2f15c4b..d193272e 100755
--- a/bin/cloud-init
+++ b/bin/cloud-init
@@ -73,8 +73,7 @@ def welcome(action):
     }
     welcome_msg = "%s" % (templater.render_string(msg, tpl_params))
     sys.stderr.write("%s\n" % (welcome_msg))
-    sys.stderr.flush()
-    LOG.info(welcome_msg)
+    LOG.debug(welcome_msg)
 
 
 def extract_fns(args):
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index e0ef6ee0..6325257c 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -133,14 +133,14 @@ class Distro(object):
             raise NotImplementedError("Unknown interface action %s" % (action))
         cmd = IFACE_ACTIONS[action]
         try:
-            LOG.info("Attempting to run %s interface action using command %s",
-                     action, cmd)
+            LOG.debug("Attempting to run %s interface action using command %s",
+                      action, cmd)
             (_out, err) = util.subp(cmd)
             if len(err):
                 LOG.warn("Running %s resulted in stderr output: %s", cmd, err)
             return True
         except util.ProcessExecutionError:
-            util.logexc(LOG, "Running %s failed", cmd)
+            util.logexc(LOG, "Running interface command %s failed", cmd)
             return False
 
 
diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py
index fd7b7b8d..15af2e7f 100644
--- a/cloudinit/distros/ubuntu.py
+++ b/cloudinit/distros/ubuntu.py
@@ -112,7 +112,11 @@ class Distro(distros.Distro):
         if not os.path.isfile(tz_file):
             raise Exception(("Invalid timezone %s,"
                              " no file found at %s") % (tz, tz_file))
-        tz_contents = "%s\n" % tz
+        tz_lines = [
+            "# Created by cloud-init",
+            str(tz),
+        ]
+        tz_contents = "\n".join(tz_lines)
         tz_fn = self._paths.join(False, "/etc/timezone")
         util.write_file(tz_fn, tz_contents)
         util.copy(tz_file, self._paths.join(False, "/etc/localtime"))
diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index b6974f3c..6751f4a5 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -161,7 +161,7 @@ class Runners(object):
         if not args:
             args = []
         if sem.has_run(name, freq):
-            LOG.info("%s already ran (freq=%s)", name, freq)
+            LOG.debug("%s already ran (freq=%s)", name, freq)
             return (False, None)
         with sem.lock(name, freq, clear_on_fail) as lk:
             if not lk:
diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index 83c577e6..751bef4f 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -98,7 +98,7 @@ class DataSourceCloudStack(sources.DataSource):
                                 timeout=timeout, status_cb=LOG.warn)
 
         if url:
-            LOG.info("Using metadata source: '%s'", url)
+            LOG.debug("Using metadata source: '%s'", url)
         else:
             LOG.critical(("Giving up on waiting for the metadata from %s"
                           " after %s seconds"),
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 9905dad4..320dd1d1 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -87,10 +87,8 @@ class DataSourceConfigDrive(sources.DataSource):
         # Update interfaces and ifup only on the local datasource
         # this way the DataSourceConfigDriveNet doesn't do it also.
         if 'network-interfaces' in md and self.dsmode == "local":
-            if md['dsmode'] == "pass":
-                LOG.info("Updating network interfaces from configdrive")
-            else:
-                LOG.debug("Updating network interfaces from configdrive")
+            LOG.debug("Updating network interfaces from config drive (%s)",
+                     md['dsmode'])
             self.distro.apply_network(md['network-interfaces'])
 
         self.seed = found
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 0598dfa2..cb460de1 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -169,7 +169,7 @@ class DataSourceEc2(sources.DataSource):
                                 timeout=timeout, status_cb=LOG.warn)
 
         if url:
-            LOG.info("Using metadata source: '%s'", url2base[url])
+            LOG.debug("Using metadata source: '%s'", url2base[url])
         else:
             LOG.critical("Giving up on md from %s after %s seconds",
                             urls, int(time.time() - start_time))
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index 104e7a54..22c90b7c 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -128,7 +128,7 @@ class DataSourceMAAS(sources.DataSource):
                                  headers_cb=self.md_headers)
 
         if url:
-            LOG.info("Using metadata source: '%s'", url)
+            LOG.debug("Using metadata source: '%s'", url)
         else:
             LOG.critical("Giving up on md from %s after %i seconds",
                             urls, int(time.time() - starttime))
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 8499a97c..bed500a2 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -152,7 +152,7 @@ class DataSourceNoCloud(sources.DataSource):
         # ('local' for NoCloud, 'net' for NoCloudNet')
         if ('network-interfaces' in md and
             (self.dsmode in ("local", seeded_interfaces))):
-            LOG.info("Updating network interfaces from %s", self)
+            LOG.debug("Updating network interfaces from %s", self)
             self.distro.apply_network(md['network-interfaces'])
 
         if md['dsmode'] == self.dsmode:
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 42e924b0..b25724a5 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -166,7 +166,7 @@ class DataSource(object):
 def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list):
     ds_list = list_sources(cfg_list, ds_deps, pkg_list)
     ds_names = [util.obj_name(f) for f in ds_list]
-    LOG.info("Searching for data source in: %s", ds_names)
+    LOG.debug("Searching for data source in: %s", ds_names)
 
     for cls in ds_list:
         try:
@@ -188,9 +188,9 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list):
 # Return an ordered list of classes that match (if any)
 def list_sources(cfg_list, depends, pkg_list):
     src_list = []
-    LOG.info(("Looking for for data source in: %s,"
-              " via packages %s that matches dependencies %s"),
-             cfg_list, pkg_list, depends)
+    LOG.debug(("Looking for for data source in: %s,"
+               " via packages %s that matches dependencies %s"),
+              cfg_list, pkg_list, depends)
     for ds_name in cfg_list:
         if not ds_name.startswith(DS_PREFIX):
             ds_name = '%s%s' % (DS_PREFIX, ds_name)
@@ -207,17 +207,17 @@ def list_sources(cfg_list, depends, pkg_list):
     return src_list
 
 
-# depends is a list of dependencies (DEP_FILESYSTEM)
-# dslist is a list of 2 item lists
-# dslist = [
+# 'depends' is a list of dependencies (DEP_FILESYSTEM)
+# ds_list is a list of 2 item lists
+# ds_list = [
 #   ( class, ( depends-that-this-class-needs ) )
 # }
-# it returns a list of 'class' that matched these deps exactly
-# it is a helper function for DataSourceCollections
-def list_from_depends(depends, dslist):
+# It returns a list of 'class' that matched these deps exactly
+# It mainly is a helper function for DataSourceCollections
+def list_from_depends(depends, ds_list):
     ret_list = []
     depset = set(depends)
-    for (cls, deps) in dslist:
+    for (cls, deps) in ds_list:
         if depset == set(deps):
             ret_list.append(cls)
     return ret_list
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 223278ce..dbf72392 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -88,8 +88,8 @@ def readurl(url, data=None, timeout=None,
     attempts = retries + 1
 
     excepts = []
-    LOG.info(("Attempting to open '%s' with %s attempts"
-              " (%s retries, timeout=%s) to be performed"),
+    LOG.debug(("Attempting to open '%s' with %s attempts"
+               " (%s retries, timeout=%s) to be performed"),
               url, attempts, retries, timeout)
     open_args = {}
     if timeout is not None:
@@ -105,8 +105,8 @@ def readurl(url, data=None, timeout=None,
                 headers = {}
                 if rh.headers:
                     headers = dict(rh.headers)
-                LOG.info("Read from %s (%s, %sb) after %s attempts",
-                         url, status, len(content), (i + 1))
+                LOG.debug("Read from %s (%s, %sb) after %s attempts",
+                          url, status, len(content), (i + 1))
                 return UrlResponse(status, content, headers)
         except urllib2.HTTPError as e:
             excepts.append(e)
@@ -165,7 +165,7 @@ def wait_for_url(urls, max_wait=None, timeout=None,
     start_time = time.time()
 
     def log_status_cb(msg):
-        LOG.info(msg)
+        LOG.debug(msg)
 
     if status_cb is None:
         status_cb = log_status_cb
@@ -219,8 +219,8 @@ def wait_for_url(urls, max_wait=None, timeout=None,
             break
 
         loop_n = loop_n + 1
-        LOG.info("Please wait %s seconds while we wait to try again",
-                 sleep_time)
+        LOG.debug("Please wait %s seconds while we wait to try again",
+                  sleep_time)
         time.sleep(sleep_time)
 
     return False
-- 
cgit v1.2.3


From 94bf83b7a66c95eb56d2c286723d7f37c8c3de04 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Thu, 21 Jun 2012 20:14:04 -0700
Subject: 1. Update comment about ipath returning None when no datasource is
 active 2. Fix pylint warning on set method of config parser.

---
 cloudinit/helpers.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 6751f4a5..7a2ac6b8 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -286,7 +286,7 @@ class Paths(object):
             cpath = os.path.join(cpath, add_on)
         return cpath
 
-    # get_ipath : get the instance path for a name in pathmap
+    # _get_ipath : get the instance path for a name in pathmap
     # (/var/lib/cloud/instances/<instance>/<name>)
     def _get_ipath(self, name=None):
         if not self.datasource:
@@ -302,6 +302,7 @@ class Paths(object):
 
     # get_ipath : get the instance path for a name in pathmap
     # (/var/lib/cloud/instances/<instance>/<name>)
+    # returns None + warns if no active datasource....
     def get_ipath(self, name=None):
         ipath = self._get_ipath(name)
         if not ipath:
@@ -337,7 +338,7 @@ class DefaultingConfigParser(RawConfigParser):
             pass
         return value
 
-    def set(self, section, option, value):
+    def set(self, section, option, value=None):
         if not self.has_section(section) and section.lower() != 'default':
             self.add_section(section)
         RawConfigParser.set(self, section, option, value)
-- 
cgit v1.2.3


From 4e938eef6581769c086e417a24206274f26b8a95 Mon Sep 17 00:00:00 2001
From: harlowja <harlowja@virtualbox.rhel>
Date: Sat, 23 Jun 2012 15:00:41 -0700
Subject: 1. Use a common config merging class now in helpers    that will be
 the central point of config fetching    for both the 'init' stage and the
 'module' stages    a. This helps those who want to understand exactly      
 what configs are fetched and how the merging occurs.

---
 cloudinit/helpers.py | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 cloudinit/stages.py  | 59 ++++++++++---------------------------
 2 files changed, 96 insertions(+), 45 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 7a2ac6b8..0f86285f 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -28,7 +28,8 @@ import os
 
 from ConfigParser import (NoSectionError, NoOptionError, RawConfigParser)
 
-from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE)
+from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE,
+                                CFG_ENV_NAME)
 
 from cloudinit import log as logging
 from cloudinit import util
@@ -176,6 +177,85 @@ class Runners(object):
                 return (True, results)
 
 
+class ConfigMerger(object):
+    def __init__(self, paths=None, datasource=None,
+                 additional_fns=None, base_cfg=None):
+        self._paths = paths
+        self._ds = datasource
+        self._fns = additional_fns
+        self._base_cfg = base_cfg
+        # Created on first use
+        self._cfg = None
+
+    def _get_datasource_configs(self):
+        d_cfgs = []
+        if self._ds:
+            try:
+                ds_cfg = self._ds.get_config_obj()
+                if ds_cfg and isinstance(ds_cfg, (dict)):
+                    d_cfgs.append(ds_cfg)
+            except:
+                util.logexc(LOG, ("Failed loading of datasource"
+                                  " config object from %s"), self._ds)
+        return d_cfgs
+
+    def _get_env_configs(self):
+        e_cfgs = []
+        if CFG_ENV_NAME in os.environ:
+            e_fn = os.environ[CFG_ENV_NAME]
+            try:
+                e_cfgs.append(util.read_conf(e_fn))
+            except:
+                util.logexc(LOG, ('Failed loading of env. config'
+                                  ' from %s'), e_fn)
+        return e_cfgs
+
+    def _get_instance_configs(self):
+        i_cfgs = []
+        # If cloud-config was written, pick it up as
+        # a configuration file to use when running...
+        if not self._paths:
+            return i_cfgs
+        cc_fn = self._paths.get_ipath_cur('cloud_config')
+        if cc_fn and os.path.isfile(cc_fn):
+            try:
+                i_cfgs.append(util.read_conf(cc_fn))
+            except:
+                util.logexc(LOG, ('Failed loading of cloud-config'
+                                      ' from %s'), cc_fn)
+        return i_cfgs
+
+    def _read_cfg(self):
+        # Input config files override
+        # env config files which
+        # override instance configs
+        # which override datasource
+        # configs which override
+        # base configuration
+        cfgs = []
+        if self._fns:
+            for c_fn in self._fns:
+                try:
+                    cfgs.append(util.read_conf(c_fn))
+                except:
+                    util.logexc(LOG, ("Failed loading of configuration"
+                                       " from %s"), c_fn)
+
+        cfgs.extend(self._get_env_configs())
+        cfgs.extend(self._get_instance_configs())
+        cfgs.extend(self._get_datasource_configs())
+        if self._base_cfg:
+            cfgs.append(self._base_cfg)
+        return util.mergemanydict(cfgs)
+
+    @property
+    def cfg(self):
+        # None check to avoid empty case causing re-reading
+        if self._cfg is None:
+            self._cfg = self._read_cfg()
+        return self._cfg
+
+
 class ContentHandlers(object):
 
     def __init__(self):
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index cfe1c071..1846dd4d 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -147,23 +147,16 @@ class Init(object):
             LOG.debug("Loaded 'init' config %s", self._cfg)
 
     def _read_cfg(self, extra_fns):
-        # Read extra files provided (if any)
-        i_cfgs = []
-        if extra_fns:
-            for fn in extra_fns:
-                try:
-                    fn_cfg = util.read_conf(fn)
-                    i_cfgs.append(fn_cfg)
-                except:
-                    util.logexc(LOG, ("Failed loading of additional"
-                                      " configuration from %s"), fn)
-        # Now read in the built-in + base
         try:
-            conf = util.get_base_cfg(builtin=util.get_builtin_cfg())
+            base_conf = util.get_base_cfg(builtin=util.get_builtin_cfg())
         except Exception:
-            conf = util.get_builtin_cfg()
-        i_cfgs.append(conf)
-        return util.mergemanydict(i_cfgs)
+            base_conf = util.get_builtin_cfg()
+        no_cfg_pths = helpers.Paths({}, self.datasource)
+        merger = helpers.ConfigMerger(paths=no_cfg_pths,
+                                      datasource=self.datasource,
+                                      additional_fns=extra_fns,
+                                      base_cfg=base_conf)
+        return merger.cfg
 
     def _restore_from_cache(self):
         pickled_fn = self.paths.get_ipath_cur('obj_pkl')
@@ -392,46 +385,24 @@ class Init(object):
 
 class Modules(object):
     def __init__(self, init, cfg_files=None):
-        self.datasource = init.datasource
-        self.cfg_files = cfg_files
-        self.base_cfg = init.cfg
         self.init = init
+        self.cfg_files = cfg_files
         # Created on first use
         self._cached_cfg = None
 
     @property
     def cfg(self):
-        # None check to avoid empty case
+        # None check to avoid empty case causing re-reading
         if self._cached_cfg is None:
-            self._cached_cfg = self._get_config()
+            merger = helpers.ConfigMerger(paths=self.init.paths,
+                                          datasource=self.init.datasource,
+                                          fns=self.cfg_files,
+                                          base_cfg=self.init.cfg)
+            self._cached_cfg = merger.cfg
             LOG.debug("Loading 'module' config %s", self._cached_cfg)
         # Only give out a copy so that others can't modify this...
         return copy.deepcopy(self._cached_cfg)
 
-    def _get_config(self):
-        t_cfgs = []
-        if self.cfg_files:
-            for fn in self.cfg_files:
-                try:
-                    t_cfgs.append(util.read_conf(fn))
-                except:
-                    util.logexc(LOG, ("Failed loading of configuration"
-                                       " from %s"), fn)
-
-        if self.datasource:
-            try:
-                d_cfg = self.datasource.get_config_obj()
-                if d_cfg:
-                    t_cfgs.append(d_cfg)
-            except:
-                util.logexc(LOG, ("Failed loading of datasource"
-                                  " config object from %s"), self.datasource)
-
-        if self.base_cfg:
-            t_cfgs.append(self.base_cfg)
-
-        return util.mergemanydict(t_cfgs)
-
     def _read_modules(self, name):
         module_list = []
         if name not in self.cfg:
-- 
cgit v1.2.3


From b760f719450d597421e519d685b46481e8ff86fe Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Fri, 29 Jun 2012 13:20:09 -0700
Subject: Don't show the args but show the name and lock that is being used.

---
 cloudinit/helpers.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'cloudinit/helpers.py')

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 0f86285f..15036a50 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -168,8 +168,7 @@ class Runners(object):
             if not lk:
                 raise LockFailure("Failed to acquire lock for %s" % name)
             else:
-                LOG.debug("Running %s with args %s using lock (%s)",
-                          functor, args, lk)
+                LOG.debug("Running %s using lock (%s)", name, lk)
                 if isinstance(args, (dict)):
                     results = functor(**args)
                 else:
-- 
cgit v1.2.3