1. Adding some new helper files that split off file inclusion, templating, importing, constant usage.

1. Move all datasources to a new sources directory
1. Rename some files to be more consistent with python file/module naming.

48 files changed, 224 insertions, 0 deletions

diff --git a/cloudinit/constants.py b/cloudinit/constants.py
new file mode 100644
index 00000000..7bc90f27
--- /dev/null
+++ b/cloudinit/constants.py
@@ -0,0 +1,37 @@
+import os
+
+VAR_LIB_DIR = '/var/lib/cloud'
+CUR_INSTANCE_LINK = os.path.join(VAR_LIB_DIR, "instance")
+BOOT_FINISHED = os.path.join(CUR_INSTANCE_LINK, "boot-finished")
+SEED_DIR = os.path.join(VAR_LIB_DIR, "seed")
+
+CFG_ENV_NAME = "CLOUD_CFG"
+CLOUD_CONFIG = '/etc/cloud/cloud.cfg'
+
+CFG_BUILTIN = {
+    'datasource_list': ['NoCloud',
+                        'ConfigDrive',
+                        'OVF',
+                        'MAAS',
+                        'Ec2',
+                        'CloudStack'],
+    'def_log_file': '/var/log/cloud-init.log',
+    'log_cfgs': [],
+    'syslog_fix_perms': 'syslog:adm'
+}
+
+PATH_MAP = {
+   "handlers": "handlers",
+   "scripts": "scripts",
+   "sem": "sem",
+   "boothooks": "boothooks",
+   "userdata_raw": "user-data.txt",
+   "userdata": "user-data.txt.i",
+   "obj_pkl": "obj.pkl",
+   "cloud_config": "cloud-config.txt",
+   "data": "data",
+}
+
+PER_INSTANCE = "once-per-instance"
+PER_ALWAYS = "always"
+PER_ONCE = "once"
diff --git a/cloudinit/DataSource.py b/cloudinit/handlers/DataSource.py
index e2a9150d..e2a9150d 100644
--- a/cloudinit/DataSource.py
+++ b/cloudinit/handlers/DataSource.py
diff --git a/cloudinit/DataSourceCloudStack.py b/cloudinit/handlers/DataSourceCloudStack.py
index 5afdf7b6..5afdf7b6 100644
--- a/cloudinit/DataSourceCloudStack.py
+++ b/cloudinit/handlers/DataSourceCloudStack.py
diff --git a/cloudinit/DataSourceConfigDrive.py b/cloudinit/handlers/DataSourceConfigDrive.py
index 2db4a76a..2db4a76a 100644
--- a/cloudinit/DataSourceConfigDrive.py
+++ b/cloudinit/handlers/DataSourceConfigDrive.py
diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/handlers/DataSourceEc2.py
index 7051ecda..7051ecda 100644
--- a/cloudinit/DataSourceEc2.py
+++ b/cloudinit/handlers/DataSourceEc2.py
diff --git a/cloudinit/DataSourceMAAS.py b/cloudinit/handlers/DataSourceMAAS.py
index 61a0038f..61a0038f 100644
--- a/cloudinit/DataSourceMAAS.py
+++ b/cloudinit/handlers/DataSourceMAAS.py
diff --git a/cloudinit/DataSourceNoCloud.py b/cloudinit/handlers/DataSourceNoCloud.py
index e8c56b8f..e8c56b8f 100644
--- a/cloudinit/DataSourceNoCloud.py
+++ b/cloudinit/handlers/DataSourceNoCloud.py
diff --git a/cloudinit/DataSourceOVF.py b/cloudinit/handlers/DataSourceOVF.py
index a0b1b518..a0b1b518 100644
--- a/cloudinit/DataSourceOVF.py
+++ b/cloudinit/handlers/DataSourceOVF.py
diff --git a/cloudinit/CloudConfig/__init__.py b/cloudinit/handlers/__init__.py
index a16bdde6..a16bdde6 100644
--- a/cloudinit/CloudConfig/__init__.py
+++ b/cloudinit/handlers/__init__.py
diff --git a/cloudinit/CloudConfig/cc_apt_pipelining.py b/cloudinit/handlers/cc_apt_pipelining.py
index 0286a9ae..0286a9ae 100644
--- a/cloudinit/CloudConfig/cc_apt_pipelining.py
+++ b/cloudinit/handlers/cc_apt_pipelining.py
diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/handlers/cc_apt_update_upgrade.py
index a7049bce..a7049bce 100644
--- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py
+++ b/cloudinit/handlers/cc_apt_update_upgrade.py
diff --git a/cloudinit/CloudConfig/cc_bootcmd.py b/cloudinit/handlers/cc_bootcmd.py
index f584da02..f584da02 100644
--- a/cloudinit/CloudConfig/cc_bootcmd.py
+++ b/cloudinit/handlers/cc_bootcmd.py
diff --git a/cloudinit/CloudConfig/cc_byobu.py b/cloudinit/handlers/cc_byobu.py
index e821b261..e821b261 100644
--- a/cloudinit/CloudConfig/cc_byobu.py
+++ b/cloudinit/handlers/cc_byobu.py
diff --git a/cloudinit/CloudConfig/cc_ca_certs.py b/cloudinit/handlers/cc_ca_certs.py
index 3af6238a..3af6238a 100644
--- a/cloudinit/CloudConfig/cc_ca_certs.py
+++ b/cloudinit/handlers/cc_ca_certs.py
diff --git a/cloudinit/CloudConfig/cc_chef.py b/cloudinit/handlers/cc_chef.py
index 941e04fe..941e04fe 100644
--- a/cloudinit/CloudConfig/cc_chef.py
+++ b/cloudinit/handlers/cc_chef.py
diff --git a/cloudinit/CloudConfig/cc_disable_ec2_metadata.py b/cloudinit/handlers/cc_disable_ec2_metadata.py
index 6b31ea8e..6b31ea8e 100644
--- a/cloudinit/CloudConfig/cc_disable_ec2_metadata.py
+++ b/cloudinit/handlers/cc_disable_ec2_metadata.py
diff --git a/cloudinit/CloudConfig/cc_final_message.py b/cloudinit/handlers/cc_final_message.py
index abb4ca32..abb4ca32 100644
--- a/cloudinit/CloudConfig/cc_final_message.py
+++ b/cloudinit/handlers/cc_final_message.py
diff --git a/cloudinit/CloudConfig/cc_foo.py b/cloudinit/handlers/cc_foo.py
index 35ec3fa7..35ec3fa7 100644
--- a/cloudinit/CloudConfig/cc_foo.py
+++ b/cloudinit/handlers/cc_foo.py
diff --git a/cloudinit/CloudConfig/cc_grub_dpkg.py b/cloudinit/handlers/cc_grub_dpkg.py
index 9f3a7eaf..9f3a7eaf 100644
--- a/cloudinit/CloudConfig/cc_grub_dpkg.py
+++ b/cloudinit/handlers/cc_grub_dpkg.py
diff --git a/cloudinit/CloudConfig/cc_keys_to_console.py b/cloudinit/handlers/cc_keys_to_console.py
index 73a477c0..73a477c0 100644
--- a/cloudinit/CloudConfig/cc_keys_to_console.py
+++ b/cloudinit/handlers/cc_keys_to_console.py
diff --git a/cloudinit/CloudConfig/cc_landscape.py b/cloudinit/handlers/cc_landscape.py
index a4113cbe..a4113cbe 100644
--- a/cloudinit/CloudConfig/cc_landscape.py
+++ b/cloudinit/handlers/cc_landscape.py
diff --git a/cloudinit/CloudConfig/cc_locale.py b/cloudinit/handlers/cc_locale.py
index 2bb22fdb..2bb22fdb 100644
--- a/cloudinit/CloudConfig/cc_locale.py
+++ b/cloudinit/handlers/cc_locale.py
diff --git a/cloudinit/CloudConfig/cc_mcollective.py b/cloudinit/handlers/cc_mcollective.py
index a2a6230c..a2a6230c 100644
--- a/cloudinit/CloudConfig/cc_mcollective.py
+++ b/cloudinit/handlers/cc_mcollective.py
diff --git a/cloudinit/CloudConfig/cc_mounts.py b/cloudinit/handlers/cc_mounts.py
index 6cdd74e8..6cdd74e8 100644
--- a/cloudinit/CloudConfig/cc_mounts.py
+++ b/cloudinit/handlers/cc_mounts.py
diff --git a/cloudinit/CloudConfig/cc_phone_home.py b/cloudinit/handlers/cc_phone_home.py
index a7ff74e1..a7ff74e1 100644
--- a/cloudinit/CloudConfig/cc_phone_home.py
+++ b/cloudinit/handlers/cc_phone_home.py
diff --git a/cloudinit/CloudConfig/cc_puppet.py b/cloudinit/handlers/cc_puppet.py
index 6fc475f6..6fc475f6 100644
--- a/cloudinit/CloudConfig/cc_puppet.py
+++ b/cloudinit/handlers/cc_puppet.py
diff --git a/cloudinit/CloudConfig/cc_resizefs.py b/cloudinit/handlers/cc_resizefs.py
index 2dc66def..2dc66def 100644
--- a/cloudinit/CloudConfig/cc_resizefs.py
+++ b/cloudinit/handlers/cc_resizefs.py
diff --git a/cloudinit/CloudConfig/cc_rightscale_userdata.py b/cloudinit/handlers/cc_rightscale_userdata.py
index 5ed0848f..5ed0848f 100644
--- a/cloudinit/CloudConfig/cc_rightscale_userdata.py
+++ b/cloudinit/handlers/cc_rightscale_userdata.py
diff --git a/cloudinit/CloudConfig/cc_rsyslog.py b/cloudinit/handlers/cc_rsyslog.py
index ac7f2c74..ac7f2c74 100644
--- a/cloudinit/CloudConfig/cc_rsyslog.py
+++ b/cloudinit/handlers/cc_rsyslog.py
diff --git a/cloudinit/CloudConfig/cc_runcmd.py b/cloudinit/handlers/cc_runcmd.py
index f7e8c671..f7e8c671 100644
--- a/cloudinit/CloudConfig/cc_runcmd.py
+++ b/cloudinit/handlers/cc_runcmd.py
diff --git a/cloudinit/CloudConfig/cc_salt_minion.py b/cloudinit/handlers/cc_salt_minion.py
index 1a3b5039..1a3b5039 100644
--- a/cloudinit/CloudConfig/cc_salt_minion.py
+++ b/cloudinit/handlers/cc_salt_minion.py
diff --git a/cloudinit/CloudConfig/cc_scripts_per_boot.py b/cloudinit/handlers/cc_scripts_per_boot.py
index 41a74754..41a74754 100644
--- a/cloudinit/CloudConfig/cc_scripts_per_boot.py
+++ b/cloudinit/handlers/cc_scripts_per_boot.py
diff --git a/cloudinit/CloudConfig/cc_scripts_per_instance.py b/cloudinit/handlers/cc_scripts_per_instance.py
index a2981eab..a2981eab 100644
--- a/cloudinit/CloudConfig/cc_scripts_per_instance.py
+++ b/cloudinit/handlers/cc_scripts_per_instance.py
diff --git a/cloudinit/CloudConfig/cc_scripts_per_once.py b/cloudinit/handlers/cc_scripts_per_once.py
index a69151da..a69151da 100644
--- a/cloudinit/CloudConfig/cc_scripts_per_once.py
+++ b/cloudinit/handlers/cc_scripts_per_once.py
diff --git a/cloudinit/CloudConfig/cc_scripts_user.py b/cloudinit/handlers/cc_scripts_user.py
index 933aa4e0..933aa4e0 100644
--- a/cloudinit/CloudConfig/cc_scripts_user.py
+++ b/cloudinit/handlers/cc_scripts_user.py
diff --git a/cloudinit/CloudConfig/cc_set_hostname.py b/cloudinit/handlers/cc_set_hostname.py
index acea74d9..acea74d9 100644
--- a/cloudinit/CloudConfig/cc_set_hostname.py
+++ b/cloudinit/handlers/cc_set_hostname.py
diff --git a/cloudinit/CloudConfig/cc_set_passwords.py b/cloudinit/handlers/cc_set_passwords.py
index 9d0bbdb8..9d0bbdb8 100644
--- a/cloudinit/CloudConfig/cc_set_passwords.py
+++ b/cloudinit/handlers/cc_set_passwords.py
diff --git a/cloudinit/CloudConfig/cc_ssh.py b/cloudinit/handlers/cc_ssh.py
index 48eb58bc..48eb58bc 100644
--- a/cloudinit/CloudConfig/cc_ssh.py
+++ b/cloudinit/handlers/cc_ssh.py
diff --git a/cloudinit/CloudConfig/cc_ssh_import_id.py b/cloudinit/handlers/cc_ssh_import_id.py
index bbf5bd83..bbf5bd83 100644
--- a/cloudinit/CloudConfig/cc_ssh_import_id.py
+++ b/cloudinit/handlers/cc_ssh_import_id.py
diff --git a/cloudinit/CloudConfig/cc_timezone.py b/cloudinit/handlers/cc_timezone.py
index e5c9901b..e5c9901b 100644
--- a/cloudinit/CloudConfig/cc_timezone.py
+++ b/cloudinit/handlers/cc_timezone.py
diff --git a/cloudinit/CloudConfig/cc_update_etc_hosts.py b/cloudinit/handlers/cc_update_etc_hosts.py
index 6ad2fca8..6ad2fca8 100644
--- a/cloudinit/CloudConfig/cc_update_etc_hosts.py
+++ b/cloudinit/handlers/cc_update_etc_hosts.py
diff --git a/cloudinit/CloudConfig/cc_update_hostname.py b/cloudinit/handlers/cc_update_hostname.py
index b9d1919a..b9d1919a 100644
--- a/cloudinit/CloudConfig/cc_update_hostname.py
+++ b/cloudinit/handlers/cc_update_hostname.py
diff --git a/cloudinit/importer.py b/cloudinit/importer.py
new file mode 100644
index 00000000..deea0956
--- /dev/null
+++ b/cloudinit/importer.py
@@ -0,0 +1,11 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+import sys
+
+
+def import_module(module_name):
+    try:
+        __import__(module_name)
+        return sys.modules.get(module_name, None)
+    except ImportError as err:
+        raise RuntimeError('Could not load module %s: %s' % (module_name, err))
diff --git a/cloudinit/includer.py b/cloudinit/includer.py
new file mode 100644
index 00000000..d1022c5a
--- /dev/null
+++ b/cloudinit/includer.py
@@ -0,0 +1,65 @@
+import os
+import re
+
+from cloudinit import downloader as down
+from cloudinit import exceptions as excp
+from cloudinit import log as logging
+from cloudinit import shell as sh
+
+INCLUDE_PATT = re.compile("^#(opt_include|include)[ \t](.*)$", re.MULTILINE)
+OPT_PATS = ['opt_include']
+
+LOG = logging.getLogger(__name__)
+
+
+class Includer(object):
+
+    def __init__(self, root_fn, stack_limit=10):
+        self.root_fn = root_fn
+        self.stack_limit = stack_limit
+
+    def _read_file(self, fname):
+        return sh.read_file(fname)
+
+    def _read(self, fname, stack, rel):
+        if len(stack) >= self.stack_limit:
+            raise excp.StackExceeded("Stack limit of %s reached while including %s" % (self.stack_limit, fname))
+
+        canon_fname = self._canon_name(fname, rel)
+        if canon_fname in stack:
+            raise excp.RecursiveInclude("File %s recursively included" % (canon_fname))
+
+        stack.add(canon_fname)
+        new_rel = os.path.dirname(canon_fname)
+        contents = self._read_file(canon_fname)
+
+        def include_cb(match):
+            is_optional = (match.group(1).lower() in OPT_PATS)
+            fn = match.group(2).strip()
+            if not fn:
+                # Should we die??
+                return match.group(0)
+            else:
+                try:
+                    LOG.debug("Including file %s", fn)
+                    return self._read(fn, stack, new_rel)
+                except IOError:
+                    if is_optional:
+                        return ''
+                    else:
+                        raise
+
+        adjusted_contents = INCLUDE_PATT.sub(include_cb, contents)
+        stack.remove(fname)
+        return adjusted_contents
+
+    def _canon_name(self, fname, rel):
+        fname = fname.strip()
+        if not fname.startswith("/"):
+            fname = os.path.sep.join([rel, fname])
+        return os.path.realpath(fname)
+
+    def read(self, relative_to="."):
+        stack = set()
+        return self._read(self.root_fn, stack, rel=relative_to)
+    
diff --git a/cloudinit/log.py b/cloudinit/log.py
new file mode 100644
index 00000000..4d270045
--- /dev/null
+++ b/cloudinit/log.py
@@ -0,0 +1,94 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+import logging
+import logging.handlers
+import sys
+
+# Logging levels for easy access
+CRITICAL = logging.CRITICAL
+FATAL = logging.FATAL
+ERROR = logging.ERROR
+WARNING = logging.WARNING
+WARN = logging.WARN
+INFO = logging.INFO
+DEBUG = logging.DEBUG
+NOTSET = logging.NOTSET
+
+# File log rotation settings
+ROTATE_AMOUNT = 10  # Only keep the past 9 + 1 active
+ROTATE_SIZE = 10 * 1024 * 1024  # 10 MB
+
+
+class ConsoleFormatter(logging.Formatter):
+
+    def _get_mini_level(self, record):
+        if record.levelno in [INFO, NOTSET] or not record.levelname:
+            return ''
+        lvlname = record.levelname
+        return lvlname[0].upper() + ": "
+
+    def format(self, record):
+        record.message = record.getMessage()
+        rdict = dict(record.__dict__)
+        rdict['minilevelname'] = self._get_mini_level(record)
+        # Skipping exception info for the console...
+        return self._fmt % (rdict)
+
+
+def setupLogging(level, filename=None, filelevel=logging.DEBUG):
+    root = getLogger()
+    consolelg = logging.StreamHandler(sys.stdout)
+    consolelg.setFormatter(ConsoleFormatter('%(minilevelname)s%(message)s'))
+    consolelg.setLevel(level)
+    root.addHandler(consolelg)
+    if filename:
+        filelg = logging.handlers.RotatingFileHandler(filename, maxBytes=ROTATE_SIZE, backupCount=ROTATE_AMOUNT)
+        filelg.setFormatter(logging.Formatter('%(levelname)s: @%(name)s : %(message)s'))
+        filelg.setLevel(filelevel)
+        root.addHandler(filelg)
+    root.setLevel(level)
+
+
+def logging_set_from_cfg(cfg):
+    log_cfgs = []
+    logcfg = util.get_cfg_option_str(cfg, "log_cfg", False)
+    if logcfg:
+        # if there is a 'logcfg' entry in the config, respect
+        # it, it is the old keyname
+        log_cfgs = [logcfg]
+    elif "log_cfgs" in cfg:
+        for cfg in cfg['log_cfgs']:
+            if isinstance(cfg, list):
+                log_cfgs.append('\n'.join(cfg))
+            else:
+                log_cfgs.append()
+
+    if not len(log_cfgs):
+        sys.stderr.write("Warning, no logging configured\n")
+        return
+
+    for logcfg in log_cfgs:
+        try:
+            logging.config.fileConfig(StringIO.StringIO(logcfg))
+            return
+        except:
+            pass
+
+    raise Exception("no valid logging found\n")
+
+
+def getLogger(name='cloudinit'):
+    return logging.getLogger(name)
+
+
+# Fixes this annoyance...
+# No handlers could be found for logger XXX annoying output...
+try:
+    from logging import NullHandler
+except ImportError:
+    class NullHandler(logging.Handler):
+        def emit(self, record):
+            pass
+
+logger = getLogger()
+logger.addHandler(NullHandler())
diff --git a/cloudinit/SshUtil.py b/cloudinit/ssh_util.py
index a081fbe8..a081fbe8 100644
--- a/cloudinit/SshUtil.py
+++ b/cloudinit/ssh_util.py
diff --git a/cloudinit/templater.py b/cloudinit/templater.py
new file mode 100644
index 00000000..b6b320ab
--- /dev/null
+++ b/cloudinit/templater.py
@@ -0,0 +1,17 @@
+import os
+
+from Cheetah.Template import Template
+
+from cloudinit import util
+
+TEMPLATE_DIR = '/etc/cloud/templates/'
+
+
+def render_to_file(template, outfile, searchList):
+    contents = Template(file=os.path.join(TEMPLATE_DIR, template),
+                 searchList=[searchList]).respond()
+    util.write_file(outfile, contents)
+
+
+def render_string(template, searchList):
+    return Template(template, searchList=[searchList]).respond()
diff --git a/cloudinit/UserDataHandler.py b/cloudinit/user_data.py
index ec914480..ec914480 100644
--- a/cloudinit/UserDataHandler.py
+++ b/cloudinit/user_data.py