make cloud-config modules configurable by cloud-config

The list of cloud-config modules is now kept in cloud config itself.
There is a builtin list in cloudinit, which is overrideable by 
 /etc/cloud/cloud.cfg or user data cloud-config.

This should make the modules more easily added or removed (as no code
needs to be edited now)

Basic summary of changes:
 - move CloudConfig.py -> cloudinit/CloudConfig/__init__.py
 - split cloud-config modules into their own files named
   cloudinit/CloudConfig/cc_<name>.py
 - remove all the upstart/cloud-config-* scripts, replacing them with
   upstart/cloud-config.conf

11 files changed, 540 insertions, 569 deletions

diff --git a/cloudinit/CloudConfig.py b/cloudinit/CloudConfig.py
deleted file mode 100644
index 9c050abc..00000000
--- a/cloudinit/CloudConfig.py
+++ /dev/null
@@ -1,564 +0,0 @@
-# vi: ts=4 expandtab
-#
-#    Copyright (C) 2008-2010 Canonical Ltd.
-#
-#    Author: Chuck Short <chuck.short@canonical.com>
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License version 3, as
-#    published by the Free Software Foundation.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-import yaml
-import re
-import cloudinit
-import cloudinit.util as util
-import pwd
-import socket
-import subprocess
-import os
-import glob
-import sys
-import time
-import re
-import string
-
-per_instance="once-per-instance"
-cronpre = "/etc/cron.d/cloudinit"
-
-class CloudConfig():
-    cfgfile = None
-    handlers = { }
-    cfg = None
-
-    def __init__(self,cfgfile):
-        self.cloud = cloudinit.CloudInit()
-        self.cfg = self.get_config_obj(cfgfile)
-        self.cloud.get_data_source()
-        self.add_handler('apt-update-upgrade', self.h_apt_update_upgrade)
-        self.add_handler('config-ssh')
-        self.add_handler('disable-ec2-metadata',
-            self.h_disable_ec2_metadata, "always")
-        self.add_handler('config-mounts')
-        self.add_handler('config-puppet')
-        self.add_handler('config-misc')
-
-    def get_config_obj(self,cfgfile):
-        f=file(cfgfile)
-        cfg=yaml.load(f.read())
-        f.close()
-        if cfg is None: cfg = { }
-        return(util.mergedict(cfg,self.cloud.cfg))
-
-    def convert_old_config(self):
-        # support reading the old ConfigObj format file and turning it
-        # into a yaml string
-        try:
-            f = file(self.conffile)
-            str=file.read().replace('=',': ')
-            f.close()
-            return str
-        except:
-            return("")
-
-    def add_handler(self, name, handler=None, freq=None):
-        if handler is None:
-            try:
-                handler=getattr(self,'h_%s' % name.replace('-','_'))
-            except:
-                raise Exception("Unknown hander for name %s" %name)
-        if freq is None:
-            freq = per_instance
-
-        self.handlers[name]= { 'handler': handler, 'freq': freq }
-
-    def get_handler_info(self, name):
-        return(self.handlers[name]['handler'], self.handlers[name]['freq'])
-
-    def parse_ssh_keys(self):
-        disableRoot = self.cfg['disable_root']
-        if disableRoot == 'true':
-            value = 'disabled_root'
-            return value
-        else:
-            ec2Key = self.cfg['ec2_fetch_key']
-            if ec2Key != 'none':
-                value = 'default_key'
-                return value
-            else:
-                return ec2Key
-
-    def handle(self, name, args):
-        handler = None
-        freq = None
-        try:
-            (handler, freq) = self.get_handler_info(name)
-        except:
-            raise Exception("Unknown config key %s\n" % name)
-
-        self.cloud.sem_and_run(name, freq, handler, [ name, args ])
-
-    def h_apt_update_upgrade(self,name,args):
-        update = util.get_cfg_option_bool(self.cfg, 'apt_update', False)
-        upgrade = util.get_cfg_option_bool(self.cfg, 'apt_upgrade', False)
-
-        if not util.get_cfg_option_bool(self.cfg, \
-            'apt_preserve_sources_list', False):
-            if self.cfg.has_key("apt_mirror"):
-                mirror = self.cfg["apt_mirror"]
-            else:
-                mirror = self.cloud.get_mirror()
-            generate_sources_list(mirror)
-            old_mir = util.get_cfg_option_str(self.cfg,'apt_old_mirror', \
-                "archive.ubuntu.com/ubuntu")
-            rename_apt_lists(old_mir, mirror)
-
-        # process 'apt_sources'
-        if self.cfg.has_key('apt_sources'):
-            errors = add_sources(self.cfg['apt_sources'])
-            for e in errors:
-                warn("Source Error: %s\n" % ':'.join(e))
-
-        pkglist = []
-        if 'packages' in self.cfg:
-            if isinstance(self.cfg['packages'],list):
-                pkglist = self.cfg['packages']
-            else: pkglist.append(self.cfg['packages'])
-
-        if update or upgrade or pkglist:
-            #retcode = subprocess.call(list)
-            subprocess.Popen(['apt-get', 'update']).communicate()
-
-        e=os.environ.copy()
-        e['DEBIAN_FRONTEND']='noninteractive'
-
-        if upgrade:
-            subprocess.Popen(['apt-get', 'upgrade', '--assume-yes'], env=e).communicate()
-
-        if pkglist:
-            cmd=['apt-get', 'install', '--assume-yes']
-            cmd.extend(pkglist)
-            subprocess.Popen(cmd, env=e).communicate()
-
-        return(True)
-
-    def h_disable_ec2_metadata(self,name,args):
-        if util.get_cfg_option_bool(self.cfg, "disable_ec2_metadata", False):
-            fwall="route add -host 169.254.169.254 reject"
-            subprocess.call(fwall.split(' '))
-
-    def h_config_ssh(self,name,args):
-        # remove the static keys from the pristine image
-        for f in glob.glob("/etc/ssh/ssh_host_*_key*"):
-            try: os.unlink(f)
-            except: pass
-
-        if self.cfg.has_key("ssh_keys"):
-            # if there are keys in cloud-config, use them
-            key2file = {
-                "rsa_private" : ("/etc/ssh/ssh_host_rsa_key", 0600),
-                "rsa_public"  : ("/etc/ssh/ssh_host_rsa_key.pub", 0644),
-                "dsa_private" : ("/etc/ssh/ssh_host_dsa_key", 0600),
-                "dsa_public"  : ("/etc/ssh/ssh_host_dsa_key.pub", 0644)
-            }
-
-            for key,val in self.cfg["ssh_keys"].items():
-                if key2file.has_key(key):
-                    util.write_file(key2file[key][0],val,key2file[key][1])
-        else:
-            # if not, generate them
-            genkeys ='ssh-keygen -f /etc/ssh/ssh_host_rsa_key -t rsa -N ""; '
-            genkeys+='ssh-keygen -f /etc/ssh/ssh_host_dsa_key -t dsa -N ""; '
-            subprocess.call(('sh', '-c', "{ %s } </dev/null" % (genkeys)))
-
-        try:
-            user = util.get_cfg_option_str(self.cfg,'user')
-            disable_root = util.get_cfg_option_bool(self.cfg, "disable_root", True)
-            keys = self.cloud.get_public_ssh_keys()
-
-            if self.cfg.has_key("ssh_authorized_keys"):
-                cfgkeys = self.cfg["ssh_authorized_keys"]
-                keys.extend(cfgkeys)
-
-            apply_credentials(keys,user,disable_root)
-        except:
-            warn("applying credentials failed!\n")
-
-        send_ssh_keys_to_console()
-
-    def h_config_misc(self,name,args):
-        handle_updates_check(self.cfg)
-        handle_runcmd(self.cfg)
-
-    def h_config_puppet(self,name,args):
-        # If there isn't a puppet key in the configuration don't do anything
-        if not self.cfg.has_key('puppet'): return
-        puppet_cfg = self.cfg['puppet']
-        # Start by installing the puppet package ...
-        e=os.environ.copy()
-        e['DEBIAN_FRONTEND']='noninteractive'
-        # Make sure that the apt database is updated since it's not run by
-        # default
-        # Note: we should have a helper to check if apt-get update 
-        # has already been run on this instance to speed the boot time.
-        subprocess.check_call(['apt-get', 'update'], env=e)
-        subprocess.check_call(['apt-get', 'install', '--assume-yes',
-                               'puppet'], env=e)
-        # ... and then update the puppet configuration
-        if puppet_cfg.has_key('conf'):
-            # Add all sections from the conf object to puppet.conf
-            puppet_conf_fh = open('/etc/puppet/puppet.conf', 'a')
-            for cfg_name, cfg in puppet_cfg['conf'].iteritems():
-                # ca_cert configuration is a special case
-                # Dump the puppetmaster ca certificate in the correct place
-                if cfg_name == 'ca_cert':
-                    # Puppet ssl sub-directory isn't created yet
-                    # Create it with the proper permissions and ownership
-                    os.makedirs('/var/lib/puppet/ssl')
-                    os.chmod('/var/lib/puppet/ssl', 0771)
-                    os.chown('/var/lib/puppet/ssl',
-                             pwd.getpwnam('puppet').pw_uid, 0)
-                    os.makedirs('/var/lib/puppet/ssl/certs/')
-                    os.chown('/var/lib/puppet/ssl/certs/',
-                             pwd.getpwnam('puppet').pw_uid, 0)
-                    ca_fh = open('/var/lib/puppet/ssl/certs/ca.pem', 'w')
-                    ca_fh.write(cfg)
-                    ca_fh.close()
-                    os.chown('/var/lib/puppet/ssl/certs/ca.pem',
-                             pwd.getpwnam('puppet').pw_uid, 0)
-                else:
-                    puppet_conf_fh.write("\n[%s]\n" % (cfg_name))
-                    for o, v in cfg.iteritems():
-                        if o == 'certname':
-                            # Expand %f as the fqdn
-                            v = v.replace("%f", socket.getfqdn())
-                            # Expand %i as the instance id
-                            v = v.replace("%i",
-                                  self.cloud.datasource.get_instance_id())
-                            # certname needs to be downcase
-                            v = v.lower()
-                        puppet_conf_fh.write("%s=\"%s\"\n" % (o, v))
-            puppet_conf_fh.close()
-        # Set puppet default file to automatically start
-        subprocess.check_call(['sed', '-i',
-                               '-e', 's/^START=.*/START=yes/',
-                               '/etc/default/puppet'])
-        # Start puppetd
-        subprocess.check_call(['service', 'puppet', 'start'])
-
-    def h_ec2_ebs_mounts(self,name,args):
-        print "Warning, not doing anything for config %s" % name
-
-    def h_config_setup_raid(self,name,args):
-        print "Warning, not doing anything for config %s" % name
-
-    def h_config_runurl(self,name,args):
-        print "Warning, not doing anything for config %s" % name
-
-    def h_config_mounts(self,name,args):
-        # handle 'mounts'
-
-        # these are our default set of mounts
-        defmnts = [ [ "ephemeral0", "/mnt", "auto", "defaults", "0", "0" ],
-                    [ "swap", "none", "swap", "sw", "0", "0" ] ]
-
-        # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno
-        defvals = [ None, None, "auto", "defaults", "0", "0" ]
-
-        cfgmnt = [ ]
-        if self.cfg.has_key("mounts"):
-            cfgmnt = self.cfg["mounts"]
-    
-        for i in range(len(cfgmnt)):
-            # skip something that wasn't a list
-            if not isinstance(cfgmnt[i],list): continue
-
-            # workaround, allow user to specify 'ephemeral'
-            # rather than more ec2 correct 'ephemeral0'
-            if cfgmnt[i][0] == "ephemeral":
-                cfgmnt[i][0] = "ephemeral0"
-
-            newname = cfgmnt[i][0]
-            if not newname.startswith("/"):
-                newname = self.cloud.device_name_to_device(cfgmnt[i][0])
-            if newname is not None:
-                cfgmnt[i][0] = newname
-            else:
-                # there is no good way of differenciating between
-                # a name that *couldn't* exist in the md service and
-                # one that merely didnt
-                # in order to allow user to specify 'sda3' rather
-                # than '/dev/sda3', go through some hoops
-                ok = False
-                for f in [ "/", "sd", "hd", "vd", "xvd" ]:
-                    if cfgmnt[i][0].startswith(f):
-                        ok = True
-                        break
-                if not ok:
-                    cfgmnt[i][1] = None
-    
-        for i in range(len(cfgmnt)):
-            # fill in values with 
-            for j in range(len(defvals)):
-                if len(cfgmnt[i]) <= j:
-                    cfgmnt[i].append(defvals[j])
-                elif cfgmnt[i][j] is None:
-                    cfgmnt[i][j] = defvals[j]
-    
-            if not cfgmnt[i][0].startswith("/"):
-                cfgmnt[i][0]="/dev/%s" % cfgmnt[i][0]
-    
-            # if the second entry in the list is 'None' this
-            # clears all previous entries of that same 'fs_spec'
-            # (fs_spec is the first field in /etc/fstab, ie, that device)
-            if cfgmnt[i][1] is None:
-                for j in range(i):
-                    if cfgmnt[j][0] == cfgmnt[i][0]:
-                        cfgmnt[j][1] = None
-    
-
-        # for each of the "default" mounts, add them only if no other
-        # entry has the same device name
-        for defmnt in defmnts:
-            devname = self.cloud.device_name_to_device(defmnt[0])
-            if devname is None: continue
-            if devname.startswith("/"):
-                defmnt[0] = devname
-            else:
-                defmnt[0] = "/dev/%s" % devname
-
-            cfgmnt_has = False
-            for cfgm in cfgmnt:
-                if cfgm[0] == defmnt[0]:
-                    cfgmnt_has = True
-                    break
-            
-            if cfgmnt_has: continue
-            cfgmnt.append(defmnt)
-
-    
-        # now, each entry in the cfgmnt list has all fstab values
-        # if the second field is None (not the string, the value) we skip it
-        actlist = filter(lambda x: x[1] is not None, cfgmnt)
-    
-        if len(actlist) == 0: return
-    
-        comment="comment=cloudconfig"
-        cc_lines = [ ]
-        needswap = False
-        dirs = [ ]
-        for line in actlist:
-            # write 'comment' in the fs_mntops, entry,  claiming this
-            line[3]="%s,comment=cloudconfig" % line[3]
-            if line[2] == "swap": needswap = True
-            if line[1].startswith("/"): dirs.append(line[1])
-            cc_lines.append('\t'.join(line))
-
-        fstab_lines = [ ]
-        fstab=open("/etc/fstab","r+")
-        ws = re.compile("[%s]+" % string.whitespace)
-        for line in fstab.read().splitlines():
-            try:
-                toks = ws.split(line)
-                if toks[3].find(comment) != -1: continue
-            except:
-                pass
-            fstab_lines.append(line)
-
-        fstab_lines.extend(cc_lines)
-            
-        fstab.seek(0)
-        fstab.write("%s\n" % '\n'.join(fstab_lines))
-        fstab.truncate()
-        fstab.close()
-    
-        if needswap:
-            try: util.subp(("swapon", "-a"))
-            except: warn("Failed to enable swap")
-
-        for d in dirs:
-            if os.path.exists(d): continue
-            try: os.makedirs(d)
-            except: warn("Failed to make '%s' config-mount\n",d)
-
-        try: util.subp(("mount","-a"))
-        except: pass
-    
-
-
-    
-def apply_credentials(keys, user, disable_root):
-    keys = set(keys)
-    if user:
-        setup_user_keys(keys, user, '')
- 
-    if disable_root:
-        key_prefix = 'command="echo \'Please login as the %s user rather than root user.\';echo;sleep 10" ' % user
-    else:
-        key_prefix = ''
-
-    setup_user_keys(keys, 'root', key_prefix)
-
-def setup_user_keys(keys, user, key_prefix):
-    import pwd
-    saved_umask = os.umask(077)
-
-    pwent = pwd.getpwnam(user)
-
-    ssh_dir = '%s/.ssh' % pwent.pw_dir
-    if not os.path.exists(ssh_dir):
-        os.mkdir(ssh_dir)
-        os.chown(ssh_dir, pwent.pw_uid, pwent.pw_gid)
-
-    authorized_keys = '%s/.ssh/authorized_keys' % pwent.pw_dir
-    fp = open(authorized_keys, 'a')
-    fp.write(''.join(['%s%s\n' % (key_prefix, key) for key in keys]))
-    fp.close()
-
-    os.chown(authorized_keys, pwent.pw_uid, pwent.pw_gid)
-
-    os.umask(saved_umask)
-
-def send_ssh_keys_to_console():
-    send_keys_sh = """
-    {
-    echo
-    echo "#############################################################"
-    echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----"
-    ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key.pub
-    ssh-keygen -l -f /etc/ssh/ssh_host_dsa_key.pub
-    echo "-----END SSH HOST KEY FINGERPRINTS-----"
-    echo "#############################################################"
-    } | logger -p user.info -s -t "ec2"
-    """
-    subprocess.call(('sh', '-c', send_keys_sh))
-
-
-def warn(str):
-   sys.stderr.write("Warning:%s\n" % str)
-
-# srclist is a list of dictionaries, 
-# each entry must have: 'source'
-# may have: key, ( keyid and keyserver)
-def add_sources(srclist):
-    elst = []
-
-    for ent in srclist:
-        if not ent.has_key('source'):
-            elst.append([ "", "missing source" ])
-            continue
-
-        source=ent['source']
-        if source.startswith("ppa:"):
-            try: util.subp(["add-apt-repository",source])
-            except:
-                elst.append([source, "add-apt-repository failed"])
-            continue
-
-        if not ent.has_key('filename'):
-            ent['filename']='cloud_config_sources.list'
-
-        if not ent['filename'].startswith("/"):
-            ent['filename'] = "%s/%s" % \
-                ("/etc/apt/sources.list.d/", ent['filename'])
-
-        if ( ent.has_key('keyid') and not ent.has_key('key') ):
-            ks = "keyserver.ubuntu.com"
-            if ent.has_key('keyserver'): ks = ent['keyserver']
-            try:
-                ent['key'] = util.getkeybyid(ent['keyid'], ks)
-            except:
-                elst.append([source,"failed to get key from %s" % ks])
-                continue
-
-        if ent.has_key('key'):
-            try: util.subp(('apt-key', 'add', '-'), ent['key'])
-            except:
-                elst.append([source, "failed add key"])
-
-        try: util.write_file(ent['filename'], source + "\n")
-        except:
-            elst.append([source, "failed write to file %s" % ent['filename']])
-
-    return(elst)
-
-
-def generate_sources_list(mirror):
-    stdout, stderr = subprocess.Popen(['lsb_release', '-cs'], stdout=subprocess.PIPE).communicate()
-    codename = stdout.strip()
-
-    util.render_to_file('sources.list', '/etc/apt/sources.list', \
-        { 'mirror' : mirror, 'codename' : codename })
-
-def handle_updates_check(cfg):
-    if not util.get_cfg_option_bool(cfg, 'updates-check', True):
-        return
-    build_info = "/etc/cloud/build.info"
-    if not os.path.isfile(build_info):
-        warn("no %s file" % build_info)
-
-    avail="%s/%s" % ( cloudinit.datadir, "available.build" )
-    cmd=( "uec-query-builds", "--system-suite", "--config", "%s" % build_info,
-          "--output", "%s" % avail, "is-update-available" )
-    try:
-        util.subp(cmd)
-    except:
-        warn("failed to execute uec-query-build for updates check")
-
-    # add a cron entry for this hour and this minute every day
-    try:
-        cron=open("%s-%s" % (cronpre, "updates") ,"w")
-        cron.write("%s root %s\n" % \
-            (time.strftime("%M %H * * *"),' '.join(cmd)))
-        cron.close()
-    except:
-        warn("failed to enable cron update system check")
-
-def handle_runcmd(cfg):
-    if not cfg.has_key("runcmd"):
-        return
-    outfile="%s/runcmd" % cloudinit.user_scripts_dir
-
-    content="#!/bin/sh\n"
-    escaped="%s%s%s%s" % ( "'", '\\', "'", "'" )
-    try:
-        for args in cfg["runcmd"]:
-            # if the item is a list, wrap all items in single tick
-            # if its not, then just write it directly
-            if isinstance(args,list):
-                fixed = [ ]
-                for f in args:
-                    fixed.append("'%s'" % str(f).replace("'",escaped))
-                content="%s%s\n" % ( content, ' '.join(fixed) )
-            else:
-                content="%s%s\n" % ( content, str(args) )
-
-        util.write_file(outfile,content,0700)
-    except:
-        warn("failed to open %s for runcmd" % outfile)
-
-def mirror2lists_fileprefix(mirror):
-    file=mirror
-    # take of http:// or ftp://
-    if file.endswith("/"): file=file[0:-1]
-    pos=file.find("://")
-    if pos >= 0:
-        file=file[pos+3:]
-    file=file.replace("/","_")
-    return file
-
-def rename_apt_lists(omirror,new_mirror,lists_d="/var/lib/apt/lists"):
-    
-    oprefix="%s/%s" % (lists_d,mirror2lists_fileprefix(omirror))
-    nprefix="%s/%s" % (lists_d,mirror2lists_fileprefix(new_mirror))
-    if(oprefix==nprefix): return
-    olen=len(oprefix)
-    for file in glob.glob("%s_*" % oprefix):
-        os.rename(file,"%s%s" % (nprefix, file[olen:]))
diff --git a/cloudinit/CloudConfig/__init__.py b/cloudinit/CloudConfig/__init__.py
new file mode 100644
index 00000000..600b21ab
--- /dev/null
+++ b/cloudinit/CloudConfig/__init__.py
@@ -0,0 +1,57 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2008-2010 Canonical Ltd.
+#
+#    Author: Chuck Short <chuck.short@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+import yaml
+import cloudinit
+import cloudinit.util as util
+import sys
+
+per_instance="once-per-instance"
+per_always="always"
+
+class CloudConfig():
+    cfgfile = None
+    cfg = None
+
+    def __init__(self,cfgfile):
+        self.cloud = cloudinit.CloudInit()
+        self.cfg = self.get_config_obj(cfgfile)
+        self.cloud.get_data_source()
+
+    def get_config_obj(self,cfgfile):
+        f=file(cfgfile)
+        cfg=yaml.load(f.read())
+        f.close()
+        if cfg is None: cfg = { }
+        return(util.mergedict(cfg,self.cloud.cfg))
+
+    def handle(self, name, args, freq=None):
+        try:
+            mod = __import__("cc_" + name.replace("-","_"),globals())
+            def_freq = getattr(mod, "frequency",per_instance)
+            handler = getattr(mod, "handle")
+
+            if not freq:
+                freq = def_freq
+
+            self.cloud.sem_and_run(name, freq, handler, 
+                [ name, self.cfg, self.cloud, cloudinit.log, args ])
+        except:
+            cloudinit.log.error(traceback.format_exc())
+            raise
+
diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py
new file mode 100644
index 00000000..ab2ece93
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py
@@ -0,0 +1,120 @@
+import cloudinit.util as util
+import subprocess
+import os
+
+def handle(name,cfg,cloud,log,args):
+    update = util.get_cfg_option_bool(cfg, 'apt_update', False)
+    upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False)
+
+    if not util.get_cfg_option_bool(cfg, \
+        'apt_preserve_sources_list', False):
+        if cfg.has_key("apt_mirror"):
+            mirror = cfg["apt_mirror"]
+        else:
+            mirror = cloud.get_mirror()
+        generate_sources_list(mirror)
+        old_mir = util.get_cfg_option_str(cfg,'apt_old_mirror', \
+            "archive.ubuntu.com/ubuntu")
+        rename_apt_lists(old_mir, mirror)
+
+    # process 'apt_sources'
+    if cfg.has_key('apt_sources'):
+        errors = add_sources(cfg['apt_sources'])
+        for e in errors:
+            log.warn("Source Error: %s\n" % ':'.join(e))
+
+    pkglist = []
+    if 'packages' in cfg:
+        if isinstance(cfg['packages'],list):
+            pkglist = cfg['packages']
+        else: pkglist.append(cfg['packages'])
+
+    if update or upgrade or pkglist:
+        #retcode = subprocess.call(list)
+        subprocess.Popen(['apt-get', 'update']).communicate()
+
+    e=os.environ.copy()
+    e['DEBIAN_FRONTEND']='noninteractive'
+
+    if upgrade:
+        subprocess.Popen(['apt-get', 'upgrade', '--assume-yes'], env=e).communicate()
+
+    if pkglist:
+        cmd=['apt-get', 'install', '--assume-yes']
+        cmd.extend(pkglist)
+        subprocess.Popen(cmd, env=e).communicate()
+
+    return(True)
+
+def mirror2lists_fileprefix(mirror):
+    file=mirror
+    # take of http:// or ftp://
+    if file.endswith("/"): file=file[0:-1]
+    pos=file.find("://")
+    if pos >= 0:
+        file=file[pos+3:]
+    file=file.replace("/","_")
+    return file
+
+def rename_apt_lists(omirror,new_mirror,lists_d="/var/lib/apt/lists"):
+    
+    oprefix="%s/%s" % (lists_d,mirror2lists_fileprefix(omirror))
+    nprefix="%s/%s" % (lists_d,mirror2lists_fileprefix(new_mirror))
+    if(oprefix==nprefix): return
+    olen=len(oprefix)
+    for file in glob.glob("%s_*" % oprefix):
+        os.rename(file,"%s%s" % (nprefix, file[olen:]))
+
+def generate_sources_list(mirror):
+    stdout, stderr = subprocess.Popen(['lsb_release', '-cs'], stdout=subprocess.PIPE).communicate()
+    codename = stdout.strip()
+
+    util.render_to_file('sources.list', '/etc/apt/sources.list', \
+        { 'mirror' : mirror, 'codename' : codename })
+
+# srclist is a list of dictionaries, 
+# each entry must have: 'source'
+# may have: key, ( keyid and keyserver)
+def add_sources(srclist):
+    elst = []
+
+    for ent in srclist:
+        if not ent.has_key('source'):
+            elst.append([ "", "missing source" ])
+            continue
+
+        source=ent['source']
+        if source.startswith("ppa:"):
+            try: util.subp(["add-apt-repository",source])
+            except:
+                elst.append([source, "add-apt-repository failed"])
+            continue
+
+        if not ent.has_key('filename'):
+            ent['filename']='cloud_config_sources.list'
+
+        if not ent['filename'].startswith("/"):
+            ent['filename'] = "%s/%s" % \
+                ("/etc/apt/sources.list.d/", ent['filename'])
+
+        if ( ent.has_key('keyid') and not ent.has_key('key') ):
+            ks = "keyserver.ubuntu.com"
+            if ent.has_key('keyserver'): ks = ent['keyserver']
+            try:
+                ent['key'] = util.getkeybyid(ent['keyid'], ks)
+            except:
+                elst.append([source,"failed to get key from %s" % ks])
+                continue
+
+        if ent.has_key('key'):
+            try: util.subp(('apt-key', 'add', '-'), ent['key'])
+            except:
+                elst.append([source, "failed add key"])
+
+        try: util.write_file(ent['filename'], source + "\n")
+        except:
+            elst.append([source, "failed write to file %s" % ent['filename']])
+
+    return(elst)
+
+
diff --git a/cloudinit/CloudConfig/cc_disable_ec2_metadata.py b/cloudinit/CloudConfig/cc_disable_ec2_metadata.py
new file mode 100644
index 00000000..a4b4280f
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_disable_ec2_metadata.py
@@ -0,0 +1,9 @@
+import cloudinit.util as util
+from cloudinit.CloudConfig import per_always
+
+frequency = per_always
+
+def handle(name,cfg,cloud,log,args):
+    if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False):
+        fwall="route add -host 169.254.169.254 reject"
+        subprocess.call(fwall.split(' '))
diff --git a/cloudinit/CloudConfig/cc_foo.py b/cloudinit/CloudConfig/cc_foo.py
new file mode 100644
index 00000000..dc5d453e
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_foo.py
@@ -0,0 +1,7 @@
+import cloudinit
+import cloudinit.util as util
+from cloudinit.CloudConfig import per_instance
+
+frequency = per_instance
+def handle(name,cfg,cloud,log,args):
+   print "hi"
diff --git a/cloudinit/CloudConfig/cc_mounts.py b/cloudinit/CloudConfig/cc_mounts.py
new file mode 100644
index 00000000..8bd16240
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_mounts.py
@@ -0,0 +1,131 @@
+import cloudinit.util as util
+import os
+import re
+import string
+
+def handle(name,cfg,cloud,log,args):
+    # these are our default set of mounts
+    defmnts = [ [ "ephemeral0", "/mnt", "auto", "defaults", "0", "0" ],
+                [ "swap", "none", "swap", "sw", "0", "0" ] ]
+
+    # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno
+    defvals = [ None, None, "auto", "defaults", "0", "0" ]
+
+    cfgmnt = [ ]
+    if cfg.has_key("mounts"):
+        cfgmnt = cfg["mounts"]
+
+    for i in range(len(cfgmnt)):
+        # skip something that wasn't a list
+        if not isinstance(cfgmnt[i],list): continue
+
+        # workaround, allow user to specify 'ephemeral'
+        # rather than more ec2 correct 'ephemeral0'
+        if cfgmnt[i][0] == "ephemeral":
+            cfgmnt[i][0] = "ephemeral0"
+
+        newname = cfgmnt[i][0]
+        if not newname.startswith("/"):
+            newname = cloud.device_name_to_device(cfgmnt[i][0])
+        if newname is not None:
+            cfgmnt[i][0] = newname
+        else:
+            # there is no good way of differenciating between
+            # a name that *couldn't* exist in the md service and
+            # one that merely didnt
+            # in order to allow user to specify 'sda3' rather
+            # than '/dev/sda3', go through some hoops
+            ok = False
+            for f in [ "/", "sd", "hd", "vd", "xvd" ]:
+                if cfgmnt[i][0].startswith(f):
+                    ok = True
+                    break
+            if not ok:
+                cfgmnt[i][1] = None
+
+    for i in range(len(cfgmnt)):
+        # fill in values with 
+        for j in range(len(defvals)):
+            if len(cfgmnt[i]) <= j:
+                cfgmnt[i].append(defvals[j])
+            elif cfgmnt[i][j] is None:
+                cfgmnt[i][j] = defvals[j]
+
+        if not cfgmnt[i][0].startswith("/"):
+            cfgmnt[i][0]="/dev/%s" % cfgmnt[i][0]
+
+        # if the second entry in the list is 'None' this
+        # clears all previous entries of that same 'fs_spec'
+        # (fs_spec is the first field in /etc/fstab, ie, that device)
+        if cfgmnt[i][1] is None:
+            for j in range(i):
+                if cfgmnt[j][0] == cfgmnt[i][0]:
+                    cfgmnt[j][1] = None
+
+
+    # for each of the "default" mounts, add them only if no other
+    # entry has the same device name
+    for defmnt in defmnts:
+        devname = cloud.device_name_to_device(defmnt[0])
+        if devname is None: continue
+        if devname.startswith("/"):
+            defmnt[0] = devname
+        else:
+            defmnt[0] = "/dev/%s" % devname
+
+        cfgmnt_has = False
+        for cfgm in cfgmnt:
+            if cfgm[0] == defmnt[0]:
+                cfgmnt_has = True
+                break
+        
+        if cfgmnt_has: continue
+        cfgmnt.append(defmnt)
+
+
+    # now, each entry in the cfgmnt list has all fstab values
+    # if the second field is None (not the string, the value) we skip it
+    actlist = filter(lambda x: x[1] is not None, cfgmnt)
+
+    if len(actlist) == 0: return
+
+    comment="comment=cloudconfig"
+    cc_lines = [ ]
+    needswap = False
+    dirs = [ ]
+    for line in actlist:
+        # write 'comment' in the fs_mntops, entry,  claiming this
+        line[3]="%s,comment=cloudconfig" % line[3]
+        if line[2] == "swap": needswap = True
+        if line[1].startswith("/"): dirs.append(line[1])
+        cc_lines.append('\t'.join(line))
+
+    fstab_lines = [ ]
+    fstab=open("/etc/fstab","r+")
+    ws = re.compile("[%s]+" % string.whitespace)
+    for line in fstab.read().splitlines():
+        try:
+            toks = ws.split(line)
+            if toks[3].find(comment) != -1: continue
+        except:
+            pass
+        fstab_lines.append(line)
+
+    fstab_lines.extend(cc_lines)
+        
+    fstab.seek(0)
+    fstab.write("%s\n" % '\n'.join(fstab_lines))
+    fstab.truncate()
+    fstab.close()
+
+    if needswap:
+        try: util.subp(("swapon", "-a"))
+        except: log.warn("Failed to enable swap")
+
+    for d in dirs:
+        if os.path.exists(d): continue
+        try: os.makedirs(d)
+        except: log.warn("Failed to make '%s' config-mount\n",d)
+
+    try: util.subp(("mount","-a"))
+    except: pass
diff --git a/cloudinit/CloudConfig/cc_puppet.py b/cloudinit/CloudConfig/cc_puppet.py
new file mode 100644
index 00000000..542bced0
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_puppet.py
@@ -0,0 +1,59 @@
+import os
+import subprocess
+
+def handle(name,cfg,cloud,log,args):
+    # If there isn't a puppet key in the configuration don't do anything
+    if not cfg.has_key('puppet'): return
+    puppet_cfg = cfg['puppet']
+    # Start by installing the puppet package ...
+    e=os.environ.copy()
+    e['DEBIAN_FRONTEND']='noninteractive'
+    # Make sure that the apt database is updated since it's not run by
+    # default
+    # Note: we should have a helper to check if apt-get update 
+    # has already been run on this instance to speed the boot time.
+    subprocess.check_call(['apt-get', 'update'], env=e)
+    subprocess.check_call(['apt-get', 'install', '--assume-yes',
+                           'puppet'], env=e)
+    # ... and then update the puppet configuration
+    if puppet_cfg.has_key('conf'):
+        # Add all sections from the conf object to puppet.conf
+        puppet_conf_fh = open('/etc/puppet/puppet.conf', 'a')
+        for cfg_name, cfg in puppet_cfg['conf'].iteritems():
+            # ca_cert configuration is a special case
+            # Dump the puppetmaster ca certificate in the correct place
+            if cfg_name == 'ca_cert':
+                # Puppet ssl sub-directory isn't created yet
+                # Create it with the proper permissions and ownership
+                os.makedirs('/var/lib/puppet/ssl')
+                os.chmod('/var/lib/puppet/ssl', 0771)
+                os.chown('/var/lib/puppet/ssl',
+                         pwd.getpwnam('puppet').pw_uid, 0)
+                os.makedirs('/var/lib/puppet/ssl/certs/')
+                os.chown('/var/lib/puppet/ssl/certs/',
+                         pwd.getpwnam('puppet').pw_uid, 0)
+                ca_fh = open('/var/lib/puppet/ssl/certs/ca.pem', 'w')
+                ca_fh.write(cfg)
+                ca_fh.close()
+                os.chown('/var/lib/puppet/ssl/certs/ca.pem',
+                         pwd.getpwnam('puppet').pw_uid, 0)
+            else:
+                puppet_conf_fh.write("\n[%s]\n" % (cfg_name))
+                for o, v in cfg.iteritems():
+                    if o == 'certname':
+                        # Expand %f as the fqdn
+                        v = v.replace("%f", socket.getfqdn())
+                        # Expand %i as the instance id
+                        v = v.replace("%i",
+                              cloud.datasource.get_instance_id())
+                        # certname needs to be downcase
+                        v = v.lower()
+                    puppet_conf_fh.write("%s=\"%s\"\n" % (o, v))
+        puppet_conf_fh.close()
+    # Set puppet default file to automatically start
+    subprocess.check_call(['sed', '-i',
+                           '-e', 's/^START=.*/START=yes/',
+                           '/etc/default/puppet'])
+    # Start puppetd
+    subprocess.check_call(['service', 'puppet', 'start'])
+
diff --git a/cloudinit/CloudConfig/cc_runcmd.py b/cloudinit/CloudConfig/cc_runcmd.py
new file mode 100644
index 00000000..fb5739da
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_runcmd.py
@@ -0,0 +1,25 @@
+import cloudinit
+import cloudinit.util as util
+
+def handle(name,cfg,cloud,log,args):
+    if not cfg.has_key("runcmd"):
+        return
+    outfile="%s/runcmd" % cloudinit.user_scripts_dir
+
+    content="#!/bin/sh\n"
+    escaped="%s%s%s%s" % ( "'", '\\', "'", "'" )
+    try:
+        for args in cfg["runcmd"]:
+            # if the item is a list, wrap all items in single tick
+            # if its not, then just write it directly
+            if isinstance(args,list):
+                fixed = [ ]
+                for f in args:
+                    fixed.append("'%s'" % str(f).replace("'",escaped))
+                content="%s%s\n" % ( content, ' '.join(fixed) )
+            else:
+                content="%s%s\n" % ( content, str(args) )
+
+        util.write_file(outfile,content,0700)
+    except:
+        log.warn("failed to open %s for runcmd" % outfile)
diff --git a/cloudinit/CloudConfig/cc_ssh.py b/cloudinit/CloudConfig/cc_ssh.py
new file mode 100644
index 00000000..08d1e243
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_ssh.py
@@ -0,0 +1,91 @@
+import cloudinit.util as util
+import os
+import glob
+import subprocess
+
+def handle(name,cfg,cloud,log,args):
+    # remove the static keys from the pristine image
+    for f in glob.glob("/etc/ssh/ssh_host_*_key*"):
+        try: os.unlink(f)
+        except: pass
+
+    if cfg.has_key("ssh_keys"):
+        # if there are keys in cloud-config, use them
+        key2file = {
+            "rsa_private" : ("/etc/ssh/ssh_host_rsa_key", 0600),
+            "rsa_public"  : ("/etc/ssh/ssh_host_rsa_key.pub", 0644),
+            "dsa_private" : ("/etc/ssh/ssh_host_dsa_key", 0600),
+            "dsa_public"  : ("/etc/ssh/ssh_host_dsa_key.pub", 0644)
+        }
+
+        for key,val in cfg["ssh_keys"].items():
+            if key2file.has_key(key):
+                util.write_file(key2file[key][0],val,key2file[key][1])
+    else:
+        # if not, generate them
+        genkeys ='ssh-keygen -f /etc/ssh/ssh_host_rsa_key -t rsa -N ""; '
+        genkeys+='ssh-keygen -f /etc/ssh/ssh_host_dsa_key -t dsa -N ""; '
+        subprocess.call(('sh', '-c', "{ %s } </dev/null" % (genkeys)))
+
+    try:
+        user = util.get_cfg_option_str(cfg,'user')
+        disable_root = util.get_cfg_option_bool(cfg, "disable_root", True)
+        keys = cloud.get_public_ssh_keys()
+
+        if cfg.has_key("ssh_authorized_keys"):
+            cfgkeys = cfg["ssh_authorized_keys"]
+            keys.extend(cfgkeys)
+
+        apply_credentials(keys,user,disable_root)
+    except:
+        log.warn("applying credentials failed!\n")
+
+    send_ssh_keys_to_console()
+
+def send_ssh_keys_to_console():
+    send_keys_sh = """
+    {
+    echo
+    echo "#############################################################"
+    echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----"
+    ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key.pub
+    ssh-keygen -l -f /etc/ssh/ssh_host_dsa_key.pub
+    echo "-----END SSH HOST KEY FINGERPRINTS-----"
+    echo "#############################################################"
+    } | logger -p user.info -s -t "ec2"
+    """
+    subprocess.call(('sh', '-c', send_keys_sh))
+
+def apply_credentials(keys, user, disable_root):
+    keys = set(keys)
+    if user:
+        setup_user_keys(keys, user, '')
+ 
+    if disable_root:
+        key_prefix = 'command="echo \'Please login as the %s user rather than root user.\';echo;sleep 10" ' % user
+    else:
+        key_prefix = ''
+
+    setup_user_keys(keys, 'root', key_prefix)
+
+def setup_user_keys(keys, user, key_prefix):
+    import pwd
+    saved_umask = os.umask(077)
+
+    pwent = pwd.getpwnam(user)
+
+    ssh_dir = '%s/.ssh' % pwent.pw_dir
+    if not os.path.exists(ssh_dir):
+        os.mkdir(ssh_dir)
+        os.chown(ssh_dir, pwent.pw_uid, pwent.pw_gid)
+
+    authorized_keys = '%s/.ssh/authorized_keys' % pwent.pw_dir
+    fp = open(authorized_keys, 'a')
+    fp.write(''.join(['%s%s\n' % (key_prefix, key) for key in keys]))
+    fp.close()
+
+    os.chown(authorized_keys, pwent.pw_uid, pwent.pw_gid)
+
+    os.umask(saved_umask)
+
+
diff --git a/cloudinit/CloudConfig/cc_updates_check.py b/cloudinit/CloudConfig/cc_updates_check.py
new file mode 100644
index 00000000..b24b8c61
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_updates_check.py
@@ -0,0 +1,32 @@
+import cloudinit.util as util
+import cloudinit
+import os
+import time
+
+cronpre = "/etc/cron.d/cloudinit"
+
+def handle(name,cfg,cloud,log,args):
+    if not util.get_cfg_option_bool(cfg, 'updates-check', True):
+        return
+    build_info = "/etc/cloud/build.info"
+    if not os.path.isfile(build_info):
+        log.warn("no %s file" % build_info)
+
+    avail="%s/%s" % ( cloudinit.datadir, "available.build" )
+    cmd=( "uec-query-builds", "--system-suite", "--config", "%s" % build_info,
+          "--output", "%s" % avail, "is-update-available" )
+    try:
+        util.subp(cmd)
+    except:
+        log.warn("failed to execute uec-query-build for updates check")
+
+    # add a cron entry for this hour and this minute every day
+    try:
+        cron=open("%s-%s" % (cronpre, "updates") ,"w")
+        cron.write("%s root %s\n" % \
+            (time.strftime("%M %H * * *"),' '.join(cmd)))
+        cron.close()
+    except:
+        log.warn("failed to enable cron update system check")
+        raise
+
diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py
index d8f2c8c9..2b76a73f 100644
--- a/cloudinit/__init__.py
+++ b/cloudinit/__init__.py
@@ -26,6 +26,7 @@ userdata_raw = datadir + '/user-data.txt'
 userdata = datadir + '/user-data.txt.i'
 user_scripts_dir = datadir + "/scripts"
 cloud_config = datadir + '/cloud-config.txt'
+#cloud_config = '/tmp/cloud-config.txt'
 data_source_cache = cachedir + '/obj.pkl'
 system_config = '/etc/cloud/cloud.cfg'
 cfg_env_name = "CLOUD_CFG"
@@ -36,12 +37,13 @@ user: ubuntu
 disable_root: 1
 
 cloud_config_modules:
+ - mounts
+ - ssh
  - apt-update-upgrade
- - config-misc
- - config-mounts
- - config-puppet
- - config-ssh
+ - puppet
+ - updates-check
  - disable-ec2-metadata
+ - runcmd
 
 log_cfg: built_in
 """
@@ -316,7 +318,9 @@ class CloudInit:
     # if 'clear_on_fail' is True and func throws an exception
     #  then remove the lock (so it would run again)
     def sem_and_run(self,semname,freq,func,args=[],clear_on_fail=False):
-        if self.sem_has_run(semname,freq): return
+        if self.sem_has_run(semname,freq):
+            log.debug("%s already ran %s", semname, freq)
+            return
         try:
             if not self.sem_acquire(semname,freq):
                 raise Exception("Failed to acquire lock on %s\n" % semname)