add function to cloud-init to run cloud-config style modules

add 'hostname' cloud-config option for setting hostname
make rsyslog and resizefs run at cloud-init time

10 files changed, 251 insertions, 128 deletions

diff --git a/ChangeLog b/ChangeLog
index c23f6503..42f54564 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -4,3 +4,15 @@
  - rework of /var/lib/cloud layout
  - remove updates-check (LP: #653220)
  - support resizing / on first boot (enabled by default)
+ - added support for running CloudConfig modules at cloud-init time
+   rather than cloud-config time, and the new 'cloud_init_modules'
+   entry in cloud.cfg to indicate which should run then.
+   The driving force behind this was to have the rsyslog module
+   able to run before rsyslog even runs so that a restart would
+   not be needed (rsyslog on ubuntu runs on 'filesystem')
+ - moved setting and updating of hostname to cloud_init_modules
+   this allows the user to easily disable these from running.
+   This also means:
+   - the semaphore name for 'set_hostname' and 'update_hostname'
+     changes to 'config_set_hostname' and 'config_update_hostname'
+ - added cloud-config option 'hostname' for setting hostname
diff --git a/cloud-init-cfg.py b/cloud-init-cfg.py
index 326062a8..b1c63a17 100755
--- a/cloud-init-cfg.py
+++ b/cloud-init-cfg.py
@@ -19,7 +19,7 @@
 
 import sys
 import cloudinit
-import cloudinit.CloudConfig
+import cloudinit.CloudConfig as CC
 import logging
 import os
 import traceback
@@ -60,47 +60,20 @@ def main():
     if os.environ.has_key(cfg_env_name):
         cfg_path = os.environ[cfg_env_name]
 
-    cc = cloudinit.CloudConfig.CloudConfig(cfg_path)
+    cc = CC.CloudConfig(cfg_path)
 
     module_list = [ ]
     if name == "all":
-        # create 'module_list', an array of arrays
-        # where array[0] = config
-        #       array[1] = freq
-        #       array[2:] = arguemnts
-        if "cloud_config_modules" in cc.cfg:
-            for item in cc.cfg["cloud_config_modules"]:
-                if isinstance(item,str):
-                    module_list.append((item,))
-                elif isinstance(item,list):
-                    module_list.append(item)
-                else:
-                    fail("Failed to parse cloud_config_modules",log)
-        else:
-            fail("No cloud_config_modules found in config",log)
+        modules_list = CC.read_cc_modules(cc.cfg,"cloud_config_modules")
+        if not len(modules_list):
+            err("no modules to run in cloud_config",log)
+            sys.exit(0)
     else:
         module_list.append( [ name, freq ] + run_args )
 
-    failures = []
-    for cfg_mod in module_list:
-        name = cfg_mod[0]
-        freq = None
-        run_args = [ ]
-        if len(cfg_mod) > 1:
-            freq = cfg_mod[1]
-        if len(cfg_mod) > 2:
-            run_args = cfg_mod[2:]
-
-        try:
-            log.debug("handling %s with freq=%s and args=%s" %
-                (name, freq, run_args ))
-            cc.handle(name, run_args, freq=freq)
-        except:
-            log.warn(traceback.format_exc())
-            err("config handling of %s, %s, %s failed\n" %
-                (name,freq,run_args), log)
-            failures.append(name)
-
+    failures = CC.run_cc_modules(cc,module_list,log)
+    if len(failures):
+        err("errors running cloud_config modules: %s" % failures)
     sys.exit(len(failures))
 
 def err(msg,log=None):
diff --git a/cloud-init.py b/cloud-init.py
index bdfa9f01..7c261c26 100755
--- a/cloud-init.py
+++ b/cloud-init.py
@@ -22,6 +22,7 @@ import sys
 
 import cloudinit
 import cloudinit.util as util
+import cloudinit.CloudConfig as CC
 import time
 import logging
 import errno
@@ -98,92 +99,23 @@ def main():
         warn("consuming user data failed!\n")
         raise
 
-    try:
-        if util.get_cfg_option_bool(cloud.cfg,"preserve_hostname",False):
-            log.debug("preserve_hostname is set. not managing hostname")
-        else:
-            hostname = cloud.get_hostname()
-            cloud.sem_and_run("set_hostname", "once-per-instance",
-                set_hostname, [ hostname, log ], False)
-            cloud.sem_and_run("update_hostname", "always",
-                update_hostname, [ hostname, log ], False)
-    except Exception, e:
-        util.logexc(log)
-        warn("failed to set hostname\n")
-
-    #print "user data is:" + cloud.get_user_data()
-
     # finish, send the cloud-config event
     cloud.initctl_emit()
 
-    sys.exit(0)
-
-# read hostname from a 'hostname' file
-# allow for comments and stripping line endings.
-# if file doesn't exist, or no contents, return default
-def read_hostname(filename, default=None):
-    try:
-        fp = open(filename,"r")
-        lines = fp.readlines()
-        fp.close()
-        for line in lines:
-            hpos = line.find("#")
-            if hpos != -1:
-                line = line[0:hpos]
-            line = line.rstrip()
-            if line:
-                return line
-    except IOError, e:
-        if e.errno == errno.ENOENT: pass
-    return default
-    
-def set_hostname(hostname, log):
-    try:
-        subprocess.Popen(['hostname', hostname]).communicate()
-        util.write_file("/etc/hostname","%s\n" % hostname, 0644)
-        log.debug("populated /etc/hostname with %s on first boot", hostname)
-    except:
-        log.error("failed to set_hostname")
-
-def update_hostname(hostname, log):
-    prev_file="%s/%s" % (cloudinit.get_cpath('datadir'),"previous-hostname")
-    etc_file = "/etc/hostname"
-
-    hostname_prev = None
-    hostname_in_etc = None
-
-    try:
-        hostname_prev = read_hostname(prev_file)
-    except:
-        log.warn("Failed to open %s" % prev_file)
-    
-    try:
-        hostname_in_etc = read_hostname(etc_file)
-    except:
-        log.warn("Failed to open %s" % etc_file)
-
-    update_files = []
-    if not hostname_prev or hostname_prev != hostname:
-        update_files.append(prev_file)
-
-    if (not hostname_in_etc or 
-        (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)):
-        update_files.append(etc_file)
-
-    try:
-        for fname in update_files:
-            util.write_file(fname,"%s\n" % hostname, 0644)
-            log.debug("wrote %s to %s" % (hostname,fname))
-    except:
-        log.warn("failed to write hostname to %s" % fname)
+    cfg_path = cloudinit.get_ipath_cur("cloud_config")
+    cc = CC.CloudConfig(cfg_path, cloud)
+    modules_list = CC.read_cc_modules(cc.cfg,"cloud_init_modules")
 
-    if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev:
-        log.debug("%s differs from %s. assuming user maintained" %
-                  (prev_file,etc_file))
+    failures = []
+    if len(modules_list):
+        failures = CC.run_cc_modules(cc,module_list,log)
+    else:
+        msg = "no cloud_init_modules to run"
+        sys.stderr.write(msg + "\n")
+        log.debug(msg)
+        sys.exit(0)
 
-    if etc_file in update_files:
-        log.debug("setting hostname to %s" % hostname)
-        subprocess.Popen(['hostname', hostname]).communicate()
+    sys.exit(len(failures))
 
 if __name__ == '__main__':
     main()
diff --git a/cloudinit/CloudConfig/__init__.py b/cloudinit/CloudConfig/__init__.py
index 0a91059a..bfed44b7 100644
--- a/cloudinit/CloudConfig/__init__.py
+++ b/cloudinit/CloudConfig/__init__.py
@@ -29,8 +29,11 @@ class CloudConfig():
     cfgfile = None
     cfg = None
 
-    def __init__(self,cfgfile):
-        self.cloud = cloudinit.CloudInit()
+    def __init__(self,cfgfile, cloud=None):
+        if cloud == None:
+            self.cloud = cloudinit.CloudInit()
+        else:
+            self.cloud = cloud
         self.cfg = self.get_config_obj(cfgfile)
         self.cloud.get_data_source()
 
@@ -38,6 +41,7 @@ class CloudConfig():
         try:
             cfg = util.read_conf(cfgfile)
         except:
+            # TODO: this 'log' could/should be passed in
             cloudinit.log.critical("Failed loading of cloud config '%s'. Continuing with empty config\n" % cfgfile)
             cloudinit.log.debug(traceback.format_exc() + "\n")
             cfg = None
@@ -58,3 +62,43 @@ class CloudConfig():
         except:
             raise
 
+# reads a cloudconfig module list, returns
+# a 2 dimensional array suitable to pass to run_cc_modules
+def read_cc_modules(cfg,name):
+    if name not in cfg: return([])
+    module_list = []
+    # create 'module_list', an array of arrays
+    # where array[0] = config
+    #       array[1] = freq
+    #       array[2:] = arguemnts
+    for item in cfg[name]:
+        if isinstance(item,str):
+            module_list.append((item,))
+        elif isinstance(item,list):
+            module_list.append(item)
+        else:
+            raise TypeError("failed to read '%s' item in config")
+    return(module_list)
+    
+def run_cc_modules(cc,module_list,log):
+    failures = []
+    for cfg_mod in module_list:
+        name = cfg_mod[0]
+        freq = None
+        run_args = [ ]
+        if len(cfg_mod) > 1:
+            freq = cfg_mod[1]
+        if len(cfg_mod) > 2:
+            run_args = cfg_mod[2:]
+
+        try:
+            log.debug("handling %s with freq=%s and args=%s" %
+                (name, freq, run_args ))
+            cc.handle(name, run_args, freq=freq)
+        except:
+            log.warn(traceback.format_exc())
+            log.err("config handling of %s, %s, %s failed\n" %
+                (name,freq,run_args))
+            failures.append(name)
+
+    return(failures)
diff --git a/cloudinit/CloudConfig/cc_rsyslog.py b/cloudinit/CloudConfig/cc_rsyslog.py
index 53fa1d23..3320dbb2 100644
--- a/cloudinit/CloudConfig/cc_rsyslog.py
+++ b/cloudinit/CloudConfig/cc_rsyslog.py
@@ -33,7 +33,7 @@ def handle(name,cfg,cloud,log,args):
     #      *.*   @@syslogd.example.com
 
     # process 'rsyslog'
-    if not 'rsyslog' in cfg: return True
+    if not 'rsyslog' in cfg: return
 
     def_dir = cfg.get('rsyslog_dir', DEF_DIR)
     def_fname = cfg.get('rsyslog_filename', DEF_FILENAME)
@@ -69,18 +69,31 @@ def handle(name,cfg,cloud,log,args):
             elst.append((content, "failed to write to %s" % filename))
 
     # need to restart syslogd
+    restarted = False
     try:
+        # if this config module is running at cloud-init time 
+        # (before rsyslog is running) we don't actually have to
+        # restart syslog.
+        #
+        # upstart actually does what we want here, in that it doesn't
+        # start a service that wasn't running already on 'restart'
+        # it will also return failure on the attempt, so 'restarted'
+        # won't get set
         log.debug("restarting rsyslog")
         p = util.subp(['service', 'rsyslog', 'restart'])
+        restarted = True
+
     except Exception, e:
         elst.append(("restart", str(e)))
     
+    if restarted:
+        # this only needs to run if we *actually* restarted
+        # syslog above.
+        cloudinit.logging_set_from_cfg_file()
+        log = logging.getLogger()
+        log.debug("rsyslog configured %s" % files)
+
     for e in elst:
         log.warn("rsyslog error: %s\n" % ':'.join(e))
-        return False
-
-    cloudinit.logging_set_from_cfg_file()
-    log = logging.getLogger()
-    log.debug("rsyslog configured %s" % files)
 
-    return True
+    return
diff --git a/cloudinit/CloudConfig/cc_set_hostname.py b/cloudinit/CloudConfig/cc_set_hostname.py
new file mode 100644
index 00000000..34621e97
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_set_hostname.py
@@ -0,0 +1,38 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2011 Canonical Ltd.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+import cloudinit.util as util
+import subprocess
+
+def handle(name,cfg,cloud,log,args):
+    if util.get_cfg_option_bool(cfg,"preserve_hostname",False):
+        log.debug("preserve_hostname is set. not setting hostname")
+        return(True)
+
+    try:
+        hostname = util.get_cfg_option_str(cfg,"hostname",cloud.get_hostname())
+        set_hostname(hostname, log)
+    except Exception, e:
+        util.logexc(log)
+        log.warn("failed to set hostname\n")
+
+    return(True)
+
+def set_hostname(hostname, log):
+    subprocess.Popen(['hostname', hostname]).communicate()
+    util.write_file("/etc/hostname","%s\n" % hostname, 0644)
+    log.debug("populated /etc/hostname with %s on first boot", hostname)
diff --git a/cloudinit/CloudConfig/cc_update_hostname.py b/cloudinit/CloudConfig/cc_update_hostname.py
new file mode 100644
index 00000000..c06a434b
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_update_hostname.py
@@ -0,0 +1,94 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2011 Canonical Ltd.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+import cloudinit.util as util
+import subprocess
+from cloudinit.CloudConfig import per_always
+
+frequency = per_always
+
+def handle(name,cfg,cloud,log,args):
+    if util.get_cfg_option_bool(cfg,"preserve_hostname",False):
+        log.debug("preserve_hostname is set. not updating hostname")
+        return
+
+    try:
+        hostname = util.get_cfg_option_str(cfg,"hostname",cloud.get_hostname())
+        prev ="%s/%s" % (cloud.get_cpath('datadir'),"previous-hostname")
+        update_hostname(hostname, prev, log)
+    except Exception, e:
+        log.warn("failed to set hostname\n")
+        raise
+
+# read hostname from a 'hostname' file
+# allow for comments and stripping line endings.
+# if file doesn't exist, or no contents, return default
+def read_hostname(filename, default=None):
+    try:
+        fp = open(filename,"r")
+        lines = fp.readlines()
+        fp.close()
+        for line in lines:
+            hpos = line.find("#")
+            if hpos != -1:
+                line = line[0:hpos]
+            line = line.rstrip()
+            if line:
+                return line
+    except IOError, e:
+        if e.errno == errno.ENOENT: pass
+    return default
+    
+def update_hostname(hostname, previous, log):
+    etc_file = "/etc/hostname"
+
+    hostname_prev = None
+    hostname_in_etc = None
+
+    try:
+        hostname_prev = read_hostname(prev_file)
+    except:
+        log.warn("Failed to open %s" % prev_file)
+    
+    try:
+        hostname_in_etc = read_hostname(etc_file)
+    except:
+        log.warn("Failed to open %s" % etc_file)
+
+    update_files = []
+    if not hostname_prev or hostname_prev != hostname:
+        update_files.append(prev_file)
+
+    if (not hostname_in_etc or 
+        (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)):
+        update_files.append(etc_file)
+
+    try:
+        for fname in update_files:
+            util.write_file(fname,"%s\n" % hostname, 0644)
+            log.debug("wrote %s to %s" % (hostname,fname))
+    except:
+        log.warn("failed to write hostname to %s" % fname)
+
+    if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev:
+        log.debug("%s differs from %s. assuming user maintained" %
+                  (prev_file,etc_file))
+
+    if etc_file in update_files:
+        log.debug("setting hostname to %s" % hostname)
+        subprocess.Popen(['hostname', hostname]).communicate()
+
diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py
index 889455d2..12bd0fe1 100644
--- a/cloudinit/__init__.py
+++ b/cloudinit/__init__.py
@@ -478,6 +478,13 @@ class CloudInit:
     def device_name_to_device(self,name):
         return(self.datasource.device_name_to_device(name))
 
+    # I really don't know if this should be here or not, but
+    # I needed it in cc_update_hostname, where that code had a valid 'cloud'
+    # reference, but did not have a cloudinit handle
+    # (ie, no cloudinit.get_cpath())
+    def get_cpath(self,name=None):
+        return(get_cpath,name)
+
 
 def initfs():
     subds = [ 'scripts/per-instance', 'scripts/per-once', 'scripts/per-boot',
@@ -507,6 +514,7 @@ def purge_cache():
         return(False)
     return(True)
 
+# get_ipath_cur: get the current instance path for an item
 def get_ipath_cur(name=None):
     return("%s/instance/%s" % (varlibdir, pathmap[name]))
 
diff --git a/config/cloud.cfg b/config/cloud.cfg
index da8c70a6..90aecb6a 100644
--- a/config/cloud.cfg
+++ b/config/cloud.cfg
@@ -3,10 +3,14 @@ user: ubuntu
 disable_root: 1
 preserve_hostname: False
 
-cloud_config_modules:
+cloud_init_modules:
  - resizefs
- - mounts
+ - set_hostname
+ - update_hostname
  - rsyslog
+
+cloud_config_modules:
+ - mounts
  - ssh-import-id
  - locale
  - ssh
diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt
index 0737aea7..41ff7924 100644
--- a/doc/examples/cloud-config.txt
+++ b/doc/examples/cloud-config.txt
@@ -281,3 +281,8 @@ rsyslog:
 # and have the instance automatically grow / to accomoddate it
 # set to 'False' to disable
 resize_rootfs: True
+
+# if hostname is set, cloud-init will set the system hostname 
+# appropriately to its value
+# if not set, it will set hostname from the cloud metadata
+# default: None