move cloud-run-user-script.conf to cloud-final, use cloud-cfg for invoking

This moves what was done as cloud-run-user-script.conf to 'cloud-final'
and makes that re-use the cloud-init-cfg code, but simply with a different
set of default configs.

Also, adds keys_to_console and final_message cloud-config modules

LP: #653271

17 files changed, 307 insertions, 37 deletions

diff --git a/ChangeLog b/ChangeLog
index 42f54564..7a312263 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -16,3 +16,14 @@
    - the semaphore name for 'set_hostname' and 'update_hostname'
      changes to 'config_set_hostname' and 'config_update_hostname'
  - added cloud-config option 'hostname' for setting hostname
+ - moved upstart/cloud-run-user-script.conf to upstart/cloud-final.conf
+ - cloud-final.conf now runs runs cloud-config modules similar
+   to cloud-config and cloud-init.
+ - LP: #653271 
+   - added writing of "boot-finished" to /var/lib/cloud/instance/boot-finished
+     this is the last thing done, indicating cloud-init is finished booting
+   - writes message to console with timestamp and uptime
+ - write ssh keys to console as one of the last things done
+   this is to ensure they don't get run off the 'get-console-ouptut' buffer
+ - user_scripts run via cloud-final and thus semaphore renamed from
+   user_scripts to config_user_scripts
diff --git a/cloud-init-cfg.py b/cloud-init-cfg.py
index 98bf63ce..dd8bd7f1 100755
--- a/cloud-init-cfg.py
+++ b/cloud-init-cfg.py
@@ -68,8 +68,8 @@ def main():
     module_list = [ ]
     if name == "all":
         modlist_cfg_name = "%s_modules" % modlist
-        modules_list = CC.read_cc_modules(cc.cfg,modlist_cfg_name)
-        if not len(modules_list):
+        module_list = CC.read_cc_modules(cc.cfg,modlist_cfg_name)
+        if not len(module_list):
             err("no modules to run in cloud_config [%s]" % modlist,log)
             sys.exit(0)
     else:
diff --git a/cloudinit/CloudConfig/__init__.py b/cloudinit/CloudConfig/__init__.py
index ab2c3573..6932acbe 100644
--- a/cloudinit/CloudConfig/__init__.py
+++ b/cloudinit/CloudConfig/__init__.py
@@ -98,7 +98,7 @@ def run_cc_modules(cc,module_list,log):
             cc.handle(name, run_args, freq=freq)
         except:
             log.warn(traceback.format_exc())
-            log.err("config handling of %s, %s, %s failed\n" %
+            log.error("config handling of %s, %s, %s failed\n" %
                 (name,freq,run_args))
             failures.append(name)
 
diff --git a/cloudinit/CloudConfig/cc_final_message.py b/cloudinit/CloudConfig/cc_final_message.py
new file mode 100644
index 00000000..febfd017
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_final_message.py
@@ -0,0 +1,55 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2011 Canonical Ltd.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+from cloudinit.CloudConfig import per_instance
+import sys
+from cloudinit import util, boot_finished
+
+frequency = per_instance
+
+final_message = "cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds"
+
+def handle(name,cfg,cloud,log,args):
+    if len(args) != 0:
+        msg_in = args[0]
+    else:
+        msg_in = util.get_cfg_option_str(cfg,"final_message",final_message)
+
+    try:
+        uptimef=open("/proc/uptime")
+        uptime=uptimef.read().split(" ")[0]
+        uptimef.close()
+    except IOError as e:
+        log.warn("unable to open /proc/uptime\n")
+        uptime = "na"
+
+
+    try:
+        from datetime import datetime
+        ts = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S +0000')
+    except:
+        ts = "na"
+
+    try:
+        subs = { 'UPTIME' : uptime, 'TIMESTAMP' : ts }
+        sys.stdout.write(util.render_string(msg_in, subs))
+    except Exception as e:
+        log.warn("failed to render string to stdout: %s" % e)
+
+    fp = open(boot_finished, "wb")
+    fp.write(uptime + "\n")
+    fp.close()
diff --git a/cloudinit/CloudConfig/cc_keys_to_console.py b/cloudinit/CloudConfig/cc_keys_to_console.py
new file mode 100644
index 00000000..47227b76
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_keys_to_console.py
@@ -0,0 +1,31 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2011 Canonical Ltd.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+from cloudinit.CloudConfig import per_instance
+import subprocess
+
+frequency = per_instance
+
+def handle(name,cfg,cloud,log,args):
+    write_ssh_prog='/usr/lib/cloud-init/write-ssh-key-fingerprints'
+    try:
+        confp = open('/dev/console',"wb")
+        subprocess.call(write_ssh_prog,stdout=confp)
+        confp.close()
+    except:
+        log.warn("writing keys to console value")
+        raise
diff --git a/cloudinit/CloudConfig/cc_phone_home.py b/cloudinit/CloudConfig/cc_phone_home.py
new file mode 100644
index 00000000..89546287
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_phone_home.py
@@ -0,0 +1,39 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2011 Canonical Ltd.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+from cloudinit.CloudConfig import per_instance
+import cloudinit.util as util
+frequency = per_instance
+
+def handle(name,cfg,cloud,log,args):
+    if len(args) != 0:
+        value = args[0]
+    else:
+        value = util.get_cfg_option_str(cfg,"phone_home_url",False)
+
+    if not value:
+        return
+
+    # TODO:
+    # implement phone_home
+    # pass to it
+    #  - ssh key fingerprints
+    #  - mac addr ?
+    #  - ip address
+    #  
+    log.warn("TODO: write cc_phone_home")
+    return
diff --git a/cloudinit/CloudConfig/cc_scripts_per_boot.py b/cloudinit/CloudConfig/cc_scripts_per_boot.py
new file mode 100644
index 00000000..4e407fb7
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_scripts_per_boot.py
@@ -0,0 +1,30 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2011 Canonical Ltd.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+import cloudinit.util as util
+from cloudinit.CloudConfig import per_once, per_always, per_instance
+from cloudinit import get_cpath, get_ipath_cur
+
+frequency = per_always
+runparts_path = "%s/%s" % (get_cpath(), "scripts/per-boot")
+
+def handle(name,cfg,cloud,log,args):
+    try:
+        util.runparts(runparts_path)
+    except:
+        log.warn("failed to run-parts in %s" % runparts_path)
+        raise
diff --git a/cloudinit/CloudConfig/cc_scripts_per_instance.py b/cloudinit/CloudConfig/cc_scripts_per_instance.py
new file mode 100644
index 00000000..22b41185
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_scripts_per_instance.py
@@ -0,0 +1,30 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2011 Canonical Ltd.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+import cloudinit.util as util
+from cloudinit.CloudConfig import per_once, per_always, per_instance
+from cloudinit import get_cpath, get_ipath_cur
+
+frequency = per_instance
+runparts_path = "%s/%s" % (get_cpath(), "scripts/per-instance")
+
+def handle(name,cfg,cloud,log,args):
+    try:
+        util.runparts(runparts_path)
+    except:
+        log.warn("failed to run-parts in %s" % runparts_path)
+        raise
diff --git a/cloudinit/CloudConfig/cc_scripts_per_once.py b/cloudinit/CloudConfig/cc_scripts_per_once.py
new file mode 100644
index 00000000..9d752325
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_scripts_per_once.py
@@ -0,0 +1,30 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2011 Canonical Ltd.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+import cloudinit.util as util
+from cloudinit.CloudConfig import per_once, per_always, per_instance
+from cloudinit import get_cpath, get_ipath_cur
+
+frequency = per_once
+runparts_path = "%s/%s" % (get_cpath(), "scripts/per-once")
+
+def handle(name,cfg,cloud,log,args):
+    try:
+        util.runparts(runparts_path)
+    except:
+        log.warn("failed to run-parts in %s" % runparts_path)
+        raise
diff --git a/cloudinit/CloudConfig/cc_scripts_user.py b/cloudinit/CloudConfig/cc_scripts_user.py
new file mode 100644
index 00000000..bafecd23
--- /dev/null
+++ b/cloudinit/CloudConfig/cc_scripts_user.py
@@ -0,0 +1,30 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2011 Canonical Ltd.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+import cloudinit.util as util
+from cloudinit.CloudConfig import per_once, per_always, per_instance
+from cloudinit import get_cpath, get_ipath_cur
+
+frequency = per_instance
+runparts_path = "%s/%s" % (get_ipath_cur(), "scripts")
+
+def handle(name,cfg,cloud,log,args):
+    try:
+        util.runparts(runparts_path)
+    except:
+        log.warn("failed to run-parts in %s" % runparts_path)
+        raise
diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py
index d5180c05..1786cc2f 100644
--- a/cloudinit/__init__.py
+++ b/cloudinit/__init__.py
@@ -20,6 +20,7 @@
 
 varlibdir = '/var/lib/cloud'
 cur_instance_link = varlibdir + "/instance"
+boot_finished = cur_instance_link + "/boot-finished"
 system_config = '/etc/cloud/cloud.cfg'
 seeddir = varlibdir + "/seed"
 cfg_env_name = "CLOUD_CFG"
@@ -509,12 +510,15 @@ def initfs():
         util.chownbyname(log_file, u, g)
 
 def purge_cache():
-    try:
-        os.unlink(cur_instance_link)
-    except OSError as e:
-        if e.errno != errno.ENOENT: return(False)
-    except:
-        return(False)
+    rmlist = ( boot_finished , cur_instance_link )
+    for f in rmlist:
+        try:
+            os.unlink(f)
+        except OSError as e:
+            if e.errno == errno.ENOENT: continue
+            return(False)
+        except:
+            return(False)
     return(True)
 
 # get_ipath_cur: get the current instance path for an item
diff --git a/cloudinit/util.py b/cloudinit/util.py
index f8c847aa..96e93af2 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -127,6 +127,16 @@ def getkeybyid(keyid,keyserver):
    args=['sh', '-c', shcmd, "export-gpg-keyid", keyid, keyserver]
    return(subp(args)[0])
 
+def runparts(dirp, skip_no_exist=True):
+    if skip_no_exist and not os.path.isdir(dirp): return
+        
+    cmd = [ 'run-parts', '--regex', '.*', dirp ]
+    sp = subprocess.Popen(cmd)
+    sp.communicate()
+    if sp.returncode is not 0:
+        raise subprocess.CalledProcessError(sp.returncode,cmd)
+    return
+
 def subp(args, input=None):
     s_in = None
     if input is not None:
diff --git a/config/cloud.cfg b/config/cloud.cfg
index 90aecb6a..c499969f 100644
--- a/config/cloud.cfg
+++ b/config/cloud.cfg
@@ -21,6 +21,15 @@ cloud_config_modules:
  - runcmd
  - byobu
 
+cloud_final_modules:
+ - scripts-per-once
+ - scripts-per-boot
+ - scripts-per-instance
+ - scripts-user
+ - keys-to-console
+ - phone-home
+ - final-message
+
 ## logging.cfg contains info on logging output for cloud-init
 #include logging.cfg
 
diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt
index 41ff7924..e6d33465 100644
--- a/doc/examples/cloud-config.txt
+++ b/doc/examples/cloud-config.txt
@@ -286,3 +286,9 @@ resize_rootfs: True
 # appropriately to its value
 # if not set, it will set hostname from the cloud metadata
 # default: None
+
+# final_message
+# default: cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds
+# this message is written by cloud-final when the system is finished
+# its first boot
+final_message: "The system is finally up, after $UPTIME seconds"
diff --git a/doc/var-lib-cloud.txt b/doc/var-lib-cloud.txt
index 2a1acd2b..0f96f267 100644
--- a/doc/var-lib-cloud.txt
+++ b/doc/var-lib-cloud.txt
@@ -35,6 +35,9 @@
          obj.pkl
          handlers/
          data/  # just a per-instance data location to be used
+         boot-finished
+            # this file indicates when "boot" is finished
+            # it is created by the 'final_message' cloud-config
           
   - sem/
       scripts.once
diff --git a/upstart/cloud-final.conf b/upstart/cloud-final.conf
new file mode 100644
index 00000000..1127647a
--- /dev/null
+++ b/upstart/cloud-final.conf
@@ -0,0 +1,10 @@
+# cloud-final.conf - run "final" jobs
+# this runs around traditional "rc.local" time.
+# and after all cloud-config jobs are run
+description "execute cloud user/final scripts"
+
+start on (stopped rc RUNLEVEL=[2345] and stopped cloud-config)
+console output
+task
+
+exec cloud-init-cfg all cloud_final
diff --git a/upstart/cloud-run-user-script.conf b/upstart/cloud-run-user-script.conf
deleted file mode 100644
index e845aa71..00000000
--- a/upstart/cloud-run-user-script.conf
+++ /dev/null
@@ -1,28 +0,0 @@
-# cloud-run-user-script - runs user scripts found in user-data, that are
-#   stored in /var/lib/cloud/scripts by the initial cloudinit upstart job
-description "execute cloud user scripts"
-
-start on (stopped rc RUNLEVEL=[2345] and stopped cloud-config)
-console output
-task
-
-script
-bd=/var/lib/cloud
-toks="
-   ${bd}/scripts/per-once:once:cloud-scripts-per-once
-   ${bd}/scripts/per-boot:always:cloud-scripts-per-boot
-   ${bd}/scripts/per-instance:once-per-instance:cloud-scripts-per-instance
-   ${bd}/instance/scripts:once-per-instance:user-scripts
-"
-oifs=${IFS}
-errors=""
-for tok in ${toks}; do
-   IFS=":"; set -- ${tok}; IFS=${oifs}
-   dir=${1}; per=${2}; name=${3}
-   [ -d "${dir}" ] || continue
-   cloud-init-run-module "${per}" "${name}" execute \
-      run-parts --regex '.*' "$dir" || errors="${errors} ${name}"
-done
-errors=${errors# }
-[ -z "${errors}" ] || { echo "errors executing ${errors}" 1>&2; exit 1; }
-end script