Merge with lp:cloud-init

13 files changed, 207 insertions, 74 deletions

diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py
index 3426099e..02056ee0 100644
--- a/cloudinit/config/cc_apt_pipelining.py
+++ b/cloudinit/config/cc_apt_pipelining.py
@@ -16,8 +16,8 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from cloudinit import util
 from cloudinit.settings import PER_INSTANCE
+from cloudinit import util
 
 frequency = PER_INSTANCE
 
@@ -50,7 +50,7 @@ def handle(_name, cfg, cloud, log, _args):
 
 
 def write_apt_snippet(cloud, setting, log, f_name):
-    """ Writes f_name with apt pipeline depth 'setting' """
+    """Writes f_name with apt pipeline depth 'setting'."""
 
     file_contents = APT_PIPE_TPL % (setting)
 
diff --git a/cloudinit/config/cc_apt_update_upgrade.py b/cloudinit/config/cc_apt_update_upgrade.py
index 1bffa47d..356bb98d 100644
--- a/cloudinit/config/cc_apt_update_upgrade.py
+++ b/cloudinit/config/cc_apt_update_upgrade.py
@@ -20,6 +20,7 @@
 
 import glob
 import os
+import time
 
 from cloudinit import templater
 from cloudinit import util
@@ -50,20 +51,25 @@ def handle(name, cfg, cloud, log, _args):
     upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False)
 
     release = get_release()
-    mirror = find_apt_mirror(cloud, cfg)
-    if not mirror:
+    mirrors = find_apt_mirror_info(cloud, cfg)
+    if not mirrors or "primary" not in mirrors:
         log.debug(("Skipping module named %s,"
                    " no package 'mirror' located"), name)
         return
 
-    log.debug("Selected mirror at: %s" % mirror)
+    # backwards compatibility
+    mirror = mirrors["primary"]
+    mirrors["mirror"] = mirror
+
+    log.debug("mirror info: %s" % mirrors)
 
     if not util.get_cfg_option_bool(cfg,
                                     'apt_preserve_sources_list', False):
-        generate_sources_list(release, mirror, cloud, log)
-        old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror',
-                                          "archive.ubuntu.com/ubuntu")
-        rename_apt_lists(old_mir, mirror)
+        generate_sources_list(release, mirrors, cloud, log)
+        old_mirrors = cfg.get('apt_old_mirrors',
+                              {"primary": "archive.ubuntu.com/ubuntu",
+                               "security": "security.ubuntu.com/ubuntu"})
+        rename_apt_lists(old_mirrors, mirrors)
 
     # Set up any apt proxy
     proxy = cfg.get("apt_proxy", None)
@@ -81,8 +87,10 @@ def handle(name, cfg, cloud, log, _args):
 
     # Process 'apt_sources'
     if 'apt_sources' in cfg:
-        errors = add_sources(cloud, cfg['apt_sources'],
-                             {'MIRROR': mirror, 'RELEASE': release})
+        params = mirrors
+        params['RELEASE'] = release
+        params['MIRROR'] = mirror
+        errors = add_sources(cloud, cfg['apt_sources'], params)
         for e in errors:
             log.warn("Source Error: %s", ':'.join(e))
 
@@ -118,6 +126,20 @@ def handle(name, cfg, cloud, log, _args):
             util.logexc(log, "Failed to install packages: %s ", pkglist)
             errors.append(e)
 
+    # kernel and openssl (possibly some other packages)
+    # write a file /var/run/reboot-required after upgrading.
+    # if that file exists and configured, then just stop right now and reboot
+    # TODO(smoser): handle this less voilently
+    reboot_file = "/var/run/reboot-required"
+    if ((upgrade or pkglist) and cfg.get("apt_reboot_if_required", False) and
+         os.path.isfile(reboot_file)):
+        log.warn("rebooting after upgrade or install per %s" % reboot_file)
+        time.sleep(1)  # give the warning time to get out
+        util.subp(["/sbin/reboot"])
+        time.sleep(60)
+        log.warn("requested reboot did not happen!")
+        errors.append(Exception("requested reboot did not happen!"))
+
     if len(errors):
         log.warn("%s failed with exceptions, re-raising the last one",
                  len(errors))
@@ -146,15 +168,18 @@ def mirror2lists_fileprefix(mirror):
     return string
 
 
-def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"):
-    oprefix = os.path.join(lists_d, mirror2lists_fileprefix(omirror))
-    nprefix = os.path.join(lists_d, mirror2lists_fileprefix(new_mirror))
-    if oprefix == nprefix:
-        return
-    olen = len(oprefix)
-    for filename in glob.glob("%s_*" % oprefix):
-        # TODO use the cloud.paths.join...
-        util.rename(filename, "%s%s" % (nprefix, filename[olen:]))
+def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"):
+    for (name, omirror) in old_mirrors.iteritems():
+        nmirror = new_mirrors.get(name)
+        if not nmirror:
+            continue
+        oprefix = os.path.join(lists_d, mirror2lists_fileprefix(omirror))
+        nprefix = os.path.join(lists_d, mirror2lists_fileprefix(nmirror))
+        if oprefix == nprefix:
+            continue
+        olen = len(oprefix)
+        for filename in glob.glob("%s_*" % oprefix):
+            util.rename(filename, "%s%s" % (nprefix, filename[olen:]))
 
 
 def get_release():
@@ -162,14 +187,17 @@ def get_release():
     return stdout.strip()
 
 
-def generate_sources_list(codename, mirror, cloud, log):
+def generate_sources_list(codename, mirrors, cloud, log):
     template_fn = cloud.get_template_filename('sources.list')
-    if template_fn:
-        params = {'mirror': mirror, 'codename': codename}
-        out_fn = cloud.paths.join(False, '/etc/apt/sources.list')
-        templater.render_to_file(template_fn, out_fn, params)
-    else:
+    if not template_fn:
         log.warn("No template found, not rendering /etc/apt/sources.list")
+        return
+
+    params = {'codename': codename}
+    for k in mirrors:
+        params[k] = mirrors[k]
+    out_fn = cloud.paths.join(False, '/etc/apt/sources.list')
+    templater.render_to_file(template_fn, out_fn, params)
 
 
 def add_sources(cloud, srclist, template_params=None):
@@ -231,43 +259,47 @@ def add_sources(cloud, srclist, template_params=None):
     return errorlist
 
 
-def find_apt_mirror(cloud, cfg):
-    """ find an apt_mirror given the cloud and cfg provided """
+def find_apt_mirror_info(cloud, cfg):
+    """find an apt_mirror given the cloud and cfg provided."""
 
     mirror = None
 
-    cfg_mirror = cfg.get("apt_mirror", None)
-    if cfg_mirror:
-        mirror = cfg["apt_mirror"]
-    elif "apt_mirror_search" in cfg:
-        mirror = util.search_for_mirror(cfg['apt_mirror_search'])
-    else:
-        mirror = cloud.get_local_mirror()
+    # this is less preferred way of specifying mirror preferred would be to
+    # use the distro's search or package_mirror.
+    mirror = cfg.get("apt_mirror", None)
 
-        mydom = ""
+    search = cfg.get("apt_mirror_search", None)
+    if not mirror and search:
+        mirror = util.search_for_mirror(search)
 
+    if (not mirror and
+        util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)):
+        mydom = ""
         doms = []
 
-        if not mirror:
-            # if we have a fqdn, then search its domain portion first
-            (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud)
-            mydom = ".".join(fqdn.split(".")[1:])
-            if mydom:
-                doms.append(".%s" % mydom)
+        # if we have a fqdn, then search its domain portion first
+        (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud)
+        mydom = ".".join(fqdn.split(".")[1:])
+        if mydom:
+            doms.append(".%s" % mydom)
+
+        doms.extend((".localdomain", "",))
 
-        if (not mirror and
-            util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)):
-            doms.extend((".localdomain", "",))
+        mirror_list = []
+        distro = cloud.distro.name
+        mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro)
+        for post in doms:
+            mirror_list.append(mirrorfmt % (post))
 
-            mirror_list = []
-            distro = cloud.distro.name
-            mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro)
-            for post in doms:
-                mirror_list.append(mirrorfmt % (post))
+        mirror = util.search_for_mirror(mirror_list)
 
-            mirror = util.search_for_mirror(mirror_list)
+    mirror_info = cloud.datasource.get_package_mirror_info()
 
-    if not mirror:
-        mirror = cloud.distro.get_package_mirror()
+    # this is a bit strange.
+    # if mirror is set, then one of the legacy options above set it
+    # but they do not cover security. so we need to get that from
+    # get_package_mirror_info
+    if mirror:
+        mirror_info.update({'primary': mirror})
 
-    return mirror
+    return mirror_info
diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py
index bae1ea54..896cb4d0 100644
--- a/cloudinit/config/cc_bootcmd.py
+++ b/cloudinit/config/cc_bootcmd.py
@@ -20,8 +20,8 @@
 
 import os
 
-from cloudinit import util
 from cloudinit.settings import PER_ALWAYS
+from cloudinit import util
 
 frequency = PER_ALWAYS
 
diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py
index 68b86ff6..6d376184 100644
--- a/cloudinit/config/cc_emit_upstart.py
+++ b/cloudinit/config/cc_emit_upstart.py
@@ -20,8 +20,8 @@
 
 import os
 
-from cloudinit import util
 from cloudinit.settings import PER_ALWAYS
+from cloudinit import util
 
 frequency = PER_ALWAYS
 
@@ -44,5 +44,5 @@ def handle(name, _cfg, cloud, log, args):
         try:
             util.subp(cmd)
         except Exception as e:
-            # TODO, use log exception from utils??
+            # TODO(harlowja), use log exception from utils??
             log.warn("Emission of upstart event %s failed due to: %s", n, e)
diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py
index aff03c4e..6b864fda 100644
--- a/cloudinit/config/cc_final_message.py
+++ b/cloudinit/config/cc_final_message.py
@@ -28,7 +28,7 @@ frequency = PER_ALWAYS
 
 # Cheetah formated default message
 FINAL_MESSAGE_DEF = ("Cloud-init v. ${version} finished at ${timestamp}."
-                     " Up ${uptime} seconds.")
+                     " Datasource ${datasource}.  Up ${uptime} seconds")
 
 
 def handle(_name, cfg, cloud, log, args):
@@ -51,6 +51,7 @@ def handle(_name, cfg, cloud, log, args):
             'uptime': uptime,
             'timestamp': ts,
             'version': cver,
+            'datasource': str(cloud.datasource),
         }
         util.multi_log("%s\n" % (templater.render_string(msg_in, subs)),
                        console=False, stderr=True)
@@ -63,3 +64,6 @@ def handle(_name, cfg, cloud, log, args):
         util.write_file(boot_fin_fn, contents)
     except:
         util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn)
+
+    if cloud.datasource.is_disconnected:
+        log.warn("Used fallback datasource")
diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py
index 467c1496..74ee18e1 100644
--- a/cloudinit/config/cc_puppet.py
+++ b/cloudinit/config/cc_puppet.py
@@ -48,7 +48,8 @@ def handle(name, cfg, cloud, log, _args):
         # Create object for reading puppet.conf values
         puppet_config = helpers.DefaultingConfigParser()
         # Read puppet.conf values from original file in order to be able to
-        # mix the rest up. First clean them up (TODO is this really needed??)
+        # mix the rest up. First clean them up
+        # (TODO(harlowja) is this really needed??)
         cleaned_lines = [i.lstrip() for i in contents.splitlines()]
         cleaned_contents = '\n'.join(cleaned_lines)
         puppet_config.readfp(StringIO(cleaned_contents),
@@ -80,7 +81,7 @@ def handle(name, cfg, cloud, log, _args):
                 for (o, v) in cfg.iteritems():
                     if o == 'certname':
                         # Expand %f as the fqdn
-                        # TODO should this use the cloud fqdn??
+                        # TODO(harlowja) should this use the cloud fqdn??
                         v = v.replace("%f", socket.getfqdn())
                         # Expand %i as the instance id
                         v = v.replace("%i", cloud.get_instance_id())
diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py
index 256a194f..e7f27944 100644
--- a/cloudinit/config/cc_resizefs.py
+++ b/cloudinit/config/cc_resizefs.py
@@ -22,8 +22,8 @@ import os
 import stat
 import time
 
-from cloudinit import util
 from cloudinit.settings import PER_ALWAYS
+from cloudinit import util
 
 frequency = PER_ALWAYS
 
@@ -72,12 +72,12 @@ def handle(name, cfg, cloud, log, args):
         log.debug("Skipping module named %s, resizing disabled", name)
         return
 
-    # TODO is the directory ok to be used??
+    # TODO(harlowja) is the directory ok to be used??
     resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run")
     resize_root_d = cloud.paths.join(False, resize_root_d)
     util.ensure_dir(resize_root_d)
 
-    # TODO: allow what is to be resized to be configurable??
+    # TODO(harlowja): allow what is to be resized to be configurable??
     resize_what = cloud.paths.join(False, "/")
     with util.ExtendedTemporaryFile(prefix="cloudinit.resizefs.",
                                     dir=resize_root_d, delete=True) as tfh:
@@ -136,5 +136,5 @@ def do_resize(resize_cmd, log):
         raise
     tot_time = time.time() - start
     log.debug("Resizing took %.3f seconds", tot_time)
-    # TODO: Should we add a fsck check after this to make
+    # TODO(harlowja): Should we add a fsck check after this to make
     # sure we didn't corrupt anything?
diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py
index 45d41b3f..4bf18516 100644
--- a/cloudinit/config/cc_rightscale_userdata.py
+++ b/cloudinit/config/cc_rightscale_userdata.py
@@ -37,9 +37,9 @@
 
 import os
 
+from cloudinit.settings import PER_INSTANCE
 from cloudinit import url_helper as uhelp
 from cloudinit import util
-from cloudinit.settings import PER_INSTANCE
 
 from urlparse import parse_qs
 
@@ -72,7 +72,7 @@ def handle(name, _cfg, cloud, log, _args):
     captured_excps = []
 
     # These will eventually be then ran by the cc_scripts_user
-    # TODO: maybe this should just be a new user data handler??
+    # TODO(harlowja): maybe this should just be a new user data handler??
     # Instead of a late module that acts like a user data handler?
     scripts_d = cloud.get_ipath_cur('scripts')
     urls = mdict[MY_HOOKNAME]
diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py
index 4019ae90..3431bd2a 100644
--- a/cloudinit/config/cc_ssh.py
+++ b/cloudinit/config/cc_ssh.py
@@ -18,11 +18,11 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import os
 import glob
+import os
 
-from cloudinit import util
 from cloudinit import ssh_util
+from cloudinit import util
 
 DISABLE_ROOT_OPTS = ("no-port-forwarding,no-agent-forwarding,"
 "no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" "
@@ -76,7 +76,7 @@ def handle(_name, cfg, cloud, log, _args):
             pair = (KEY_2_FILE[priv][0], KEY_2_FILE[pub][0])
             cmd = ['sh', '-xc', KEY_GEN_TPL % pair]
             try:
-                # TODO: Is this guard needed?
+                # TODO(harlowja): Is this guard needed?
                 with util.SeLinuxGuard("/etc/ssh", recursive=True):
                     util.subp(cmd, capture=False)
                 log.debug("Generated a key for %s from %s", pair[0], pair[1])
@@ -94,7 +94,7 @@ def handle(_name, cfg, cloud, log, _args):
             if not os.path.exists(keyfile):
                 cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile]
                 try:
-                    # TODO: Is this guard needed?
+                    # TODO(harlowja): Is this guard needed?
                     with util.SeLinuxGuard("/etc/ssh", recursive=True):
                         util.subp(cmd, capture=False)
                 except:
diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py
new file mode 100644
index 00000000..23f5755a
--- /dev/null
+++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py
@@ -0,0 +1,96 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2012 Yahoo! Inc.
+#
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import base64
+import hashlib
+
+from prettytable import PrettyTable
+
+from cloudinit import ssh_util
+from cloudinit import util
+
+
+def _split_hash(bin_hash):
+    split_up = []
+    for i in xrange(0, len(bin_hash), 2):
+        split_up.append(bin_hash[i:i + 2])
+    return split_up
+
+
+def _gen_fingerprint(b64_text, hash_meth='md5'):
+    if not b64_text:
+        return ''
+    # TBD(harlowja): Maybe we should feed this into 'ssh -lf'?
+    try:
+        hasher = hashlib.new(hash_meth)
+        hasher.update(base64.b64decode(b64_text))
+        return ":".join(_split_hash(hasher.hexdigest()))
+    except TypeError:
+        # Raised when b64 not really b64...
+        return '?'
+
+
+def _is_printable_key(entry):
+    if any([entry.keytype, entry.base64, entry.comment, entry.options]):
+        if (entry.keytype and
+            entry.keytype.lower().strip() in ['ssh-dss', 'ssh-rsa']):
+            return True
+    return False
+
+
+def _pprint_key_entries(user, key_fn, key_entries, hash_meth='md5',
+                        prefix='ci-info: '):
+    if not key_entries:
+        message = ("%sno authorized ssh keys fingerprints found for user %s."
+                   % (prefix, user))
+        util.multi_log(message)
+        return
+    tbl_fields = ['Keytype', 'Fingerprint (%s)' % (hash_meth), 'Options',
+                  'Comment']
+    tbl = PrettyTable(tbl_fields)
+    for entry in key_entries:
+        if _is_printable_key(entry):
+            row = []
+            row.append(entry.keytype or '-')
+            row.append(_gen_fingerprint(entry.base64, hash_meth) or '-')
+            row.append(entry.options or '-')
+            row.append(entry.comment or '-')
+            tbl.add_row(row)
+    authtbl_s = tbl.get_string()
+    authtbl_lines = authtbl_s.splitlines()
+    max_len = len(max(authtbl_lines, key=len))
+    lines = [
+        util.center("Authorized keys from %s for user %s" %
+                    (key_fn, user), "+", max_len),
+    ]
+    lines.extend(authtbl_lines)
+    for line in lines:
+        util.multi_log(text="%s%s\n" % (prefix, line),
+                       stderr=False, console=True)
+
+
+def handle(name, cfg, cloud, log, _args):
+    if 'no_ssh_fingerprints' in cfg:
+        log.debug(("Skipping module named %s, "
+                   "logging of ssh fingerprints disabled"), name)
+
+    user_name = util.get_cfg_option_str(cfg, "user", "ubuntu")
+    hash_meth = util.get_cfg_option_str(cfg, "authkey_hash", "md5")
+    extract = ssh_util.extract_authorized_keys
+    (auth_key_fn, auth_key_entries) = extract(user_name, cloud.paths)
+    _pprint_key_entries(user_name, auth_key_fn, auth_key_entries, hash_meth)
diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py
index 38108da7..4d75000f 100644
--- a/cloudinit/config/cc_update_etc_hosts.py
+++ b/cloudinit/config/cc_update_etc_hosts.py
@@ -18,8 +18,8 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from cloudinit import util
 from cloudinit import templater
+from cloudinit import util
 
 from cloudinit.settings import PER_ALWAYS
 
diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py
index b84a1a06..1d6679ea 100644
--- a/cloudinit/config/cc_update_hostname.py
+++ b/cloudinit/config/cc_update_hostname.py
@@ -20,8 +20,8 @@
 
 import os
 
-from cloudinit import util
 from cloudinit.settings import PER_ALWAYS
+from cloudinit import util
 
 frequency = PER_ALWAYS
 
diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py
index 1bfa4c25..a73d6f4e 100644
--- a/cloudinit/config/cc_write_files.py
+++ b/cloudinit/config/cc_write_files.py
@@ -19,8 +19,8 @@
 import base64
 import os
 
-from cloudinit import util
 from cloudinit.settings import PER_INSTANCE
+from cloudinit import util
 
 frequency = PER_INSTANCE
 
@@ -46,7 +46,7 @@ def canonicalize_extraction(encoding_type, log):
         return ['application/x-gzip']
     if encoding_type in ['gz+base64', 'gzip+base64', 'gz+b64', 'gzip+b64']:
         return ['application/base64', 'application/x-gzip']
-    # Yaml already encodes binary data as base64 if it is given to the 
+    # Yaml already encodes binary data as base64 if it is given to the
     # yaml file as binary, so those will be automatically decoded for you.
     # But the above b64 is just for people that are more 'comfortable'
     # specifing it manually (which might be a possiblity)