Update to code on trunk.

8 files changed, 192 insertions, 135 deletions

diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index 69c376a5..9812bdcb 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -47,7 +47,7 @@ META_DATA_NOT_SUPPORTED = {
     'instance-id': 455,
     'local-hostname': 'localhost',
     'placement': {},
-    }
+}
 
 
 def read_user_data_callback(mount_dir):
@@ -73,13 +73,11 @@ def read_user_data_callback(mount_dir):
 
     # First try deltacloud_user_data_file. On failure try user_data_file.
     try:
-        with open(deltacloud_user_data_file, 'r') as user_data_f:
-            user_data = user_data_f.read().strip()
-    except:
+        user_data = util.load_file(deltacloud_user_data_file).strip()
+    except IOError:
         try:
-            with open(user_data_file, 'r') as user_data_f:
-                user_data = user_data_f.read().strip()
-        except:
+            user_data = util.load_file(user_data_file).strip()
+        except IOError:
             util.logexc(LOG, ('Failed accessing user data file.'))
             return None
 
@@ -157,11 +155,10 @@ class DataSourceAltCloud(sources.DataSource):
 
         if os.path.exists(CLOUD_INFO_FILE):
             try:
-                cloud_info = open(CLOUD_INFO_FILE)
-                cloud_type = cloud_info.read().strip().upper()
-                cloud_info.close()
-            except:
-                util.logexc(LOG, 'Unable to access cloud info file.')
+                cloud_type = util.load_file(CLOUD_INFO_FILE).strip().upper()
+            except IOError:
+                util.logexc(LOG, 'Unable to access cloud info file at %s.',
+                            CLOUD_INFO_FILE)
                 return False
         else:
             cloud_type = self.get_cloud_type()
diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index 2654df53..1335b63d 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -3,10 +3,12 @@
 #    Copyright (C) 2012 Canonical Ltd.
 #    Copyright (C) 2012 Cosmin Luta
 #    Copyright (C) 2012 Yahoo! Inc.
+#    Copyright (C) 2012 Gerard Dethier
 #
 #    Author: Cosmin Luta <q4break@gmail.com>
 #    Author: Scott Moser <scott.moser@canonical.com>
 #    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#    Author: Gerard Dethier <g.dethier@gmail.com>
 #
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License version 3, as
@@ -20,9 +22,6 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from socket import inet_ntoa
-from struct import pack
-
 import os
 import time
 
@@ -31,6 +30,8 @@ from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import url_helper as uhelp
 from cloudinit import util
+from socket import inet_ntoa
+from struct import pack
 
 LOG = logging.getLogger(__name__)
 
@@ -40,24 +41,12 @@ class DataSourceCloudStack(sources.DataSource):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
         self.seed_dir = os.path.join(paths.seed_dir, 'cs')
         # Cloudstack has its metadata/userdata URLs located at
-        # http://<default-gateway-ip>/latest/
+        # http://<virtual-router-ip>/latest/
         self.api_ver = 'latest'
-        gw_addr = self.get_default_gateway()
-        if not gw_addr:
-            raise RuntimeError("No default gateway found!")
-        self.metadata_address = "http://%s/" % (gw_addr)
-
-    def get_default_gateway(self):
-        """Returns the default gateway ip address in the dotted format."""
-        lines = util.load_file("/proc/net/route").splitlines()
-        for line in lines:
-            items = line.split("\t")
-            if items[1] == "00000000":
-                # Found the default route, get the gateway
-                gw = inet_ntoa(pack("<L", int(items[2], 16)))
-                LOG.debug("Found default route, gateway is %s", gw)
-                return gw
-        return None
+        vr_addr = get_vr_address()
+        if not vr_addr:
+            raise RuntimeError("No virtual router found!")
+        self.metadata_address = "http://%s/" % (vr_addr)
 
     def __str__(self):
         return util.obj_name(self)
@@ -90,7 +79,7 @@ class DataSourceCloudStack(sources.DataSource):
 
         (max_wait, timeout) = self._get_url_settings()
 
-        urls = [self.metadata_address]
+        urls = [self.metadata_address + "/latest/meta-data/instance-id"]
         start_time = time.time()
         url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
                                 timeout=timeout, status_cb=LOG.warn)
@@ -115,8 +104,14 @@ class DataSourceCloudStack(sources.DataSource):
             if not self.wait_for_metadata_service():
                 return False
             start_time = time.time()
-            self.userdata_raw = ec2_utils.get_instance_userdata(self.metadata_address, self.api_ver)
-            self.metadata = ec2_utils.get_instance_metadata(self.metadata_address, self.api_ver)
+            md_addr = self.metadata_address
+            ssl_details = util.fetch_ssl_details(self.paths)
+            self.userdata_raw = ec2_utils.get_instance_userdata(self.api_ver,
+                                                                md_addr,
+                                                                ssl_details)
+            self.metadata = ec2_utils.get_instance_metadata(self.api_ver,
+                                                            md_addr,
+                                                            ssl_details)
             LOG.debug("Crawl of metadata service took %s seconds",
                       int(time.time() - start_time))
             return True
@@ -133,6 +128,72 @@ class DataSourceCloudStack(sources.DataSource):
         return self.metadata['availability-zone']
 
 
+def get_default_gateway():
+    # Returns the default gateway ip address in the dotted format.
+    lines = util.load_file("/proc/net/route").splitlines()
+    for line in lines:
+        items = line.split("\t")
+        if items[1] == "00000000":
+            # Found the default route, get the gateway
+            gw = inet_ntoa(pack("<L", int(items[2], 16)))
+            LOG.debug("Found default route, gateway is %s", gw)
+            return gw
+    return None
+
+
+def get_dhclient_d():
+    # find lease files directory
+    supported_dirs = ["/var/lib/dhclient", "/var/lib/dhcp"]
+    for d in supported_dirs:
+        if os.path.exists(d):
+            LOG.debug("Using %s lease directory", d)
+            return d
+    return None
+
+
+def get_latest_lease():
+    # find latest lease file
+    lease_d = get_dhclient_d()
+    if not lease_d:
+        return None
+    lease_files = os.listdir(lease_d)
+    latest_mtime = -1
+    latest_file = None
+    for file_name in lease_files:
+        if file_name.endswith(".lease") or file_name.endswith(".leases"):
+            abs_path = os.path.join(lease_d, file_name)
+            mtime = os.path.getmtime(abs_path)
+            if mtime > latest_mtime:
+                latest_mtime = mtime
+                latest_file = abs_path
+    return latest_file
+
+
+def get_vr_address():
+    # Get the address of the virtual router via dhcp leases
+    # see http://bit.ly/T76eKC for documentation on the virtual router.
+    # If no virtual router is detected, fallback on default gateway.
+    lease_file = get_latest_lease()
+    if not lease_file:
+        LOG.debug("No lease file found, using default gateway")
+        return get_default_gateway()
+
+    latest_address = None
+    with open(lease_file, "r") as fd:
+        for line in fd:
+            if "dhcp-server-identifier" in line:
+                words = line.strip(" ;\r\n").split(" ")
+                if len(words) > 2:
+                    dhcp = words[2]
+                    LOG.debug("Found DHCP identifier %s", dhcp)
+                    latest_address = dhcp
+    if not latest_address:
+        # No virtual router found, fallback on default gateway
+        LOG.debug("No DHCP found, using default gateway")
+        return get_default_gateway()
+    return latest_address
+
+
 # Used to match classes to dependencies
 datasources = [
   (DataSourceCloudStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 4af2e5ae..ec016a1d 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -219,9 +219,10 @@ class DataSourceConfigDrive(sources.DataSource):
         return True
 
     def get_public_ssh_keys(self):
-        if not 'public-keys' in self.metadata:
-            return []
-        return self.metadata['public-keys']
+        name = "public_keys"
+        if self.version == 1:
+            name = "public-keys"
+        return sources.normalize_pubkey_data(self.metadata.get(name))
 
 
 class DataSourceConfigDriveNet(DataSourceConfigDrive):
@@ -269,7 +270,7 @@ def find_candidate_devs():
     combined = (by_label + [d for d in by_fstype if d not in by_label])
 
     # We are looking for block device (sda, not sda1), ignore partitions
-    combined = [d for d in combined if d[-1] not in "0123456789"]
+    combined = [d for d in combined if not util.is_partition(d)]
 
     return combined
 
@@ -307,19 +308,19 @@ def read_config_drive_dir_v2(source_dir, version="2012-08-10"):
         found = False
         if os.path.isfile(fpath):
             try:
-                with open(fpath) as fp:
-                    data = fp.read()
-            except Exception as exc:
-                raise BrokenConfigDriveDir("failed to read: %s" % fpath)
+                data = util.load_file(fpath)
+            except IOError:
+                raise BrokenConfigDriveDir("Failed to read: %s" % fpath)
             found = True
         elif required:
-            raise NonConfigDriveDir("missing mandatory %s" % fpath)
+            raise NonConfigDriveDir("Missing mandatory path: %s" % fpath)
 
         if found and process:
             try:
                 data = process(data)
             except Exception as exc:
-                raise BrokenConfigDriveDir("failed to process: %s" % fpath)
+                raise BrokenConfigDriveDir(("Failed to process "
+                                            "path: %s") % fpath)
 
         if found:
             results[name] = data
@@ -335,8 +336,7 @@ def read_config_drive_dir_v2(source_dir, version="2012-08-10"):
         # do not use os.path.join here, as content_path starts with /
         cpath = os.path.sep.join((source_dir, "openstack",
                                   "./%s" % item['content_path']))
-        with open(cpath) as fp:
-            return(fp.read())
+        return util.load_file(cpath)
 
     files = {}
     try:
@@ -350,7 +350,7 @@ def read_config_drive_dir_v2(source_dir, version="2012-08-10"):
         if item:
             results['network_config'] = read_content_path(item)
     except Exception as exc:
-        raise BrokenConfigDriveDir("failed to read file %s: %s" % (item, exc))
+        raise BrokenConfigDriveDir("Failed to read file %s: %s" % (item, exc))
 
     # to openstack, user can specify meta ('nova boot --meta=key=value') and
     # those will appear under metadata['meta'].
@@ -465,8 +465,7 @@ def get_previous_iid(paths):
     # hasn't declared itself found.
     fname = os.path.join(paths.get_cpath('data'), 'instance-id')
     try:
-        with open(fname) as fp:
-            return fp.read()
+        return util.load_file(fname)
     except IOError:
         return None
 
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index eee4e6bc..0a58820b 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -64,10 +64,14 @@ class DataSourceEc2(sources.DataSource):
             if not self.wait_for_metadata_service():
                 return False
             start_time = time.time()
-            self.userdata_raw = ec2_utils.get_instance_userdata(self.metadata_address, self.api_ver,
-                                                                ssl_details=util.fetch_ssl_details(self.paths))
-            self.metadata = ec2_utils.get_instance_metadata(self.metadata_address, self.api_ver,
-                                                            ssl_details=util.fetch_ssl_details(self.paths))
+            md_addr = self.metadata_address
+            ssl_details = util.fetch_ssl_details(self.paths)
+            self.userdata_raw = ec2_utils.get_instance_userdata(self.api_ver,
+                                                                md_addr,
+                                                                ssl_details)
+            self.metadata = ec2_utils.get_instance_metadata(self.api_ver,
+                                                            md_addr,
+                                                            ssl_details)
             LOG.debug("Crawl of metadata service took %s seconds",
                        int(time.time() - start_time))
             return True
@@ -85,9 +89,6 @@ class DataSourceEc2(sources.DataSource):
     def get_instance_id(self):
         return self.metadata['instance-id']
 
-    def get_availability_zone(self):
-        return self.metadata['placement']['availability-zone']
-
     def _get_url_settings(self):
         mcfg = self.ds_cfg
         if not mcfg:
@@ -197,19 +198,6 @@ class DataSourceEc2(sources.DataSource):
             return None
         return ofound
 
-    def is_vpc(self):
-        # See: https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/615545
-        # Detect that the machine was launched in a VPC.
-        # But I did notice that when in a VPC, meta-data
-        # does not have public-ipv4 and public-hostname
-        # listed as a possibility.
-        ph = "public-hostname"
-        p4 = "public-ipv4"
-        if ((ph not in self.metadata or self.metadata[ph] == "") and
-            (p4 not in self.metadata or self.metadata[p4] == "")):
-            return True
-        return False
-
     @property
     def availability_zone(self):
         try:
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index dc048943..6e1133b2 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -340,8 +340,7 @@ if __name__ == "__main__":
             'token_secret': args.tsec, 'consumer_secret': args.csec}
 
         if args.config:
-            with open(args.config) as fp:
-                cfg = util.load_yaml(fp.read())
+            cfg = util.read_conf(args.config)
             if 'datasource' in cfg:
                 cfg = cfg['datasource']['MAAS']
             for key in creds.keys():
@@ -350,7 +349,7 @@ if __name__ == "__main__":
 
         def geturl(url, headers_cb):
             req = urllib2.Request(url, data=None, headers=headers_cb(url))
-            return(urllib2.urlopen(req).read())
+            return (urllib2.urlopen(req).read())
 
         def printurl(url, headers_cb):
             print "== %s ==\n%s\n" % (url, geturl(url, headers_cb))
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index bed500a2..097bbc52 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -77,37 +77,47 @@ class DataSourceNoCloud(sources.DataSource):
             found.append("ds_config")
             md["seedfrom"] = self.ds_cfg['seedfrom']
 
-        fslist = util.find_devs_with("TYPE=vfat")
-        fslist.extend(util.find_devs_with("TYPE=iso9660"))
-
-        label_list = util.find_devs_with("LABEL=cidata")
-        devlist = list(set(fslist) & set(label_list))
-        devlist.sort(reverse=True)
-
-        for dev in devlist:
-            try:
-                LOG.debug("Attempting to use data from %s", dev)
-
-                (newmd, newud) = util.mount_cb(dev, util.read_seeded)
-                md = util.mergedict(newmd, md)
-                ud = newud
-
-                # For seed from a device, the default mode is 'net'.
-                # that is more likely to be what is desired.
-                # If they want dsmode of local, then they must
-                # specify that.
-                if 'dsmode' not in md:
-                    md['dsmode'] = "net"
-
-                LOG.debug("Using data from %s", dev)
-                found.append(dev)
-                break
-            except OSError as e:
-                if e.errno != errno.ENOENT:
-                    raise
-            except util.MountFailedError:
-                util.logexc(LOG, ("Failed to mount %s"
-                                  " when looking for data"), dev)
+        # if ds_cfg has 'user-data' and 'meta-data'
+        if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg:
+            if self.ds_cfg['user-data']:
+                ud = self.ds_cfg['user-data']
+            if self.ds_cfg['meta-data'] is not False:
+                md = util.mergedict(md, self.ds_cfg['meta-data'])
+            if 'ds_config' not in found:
+                found.append("ds_config")
+
+        if self.ds_cfg.get('fs_label', "cidata"):
+            fslist = util.find_devs_with("TYPE=vfat")
+            fslist.extend(util.find_devs_with("TYPE=iso9660"))
+
+            label = self.ds_cfg.get('fs_label')
+            label_list = util.find_devs_with("LABEL=%s" % label)
+            devlist = list(set(fslist) & set(label_list))
+            devlist.sort(reverse=True)
+
+            for dev in devlist:
+                try:
+                    LOG.debug("Attempting to use data from %s", dev)
+
+                    (newmd, newud) = util.mount_cb(dev, util.read_seeded)
+                    md = util.mergedict(newmd, md)
+                    ud = newud
+
+                    # For seed from a device, the default mode is 'net'.
+                    # that is more likely to be what is desired.  If they want
+                    # dsmode of local, then they must specify that.
+                    if 'dsmode' not in md:
+                        md['dsmode'] = "net"
+
+                    LOG.debug("Using data from %s", dev)
+                    found.append(dev)
+                    break
+                except OSError as e:
+                    if e.errno != errno.ENOENT:
+                        raise
+                except util.MountFailedError:
+                    util.logexc(LOG, ("Failed to mount %s"
+                                      " when looking for data"), dev)
 
         # There was no indication on kernel cmdline or data
         # in the seeddir suggesting this handler should be used.
@@ -195,6 +205,8 @@ def parse_cmdline_data(ds_id, fill, cmdline=None):
     # short2long mapping to save cmdline typing
     s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"}
     for item in kvpairs:
+        if item == "":
+            continue
         try:
             (k, v) = item.split("=", 1)
         except:
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index 771e64eb..e90150c6 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -204,9 +204,8 @@ def transport_iso9660(require_iso=True):
 
         try:
             # See if we can read anything at all...??
-            with open(fullp, 'rb') as fp:
-                fp.read(512)
-        except:
+            util.peek_file(fullp, 512)
+        except IOError:
             continue
 
         try:
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index b22369a8..96baff90 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -20,8 +20,6 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from email.mime.multipart import MIMEMultipart
-
 import abc
 import os
 
@@ -102,32 +100,7 @@ class DataSource(object):
         return {}
 
     def get_public_ssh_keys(self):
-        keys = []
-
-        if not self.metadata or 'public-keys' not in self.metadata:
-            return keys
-
-        if isinstance(self.metadata['public-keys'], (basestring, str)):
-            return str(self.metadata['public-keys']).splitlines()
-
-        if isinstance(self.metadata['public-keys'], (list, set)):
-            return list(self.metadata['public-keys'])
-
-        if isinstance(self.metadata['public-keys'], (dict)):
-            for (_keyname, klist) in self.metadata['public-keys'].iteritems():
-                # lp:506332 uec metadata service responds with
-                # data that makes boto populate a string for 'klist' rather
-                # than a list.
-                if isinstance(klist, (str, basestring)):
-                    klist = [klist]
-                if isinstance(klist, (list, set)):
-                    for pkey in klist:
-                        # There is an empty string at
-                        # the end of the keylist, trim it
-                        if pkey:
-                            keys.append(pkey)
-
-        return keys
+        return normalize_pubkey_data(self.metadata.get('public-keys'))
 
     def _remap_device(self, short_name):
         # LP: #611137
@@ -210,6 +183,35 @@ class DataSource(object):
             availability_zone=self.availability_zone)
 
 
+def normalize_pubkey_data(pubkey_data):
+    keys = []
+
+    if not pubkey_data:
+        return keys
+
+    if isinstance(pubkey_data, (basestring, str)):
+        return str(pubkey_data).splitlines()
+
+    if isinstance(pubkey_data, (list, set)):
+        return list(pubkey_data)
+
+    if isinstance(pubkey_data, (dict)):
+        for (_keyname, klist) in pubkey_data.iteritems():
+            # lp:506332 uec metadata service responds with
+            # data that makes boto populate a string for 'klist' rather
+            # than a list.
+            if isinstance(klist, (str, basestring)):
+                klist = [klist]
+            if isinstance(klist, (list, set)):
+                for pkey in klist:
+                    # There is an empty string at
+                    # the end of the keylist, trim it
+                    if pkey:
+                        keys.append(pkey)
+
+    return keys
+
+
 def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list):
     ds_list = list_sources(cfg_list, ds_deps, pkg_list)
     ds_names = [util.obj_name(f) for f in ds_list]