Complete cleanup for refactoring/rework that makes it comptabile with the new structure, using unified util functions, logging and eliminating code and calls.

1 files changed, 123 insertions, 86 deletions

diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 7051ecda..38be71fa 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -2,9 +2,11 @@
 #
 #    Copyright (C) 2009-2010 Canonical Ltd.
 #    Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
+#    Copyright (C) 2012 Yahoo! Inc.
 #
 #    Author: Scott Moser <scott.moser@canonical.com>
 #    Author: Juerg Hafliger <juerg.haefliger@hp.com>
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
 #
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License version 3, as
@@ -18,31 +20,38 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import cloudinit.DataSource as DataSource
-
-from cloudinit import seeddir as base_seeddir
-from cloudinit import log
-import cloudinit.util as util
-import socket
+import os
 import time
+
 import boto.utils as boto_utils
-import os.path
 
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import url_helper as uhelp
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+DEF_MD_URL = "http://169.254.169.254"
+DEF_MD_VERSION = '2009-04-04'
+DEF_MD_URLS = [DEF_MD_URL, "http://instance-data:8773"]
 
-class DataSourceEc2(DataSource.DataSource):
-    api_ver = '2009-04-04'
-    seeddir = base_seeddir + '/ec2'
-    metadata_address = "http://169.254.169.254"
+
+class DataSourceEc2(sources.DataSource):
+    def __init__(self, sys_cfg, distro, paths):
+        sources.DataSource.__init__(self, sys_cfg, distro, paths)
+        self.metadata_address = DEF_MD_URL
+        self.seed_dir = os.path.join(paths.seed_dir, "ec2")
+        self.api_ver = DEF_MD_VERSION
 
     def __str__(self):
-        return("DataSourceEc2")
+        return util.obj_name(self)
 
     def get_data(self):
-        seedret = {}
-        if util.read_optional_seed(seedret, base=self.seeddir + "/"):
-            self.userdata_raw = seedret['user-data']
-            self.metadata = seedret['meta-data']
-            log.debug("using seeded ec2 data in %s" % self.seeddir)
+        seed_ret = {}
+        if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")):
+            self.userdata_raw = seed_ret['user-data']
+            self.metadata = seed_ret['meta-data']
+            LOG.debug("Using seeded ec2 data from %s", self.seed_dir)
             return True
 
         try:
@@ -53,51 +62,61 @@ class DataSourceEc2(DataSource.DataSource):
                 None, self.metadata_address)
             self.metadata = boto_utils.get_instance_metadata(self.api_ver,
                 self.metadata_address)
-            log.debug("crawl of metadata service took %ds" % (time.time() -
-                                                              start))
+            tot_time = int(time.time() - start)
+            LOG.debug("Crawl of metadata service took %s", tot_time)
             return True
-        except Exception as e:
-            print e
+        except Exception:
+            util.logexc(LOG, "Failed reading from metadata address %s",
+                        self.metadata_address)
             return False
 
     def get_instance_id(self):
-        return(self.metadata['instance-id'])
+        return self.metadata['instance-id']
 
     def get_availability_zone(self):
-        return(self.metadata['placement']['availability-zone'])
+        return self.metadata['placement']['availability-zone']
 
     def get_local_mirror(self):
-        return(self.get_mirror_from_availability_zone())
+        return self.get_mirror_from_availability_zone()
 
     def get_mirror_from_availability_zone(self, availability_zone=None):
-        # availability is like 'us-west-1b' or 'eu-west-1a'
-        if availability_zone == None:
+        # Availability is like 'us-west-1b' or 'eu-west-1a'
+        if availability_zone is None:
             availability_zone = self.get_availability_zone()
 
-        fallback = None
-
         if self.is_vpc():
-            return fallback
+            return None
 
-        try:
-            host = "%s.ec2.archive.ubuntu.com" % availability_zone[:-1]
-            socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM)
-            return 'http://%s/ubuntu/' % host
-        except:
-            return fallback
+        # Use the distro to get the mirror
+        if not availability_zone:
+            return None
 
-    def wait_for_metadata_service(self):
-        mcfg = self.ds_cfg
+        mirror_tpl = self.distro.get_option('availability_zone_template')
+        if not mirror_tpl:
+            return None
 
-        if not hasattr(mcfg, "get"):
-            mcfg = {}
+        tpl_params = {
+            'zone': availability_zone.strip(),
+        }
+        mirror_url = mirror_tpl % (tpl_params)
+
+        (max_wait, timeout) = self._get_url_settings()
+        worked = uhelp.wait_for_url([mirror_url], max_wait=max_wait,
+                                timeout=timeout, status_cb=LOG.warn)
+        if not worked:
+            return None
+
+        return mirror_url
 
+    def _get_url_settings(self):
+        mcfg = self.ds_cfg
+        if not mcfg:
+            mcfg = {}
         max_wait = 120
         try:
             max_wait = int(mcfg.get("max_wait", max_wait))
         except Exception:
-            util.logexc(log)
-            log.warn("Failed to get max wait. using %s" % max_wait)
+            util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
 
         if max_wait == 0:
             return False
@@ -106,91 +125,104 @@ class DataSourceEc2(DataSource.DataSource):
         try:
             timeout = int(mcfg.get("timeout", timeout))
         except Exception:
-            util.logexc(log)
-            log.warn("Failed to get timeout, using %s" % timeout)
+            util.logexc(LOG, "Failed to get timeout, using %s", timeout)
+        return (max_wait, timeout)
+
+    def wait_for_metadata_service(self):
+        mcfg = self.ds_cfg
+        if not mcfg:
+            mcfg = {}
 
-        def_mdurls = ["http://169.254.169.254", "http://instance-data:8773"]
-        mdurls = mcfg.get("metadata_urls", def_mdurls)
+        (max_wait, timeout) = self._get_url_settings()
 
         # Remove addresses from the list that wont resolve.
+        mdurls = mcfg.get("metadata_urls", DEF_MD_URLS)
         filtered = [x for x in mdurls if util.is_resolvable_url(x)]
 
         if set(filtered) != set(mdurls):
-            log.debug("removed the following from metadata urls: %s" %
-                list((set(mdurls) - set(filtered))))
+            LOG.debug("Removed the following from metadata urls: %s",
+                      list((set(mdurls) - set(filtered))))
 
         if len(filtered):
             mdurls = filtered
         else:
-            log.warn("Empty metadata url list! using default list")
-            mdurls = def_mdurls
+            LOG.warn("Empty metadata url list! using default list")
+            mdurls = DEF_MD_URLS
 
         urls = []
-        url2base = {False: False}
+        url2base = {}
         for url in mdurls:
             cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver)
             urls.append(cur)
             url2base[cur] = url
 
         starttime = time.time()
-        url = util.wait_for_url(urls=urls, max_wait=max_wait,
-                                timeout=timeout, status_cb=log.warn)
+        url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
+                                timeout=timeout, status_cb=LOG.warn)
 
         if url:
-            log.debug("Using metadata source: '%s'" % url2base[url])
+            LOG.info("Using metadata source: '%s'", url2base[url])
         else:
-            log.critical("giving up on md after %i seconds\n" %
-                         int(time.time() - starttime))
+            LOG.critical("Giving up on md from %s after %i seconds",
+                            urls, int(time.time() - starttime))
 
-        self.metadata_address = url2base[url]
-        return (bool(url))
+        self.metadata_address = url2base.get(url)
+        return bool(url)
+
+    def _remap_device(self, short_name):
+        # LP: #611137
+        # the metadata service may believe that devices are named 'sda'
+        # when the kernel named them 'vda' or 'xvda'
+        # we want to return the correct value for what will actually
+        # exist in this instance
+        mappings = {"sd": ("vd", "xvd")}
+        for (nfrom, tlist) in mappings.iteritems():
+            if not short_name.startswith(nfrom):
+                continue
+            for nto in tlist:
+                cand = "/dev/%s%s" % (nto, short_name[len(nfrom):])
+                if os.path.exists(cand):
+                    return cand
+        return None
 
     def device_name_to_device(self, name):
-        # consult metadata service, that has
+        # Consult metadata service, that has
         #  ephemeral0: sdb
         # and return 'sdb' for input 'ephemeral0'
         if 'block-device-mapping' not in self.metadata:
-            return(None)
+            return None
 
+        # Example:
+        # 'block-device-mapping': 
+        # {'ami': '/dev/sda1',
+        # 'ephemeral0': '/dev/sdb',
+        # 'root': '/dev/sda1'}
         found = None
-        for entname, device in self.metadata['block-device-mapping'].items():
+        for (entname, device) in self.metadata['block-device-mapping'].items():
             if entname == name:
                 found = device
                 break
             # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0'
             if entname == "ephemeral" and name == "ephemeral0":
                 found = device
-        if found == None:
-            log.debug("unable to convert %s to a device" % name)
+        if found is None:
+            LOG.debug("Unable to convert %s to a device", name)
             return None
 
-        # LP: #611137
-        # the metadata service may believe that devices are named 'sda'
-        # when the kernel named them 'vda' or 'xvda'
-        # we want to return the correct value for what will actually
-        # exist in this instance
-        mappings = {"sd": ("vd", "xvd")}
         ofound = found
-        short = os.path.basename(found)
-
         if not found.startswith("/"):
             found = "/dev/%s" % found
-
         if os.path.exists(found):
-            return(found)
+            return found
 
-        for nfrom, tlist in mappings.items():
-            if not short.startswith(nfrom):
-                continue
-            for nto in tlist:
-                cand = "/dev/%s%s" % (nto, short[len(nfrom):])
-                if os.path.exists(cand):
-                    log.debug("remapped device name %s => %s" % (found, cand))
-                    return(cand)
+        remapped = self._remap_device(os.path.basename(found))
+        if remapped:
+            LOG.debug("Remapped device name %s => %s", (found, remapped))
+            return remapped
 
-        # on t1.micro, ephemeral0 will appear in block-device-mapping from
+        # On t1.micro, ephemeral0 will appear in block-device-mapping from
         # metadata, but it will not exist on disk (and never will)
-        # at this pint, we've verified that the path did not exist
+        # at this point, we've verified that the path did not exist
         # in the special case of 'ephemeral0' return None to avoid bogus
         # fstab entry (LP: #744019)
         if name == "ephemeral0":
@@ -198,7 +230,11 @@ class DataSourceEc2(DataSource.DataSource):
         return ofound
 
     def is_vpc(self):
-        # per comment in LP: #615545
+        # See: https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/615545
+        # Detect that the machine was launched in a VPC.
+        # But I did notice that when in a VPC, meta-data
+        # does not have public-ipv4 and public-hostname
+        # listed as a possibility.
         ph = "public-hostname"
         p4 = "public-ipv4"
         if ((ph not in self.metadata or self.metadata[ph] == "") and
@@ -207,11 +243,12 @@ class DataSourceEc2(DataSource.DataSource):
         return False
 
 
+# Used to match classes to dependencies
 datasources = [
-  (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)),
+  (DataSourceEc2, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
 ]
 
 
-# return a list of data sources that match this set of dependencies
+# Return a list of data sources that match this set of dependencies
 def get_datasource_list(depends):
-    return(DataSource.list_from_depends(depends, datasources))
+    return sources.list_from_depends(depends, datasources)