diff options
| -rw-r--r-- | cloudinit/sources/DataSourceEc2.py | 209 |
1 files changed, 123 insertions, 86 deletions
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 7051ecda..38be71fa 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -2,9 +2,11 @@ # # Copyright (C) 2009-2010 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser <scott.moser@canonical.com> # Author: Juerg Hafliger <juerg.haefliger@hp.com> +# Author: Joshua Harlow <harlowja@yahoo-inc.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -18,31 +20,38 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import socket +import os import time + import boto.utils as boto_utils -import os.path +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import url_helper as uhelp +from cloudinit import util + +LOG = logging.getLogger(__name__) +DEF_MD_URL = "http://169.254.169.254" +DEF_MD_VERSION = '2009-04-04' +DEF_MD_URLS = [DEF_MD_URL, "http://instance-data:8773"] -class DataSourceEc2(DataSource.DataSource): - api_ver = '2009-04-04' - seeddir = base_seeddir + '/ec2' - metadata_address = "http://169.254.169.254" + +class DataSourceEc2(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.metadata_address = DEF_MD_URL + self.seed_dir = os.path.join(paths.seed_dir, "ec2") + self.api_ver = DEF_MD_VERSION def __str__(self): - return("DataSourceEc2") + return util.obj_name(self) def get_data(self): - seedret = {} - if util.read_optional_seed(seedret, base=self.seeddir + "/"): - self.userdata_raw = seedret['user-data'] - self.metadata = seedret['meta-data'] - log.debug("using seeded ec2 data in %s" % self.seeddir) + seed_ret = {} + if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")): + self.userdata_raw = seed_ret['user-data'] + self.metadata = seed_ret['meta-data'] + LOG.debug("Using seeded ec2 data from %s", self.seed_dir) return True try: @@ -53,51 +62,61 @@ class DataSourceEc2(DataSource.DataSource): None, self.metadata_address) self.metadata = boto_utils.get_instance_metadata(self.api_ver, self.metadata_address) - log.debug("crawl of metadata service took %ds" % (time.time() - - start)) + tot_time = int(time.time() - start) + LOG.debug("Crawl of metadata service took %s", tot_time) return True - except Exception as e: - print e + except Exception: + util.logexc(LOG, "Failed reading from metadata address %s", + self.metadata_address) return False def get_instance_id(self): - return(self.metadata['instance-id']) + return self.metadata['instance-id'] def get_availability_zone(self): - return(self.metadata['placement']['availability-zone']) + return self.metadata['placement']['availability-zone'] def get_local_mirror(self): - return(self.get_mirror_from_availability_zone()) + return self.get_mirror_from_availability_zone() def get_mirror_from_availability_zone(self, availability_zone=None): - # availability is like 'us-west-1b' or 'eu-west-1a' - if availability_zone == None: + # Availability is like 'us-west-1b' or 'eu-west-1a' + if availability_zone is None: availability_zone = self.get_availability_zone() - fallback = None - if self.is_vpc(): - return fallback + return None - try: - host = "%s.ec2.archive.ubuntu.com" % availability_zone[:-1] - socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM) - return 'http://%s/ubuntu/' % host - except: - return fallback + # Use the distro to get the mirror + if not availability_zone: + return None - def wait_for_metadata_service(self): - mcfg = self.ds_cfg + mirror_tpl = self.distro.get_option('availability_zone_template') + if not mirror_tpl: + return None - if not hasattr(mcfg, "get"): - mcfg = {} + tpl_params = { + 'zone': availability_zone.strip(), + } + mirror_url = mirror_tpl % (tpl_params) + + (max_wait, timeout) = self._get_url_settings() + worked = uhelp.wait_for_url([mirror_url], max_wait=max_wait, + timeout=timeout, status_cb=LOG.warn) + if not worked: + return None + + return mirror_url + def _get_url_settings(self): + mcfg = self.ds_cfg + if not mcfg: + mcfg = {} max_wait = 120 try: max_wait = int(mcfg.get("max_wait", max_wait)) except Exception: - util.logexc(log) - log.warn("Failed to get max wait. using %s" % max_wait) + util.logexc(LOG, "Failed to get max wait. using %s", max_wait) if max_wait == 0: return False @@ -106,91 +125,104 @@ class DataSourceEc2(DataSource.DataSource): try: timeout = int(mcfg.get("timeout", timeout)) except Exception: - util.logexc(log) - log.warn("Failed to get timeout, using %s" % timeout) + util.logexc(LOG, "Failed to get timeout, using %s", timeout) + return (max_wait, timeout) + + def wait_for_metadata_service(self): + mcfg = self.ds_cfg + if not mcfg: + mcfg = {} - def_mdurls = ["http://169.254.169.254", "http://instance-data:8773"] - mdurls = mcfg.get("metadata_urls", def_mdurls) + (max_wait, timeout) = self._get_url_settings() # Remove addresses from the list that wont resolve. + mdurls = mcfg.get("metadata_urls", DEF_MD_URLS) filtered = [x for x in mdurls if util.is_resolvable_url(x)] if set(filtered) != set(mdurls): - log.debug("removed the following from metadata urls: %s" % - list((set(mdurls) - set(filtered)))) + LOG.debug("Removed the following from metadata urls: %s", + list((set(mdurls) - set(filtered)))) if len(filtered): mdurls = filtered else: - log.warn("Empty metadata url list! using default list") - mdurls = def_mdurls + LOG.warn("Empty metadata url list! using default list") + mdurls = DEF_MD_URLS urls = [] - url2base = {False: False} + url2base = {} for url in mdurls: cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver) urls.append(cur) url2base[cur] = url starttime = time.time() - url = util.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=log.warn) + url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, status_cb=LOG.warn) if url: - log.debug("Using metadata source: '%s'" % url2base[url]) + LOG.info("Using metadata source: '%s'", url2base[url]) else: - log.critical("giving up on md after %i seconds\n" % - int(time.time() - starttime)) + LOG.critical("Giving up on md from %s after %i seconds", + urls, int(time.time() - starttime)) - self.metadata_address = url2base[url] - return (bool(url)) + self.metadata_address = url2base.get(url) + return bool(url) + + def _remap_device(self, short_name): + # LP: #611137 + # the metadata service may believe that devices are named 'sda' + # when the kernel named them 'vda' or 'xvda' + # we want to return the correct value for what will actually + # exist in this instance + mappings = {"sd": ("vd", "xvd")} + for (nfrom, tlist) in mappings.iteritems(): + if not short_name.startswith(nfrom): + continue + for nto in tlist: + cand = "/dev/%s%s" % (nto, short_name[len(nfrom):]) + if os.path.exists(cand): + return cand + return None def device_name_to_device(self, name): - # consult metadata service, that has + # Consult metadata service, that has # ephemeral0: sdb # and return 'sdb' for input 'ephemeral0' if 'block-device-mapping' not in self.metadata: - return(None) + return None + # Example: + # 'block-device-mapping': + # {'ami': '/dev/sda1', + # 'ephemeral0': '/dev/sdb', + # 'root': '/dev/sda1'} found = None - for entname, device in self.metadata['block-device-mapping'].items(): + for (entname, device) in self.metadata['block-device-mapping'].items(): if entname == name: found = device break # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0' if entname == "ephemeral" and name == "ephemeral0": found = device - if found == None: - log.debug("unable to convert %s to a device" % name) + if found is None: + LOG.debug("Unable to convert %s to a device", name) return None - # LP: #611137 - # the metadata service may believe that devices are named 'sda' - # when the kernel named them 'vda' or 'xvda' - # we want to return the correct value for what will actually - # exist in this instance - mappings = {"sd": ("vd", "xvd")} ofound = found - short = os.path.basename(found) - if not found.startswith("/"): found = "/dev/%s" % found - if os.path.exists(found): - return(found) + return found - for nfrom, tlist in mappings.items(): - if not short.startswith(nfrom): - continue - for nto in tlist: - cand = "/dev/%s%s" % (nto, short[len(nfrom):]) - if os.path.exists(cand): - log.debug("remapped device name %s => %s" % (found, cand)) - return(cand) + remapped = self._remap_device(os.path.basename(found)) + if remapped: + LOG.debug("Remapped device name %s => %s", (found, remapped)) + return remapped - # on t1.micro, ephemeral0 will appear in block-device-mapping from + # On t1.micro, ephemeral0 will appear in block-device-mapping from # metadata, but it will not exist on disk (and never will) - # at this pint, we've verified that the path did not exist + # at this point, we've verified that the path did not exist # in the special case of 'ephemeral0' return None to avoid bogus # fstab entry (LP: #744019) if name == "ephemeral0": @@ -198,7 +230,11 @@ class DataSourceEc2(DataSource.DataSource): return ofound def is_vpc(self): - # per comment in LP: #615545 + # See: https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/615545 + # Detect that the machine was launched in a VPC. + # But I did notice that when in a VPC, meta-data + # does not have public-ipv4 and public-hostname + # listed as a possibility. ph = "public-hostname" p4 = "public-ipv4" if ((ph not in self.metadata or self.metadata[ph] == "") and @@ -207,11 +243,12 @@ class DataSourceEc2(DataSource.DataSource): return False +# Used to match classes to dependencies datasources = [ - (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), + (DataSourceEc2, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] -# return a list of data sources that match this set of dependencies +# Return a list of data sources that match this set of dependencies def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) + return sources.list_from_depends(depends, datasources) |