From ffa19cbd80ba10453f0ef448c1c10dbcbf5be504 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 5 Oct 2012 13:38:54 -0700 Subject: Ensure that config drive datasource attempts to translate the device name to a actual device using logic that will try the ec2 metadata (if avail) or will try using 'blkid' to find a corresponding label. LP: #1062540 --- cloudinit/sources/DataSourceEc2.py | 16 ---------------- 1 file changed, 16 deletions(-) (limited to 'cloudinit/sources/DataSourceEc2.py') diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index c7ad6d54..3686fa10 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -151,22 +151,6 @@ class DataSourceEc2(sources.DataSource): self.metadata_address = url2base.get(url) return bool(url) - def _remap_device(self, short_name): - # LP: #611137 - # the metadata service may believe that devices are named 'sda' - # when the kernel named them 'vda' or 'xvda' - # we want to return the correct value for what will actually - # exist in this instance - mappings = {"sd": ("vd", "xvd")} - for (nfrom, tlist) in mappings.iteritems(): - if not short_name.startswith(nfrom): - continue - for nto in tlist: - cand = "/dev/%s%s" % (nto, short_name[len(nfrom):]) - if os.path.exists(cand): - return cand - return None - def device_name_to_device(self, name): # Consult metadata service, that has # ephemeral0: sdb -- cgit v1.2.3 From 758e152721891c707573757fe7a7ff410ec446e2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 24 Oct 2012 20:31:19 -0700 Subject: Handle the case where newer versions of boto are used that lazily load the metadata from the ec2 metadata service. 1. Add a ec2_utils module that checks which version of boto is being used and under the right versions the metadata dictionary will be expanded. 2. Use this new ec2_utils module in the cloudstack and ec2 datasources as there entrypoints into boto. --- cloudinit/ec2_utils.py | 65 +++++++++++++++++++++++++++++++ cloudinit/sources/DataSourceCloudStack.py | 11 +++--- cloudinit/sources/DataSourceEc2.py | 15 ++++--- 3 files changed, 79 insertions(+), 12 deletions(-) create mode 100644 cloudinit/ec2_utils.py (limited to 'cloudinit/sources/DataSourceEc2.py') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py new file mode 100644 index 00000000..76699102 --- /dev/null +++ b/cloudinit/ec2_utils.py @@ -0,0 +1,65 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import pkg_resources +from pkg_resources import parse_version + +import cloudinit.util as util +import cloudinit.url_helper as uh + +import boto.utils as boto_utils + + +BOTO_LAZY = False +try: + _boto_lib = pkg_resources.get_distribution('boto') + if _boto_lib.parsed_version > parse_version("2.5.2"): + BOTO_LAZY = True +except pkg_resources.DistributionNotFound: + pass + + +# Versions of boto >= 2.6.0 try to lazily load +# the metadata backing, which doesn't work so well +# in cloud-init especially since the metadata is +# serialized and actions are performed where the +# metadata server may be blocked (thus the datasource +# will start failing) resulting in url exceptions +# when fields that do exist (or would have existed) +# do not exist due to the blocking that occurred. +def _unlazy_dict(mp): + if not isinstance(mp, (dict)): + return mp + if not BOTO_LAZY: + return mp + for (k, v) in mp.items(): + _unlazy_dict(v) + + +def get_instance_userdata(api_version, metadata_address): + ud = boto_utils.get_instance_userdata(api_version, None, metadata_address) + if not ud: + ud = '' + return ud + + +def get_instance_metadata(api_version, metadata_address): + metadata = boto_utils.get_instance_metadata(api_version, metadata_address) + if not isinstance(metadata, (dict)): + metadata = {} + return _unlazy_dict(metadata) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index f7ffa7cb..78cf24d7 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -26,8 +26,7 @@ from struct import pack import os import time -import boto.utils as boto_utils - +from cloudinit import ec2_utils as ec2 from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -116,10 +115,10 @@ class DataSourceCloudStack(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, - None, self.metadata_address) - self.metadata = boto_utils.get_instance_metadata(self.api_ver, - self.metadata_address) + self.userdata_raw = ec2.get_instance_userdata(self.api_ver, + self.metadata_address) + self.metadata = ec2.get_instance_metadata(self.api_ver, + self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 3686fa10..3da7b54e 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -23,8 +23,7 @@ import os import time -import boto.utils as boto_utils - +from cloudinit import ec2_utils as ec2 from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -53,6 +52,10 @@ class DataSourceEc2(sources.DataSource): def __str__(self): return util.obj_name(self) + def __getstate__(self): + # Versions of boto + pass + def get_data(self): seed_ret = {} if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")): @@ -65,10 +68,10 @@ class DataSourceEc2(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, - None, self.metadata_address) - self.metadata = boto_utils.get_instance_metadata(self.api_ver, - self.metadata_address) + self.userdata_raw = ec2.get_instance_userdata(self.api_ver, + self.metadata_address) + self.metadata = ec2.get_instance_metadata(self.api_ver, + self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True -- cgit v1.2.3 From 8ffc2c8f791b7694a121ec30dac7437c6e8fdb9b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 24 Oct 2012 20:35:30 -0700 Subject: Remove function that shouldn't have shown up. --- cloudinit/sources/DataSourceEc2.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'cloudinit/sources/DataSourceEc2.py') diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 3da7b54e..0fc79b32 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -52,10 +52,6 @@ class DataSourceEc2(sources.DataSource): def __str__(self): return util.obj_name(self) - def __getstate__(self): - # Versions of boto - pass - def get_data(self): seed_ret = {} if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")): -- cgit v1.2.3 From 1e6fc277a1c8d695c37741cc31f5ddab3d5b5600 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 7 Nov 2012 16:08:17 -0500 Subject: remove dead code from DataSourceEc2 --- cloudinit/sources/DataSourceEc2.py | 16 ---------------- 1 file changed, 16 deletions(-) (limited to 'cloudinit/sources/DataSourceEc2.py') diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 3686fa10..cff50669 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -86,9 +86,6 @@ class DataSourceEc2(sources.DataSource): def get_instance_id(self): return self.metadata['instance-id'] - def get_availability_zone(self): - return self.metadata['placement']['availability-zone'] - def _get_url_settings(self): mcfg = self.ds_cfg if not mcfg: @@ -198,19 +195,6 @@ class DataSourceEc2(sources.DataSource): return None return ofound - def is_vpc(self): - # See: https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/615545 - # Detect that the machine was launched in a VPC. - # But I did notice that when in a VPC, meta-data - # does not have public-ipv4 and public-hostname - # listed as a possibility. - ph = "public-hostname" - p4 = "public-ipv4" - if ((ph not in self.metadata or self.metadata[ph] == "") and - (p4 not in self.metadata or self.metadata[p4] == "")): - return True - return False - @property def availability_zone(self): try: -- cgit v1.2.3 From 8730e143ec07372107d794abe9f4857ead6d4718 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 12 Nov 2012 12:23:44 -0500 Subject: pep8 and pylint fixups --- cloudinit/ec2_utils.py | 9 +++------ cloudinit/sources/DataSourceCloudStack.py | 2 +- cloudinit/sources/DataSourceEc2.py | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) (limited to 'cloudinit/sources/DataSourceEc2.py') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index a278ef04..32bf3968 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -17,10 +17,7 @@ # along with this program. If not, see . import pkg_resources -from pkg_resources import parse_version - -import cloudinit.util as util -import cloudinit.url_helper as uh +from pkg_resources import parse_version as pver import boto.utils as boto_utils @@ -36,7 +33,7 @@ import boto.utils as boto_utils BOTO_LAZY = False try: _boto_lib = pkg_resources.get_distribution('boto') - if _boto_lib.parsed_version > parse_version("2.5.2"): + if _boto_lib.parsed_version > pver("2.5.2"): # pylint: disable=E1103 BOTO_LAZY = True except pkg_resources.DistributionNotFound: pass @@ -47,7 +44,7 @@ def _unlazy_dict(mp): return mp if not BOTO_LAZY: return mp - for (k, v) in mp.items(): + for (_k, v) in mp.items(): _unlazy_dict(v) return mp diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 78cf24d7..076dba5a 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -116,7 +116,7 @@ class DataSourceCloudStack(sources.DataSource): return False start_time = time.time() self.userdata_raw = ec2.get_instance_userdata(self.api_ver, - self.metadata_address) + self.metadata_address) self.metadata = ec2.get_instance_metadata(self.api_ver, self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 6f51dfae..2db53446 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -65,7 +65,7 @@ class DataSourceEc2(sources.DataSource): return False start_time = time.time() self.userdata_raw = ec2.get_instance_userdata(self.api_ver, - self.metadata_address) + self.metadata_address) self.metadata = ec2.get_instance_metadata(self.api_ver, self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", -- cgit v1.2.3