From 758e152721891c707573757fe7a7ff410ec446e2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 24 Oct 2012 20:31:19 -0700 Subject: Handle the case where newer versions of boto are used that lazily load the metadata from the ec2 metadata service. 1. Add a ec2_utils module that checks which version of boto is being used and under the right versions the metadata dictionary will be expanded. 2. Use this new ec2_utils module in the cloudstack and ec2 datasources as there entrypoints into boto. --- cloudinit/ec2_utils.py | 65 +++++++++++++++++++++++++++++++ cloudinit/sources/DataSourceCloudStack.py | 11 +++--- cloudinit/sources/DataSourceEc2.py | 15 ++++--- 3 files changed, 79 insertions(+), 12 deletions(-) create mode 100644 cloudinit/ec2_utils.py diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py new file mode 100644 index 00000000..76699102 --- /dev/null +++ b/cloudinit/ec2_utils.py @@ -0,0 +1,65 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import pkg_resources +from pkg_resources import parse_version + +import cloudinit.util as util +import cloudinit.url_helper as uh + +import boto.utils as boto_utils + + +BOTO_LAZY = False +try: + _boto_lib = pkg_resources.get_distribution('boto') + if _boto_lib.parsed_version > parse_version("2.5.2"): + BOTO_LAZY = True +except pkg_resources.DistributionNotFound: + pass + + +# Versions of boto >= 2.6.0 try to lazily load +# the metadata backing, which doesn't work so well +# in cloud-init especially since the metadata is +# serialized and actions are performed where the +# metadata server may be blocked (thus the datasource +# will start failing) resulting in url exceptions +# when fields that do exist (or would have existed) +# do not exist due to the blocking that occurred. +def _unlazy_dict(mp): + if not isinstance(mp, (dict)): + return mp + if not BOTO_LAZY: + return mp + for (k, v) in mp.items(): + _unlazy_dict(v) + + +def get_instance_userdata(api_version, metadata_address): + ud = boto_utils.get_instance_userdata(api_version, None, metadata_address) + if not ud: + ud = '' + return ud + + +def get_instance_metadata(api_version, metadata_address): + metadata = boto_utils.get_instance_metadata(api_version, metadata_address) + if not isinstance(metadata, (dict)): + metadata = {} + return _unlazy_dict(metadata) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index f7ffa7cb..78cf24d7 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -26,8 +26,7 @@ from struct import pack import os import time -import boto.utils as boto_utils - +from cloudinit import ec2_utils as ec2 from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -116,10 +115,10 @@ class DataSourceCloudStack(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, - None, self.metadata_address) - self.metadata = boto_utils.get_instance_metadata(self.api_ver, - self.metadata_address) + self.userdata_raw = ec2.get_instance_userdata(self.api_ver, + self.metadata_address) + self.metadata = ec2.get_instance_metadata(self.api_ver, + self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 3686fa10..3da7b54e 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -23,8 +23,7 @@ import os import time -import boto.utils as boto_utils - +from cloudinit import ec2_utils as ec2 from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -53,6 +52,10 @@ class DataSourceEc2(sources.DataSource): def __str__(self): return util.obj_name(self) + def __getstate__(self): + # Versions of boto + pass + def get_data(self): seed_ret = {} if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")): @@ -65,10 +68,10 @@ class DataSourceEc2(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, - None, self.metadata_address) - self.metadata = boto_utils.get_instance_metadata(self.api_ver, - self.metadata_address) + self.userdata_raw = ec2.get_instance_userdata(self.api_ver, + self.metadata_address) + self.metadata = ec2.get_instance_metadata(self.api_ver, + self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True -- cgit v1.2.3 From 8ffc2c8f791b7694a121ec30dac7437c6e8fdb9b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 24 Oct 2012 20:35:30 -0700 Subject: Remove function that shouldn't have shown up. --- cloudinit/sources/DataSourceEc2.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 3da7b54e..0fc79b32 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -52,10 +52,6 @@ class DataSourceEc2(sources.DataSource): def __str__(self): return util.obj_name(self) - def __getstate__(self): - # Versions of boto - pass - def get_data(self): seed_ret = {} if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")): -- cgit v1.2.3 From ec28772204c9b262e34cc7837e3baac0dac5ec5e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 24 Oct 2012 20:37:12 -0700 Subject: Move the comment to the top + mark as fixing. LP: #1068801 --- cloudinit/ec2_utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 76699102..06b302f2 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -24,6 +24,14 @@ import cloudinit.url_helper as uh import boto.utils as boto_utils +# Versions of boto >= 2.6.0 try to lazily load +# the metadata backing, which doesn't work so well +# in cloud-init especially since the metadata is +# serialized and actions are performed where the +# metadata server may be blocked (thus the datasource +# will start failing) resulting in url exceptions +# when fields that do exist (or would have existed) +# do not exist due to the blocking that occurred. BOTO_LAZY = False try: @@ -34,14 +42,6 @@ except pkg_resources.DistributionNotFound: pass -# Versions of boto >= 2.6.0 try to lazily load -# the metadata backing, which doesn't work so well -# in cloud-init especially since the metadata is -# serialized and actions are performed where the -# metadata server may be blocked (thus the datasource -# will start failing) resulting in url exceptions -# when fields that do exist (or would have existed) -# do not exist due to the blocking that occurred. def _unlazy_dict(mp): if not isinstance(mp, (dict)): return mp -- cgit v1.2.3 From b0f6c7bfa94a5ba302debdc16a175cb0017f9634 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 9 Nov 2012 11:16:03 -0800 Subject: Fix the none return problem. --- cloudinit/ec2_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 06b302f2..eb5b3884 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -49,6 +49,7 @@ def _unlazy_dict(mp): return mp for (k, v) in mp.items(): _unlazy_dict(v) + return mp def get_instance_userdata(api_version, metadata_address): -- cgit v1.2.3 From 54828f025dede5d5bc1d26419083e6014f69212e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 12 Nov 2012 09:55:39 -0500 Subject: add comment to get_instance_userdata reguarding empty/un-provided userdata --- cloudinit/ec2_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index eb5b3884..a278ef04 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -53,6 +53,10 @@ def _unlazy_dict(mp): def get_instance_userdata(api_version, metadata_address): + # Note: boto.utils.get_instance_metadata returns '' for empty string + # so the change from non-true to '' is not specifically necessary, but + # this way cloud-init will get consistent behavior even if boto changed + # in the future to return a None on "no user-data provided". ud = boto_utils.get_instance_userdata(api_version, None, metadata_address) if not ud: ud = '' -- cgit v1.2.3 From 7fd838c187ad004d124c9293d91fdb4fca083f66 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 12 Nov 2012 09:55:48 -0500 Subject: add ChangeLog entry --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 5408a941..a208ab0d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -23,6 +23,8 @@ - Fix the merging of group configuration when that group configuration is a dict => members. [revno 707] - add yum_add_repo configuration module for adding additional yum repos + - work around the lazy loading of get_instance_metadata in boto >= 2.6.0 + by fully walking the dictionary. (LP: #1068801) 0.7.0: - add a 'exception_cb' argument to 'wait_for_url'. If provided, this method will be called back with the exception received and the message. -- cgit v1.2.3