From 758e152721891c707573757fe7a7ff410ec446e2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 24 Oct 2012 20:31:19 -0700 Subject: Handle the case where newer versions of boto are used that lazily load the metadata from the ec2 metadata service. 1. Add a ec2_utils module that checks which version of boto is being used and under the right versions the metadata dictionary will be expanded. 2. Use this new ec2_utils module in the cloudstack and ec2 datasources as there entrypoints into boto. --- cloudinit/ec2_utils.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 cloudinit/ec2_utils.py (limited to 'cloudinit/ec2_utils.py') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py new file mode 100644 index 00000000..76699102 --- /dev/null +++ b/cloudinit/ec2_utils.py @@ -0,0 +1,65 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import pkg_resources +from pkg_resources import parse_version + +import cloudinit.util as util +import cloudinit.url_helper as uh + +import boto.utils as boto_utils + + +BOTO_LAZY = False +try: + _boto_lib = pkg_resources.get_distribution('boto') + if _boto_lib.parsed_version > parse_version("2.5.2"): + BOTO_LAZY = True +except pkg_resources.DistributionNotFound: + pass + + +# Versions of boto >= 2.6.0 try to lazily load +# the metadata backing, which doesn't work so well +# in cloud-init especially since the metadata is +# serialized and actions are performed where the +# metadata server may be blocked (thus the datasource +# will start failing) resulting in url exceptions +# when fields that do exist (or would have existed) +# do not exist due to the blocking that occurred. +def _unlazy_dict(mp): + if not isinstance(mp, (dict)): + return mp + if not BOTO_LAZY: + return mp + for (k, v) in mp.items(): + _unlazy_dict(v) + + +def get_instance_userdata(api_version, metadata_address): + ud = boto_utils.get_instance_userdata(api_version, None, metadata_address) + if not ud: + ud = '' + return ud + + +def get_instance_metadata(api_version, metadata_address): + metadata = boto_utils.get_instance_metadata(api_version, metadata_address) + if not isinstance(metadata, (dict)): + metadata = {} + return _unlazy_dict(metadata) -- cgit v1.2.3 From ec28772204c9b262e34cc7837e3baac0dac5ec5e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 24 Oct 2012 20:37:12 -0700 Subject: Move the comment to the top + mark as fixing. LP: #1068801 --- cloudinit/ec2_utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'cloudinit/ec2_utils.py') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 76699102..06b302f2 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -24,6 +24,14 @@ import cloudinit.url_helper as uh import boto.utils as boto_utils +# Versions of boto >= 2.6.0 try to lazily load +# the metadata backing, which doesn't work so well +# in cloud-init especially since the metadata is +# serialized and actions are performed where the +# metadata server may be blocked (thus the datasource +# will start failing) resulting in url exceptions +# when fields that do exist (or would have existed) +# do not exist due to the blocking that occurred. BOTO_LAZY = False try: @@ -34,14 +42,6 @@ except pkg_resources.DistributionNotFound: pass -# Versions of boto >= 2.6.0 try to lazily load -# the metadata backing, which doesn't work so well -# in cloud-init especially since the metadata is -# serialized and actions are performed where the -# metadata server may be blocked (thus the datasource -# will start failing) resulting in url exceptions -# when fields that do exist (or would have existed) -# do not exist due to the blocking that occurred. def _unlazy_dict(mp): if not isinstance(mp, (dict)): return mp -- cgit v1.2.3 From b0f6c7bfa94a5ba302debdc16a175cb0017f9634 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 9 Nov 2012 11:16:03 -0800 Subject: Fix the none return problem. --- cloudinit/ec2_utils.py | 1 + 1 file changed, 1 insertion(+) (limited to 'cloudinit/ec2_utils.py') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 06b302f2..eb5b3884 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -49,6 +49,7 @@ def _unlazy_dict(mp): return mp for (k, v) in mp.items(): _unlazy_dict(v) + return mp def get_instance_userdata(api_version, metadata_address): -- cgit v1.2.3 From 54828f025dede5d5bc1d26419083e6014f69212e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 12 Nov 2012 09:55:39 -0500 Subject: add comment to get_instance_userdata reguarding empty/un-provided userdata --- cloudinit/ec2_utils.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'cloudinit/ec2_utils.py') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index eb5b3884..a278ef04 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -53,6 +53,10 @@ def _unlazy_dict(mp): def get_instance_userdata(api_version, metadata_address): + # Note: boto.utils.get_instance_metadata returns '' for empty string + # so the change from non-true to '' is not specifically necessary, but + # this way cloud-init will get consistent behavior even if boto changed + # in the future to return a None on "no user-data provided". ud = boto_utils.get_instance_userdata(api_version, None, metadata_address) if not ud: ud = '' -- cgit v1.2.3 From 8730e143ec07372107d794abe9f4857ead6d4718 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 12 Nov 2012 12:23:44 -0500 Subject: pep8 and pylint fixups --- cloudinit/ec2_utils.py | 9 +++------ cloudinit/sources/DataSourceCloudStack.py | 2 +- cloudinit/sources/DataSourceEc2.py | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) (limited to 'cloudinit/ec2_utils.py') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index a278ef04..32bf3968 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -17,10 +17,7 @@ # along with this program. If not, see . import pkg_resources -from pkg_resources import parse_version - -import cloudinit.util as util -import cloudinit.url_helper as uh +from pkg_resources import parse_version as pver import boto.utils as boto_utils @@ -36,7 +33,7 @@ import boto.utils as boto_utils BOTO_LAZY = False try: _boto_lib = pkg_resources.get_distribution('boto') - if _boto_lib.parsed_version > parse_version("2.5.2"): + if _boto_lib.parsed_version > pver("2.5.2"): # pylint: disable=E1103 BOTO_LAZY = True except pkg_resources.DistributionNotFound: pass @@ -47,7 +44,7 @@ def _unlazy_dict(mp): return mp if not BOTO_LAZY: return mp - for (k, v) in mp.items(): + for (_k, v) in mp.items(): _unlazy_dict(v) return mp diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 78cf24d7..076dba5a 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -116,7 +116,7 @@ class DataSourceCloudStack(sources.DataSource): return False start_time = time.time() self.userdata_raw = ec2.get_instance_userdata(self.api_ver, - self.metadata_address) + self.metadata_address) self.metadata = ec2.get_instance_metadata(self.api_ver, self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 6f51dfae..2db53446 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -65,7 +65,7 @@ class DataSourceEc2(sources.DataSource): return False start_time = time.time() self.userdata_raw = ec2.get_instance_userdata(self.api_ver, - self.metadata_address) + self.metadata_address) self.metadata = ec2.get_instance_metadata(self.api_ver, self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", -- cgit v1.2.3 From 10147364638595d6c1a7077cc47c205413ef13a6 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 13 Nov 2012 11:02:03 -0800 Subject: Even when using boto < 2.6 force the unlazying to occur It seems like its possible that boto 2.5.2 and below have the lazy loading metadata dictionary so as a precaution we will always take the hit of unlazying the metadata dictionary by traversing it which in the non-lazy dictionary case has no effect (its marginal). This also removes the need to check the boto version and the dependency on setup tools just for this case. --- cloudinit/ec2_utils.py | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) (limited to 'cloudinit/ec2_utils.py') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 32bf3968..46b93f39 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -16,34 +16,26 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import pkg_resources -from pkg_resources import parse_version as pver - import boto.utils as boto_utils -# Versions of boto >= 2.6.0 try to lazily load -# the metadata backing, which doesn't work so well -# in cloud-init especially since the metadata is -# serialized and actions are performed where the -# metadata server may be blocked (thus the datasource -# will start failing) resulting in url exceptions -# when fields that do exist (or would have existed) -# do not exist due to the blocking that occurred. - -BOTO_LAZY = False -try: - _boto_lib = pkg_resources.get_distribution('boto') - if _boto_lib.parsed_version > pver("2.5.2"): # pylint: disable=E1103 - BOTO_LAZY = True -except pkg_resources.DistributionNotFound: - pass +# Versions of boto >= 2.6.0 (and possibly 2.5.2) +# try to lazily load the metadata backing, which +# doesn't work so well in cloud-init especially +# since the metadata is serialized and actions are +# performed where the metadata server may be blocked +# (thus the datasource will start failing) resulting +# in url exceptions when fields that do exist (or +# would have existed) do not exist due to the blocking +# that occurred. def _unlazy_dict(mp): if not isinstance(mp, (dict)): return mp - if not BOTO_LAZY: - return mp + # Walk over the keys/values which + # forces boto to unlazy itself and + # has no effect on dictionaries that + # already have there items. for (_k, v) in mp.items(): _unlazy_dict(v) return mp -- cgit v1.2.3