Handle the case where newer versions of boto

are used that lazily load the metadata from the
ec2 metadata service.

1. Add a ec2_utils module that checks which version
   of boto is being used and under the right versions
   the metadata dictionary will be expanded.
2. Use this new ec2_utils module in the cloudstack and ec2
   datasources as there entrypoints into boto.

3 files changed, 79 insertions, 12 deletions

diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py
new file mode 100644
index 00000000..76699102
--- /dev/null
+++ b/cloudinit/ec2_utils.py
@@ -0,0 +1,65 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2012 Yahoo! Inc.
+#
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import pkg_resources
+from pkg_resources import parse_version
+
+import cloudinit.util as util
+import cloudinit.url_helper as uh
+
+import boto.utils as boto_utils
+
+
+BOTO_LAZY = False
+try:
+    _boto_lib = pkg_resources.get_distribution('boto')
+    if _boto_lib.parsed_version > parse_version("2.5.2"):
+        BOTO_LAZY = True
+except pkg_resources.DistributionNotFound:
+    pass
+
+
+# Versions of boto >= 2.6.0 try to lazily load
+# the metadata backing, which doesn't work so well
+# in cloud-init especially since the metadata is
+# serialized and actions are performed where the
+# metadata server may be blocked (thus the datasource
+# will start failing) resulting in url exceptions
+# when fields that do exist (or would have existed)
+# do not exist due to the blocking that occurred.
+def _unlazy_dict(mp):
+    if not isinstance(mp, (dict)):
+        return mp
+    if not BOTO_LAZY:
+        return mp
+    for (k, v) in mp.items():
+        _unlazy_dict(v)
+
+
+def get_instance_userdata(api_version, metadata_address):
+    ud = boto_utils.get_instance_userdata(api_version, None, metadata_address)
+    if not ud:
+        ud = ''
+    return ud
+
+
+def get_instance_metadata(api_version, metadata_address):
+    metadata = boto_utils.get_instance_metadata(api_version, metadata_address)
+    if not isinstance(metadata, (dict)):
+        metadata = {}
+    return _unlazy_dict(metadata)
diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index f7ffa7cb..78cf24d7 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -26,8 +26,7 @@ from struct import pack
 import os
 import time
 
-import boto.utils as boto_utils
-
+from cloudinit import ec2_utils as ec2
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import url_helper as uhelp
@@ -116,10 +115,10 @@ class DataSourceCloudStack(sources.DataSource):
             if not self.wait_for_metadata_service():
                 return False
             start_time = time.time()
-            self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver,
-                None, self.metadata_address)
-            self.metadata = boto_utils.get_instance_metadata(self.api_ver,
-                self.metadata_address)
+            self.userdata_raw = ec2.get_instance_userdata(self.api_ver,
+                                                          self.metadata_address)
+            self.metadata = ec2.get_instance_metadata(self.api_ver,
+                                                      self.metadata_address)
             LOG.debug("Crawl of metadata service took %s seconds",
                       int(time.time() - start_time))
             return True
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 3686fa10..3da7b54e 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -23,8 +23,7 @@
 import os
 import time
 
-import boto.utils as boto_utils
-
+from cloudinit import ec2_utils as ec2
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import url_helper as uhelp
@@ -53,6 +52,10 @@ class DataSourceEc2(sources.DataSource):
     def __str__(self):
         return util.obj_name(self)
 
+    def __getstate__(self):
+        # Versions of boto
+        pass
+
     def get_data(self):
         seed_ret = {}
         if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")):
@@ -65,10 +68,10 @@ class DataSourceEc2(sources.DataSource):
             if not self.wait_for_metadata_service():
                 return False
             start_time = time.time()
-            self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver,
-                None, self.metadata_address)
-            self.metadata = boto_utils.get_instance_metadata(self.api_ver,
-                self.metadata_address)
+            self.userdata_raw = ec2.get_instance_userdata(self.api_ver,
+                                                          self.metadata_address)
+            self.metadata = ec2.get_instance_metadata(self.api_ver,
+                                                      self.metadata_address)
             LOG.debug("Crawl of metadata service took %s seconds",
                        int(time.time() - start_time))
             return True