From 5fc46193d3e65d0eaaaa45bcf41c5b35b4e80df7 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Tue, 26 Aug 2014 13:41:46 -0700
Subject: Optimize away most of the path_exists checks

---
 cloudinit/sources/helpers/openstack.py | 101 +++++++++++++++++----------------
 1 file changed, 52 insertions(+), 49 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 0fac0335..361c9994 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -149,10 +149,6 @@ class BaseReader(object):
     def _path_join(self, base, *add_ons):
         pass
 
-    @abc.abstractmethod
-    def _path_exists(self, path):
-        pass
-
     @abc.abstractmethod
     def _path_read(self, path):
         pass
@@ -170,22 +166,9 @@ class BaseReader(object):
         path = self._path_join(self.base_path, "openstack", *path_pieces)
         return self._path_read(path)
 
+    @abc.abstractmethod
     def _find_working_version(self, version):
-        search_versions = [version] + list(OS_VERSIONS)
-        for potential_version in search_versions:
-            if not potential_version:
-                continue
-            path = self._path_join(self.base_path, "openstack",
-                                   potential_version)
-            if self._path_exists(path):
-                if potential_version != version:
-                    LOG.debug("Version '%s' not available, attempting to use"
-                              " version '%s' instead", version,
-                              potential_version)
-                return potential_version
-        LOG.debug("Version '%s' not available, attempting to use '%s'"
-                  " instead", version, OS_LATEST)
-        return OS_LATEST
+        pass
 
     def read_v2(self, version=None):
         """Reads a version 2 formatted location.
@@ -228,15 +211,14 @@ class BaseReader(object):
             path = self._path_join(self.base_path, path)
             data = None
             found = False
-            if self._path_exists(path):
-                try:
-                    data = self._path_read(path)
-                except IOError:
-                    raise NonReadable("Failed to read: %s" % path)
-                found = True
+            try:
+                data = self._path_read(path)
+            except IOError:
+                pass
             else:
-                if required:
-                    raise NonReadable("Missing mandatory path: %s" % path)
+                found = True
+            if required and not found:
+                raise NonReadable("Missing mandatory path: %s" % path)
             if found and translator:
                 try:
                     data = translator(data)
@@ -315,6 +297,23 @@ class ConfigDriveReader(BaseReader):
     def _path_read(self, path):
         return util.load_file(path)
 
+    def _find_working_version(self, version):
+        search_versions = [version] + list(OS_VERSIONS)
+        for potential_version in search_versions:
+            if not potential_version:
+                continue
+            path = self._path_join(self.base_path, "openstack",
+                                   potential_version)
+            if self._path_exists(path):
+                if potential_version != version:
+                    LOG.debug("Version '%s' not available, attempting to use"
+                              " version '%s' instead", version,
+                              potential_version)
+                return potential_version
+        LOG.debug("Version '%s' not available, attempting to use '%s'"
+                  " instead", version, OS_LATEST)
+        return OS_LATEST
+
     def _read_ec2_metadata(self):
         path = self._path_join(self.base_path,
                                'ec2', 'latest', 'meta-data.json')
@@ -401,6 +400,32 @@ class MetadataReader(BaseReader):
         self.timeout = float(timeout)
         self.retries = int(retries)
 
+    def _find_working_version(self, version):
+        search_versions = [version] + list(OS_VERSIONS)
+        version_path = self._path_join(self.base_path, "openstack")
+        versions_available = []
+        try:
+            versions = self._path_read(version_path)
+        except IOError:
+            pass
+        else:
+            for line in versions.splitlines():
+                line = line.strip()
+                if not line:
+                    continue
+                versions_available.append(line)
+        for potential_version in search_versions:
+            if potential_version not in versions_available:
+                continue
+            if potential_version != version:
+                LOG.debug("Version '%s' not available, attempting to use"
+                          " version '%s' instead", version,
+                          potential_version)
+            return potential_version
+        LOG.debug("Version '%s' not available, searched %s, attempting to"
+                  " use '%s' instead", version, search_versions, OS_LATEST)
+        return OS_LATEST
+
     def _path_read(self, path):
         response = url_helper.readurl(path,
                                       retries=self.retries,
@@ -408,28 +433,6 @@ class MetadataReader(BaseReader):
                                       timeout=self.timeout)
         return response.contents
 
-    def _path_exists(self, path):
-
-        def should_retry_cb(request, cause):
-            try:
-                code = int(cause.code)
-                if code >= 400:
-                    return False
-            except (TypeError, ValueError):
-                # Older versions of requests didn't have a code.
-                pass
-            return True
-
-        try:
-            response = url_helper.readurl(path,
-                                          retries=self.retries,
-                                          ssl_details=self.ssl_details,
-                                          timeout=self.timeout,
-                                          exception_cb=should_retry_cb)
-            return response.ok()
-        except IOError:
-            return False
-
     def _path_join(self, base, *add_ons):
         return url_helper.combine_url(base, *add_ons)
 
-- 
cgit v1.2.3


From cc9e3af6c95b3263a49d4590d9dd176bdc570c99 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@gmail.com>
Date: Tue, 26 Aug 2014 23:02:58 -0700
Subject: Add logging around failure to read optional/mandatory paths

---
 cloudinit/sources/helpers/openstack.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 361c9994..e5a38de0 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -213,8 +213,12 @@ class BaseReader(object):
             found = False
             try:
                 data = self._path_read(path)
-            except IOError:
-                pass
+            except IOError as e:
+                if not required:
+                    LOG.debug("Failed reading optional path %s due"
+                              " to: %s", path, e)
+                else:
+                    LOG.exception("Failed reading mandatory path %s", path)
             else:
                 found = True
             if required and not found:
-- 
cgit v1.2.3


From 6a6d3a499c2327b03993bbaea2b9d0df5dc7eb64 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@gmail.com>
Date: Tue, 26 Aug 2014 23:04:56 -0700
Subject: Just use os.path.exists directly

---
 cloudinit/sources/helpers/openstack.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index e5a38de0..418ab4d1 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -295,9 +295,6 @@ class ConfigDriveReader(BaseReader):
         components = [base] + list(add_ons)
         return os.path.join(*components)
 
-    def _path_exists(self, path):
-        return os.path.exists(path)
-
     def _path_read(self, path):
         return util.load_file(path)
 
@@ -308,7 +305,7 @@ class ConfigDriveReader(BaseReader):
                 continue
             path = self._path_join(self.base_path, "openstack",
                                    potential_version)
-            if self._path_exists(path):
+            if os.path.exists(path):
                 if potential_version != version:
                     LOG.debug("Version '%s' not available, attempting to use"
                               " version '%s' instead", version,
@@ -321,7 +318,7 @@ class ConfigDriveReader(BaseReader):
     def _read_ec2_metadata(self):
         path = self._path_join(self.base_path,
                                'ec2', 'latest', 'meta-data.json')
-        if not self._path_exists(path):
+        if not os.path.exists(path):
             return {}
         else:
             try:
@@ -341,7 +338,7 @@ class ConfigDriveReader(BaseReader):
         found = {}
         for name in FILES_V1.keys():
             path = self._path_join(self.base_path, name)
-            if self._path_exists(path):
+            if os.path.exists(path):
                 found[name] = path
         if len(found) == 0:
             raise NonReadable("%s: no files found" % (self.base_path))
-- 
cgit v1.2.3


From 9a7587f35f327b9f8cafce8687832e9e77c1afde Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@gmail.com>
Date: Tue, 26 Aug 2014 23:08:14 -0700
Subject: Log a warning when unable to fetch the openstack versions

---
 cloudinit/sources/helpers/openstack.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 418ab4d1..56986a94 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -407,8 +407,9 @@ class MetadataReader(BaseReader):
         versions_available = []
         try:
             versions = self._path_read(version_path)
-        except IOError:
-            pass
+        except IOError as e:
+            LOG.warn("Unable to read openstack versions from %s due"
+                     " to: %s", version_path, e)
         else:
             for line in versions.splitlines():
                 line = line.strip()
-- 
cgit v1.2.3


From fb482ce4a36d9b4be75a8bf5b428189548a205d9 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@gmail.com>
Date: Tue, 26 Aug 2014 23:10:50 -0700
Subject: Show the available versions in the debug log message

---
 cloudinit/sources/helpers/openstack.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 56986a94..3ceec837 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -424,8 +424,9 @@ class MetadataReader(BaseReader):
                           " version '%s' instead", version,
                           potential_version)
             return potential_version
-        LOG.debug("Version '%s' not available, searched %s, attempting to"
-                  " use '%s' instead", version, search_versions, OS_LATEST)
+        LOG.debug("Version '%s' not available, searched for %s (with available"
+                  " versions being %s), attempting to use '%s' instead",
+                  version, search_versions, versions_available, OS_LATEST)
         return OS_LATEST
 
     def _path_read(self, path):
-- 
cgit v1.2.3


From d701035265c765bc42cb3bc358f2bfd0b41f484b Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Wed, 27 Aug 2014 12:30:23 -0700
Subject: Fixed more of the slowness around fetching and retrying

---
 cloudinit/sources/helpers/openstack.py            | 112 +++++++++++++---------
 tests/unittests/test_datasource/test_openstack.py |  23 ++++-
 2 files changed, 84 insertions(+), 51 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 3ceec837..2d0fc70e 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -21,6 +21,7 @@
 import abc
 import base64
 import copy
+import httplib
 import os
 
 from cloudinit import ec2_utils
@@ -153,10 +154,31 @@ class BaseReader(object):
     def _path_read(self, path):
         pass
 
+    @abc.abstractmethod
+    def _fetch_available_versions(self):
+        pass
+
     @abc.abstractmethod
     def _read_ec2_metadata(self):
         pass
 
+    def _find_working_version(self, version):
+        search_versions = [version] + list(OS_VERSIONS)
+        available_versions = self._fetch_available_versions()
+        for potential_version in search_versions:
+            if not potential_version:
+                continue
+            if potential_version not in available_versions:
+                continue
+            if potential_version != version:
+                LOG.debug("Version '%s' not available, attempting to use"
+                          " version '%s' instead", version,
+                          potential_version)
+                return potential_version
+        LOG.debug("Version '%s' not available, attempting to use '%s'"
+                  " instead", version, OS_LATEST)
+        return OS_LATEST
+
     def _read_content_path(self, item):
         path = item.get('content_path', '').lstrip("/")
         path_pieces = path.split("/")
@@ -166,10 +188,6 @@ class BaseReader(object):
         path = self._path_join(self.base_path, "openstack", *path_pieces)
         return self._path_read(path)
 
-    @abc.abstractmethod
-    def _find_working_version(self, version):
-        pass
-
     def read_v2(self, version=None):
         """Reads a version 2 formatted location.
 
@@ -290,6 +308,7 @@ class BaseReader(object):
 class ConfigDriveReader(BaseReader):
     def __init__(self, base_path):
         super(ConfigDriveReader, self).__init__(base_path)
+        self._versions = None
 
     def _path_join(self, base, *add_ons):
         components = [base] + list(add_ons)
@@ -298,22 +317,21 @@ class ConfigDriveReader(BaseReader):
     def _path_read(self, path):
         return util.load_file(path)
 
-    def _find_working_version(self, version):
-        search_versions = [version] + list(OS_VERSIONS)
-        for potential_version in search_versions:
-            if not potential_version:
-                continue
-            path = self._path_join(self.base_path, "openstack",
-                                   potential_version)
-            if os.path.exists(path):
-                if potential_version != version:
-                    LOG.debug("Version '%s' not available, attempting to use"
-                              " version '%s' instead", version,
-                              potential_version)
-                return potential_version
-        LOG.debug("Version '%s' not available, attempting to use '%s'"
-                  " instead", version, OS_LATEST)
-        return OS_LATEST
+    def _fetch_available_versions(self):
+        if self._versions is not None:
+            return self._versions
+        else:
+            versions_available = []
+            path = self._path_join(self.base_path, 'openstack')
+            try:
+                for child in os.listdir(path):
+                    child_path = os.path.join(path, child)
+                    if os.path.isdir(child_path):
+                        versions_available.append(child)
+            except (OSError, IOError):
+                pass
+            self._versions = tuple(versions_available)
+            return self._versions
 
     def _read_ec2_metadata(self):
         path = self._path_join(self.base_path,
@@ -400,40 +418,40 @@ class MetadataReader(BaseReader):
         self.ssl_details = ssl_details
         self.timeout = float(timeout)
         self.retries = int(retries)
+        self._versions = None
 
-    def _find_working_version(self, version):
-        search_versions = [version] + list(OS_VERSIONS)
-        version_path = self._path_join(self.base_path, "openstack")
-        versions_available = []
-        try:
-            versions = self._path_read(version_path)
-        except IOError as e:
-            LOG.warn("Unable to read openstack versions from %s due"
-                     " to: %s", version_path, e)
+    def _fetch_available_versions(self):
+        if self._versions is not None:
+            return self._versions
         else:
-            for line in versions.splitlines():
-                line = line.strip()
-                if not line:
-                    continue
-                versions_available.append(line)
-        for potential_version in search_versions:
-            if potential_version not in versions_available:
-                continue
-            if potential_version != version:
-                LOG.debug("Version '%s' not available, attempting to use"
-                          " version '%s' instead", version,
-                          potential_version)
-            return potential_version
-        LOG.debug("Version '%s' not available, searched for %s (with available"
-                  " versions being %s), attempting to use '%s' instead",
-                  version, search_versions, versions_available, OS_LATEST)
-        return OS_LATEST
+            path = self._path_join(self.base_path, "openstack")
+            versions_available = []
+            try:
+                versions = self._path_read(path)
+            except IOError as e:
+                LOG.warn("Unable to read openstack versions from %s due"
+                         " to: %s", path, e)
+            else:
+                for line in versions.splitlines():
+                    line = line.strip()
+                    if not line:
+                        continue
+                    versions_available.append(line)
+            self._versions = tuple(versions_available)
+            return self._versions
 
     def _path_read(self, path):
+
+        def should_retry(_request_args, cause):
+            if cause.code == httplib.NOT_FOUND:
+                return False
+            return True
+
         response = url_helper.readurl(path,
                                       retries=self.retries,
                                       ssl_details=self.ssl_details,
-                                      timeout=self.timeout)
+                                      timeout=self.timeout,
+                                      exception_cb=should_retry)
         return response.contents
 
     def _path_join(self, base, *add_ons):
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index f43cbec8..412ae5a4 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -67,8 +67,8 @@ OSTACK_META = {
 CONTENT_0 = 'This is contents of /etc/foo.cfg\n'
 CONTENT_1 = '# this is /etc/bar/bar.cfg\n'
 OS_FILES = {
-    'openstack/2012-08-10/meta_data.json': json.dumps(OSTACK_META),
-    'openstack/2012-08-10/user_data': USER_DATA,
+    'openstack/latest/meta_data.json': json.dumps(OSTACK_META),
+    'openstack/latest/user_data': USER_DATA,
     'openstack/content/0000': CONTENT_0,
     'openstack/content/0001': CONTENT_1,
     'openstack/latest/meta_data.json': json.dumps(OSTACK_META),
@@ -78,6 +78,9 @@ OS_FILES = {
 EC2_FILES = {
     'latest/user-data': USER_DATA,
 }
+EC2_VERSIONS = [
+    'lastest',
+]
 
 
 def _register_uris(version, ec2_files, ec2_meta, os_files):
@@ -85,6 +88,9 @@ def _register_uris(version, ec2_files, ec2_meta, os_files):
     same data returned by the openstack metadata service (and ec2 service)."""
 
     def match_ec2_url(uri, headers):
+        path = uri.path.strip("/")
+        if len(path) == 0:
+            return (200, headers, "\n".join(EC2_VERSIONS))
         path = uri.path.lstrip("/")
         if path in ec2_files:
             return (200, headers, ec2_files.get(path))
@@ -110,11 +116,20 @@ def _register_uris(version, ec2_files, ec2_meta, os_files):
                 return (200, headers, str(value))
         return (404, headers, '')
 
-    def get_request_callback(method, uri, headers):
-        uri = urlparse(uri)
+    def match_os_uri(uri, headers):
+        path = uri.path.strip("/")
+        if path == 'openstack':
+            return (200, headers, "\n".join([openstack.OS_LATEST]))
         path = uri.path.lstrip("/")
         if path in os_files:
             return (200, headers, os_files.get(path))
+        return (404, headers, '')
+
+    def get_request_callback(method, uri, headers):
+        uri = urlparse(uri)
+        path = uri.path.lstrip("/").split("/")
+        if path[0] == 'openstack':
+            return match_os_uri(uri, headers)
         return match_ec2_url(uri, headers)
 
     hp.register_uri(hp.GET, re.compile(r'http://169.254.169.254/.*'),
-- 
cgit v1.2.3


From 984d36fb8c5feb82c31121ffd5d6a72b4f593499 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Wed, 27 Aug 2014 12:56:50 -0700
Subject: Fix retry cb to reflect what used to exist

---
 cloudinit/sources/helpers/openstack.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 2d0fc70e..025a2404 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -21,7 +21,6 @@
 import abc
 import base64
 import copy
-import httplib
 import os
 
 from cloudinit import ec2_utils
@@ -442,16 +441,21 @@ class MetadataReader(BaseReader):
 
     def _path_read(self, path):
 
-        def should_retry(_request_args, cause):
-            if cause.code == httplib.NOT_FOUND:
-                return False
+        def should_retry_cb(_request_args, cause):
+            try:
+                code = int(cause.code)
+                if code >= 400:
+                    return False
+            except (TypeError, ValueError):
+                # Older versions of requests didn't have a code.
+                pass
             return True
 
         response = url_helper.readurl(path,
                                       retries=self.retries,
                                       ssl_details=self.ssl_details,
                                       timeout=self.timeout,
-                                      exception_cb=should_retry)
+                                      exception_cb=should_retry_cb)
         return response.contents
 
     def _path_join(self, base, *add_ons):
-- 
cgit v1.2.3


From 419e0caab7e005827485460372c7f0d54ac0e9c9 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 27 Aug 2014 17:03:43 -0400
Subject: no functional changes, but some minor changes.

---
 cloudinit/sources/helpers/openstack.py            | 75 ++++++++++-------------
 tests/unittests/test_datasource/test_openstack.py |  2 +-
 2 files changed, 34 insertions(+), 43 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 025a2404..3c6bb6aa 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -162,21 +162,25 @@ class BaseReader(object):
         pass
 
     def _find_working_version(self, version):
+        try:
+            versions_available = self._fetch_available_versions(self)
+        except Exception as e:
+            LOG.warn("Unable to read openstack versions from %s due to: %s",
+                     self.base_path, e)
+            versions_available = []
+
         search_versions = [version] + list(OS_VERSIONS)
-        available_versions = self._fetch_available_versions()
+        selected_version = OS_LATEST
         for potential_version in search_versions:
-            if not potential_version:
+            if potential_version not in versions_available:
                 continue
-            if potential_version not in available_versions:
-                continue
-            if potential_version != version:
-                LOG.debug("Version '%s' not available, attempting to use"
-                          " version '%s' instead", version,
-                          potential_version)
-                return potential_version
-        LOG.debug("Version '%s' not available, attempting to use '%s'"
-                  " instead", version, OS_LATEST)
-        return OS_LATEST
+            selected_version = potential_version
+            break
+
+        if selected_version != version:
+            LOG.warn("Version '%s' not available, attempting to use"
+                     " version '%s' instead", version, selected_version)
+        return selected_version
 
     def _read_content_path(self, item):
         path = item.get('content_path', '').lstrip("/")
@@ -317,20 +321,12 @@ class ConfigDriveReader(BaseReader):
         return util.load_file(path)
 
     def _fetch_available_versions(self):
-        if self._versions is not None:
-            return self._versions
-        else:
-            versions_available = []
+        if self._versions is None:
             path = self._path_join(self.base_path, 'openstack')
-            try:
-                for child in os.listdir(path):
-                    child_path = os.path.join(path, child)
-                    if os.path.isdir(child_path):
-                        versions_available.append(child)
-            except (OSError, IOError):
-                pass
-            self._versions = tuple(versions_available)
-            return self._versions
+            found = [d for d in os.listdir(path)
+                     if os.path.isdir(os.path.join(path))]
+            self._versions = tuple(found)
+        return self._versions
 
     def _read_ec2_metadata(self):
         path = self._path_join(self.base_path,
@@ -420,24 +416,19 @@ class MetadataReader(BaseReader):
         self._versions = None
 
     def _fetch_available_versions(self):
+        # <baseurl>/openstack/ returns a newline separated list of versions
         if self._versions is not None:
-            return self._versions
-        else:
-            path = self._path_join(self.base_path, "openstack")
-            versions_available = []
-            try:
-                versions = self._path_read(path)
-            except IOError as e:
-                LOG.warn("Unable to read openstack versions from %s due"
-                         " to: %s", path, e)
-            else:
-                for line in versions.splitlines():
-                    line = line.strip()
-                    if not line:
-                        continue
-                    versions_available.append(line)
-            self._versions = tuple(versions_available)
-            return self._versions
+            return self.os_versions
+        found = []
+        content = self._path_read(version_path)
+        for line in content.splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            found.append(line)
+        self._versions = tuple(found)
+        return self._versions
+
 
     def _path_read(self, path):
 
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index 412ae5a4..530fba20 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -79,7 +79,7 @@ EC2_FILES = {
     'latest/user-data': USER_DATA,
 }
 EC2_VERSIONS = [
-    'lastest',
+    'latest',
 ]
 
 
-- 
cgit v1.2.3


From 6f8bd54650e78bd383b29a59bee95a887ff13c81 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Thu, 4 Sep 2014 10:33:37 -0700
Subject: Reduce logging levels and fix broken call to
 _fetch_available_versions

---
 cloudinit/sources/helpers/openstack.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 3c6bb6aa..61c61570 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -163,10 +163,10 @@ class BaseReader(object):
 
     def _find_working_version(self, version):
         try:
-            versions_available = self._fetch_available_versions(self)
+            versions_available = self._fetch_available_versions()
         except Exception as e:
-            LOG.warn("Unable to read openstack versions from %s due to: %s",
-                     self.base_path, e)
+            LOG.debug("Unable to read openstack versions from %s due to: %s",
+                      self.base_path, e)
             versions_available = []
 
         search_versions = [version] + list(OS_VERSIONS)
@@ -178,8 +178,8 @@ class BaseReader(object):
             break
 
         if selected_version != version:
-            LOG.warn("Version '%s' not available, attempting to use"
-                     " version '%s' instead", version, selected_version)
+            LOG.debug("Version '%s' not available, attempting to use"
+                      " version '%s' instead", version, selected_version)
         return selected_version
 
     def _read_content_path(self, item):
@@ -239,7 +239,8 @@ class BaseReader(object):
                     LOG.debug("Failed reading optional path %s due"
                               " to: %s", path, e)
                 else:
-                    LOG.exception("Failed reading mandatory path %s", path)
+                    LOG.debug("Failed reading mandatory path %s due"
+                              " to: %s", path, e)
             else:
                 found = True
             if required and not found:
-- 
cgit v1.2.3


From 1b975098ea2cc82a168203c720ef936db4864467 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 10 Sep 2014 14:39:59 -0400
Subject: log as warn if things are obviously wrong

If something is broken as in a built in config, or code
just broken, then logging warning during search for metadata
is ok.
---
 cloudinit/sources/DataSourceOpenStack.py | 16 ++++++++--------
 cloudinit/sources/helpers/openstack.py   |  6 ++++--
 2 files changed, 12 insertions(+), 10 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index 221759e1..0970d07b 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -64,13 +64,13 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
 
         try:
             max_wait = int(self.ds_cfg.get("max_wait", max_wait))
-        except Exception as e:
-            LOG.debug("Failed to get max wait. using %s: %s", max_wait, e)
+        except Exception:
+            util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
 
         try:
             timeout = max(0, int(self.ds_cfg.get("timeout", timeout)))
-        except Exception as e:
-            LOG.debug("Failed to get timeout, using %s: %s", timeout, e)
+        except Exception:
+            util.logexc(LOG, "Failed to get timeout, using %s", timeout)
         return (max_wait, timeout)
 
     def wait_for_metadata_service(self):
@@ -82,7 +82,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
         if len(filtered):
             urls = filtered
         else:
-            LOG.debug("Empty metadata url list! using default list")
+            LOG.warn("Empty metadata url list! using default list")
             urls = [DEF_MD_URL]
 
         md_urls = []
@@ -123,9 +123,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
                                             'version': openstack.OS_HAVANA})
         except openstack.NonReadable:
             return False
-        except (openstack.BrokenMetadata, IOError) as e:
-            LOG.debug("Broken metadata address %s: %s",
-                      self.metadata_address, e)
+        except (openstack.BrokenMetadata, IOError):
+            util.logexc(LOG, "Broken metadata address %s",
+                        self.metadata_address)
             return False
 
         user_dsmode = results.get('dsmode', None)
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 7b27621c..0c6b9b5a 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -178,8 +178,10 @@ class BaseReader(object):
             break
 
         if selected_version != version:
-            LOG.debug("Version '%s' not available, attempting to use"
-                      " version '%s' instead", version, selected_version)
+            LOG.warn("Version '%s' not available, attempting to use "
+                     "version '%s' instead", version, selected_version)
+        else:
+            LOG.debug("Version '%s' was available.", version)
         return selected_version
 
     def _read_content_path(self, item):
-- 
cgit v1.2.3


From 64f4a63c174a4f7c21e6e372927906099103d64c Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 10 Sep 2014 14:42:33 -0400
Subject: fix a bug, use a list instead of tuple for _versions

using tuple for _versions was just not necessary.
fix reference to undefined os_versions.
---
 cloudinit/sources/helpers/openstack.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 0c6b9b5a..3c3de7e4 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -328,7 +328,7 @@ class ConfigDriveReader(BaseReader):
             path = self._path_join(self.base_path, 'openstack')
             found = [d for d in os.listdir(path)
                      if os.path.isdir(os.path.join(path))]
-            self._versions = tuple(found)
+            self._versions = found
         return self._versions
 
     def _read_ec2_metadata(self):
@@ -421,7 +421,7 @@ class MetadataReader(BaseReader):
     def _fetch_available_versions(self):
         # <baseurl>/openstack/ returns a newline separated list of versions
         if self._versions is not None:
-            return self.os_versions
+            return self._versions
         found = []
         version_path = self._path_join(self.base_path, "openstack")
         content = self._path_read(version_path)
@@ -430,7 +430,7 @@ class MetadataReader(BaseReader):
             if not line:
                 continue
             found.append(line)
-        self._versions = tuple(found)
+        self._versions = found
         return self._versions
 
 
-- 
cgit v1.2.3


From c4b09a27239bc88bcf6b4ec536410bcc02cdf11c Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 10 Sep 2014 15:37:09 -0400
Subject: make BaseReader select latest supported version

---
 cloudinit/sources/DataSourceConfigDrive.py | 16 ++++++----------
 cloudinit/sources/DataSourceOpenStack.py   |  3 +--
 cloudinit/sources/helpers/openstack.py     | 19 +++++++++++++------
 3 files changed, 20 insertions(+), 18 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 83cc6b25..b8c16361 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -167,17 +167,13 @@ def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None):
     return "net"
 
 
-def read_config_drive(source_dir):
-    excps = []
-    finders = []
+def read_config_drive(source_dir, version=None):
     reader = openstack.ConfigDriveReader(source_dir)
-
-    # openstack.OS_VERSIONS is stored in chronological order, so to check the
-    # newest first, use reversed()
-    for version in reversed(openstack.OS_VERSIONS):
-        finders.append((reader.read_v2, [], {'version': version}))
-    finders.append((reader.read_v1, [], {}))
-
+    finders = [
+        (reader.read_v2, [], {'version': version}),
+        (reader.read_v1, [], {}),
+    ]
+    excps = []
     for (functor, args, kwargs) in finders:
         try:
             return functor(*args, **kwargs)
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index 0970d07b..ce8e8364 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -119,8 +119,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
                                     'Crawl of openstack metadata service',
                                     read_metadata_service,
                                     args=[self.metadata_address],
-                                    kwargs={'ssl_details': self.ssl_details,
-                                            'version': openstack.OS_HAVANA})
+                                    kwargs={'ssl_details': self.ssl_details})
         except openstack.NonReadable:
             return False
         except (openstack.BrokenMetadata, IOError):
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index ed91a55c..ed9cdbaa 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -48,7 +48,7 @@ OS_LATEST = 'latest'
 OS_FOLSOM = '2012-08-10'
 OS_GRIZZLY = '2013-04-04'
 OS_HAVANA = '2013-10-17'
-# keep this in chronological order by time: add new entries to the end
+# keep this in chronological order. new supported versions go at the end.
 OS_VERSIONS = (
     OS_FOLSOM,
     OS_GRIZZLY,
@@ -162,7 +162,7 @@ class BaseReader(object):
     def _read_ec2_metadata(self):
         pass
 
-    def _find_working_version(self, version):
+    def _find_working_version(self, version=None):
         try:
             versions_available = self._fetch_available_versions()
         except Exception as e:
@@ -170,7 +170,14 @@ class BaseReader(object):
                       self.base_path, e)
             versions_available = []
 
-        search_versions = [version] + list(OS_VERSIONS)
+        # openstack.OS_VERSIONS is stored in chronological order, so
+        # reverse it to check newest first.
+        supported = [v for v in reversed(list(OS_VERSIONS))]
+        if version is not None:
+            search_versions = [version] + supported
+        else:
+            search_versions = supported
+
         selected_version = OS_LATEST
         for potential_version in search_versions:
             if potential_version not in versions_available:
@@ -178,11 +185,12 @@ class BaseReader(object):
             selected_version = potential_version
             break
 
-        if selected_version != version:
+        if version is not None and selected_version != version:
             LOG.warn("Version '%s' not available, attempting to use "
                      "version '%s' instead", version, selected_version)
         else:
-            LOG.debug("Version '%s' was available.", version)
+            LOG.debug("Selected version '%s' from %s", version,
+                      versions_available)
         return selected_version
 
     def _read_content_path(self, item):
@@ -434,7 +442,6 @@ class MetadataReader(BaseReader):
         self._versions = found
         return self._versions
 
-
     def _path_read(self, path):
 
         def should_retry_cb(_request_args, cause):
-- 
cgit v1.2.3


From c548fdf3201519c1c30c815ba9feec643b87e0bf Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 10 Sep 2014 15:40:30 -0400
Subject: fix log message

---
 cloudinit/sources/helpers/openstack.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index ed9cdbaa..3d903a3c 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -189,7 +189,7 @@ class BaseReader(object):
             LOG.warn("Version '%s' not available, attempting to use "
                      "version '%s' instead", version, selected_version)
         else:
-            LOG.debug("Selected version '%s' from %s", version,
+            LOG.debug("Selected version '%s' from %s", selected_version,
                       versions_available)
         return selected_version
 
-- 
cgit v1.2.3


From 44113217719ebee756325b40a5d14045ba8f3a3a Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 10 Sep 2014 16:00:00 -0400
Subject: drop version= from readers

instead of taking a version that they should look for,
the readers now just select the highest supported version.

definitely a use case later for having version= but nothing
is using it now.
---
 cloudinit/sources/DataSourceConfigDrive.py        |  4 ++--
 cloudinit/sources/DataSourceOpenStack.py          |  4 ++--
 cloudinit/sources/helpers/openstack.py            | 23 +++++++----------------
 tests/unittests/test_datasource/test_openstack.py | 16 ++++++++--------
 4 files changed, 19 insertions(+), 28 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index b8c16361..a27d07fb 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -167,10 +167,10 @@ def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None):
     return "net"
 
 
-def read_config_drive(source_dir, version=None):
+def read_config_drive(source_dir):
     reader = openstack.ConfigDriveReader(source_dir)
     finders = [
-        (reader.read_v2, [], {'version': version}),
+        (reader.read_v2, [], {}),
         (reader.read_v1, [], {}),
     ]
     excps = []
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index ce8e8364..466de8f4 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -153,9 +153,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
         return True
 
 
-def read_metadata_service(base_url, version=None, ssl_details=None):
+def read_metadata_service(base_url, ssl_details=None):
     reader = openstack.MetadataReader(base_url, ssl_details=ssl_details)
-    return reader.read_v2(version=version)
+    return reader.read_v2()
 
 
 # Used to match classes to dependencies
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 3d903a3c..a7dd05df 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -162,7 +162,7 @@ class BaseReader(object):
     def _read_ec2_metadata(self):
         pass
 
-    def _find_working_version(self, version=None):
+    def _find_working_version(self):
         try:
             versions_available = self._fetch_available_versions()
         except Exception as e:
@@ -173,24 +173,16 @@ class BaseReader(object):
         # openstack.OS_VERSIONS is stored in chronological order, so
         # reverse it to check newest first.
         supported = [v for v in reversed(list(OS_VERSIONS))]
-        if version is not None:
-            search_versions = [version] + supported
-        else:
-            search_versions = supported
-
         selected_version = OS_LATEST
-        for potential_version in search_versions:
+
+        for potential_version in supported:
             if potential_version not in versions_available:
                 continue
             selected_version = potential_version
             break
 
-        if version is not None and selected_version != version:
-            LOG.warn("Version '%s' not available, attempting to use "
-                     "version '%s' instead", version, selected_version)
-        else:
-            LOG.debug("Selected version '%s' from %s", selected_version,
-                      versions_available)
+        LOG.debug("Selected version '%s' from %s", selected_version,
+                  versions_available)
         return selected_version
 
     def _read_content_path(self, item):
@@ -202,7 +194,7 @@ class BaseReader(object):
         path = self._path_join(self.base_path, "openstack", *path_pieces)
         return self._path_read(path)
 
-    def read_v2(self, version=None):
+    def read_v2(self):
         """Reads a version 2 formatted location.
 
         Return a dict with metadata, userdata, ec2-metadata, dsmode,
@@ -233,12 +225,11 @@ class BaseReader(object):
             )
             return files
 
-        version = self._find_working_version(version)
         results = {
             'userdata': '',
             'version': 2,
         }
-        data = datafiles(version)
+        data = datafiles(self._find_working_version())
         for (name, (path, required, translator)) in data.iteritems():
             path = self._path_join(self.base_path, path)
             data = None
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index 530fba20..6809823e 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -142,7 +142,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
     @hp.activate
     def test_successful(self):
         _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES)
-        f = ds.read_metadata_service(BASE_URL, version=self.VERSION)
+        f = ds.read_metadata_service(BASE_URL)
         self.assertEquals(VENDOR_DATA, f.get('vendordata'))
         self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg'])
         self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg'])
@@ -164,7 +164,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
     @hp.activate
     def test_no_ec2(self):
         _register_uris(self.VERSION, {}, {}, OS_FILES)
-        f = ds.read_metadata_service(BASE_URL, version=self.VERSION)
+        f = ds.read_metadata_service(BASE_URL)
         self.assertEquals(VENDOR_DATA, f.get('vendordata'))
         self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg'])
         self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg'])
@@ -180,7 +180,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
                 os_files.pop(k, None)
         _register_uris(self.VERSION, {}, {}, os_files)
         self.assertRaises(openstack.NonReadable, ds.read_metadata_service,
-                          BASE_URL, version=self.VERSION)
+                          BASE_URL)
 
     @hp.activate
     def test_bad_uuid(self):
@@ -192,7 +192,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
                 os_files[k] = json.dumps(os_meta)
         _register_uris(self.VERSION, {}, {}, os_files)
         self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service,
-                          BASE_URL, version=self.VERSION)
+                          BASE_URL)
 
     @hp.activate
     def test_userdata_empty(self):
@@ -201,7 +201,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('user_data'):
                 os_files.pop(k, None)
         _register_uris(self.VERSION, {}, {}, os_files)
-        f = ds.read_metadata_service(BASE_URL, version=self.VERSION)
+        f = ds.read_metadata_service(BASE_URL)
         self.assertEquals(VENDOR_DATA, f.get('vendordata'))
         self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg'])
         self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg'])
@@ -214,7 +214,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             if k.endswith('vendor_data.json'):
                 os_files.pop(k, None)
         _register_uris(self.VERSION, {}, {}, os_files)
-        f = ds.read_metadata_service(BASE_URL, version=self.VERSION)
+        f = ds.read_metadata_service(BASE_URL)
         self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg'])
         self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg'])
         self.assertFalse(f.get('vendordata'))
@@ -227,7 +227,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
                 os_files[k] = '{'  # some invalid json
         _register_uris(self.VERSION, {}, {}, os_files)
         self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service,
-                          BASE_URL, version=self.VERSION)
+                          BASE_URL)
 
     @hp.activate
     def test_metadata_invalid(self):
@@ -237,7 +237,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
                 os_files[k] = '{'  # some invalid json
         _register_uris(self.VERSION, {}, {}, os_files)
         self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service,
-                          BASE_URL, version=self.VERSION)
+                          BASE_URL)
 
     @hp.activate
     def test_datasource(self):
-- 
cgit v1.2.3


From 932c073db79e667623d27174c55e5b16ea439578 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 10 Sep 2014 21:17:40 -0400
Subject: Openstack: Vendor data cleanup

For now, this vendor data handling is just added to openstack.
However, in an effort to allow sanely handling of multi-part vendor-data
that is namespaced, we add openstack.convert_vendordata_json .

That basically takes whatever was loaded from vendordata and takes
the 'cloud-init' key if it is a dict.  This way the author can
namespace cloud-init, basically telling it to ignore everything else.
---
 cloudinit/sources/DataSourceConfigDrive.py        | 13 +++++----
 cloudinit/sources/DataSourceOpenStack.py          | 12 ++++----
 cloudinit/sources/helpers/openstack.py            | 25 ++++++++++++++++
 tests/unittests/test_datasource/test_openstack.py | 35 ++++++++++++++++++++++-
 4 files changed, 72 insertions(+), 13 deletions(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index a27d07fb..4e5d90de 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -126,12 +126,13 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
         self.version = results['version']
         self.files.update(results.get('files', {}))
 
-        # If there is no vendordata, set vd to an empty dict instead of None
-        vd = results.get('vendordata', {})
-        # if vendordata includes 'cloud-init', then read that explicitly
-        # for cloud-init (for namespacing).
-        if 'cloud-init' in vd:
-            self.vendordata_raw = vd['cloud-init']
+        vd = results.get('vendordata')
+        self.vendordata_pure = vd
+        try:
+            self.vendordata_raw = openstack.convert_vendordata_json(vd)
+        except ValueError as e:
+            LOG.warn("Invalid content in vendor-data: %s", e)
+            self.vendordata_raw = None
 
         return True
 
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index 765137c6..469c2e2a 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -140,13 +140,13 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
         self.version = results['version']
         self.files.update(results.get('files', {}))
 
-        # if vendordata includes 'cloud-init', then read that explicitly
-        # for cloud-init (for namespacing).
         vd = results.get('vendordata')
-        if isinstance(vd, dict) and 'cloud-init' in vd:
-            self.vendordata_raw = vd['cloud-init']
-        else:
-            self.vendordata_raw = vd
+        self.vendordata_pure = vd
+        try:
+            self.vendordata_raw = openstack.convert_vendordata_json(vd)
+        except ValueError as e:
+            LOG.warn("Invalid content in vendor-data: %s", e)
+            self.vendordata_raw = None
 
         return True
 
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index a7dd05df..9718b0be 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -459,3 +459,28 @@ class MetadataReader(BaseReader):
         return ec2_utils.get_instance_metadata(ssl_details=self.ssl_details,
                                                timeout=self.timeout,
                                                retries=self.retries)
+
+
+def convert_vendordata_json(data, recurse=True):
+    """ data: a loaded json *object* (strings, arrays, dicts).
+    return something suitable for cloudinit vendordata_raw.
+
+    if data is:
+       None: return None
+       string: return string
+       list: return data
+             the list is then processed in UserDataProcessor
+       dict: return convert_vendordata_json(data.get('cloud-init'))
+    """
+    if not data:
+        return None
+    if isinstance(data, (str, unicode, basestring)):
+        return data
+    if isinstance(data, list):
+        return copy.deepcopy(data)
+    if isinstance(data, dict):
+        if recurse is True:
+            return convert_vendordata_json(data.get('cloud-init'),
+                                           recurse=False)
+        raise ValueError("vendordata['cloud-init'] cannot be dict")
+    raise ValueError("Unknown data type for vendordata: %s" % type(data))
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index 6809823e..7b4e651a 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -19,6 +19,7 @@
 import copy
 import json
 import re
+import unittest
 
 from StringIO import StringIO
 
@@ -256,7 +257,8 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         self.assertEquals(EC2_META, ds_os.ec2_metadata)
         self.assertEquals(USER_DATA, ds_os.userdata_raw)
         self.assertEquals(2, len(ds_os.files))
-        self.assertEquals(VENDOR_DATA, ds_os.vendordata_raw)
+        self.assertEquals(VENDOR_DATA, ds_os.vendordata_pure)
+        self.assertEquals(ds_os.vendordata_raw, None)
 
     @hp.activate
     def test_bad_datasource_meta(self):
@@ -314,3 +316,34 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         found = ds_os.get_data()
         self.assertFalse(found)
         self.assertIsNone(ds_os.version)
+
+
+class TestVendorDataLoading(unittest.TestCase):
+    def cvj(self, data):
+        return openstack.convert_vendordata_json(data)
+
+    def test_vd_load_none(self):
+        # non-existant vendor-data should return none
+        self.assertIsNone(self.cvj(None))
+
+    def test_vd_load_string(self):
+        self.assertEqual(self.cvj("foobar"), "foobar")
+
+    def test_vd_load_list(self):
+        data = [{'foo': 'bar'}, 'mystring', list(['another', 'list'])]
+        self.assertEqual(self.cvj(data), data)
+
+    def test_vd_load_dict_no_ci(self):
+        self.assertEqual(self.cvj({'foo': 'bar'}), None)
+
+    def test_vd_load_dict_ci_dict(self):
+        self.assertRaises(ValueError, self.cvj,
+                          {'foo': 'bar', 'cloud-init': {'x': 1}})
+
+    def test_vd_load_dict_ci_string(self):
+        data = {'foo': 'bar', 'cloud-init': 'VENDOR_DATA'}
+        self.assertEqual(self.cvj(data), data['cloud-init'])
+
+    def test_vd_load_dict_ci_list(self):
+        data = {'foo': 'bar', 'cloud-init': ['VD_1', 'VD_2']}
+        self.assertEqual(self.cvj(data), data['cloud-init'])
-- 
cgit v1.2.3


From b2ba18dcc48b4721b097dc108f5a5eceba6b87ab Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Thu, 11 Sep 2014 10:41:10 -0400
Subject: when loading vendordata allow it to be string or list

---
 cloudinit/sources/helpers/openstack.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'cloudinit/sources/helpers')

diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 9718b0be..b7e19314 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -21,6 +21,7 @@
 import abc
 import base64
 import copy
+import functools
 import os
 
 from cloudinit import ec2_utils
@@ -203,6 +204,9 @@ class BaseReader(object):
         If not a valid location, raise a NonReadable exception.
         """
 
+        load_json_anytype = functools.partial(
+            util.load_json, root_types=(dict, basestring, list))
+
         def datafiles(version):
             files = {}
             files['metadata'] = (
@@ -221,7 +225,7 @@ class BaseReader(object):
             files['vendordata'] = (
                 self._path_join("openstack", version, 'vendor_data.json'),
                 False,
-                util.load_json,
+                load_json_anytype,
             )
             return files
 
-- 
cgit v1.2.3