merge from trunk

3 files changed, 78 insertions, 55 deletions

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 34557f8b..e2469f6e 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -28,7 +28,7 @@ import base64
 import os
 import pwd
 import re
-import string  # pylint: disable=W0402
+import string
 
 from cloudinit import log as logging
 from cloudinit import sources
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 7d52a2e6..7c7ef9ab 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -66,7 +66,7 @@ class DataSource(object):
             name = name[0:-3]
 
         self.ds_cfg = util.get_cfg_by_path(self.sys_cfg,
-                                          ("datasource", name), {})
+                                           ("datasource", name), {})
         if not ud_proc:
             self.ud_proc = ud.UserDataProcessor(self.paths)
         else:
@@ -166,7 +166,7 @@ class DataSource(object):
         defhost = "localhost"
         domain = defdomain
 
-        if self.metadata or 'local-hostname' not in self.metadata:
+        if not self.metadata or 'local-hostname' not in self.metadata:
             # this is somewhat questionable really.
             # the cloud datasource was asked for a hostname
             # and didn't have one. raising error might be more appropriate
@@ -272,9 +272,9 @@ def list_sources(cfg_list, depends, pkg_list):
     for ds_name in cfg_list:
         if not ds_name.startswith(DS_PREFIX):
             ds_name = '%s%s' % (DS_PREFIX, ds_name)
-        m_locs = importer.find_module(ds_name,
-                                      pkg_list,
-                                      ['get_datasource_list'])
+        m_locs, _looked_locs = importer.find_module(ds_name,
+                                                    pkg_list,
+                                                    ['get_datasource_list'])
         for m_loc in m_locs:
             mod = importer.import_module(m_loc)
             lister = getattr(mod, "get_datasource_list")
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index ed102c4c..ed91a55c 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -151,17 +151,40 @@ class BaseReader(object):
         pass
 
     @abc.abstractmethod
-    def _path_exists(self, path):
+    def _path_read(self, path):
         pass
 
     @abc.abstractmethod
-    def _path_read(self, path):
+    def _fetch_available_versions(self):
         pass
 
     @abc.abstractmethod
     def _read_ec2_metadata(self):
         pass
 
+    def _find_working_version(self, version):
+        try:
+            versions_available = self._fetch_available_versions()
+        except Exception as e:
+            LOG.debug("Unable to read openstack versions from %s due to: %s",
+                      self.base_path, e)
+            versions_available = []
+
+        search_versions = [version] + list(OS_VERSIONS)
+        selected_version = OS_LATEST
+        for potential_version in search_versions:
+            if potential_version not in versions_available:
+                continue
+            selected_version = potential_version
+            break
+
+        if selected_version != version:
+            LOG.warn("Version '%s' not available, attempting to use "
+                     "version '%s' instead", version, selected_version)
+        else:
+            LOG.debug("Version '%s' was available.", version)
+        return selected_version
+
     def _read_content_path(self, item):
         path = item.get('content_path', '').lstrip("/")
         path_pieces = path.split("/")
@@ -171,23 +194,6 @@ class BaseReader(object):
         path = self._path_join(self.base_path, "openstack", *path_pieces)
         return self._path_read(path)
 
-    def _find_working_version(self, version):
-        search_versions = [version] + list(OS_VERSIONS)
-        for potential_version in search_versions:
-            if not potential_version:
-                continue
-            path = self._path_join(self.base_path, "openstack",
-                                   potential_version)
-            if self._path_exists(path):
-                if potential_version != version:
-                    LOG.debug("Version '%s' not available, attempting to use"
-                              " version '%s' instead", version,
-                              potential_version)
-                return potential_version
-        LOG.debug("Version '%s' not available, attempting to use '%s'"
-                  " instead", version, OS_LATEST)
-        return OS_LATEST
-
     def read_v2(self, version=None):
         """Reads a version 2 formatted location.
 
@@ -229,15 +235,19 @@ class BaseReader(object):
             path = self._path_join(self.base_path, path)
             data = None
             found = False
-            if self._path_exists(path):
-                try:
-                    data = self._path_read(path)
-                except IOError:
-                    raise NonReadable("Failed to read: %s" % path)
-                found = True
+            try:
+                data = self._path_read(path)
+            except IOError as e:
+                if not required:
+                    LOG.debug("Failed reading optional path %s due"
+                              " to: %s", path, e)
+                else:
+                    LOG.debug("Failed reading mandatory path %s due"
+                              " to: %s", path, e)
             else:
-                if required:
-                    raise NonReadable("Missing mandatory path: %s" % path)
+                found = True
+            if required and not found:
+                raise NonReadable("Missing mandatory path: %s" % path)
             if found and translator:
                 try:
                     data = translator(data)
@@ -305,21 +315,27 @@ class BaseReader(object):
 class ConfigDriveReader(BaseReader):
     def __init__(self, base_path):
         super(ConfigDriveReader, self).__init__(base_path)
+        self._versions = None
 
     def _path_join(self, base, *add_ons):
         components = [base] + list(add_ons)
         return os.path.join(*components)
 
-    def _path_exists(self, path):
-        return os.path.exists(path)
-
     def _path_read(self, path):
         return util.load_file(path)
 
+    def _fetch_available_versions(self):
+        if self._versions is None:
+            path = self._path_join(self.base_path, 'openstack')
+            found = [d for d in os.listdir(path)
+                     if os.path.isdir(os.path.join(path))]
+            self._versions = found
+        return self._versions
+
     def _read_ec2_metadata(self):
         path = self._path_join(self.base_path,
                                'ec2', 'latest', 'meta-data.json')
-        if not self._path_exists(path):
+        if not os.path.exists(path):
             return {}
         else:
             try:
@@ -339,7 +355,7 @@ class ConfigDriveReader(BaseReader):
         found = {}
         for name in FILES_V1.keys():
             path = self._path_join(self.base_path, name)
-            if self._path_exists(path):
+            if os.path.exists(path):
                 found[name] = path
         if len(found) == 0:
             raise NonReadable("%s: no files found" % (self.base_path))
@@ -401,17 +417,27 @@ class MetadataReader(BaseReader):
         self.ssl_details = ssl_details
         self.timeout = float(timeout)
         self.retries = int(retries)
+        self._versions = None
+
+    def _fetch_available_versions(self):
+        # <baseurl>/openstack/ returns a newline separated list of versions
+        if self._versions is not None:
+            return self._versions
+        found = []
+        version_path = self._path_join(self.base_path, "openstack")
+        content = self._path_read(version_path)
+        for line in content.splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            found.append(line)
+        self._versions = found
+        return self._versions
 
-    def _path_read(self, path):
-        response = url_helper.readurl(path,
-                                      retries=self.retries,
-                                      ssl_details=self.ssl_details,
-                                      timeout=self.timeout)
-        return response.contents
 
-    def _path_exists(self, path):
+    def _path_read(self, path):
 
-        def should_retry_cb(request, cause):
+        def should_retry_cb(_request_args, cause):
             try:
                 code = int(cause.code)
                 if code >= 400:
@@ -421,15 +447,12 @@ class MetadataReader(BaseReader):
                 pass
             return True
 
-        try:
-            response = url_helper.readurl(path,
-                                          retries=self.retries,
-                                          ssl_details=self.ssl_details,
-                                          timeout=self.timeout,
-                                          exception_cb=should_retry_cb)
-            return response.ok()
-        except IOError:
-            return False
+        response = url_helper.readurl(path,
+                                      retries=self.retries,
+                                      ssl_details=self.ssl_details,
+                                      timeout=self.timeout,
+                                      exception_cb=should_retry_cb)
+        return response.contents
 
     def _path_join(self, base, *add_ons):
         return url_helper.combine_url(base, *add_ons)