From 5fb6482692cfffba5ba45102858b14ba3acc5bc7 Mon Sep 17 00:00:00 2001 From: Scott Moser <smoser@ubuntu.com> Date: Tue, 26 Aug 2014 15:53:41 -0400 Subject: further remove evidence of pylint. This just removes comments '# pylint:' things and other code remnents of pylint. --- cloudinit/sources/DataSourceOpenNebula.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 34557f8b..e2469f6e 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -28,7 +28,7 @@ import base64 import os import pwd import re -import string # pylint: disable=W0402 +import string from cloudinit import log as logging from cloudinit import sources -- cgit v1.2.3 From 5fc46193d3e65d0eaaaa45bcf41c5b35b4e80df7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow <harlowja@yahoo-inc.com> Date: Tue, 26 Aug 2014 13:41:46 -0700 Subject: Optimize away most of the path_exists checks --- cloudinit/sources/helpers/openstack.py | 101 +++++++++++++++++---------------- 1 file changed, 52 insertions(+), 49 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 0fac0335..361c9994 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -149,10 +149,6 @@ class BaseReader(object): def _path_join(self, base, *add_ons): pass - @abc.abstractmethod - def _path_exists(self, path): - pass - @abc.abstractmethod def _path_read(self, path): pass @@ -170,22 +166,9 @@ class BaseReader(object): path = self._path_join(self.base_path, "openstack", *path_pieces) return self._path_read(path) + @abc.abstractmethod def _find_working_version(self, version): - search_versions = [version] + list(OS_VERSIONS) - for potential_version in search_versions: - if not potential_version: - continue - path = self._path_join(self.base_path, "openstack", - potential_version) - if self._path_exists(path): - if potential_version != version: - LOG.debug("Version '%s' not available, attempting to use" - " version '%s' instead", version, - potential_version) - return potential_version - LOG.debug("Version '%s' not available, attempting to use '%s'" - " instead", version, OS_LATEST) - return OS_LATEST + pass def read_v2(self, version=None): """Reads a version 2 formatted location. @@ -228,15 +211,14 @@ class BaseReader(object): path = self._path_join(self.base_path, path) data = None found = False - if self._path_exists(path): - try: - data = self._path_read(path) - except IOError: - raise NonReadable("Failed to read: %s" % path) - found = True + try: + data = self._path_read(path) + except IOError: + pass else: - if required: - raise NonReadable("Missing mandatory path: %s" % path) + found = True + if required and not found: + raise NonReadable("Missing mandatory path: %s" % path) if found and translator: try: data = translator(data) @@ -315,6 +297,23 @@ class ConfigDriveReader(BaseReader): def _path_read(self, path): return util.load_file(path) + def _find_working_version(self, version): + search_versions = [version] + list(OS_VERSIONS) + for potential_version in search_versions: + if not potential_version: + continue + path = self._path_join(self.base_path, "openstack", + potential_version) + if self._path_exists(path): + if potential_version != version: + LOG.debug("Version '%s' not available, attempting to use" + " version '%s' instead", version, + potential_version) + return potential_version + LOG.debug("Version '%s' not available, attempting to use '%s'" + " instead", version, OS_LATEST) + return OS_LATEST + def _read_ec2_metadata(self): path = self._path_join(self.base_path, 'ec2', 'latest', 'meta-data.json') @@ -401,6 +400,32 @@ class MetadataReader(BaseReader): self.timeout = float(timeout) self.retries = int(retries) + def _find_working_version(self, version): + search_versions = [version] + list(OS_VERSIONS) + version_path = self._path_join(self.base_path, "openstack") + versions_available = [] + try: + versions = self._path_read(version_path) + except IOError: + pass + else: + for line in versions.splitlines(): + line = line.strip() + if not line: + continue + versions_available.append(line) + for potential_version in search_versions: + if potential_version not in versions_available: + continue + if potential_version != version: + LOG.debug("Version '%s' not available, attempting to use" + " version '%s' instead", version, + potential_version) + return potential_version + LOG.debug("Version '%s' not available, searched %s, attempting to" + " use '%s' instead", version, search_versions, OS_LATEST) + return OS_LATEST + def _path_read(self, path): response = url_helper.readurl(path, retries=self.retries, @@ -408,28 +433,6 @@ class MetadataReader(BaseReader): timeout=self.timeout) return response.contents - def _path_exists(self, path): - - def should_retry_cb(request, cause): - try: - code = int(cause.code) - if code >= 400: - return False - except (TypeError, ValueError): - # Older versions of requests didn't have a code. - pass - return True - - try: - response = url_helper.readurl(path, - retries=self.retries, - ssl_details=self.ssl_details, - timeout=self.timeout, - exception_cb=should_retry_cb) - return response.ok() - except IOError: - return False - def _path_join(self, base, *add_ons): return url_helper.combine_url(base, *add_ons) -- cgit v1.2.3 From cc9e3af6c95b3263a49d4590d9dd176bdc570c99 Mon Sep 17 00:00:00 2001 From: Joshua Harlow <harlowja@gmail.com> Date: Tue, 26 Aug 2014 23:02:58 -0700 Subject: Add logging around failure to read optional/mandatory paths --- cloudinit/sources/helpers/openstack.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 361c9994..e5a38de0 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -213,8 +213,12 @@ class BaseReader(object): found = False try: data = self._path_read(path) - except IOError: - pass + except IOError as e: + if not required: + LOG.debug("Failed reading optional path %s due" + " to: %s", path, e) + else: + LOG.exception("Failed reading mandatory path %s", path) else: found = True if required and not found: -- cgit v1.2.3 From 6a6d3a499c2327b03993bbaea2b9d0df5dc7eb64 Mon Sep 17 00:00:00 2001 From: Joshua Harlow <harlowja@gmail.com> Date: Tue, 26 Aug 2014 23:04:56 -0700 Subject: Just use os.path.exists directly --- cloudinit/sources/helpers/openstack.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index e5a38de0..418ab4d1 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -295,9 +295,6 @@ class ConfigDriveReader(BaseReader): components = [base] + list(add_ons) return os.path.join(*components) - def _path_exists(self, path): - return os.path.exists(path) - def _path_read(self, path): return util.load_file(path) @@ -308,7 +305,7 @@ class ConfigDriveReader(BaseReader): continue path = self._path_join(self.base_path, "openstack", potential_version) - if self._path_exists(path): + if os.path.exists(path): if potential_version != version: LOG.debug("Version '%s' not available, attempting to use" " version '%s' instead", version, @@ -321,7 +318,7 @@ class ConfigDriveReader(BaseReader): def _read_ec2_metadata(self): path = self._path_join(self.base_path, 'ec2', 'latest', 'meta-data.json') - if not self._path_exists(path): + if not os.path.exists(path): return {} else: try: @@ -341,7 +338,7 @@ class ConfigDriveReader(BaseReader): found = {} for name in FILES_V1.keys(): path = self._path_join(self.base_path, name) - if self._path_exists(path): + if os.path.exists(path): found[name] = path if len(found) == 0: raise NonReadable("%s: no files found" % (self.base_path)) -- cgit v1.2.3 From 9a7587f35f327b9f8cafce8687832e9e77c1afde Mon Sep 17 00:00:00 2001 From: Joshua Harlow <harlowja@gmail.com> Date: Tue, 26 Aug 2014 23:08:14 -0700 Subject: Log a warning when unable to fetch the openstack versions --- cloudinit/sources/helpers/openstack.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 418ab4d1..56986a94 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -407,8 +407,9 @@ class MetadataReader(BaseReader): versions_available = [] try: versions = self._path_read(version_path) - except IOError: - pass + except IOError as e: + LOG.warn("Unable to read openstack versions from %s due" + " to: %s", version_path, e) else: for line in versions.splitlines(): line = line.strip() -- cgit v1.2.3 From fb482ce4a36d9b4be75a8bf5b428189548a205d9 Mon Sep 17 00:00:00 2001 From: Joshua Harlow <harlowja@gmail.com> Date: Tue, 26 Aug 2014 23:10:50 -0700 Subject: Show the available versions in the debug log message --- cloudinit/sources/helpers/openstack.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 56986a94..3ceec837 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -424,8 +424,9 @@ class MetadataReader(BaseReader): " version '%s' instead", version, potential_version) return potential_version - LOG.debug("Version '%s' not available, searched %s, attempting to" - " use '%s' instead", version, search_versions, OS_LATEST) + LOG.debug("Version '%s' not available, searched for %s (with available" + " versions being %s), attempting to use '%s' instead", + version, search_versions, versions_available, OS_LATEST) return OS_LATEST def _path_read(self, path): -- cgit v1.2.3 From d701035265c765bc42cb3bc358f2bfd0b41f484b Mon Sep 17 00:00:00 2001 From: Joshua Harlow <harlowja@yahoo-inc.com> Date: Wed, 27 Aug 2014 12:30:23 -0700 Subject: Fixed more of the slowness around fetching and retrying --- cloudinit/sources/helpers/openstack.py | 112 +++++++++++++--------- tests/unittests/test_datasource/test_openstack.py | 23 ++++- 2 files changed, 84 insertions(+), 51 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 3ceec837..2d0fc70e 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -21,6 +21,7 @@ import abc import base64 import copy +import httplib import os from cloudinit import ec2_utils @@ -153,10 +154,31 @@ class BaseReader(object): def _path_read(self, path): pass + @abc.abstractmethod + def _fetch_available_versions(self): + pass + @abc.abstractmethod def _read_ec2_metadata(self): pass + def _find_working_version(self, version): + search_versions = [version] + list(OS_VERSIONS) + available_versions = self._fetch_available_versions() + for potential_version in search_versions: + if not potential_version: + continue + if potential_version not in available_versions: + continue + if potential_version != version: + LOG.debug("Version '%s' not available, attempting to use" + " version '%s' instead", version, + potential_version) + return potential_version + LOG.debug("Version '%s' not available, attempting to use '%s'" + " instead", version, OS_LATEST) + return OS_LATEST + def _read_content_path(self, item): path = item.get('content_path', '').lstrip("/") path_pieces = path.split("/") @@ -166,10 +188,6 @@ class BaseReader(object): path = self._path_join(self.base_path, "openstack", *path_pieces) return self._path_read(path) - @abc.abstractmethod - def _find_working_version(self, version): - pass - def read_v2(self, version=None): """Reads a version 2 formatted location. @@ -290,6 +308,7 @@ class BaseReader(object): class ConfigDriveReader(BaseReader): def __init__(self, base_path): super(ConfigDriveReader, self).__init__(base_path) + self._versions = None def _path_join(self, base, *add_ons): components = [base] + list(add_ons) @@ -298,22 +317,21 @@ class ConfigDriveReader(BaseReader): def _path_read(self, path): return util.load_file(path) - def _find_working_version(self, version): - search_versions = [version] + list(OS_VERSIONS) - for potential_version in search_versions: - if not potential_version: - continue - path = self._path_join(self.base_path, "openstack", - potential_version) - if os.path.exists(path): - if potential_version != version: - LOG.debug("Version '%s' not available, attempting to use" - " version '%s' instead", version, - potential_version) - return potential_version - LOG.debug("Version '%s' not available, attempting to use '%s'" - " instead", version, OS_LATEST) - return OS_LATEST + def _fetch_available_versions(self): + if self._versions is not None: + return self._versions + else: + versions_available = [] + path = self._path_join(self.base_path, 'openstack') + try: + for child in os.listdir(path): + child_path = os.path.join(path, child) + if os.path.isdir(child_path): + versions_available.append(child) + except (OSError, IOError): + pass + self._versions = tuple(versions_available) + return self._versions def _read_ec2_metadata(self): path = self._path_join(self.base_path, @@ -400,40 +418,40 @@ class MetadataReader(BaseReader): self.ssl_details = ssl_details self.timeout = float(timeout) self.retries = int(retries) + self._versions = None - def _find_working_version(self, version): - search_versions = [version] + list(OS_VERSIONS) - version_path = self._path_join(self.base_path, "openstack") - versions_available = [] - try: - versions = self._path_read(version_path) - except IOError as e: - LOG.warn("Unable to read openstack versions from %s due" - " to: %s", version_path, e) + def _fetch_available_versions(self): + if self._versions is not None: + return self._versions else: - for line in versions.splitlines(): - line = line.strip() - if not line: - continue - versions_available.append(line) - for potential_version in search_versions: - if potential_version not in versions_available: - continue - if potential_version != version: - LOG.debug("Version '%s' not available, attempting to use" - " version '%s' instead", version, - potential_version) - return potential_version - LOG.debug("Version '%s' not available, searched for %s (with available" - " versions being %s), attempting to use '%s' instead", - version, search_versions, versions_available, OS_LATEST) - return OS_LATEST + path = self._path_join(self.base_path, "openstack") + versions_available = [] + try: + versions = self._path_read(path) + except IOError as e: + LOG.warn("Unable to read openstack versions from %s due" + " to: %s", path, e) + else: + for line in versions.splitlines(): + line = line.strip() + if not line: + continue + versions_available.append(line) + self._versions = tuple(versions_available) + return self._versions def _path_read(self, path): + + def should_retry(_request_args, cause): + if cause.code == httplib.NOT_FOUND: + return False + return True + response = url_helper.readurl(path, retries=self.retries, ssl_details=self.ssl_details, - timeout=self.timeout) + timeout=self.timeout, + exception_cb=should_retry) return response.contents def _path_join(self, base, *add_ons): diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index f43cbec8..412ae5a4 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -67,8 +67,8 @@ OSTACK_META = { CONTENT_0 = 'This is contents of /etc/foo.cfg\n' CONTENT_1 = '# this is /etc/bar/bar.cfg\n' OS_FILES = { - 'openstack/2012-08-10/meta_data.json': json.dumps(OSTACK_META), - 'openstack/2012-08-10/user_data': USER_DATA, + 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), + 'openstack/latest/user_data': USER_DATA, 'openstack/content/0000': CONTENT_0, 'openstack/content/0001': CONTENT_1, 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), @@ -78,6 +78,9 @@ OS_FILES = { EC2_FILES = { 'latest/user-data': USER_DATA, } +EC2_VERSIONS = [ + 'lastest', +] def _register_uris(version, ec2_files, ec2_meta, os_files): @@ -85,6 +88,9 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): same data returned by the openstack metadata service (and ec2 service).""" def match_ec2_url(uri, headers): + path = uri.path.strip("/") + if len(path) == 0: + return (200, headers, "\n".join(EC2_VERSIONS)) path = uri.path.lstrip("/") if path in ec2_files: return (200, headers, ec2_files.get(path)) @@ -110,11 +116,20 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): return (200, headers, str(value)) return (404, headers, '') - def get_request_callback(method, uri, headers): - uri = urlparse(uri) + def match_os_uri(uri, headers): + path = uri.path.strip("/") + if path == 'openstack': + return (200, headers, "\n".join([openstack.OS_LATEST])) path = uri.path.lstrip("/") if path in os_files: return (200, headers, os_files.get(path)) + return (404, headers, '') + + def get_request_callback(method, uri, headers): + uri = urlparse(uri) + path = uri.path.lstrip("/").split("/") + if path[0] == 'openstack': + return match_os_uri(uri, headers) return match_ec2_url(uri, headers) hp.register_uri(hp.GET, re.compile(r'http://169.254.169.254/.*'), -- cgit v1.2.3 From 984d36fb8c5feb82c31121ffd5d6a72b4f593499 Mon Sep 17 00:00:00 2001 From: Joshua Harlow <harlowja@yahoo-inc.com> Date: Wed, 27 Aug 2014 12:56:50 -0700 Subject: Fix retry cb to reflect what used to exist --- cloudinit/sources/helpers/openstack.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 2d0fc70e..025a2404 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -21,7 +21,6 @@ import abc import base64 import copy -import httplib import os from cloudinit import ec2_utils @@ -442,16 +441,21 @@ class MetadataReader(BaseReader): def _path_read(self, path): - def should_retry(_request_args, cause): - if cause.code == httplib.NOT_FOUND: - return False + def should_retry_cb(_request_args, cause): + try: + code = int(cause.code) + if code >= 400: + return False + except (TypeError, ValueError): + # Older versions of requests didn't have a code. + pass return True response = url_helper.readurl(path, retries=self.retries, ssl_details=self.ssl_details, timeout=self.timeout, - exception_cb=should_retry) + exception_cb=should_retry_cb) return response.contents def _path_join(self, base, *add_ons): -- cgit v1.2.3 From 419e0caab7e005827485460372c7f0d54ac0e9c9 Mon Sep 17 00:00:00 2001 From: Scott Moser <smoser@ubuntu.com> Date: Wed, 27 Aug 2014 17:03:43 -0400 Subject: no functional changes, but some minor changes. --- cloudinit/sources/helpers/openstack.py | 75 ++++++++++------------- tests/unittests/test_datasource/test_openstack.py | 2 +- 2 files changed, 34 insertions(+), 43 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 025a2404..3c6bb6aa 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -162,21 +162,25 @@ class BaseReader(object): pass def _find_working_version(self, version): + try: + versions_available = self._fetch_available_versions(self) + except Exception as e: + LOG.warn("Unable to read openstack versions from %s due to: %s", + self.base_path, e) + versions_available = [] + search_versions = [version] + list(OS_VERSIONS) - available_versions = self._fetch_available_versions() + selected_version = OS_LATEST for potential_version in search_versions: - if not potential_version: + if potential_version not in versions_available: continue - if potential_version not in available_versions: - continue - if potential_version != version: - LOG.debug("Version '%s' not available, attempting to use" - " version '%s' instead", version, - potential_version) - return potential_version - LOG.debug("Version '%s' not available, attempting to use '%s'" - " instead", version, OS_LATEST) - return OS_LATEST + selected_version = potential_version + break + + if selected_version != version: + LOG.warn("Version '%s' not available, attempting to use" + " version '%s' instead", version, selected_version) + return selected_version def _read_content_path(self, item): path = item.get('content_path', '').lstrip("/") @@ -317,20 +321,12 @@ class ConfigDriveReader(BaseReader): return util.load_file(path) def _fetch_available_versions(self): - if self._versions is not None: - return self._versions - else: - versions_available = [] + if self._versions is None: path = self._path_join(self.base_path, 'openstack') - try: - for child in os.listdir(path): - child_path = os.path.join(path, child) - if os.path.isdir(child_path): - versions_available.append(child) - except (OSError, IOError): - pass - self._versions = tuple(versions_available) - return self._versions + found = [d for d in os.listdir(path) + if os.path.isdir(os.path.join(path))] + self._versions = tuple(found) + return self._versions def _read_ec2_metadata(self): path = self._path_join(self.base_path, @@ -420,24 +416,19 @@ class MetadataReader(BaseReader): self._versions = None def _fetch_available_versions(self): + # <baseurl>/openstack/ returns a newline separated list of versions if self._versions is not None: - return self._versions - else: - path = self._path_join(self.base_path, "openstack") - versions_available = [] - try: - versions = self._path_read(path) - except IOError as e: - LOG.warn("Unable to read openstack versions from %s due" - " to: %s", path, e) - else: - for line in versions.splitlines(): - line = line.strip() - if not line: - continue - versions_available.append(line) - self._versions = tuple(versions_available) - return self._versions + return self.os_versions + found = [] + content = self._path_read(version_path) + for line in content.splitlines(): + line = line.strip() + if not line: + continue + found.append(line) + self._versions = tuple(found) + return self._versions + def _path_read(self, path): diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 412ae5a4..530fba20 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -79,7 +79,7 @@ EC2_FILES = { 'latest/user-data': USER_DATA, } EC2_VERSIONS = [ - 'lastest', + 'latest', ] -- cgit v1.2.3 From 141caf7f3b224c0265c7bb0014b96ca08aa67193 Mon Sep 17 00:00:00 2001 From: Joshua Harlow <harlowja@yahoo-inc.com> Date: Tue, 2 Sep 2014 13:31:18 -0700 Subject: Remove/adjust the verbose 'failed at attempted import of' log Instead of using this log (which really isn't a failure) we should instead of just return the looked up locations and then if there really is an error the caller can handle the usage of the looked up locations as they choose fit. --- cloudinit/distros/__init__.py | 8 +++----- cloudinit/importer.py | 21 ++++++++------------- cloudinit/mergers/__init__.py | 10 ++++++---- cloudinit/sources/__init__.py | 6 +++--- cloudinit/stages.py | 18 +++++++++--------- 5 files changed, 29 insertions(+), 34 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 4b41220e..9c9211fe 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -847,12 +847,10 @@ def extract_default(users, default_name=None, default_config=None): def fetch(name): - locs = importer.find_module(name, - ['', __name__], - ['Distro']) + locs, looked_locs = importer.find_module(name, ['', __name__], ['Distro']) if not locs: - raise ImportError("No distribution found for distro %s" - % (name)) + raise ImportError("No distribution found for distro %s (searched %s)" + % (name, looked_locs)) mod = importer.import_module(locs[0]) cls = getattr(mod, 'Distro') return cls diff --git a/cloudinit/importer.py b/cloudinit/importer.py index a1929137..fb57253c 100644 --- a/cloudinit/importer.py +++ b/cloudinit/importer.py @@ -22,10 +22,6 @@ import sys -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) - def import_module(module_name): __import__(module_name) @@ -33,25 +29,24 @@ def import_module(module_name): def find_module(base_name, search_paths, required_attrs=None): - found_places = [] if not required_attrs: required_attrs = [] # NOTE(harlowja): translate the search paths to include the base name. - real_paths = [] + lookup_paths = [] for path in search_paths: real_path = [] if path: real_path.extend(path.split(".")) real_path.append(base_name) full_path = '.'.join(real_path) - real_paths.append(full_path) - for full_path in real_paths: + lookup_paths.append(full_path) + found_paths = [] + for full_path in lookup_paths: mod = None try: mod = import_module(full_path) - except ImportError as e: - LOG.debug("Failed at attempted import of '%s' due to: %s", - full_path, e) + except ImportError: + pass if not mod: continue found_attrs = 0 @@ -59,5 +54,5 @@ def find_module(base_name, search_paths, required_attrs=None): if hasattr(mod, attr): found_attrs += 1 if found_attrs == len(required_attrs): - found_places.append(full_path) - return found_places + found_paths.append(full_path) + return (found_paths, lookup_paths) diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py index 650b42a9..03aa1ee1 100644 --- a/cloudinit/mergers/__init__.py +++ b/cloudinit/mergers/__init__.py @@ -143,12 +143,14 @@ def construct(parsed_mergers): for (m_name, m_ops) in parsed_mergers: if not m_name.startswith(MERGER_PREFIX): m_name = MERGER_PREFIX + str(m_name) - merger_locs = importer.find_module(m_name, - [__name__], - [MERGER_ATTR]) + merger_locs, looked_locs = importer.find_module(m_name, + [__name__], + [MERGER_ATTR]) if not merger_locs: msg = ("Could not find merger module named '%s' " - "with attribute '%s'") % (m_name, MERGER_ATTR) + "with attribute '%s' (searched %s)") % (m_name, + MERGER_ATTR, + looked_locs) raise ImportError(msg) else: mod = importer.import_module(merger_locs[0]) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 7d52a2e6..5d58bebd 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -272,9 +272,9 @@ def list_sources(cfg_list, depends, pkg_list): for ds_name in cfg_list: if not ds_name.startswith(DS_PREFIX): ds_name = '%s%s' % (DS_PREFIX, ds_name) - m_locs = importer.find_module(ds_name, - pkg_list, - ['get_datasource_list']) + m_locs, _looked_locs = importer.find_module(ds_name, + pkg_list, + ['get_datasource_list']) for m_loc in m_locs: mod = importer.import_module(m_loc) lister = getattr(mod, "get_datasource_list") diff --git a/cloudinit/stages.py b/cloudinit/stages.py index d29d480a..67f467f7 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -386,12 +386,12 @@ class Init(object): potential_handlers = util.find_modules(path) for (fname, mod_name) in potential_handlers.iteritems(): try: - mod_locs = importer.find_module(mod_name, [''], - ['list_types', - 'handle_part']) + mod_locs, looked_locs = importer.find_module( + mod_name, [''], ['list_types', 'handle_part']) if not mod_locs: - LOG.warn(("Could not find a valid user-data handler" - " named %s in file %s"), mod_name, fname) + LOG.warn("Could not find a valid user-data handler" + " named %s in file %s (searched %s)", + mod_name, fname, looked_locs) continue mod = importer.import_module(mod_locs[0]) mod = handlers.fixup_handler(mod) @@ -621,11 +621,11 @@ class Modules(object): " has an unknown frequency %s"), raw_name, freq) # Reset it so when ran it will get set to a known value freq = None - mod_locs = importer.find_module(mod_name, - ['', type_utils.obj_name(config)], - ['handle']) + mod_locs, looked_locs = importer.find_module( + mod_name, ['', type_utils.obj_name(config)], ['handle']) if not mod_locs: - LOG.warn("Could not find module named %s", mod_name) + LOG.warn("Could not find module named %s (searched %s)", + mod_name, looked_locs) continue mod = config.fixup_module(importer.import_module(mod_locs[0])) mostly_mods.append([mod, raw_name, freq, run_args]) -- cgit v1.2.3 From 7dade55ae5e429e0abb1687fb94799226c211e18 Mon Sep 17 00:00:00 2001 From: Joshua Harlow <harlowja@yahoo-inc.com> Date: Tue, 2 Sep 2014 14:17:38 -0700 Subject: Fix logic statement and pep8 issue --- cloudinit/sources/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 7d52a2e6..a2a8f492 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -66,7 +66,7 @@ class DataSource(object): name = name[0:-3] self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, - ("datasource", name), {}) + ("datasource", name), {}) if not ud_proc: self.ud_proc = ud.UserDataProcessor(self.paths) else: @@ -166,7 +166,7 @@ class DataSource(object): defhost = "localhost" domain = defdomain - if self.metadata or 'local-hostname' not in self.metadata: + if not self.metadata or 'local-hostname' not in self.metadata: # this is somewhat questionable really. # the cloud datasource was asked for a hostname # and didn't have one. raising error might be more appropriate -- cgit v1.2.3 From 6f8bd54650e78bd383b29a59bee95a887ff13c81 Mon Sep 17 00:00:00 2001 From: Joshua Harlow <harlowja@yahoo-inc.com> Date: Thu, 4 Sep 2014 10:33:37 -0700 Subject: Reduce logging levels and fix broken call to _fetch_available_versions --- cloudinit/sources/helpers/openstack.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 3c6bb6aa..61c61570 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -163,10 +163,10 @@ class BaseReader(object): def _find_working_version(self, version): try: - versions_available = self._fetch_available_versions(self) + versions_available = self._fetch_available_versions() except Exception as e: - LOG.warn("Unable to read openstack versions from %s due to: %s", - self.base_path, e) + LOG.debug("Unable to read openstack versions from %s due to: %s", + self.base_path, e) versions_available = [] search_versions = [version] + list(OS_VERSIONS) @@ -178,8 +178,8 @@ class BaseReader(object): break if selected_version != version: - LOG.warn("Version '%s' not available, attempting to use" - " version '%s' instead", version, selected_version) + LOG.debug("Version '%s' not available, attempting to use" + " version '%s' instead", version, selected_version) return selected_version def _read_content_path(self, item): @@ -239,7 +239,8 @@ class BaseReader(object): LOG.debug("Failed reading optional path %s due" " to: %s", path, e) else: - LOG.exception("Failed reading mandatory path %s", path) + LOG.debug("Failed reading mandatory path %s due" + " to: %s", path, e) else: found = True if required and not found: -- cgit v1.2.3 From 67efb99bc8e4628e86aef76fed3411c1cc763571 Mon Sep 17 00:00:00 2001 From: Joshua Harlow <harlowja@yahoo-inc.com> Date: Thu, 4 Sep 2014 12:33:41 -0700 Subject: Fix a couple more cases of exceptional logs --- cloudinit/sources/DataSourceOpenStack.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 0970d07b..221759e1 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -64,13 +64,13 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): try: max_wait = int(self.ds_cfg.get("max_wait", max_wait)) - except Exception: - util.logexc(LOG, "Failed to get max wait. using %s", max_wait) + except Exception as e: + LOG.debug("Failed to get max wait. using %s: %s", max_wait, e) try: timeout = max(0, int(self.ds_cfg.get("timeout", timeout))) - except Exception: - util.logexc(LOG, "Failed to get timeout, using %s", timeout) + except Exception as e: + LOG.debug("Failed to get timeout, using %s: %s", timeout, e) return (max_wait, timeout) def wait_for_metadata_service(self): @@ -82,7 +82,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): if len(filtered): urls = filtered else: - LOG.warn("Empty metadata url list! using default list") + LOG.debug("Empty metadata url list! using default list") urls = [DEF_MD_URL] md_urls = [] @@ -123,9 +123,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'version': openstack.OS_HAVANA}) except openstack.NonReadable: return False - except (openstack.BrokenMetadata, IOError): - util.logexc(LOG, "Broken metadata address %s", - self.metadata_address) + except (openstack.BrokenMetadata, IOError) as e: + LOG.debug("Broken metadata address %s: %s", + self.metadata_address, e) return False user_dsmode = results.get('dsmode', None) -- cgit v1.2.3 From 1b975098ea2cc82a168203c720ef936db4864467 Mon Sep 17 00:00:00 2001 From: Scott Moser <smoser@ubuntu.com> Date: Wed, 10 Sep 2014 14:39:59 -0400 Subject: log as warn if things are obviously wrong If something is broken as in a built in config, or code just broken, then logging warning during search for metadata is ok. --- cloudinit/sources/DataSourceOpenStack.py | 16 ++++++++-------- cloudinit/sources/helpers/openstack.py | 6 ++++-- 2 files changed, 12 insertions(+), 10 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 221759e1..0970d07b 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -64,13 +64,13 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): try: max_wait = int(self.ds_cfg.get("max_wait", max_wait)) - except Exception as e: - LOG.debug("Failed to get max wait. using %s: %s", max_wait, e) + except Exception: + util.logexc(LOG, "Failed to get max wait. using %s", max_wait) try: timeout = max(0, int(self.ds_cfg.get("timeout", timeout))) - except Exception as e: - LOG.debug("Failed to get timeout, using %s: %s", timeout, e) + except Exception: + util.logexc(LOG, "Failed to get timeout, using %s", timeout) return (max_wait, timeout) def wait_for_metadata_service(self): @@ -82,7 +82,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): if len(filtered): urls = filtered else: - LOG.debug("Empty metadata url list! using default list") + LOG.warn("Empty metadata url list! using default list") urls = [DEF_MD_URL] md_urls = [] @@ -123,9 +123,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'version': openstack.OS_HAVANA}) except openstack.NonReadable: return False - except (openstack.BrokenMetadata, IOError) as e: - LOG.debug("Broken metadata address %s: %s", - self.metadata_address, e) + except (openstack.BrokenMetadata, IOError): + util.logexc(LOG, "Broken metadata address %s", + self.metadata_address) return False user_dsmode = results.get('dsmode', None) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 7b27621c..0c6b9b5a 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -178,8 +178,10 @@ class BaseReader(object): break if selected_version != version: - LOG.debug("Version '%s' not available, attempting to use" - " version '%s' instead", version, selected_version) + LOG.warn("Version '%s' not available, attempting to use " + "version '%s' instead", version, selected_version) + else: + LOG.debug("Version '%s' was available.", version) return selected_version def _read_content_path(self, item): -- cgit v1.2.3 From 64f4a63c174a4f7c21e6e372927906099103d64c Mon Sep 17 00:00:00 2001 From: Scott Moser <smoser@ubuntu.com> Date: Wed, 10 Sep 2014 14:42:33 -0400 Subject: fix a bug, use a list instead of tuple for _versions using tuple for _versions was just not necessary. fix reference to undefined os_versions. --- cloudinit/sources/helpers/openstack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 0c6b9b5a..3c3de7e4 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -328,7 +328,7 @@ class ConfigDriveReader(BaseReader): path = self._path_join(self.base_path, 'openstack') found = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path))] - self._versions = tuple(found) + self._versions = found return self._versions def _read_ec2_metadata(self): @@ -421,7 +421,7 @@ class MetadataReader(BaseReader): def _fetch_available_versions(self): # <baseurl>/openstack/ returns a newline separated list of versions if self._versions is not None: - return self.os_versions + return self._versions found = [] version_path = self._path_join(self.base_path, "openstack") content = self._path_read(version_path) @@ -430,7 +430,7 @@ class MetadataReader(BaseReader): if not line: continue found.append(line) - self._versions = tuple(found) + self._versions = found return self._versions -- cgit v1.2.3