1. Add an importer function that will search for a given module in a set of search module 'prefixes'

   that also has a potential set of required attributes.
2. Use this new importer to find the distro class, the userdata handler modules, the config modules
   and the datasource modules, if none can be found error out accordingly.

6 files changed, 74 insertions, 61 deletions

diff --git a/cloudinit/config/__init__.py b/cloudinit/config/__init__.py
index 02e32462..ab13045f 100644
--- a/cloudinit/config/__init__.py
+++ b/cloudinit/config/__init__.py
@@ -51,10 +51,6 @@ def fixup_module(mod, def_freq=PER_INSTANCE):
         freq = mod.frequency
         if freq and freq not in FREQUENCIES:
             LOG.warn("Module %s has an unknown frequency %s", mod, freq)
-    if not hasattr(mod, 'handle'):
-        def empty_handle(_name, _cfg, _cloud, _log, _args):
-            pass
-        setattr(mod, 'handle', empty_handle)
     if not hasattr(mod, 'distros'):
         setattr(mod, 'distros', None)
     return mod
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 25a60c52..e0ef6ee0 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -144,16 +144,13 @@ class Distro(object):
             return False
 
 
-def fetch(distro_name, mods=(__name__, )):
-    mod = None
-    for m in mods:
-        mod_name = "%s.%s" % (m, distro_name)
-        try:
-            mod = importer.import_module(mod_name)
-        except ImportError:
-            pass
-    if not mod:
-        raise RuntimeError("No distribution found for distro %s"
-                           % (distro_name))
-    distro_cls = getattr(mod, 'Distro')
-    return distro_cls
+def fetch(name):
+    locs = importer.find_module(name,
+                                ['', __name__],
+                                ['Distro'])
+    if not locs:
+        raise ImportError("No distribution found for distro %s"
+                           % (name))
+    mod = importer.import_module(locs[0])
+    cls = getattr(mod, 'Distro')
+    return cls
diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py
index d52b1cba..0f7432e5 100644
--- a/cloudinit/handlers/__init__.py
+++ b/cloudinit/handlers/__init__.py
@@ -202,20 +202,12 @@ def walk(msg, callback, data):
 def fixup_handler(mod, def_freq=PER_INSTANCE):
     if not hasattr(mod, "handler_version"):
         setattr(mod, "handler_version", 1)
-    if not hasattr(mod, 'list_types'):
-        def empty_types():
-            return []
-        setattr(mod, 'list_types', empty_types)
     if not hasattr(mod, 'frequency'):
         setattr(mod, 'frequency', def_freq)
     else:
         freq = mod.frequency
         if freq and freq not in FREQUENCIES:
             LOG.warn("Handler %s has an unknown frequency %s", mod, freq)
-    if not hasattr(mod, 'handle_part'):
-        def empty_handler(_data, _ctype, _filename, _payload):
-            pass
-        setattr(mod, 'handle_part', empty_handler)
     return mod
 
 
diff --git a/cloudinit/importer.py b/cloudinit/importer.py
index a36b87bc..71cf2726 100644
--- a/cloudinit/importer.py
+++ b/cloudinit/importer.py
@@ -23,17 +23,43 @@
 import sys
 
 from cloudinit import log as logging
-from cloudinit import util
 
 LOG = logging.getLogger(__name__)
 
 
-# Simple wrapper that allows us to add more logging in...
 def import_module(module_name):
-    try:
-        LOG.debug("Attempting to import module %s", module_name)
-        __import__(module_name)
-        return sys.modules[module_name]
-    except:
-        util.logexc(LOG, 'Failed at importing %s', module_name)
-        raise
+    __import__(module_name)
+    return sys.modules[module_name]
+
+
+def find_module(base_name, search_paths, required_attrs=None):
+    found_places = []
+    if not required_attrs:
+        required_attrs = []
+    real_paths = []
+    for path in search_paths:
+        real_path = []
+        if path:
+            real_path.extend(path.split("."))
+        real_path.append(base_name)
+        full_path = '.'.join(real_path)
+        real_paths.append(full_path)
+    LOG.debug("Looking for modules %s that have attributes %s",
+              real_paths, required_attrs)
+    for full_path in real_paths:
+        mod = None
+        try:
+            mod = import_module(full_path)
+        except ImportError:
+            pass
+        if not mod:
+            continue
+        found_attrs = 0
+        for attr in required_attrs:
+            if hasattr(mod, attr):
+                found_attrs += 1
+        if found_attrs == len(required_attrs):
+            found_places.append(full_path)
+    LOG.debug("Found %s with attributes %s in %s", base_name,
+              required_attrs, found_places)
+    return found_places
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 8ab7cf54..42e924b0 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -191,31 +191,19 @@ def list_sources(cfg_list, depends, pkg_list):
     LOG.info(("Looking for for data source in: %s,"
               " via packages %s that matches dependencies %s"),
              cfg_list, pkg_list, depends)
-    for ds_coll in cfg_list:
-        ds_name = str(ds_coll)
+    for ds_name in cfg_list:
         if not ds_name.startswith(DS_PREFIX):
             ds_name = '%s%s' % (DS_PREFIX, ds_name)
-        for pkg in pkg_list:
-            pkg_name = []
-            if pkg:
-                # Any package name given, this affects
-                # the lookup path
-                pkg_name.append(str(pkg))
-            pkg_name.append(ds_name)
-            try:
-                mod = importer.import_module(".".join(pkg_name))
-            except ImportError:
-                continue
-            lister = getattr(mod, "get_datasource_list", None)
-            if not lister:
-                continue
-            cls_matches = lister(depends)
-            if not cls_matches:
-                continue
-            src_list.extend(cls_matches)
-            LOG.debug(("Found a match"
-                       " in %s with matches %s"), mod, cls_matches)
-            break
+        m_locs = importer.find_module(ds_name,
+                                      pkg_list,
+                                      ['get_datasource_list'])
+        for m_loc in m_locs:
+            mod = importer.import_module(m_loc)
+            lister = getattr(mod, "get_datasource_list")
+            matches = lister(depends)
+            if matches:
+                src_list.extend(matches)
+                break
     return src_list
 
 
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 84a965c2..1997301a 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -205,7 +205,7 @@ class Init(object):
         # Any config provided???
         pkg_list = self.cfg.get('datasource_pkg_list') or []
         # Add the defaults at the end
-        for n in [util.obj_name(sources), '']:
+        for n in ['', util.obj_name(sources)]:
             if n not in pkg_list:
                 pkg_list.append(n)
         cfg_list = self.cfg.get('datasource_list') or []
@@ -334,9 +334,17 @@ class Init(object):
 
         # Add handlers in cdir
         potential_handlers = util.find_modules(cdir)
-        for (fname, modname) in potential_handlers.iteritems():
+        for (fname, mod_name) in potential_handlers.iteritems():
             try:
-                mod = handlers.fixup_handler(importer.import_module(modname))
+                mod_locs = importer.find_module(mod_name, [''],
+                                                ['list_types',
+                                                 'handle_part'])
+                if not mod_locs:
+                    LOG.warn(("Could not find a valid user-data handler"
+                              " named %s in file %s"), mod_name, fname)
+                    continue
+                mod = importer.import_module(mod_locs[0])
+                mod = handlers.fixup_handler(mod)
                 types = c_handlers.register(mod)
                 LOG.debug("Added handler for %s from %s", types, fname)
             except:
@@ -482,7 +490,13 @@ class Modules(object):
                           " has an unknown frequency %s"), raw_name, freq)
                 # Reset it so when ran it will get set to a known value
                 freq = None
-            mod = config.fixup_module(importer.import_module(mod_name))
+            mod_locs = importer.find_module(mod_name,
+                                            ['', util.obj_name(config)],
+                                            ['handle'])
+            if not mod_locs:
+                LOG.warn("Could not find module named %s", mod_name)
+                continue
+            mod = config.fixup_module(importer.import_module(mod_locs[0]))
             mostly_mods.append([mod, raw_name, freq, run_args])
         return mostly_mods