Complete initial cleanup for refactoring/rework.

Some of the cleanups were the following

1. Using standard (logged) utility functions for sub process work, writing, reading files, and other file system/operating system options
2. Having distrobutions impelement there own subclasses to handle system specifics (if applicable)
3. Having a cloud wrapper that provides just the functionality we want to expose (cloud.py)
4. Using a path class instead of globals for all cloud init paths (it is configured via config)
5. Removal of as much shared global state as possible (there should be none, minus a set of constants)
6. Other various cleanups that remove transforms/handlers/modules from reading/writing/chmoding there own files. 
 a. They should be using util functions to take advantage of the logging that is now enabled in those util functions (very useful for debugging)
7. Urls being read and checked from a single module that serves this and only this purpose (+1 for code organization)
8. Updates to log whenever a transform decides not to run
9. Ensure whenever a exception is thrown (and possibly captured) that the util.logexc function is called 
 a. For debugging, tracing this is important to not just drop them on the floor.
10. Code shuffling into utils.py where it makes sense (and where it could serve a benefit for other code now or in the future)

1 files changed, 12 insertions, 178 deletions

diff --git a/cloudinit/transforms/__init__.py b/cloudinit/transforms/__init__.py
index 5d70ac43..8275b375 100644
--- a/cloudinit/transforms/__init__.py
+++ b/cloudinit/transforms/__init__.py
@@ -19,183 +19,12 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-import os
-import subprocess
-import sys
-import time
-import traceback
-
-import yaml
-
-from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE)
+from cloudinit.settings import (PER_INSTANCE, FREQUENCIES)
 
 from cloudinit import log as logging
-from cloudinit import util
 
 LOG = logging.getLogger(__name__)
 
-DEF_HANDLER_VERSION = 1
-DEF_FREQ = PER_INSTANCE
-
-
-# reads a cloudconfig module list, returns
-# a 2 dimensional array suitable to pass to run_cc_modules
-def read_cc_modules(cfg, name):
-    if name not in cfg:
-        return([])
-    module_list = []
-    # create 'module_list', an array of arrays
-    # where array[0] = config
-    #       array[1] = freq
-    #       array[2:] = arguemnts
-    for item in cfg[name]:
-        if isinstance(item, str):
-            module_list.append((item,))
-        elif isinstance(item, list):
-            module_list.append(item)
-        else:
-            raise TypeError("failed to read '%s' item in config")
-    return(module_list)
-
-
-def run_cc_modules(cc, module_list, log):
-    failures = []
-    for cfg_mod in module_list:
-        name = cfg_mod[0]
-        freq = None
-        run_args = []
-        if len(cfg_mod) > 1:
-            freq = cfg_mod[1]
-        if len(cfg_mod) > 2:
-            run_args = cfg_mod[2:]
-
-        try:
-            log.debug("handling %s with freq=%s and args=%s" %
-                (name, freq, run_args))
-            cc.handle(name, run_args, freq=freq)
-        except:
-            log.warn(traceback.format_exc())
-            log.error("config handling of %s, %s, %s failed\n" %
-                (name, freq, run_args))
-            failures.append(name)
-
-    return(failures)
-
-
-# always returns well formated values
-# cfg is expected to have an entry 'output' in it, which is a dictionary
-# that includes entries for 'init', 'config', 'final' or 'all'
-#   init: /var/log/cloud.out
-#   config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ]
-#   final:
-#     output: "| logger -p"
-#     error: "> /dev/null"
-# this returns the specific 'mode' entry, cleanly formatted, with value
-# None if if none is given
-def get_output_cfg(cfg, mode="init"):
-    ret = [None, None]
-    if not 'output' in cfg:
-        return ret
-
-    outcfg = cfg['output']
-    if mode in outcfg:
-        modecfg = outcfg[mode]
-    else:
-        if 'all' not in outcfg:
-            return ret
-        # if there is a 'all' item in the output list
-        # then it applies to all users of this (init, config, final)
-        modecfg = outcfg['all']
-
-    # if value is a string, it specifies stdout and stderr
-    if isinstance(modecfg, str):
-        ret = [modecfg, modecfg]
-
-    # if its a list, then we expect (stdout, stderr)
-    if isinstance(modecfg, list):
-        if len(modecfg) > 0:
-            ret[0] = modecfg[0]
-        if len(modecfg) > 1:
-            ret[1] = modecfg[1]
-
-    # if it is a dictionary, expect 'out' and 'error'
-    # items, which indicate out and error
-    if isinstance(modecfg, dict):
-        if 'output' in modecfg:
-            ret[0] = modecfg['output']
-        if 'error' in modecfg:
-            ret[1] = modecfg['error']
-
-    # if err's entry == "&1", then make it same as stdout
-    # as in shell syntax of "echo foo >/dev/null 2>&1"
-    if ret[1] == "&1":
-        ret[1] = ret[0]
-
-    swlist = [">>", ">", "|"]
-    for i in range(len(ret)):
-        if not ret[i]:
-            continue
-        val = ret[i].lstrip()
-        found = False
-        for s in swlist:
-            if val.startswith(s):
-                val = "%s %s" % (s, val[len(s):].strip())
-                found = True
-                break
-        if not found:
-            # default behavior is append
-            val = "%s %s" % (">>", val.strip())
-        ret[i] = val
-
-    return(ret)
-
-
-# redirect_output(outfmt, errfmt, orig_out, orig_err)
-#  replace orig_out and orig_err with filehandles specified in outfmt or errfmt
-#  fmt can be:
-#   > FILEPATH
-#   >> FILEPATH
-#   | program [ arg1 [ arg2 [ ... ] ] ]
-#
-#   with a '|', arguments are passed to shell, so one level of
-#   shell escape is required.
-def redirect_output(outfmt, errfmt, o_out=sys.stdout, o_err=sys.stderr):
-    if outfmt:
-        (mode, arg) = outfmt.split(" ", 1)
-        if mode == ">" or mode == ">>":
-            owith = "ab"
-            if mode == ">":
-                owith = "wb"
-            new_fp = open(arg, owith)
-        elif mode == "|":
-            proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE)
-            new_fp = proc.stdin
-        else:
-            raise TypeError("invalid type for outfmt: %s" % outfmt)
-
-        if o_out:
-            os.dup2(new_fp.fileno(), o_out.fileno())
-        if errfmt == outfmt:
-            os.dup2(new_fp.fileno(), o_err.fileno())
-            return
-
-    if errfmt:
-        (mode, arg) = errfmt.split(" ", 1)
-        if mode == ">" or mode == ">>":
-            owith = "ab"
-            if mode == ">":
-                owith = "wb"
-            new_fp = open(arg, owith)
-        elif mode == "|":
-            proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE)
-            new_fp = proc.stdin
-        else:
-            raise TypeError("invalid type for outfmt: %s" % outfmt)
-
-        if o_err:
-            os.dup2(new_fp.fileno(), o_err.fileno())
-    return
-
 
 def form_module_name(name):
     canon_name = name.replace("-", "_")
@@ -209,13 +38,18 @@ def form_module_name(name):
     return canon_name
 
 
-def fixup_module(mod):
-    freq = getattr(mod, "frequency", None)
-    if not freq:
-        setattr(mod, 'frequency', PER_INSTANCE)
-    handler = getattr(mod, "handle", None)
-    if not handler:
+def fixup_module(mod, def_freq=PER_INSTANCE):
+    if not hasattr(mod, 'frequency'):
+        setattr(mod, 'frequency', def_freq)
+    else:
+        freq = mod.frequency
+        if freq and freq not in FREQUENCIES:
+            LOG.warn("Module %s has an unknown frequency %s", mod, freq)
+    if not hasattr(mod, 'handle'):
         def empty_handle(_name, _cfg, _cloud, _log, _args):
             pass
         setattr(mod, 'handle', empty_handle)
+    # Used only for warning if possibly running on a not checked distro...
+    if not hasattr(mod, 'distros'):
+        setattr(mod, 'distros', None)
     return mod