Merge branch 'master' into ubuntu/xenial

11 files changed, 824 insertions, 199 deletions

diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 05ad4b03..fa9505a7 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -23,80 +23,182 @@ import os
 import re
 
 from cloudinit import gpg
+from cloudinit import log as logging
 from cloudinit import templater
 from cloudinit import util
 
-distros = ['ubuntu', 'debian']
-
-PROXY_TPL = "Acquire::HTTP::Proxy \"%s\";\n"
-APT_CONFIG_FN = "/etc/apt/apt.conf.d/94cloud-init-config"
-APT_PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy"
+LOG = logging.getLogger(__name__)
 
 # this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar')
 ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
 
+# place where apt stores cached repository data
+APT_LISTS = "/var/lib/apt/lists"
 
-def handle(name, cfg, cloud, log, _args):
-    if util.is_false(cfg.get('apt_configure_enabled', True)):
-        log.debug("Skipping module named %s, disabled by config.", name)
-        return
-
-    release = get_release()
-    mirrors = find_apt_mirror_info(cloud, cfg)
-    if not mirrors or "primary" not in mirrors:
-        log.debug(("Skipping module named %s,"
-                   " no package 'mirror' located"), name)
-        return
-
-    # backwards compatibility
-    mirror = mirrors["primary"]
-    mirrors["mirror"] = mirror
-
-    log.debug("Mirror info: %s" % mirrors)
-
-    if not util.get_cfg_option_bool(cfg,
-                                    'apt_preserve_sources_list', False):
-        generate_sources_list(cfg, release, mirrors, cloud, log)
-        old_mirrors = cfg.get('apt_old_mirrors',
-                              {"primary": "archive.ubuntu.com/ubuntu",
-                               "security": "security.ubuntu.com/ubuntu"})
-        rename_apt_lists(old_mirrors, mirrors)
+# Files to store proxy information
+APT_CONFIG_FN = "/etc/apt/apt.conf.d/94cloud-init-config"
+APT_PROXY_FN = "/etc/apt/apt.conf.d/90cloud-init-aptproxy"
+
+# Default keyserver to use
+DEFAULT_KEYSERVER = "keyserver.ubuntu.com"
+
+# Default archive mirrors
+PRIMARY_ARCH_MIRRORS = {"PRIMARY": "http://archive.ubuntu.com/ubuntu/",
+                        "SECURITY": "http://security.ubuntu.com/ubuntu/"}
+PORTS_MIRRORS = {"PRIMARY": "http://ports.ubuntu.com/ubuntu-ports",
+                 "SECURITY": "http://ports.ubuntu.com/ubuntu-ports"}
+PRIMARY_ARCHES = ['amd64', 'i386']
+PORTS_ARCHES = ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el']
+
+
+def get_default_mirrors(arch=None, target=None):
+    """returns the default mirrors for the target. These depend on the
+       architecture, for more see:
+       https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports"""
+    if arch is None:
+        arch = util.get_architecture(target)
+    if arch in PRIMARY_ARCHES:
+        return PRIMARY_ARCH_MIRRORS.copy()
+    if arch in PORTS_ARCHES:
+        return PORTS_MIRRORS.copy()
+    raise ValueError("No default mirror known for arch %s" % arch)
+
+
+def handle(name, ocfg, cloud, log, _):
+    """process the config for apt_config. This can be called from
+       curthooks if a global apt config was provided or via the "apt"
+       standalone command."""
+    # keeping code close to curtin codebase via entry handler
+    target = None
+    if log is not None:
+        global LOG
+        LOG = log
+    # feed back converted config, but only work on the subset under 'apt'
+    ocfg = convert_to_v3_apt_format(ocfg)
+    cfg = ocfg.get('apt', {})
+
+    if not isinstance(cfg, dict):
+        raise ValueError("Expected dictionary for 'apt' config, found %s",
+                         type(cfg))
+
+    LOG.debug("handling apt (module %s) with apt config '%s'", name, cfg)
+
+    release = util.lsb_release(target=target)['codename']
+    arch = util.get_architecture(target)
+    mirrors = find_apt_mirror_info(cfg, cloud, arch=arch)
+    LOG.debug("Apt Mirror info: %s", mirrors)
+
+    apply_debconf_selections(cfg, target)
+
+    if util.is_false(cfg.get('preserve_sources_list', False)):
+        generate_sources_list(cfg, release, mirrors, cloud)
+        rename_apt_lists(mirrors, target)
 
     try:
         apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN)
-    except Exception as e:
-        log.warn("failed to proxy or apt config info: %s", e)
+    except (IOError, OSError):
+        LOG.exception("Failed to apply proxy or apt config info:")
 
-    # Process 'apt_sources'
-    if 'apt_sources' in cfg:
+    # Process 'apt_source -> sources {dict}'
+    if 'sources' in cfg:
         params = mirrors
         params['RELEASE'] = release
-        params['MIRROR'] = mirror
+        params['MIRROR'] = mirrors["MIRROR"]
 
+        matcher = None
         matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH)
         if matchcfg:
             matcher = re.compile(matchcfg).search
+
+        add_apt_sources(cfg['sources'], cloud, target=target,
+                        template_params=params, aa_repo_match=matcher)
+
+
+def debconf_set_selections(selections, target=None):
+    util.subp(['debconf-set-selections'], data=selections, target=target,
+              capture=True)
+
+
+def dpkg_reconfigure(packages, target=None):
+    # For any packages that are already installed, but have preseed data
+    # we populate the debconf database, but the filesystem configuration
+    # would be preferred on a subsequent dpkg-reconfigure.
+    # so, what we have to do is "know" information about certain packages
+    # to unconfigure them.
+    unhandled = []
+    to_config = []
+    for pkg in packages:
+        if pkg in CONFIG_CLEANERS:
+            LOG.debug("unconfiguring %s", pkg)
+            CONFIG_CLEANERS[pkg](target)
+            to_config.append(pkg)
         else:
-            def matcher(x):
-                return False
+            unhandled.append(pkg)
+
+    if len(unhandled):
+        LOG.warn("The following packages were installed and preseeded, "
+                 "but cannot be unconfigured: %s", unhandled)
+
+    if len(to_config):
+        util.subp(['dpkg-reconfigure', '--frontend=noninteractive'] +
+                  list(to_config), data=None, target=target, capture=True)
+
+
+def apply_debconf_selections(cfg, target=None):
+    """apply_debconf_selections - push content to debconf"""
+    # debconf_selections:
+    #  set1: |
+    #   cloud-init cloud-init/datasources multiselect MAAS
+    #  set2: pkg pkg/value string bar
+    selsets = cfg.get('debconf_selections')
+    if not selsets:
+        LOG.debug("debconf_selections was not set in config")
+        return
 
-        errors = add_apt_sources(cfg['apt_sources'], params,
-                                 aa_repo_match=matcher)
-        for e in errors:
-            log.warn("Add source error: %s", ':'.join(e))
+    selections = '\n'.join(
+        [selsets[key] for key in sorted(selsets.keys())])
+    debconf_set_selections(selections.encode() + b"\n", target=target)
 
-    dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False)
-    if dconf_sel:
-        log.debug("Setting debconf selections per cloud config")
-        try:
-            util.subp(('debconf-set-selections', '-'), dconf_sel)
-        except Exception:
-            util.logexc(log, "Failed to run debconf-set-selections")
+    # get a complete list of packages listed in input
+    pkgs_cfgd = set()
+    for key, content in selsets.items():
+        for line in content.splitlines():
+            if line.startswith("#"):
+                continue
+            pkg = re.sub(r"[:\s].*", "", line)
+            pkgs_cfgd.add(pkg)
+
+    pkgs_installed = util.get_installed_packages(target)
+
+    LOG.debug("pkgs_cfgd: %s", pkgs_cfgd)
+    need_reconfig = pkgs_cfgd.intersection(pkgs_installed)
+
+    if len(need_reconfig) == 0:
+        LOG.debug("no need for reconfig")
+        return
+
+    dpkg_reconfigure(need_reconfig, target=target)
+
+
+def clean_cloud_init(target):
+    """clean out any local cloud-init config"""
+    flist = glob.glob(
+        util.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*"))
+
+    LOG.debug("cleaning cloud-init config from: %s", flist)
+    for dpkg_cfg in flist:
+        os.unlink(dpkg_cfg)
 
 
 def mirrorurl_to_apt_fileprefix(mirror):
+    """mirrorurl_to_apt_fileprefix
+       Convert a mirror url to the file prefix used by apt on disk to
+       store cache information for that mirror.
+       To do so do:
+       - take off ???://
+       - drop tailing /
+       - convert in string / to _"""
     string = mirror
-    # take off http:// or ftp://
     if string.endswith("/"):
         string = string[0:-1]
     pos = string.find("://")
@@ -106,174 +208,379 @@ def mirrorurl_to_apt_fileprefix(mirror):
     return string
 
 
-def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"):
-    for (name, omirror) in old_mirrors.items():
+def rename_apt_lists(new_mirrors, target=None):
+    """rename_apt_lists - rename apt lists to preserve old cache data"""
+    default_mirrors = get_default_mirrors(util.get_architecture(target))
+
+    pre = util.target_path(target, APT_LISTS)
+    for (name, omirror) in default_mirrors.items():
         nmirror = new_mirrors.get(name)
         if not nmirror:
             continue
-        oprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(omirror))
-        nprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(nmirror))
+
+        oprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(omirror)
+        nprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(nmirror)
         if oprefix == nprefix:
             continue
         olen = len(oprefix)
         for filename in glob.glob("%s_*" % oprefix):
-            util.rename(filename, "%s%s" % (nprefix, filename[olen:]))
-
-
-def get_release():
-    (stdout, _stderr) = util.subp(['lsb_release', '-cs'])
-    return stdout.strip()
-
-
-def generate_sources_list(cfg, codename, mirrors, cloud, log):
-    params = {'codename': codename}
+            newname = "%s%s" % (nprefix, filename[olen:])
+            LOG.debug("Renaming apt list %s to %s", filename, newname)
+            try:
+                os.rename(filename, newname)
+            except OSError:
+                # since this is a best effort task, warn with but don't fail
+                LOG.warn("Failed to rename apt list:", exc_info=True)
+
+
+def mirror_to_placeholder(tmpl, mirror, placeholder):
+    """mirror_to_placeholder
+       replace the specified mirror in a template with a placeholder string
+       Checks for existance of the expected mirror and warns if not found"""
+    if mirror not in tmpl:
+        LOG.warn("Expected mirror '%s' not found in: %s", mirror, tmpl)
+    return tmpl.replace(mirror, placeholder)
+
+
+def map_known_suites(suite):
+    """there are a few default names which will be auto-extended.
+       This comes at the inability to use those names literally as suites,
+       but on the other hand increases readability of the cfg quite a lot"""
+    mapping = {'updates': '$RELEASE-updates',
+               'backports': '$RELEASE-backports',
+               'security': '$RELEASE-security',
+               'proposed': '$RELEASE-proposed',
+               'release': '$RELEASE'}
+    try:
+        retsuite = mapping[suite]
+    except KeyError:
+        retsuite = suite
+    return retsuite
+
+
+def disable_suites(disabled, src, release):
+    """reads the config for suites to be disabled and removes those
+       from the template"""
+    if not disabled:
+        return src
+
+    retsrc = src
+    for suite in disabled:
+        suite = map_known_suites(suite)
+        releasesuite = templater.render_string(suite, {'RELEASE': release})
+        LOG.debug("Disabling suite %s as %s", suite, releasesuite)
+
+        newsrc = ""
+        for line in retsrc.splitlines(True):
+            if line.startswith("#"):
+                newsrc += line
+                continue
+
+            # sources.list allow options in cols[1] which can have spaces
+            # so the actual suite can be [2] or later. example:
+            # deb [ arch=amd64,armel k=v ] http://example.com/debian
+            cols = line.split()
+            if len(cols) > 1:
+                pcol = 2
+                if cols[1].startswith("["):
+                    for col in cols[1:]:
+                        pcol += 1
+                        if col.endswith("]"):
+                            break
+
+                if cols[pcol] == releasesuite:
+                    line = '# suite disabled by cloud-init: %s' % line
+            newsrc += line
+        retsrc = newsrc
+
+    return retsrc
+
+
+def generate_sources_list(cfg, release, mirrors, cloud):
+    """generate_sources_list
+       create a source.list file based on a custom or default template
+       by replacing mirrors and release in the template"""
+    aptsrc = "/etc/apt/sources.list"
+    params = {'RELEASE': release, 'codename': release}
     for k in mirrors:
         params[k] = mirrors[k]
+        params[k.lower()] = mirrors[k]
 
-    custtmpl = cfg.get('apt_custom_sources_list', None)
-    if custtmpl is not None:
-        templater.render_string_to_file(custtmpl,
-                                        '/etc/apt/sources.list', params)
-        return
-
-    template_fn = cloud.get_template_filename('sources.list.%s' %
-                                              (cloud.distro.name))
-    if not template_fn:
-        template_fn = cloud.get_template_filename('sources.list')
+    tmpl = cfg.get('sources_list', None)
+    if tmpl is None:
+        LOG.info("No custom template provided, fall back to builtin")
+        template_fn = cloud.get_template_filename('sources.list.%s' %
+                                                  (cloud.distro.name))
         if not template_fn:
-            log.warn("No template found, not rendering /etc/apt/sources.list")
+            template_fn = cloud.get_template_filename('sources.list')
+        if not template_fn:
+            LOG.warn("No template found, not rendering /etc/apt/sources.list")
             return
+        tmpl = util.load_file(template_fn)
 
-    templater.render_to_file(template_fn, '/etc/apt/sources.list', params)
+    rendered = templater.render_string(tmpl, params)
+    disabled = disable_suites(cfg.get('disable_suites'), rendered, release)
+    util.write_file(aptsrc, disabled, mode=0o644)
 
 
-def add_apt_key_raw(key):
+def add_apt_key_raw(key, target=None):
     """
     actual adding of a key as defined in key argument
     to the system
     """
+    LOG.debug("Adding key:\n'%s'", key)
     try:
-        util.subp(('apt-key', 'add', '-'), key)
+        util.subp(['apt-key', 'add', '-'], data=key.encode(), target=target)
     except util.ProcessExecutionError:
-        raise ValueError('failed to add apt GPG Key to apt keyring')
+        LOG.exception("failed to add apt GPG Key to apt keyring")
+        raise
 
 
-def add_apt_key(ent):
+def add_apt_key(ent, target=None):
     """
-    add key to the system as defined in ent (if any)
-    supports raw keys or keyid's
-    The latter will as a first step fetch the raw key from a keyserver
+    Add key to the system as defined in ent (if any).
+    Supports raw keys or keyid's
+    The latter will as a first step fetched to get the raw key
     """
     if 'keyid' in ent and 'key' not in ent:
-        keyserver = "keyserver.ubuntu.com"
+        keyserver = DEFAULT_KEYSERVER
         if 'keyserver' in ent:
             keyserver = ent['keyserver']
-        ent['key'] = gpg.get_key_by_id(ent['keyid'], keyserver)
 
-    if 'key' in ent:
-        add_apt_key_raw(ent['key'])
+        ent['key'] = gpg.getkeybyid(ent['keyid'], keyserver)
 
+    if 'key' in ent:
+        add_apt_key_raw(ent['key'], target)
 
-def convert_to_new_format(srclist):
-    """convert_to_new_format
-       convert the old list based format to the new dict based one
-    """
-    srcdict = {}
-    if isinstance(srclist, list):
-        for srcent in srclist:
-            if 'filename' not in srcent:
-                # file collides for multiple !filename cases for compatibility
-                # yet we need them all processed, so not same dictionary key
-                srcent['filename'] = "cloud_config_sources.list"
-                key = util.rand_dict_key(srcdict, "cloud_config_sources.list")
-            else:
-                # all with filename use that as key (matching new format)
-                key = srcent['filename']
-            srcdict[key] = srcent
-    elif isinstance(srclist, dict):
-        srcdict = srclist
-    else:
-        raise ValueError("unknown apt_sources format")
 
-    return srcdict
+def update_packages(cloud):
+    cloud.distro.update_package_sources()
 
 
-def add_apt_sources(srclist, template_params=None, aa_repo_match=None):
+def add_apt_sources(srcdict, cloud, target=None, template_params=None,
+                    aa_repo_match=None):
     """
     add entries in /etc/apt/sources.list.d for each abbreviated
-    sources.list entry in 'srclist'.  When rendering template, also
+    sources.list entry in 'srcdict'.  When rendering template, also
     include the values in dictionary searchList
     """
     if template_params is None:
         template_params = {}
 
     if aa_repo_match is None:
-        def _aa_repo_match(x):
-            return False
-        aa_repo_match = _aa_repo_match
+        raise ValueError('did not get a valid repo matcher')
 
-    errorlist = []
-    srcdict = convert_to_new_format(srclist)
+    if not isinstance(srcdict, dict):
+        raise TypeError('unknown apt format: %s' % (srcdict))
 
     for filename in srcdict:
         ent = srcdict[filename]
+        LOG.debug("adding source/key '%s'", ent)
         if 'filename' not in ent:
             ent['filename'] = filename
 
-        # keys can be added without specifying a source
-        try:
-            add_apt_key(ent)
-        except ValueError as detail:
-            errorlist.append([ent, detail])
+        add_apt_key(ent, target)
 
         if 'source' not in ent:
-            errorlist.append(["", "missing source"])
             continue
         source = ent['source']
         source = templater.render_string(source, template_params)
 
-        if not ent['filename'].startswith(os.path.sep):
+        if not ent['filename'].startswith("/"):
             ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
                                            ent['filename'])
+        if not ent['filename'].endswith(".list"):
+            ent['filename'] += ".list"
 
         if aa_repo_match(source):
             try:
-                util.subp(["add-apt-repository", source])
-            except util.ProcessExecutionError as e:
-                errorlist.append([source,
-                                  ("add-apt-repository failed. " + str(e))])
+                util.subp(["add-apt-repository", source], target=target)
+            except util.ProcessExecutionError:
+                LOG.exception("add-apt-repository failed.")
+                raise
             continue
 
+        sourcefn = util.target_path(target, ent['filename'])
         try:
             contents = "%s\n" % (source)
-            util.write_file(ent['filename'], contents, omode="ab")
-        except Exception:
-            errorlist.append([source,
-                             "failed write to file %s" % ent['filename']])
+            util.write_file(sourcefn, contents, omode="a")
+        except IOError as detail:
+            LOG.exception("failed write to file %s: %s", sourcefn, detail)
+            raise
 
-    return errorlist
+    update_packages(cloud)
 
+    return
 
-def find_apt_mirror_info(cloud, cfg):
-    """find an apt_mirror given the cloud and cfg provided."""
 
-    mirror = None
+def convert_v1_to_v2_apt_format(srclist):
+    """convert v1 apt format to v2 (dict in apt_sources)"""
+    srcdict = {}
+    if isinstance(srclist, list):
+        LOG.debug("apt config: convert V1 to V2 format (source list to dict)")
+        for srcent in srclist:
+            if 'filename' not in srcent:
+                # file collides for multiple !filename cases for compatibility
+                # yet we need them all processed, so not same dictionary key
+                srcent['filename'] = "cloud_config_sources.list"
+                key = util.rand_dict_key(srcdict, "cloud_config_sources.list")
+            else:
+                # all with filename use that as key (matching new format)
+                key = srcent['filename']
+            srcdict[key] = srcent
+    elif isinstance(srclist, dict):
+        srcdict = srclist
+    else:
+        raise ValueError("unknown apt_sources format")
+
+    return srcdict
 
-    # this is less preferred way of specifying mirror preferred would be to
-    # use the distro's search or package_mirror.
-    mirror = cfg.get("apt_mirror", None)
 
-    search = cfg.get("apt_mirror_search", None)
-    if not mirror and search:
-        mirror = util.search_for_mirror(search)
+def convert_key(oldcfg, aptcfg, oldkey, newkey):
+    """convert an old key to the new one if the old one exists
+       returns true if a key was found and converted"""
+    if oldcfg.get(oldkey, None) is not None:
+        aptcfg[newkey] = oldcfg.get(oldkey)
+        del oldcfg[oldkey]
+        return True
+    return False
+
+
+def convert_mirror(oldcfg, aptcfg):
+    """convert old apt_mirror keys into the new more advanced mirror spec"""
+    keymap = [('apt_mirror', 'uri'),
+              ('apt_mirror_search', 'search'),
+              ('apt_mirror_search_dns', 'search_dns')]
+    converted = False
+    newmcfg = {'arches': ['default']}
+    for oldkey, newkey in keymap:
+        if convert_key(oldcfg, newmcfg, oldkey, newkey):
+            converted = True
+
+    # only insert new style config if anything was converted
+    if converted:
+        aptcfg['primary'] = [newmcfg]
+
+
+def convert_v2_to_v3_apt_format(oldcfg):
+    """convert old to new keys and adapt restructured mirror spec"""
+    mapoldkeys = {'apt_sources': 'sources',
+                  'apt_mirror': None,
+                  'apt_mirror_search': None,
+                  'apt_mirror_search_dns': None,
+                  'apt_proxy': 'proxy',
+                  'apt_http_proxy': 'http_proxy',
+                  'apt_ftp_proxy': 'https_proxy',
+                  'apt_https_proxy': 'ftp_proxy',
+                  'apt_preserve_sources_list': 'preserve_sources_list',
+                  'apt_custom_sources_list': 'sources_list',
+                  'add_apt_repo_match': 'add_apt_repo_match'}
+    needtoconvert = []
+    for oldkey in mapoldkeys:
+        if oldkey in oldcfg:
+            if oldcfg[oldkey] in (None, ""):
+                del oldcfg[oldkey]
+            else:
+                needtoconvert.append(oldkey)
+
+    # no old config, so no new one to be created
+    if not needtoconvert:
+        return oldcfg
+    LOG.debug("apt config: convert V2 to V3 format for keys '%s'",
+              ", ".join(needtoconvert))
+
+    # if old AND new config are provided, prefer the new one (LP #1616831)
+    newaptcfg = oldcfg.get('apt', None)
+    if newaptcfg is not None:
+        LOG.debug("apt config: V1/2 and V3 format specified, preferring V3")
+        for oldkey in needtoconvert:
+            newkey = mapoldkeys[oldkey]
+            verify = oldcfg[oldkey]  # drop, but keep a ref for verification
+            del oldcfg[oldkey]
+            if newkey is None or newaptcfg.get(newkey, None) is None:
+                # no simple mapping or no collision on this particular key
+                continue
+            if verify != newaptcfg[newkey]:
+                raise ValueError("Old and New apt format defined with unequal "
+                                 "values %s vs %s @ %s" % (verify,
+                                                           newaptcfg[newkey],
+                                                           oldkey))
+        # return conf after clearing conflicting V1/2 keys
+        return oldcfg
+
+    # create new format from old keys
+    aptcfg = {}
+
+    # simple renames / moves under the apt key
+    for oldkey in mapoldkeys:
+        if mapoldkeys[oldkey] is not None:
+            convert_key(oldcfg, aptcfg, oldkey, mapoldkeys[oldkey])
+
+    # mirrors changed in a more complex way
+    convert_mirror(oldcfg, aptcfg)
+
+    for oldkey in mapoldkeys:
+        if oldcfg.get(oldkey, None) is not None:
+            raise ValueError("old apt key '%s' left after conversion" % oldkey)
+
+    # insert new format into config and return full cfg with only v3 content
+    oldcfg['apt'] = aptcfg
+    return oldcfg
+
+
+def convert_to_v3_apt_format(cfg):
+    """convert the old list based format to the new dict based one. After that
+       convert the old dict keys/format to v3 a.k.a 'new apt config'"""
+    # V1 -> V2, the apt_sources entry from list to dict
+    apt_sources = cfg.get('apt_sources', None)
+    if apt_sources is not None:
+        cfg['apt_sources'] = convert_v1_to_v2_apt_format(apt_sources)
+
+    # V2 -> V3, move all former globals under the "apt" key
+    # Restructure into new key names and mirror hierarchy
+    cfg = convert_v2_to_v3_apt_format(cfg)
+
+    return cfg
+
+
+def search_for_mirror(candidates):
+    """
+    Search through a list of mirror urls for one that works
+    This needs to return quickly.
+    """
+    if candidates is None:
+        return None
+
+    LOG.debug("search for mirror in candidates: '%s'", candidates)
+    for cand in candidates:
+        try:
+            if util.is_resolvable_url(cand):
+                LOG.debug("found working mirror: '%s'", cand)
+                return cand
+        except Exception:
+            pass
+    return None
+
+
+def search_for_mirror_dns(configured, mirrortype, cfg, cloud):
+    """
+    Try to resolve a list of predefines DNS names to pick mirrors
+    """
+    mirror = None
 
-    if (not mirror and
-            util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)):
+    if configured:
         mydom = ""
         doms = []
 
+        if mirrortype == "primary":
+            mirrordns = "mirror"
+        elif mirrortype == "security":
+            mirrordns = "security-mirror"
+        else:
+            raise ValueError("unknown mirror type")
+
         # if we have a fqdn, then search its domain portion first
-        (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud)
+        (_, fqdn) = util.get_hostname_fqdn(cfg, cloud)
         mydom = ".".join(fqdn.split(".")[1:])
         if mydom:
             doms.append(".%s" % mydom)
@@ -282,38 +589,136 @@ def find_apt_mirror_info(cloud, cfg):
 
         mirror_list = []
         distro = cloud.distro.name
-        mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro)
+        mirrorfmt = "http://%s-%s%s/%s" % (distro, mirrordns, "%s", distro)
         for post in doms:
             mirror_list.append(mirrorfmt % (post))
 
-        mirror = util.search_for_mirror(mirror_list)
+        mirror = search_for_mirror(mirror_list)
+
+    return mirror
 
+
+def update_mirror_info(pmirror, smirror, arch, cloud):
+    """sets security mirror to primary if not defined.
+       returns defaults if no mirrors are defined"""
+    if pmirror is not None:
+        if smirror is None:
+            smirror = pmirror
+        return {'PRIMARY': pmirror,
+                'SECURITY': smirror}
+
+    # None specified at all, get default mirrors from cloud
     mirror_info = cloud.datasource.get_package_mirror_info()
+    if mirror_info:
+        # get_package_mirror_info() returns a dictionary with
+        # arbitrary key/value pairs including 'primary' and 'security' keys.
+        # caller expects dict with PRIMARY and SECURITY.
+        m = mirror_info.copy()
+        m['PRIMARY'] = m['primary']
+        m['SECURITY'] = m['security']
+
+        return m
+
+    # if neither apt nor cloud configured mirrors fall back to
+    return get_default_mirrors(arch)
+
+
+def get_arch_mirrorconfig(cfg, mirrortype, arch):
+    """out of a list of potential mirror configurations select
+       and return the one matching the architecture (or default)"""
+    # select the mirror specification (if-any)
+    mirror_cfg_list = cfg.get(mirrortype, None)
+    if mirror_cfg_list is None:
+        return None
+
+    # select the specification matching the target arch
+    default = None
+    for mirror_cfg_elem in mirror_cfg_list:
+        arches = mirror_cfg_elem.get("arches")
+        if arch in arches:
+            return mirror_cfg_elem
+        if "default" in arches:
+            default = mirror_cfg_elem
+    return default
+
+
+def get_mirror(cfg, mirrortype, arch, cloud):
+    """pass the three potential stages of mirror specification
+       returns None is neither of them found anything otherwise the first
+       hit is returned"""
+    mcfg = get_arch_mirrorconfig(cfg, mirrortype, arch)
+    if mcfg is None:
+        return None
+
+    # directly specified
+    mirror = mcfg.get("uri", None)
+
+    # fallback to search if specified
+    if mirror is None:
+        # list of mirrors to try to resolve
+        mirror = search_for_mirror(mcfg.get("search", None))
+
+    # fallback to search_dns if specified
+    if mirror is None:
+        # list of mirrors to try to resolve
+        mirror = search_for_mirror_dns(mcfg.get("search_dns", None),
+                                       mirrortype, cfg, cloud)
+
+    return mirror
+
+
+def find_apt_mirror_info(cfg, cloud, arch=None):
+    """find_apt_mirror_info
+       find an apt_mirror given the cfg provided.
+       It can check for separate config of primary and security mirrors
+       If only primary is given security is assumed to be equal to primary
+       If the generic apt_mirror is given that is defining for both
+    """
 
-    # this is a bit strange.
-    # if mirror is set, then one of the legacy options above set it
-    # but they do not cover security. so we need to get that from
-    # get_package_mirror_info
-    if mirror:
-        mirror_info.update({'primary': mirror})
+    if arch is None:
+        arch = util.get_architecture()
+        LOG.debug("got arch for mirror selection: %s", arch)
+    pmirror = get_mirror(cfg, "primary", arch, cloud)
+    LOG.debug("got primary mirror: %s", pmirror)
+    smirror = get_mirror(cfg, "security", arch, cloud)
+    LOG.debug("got security mirror: %s", smirror)
+
+    mirror_info = update_mirror_info(pmirror, smirror, arch, cloud)
+
+    # less complex replacements use only MIRROR, derive from primary
+    mirror_info["MIRROR"] = mirror_info["PRIMARY"]
 
     return mirror_info
 
 
 def apply_apt_config(cfg, proxy_fname, config_fname):
+    """apply_apt_config
+       Applies any apt*proxy config from if specified
+    """
     # Set up any apt proxy
-    cfgs = (('apt_proxy', 'Acquire::HTTP::Proxy "%s";'),
-            ('apt_http_proxy', 'Acquire::HTTP::Proxy "%s";'),
-            ('apt_ftp_proxy', 'Acquire::FTP::Proxy "%s";'),
-            ('apt_https_proxy', 'Acquire::HTTPS::Proxy "%s";'))
+    cfgs = (('proxy', 'Acquire::http::Proxy "%s";'),
+            ('http_proxy', 'Acquire::http::Proxy "%s";'),
+            ('ftp_proxy', 'Acquire::ftp::Proxy "%s";'),
+            ('https_proxy', 'Acquire::https::Proxy "%s";'))
 
     proxies = [fmt % cfg.get(name) for (name, fmt) in cfgs if cfg.get(name)]
     if len(proxies):
+        LOG.debug("write apt proxy info to %s", proxy_fname)
         util.write_file(proxy_fname, '\n'.join(proxies) + '\n')
     elif os.path.isfile(proxy_fname):
         util.del_file(proxy_fname)
+        LOG.debug("no apt proxy configured, removed %s", proxy_fname)
 
-    if cfg.get('apt_config', None):
-        util.write_file(config_fname, cfg.get('apt_config'))
+    if cfg.get('conf', None):
+        LOG.debug("write apt config info to %s", config_fname)
+        util.write_file(config_fname, cfg.get('conf'))
     elif os.path.isfile(config_fname):
         util.del_file(config_fname)
+        LOG.debug("no apt config configured, removed %s", config_fname)
+
+
+CONFIG_CLEANERS = {
+    'cloud-init': clean_cloud_init,
+}
+
+# vi: ts=4 expandtab syntax=python
diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py
index 70d4e7c3..0086840f 100644
--- a/cloudinit/config/cc_lxd.py
+++ b/cloudinit/config/cc_lxd.py
@@ -47,6 +47,8 @@ Example config:
 
 from cloudinit import util
 
+distros = ['ubuntu']
+
 
 def handle(name, cfg, cloud, log, args):
     # Get config
diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py
index 0c84d600..b3089f30 100644
--- a/cloudinit/config/cc_mcollective.py
+++ b/cloudinit/config/cc_mcollective.py
@@ -19,6 +19,8 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import errno
+
 import six
 from six import BytesIO
 
@@ -36,49 +38,61 @@ SERVER_CFG = '/etc/mcollective/server.cfg'
 LOG = logging.getLogger(__name__)
 
 
-def configure(config):
-    # Read server.cfg values from the
-    # original file in order to be able to mix the rest up
+def configure(config, server_cfg=SERVER_CFG,
+              pubcert_file=PUBCERT_FILE, pricert_file=PRICERT_FILE):
+    # Read server.cfg (if it exists) values from the
+    # original file in order to be able to mix the rest up.
     try:
-        mcollective_config = ConfigObj(SERVER_CFG, file_error=True)
-    except IOError:
-        LOG.warn("Did not find file %s", SERVER_CFG)
-        mcollective_config = ConfigObj(config)
-    else:
-        for (cfg_name, cfg) in config.items():
-            if cfg_name == 'public-cert':
-                util.write_file(PUBCERT_FILE, cfg, mode=0o644)
-                mcollective_config[
-                    'plugin.ssl_server_public'] = PUBCERT_FILE
-                mcollective_config['securityprovider'] = 'ssl'
-            elif cfg_name == 'private-cert':
-                util.write_file(PRICERT_FILE, cfg, mode=0o600)
-                mcollective_config[
-                    'plugin.ssl_server_private'] = PRICERT_FILE
-                mcollective_config['securityprovider'] = 'ssl'
+        old_contents = util.load_file(server_cfg, quiet=False, decode=False)
+        mcollective_config = ConfigObj(BytesIO(old_contents))
+    except IOError as e:
+        if e.errno != errno.ENOENT:
+            raise
+        else:
+            LOG.debug("Did not find file %s (starting with an empty"
+                      " config)", server_cfg)
+            mcollective_config = ConfigObj()
+    for (cfg_name, cfg) in config.items():
+        if cfg_name == 'public-cert':
+            util.write_file(pubcert_file, cfg, mode=0o644)
+            mcollective_config[
+                'plugin.ssl_server_public'] = pubcert_file
+            mcollective_config['securityprovider'] = 'ssl'
+        elif cfg_name == 'private-cert':
+            util.write_file(pricert_file, cfg, mode=0o600)
+            mcollective_config[
+                'plugin.ssl_server_private'] = pricert_file
+            mcollective_config['securityprovider'] = 'ssl'
+        else:
+            if isinstance(cfg, six.string_types):
+                # Just set it in the 'main' section
+                mcollective_config[cfg_name] = cfg
+            elif isinstance(cfg, (dict)):
+                # Iterate through the config items, create a section if
+                # it is needed and then add/or create items as needed
+                if cfg_name not in mcollective_config.sections:
+                    mcollective_config[cfg_name] = {}
+                for (o, v) in cfg.items():
+                    mcollective_config[cfg_name][o] = v
             else:
-                if isinstance(cfg, six.string_types):
-                    # Just set it in the 'main' section
-                    mcollective_config[cfg_name] = cfg
-                elif isinstance(cfg, (dict)):
-                    # Iterate through the config items, create a section if
-                    # it is needed and then add/or create items as needed
-                    if cfg_name not in mcollective_config.sections:
-                        mcollective_config[cfg_name] = {}
-                    for (o, v) in cfg.items():
-                        mcollective_config[cfg_name][o] = v
-                else:
-                    # Otherwise just try to convert it to a string
-                    mcollective_config[cfg_name] = str(cfg)
-        # We got all our config as wanted we'll rename
-        # the previous server.cfg and create our new one
-        util.rename(SERVER_CFG, "%s.old" % (SERVER_CFG))
-
-    # Now we got the whole file, write to disk...
+                # Otherwise just try to convert it to a string
+                mcollective_config[cfg_name] = str(cfg)
+
+    try:
+        # We got all our config as wanted we'll copy
+        # the previous server.cfg and overwrite the old with our new one
+        util.copy(server_cfg, "%s.old" % (server_cfg))
+    except IOError as e:
+        if e.errno == errno.ENOENT:
+            # Doesn't exist to copy...
+            pass
+        else:
+            raise
+
+    # Now we got the whole (new) file, write to disk...
     contents = BytesIO()
     mcollective_config.write(contents)
-    contents = contents.getvalue()
-    util.write_file(SERVER_CFG, contents, mode=0o644)
+    util.write_file(server_cfg, contents.getvalue(), mode=0o644)
 
 
 def handle(name, cfg, cloud, log, _args):
@@ -98,5 +112,5 @@ def handle(name, cfg, cloud, log, _args):
     if 'conf' in mcollective_cfg:
         configure(config=mcollective_cfg['conf'])
 
-    # Start mcollective
-    util.subp(['service', 'mcollective', 'start'], capture=False)
+    # restart mcollective to handle updated config
+    util.subp(['service', 'mcollective', 'restart'], capture=False)
diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py
new file mode 100644
index 00000000..ad69aa34
--- /dev/null
+++ b/cloudinit/config/cc_ntp.py
@@ -0,0 +1,106 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2016 Canonical Ltd.
+#
+#    Author: Ryan Harper <ryan.harper@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from cloudinit import log as logging
+from cloudinit.settings import PER_INSTANCE
+from cloudinit import templater
+from cloudinit import type_utils
+from cloudinit import util
+
+import os
+
+LOG = logging.getLogger(__name__)
+
+frequency = PER_INSTANCE
+NTP_CONF = '/etc/ntp.conf'
+NR_POOL_SERVERS = 4
+distros = ['centos', 'debian', 'fedora', 'opensuse', 'ubuntu']
+
+
+def handle(name, cfg, cloud, log, _args):
+    """
+    Enable and configure ntp
+
+    ntp:
+       pools: ['0.{{distro}}.pool.ntp.org', '1.{{distro}}.pool.ntp.org']
+       servers: ['192.168.2.1']
+
+    """
+
+    ntp_cfg = cfg.get('ntp', {})
+
+    if not isinstance(ntp_cfg, (dict)):
+        raise RuntimeError(("'ntp' key existed in config,"
+                            " but not a dictionary type,"
+                            " is a %s %instead"), type_utils.obj_name(ntp_cfg))
+
+    if 'ntp' not in cfg:
+        LOG.debug("Skipping module named %s,"
+                  "not present or disabled by cfg", name)
+        return True
+
+    install_ntp(cloud.distro.install_packages, packages=['ntp'],
+                check_exe="ntpd")
+    rename_ntp_conf()
+    write_ntp_config_template(ntp_cfg, cloud)
+
+
+def install_ntp(install_func, packages=None, check_exe="ntpd"):
+    if util.which(check_exe):
+        return
+    if packages is None:
+        packages = ['ntp']
+
+    install_func(packages)
+
+
+def rename_ntp_conf(config=NTP_CONF):
+    if os.path.exists(config):
+        util.rename(config, config + ".dist")
+
+
+def generate_server_names(distro):
+    names = []
+    for x in range(0, NR_POOL_SERVERS):
+        name = "%d.%s.pool.ntp.org" % (x, distro)
+        names.append(name)
+    return names
+
+
+def write_ntp_config_template(cfg, cloud):
+    servers = cfg.get('servers', [])
+    pools = cfg.get('pools', [])
+
+    if len(servers) == 0 and len(pools) == 0:
+        LOG.debug('Adding distro default ntp pool servers')
+        pools = generate_server_names(cloud.distro.name)
+
+    params = {
+        'servers': servers,
+        'pools': pools,
+    }
+
+    template_fn = cloud.get_template_filename('ntp.conf.%s' %
+                                              (cloud.distro.name))
+    if not template_fn:
+        template_fn = cloud.get_template_filename('ntp.conf')
+        if not template_fn:
+            raise RuntimeError(("No template found, "
+                                "not rendering %s"), NTP_CONF)
+
+    templater.render_to_file(template_fn, NTP_CONF, params)
diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py
index 72176d42..ae720bd2 100644
--- a/cloudinit/config/cc_phone_home.py
+++ b/cloudinit/config/cc_phone_home.py
@@ -31,7 +31,7 @@ POST_LIST_ALL = [
     'pub_key_ecdsa',
     'instance_id',
     'hostname',
-    'fdqn'
+    'fqdn'
 ]
 
 
diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py
index 3a113aea..d4ad724a 100644
--- a/cloudinit/config/cc_rh_subscription.py
+++ b/cloudinit/config/cc_rh_subscription.py
@@ -18,6 +18,8 @@
 
 from cloudinit import util
 
+distros = ['fedora', 'rhel']
+
 
 def handle(name, cfg, _cloud, log, _args):
     sm = SubscriptionManager(cfg)
diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py
index f5786a31..13d70c8e 100644
--- a/cloudinit/config/cc_salt_minion.py
+++ b/cloudinit/config/cc_salt_minion.py
@@ -46,7 +46,12 @@ def handle(name, cfg, cloud, log, _args):
 
     # ... copy the key pair if specified
     if 'public_key' in salt_cfg and 'private_key' in salt_cfg:
-        pki_dir = salt_cfg.get('pki_dir', '/etc/salt/pki')
+        if os.path.isdir("/etc/salt/pki/minion"):
+            pki_dir_default = "/etc/salt/pki/minion"
+        else:
+            pki_dir_default = "/etc/salt/pki"
+
+        pki_dir = salt_cfg.get('pki_dir', pki_dir_default)
         with util.umask(0o77):
             util.ensure_dir(pki_dir)
             pub_name = os.path.join(pki_dir, 'minion.pub')
diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py
index 1a485ee6..6bcd8382 100644
--- a/cloudinit/config/cc_snappy.py
+++ b/cloudinit/config/cc_snappy.py
@@ -68,6 +68,8 @@ BUILTIN_CFG = {
     'config': {},
 }
 
+distros = ['ubuntu']
+
 
 def parse_filename(fname):
     fname = os.path.basename(fname)
diff --git a/cloudinit/config/cc_spacewalk.py b/cloudinit/config/cc_spacewalk.py
new file mode 100644
index 00000000..f3c1a664
--- /dev/null
+++ b/cloudinit/config/cc_spacewalk.py
@@ -0,0 +1,85 @@
+# vi: ts=4 expandtab
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+**Summary:** helper to setup https://fedorahosted.org/spacewalk/
+
+**Description:** This module will enable for configuring the needed
+actions to setup spacewalk on redhat based systems.
+
+It can be configured with the following option structure::
+
+    spacewalk:
+       server: spacewalk api server (required)
+"""
+
+from cloudinit import util
+
+
+distros = ['redhat', 'fedora']
+required_packages = ['rhn-setup']
+def_ca_cert_path = "/usr/share/rhn/RHN-ORG-TRUSTED-SSL-CERT"
+
+
+def is_registered():
+    # Check to see if already registered and don't bother; this is
+    # apparently done by trying to sync and if that fails then we
+    # assume we aren't registered; which is sorta ghetto...
+    already_registered = False
+    try:
+        util.subp(['rhn-profile-sync', '--verbose'], capture=False)
+        already_registered = True
+    except util.ProcessExecutionError as e:
+        if e.exit_code != 1:
+            raise
+    return already_registered
+
+
+def do_register(server, profile_name,
+                ca_cert_path=def_ca_cert_path,
+                proxy=None, log=None,
+                activation_key=None):
+    if log is not None:
+        log.info("Registering using `rhnreg_ks` profile '%s'"
+                 " into server '%s'", profile_name, server)
+    cmd = ['rhnreg_ks']
+    cmd.extend(['--serverUrl', 'https://%s/XMLRPC' % server])
+    cmd.extend(['--profilename', str(profile_name)])
+    if proxy:
+        cmd.extend(["--proxy", str(proxy)])
+    if ca_cert_path:
+        cmd.extend(['--sslCACert', str(ca_cert_path)])
+    if activation_key:
+        cmd.extend(['--activationkey', str(activation_key)])
+    util.subp(cmd, capture=False)
+
+
+def handle(name, cfg, cloud, log, _args):
+    if 'spacewalk' not in cfg:
+        log.debug(("Skipping module named %s,"
+                   " no 'spacewalk' key in configuration"), name)
+        return
+    cfg = cfg['spacewalk']
+    spacewalk_server = cfg.get('server')
+    if spacewalk_server:
+        # Need to have this installed before further things will work.
+        cloud.distro.install_packages(required_packages)
+        if not is_registered():
+            do_register(spacewalk_server,
+                        cloud.datasource.get_hostname(fqdn=True),
+                        proxy=cfg.get("proxy"), log=log,
+                        activation_key=cfg.get('activation_key'))
+    else:
+        log.debug("Skipping module named %s, 'spacewalk/server' key"
+                  " was not found in configuration", name)
diff --git a/cloudinit/config/cc_ubuntu_init_switch.py b/cloudinit/config/cc_ubuntu_init_switch.py
index 884d79f1..bffb4380 100644
--- a/cloudinit/config/cc_ubuntu_init_switch.py
+++ b/cloudinit/config/cc_ubuntu_init_switch.py
@@ -86,6 +86,8 @@ else
 fi
 """
 
+distros = ['ubuntu']
+
 
 def handle(name, cfg, cloud, log, args):
     """Handler method activated by cloud-init."""
diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py
index 64fba869..22549e62 100644
--- a/cloudinit/config/cc_yum_add_repo.py
+++ b/cloudinit/config/cc_yum_add_repo.py
@@ -23,6 +23,8 @@ import six
 
 from cloudinit import util
 
+distros = ['fedora', 'rhel']
+
 
 def _canonicalize_id(repo_id):
     repo_id = repo_id.lower().replace("-", "_")