18 files changed, 579 insertions, 352 deletions
diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py
index 1f00dd90..4a616e26 100644..100755
--- a/cloudinit/config/cc_byobu.py
+++ b/cloudinit/config/cc_byobu.py
@@ -50,12 +50,7 @@ Valid configuration options for this module are:
     byobu_by_default: <user/system>
 """
 
-
-# Ensure this is aliased to a name not 'distros'
-# since the module attribute 'distros'
-# is a list of distros that are supported, not a sub-module
-from cloudinit import distros as ds
-
+from cloudinit.distros import ug_util
 from cloudinit import util
 
 distros = ['ubuntu', 'debian']
@@ -94,8 +89,8 @@ def handle(name, cfg, cloud, log, args):
 
     shcmd = ""
     if mod_user:
-        (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro)
-        (user, _user_config) = ds.extract_default(users)
+        (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro)
+        (user, _user_config) = ug_util.extract_default(users)
         if not user:
             log.warn(("No default byobu user provided, "
                       "can not launch %s for the default user"), bl_inst)
diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py
index efa7a226..0c4b794d 100644
--- a/cloudinit/config/cc_disk_setup.py
+++ b/cloudinit/config/cc_disk_setup.py
@@ -716,6 +716,8 @@ def exec_mkpart_gpt(device, layout):
         LOG.warn("Failed to partition device %s" % device)
         raise
 
+    read_parttbl(device)
+
 
 def exec_mkpart(table_type, device, layout):
     """
diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py
index 48f18620..1c12e567 100644
--- a/cloudinit/config/cc_rsyslog.py
+++ b/cloudinit/config/cc_rsyslog.py
@@ -18,6 +18,8 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
+.. _cc_rsyslog:
+
 Rsyslog
 -------
 **Summary:** configure system loggig via rsyslog
diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py
index 94716017..6fc00517 100644..100755
--- a/cloudinit/config/cc_set_passwords.py
+++ b/cloudinit/config/cc_set_passwords.py
@@ -66,11 +66,7 @@ enabled, disabled, or left to system defaults using ``ssh_pwauth``.
 
 import sys
 
-# Ensure this is aliased to a name not 'distros'
-# since the module attribute 'distros'
-# is a list of distros that are supported, not a sub-module
-from cloudinit import distros as ds
-
+from cloudinit.distros import ug_util
 from cloudinit import ssh_util
 from cloudinit import util
 
@@ -99,8 +95,8 @@ def handle(_name, cfg, cloud, log, args):
         expire = util.get_cfg_option_bool(chfg, 'expire', expire)
 
     if not plist and password:
-        (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro)
-        (user, _user_config) = ds.extract_default(users)
+        (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro)
+        (user, _user_config) = ug_util.extract_default(users)
         if user:
             plist = "%s:%s" % (user, password)
         else:
diff --git a/cloudinit/config/cc_snap_config.py b/cloudinit/config/cc_snap_config.py
new file mode 100644
index 00000000..275a2d09
--- /dev/null
+++ b/cloudinit/config/cc_snap_config.py
@@ -0,0 +1,184 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2016 Canonical Ltd.
+#
+#    Author: Ryan Harper <ryan.harper@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Snappy
+------
+**Summary:** snap_config modules allows configuration of snapd.
+
+This module uses the same ``snappy`` namespace for configuration but
+acts only only a subset of the configuration.
+
+If ``assertions`` is set and the user has included a list of assertions
+then cloud-init will collect the assertions into a single assertion file
+and invoke ``snap ack <path to file with assertions>`` which will attempt
+to load the provided assertions into the snapd assertion database.
+
+If ``email`` is set, this value is used to create an authorized user for
+contacting and installing snaps from the Ubuntu Store.  This is done by
+calling ``snap create-user`` command.
+
+If ``known`` is set to True, then it is expected the user also included
+an assertion of type ``system-user``.  When ``snap create-user`` is called
+cloud-init will append '--known' flag which instructs snapd to look for
+a system-user assertion with the details.  If ``known`` is not set, then
+``snap create-user`` will contact the Ubuntu SSO for validating and importing
+a system-user for the instance.
+
+.. note::
+    If the system is already managed, then cloud-init will not attempt to
+    create a system-user.
+
+**Internal name:** ``cc_snap_config``
+
+**Module frequency:** per instance
+
+**Supported distros:** any with 'snapd' available
+
+**Config keys**::
+
+    #cloud-config
+    snappy:
+        assertions:
+        - |
+        <assertion 1>
+        - |
+        <assertion 2>
+        email: user@user.org
+        known: true
+
+"""
+
+from cloudinit import log as logging
+from cloudinit.settings import PER_INSTANCE
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+frequency = PER_INSTANCE
+SNAPPY_CMD = "snap"
+ASSERTIONS_FILE = "/var/lib/cloud/instance/snapd.assertions"
+
+
+"""
+snappy:
+  assertions:
+  - |
+  <snap assertion 1>
+  - |
+  <snap assertion 2>
+  email: foo@foo.io
+  known: true
+"""
+
+
+def add_assertions(assertions=None):
+    """Import list of assertions.
+
+    Import assertions by concatenating each assertion into a
+    string separated by a '\n'.  Write this string to a instance file and
+    then invoke `snap ack /path/to/file` and check for errors.
+    If snap exits 0, then all assertions are imported.
+    """
+    if not assertions:
+        assertions = []
+
+    if not isinstance(assertions, list):
+        raise ValueError('assertion parameter was not a list: %s', assertions)
+
+    snap_cmd = [SNAPPY_CMD, 'ack']
+    combined = "\n".join(assertions)
+    if len(combined) == 0:
+        raise ValueError("Assertion list is empty")
+
+    for asrt in assertions:
+        LOG.debug('Acking: %s', asrt.split('\n')[0:2])
+
+    util.write_file(ASSERTIONS_FILE, combined.encode('utf-8'))
+    util.subp(snap_cmd + [ASSERTIONS_FILE], capture=True)
+
+
+def add_snap_user(cfg=None):
+    """Add a snap system-user if provided with email under snappy config.
+
+      - Check that system is not already managed.
+      - Check that if using a system-user assertion, that it's
+        imported into snapd.
+
+    Returns a dictionary to be passed to Distro.create_user
+    """
+
+    if not cfg:
+        cfg = {}
+
+    if not isinstance(cfg, dict):
+        raise ValueError('configuration parameter was not a dict: %s', cfg)
+
+    snapuser = cfg.get('email', None)
+    if not snapuser:
+        return
+
+    usercfg = {
+        'snapuser': snapuser,
+        'known': cfg.get('known', False),
+    }
+
+    # query if we're already registered
+    out, _ = util.subp([SNAPPY_CMD, 'managed'], capture=True)
+    if out.strip() == "true":
+        LOG.warning('This device is already managed. '
+                    'Skipping system-user creation')
+        return
+
+    if usercfg.get('known'):
+        # Check that we imported a system-user assertion
+        out, _ = util.subp([SNAPPY_CMD, 'known', 'system-user'],
+                           capture=True)
+        if len(out) == 0:
+            LOG.error('Missing "system-user" assertion. '
+                      'Check "snappy" user-data assertions.')
+            return
+
+    return usercfg
+
+
+def handle(name, cfg, cloud, log, args):
+    cfgin = cfg.get('snappy')
+    if not cfgin:
+        LOG.debug('No snappy config provided, skipping')
+        return
+
+    if not(util.system_is_snappy()):
+        LOG.debug("%s: system not snappy", name)
+        return
+
+    assertions = cfgin.get('assertions', [])
+    if len(assertions) > 0:
+        LOG.debug('Importing user-provided snap assertions')
+        add_assertions(assertions)
+
+    # Create a snap user if requested.
+    # Snap systems contact the store with a user's email
+    # and extract information needed to create a local user.
+    # A user may provide a 'system-user' assertion which includes
+    # the required information. Using such an assertion to create
+    # a local user requires specifying 'known: true' in the supplied
+    # user-data.
+    usercfg = add_snap_user(cfg=cfgin)
+    if usercfg:
+        cloud.distro.create_user(usercfg.get('snapuser'), **usercfg)
diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py
index 36db9e67..e03ec483 100644
--- a/cloudinit/config/cc_snappy.py
+++ b/cloudinit/config/cc_snappy.py
@@ -257,24 +257,14 @@ def disable_enable_ssh(enabled):
         util.write_file(not_to_be_run, "cloud-init\n")
 
 
-def system_is_snappy():
-    # channel.ini is configparser loadable.
-    # snappy will move to using /etc/system-image/config.d/*.ini
-    # this is certainly not a perfect test, but good enough for now.
-    content = util.load_file("/etc/system-image/channel.ini", quiet=True)
-    if 'ubuntu-core' in content.lower():
-        return True
-    if os.path.isdir("/etc/system-image/config.d/"):
-        return True
-    return False
-
-
 def set_snappy_command():
     global SNAPPY_CMD
     if util.which("snappy-go"):
         SNAPPY_CMD = "snappy-go"
-    else:
+    elif util.which("snappy"):
         SNAPPY_CMD = "snappy"
+    else:
+        SNAPPY_CMD = "snap"
     LOG.debug("snappy command is '%s'", SNAPPY_CMD)
 
 
@@ -289,7 +279,7 @@ def handle(name, cfg, cloud, log, args):
         LOG.debug("%s: System is not snappy. disabling", name)
         return
 
-    if sys_snappy.lower() == "auto" and not(system_is_snappy()):
+    if sys_snappy.lower() == "auto" and not(util.system_is_snappy()):
         LOG.debug("%s: 'auto' mode, and system not snappy", name)
         return
 
diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py
index 6138fb53..576fa58a 100644..100755
--- a/cloudinit/config/cc_ssh.py
+++ b/cloudinit/config/cc_ssh.py
@@ -109,11 +109,7 @@ import glob
 import os
 import sys
 
-# Ensure this is aliased to a name not 'distros'
-# since the module attribute 'distros'
-# is a list of distros that are supported, not a sub-module
-from cloudinit import distros as ds
-
+from cloudinit.distros import ug_util
 from cloudinit import ssh_util
 from cloudinit import util
 
@@ -197,8 +193,8 @@ def handle(_name, cfg, cloud, log, _args):
                                     "file %s", keytype, keyfile)
 
     try:
-        (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro)
-        (user, _user_config) = ds.extract_default(users)
+        (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro)
+        (user, _user_config) = ug_util.extract_default(users)
         disable_root = util.get_cfg_option_bool(cfg, "disable_root", True)
         disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts",
                                                     DISABLE_ROOT_OPTS)
diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py
index 6f3d0ee2..7eeb0f84 100644..100755
--- a/cloudinit/config/cc_ssh_authkey_fingerprints.py
+++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py
@@ -42,11 +42,7 @@ import hashlib
 
 from prettytable import PrettyTable
 
-# Ensure this is aliased to a name not 'distros'
-# since the module attribute 'distros'
-# is a list of distros that are supported, not a sub-module
-from cloudinit import distros as ds
-
+from cloudinit.distros import ug_util
 from cloudinit import ssh_util
 from cloudinit import util
 
@@ -119,7 +115,7 @@ def handle(name, cfg, cloud, log, _args):
         return
 
     hash_meth = util.get_cfg_option_str(cfg, "authkey_hash", "md5")
-    (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro)
+    (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro)
     for (user_name, _cfg) in users.items():
         (key_fn, key_entries) = ssh_util.extract_authorized_keys(user_name)
         _pprint_key_entries(user_name, key_fn,
diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py
index 99359c87..1be96dc5 100644..100755
--- a/cloudinit/config/cc_ssh_import_id.py
+++ b/cloudinit/config/cc_ssh_import_id.py
@@ -42,11 +42,7 @@ either ``lp:`` for launchpad or ``gh:`` for github to the username.
         - lp:user
 """
 
-# Ensure this is aliased to a name not 'distros'
-# since the module attribute 'distros'
-# is a list of distros that are supported, not a sub-module
-from cloudinit import distros as ds
-
+from cloudinit.distros import ug_util
 from cloudinit import util
 import pwd
 
@@ -67,7 +63,7 @@ def handle(_name, cfg, cloud, log, args):
         return
 
     # import for cloudinit created users
-    (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro)
+    (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro)
     elist = []
     for (user, user_cfg) in users.items():
         import_ids = []
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index b1192e84..4a726430 100644..100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -367,6 +367,9 @@ class Distro(object):
 
         adduser_cmd = ['useradd', name]
         log_adduser_cmd = ['useradd', name]
+        if util.system_is_snappy():
+            adduser_cmd.append('--extrausers')
+            log_adduser_cmd.append('--extrausers')
 
         # Since we are creating users, we want to carefully validate the
         # inputs. If something goes wrong, we can end up with a system
@@ -445,6 +448,32 @@ class Distro(object):
             util.logexc(LOG, "Failed to create user %s", name)
             raise e
 
+    def add_snap_user(self, name, **kwargs):
+        """
+        Add a snappy user to the system using snappy tools
+        """
+
+        snapuser = kwargs.get('snapuser')
+        known = kwargs.get('known', False)
+        adduser_cmd = ["snap", "create-user", "--sudoer", "--json"]
+        if known:
+            adduser_cmd.append("--known")
+        adduser_cmd.append(snapuser)
+
+        # Run the command
+        LOG.debug("Adding snap user %s", name)
+        try:
+            (out, err) = util.subp(adduser_cmd, logstring=adduser_cmd,
+                                   capture=True)
+            LOG.debug("snap create-user returned: %s:%s", out, err)
+            jobj = util.load_json(out)
+            username = jobj.get('username', None)
+        except Exception as e:
+            util.logexc(LOG, "Failed to create snap user %s", name)
+            raise e
+
+        return username
+
     def create_user(self, name, **kwargs):
         """
         Creates users for the system using the GNU passwd tools. This
@@ -452,6 +481,10 @@ class Distro(object):
         distros where useradd is not desirable or not available.
         """
 
+        # Add a snap user, if requested
+        if 'snapuser' in kwargs:
+            return self.add_snap_user(name, **kwargs)
+
         # Add the user
         self.add_user(name, **kwargs)
 
@@ -602,6 +635,8 @@ class Distro(object):
 
     def create_group(self, name, members=None):
         group_add_cmd = ['groupadd', name]
+        if util.system_is_snappy():
+            group_add_cmd.append('--extrausers')
         if not members:
             members = []
 
@@ -685,275 +720,6 @@ def _get_arch_package_mirror_info(package_mirrors, arch):
     return default
 
 
-# Normalizes a input group configuration
-# which can be a comma seperated list of
-# group names, or a list of group names
-# or a python dictionary of group names
-# to a list of members of that group.
-#
-# The output is a dictionary of group
-# names => members of that group which
-# is the standard form used in the rest
-# of cloud-init
-def _normalize_groups(grp_cfg):
-    if isinstance(grp_cfg, six.string_types):
-        grp_cfg = grp_cfg.strip().split(",")
-    if isinstance(grp_cfg, list):
-        c_grp_cfg = {}
-        for i in grp_cfg:
-            if isinstance(i, dict):
-                for k, v in i.items():
-                    if k not in c_grp_cfg:
-                        if isinstance(v, list):
-                            c_grp_cfg[k] = list(v)
-                        elif isinstance(v, six.string_types):
-                            c_grp_cfg[k] = [v]
-                        else:
-                            raise TypeError("Bad group member type %s" %
-                                            type_utils.obj_name(v))
-                    else:
-                        if isinstance(v, list):
-                            c_grp_cfg[k].extend(v)
-                        elif isinstance(v, six.string_types):
-                            c_grp_cfg[k].append(v)
-                        else:
-                            raise TypeError("Bad group member type %s" %
-                                            type_utils.obj_name(v))
-            elif isinstance(i, six.string_types):
-                if i not in c_grp_cfg:
-                    c_grp_cfg[i] = []
-            else:
-                raise TypeError("Unknown group name type %s" %
-                                type_utils.obj_name(i))
-        grp_cfg = c_grp_cfg
-    groups = {}
-    if isinstance(grp_cfg, dict):
-        for (grp_name, grp_members) in grp_cfg.items():
-            groups[grp_name] = util.uniq_merge_sorted(grp_members)
-    else:
-        raise TypeError(("Group config must be list, dict "
-                         " or string types only and not %s") %
-                        type_utils.obj_name(grp_cfg))
-    return groups
-
-
-# Normalizes a input group configuration
-# which can be a comma seperated list of
-# user names, or a list of string user names
-# or a list of dictionaries with components
-# that define the user config + 'name' (if
-# a 'name' field does not exist then the
-# default user is assumed to 'own' that
-# configuration.
-#
-# The output is a dictionary of user
-# names => user config which is the standard
-# form used in the rest of cloud-init. Note
-# the default user will have a special config
-# entry 'default' which will be marked as true
-# all other users will be marked as false.
-def _normalize_users(u_cfg, def_user_cfg=None):
-    if isinstance(u_cfg, dict):
-        ad_ucfg = []
-        for (k, v) in u_cfg.items():
-            if isinstance(v, (bool, int, float) + six.string_types):
-                if util.is_true(v):
-                    ad_ucfg.append(str(k))
-            elif isinstance(v, dict):
-                v['name'] = k
-                ad_ucfg.append(v)
-            else:
-                raise TypeError(("Unmappable user value type %s"
-                                 " for key %s") % (type_utils.obj_name(v), k))
-        u_cfg = ad_ucfg
-    elif isinstance(u_cfg, six.string_types):
-        u_cfg = util.uniq_merge_sorted(u_cfg)
-
-    users = {}
-    for user_config in u_cfg:
-        if isinstance(user_config, (list,) + six.string_types):
-            for u in util.uniq_merge(user_config):
-                if u and u not in users:
-                    users[u] = {}
-        elif isinstance(user_config, dict):
-            if 'name' in user_config:
-                n = user_config.pop('name')
-                prev_config = users.get(n) or {}
-                users[n] = util.mergemanydict([prev_config,
-                                               user_config])
-            else:
-                # Assume the default user then
-                prev_config = users.get('default') or {}
-                users['default'] = util.mergemanydict([prev_config,
-                                                       user_config])
-        else:
-            raise TypeError(("User config must be dictionary/list "
-                             " or string types only and not %s") %
-                            type_utils.obj_name(user_config))
-
-    # Ensure user options are in the right python friendly format
-    if users:
-        c_users = {}
-        for (uname, uconfig) in users.items():
-            c_uconfig = {}
-            for (k, v) in uconfig.items():
-                k = k.replace('-', '_').strip()
-                if k:
-                    c_uconfig[k] = v
-            c_users[uname] = c_uconfig
-        users = c_users
-
-    # Fixup the default user into the real
-    # default user name and replace it...
-    def_user = None
-    if users and 'default' in users:
-        def_config = users.pop('default')
-        if def_user_cfg:
-            # Pickup what the default 'real name' is
-            # and any groups that are provided by the
-            # default config
-            def_user_cfg = def_user_cfg.copy()
-            def_user = def_user_cfg.pop('name')
-            def_groups = def_user_cfg.pop('groups', [])
-            # Pickup any config + groups for that user name
-            # that we may have previously extracted
-            parsed_config = users.pop(def_user, {})
-            parsed_groups = parsed_config.get('groups', [])
-            # Now merge our extracted groups with
-            # anything the default config provided
-            users_groups = util.uniq_merge_sorted(parsed_groups, def_groups)
-            parsed_config['groups'] = ",".join(users_groups)
-            # The real config for the default user is the
-            # combination of the default user config provided
-            # by the distro, the default user config provided
-            # by the above merging for the user 'default' and
-            # then the parsed config from the user's 'real name'
-            # which does not have to be 'default' (but could be)
-            users[def_user] = util.mergemanydict([def_user_cfg,
-                                                  def_config,
-                                                  parsed_config])
-
-    # Ensure that only the default user that we
-    # found (if any) is actually marked as being
-    # the default user
-    if users:
-        for (uname, uconfig) in users.items():
-            if def_user and uname == def_user:
-                uconfig['default'] = True
-            else:
-                uconfig['default'] = False
-
-    return users
-
-
-# Normalizes a set of user/users and group
-# dictionary configuration into a useable
-# format that the rest of cloud-init can
-# understand using the default user
-# provided by the input distrobution (if any)
-# to allow for mapping of the 'default' user.
-#
-# Output is a dictionary of group names -> [member] (list)
-# and a dictionary of user names -> user configuration (dict)
-#
-# If 'user' exists it will override
-# the 'users'[0] entry (if a list) otherwise it will
-# just become an entry in the returned dictionary (no override)
-def normalize_users_groups(cfg, distro):
-    if not cfg:
-        cfg = {}
-
-    users = {}
-    groups = {}
-    if 'groups' in cfg:
-        groups = _normalize_groups(cfg['groups'])
-
-    # Handle the previous style of doing this where the first user
-    # overrides the concept of the default user if provided in the user: XYZ
-    # format.
-    old_user = {}
-    if 'user' in cfg and cfg['user']:
-        old_user = cfg['user']
-        # Translate it into the format that is more useful
-        # going forward
-        if isinstance(old_user, six.string_types):
-            old_user = {
-                'name': old_user,
-            }
-        if not isinstance(old_user, dict):
-            LOG.warn(("Format for 'user' key must be a string or "
-                      "dictionary and not %s"), type_utils.obj_name(old_user))
-            old_user = {}
-
-    # If no old user format, then assume the distro
-    # provides what the 'default' user maps to, but notice
-    # that if this is provided, we won't automatically inject
-    # a 'default' user into the users list, while if a old user
-    # format is provided we will.
-    distro_user_config = {}
-    try:
-        distro_user_config = distro.get_default_user()
-    except NotImplementedError:
-        LOG.warn(("Distro has not implemented default user "
-                  "access. No distribution provided default user"
-                  " will be normalized."))
-
-    # Merge the old user (which may just be an empty dict when not
-    # present with the distro provided default user configuration so
-    # that the old user style picks up all the distribution specific
-    # attributes (if any)
-    default_user_config = util.mergemanydict([old_user, distro_user_config])
-
-    base_users = cfg.get('users', [])
-    if not isinstance(base_users, (list, dict) + six.string_types):
-        LOG.warn(("Format for 'users' key must be a comma separated string"
-                  " or a dictionary or a list and not %s"),
-                 type_utils.obj_name(base_users))
-        base_users = []
-
-    if old_user:
-        # Ensure that when user: is provided that this user
-        # always gets added (as the default user)
-        if isinstance(base_users, list):
-            # Just add it on at the end...
-            base_users.append({'name': 'default'})
-        elif isinstance(base_users, dict):
-            base_users['default'] = dict(base_users).get('default', True)
-        elif isinstance(base_users, six.string_types):
-            # Just append it on to be re-parsed later
-            base_users += ",default"
-
-    users = _normalize_users(base_users, default_user_config)
-    return (users, groups)
-
-
-# Given a user dictionary config it will
-# extract the default user name and user config
-# from that list and return that tuple or
-# return (None, None) if no default user is
-# found in the given input
-def extract_default(users, default_name=None, default_config=None):
-    if not users:
-        users = {}
-
-    def safe_find(entry):
-        config = entry[1]
-        if not config or 'default' not in config:
-            return False
-        else:
-            return config['default']
-
-    tmp_users = users.items()
-    tmp_users = dict(filter(safe_find, tmp_users))
-    if not tmp_users:
-        return (default_name, default_config)
-    else:
-        name = list(tmp_users)[0]
-        config = tmp_users[name]
-        config.pop('default', None)
-        return (name, config)
-
-
 def fetch(name):
     locs, looked_locs = importer.find_module(name, ['', __name__], ['Distro'])
     if not locs:
diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py
index 1aa42d75..e574e1b9 100644
--- a/cloudinit/distros/rhel.py
+++ b/cloudinit/distros/rhel.py
@@ -62,6 +62,7 @@ class Distro(distros.Distro):
         self._runner = helpers.Runners(paths)
         self.osfamily = 'redhat'
         self._net_renderer = sysconfig.Renderer()
+        cfg['ssh_svcname'] = 'sshd'
 
     def install_packages(self, pkglist):
         self.package_command('install', pkgs=pkglist)
diff --git a/cloudinit/distros/ug_util.py b/cloudinit/distros/ug_util.py
new file mode 100755
index 00000000..99301530
--- /dev/null
+++ b/cloudinit/distros/ug_util.py
@@ -0,0 +1,299 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2012 Canonical Ltd.
+#    Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P.
+#    Copyright (C) 2012 Yahoo! Inc.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#    Author: Juerg Haefliger <juerg.haefliger@hp.com>
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#    Author: Ben Howard <ben.howard@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import six
+
+from cloudinit import log as logging
+from cloudinit import type_utils
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+# Normalizes a input group configuration
+# which can be a comma seperated list of
+# group names, or a list of group names
+# or a python dictionary of group names
+# to a list of members of that group.
+#
+# The output is a dictionary of group
+# names => members of that group which
+# is the standard form used in the rest
+# of cloud-init
+def _normalize_groups(grp_cfg):
+    if isinstance(grp_cfg, six.string_types):
+        grp_cfg = grp_cfg.strip().split(",")
+    if isinstance(grp_cfg, list):
+        c_grp_cfg = {}
+        for i in grp_cfg:
+            if isinstance(i, dict):
+                for k, v in i.items():
+                    if k not in c_grp_cfg:
+                        if isinstance(v, list):
+                            c_grp_cfg[k] = list(v)
+                        elif isinstance(v, six.string_types):
+                            c_grp_cfg[k] = [v]
+                        else:
+                            raise TypeError("Bad group member type %s" %
+                                            type_utils.obj_name(v))
+                    else:
+                        if isinstance(v, list):
+                            c_grp_cfg[k].extend(v)
+                        elif isinstance(v, six.string_types):
+                            c_grp_cfg[k].append(v)
+                        else:
+                            raise TypeError("Bad group member type %s" %
+                                            type_utils.obj_name(v))
+            elif isinstance(i, six.string_types):
+                if i not in c_grp_cfg:
+                    c_grp_cfg[i] = []
+            else:
+                raise TypeError("Unknown group name type %s" %
+                                type_utils.obj_name(i))
+        grp_cfg = c_grp_cfg
+    groups = {}
+    if isinstance(grp_cfg, dict):
+        for (grp_name, grp_members) in grp_cfg.items():
+            groups[grp_name] = util.uniq_merge_sorted(grp_members)
+    else:
+        raise TypeError(("Group config must be list, dict "
+                         " or string types only and not %s") %
+                        type_utils.obj_name(grp_cfg))
+    return groups
+
+
+# Normalizes a input group configuration
+# which can be a comma seperated list of
+# user names, or a list of string user names
+# or a list of dictionaries with components
+# that define the user config + 'name' (if
+# a 'name' field does not exist then the
+# default user is assumed to 'own' that
+# configuration.
+#
+# The output is a dictionary of user
+# names => user config which is the standard
+# form used in the rest of cloud-init. Note
+# the default user will have a special config
+# entry 'default' which will be marked as true
+# all other users will be marked as false.
+def _normalize_users(u_cfg, def_user_cfg=None):
+    if isinstance(u_cfg, dict):
+        ad_ucfg = []
+        for (k, v) in u_cfg.items():
+            if isinstance(v, (bool, int, float) + six.string_types):
+                if util.is_true(v):
+                    ad_ucfg.append(str(k))
+            elif isinstance(v, dict):
+                v['name'] = k
+                ad_ucfg.append(v)
+            else:
+                raise TypeError(("Unmappable user value type %s"
+                                 " for key %s") % (type_utils.obj_name(v), k))
+        u_cfg = ad_ucfg
+    elif isinstance(u_cfg, six.string_types):
+        u_cfg = util.uniq_merge_sorted(u_cfg)
+
+    users = {}
+    for user_config in u_cfg:
+        if isinstance(user_config, (list,) + six.string_types):
+            for u in util.uniq_merge(user_config):
+                if u and u not in users:
+                    users[u] = {}
+        elif isinstance(user_config, dict):
+            if 'name' in user_config:
+                n = user_config.pop('name')
+                prev_config = users.get(n) or {}
+                users[n] = util.mergemanydict([prev_config,
+                                               user_config])
+            else:
+                # Assume the default user then
+                prev_config = users.get('default') or {}
+                users['default'] = util.mergemanydict([prev_config,
+                                                       user_config])
+        else:
+            raise TypeError(("User config must be dictionary/list "
+                             " or string types only and not %s") %
+                            type_utils.obj_name(user_config))
+
+    # Ensure user options are in the right python friendly format
+    if users:
+        c_users = {}
+        for (uname, uconfig) in users.items():
+            c_uconfig = {}
+            for (k, v) in uconfig.items():
+                k = k.replace('-', '_').strip()
+                if k:
+                    c_uconfig[k] = v
+            c_users[uname] = c_uconfig
+        users = c_users
+
+    # Fixup the default user into the real
+    # default user name and replace it...
+    def_user = None
+    if users and 'default' in users:
+        def_config = users.pop('default')
+        if def_user_cfg:
+            # Pickup what the default 'real name' is
+            # and any groups that are provided by the
+            # default config
+            def_user_cfg = def_user_cfg.copy()
+            def_user = def_user_cfg.pop('name')
+            def_groups = def_user_cfg.pop('groups', [])
+            # Pickup any config + groups for that user name
+            # that we may have previously extracted
+            parsed_config = users.pop(def_user, {})
+            parsed_groups = parsed_config.get('groups', [])
+            # Now merge our extracted groups with
+            # anything the default config provided
+            users_groups = util.uniq_merge_sorted(parsed_groups, def_groups)
+            parsed_config['groups'] = ",".join(users_groups)
+            # The real config for the default user is the
+            # combination of the default user config provided
+            # by the distro, the default user config provided
+            # by the above merging for the user 'default' and
+            # then the parsed config from the user's 'real name'
+            # which does not have to be 'default' (but could be)
+            users[def_user] = util.mergemanydict([def_user_cfg,
+                                                  def_config,
+                                                  parsed_config])
+
+    # Ensure that only the default user that we
+    # found (if any) is actually marked as being
+    # the default user
+    if users:
+        for (uname, uconfig) in users.items():
+            if def_user and uname == def_user:
+                uconfig['default'] = True
+            else:
+                uconfig['default'] = False
+
+    return users
+
+
+# Normalizes a set of user/users and group
+# dictionary configuration into a useable
+# format that the rest of cloud-init can
+# understand using the default user
+# provided by the input distrobution (if any)
+# to allow for mapping of the 'default' user.
+#
+# Output is a dictionary of group names -> [member] (list)
+# and a dictionary of user names -> user configuration (dict)
+#
+# If 'user' exists it will override
+# the 'users'[0] entry (if a list) otherwise it will
+# just become an entry in the returned dictionary (no override)
+def normalize_users_groups(cfg, distro):
+    if not cfg:
+        cfg = {}
+
+    users = {}
+    groups = {}
+    if 'groups' in cfg:
+        groups = _normalize_groups(cfg['groups'])
+
+    # Handle the previous style of doing this where the first user
+    # overrides the concept of the default user if provided in the user: XYZ
+    # format.
+    old_user = {}
+    if 'user' in cfg and cfg['user']:
+        old_user = cfg['user']
+        # Translate it into the format that is more useful
+        # going forward
+        if isinstance(old_user, six.string_types):
+            old_user = {
+                'name': old_user,
+            }
+        if not isinstance(old_user, dict):
+            LOG.warn(("Format for 'user' key must be a string or "
+                      "dictionary and not %s"), type_utils.obj_name(old_user))
+            old_user = {}
+
+    # If no old user format, then assume the distro
+    # provides what the 'default' user maps to, but notice
+    # that if this is provided, we won't automatically inject
+    # a 'default' user into the users list, while if a old user
+    # format is provided we will.
+    distro_user_config = {}
+    try:
+        distro_user_config = distro.get_default_user()
+    except NotImplementedError:
+        LOG.warn(("Distro has not implemented default user "
+                  "access. No distribution provided default user"
+                  " will be normalized."))
+
+    # Merge the old user (which may just be an empty dict when not
+    # present with the distro provided default user configuration so
+    # that the old user style picks up all the distribution specific
+    # attributes (if any)
+    default_user_config = util.mergemanydict([old_user, distro_user_config])
+
+    base_users = cfg.get('users', [])
+    if not isinstance(base_users, (list, dict) + six.string_types):
+        LOG.warn(("Format for 'users' key must be a comma separated string"
+                  " or a dictionary or a list and not %s"),
+                 type_utils.obj_name(base_users))
+        base_users = []
+
+    if old_user:
+        # Ensure that when user: is provided that this user
+        # always gets added (as the default user)
+        if isinstance(base_users, list):
+            # Just add it on at the end...
+            base_users.append({'name': 'default'})
+        elif isinstance(base_users, dict):
+            base_users['default'] = dict(base_users).get('default', True)
+        elif isinstance(base_users, six.string_types):
+            # Just append it on to be re-parsed later
+            base_users += ",default"
+
+    users = _normalize_users(base_users, default_user_config)
+    return (users, groups)
+
+
+# Given a user dictionary config it will
+# extract the default user name and user config
+# from that list and return that tuple or
+# return (None, None) if no default user is
+# found in the given input
+def extract_default(users, default_name=None, default_config=None):
+    if not users:
+        users = {}
+
+    def safe_find(entry):
+        config = entry[1]
+        if not config or 'default' not in config:
+            return False
+        else:
+            return config['default']
+
+    tmp_users = users.items()
+    tmp_users = dict(filter(safe_find, tmp_users))
+    if not tmp_users:
+        return (default_name, default_config)
+    else:
+        name = list(tmp_users)[0]
+        config = tmp_users[name]
+        config.pop('default', None)
+        return (name, config)
diff --git a/cloudinit/settings.py b/cloudinit/settings.py
index 8c258ea1..a9682716 100644
--- a/cloudinit/settings.py
+++ b/cloudinit/settings.py
@@ -32,6 +32,7 @@ CFG_BUILTIN = {
         'NoCloud',
         'ConfigDrive',
         'OpenNebula',
+        'DigitalOcean',
         'Azure',
         'AltCloud',
         'OVF',
diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index 48136f7c..20345389 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -195,8 +195,7 @@ class DataSourceAltCloud(sources.DataSource):
             (cmd_out, _err) = util.subp(cmd)
             LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out))
         except ProcessExecutionError as _err:
-            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
-                        _err.message)
+            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err)
             return False
         except OSError as _err:
             util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err)
@@ -211,8 +210,7 @@ class DataSourceAltCloud(sources.DataSource):
             (cmd_out, _err) = util.subp(cmd)
             LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out))
         except ProcessExecutionError as _err:
-            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
-                        _err.message)
+            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err)
             return False
         except OSError as _err:
             util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index ab93c0a2..81abcd47 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -310,12 +310,12 @@ if __name__ == "__main__":
         creds = {'consumer_key': args.ckey, 'token_key': args.tkey,
                  'token_secret': args.tsec, 'consumer_secret': args.csec}
 
-        maaspkg_cfg = "/etc/cloud/cloud.cfg.d/90_dpkg_maas.cfg"
-        if (args.config is None and args.url is None and
-                os.path.exists(maaspkg_cfg) and
-                os.access(maaspkg_cfg, os.R_OK)):
-            sys.stderr.write("Used config in %s.\n" % maaspkg_cfg)
-            args.config = maaspkg_cfg
+        if args.config is None:
+            for fname in ('91_kernel_cmdline_url', '90_dpkg_maas'):
+                fpath = "/etc/cloud/cloud.cfg.d/" + fname + ".cfg"
+                if os.path.exists(fpath) and os.access(fpath, os.R_OK):
+                    sys.stderr.write("Used config in %s.\n" % fpath)
+                    args.config = fpath
 
         if args.config:
             cfg = util.read_conf(args.config)
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 635a836c..ba5f3f92 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -30,6 +30,7 @@ import re
 import string
 
 from cloudinit import log as logging
+from cloudinit import net
 from cloudinit import sources
 from cloudinit import util
 
@@ -120,17 +121,11 @@ class BrokenContextDiskDir(Exception):
 
 
 class OpenNebulaNetwork(object):
-    REG_DEV_MAC = re.compile(
-        r'^\d+: (eth\d+):.*?link\/ether (..:..:..:..:..:..) ?',
-        re.MULTILINE | re.DOTALL)
-
-    def __init__(self, ip, context):
-        self.ip = ip
+    def __init__(self, context, system_nics_by_mac=None):
         self.context = context
-        self.ifaces = self.get_ifaces()
-
-    def get_ifaces(self):
-        return self.REG_DEV_MAC.findall(self.ip)
+        if system_nics_by_mac is None:
+            system_nics_by_mac = get_physical_nics_by_mac()
+        self.ifaces = system_nics_by_mac
 
     def mac2ip(self, mac):
         components = mac.split(':')[2:]
@@ -188,9 +183,7 @@ class OpenNebulaNetwork(object):
         conf.append('iface lo inet loopback')
         conf.append('')
 
-        for i in self.ifaces:
-            dev = i[0]
-            mac = i[1]
+        for mac, dev in self.ifaces.items():
             ip_components = self.mac2ip(mac)
 
             conf.append('auto ' + dev)
@@ -405,16 +398,19 @@ def read_context_disk_dir(source_dir, asuser=None):
     # generate static /etc/network/interfaces
     # only if there are any required context variables
     # http://opennebula.org/documentation:rel3.8:cong#network_configuration
-    for k in context:
-        if re.match(r'^ETH\d+_IP$', k):
-            (out, _) = util.subp(['ip', 'link'])
-            net = OpenNebulaNetwork(out, context)
-            results['network-interfaces'] = net.gen_conf()
-            break
+    ipaddr_keys = [k for k in context if re.match(r'^ETH\d+_IP$', k)]
+    if ipaddr_keys:
+        onet = OpenNebulaNetwork(context)
+        results['network-interfaces'] = onet.gen_conf()
 
     return results
 
 
+def get_physical_nics_by_mac():
+    devs = net.get_interfaces_by_mac()
+    return dict([(m, n) for m, n in devs.items() if net.is_physical(n)])
+
+
 # Legacy: Must be present in case we load an old pkl object
 DataSourceOpenNebulaNet = DataSourceOpenNebula
 
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index 689ed4cc..1b3e9b70 100644
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -232,7 +232,7 @@ class WALinuxAgentShim(object):
     def _get_value_from_leases_file(fallback_lease_file):
         leases = []
         content = util.load_file(fallback_lease_file)
-        LOG.debug("content is {}".format(content))
+        LOG.debug("content is %s", content)
         for line in content.splitlines():
             if 'unknown-245' in line:
                 # Example line from Ubuntu
diff --git a/cloudinit/util.py b/cloudinit/util.py
index eb3e5899..4b3fd0cb 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -199,7 +199,7 @@ def fully_decoded_payload(part):
             encoding = charset.input_codec
         else:
             encoding = 'utf-8'
-        return cte_payload.decode(encoding, errors='surrogateescape')
+        return cte_payload.decode(encoding, 'surrogateescape')
     return cte_payload
 
 
@@ -282,9 +282,6 @@ class ProcessExecutionError(IOError):
             'reason': self.reason,
         }
         IOError.__init__(self, message)
-        # For backward compatibility with Python 2.
-        if not hasattr(self, 'message'):
-            self.message = message
 
 
 class SeLinuxGuard(object):
@@ -1821,7 +1818,7 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False,
             def ldecode(data, m='utf-8'):
                 if not isinstance(data, bytes):
                     return data
-                return data.decode(m, errors=decode)
+                return data.decode(m, decode)
 
             out = ldecode(out)
             err = ldecode(err)
@@ -2377,3 +2374,15 @@ def get_installed_packages(target=None):
             pkgs_inst.add(re.sub(":.*", "", pkg))
 
     return pkgs_inst
+
+
+def system_is_snappy():
+    # channel.ini is configparser loadable.
+    # snappy will move to using /etc/system-image/config.d/*.ini
+    # this is certainly not a perfect test, but good enough for now.
+    content = load_file("/etc/system-image/channel.ini", quiet=True)
+    if 'ubuntu-core' in content.lower():
+        return True
+    if os.path.isdir("/etc/system-image/config.d/"):
+        return True
+    return False