cleanup the user/group lists

The primary utility here is normalize_user_groups, which would
be called by config modules to get a list of users or groups.
This centralizes what was copied code into this one location.

6 files changed, 448 insertions, 115 deletions

diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py
index 23f5755a..2b9a6e0e 100644
--- a/cloudinit/config/cc_ssh_authkey_fingerprints.py
+++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py
@@ -21,6 +21,7 @@ import hashlib
 
 from prettytable import PrettyTable
 
+from cloudinit import distros
 from cloudinit import ssh_util
 from cloudinit import util
 
@@ -89,8 +90,9 @@ def handle(name, cfg, cloud, log, _args):
         log.debug(("Skipping module named %s, "
                    "logging of ssh fingerprints disabled"), name)
 
-    user_name = util.get_cfg_option_str(cfg, "user", "ubuntu")
     hash_meth = util.get_cfg_option_str(cfg, "authkey_hash", "md5")
-    extract = ssh_util.extract_authorized_keys
-    (auth_key_fn, auth_key_entries) = extract(user_name, cloud.paths)
-    _pprint_key_entries(user_name, auth_key_fn, auth_key_entries, hash_meth)
+    extract_func = ssh_util.extract_authorized_keys
+    (users, _groups) = distros.normalize_users_groups(cfg, cloud.distro)
+    for (user_name, _cfg) in users.items():
+        (auth_key_fn, auth_key_entries) = extract_func(user_name, cloud.paths)
+        _pprint_key_entries(user_name, auth_key_fn, auth_key_entries, hash_meth)
diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py
index 418f3330..464f55c3 100644
--- a/cloudinit/config/cc_users_groups.py
+++ b/cloudinit/config/cc_users_groups.py
@@ -16,63 +16,17 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from cloudinit import distros
+from cloudinit import util
+
 from cloudinit.settings import PER_INSTANCE
 
 frequency = PER_INSTANCE
 
 
 def handle(name, cfg, cloud, log, _args):
-    user_zero = None
-
-    if 'groups' in cfg:
-        for group in cfg['groups']:
-            if isinstance(group, dict):
-                for name, values in group.iteritems():
-                    if isinstance(values, list):
-                        cloud.distro.create_group(name, values)
-                    elif isinstance(values, str):
-                        cloud.distro.create_group(name, values.split(','))
-            else:
-                cloud.distro.create_group(group, [])
-
-    if 'users' in cfg:
-        user_zero = None
-
-        for user_config in cfg['users']:
-
-            # Handle the default user creation
-            if 'default' in user_config:
-                log.info("Creating default user")
-
-                # Create the default user if so defined
-                try:
-                    cloud.distro.add_default_user()
-
-                    if not user_zero:
-                        user_zero = cloud.distro.get_default_user()
-
-                except NotImplementedError:
-
-                    if user_zero == name:
-                        user_zero = None
-
-                    log.warn("Distro has not implemented default user "
-                             "creation. No default user will be created")
-
-            elif isinstance(user_config, dict) and 'name' in user_config:
-
-                name = user_config['name']
-                if not user_zero:
-                    user_zero = name
-
-                # Make options friendly for distro.create_user
-                new_opts = {}
-                if isinstance(user_config, dict):
-                    for opt in user_config:
-                        new_opts[opt.replace('-', '_')] = user_config[opt]
-
-                cloud.distro.create_user(**new_opts)
-
-            else:
-                # create user with no configuration
-                cloud.distro.create_user(user_config)
+    (users, groups) = distros.normalize_users_groups(cfg, cloud.distro)
+    for (name, members) in groups.items():
+        cloud.distro.create_group(name, members)
+    for (user, config) in users.items():
+        cloud.distro.create_user(user, **config)
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 549c1612..f07ba3fa 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -24,9 +24,7 @@
 from StringIO import StringIO
 
 import abc
-import grp
 import os
-import pwd
 import re
 
 from cloudinit import importer
@@ -48,34 +46,6 @@ class Distro(object):
         self._cfg = cfg
         self.name = name
 
-    def add_default_user(self):
-        # Adds the distro user using the rules:
-        #  - Password is same as username but is locked
-        #  - nopasswd sudo access
-
-        user = self.get_default_user()
-        groups = self.get_default_user_groups()
-
-        if not user:
-            raise NotImplementedError("No Default user")
-
-        user_dict = {
-                    'name': user,
-                    'plain_text_passwd': user,
-                    'home': "/home/%s" % user,
-                    'shell': "/bin/bash",
-                    'lock_passwd': True,
-                    'gecos': "%s%s" % (user[0:1].upper(), user[1:]),
-                    'sudo': "ALL=(ALL) NOPASSWD:ALL",
-                    }
-
-        if groups:
-            user_dict['groups'] = groups
-
-        self.create_user(**user_dict)
-
-        LOG.info("Added default '%s' user with passwordless sudo", user)
-
     @abc.abstractmethod
     def install_packages(self, pkglist):
         raise NotImplementedError()
@@ -205,18 +175,21 @@ class Distro(object):
             return True
         return False
 
-    def isuser(self, name):
-        try:
-            if pwd.getpwnam(name):
-                return True
-        except KeyError:
-            return False
-
     def get_default_user(self):
-        return self.default_user
-
-    def get_default_user_groups(self):
-        return self.default_user_groups
+        if not self.default_user:
+            return None
+        user_cfg = {
+            'name': self.default_user,
+            'plain_text_passwd': self.default_user,
+            'home': "/home/%s" % (self.default_user),
+            'shell': "/bin/bash",
+            'lock_passwd': True,
+            'gecos': "%s" % (self.default_user.title()),
+            'sudo': "ALL=(ALL) NOPASSWD:ALL",
+        }
+        if self.default_user_groups:
+            user_cfg['groups'] = _uniq_merge_sorted(self.default_user_groups)
+        return user_cfg
 
     def create_user(self, name, **kwargs):
         """
@@ -273,7 +246,7 @@ class Distro(object):
             adduser_cmd.append('-m')
 
         # Create the user
-        if self.isuser(name):
+        if util.is_user(name):
             LOG.warn("User %s already exists, skipping." % name)
         else:
             LOG.debug("Creating name %s" % name)
@@ -350,18 +323,11 @@ class Distro(object):
                 util.logexc(LOG, "Failed to write %s" % sudo_file, e)
                 raise e
 
-    def isgroup(self, name):
-        try:
-            if grp.getgrnam(name):
-                return True
-        except:
-            return False
-
     def create_group(self, name, members):
         group_add_cmd = ['groupadd', name]
 
         # Check if group exists, and then add it doesn't
-        if self.isgroup(name):
+        if util.is_group(name):
             LOG.warn("Skipping creation of existing group '%s'" % name)
         else:
             try:
@@ -373,7 +339,7 @@ class Distro(object):
         # Add members to the group, if so defined
         if len(members) > 0:
             for member in members:
-                if not self.isuser(member):
+                if not util.is_user(member):
                     LOG.warn("Unable to add group member '%s' to group '%s'"
                             "; user does not exist." % (member, name))
                     continue
@@ -431,6 +397,156 @@ def _get_arch_package_mirror_info(package_mirrors, arch):
     return default
 
 
+def _uniq_merge_sorted(*lists):
+    return sorted(_uniq_merge(*lists))
+
+
+def _uniq_merge(*lists):
+    combined_list = []
+    for a_list in lists:
+        if isinstance(a_list, (str, basestring)):
+            a_list = a_list.strip().split(",")
+        else:
+            a_list = [str(a) for a in a_list]
+        a_list = [a.strip() for a in a_list if a.strip()]
+        combined_list.extend(a_list)
+    uniq_list = []
+    for a in combined_list:
+        if a in uniq_list:
+            continue
+        else:
+            uniq_list.append(a)
+    return uniq_list
+
+
+def _normalize_groups(grp_cfg):
+    if isinstance(grp_cfg, (str, basestring, list)):
+        c_grp_cfg = {}
+        for i in _uniq_merge(grp_cfg):
+            c_grp_cfg[i] = []
+        grp_cfg = c_grp_cfg
+
+    groups = {}
+    if isinstance(grp_cfg, (dict)):
+        for (grp_name, grp_members) in grp_cfg.items():
+            groups[grp_name] = _uniq_merge_sorted(grp_members)
+    else:
+        raise TypeError(("Group config must be list, dict "
+                         " or string types only and not %s") %
+                        util.obj_name(grp_cfg))
+    return groups
+
+
+def _normalize_users(u_cfg, def_user_cfg=None):
+    if isinstance(u_cfg, (dict)):
+        ad_ucfg = []
+        for (k, v) in u_cfg.items():
+            if isinstance(v, (bool, int, basestring, str, float)):
+                if util.is_true(v):
+                    ad_ucfg.append(str(k))
+            elif isinstance(v, (dict)):
+                v['name'] = k
+                ad_ucfg.append(v)
+            else:
+                raise TypeError(("Unmappable user value type %s"
+                                 " for key %s") % (util.obj_name(v), k))
+        u_cfg = ad_ucfg
+    elif isinstance(u_cfg, (str, basestring)):
+        u_cfg = _uniq_merge_sorted(u_cfg)
+
+    users = {}
+    for user_config in u_cfg:
+        if isinstance(user_config, (str, basestring, list)):
+            for u in _uniq_merge(user_config):
+                if u and u not in users:
+                    users[u] = {}
+        elif isinstance(user_config, (dict)):
+            if 'name' in user_config:
+                n = user_config.pop('name')
+                prev_config = users.get(n) or {}
+                users[n] = util.mergemanydict([prev_config,
+                                               user_config])
+            else:
+                # Assume the default user then
+                prev_config = users.get('default') or {}
+                users['default'] = util.mergemanydict([prev_config,
+                                                       user_config])
+        else:
+            raise TypeError(("User config must be dictionary/list "
+                             " or string types only and not %s") %
+                            util.obj_name(user_config))
+
+    # Ensure user options are in the right python friendly format
+    if users:
+        c_users = {}
+        for (uname, uconfig) in users.items():
+            c_uconfig = {}
+            for (k, v) in uconfig.items():
+                k = k.replace('-', '_').strip()
+                if k:
+                    c_uconfig[k] = v
+            c_users[uname] = c_uconfig
+        users = c_users
+
+    # Fixup the default user into the real
+    # default user name and replace it...
+    if users and 'default' in users:
+        def_config = users.pop('default')
+        if def_user_cfg:
+            def_user = def_user_cfg.pop('name')
+            def_groups = def_user_cfg.pop('groups', [])
+            parsed_config = users.pop(def_user, {})
+            users_groups = _uniq_merge_sorted(parsed_config.get('groups', []),
+                                              def_groups)
+            parsed_config['groups'] = ",".join(users_groups)
+            users[def_user] = util.mergemanydict([def_user_cfg,
+                                                  def_config,
+                                                  parsed_config])
+
+    return users
+
+
+def normalize_users_groups(cfg, distro):
+    if not cfg:
+        cfg = {}
+    users = {}
+    groups = {}
+    if 'groups' in cfg:
+        groups = _normalize_groups(cfg['groups'])
+
+    # Handle the previous style of doing this...
+    old_user = None
+    if 'user' in cfg and cfg['user']:
+        old_user = str(cfg['user'])
+        if not 'users' in cfg:
+            cfg['users'] = old_user
+            old_user = None
+    if 'users' in cfg:
+        default_user_config = None
+        try:
+            default_user_config = distro.get_default_user()
+        except NotImplementedError:
+            LOG.warn(("Distro has not implemented default user "
+                      "access. No default user will be normalized."))
+        base_users = cfg['users']
+        if old_user:
+            if isinstance(base_users, (list)):
+                if len(base_users):
+                    # The old user replaces user[0]
+                    base_users[0] = {'name': old_user}
+                else:
+                    # Just add it on at the end...
+                    base_users.append({'name': old_user})
+            elif isinstance(base_users, (dict)):
+                if old_user not in base_users:
+                    base_users[old_user] = True
+            elif isinstance(base_users, (str, basestring)):
+                # Just append it on to be re-parsed later
+                base_users += ",%s" % (old_user)
+        users = _normalize_users(base_users, default_user_config)
+    return (users, groups)
+
+
 def fetch(name):
     locs = importer.find_module(name,
                                 ['', __name__],
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 33da73eb..94b17dfa 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -1104,6 +1104,22 @@ def hash_blob(blob, routine, mlen=None):
         return digest
 
 
+def is_user(name):
+    try:
+        if pwd.getpwnam(name):
+            return True
+    except KeyError:
+        return False
+
+
+def is_group(name):
+    try:
+        if grp.getgrnam(name):
+            return True
+    except KeyError:
+        return False
+
+
 def rename(src, dest):
     LOG.debug("Renaming %s to %s", src, dest)
     # TODO(harlowja) use a se guard here??
diff --git a/doc/examples/cloud-config-user-groups.txt b/doc/examples/cloud-config-user-groups.txt
index 1da0d717..1a46c540 100644
--- a/doc/examples/cloud-config-user-groups.txt
+++ b/doc/examples/cloud-config-user-groups.txt
@@ -1,11 +1,11 @@
-# add groups to the system
+# Add groups to the system
 # The following example adds the ubuntu group with members foo and bar and
 # the group cloud-users.
 groups:
   - ubuntu: [foo,bar]
   - cloud-users
 
-# add users to the system. Users are added after groups are added.
+# Add users to the system. Users are added after groups are added.
 users:
   - default
   - name: foobar
@@ -81,14 +81,18 @@ users:
 #               directive.
 #   system: Create the user as a system user. This means no home directory.
 #
-# Default user creation: Ubuntu Only
-# Unless you define users, you will get a Ubuntu user on Ubuntu systems with the
+
+# Default user creation:
+#
+# Unless you define users, you will get a 'ubuntu' user on ubuntu systems with the
 # legacy permission (no password sudo, locked user, etc). If however, you want
-# to have the ubuntu user in addition to other users, you need to instruct
+# to have the 'ubuntu' user in addition to other users, you need to instruct
 # cloud-init that you also want the default user. To do this use the following
 # syntax:
 #    users:
-#      default: True
+#      - default
+#      - bob
+#      - ....
 #  foobar: ...
 #
 # users[0] (the first user in users) overrides the user directive.
diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py
new file mode 100644
index 00000000..b319b673
--- /dev/null
+++ b/tests/unittests/test_distros/test_user_data_normalize.py
@@ -0,0 +1,241 @@
+from mocker import MockerTestCase
+
+from cloudinit import distros
+from cloudinit import helpers
+from cloudinit import settings
+
+
+class TestUGNormalize(MockerTestCase):
+
+    def _make_distro(self, dtype, def_user=None, def_groups=None):
+        cfg = dict(settings.CFG_BUILTIN)
+        cfg['system_info']['distro'] = dtype
+        paths = helpers.Paths(cfg['system_info']['paths'])
+        distro_cls = distros.fetch(dtype)
+        distro = distro_cls(dtype, cfg['system_info'], paths)
+        if def_user:
+            distro.default_user = def_user
+        if def_groups:
+            distro.default_user_groups = def_groups
+        return distro
+
+    def _norm(self, cfg, distro):
+        return distros.normalize_users_groups(cfg, distro)
+
+    def test_basic_groups(self):
+        distro = self._make_distro('ubuntu')
+        ug_cfg = {
+            'groups': ['bob'],
+        }
+        (users, groups) = self._norm(ug_cfg, distro)
+        self.assertIn('bob', groups)
+        self.assertEquals({}, users)
+
+    def test_csv_groups(self):
+        distro = self._make_distro('ubuntu')
+        ug_cfg = {
+            'groups': 'bob,joe,steve',
+        }
+        (users, groups) = self._norm(ug_cfg, distro)
+        self.assertIn('bob', groups)
+        self.assertIn('joe', groups)
+        self.assertIn('steve', groups)
+        self.assertEquals({}, users)
+
+    def test_more_groups(self):
+        distro = self._make_distro('ubuntu')
+        ug_cfg = {
+            'groups': ['bob', 'joe', 'steve']
+        }
+        (users, groups) = self._norm(ug_cfg, distro)
+        self.assertIn('bob', groups)
+        self.assertIn('joe', groups)
+        self.assertIn('steve', groups)
+        self.assertEquals({}, users)
+
+    def test_member_groups(self):
+        distro = self._make_distro('ubuntu')
+        ug_cfg = {
+            'groups': {
+                'bob': ['s'],
+                'joe': [],
+                'steve': [],
+            }
+        }
+        (users, groups) = self._norm(ug_cfg, distro)
+        self.assertIn('bob', groups)
+        self.assertEquals(['s'], groups['bob'])
+        self.assertEquals([], groups['joe'])
+        self.assertIn('joe', groups)
+        self.assertIn('steve', groups)
+        self.assertEquals({}, users)
+
+    def test_users_simple_dict(self):
+        distro = self._make_distro('ubuntu', 'bob')
+        ug_cfg = {
+            'users': {
+                'default': True,
+            }
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertIn('bob', users)
+        ug_cfg = {
+            'users': {
+                'default': 'yes',
+            }
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertIn('bob', users)
+        ug_cfg = {
+            'users': {
+                'default': '1',
+            }
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertIn('bob', users)
+
+    def test_users_simple_dict_no(self):
+        distro = self._make_distro('ubuntu', 'bob')
+        ug_cfg = {
+            'users': {
+                'default': False,
+            }
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertEquals({}, users)
+        ug_cfg = {
+            'users': {
+                'default': 'no',
+            }
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertEquals({}, users)
+
+    def test_users_simple_csv(self):
+        distro = self._make_distro('ubuntu')
+        ug_cfg = {
+            'users': 'joe,bob',
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertIn('joe', users)
+        self.assertIn('bob', users)
+        self.assertEquals({}, users['joe'])
+        self.assertEquals({}, users['bob'])
+
+    def test_users_simple(self):
+        distro = self._make_distro('ubuntu')
+        ug_cfg = {
+            'users': [
+                'joe',
+                'bob'
+            ],
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertIn('joe', users)
+        self.assertIn('bob', users)
+        self.assertEquals({}, users['joe'])
+        self.assertEquals({}, users['bob'])
+
+    def test_users_old_user(self):
+        distro = self._make_distro('ubuntu', 'bob')
+        ug_cfg = {
+            'user': 'zetta',
+            'users': 'default'
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertIn('bob', users)
+        self.assertIn('zetta', users)
+        self.assertNotIn('default', users)
+        ug_cfg = {
+            'user': 'zetta',
+            'users': 'default, joe'
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertIn('bob', users)
+        self.assertIn('joe', users)
+        self.assertIn('zetta', users)
+        self.assertNotIn('default', users)
+        ug_cfg = {
+            'user': 'zetta',
+            'users': ['bob', 'joe']
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertNotIn('bob', users)
+        self.assertIn('joe', users)
+        self.assertIn('zetta', users)
+        ug_cfg = {
+            'user': 'zetta',
+            'users': {
+                'bob': True,
+                'joe': True,
+            }
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertIn('bob', users)
+        self.assertIn('joe', users)
+        self.assertIn('zetta', users)
+        ug_cfg = {
+            'user': 'zetta',
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertIn('zetta', users)
+        ug_cfg = {
+        }
+        (users, groups) = self._norm(ug_cfg, distro)
+        self.assertEquals({}, users)
+        self.assertEquals({}, groups)
+
+    def test_users_dict_default_additional(self):
+        distro = self._make_distro('ubuntu', 'bob')
+        ug_cfg = {
+            'users': [
+                {'name': 'default', 'blah': True}
+            ],
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertIn('bob', users)
+        self.assertEquals(",".join(distro.get_default_user()['groups']),
+                          users['bob']['groups'])
+        self.assertEquals(True,
+                          users['bob']['blah'])
+
+    def test_users_dict_default(self):
+        distro = self._make_distro('ubuntu', 'bob')
+        ug_cfg = {
+            'users': [
+                'default',
+            ],
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertIn('bob', users)
+        self.assertEquals(",".join(distro.get_default_user()['groups']),
+                          users['bob']['groups'])
+
+    def test_users_dict_trans(self):
+        distro = self._make_distro('ubuntu')
+        ug_cfg = {
+            'users': [
+                {'name': 'joe',
+                 'tr-me': True},
+                {'name': 'bob'},
+            ],
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertIn('joe', users)
+        self.assertIn('bob', users)
+        self.assertEquals({'tr_me': True}, users['joe'])
+        self.assertEquals({}, users['bob'])
+
+    def test_users_dict(self):
+        distro = self._make_distro('ubuntu')
+        ug_cfg = {
+            'users': [
+                {'name': 'joe'},
+                {'name': 'bob'},
+            ],
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertIn('joe', users)
+        self.assertIn('bob', users)
+        self.assertEquals({}, users['joe'])
+        self.assertEquals({}, users['bob'])