Make the normalization a module level function

since it has little dependence on the distros
class itself. Readjust the using code to use
this new module level function instead.

3 files changed, 160 insertions, 154 deletions

diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py
index a6ce49ac..13eb1102 100644
--- a/cloudinit/config/cc_users_groups.py
+++ b/cloudinit/config/cc_users_groups.py
@@ -16,6 +16,7 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from cloudinit import distros
 from cloudinit import util
 
 from cloudinit.settings import PER_INSTANCE
@@ -24,10 +25,20 @@ frequency = PER_INSTANCE
 
 
 def handle(name, cfg, cloud, log, _args):
-    distro = cloud.distro
-    ((users, default_user), groups) = distro.normalize_users_groups(cfg)
+    def_u = None
+    def_u_gs = None
+    try:
+        def_u = cloud.distro.get_default_user()
+        def_u_gs = cloud.distro.get_default_user_groups()
+    except NotImplementedError:
+        log.warn(("Distro has not implemented default user "
+                  "creation. No default user will be added."))
+
+    ((users, default_user), groups) = distros.normalize_users_groups(cfg,
+                                                                     def_u,
+                                                                     def_u_gs)
     for (name, members) in groups.items():
-        distro.create_group(name, members)
+        cloud.distro.create_group(name, members)
 
     if default_user:
         user = default_user['name']
@@ -41,8 +52,8 @@ def handle(name, cfg, cloud, log, _args):
             'sudo': "ALL=(ALL) NOPASSWD:ALL",
         }
         u_config = util.mergemanydict([def_base_config, config])
-        distro.create_user(user, **u_config)
+        cloud.distro.create_user(user, **u_config)
         log.info("Added default '%s' user with passwordless sudo", user)
 
     for (user, config) in users.items():
-        distro.create_user(user, **config)
+        cloud.distro.create_user(user, **config)
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 3de5be36..b43d662e 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -180,13 +180,7 @@ class Distro(object):
     def get_default_user_groups(self):
         if not self.default_user_groups:
             return []
-        def_groups = []
-        if isinstance(self.default_user_groups, (str, basestring)):
-            def_groups = self.default_user_groups.split(",")
-        else:
-            def_groups = list(self.default_user_groups)
-        def_groups = list(sorted(set(def_groups)))
-        return def_groups
+        return _uniq_merge_sorted(self.default_user_groups)
 
     def create_user(self, name, **kwargs):
         """
@@ -294,133 +288,6 @@ class Distro(object):
 
         return True
 
-    def _normalize_groups(self, grp_cfg):
-        groups = {}
-        if isinstance(grp_cfg, (str, basestring)):
-            grp_cfg = grp_cfg.strip().split(",")
-
-        if isinstance(grp_cfg, (list)):
-            for g in grp_cfg:
-                g = g.strip()
-                if g:
-                    groups[g] = []
-        elif isinstance(grp_cfg, (dict)):
-            for grp_name, grp_members in grp_cfg.items():
-                if isinstance(grp_members, (str, basestring)):
-                    r_grp_members = []
-                    for gc in grp_members.strip().split(','):
-                        gc = gc.strip()
-                        if gc and gc not in r_grp_members:
-                            r_grp_members.append(gc)
-                    grp_members = r_grp_members
-                elif not isinstance(grp_members, (list)):
-                    raise TypeError(("Group member config must be list "
-                                     " or string types only and not %s") %
-                                    util.obj_name(grp_members))
-                groups[grp_name] = grp_members
-        else:
-            raise TypeError(("Group config must be list, dict "
-                             " or string types only and not %s") %
-                            util.obj_name(grp_cfg))
-        return groups
-
-    def _normalize_users(self, u_cfg):
-        if isinstance(u_cfg, (dict)):
-            ad_ucfg = []
-            for (k, v) in u_cfg.items():
-                if isinstance(v, (bool, int, basestring, str)):
-                    if util.is_true(v):
-                        ad_ucfg.append(str(k))
-                elif isinstance(v, (dict)):
-                    v['name'] = k
-                    ad_ucfg.append(v)
-                else:
-                    raise TypeError(("Unmappable user value type %s"
-                                     " for key %s") % (util.obj_name(v), k))
-            u_cfg = ad_ucfg
-        elif isinstance(u_cfg, (str, basestring)):
-            u_cfg = u_cfg.strip().split(",")
-
-        users = {}
-        for user_config in u_cfg:
-            if isinstance(user_config, (str, basestring)):
-                for u in user_config.strip().split(","):
-                    u = u.strip()
-                    if u and u not in users:
-                        users[u] = {}
-            elif isinstance(user_config, (dict)):
-                if 'name' in user_config:
-                    n = user_config.pop('name')
-                    prev_config = users.get(n) or {}
-                    users[n] = util.mergemanydict([prev_config,
-                                                   user_config])
-                else:
-                    # Assume the default user then
-                    prev_config = users.get('default') or {}
-                    users['default'] = util.mergemanydict([prev_config,
-                                                           user_config])
-            elif isinstance(user_config, (bool, int)):
-                pass
-            else:
-                raise TypeError(("User config must be dictionary "
-                                 " or string types only and not %s") %
-                                util.obj_name(user_config))
-
-        # Ensure user options are in the right python friendly format
-        if users:
-            c_users = {}
-            for (uname, uconfig) in users.items():
-                c_uconfig = {}
-                for (k, v) in uconfig.items():
-                    k = k.replace('-', '_').strip()
-                    if k:
-                        c_uconfig[k] = v
-                c_users[uname] = c_uconfig
-            users = c_users
-
-        # Fixup the default user into the real
-        # default user name and extract it
-        default_user = {}
-        if users and 'default' in users:
-            try:
-                def_config = users.pop('default')
-                def_user = self.get_default_user()
-                def_groups = self.get_default_user_groups()
-                if def_user:
-                    u_config = users.pop(def_user, None) or {}
-                    u_groups = u_config.get('groups') or []
-                    if isinstance(u_groups, (str, basestring)):
-                        u_groups = u_groups.strip().split(",")
-                    u_groups.extend(def_groups)
-                    u_groups = set([x.strip() for x in u_groups if x.strip()])
-                    u_config['groups'] = ",".join(sorted(u_groups))
-                    default_user = {
-                        'name': def_user,
-                        'config': util.mergemanydict([def_config, u_config]),
-                    }
-                else:
-                    LOG.warn(("Distro has not provided a default user "
-                              "for creation. No default user will be "
-                              "normalized."))
-                    users.pop('default', None)
-            except NotImplementedError:
-                LOG.warn(("Distro has not implemented default user "
-                         "creation. No default user will be normalized."))
-                users.pop('default', None)
-
-        return (default_user, users)
-
-    def normalize_users_groups(self, ug_cfg):
-        users = {}
-        groups = {}
-        default_user = {}
-        if 'groups' in ug_cfg:
-            groups = self._normalize_groups(ug_cfg['groups'])
-
-        if 'users' in ug_cfg:
-            default_user, users = self._normalize_users(ug_cfg['users'])
-        return ((users, default_user), groups)
-
     def write_sudo_rules(self,
         user,
         rules,
@@ -521,6 +388,129 @@ def _get_arch_package_mirror_info(package_mirrors, arch):
     return default
 
 
+def _uniq_merge_sorted(*lists):
+    return sorted(_uniq_merge(*lists))
+
+
+def _uniq_merge(*lists):
+    combined_list = []
+    for a_list in lists:
+        if isinstance(a_list, (str, basestring)):
+            a_list = a_list.strip().split(",")
+        else:
+            a_list = [str(a) for a in a_list]
+        a_list = [a.strip() for a in a_list if a.strip()]
+        combined_list.extend(a_list)
+    uniq_list = []
+    for a in combined_list:
+        if a in uniq_list:
+            continue
+        else:
+            uniq_list.append(a)
+    return uniq_list
+
+
+def _normalize_groups(grp_cfg):
+    if isinstance(grp_cfg, (str, basestring, list)):
+        c_grp_cfg = {}
+        for i in _uniq_merge(grp_cfg):
+            c_grp_cfg[i] = []
+        grp_cfg = c_grp_cfg
+
+    groups = {}
+    if isinstance(grp_cfg, (dict)):
+        for (grp_name, grp_members) in grp_cfg.items():
+            groups[grp_name] = _uniq_merge_sorted(grp_members)
+    else:
+        raise TypeError(("Group config must be list, dict "
+                         " or string types only and not %s") %
+                        util.obj_name(grp_cfg))
+    return groups
+
+
+def _normalize_users(u_cfg, def_user=None, def_user_groups=None):
+    if isinstance(u_cfg, (dict)):
+        ad_ucfg = []
+        for (k, v) in u_cfg.items():
+            if isinstance(v, (bool, int, basestring, str, float)):
+                if util.is_true(v):
+                    ad_ucfg.append(str(k))
+            elif isinstance(v, (dict)):
+                v['name'] = k
+                ad_ucfg.append(v)
+            else:
+                raise TypeError(("Unmappable user value type %s"
+                                 " for key %s") % (util.obj_name(v), k))
+        u_cfg = ad_ucfg
+    elif isinstance(u_cfg, (str, basestring)):
+        u_cfg = _uniq_merge_sorted(u_cfg)
+
+    users = {}
+    for user_config in u_cfg:
+        if isinstance(user_config, (str, basestring, list)):
+            for u in _uniq_merge(user_config):
+                if u and u not in users:
+                    users[u] = {}
+        elif isinstance(user_config, (dict)):
+            if 'name' in user_config:
+                n = user_config.pop('name')
+                prev_config = users.get(n) or {}
+                users[n] = util.mergemanydict([prev_config,
+                                               user_config])
+            else:
+                # Assume the default user then
+                prev_config = users.get('default') or {}
+                users['default'] = util.mergemanydict([prev_config,
+                                                       user_config])
+        else:
+            raise TypeError(("User config must be dictionary/list "
+                             " or string types only and not %s") %
+                            util.obj_name(user_config))
+
+    # Ensure user options are in the right python friendly format
+    if users:
+        c_users = {}
+        for (uname, uconfig) in users.items():
+            c_uconfig = {}
+            for (k, v) in uconfig.items():
+                k = k.replace('-', '_').strip()
+                if k:
+                    c_uconfig[k] = v
+            c_users[uname] = c_uconfig
+        users = c_users
+
+    # Fixup the default user into the real
+    # default user name and extract it
+    default_user = {}
+    if users and 'default' in users:
+        def_config = users.pop('default')
+        def_groups = def_user_groups or []
+        if def_user:
+            u_config = users.pop(def_user, None) or {}
+            u_groups = u_config.get('groups') or []
+            u_groups = _uniq_merge_sorted(u_groups, def_groups)
+            u_config['groups'] = ",".join(u_groups)
+            default_user = {
+                'name': def_user,
+                'config': util.mergemanydict([def_config, u_config]),
+            }
+
+    return (default_user, users)
+
+
+def normalize_users_groups(cfg, def_user=None, def_user_groups=None):
+    users = {}
+    groups = {}
+    default_user = {}
+    if 'groups' in cfg:
+        groups = _normalize_groups(cfg['groups'])
+    if 'users' in cfg:
+        (default_user, users) = _normalize_users(cfg['users'],
+                                                 def_user,
+                                                 def_user_groups)
+    return ((users, default_user), groups)
+
+
 def fetch(name):
     locs = importer.find_module(name,
                                 ['', __name__],
diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py
index d636bb84..46733452 100644
--- a/tests/unittests/test_distros/test_user_data_normalize.py
+++ b/tests/unittests/test_distros/test_user_data_normalize.py
@@ -19,12 +19,17 @@ class TestUGNormalize(MockerTestCase):
             distro.default_user_groups = def_groups
         return distro
 
+    def _norm(self, cfg, distro):
+        def_u = distro.get_default_user()
+        def_u_gs = distro.get_default_user_groups()
+        return distros.normalize_users_groups(cfg, def_u, def_u_gs)
+
     def test_basic_groups(self):
         distro = self._make_distro('ubuntu')
         ug_cfg = {
             'groups': ['bob'],
         }
-        ((users, def_user), groups) = distro.normalize_users_groups(ug_cfg)
+        ((users, def_user), groups) = self._norm(ug_cfg, distro)
         self.assertIn('bob', groups)
         self.assertEquals({}, users)
         self.assertEquals({}, def_user)
@@ -34,7 +39,7 @@ class TestUGNormalize(MockerTestCase):
         ug_cfg = {
             'groups': 'bob,joe,steve',
         }
-        ((users, def_user), groups) = distro.normalize_users_groups(ug_cfg)
+        ((users, def_user), groups) = self._norm(ug_cfg, distro)
         self.assertIn('bob', groups)
         self.assertIn('joe', groups)
         self.assertIn('steve', groups)
@@ -46,7 +51,7 @@ class TestUGNormalize(MockerTestCase):
         ug_cfg = {
             'groups': ['bob', 'joe', 'steve',]
         }
-        ((users, def_user), groups) = distro.normalize_users_groups(ug_cfg)
+        ((users, def_user), groups) = self._norm(ug_cfg, distro)
         self.assertIn('bob', groups)
         self.assertIn('joe', groups)
         self.assertIn('steve', groups)
@@ -62,7 +67,7 @@ class TestUGNormalize(MockerTestCase):
                 'steve': [],
             }
         }
-        ((users, def_user), groups) = distro.normalize_users_groups(ug_cfg)
+        ((users, def_user), groups) = self._norm(ug_cfg, distro)
         self.assertIn('bob', groups)
         self.assertEquals(['s'], groups['bob'])
         self.assertEquals([], groups['joe'])
@@ -78,21 +83,21 @@ class TestUGNormalize(MockerTestCase):
                 'default': True,
             }
         }
-        ((_users, def_user), _groups) = distro.normalize_users_groups(ug_cfg)
+        ((_users, def_user), _groups) = self._norm(ug_cfg, distro)
         self.assertEquals('bob', def_user['name'])
         ug_cfg = {
             'users': {
                 'default': 'yes',
             }
         }
-        ((_users, def_user), _groups) = distro.normalize_users_groups(ug_cfg)
+        ((_users, def_user), _groups) = self._norm(ug_cfg, distro)
         self.assertEquals('bob', def_user['name'])
         ug_cfg = {
             'users': {
                 'default': '1',
             }
         }
-        ((_users, def_user), _groups) = distro.normalize_users_groups(ug_cfg)
+        ((_users, def_user), _groups) = self._norm(ug_cfg, distro)
         self.assertEquals('bob', def_user['name'])
 
     def test_users_simple_dict_no(self):
@@ -102,14 +107,14 @@ class TestUGNormalize(MockerTestCase):
                 'default': False,
             }
         }
-        ((_users, def_user), _groups) = distro.normalize_users_groups(ug_cfg)
+        ((_users, def_user), _groups) = self._norm(ug_cfg, distro)
         self.assertEquals({}, def_user)
         ug_cfg = {
             'users': {
                 'default': 'no',
             }
         }
-        ((_users, def_user), _groups) = distro.normalize_users_groups(ug_cfg)
+        ((_users, def_user), _groups) = self._norm(ug_cfg, distro)
         self.assertEquals({}, def_user)
 
     def test_users_simple_csv(self):
@@ -117,7 +122,7 @@ class TestUGNormalize(MockerTestCase):
         ug_cfg = {
             'users': 'joe,bob',
         }
-        ((users, _def_user), _groups) = distro.normalize_users_groups(ug_cfg)
+        ((users, _def_user), _groups) = self._norm(ug_cfg, distro)
         self.assertIn('joe', users)
         self.assertIn('bob', users)
         self.assertEquals({}, users['joe'])
@@ -131,7 +136,7 @@ class TestUGNormalize(MockerTestCase):
                 'bob'
             ],
         }
-        ((users, _def_user), _groups) = distro.normalize_users_groups(ug_cfg)
+        ((users, _def_user), _groups) = self._norm(ug_cfg, distro)
         self.assertIn('joe', users)
         self.assertIn('bob', users)
         self.assertEquals({}, users['joe'])
@@ -144,7 +149,7 @@ class TestUGNormalize(MockerTestCase):
                 {'name': 'default', 'blah': True}
             ],
         }
-        ((users, def_user), _groups) = distro.normalize_users_groups(ug_cfg)
+        ((users, def_user), _groups) = self._norm(ug_cfg, distro)
         self.assertIn('bob', def_user['name'])
         self.assertEquals(",".join(distro.get_default_user_groups()),
                           def_user['config']['groups'])
@@ -159,7 +164,7 @@ class TestUGNormalize(MockerTestCase):
                 'default',
             ],
         }
-        ((users, def_user), _groups) = distro.normalize_users_groups(ug_cfg)
+        ((users, def_user), _groups) = self._norm(ug_cfg, distro)
         self.assertIn('bob', def_user['name'])
         self.assertEquals(",".join(distro.get_default_user_groups()),
                           def_user['config']['groups'])
@@ -174,7 +179,7 @@ class TestUGNormalize(MockerTestCase):
                 {'name': 'bob'},
             ],
         }
-        ((users, _def_user), _groups) = distro.normalize_users_groups(ug_cfg)
+        ((users, _def_user), _groups) = self._norm(ug_cfg, distro)
         self.assertIn('joe', users)
         self.assertIn('bob', users)
         self.assertEquals({'tr_me': True}, users['joe'])
@@ -188,7 +193,7 @@ class TestUGNormalize(MockerTestCase):
                 {'name': 'bob'},
             ],
         }
-        ((users, _def_user), _groups) = distro.normalize_users_groups(ug_cfg)
+        ((users, _def_user), _groups) = self._norm(ug_cfg, distro)
         self.assertIn('joe', users)
         self.assertIn('bob', users)
         self.assertEquals({}, users['joe'])