From cf3dd1ba86d4ddde149f451e026c697c07b4d732 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Fri, 28 Sep 2012 13:53:56 -0700
Subject: Rework the rest of the locations that used the previous 'user' and
 make those locations go through the new distros functions to select the
 default user or the user list (depending on usage).

Adjust the tests to check the new 'default' field
that signifies the default user + test the new method
to extract just the default user from a normalized
user dictionary.
---
 cloudinit/config/cc_byobu.py                    |   6 +-
 cloudinit/config/cc_set_passwords.py            |  13 +--
 cloudinit/config/cc_ssh.py                      |  13 +--
 cloudinit/config/cc_ssh_authkey_fingerprints.py |   7 +-
 cloudinit/config/cc_ssh_import_id.py            |  33 +++---
 cloudinit/config/cc_users_groups.py             |   7 +-
 cloudinit/distros/__init__.py                   | 130 ++++++++++++++++++------
 cloudinit/util.py                               |  30 ++++++
 8 files changed, 162 insertions(+), 77 deletions(-)

(limited to 'cloudinit')

diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py
index 4e2e06bb..e1ec5af5 100644
--- a/cloudinit/config/cc_byobu.py
+++ b/cloudinit/config/cc_byobu.py
@@ -18,12 +18,13 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from cloudinit import distros as ds
 from cloudinit import util
 
 distros = ['ubuntu', 'debian']
 
 
-def handle(name, cfg, _cloud, log, args):
+def handle(name, cfg, cloud, log, args):
     if len(args) != 0:
         value = args[0]
     else:
@@ -56,7 +57,8 @@ def handle(name, cfg, _cloud, log, args):
 
     shcmd = ""
     if mod_user:
-        user = util.get_cfg_option_str(cfg, "user", "ubuntu")
+        (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro)
+        (user, _user_config) = ds.extract_default(users, 'ubuntu')
         shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst)
         shcmd += " || X=$(($X+1)); "
     if mod_sys:
diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py
index a017e6b6..bb95f948 100644
--- a/cloudinit/config/cc_set_passwords.py
+++ b/cloudinit/config/cc_set_passwords.py
@@ -20,6 +20,7 @@
 
 import sys
 
+from cloudinit import distros as ds
 from cloudinit import ssh_util
 from cloudinit import util
 
@@ -50,18 +51,10 @@ def handle(_name, cfg, cloud, log, args):
         expire = util.get_cfg_option_bool(chfg, 'expire', expire)
 
     if not plist and password:
-        user = cloud.distro.get_default_user()
-
-        if 'users' in cfg:
-
-            user_zero = cfg['users'][0]
-
-            if isinstance(user_zero, dict) and 'name' in user_zero:
-                user = user_zero['name']
-
+        (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro)
+        (user, _user_config) = ds.extract_default(users)
         if user:
             plist = "%s:%s" % (user, password)
-
         else:
             log.warn("No default or defined user to change password for.")
 
diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py
index 0ded62ba..c2ee4635 100644
--- a/cloudinit/config/cc_ssh.py
+++ b/cloudinit/config/cc_ssh.py
@@ -21,6 +21,7 @@
 import glob
 import os
 
+from cloudinit import distros as ds
 from cloudinit import ssh_util
 from cloudinit import util
 
@@ -102,16 +103,8 @@ def handle(_name, cfg, cloud, log, _args):
                                       " %s to file %s"), keytype, keyfile)
 
     try:
-        # TODO(utlemming): consolidate this stanza that occurs in:
-        # cc_ssh_import_id, cc_set_passwords, maybe cc_users_groups.py
-        user = cloud.distro.get_default_user()
-
-        if 'users' in cfg:
-            user_zero = cfg['users'][0]
-
-            if user_zero != "default":
-                user = user_zero
-
+        (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro)
+        (user, _user_config) = ds.extract_default(users)
         disable_root = util.get_cfg_option_bool(cfg, "disable_root", True)
         disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts",
                                                     DISABLE_ROOT_OPTS)
diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py
index 2b9a6e0e..32214fba 100644
--- a/cloudinit/config/cc_ssh_authkey_fingerprints.py
+++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py
@@ -41,8 +41,10 @@ def _gen_fingerprint(b64_text, hash_meth='md5'):
         hasher = hashlib.new(hash_meth)
         hasher.update(base64.b64decode(b64_text))
         return ":".join(_split_hash(hasher.hexdigest()))
-    except TypeError:
+    except (TypeError, ValueError):
         # Raised when b64 not really b64...
+        # or when the hash type is not really
+        # a known/supported hash type...
         return '?'
 
 
@@ -95,4 +97,5 @@ def handle(name, cfg, cloud, log, _args):
     (users, _groups) = distros.normalize_users_groups(cfg, cloud.distro)
     for (user_name, _cfg) in users.items():
         (auth_key_fn, auth_key_entries) = extract_func(user_name, cloud.paths)
-        _pprint_key_entries(user_name, auth_key_fn, auth_key_entries, hash_meth)
+        _pprint_key_entries(user_name, auth_key_fn,
+                            auth_key_entries, hash_meth)
diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py
index 08fb63c6..a781cd7c 100644
--- a/cloudinit/config/cc_ssh_import_id.py
+++ b/cloudinit/config/cc_ssh_import_id.py
@@ -18,6 +18,7 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from cloudinit import distros as ds
 from cloudinit import util
 import pwd
 
@@ -39,33 +40,27 @@ def handle(_name, cfg, cloud, log, args):
         return
 
     # import for cloudinit created users
+    (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro)
     elist = []
-    for user_cfg in cfg['users']:
-        user = None
+    for (user, user_cfg) in users.items():
         import_ids = []
-
-        if isinstance(user_cfg, str) and user_cfg == "default":
-            user = cloud.distro.get_default_user()
-            if not user:
-                continue
-
+        if user_cfg['default']:
             import_ids = util.get_cfg_option_list(cfg, "ssh_import_id", [])
-
-        elif isinstance(user_cfg, dict):
-            user = None
-            import_ids = []
-
+        else:
             try:
-                user = user_cfg['name']
                 import_ids = user_cfg['ssh_import_id']
-
-                if import_ids and isinstance(import_ids, str):
-                    import_ids = str(import_ids).split(',')
-
             except:
-                log.debug("user %s is not configured for ssh_import" % user)
+                log.debug("User %s is not configured for ssh_import_id", user)
                 continue
 
+        try:
+            import_ids = util.uniq_merge(import_ids)
+            import_ids = [str(i) for i in import_ids]
+        except:
+            log.debug("User %s is not correctly configured for ssh_import_id",
+                      user)
+            continue
+
         if not len(import_ids):
             continue
 
diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py
index 464f55c3..da587fb3 100644
--- a/cloudinit/config/cc_users_groups.py
+++ b/cloudinit/config/cc_users_groups.py
@@ -16,16 +16,15 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from cloudinit import distros
-from cloudinit import util
+from cloudinit import distros as ds
 
 from cloudinit.settings import PER_INSTANCE
 
 frequency = PER_INSTANCE
 
 
-def handle(name, cfg, cloud, log, _args):
-    (users, groups) = distros.normalize_users_groups(cfg, cloud.distro)
+def handle(name, cfg, cloud, _log, _args):
+    (users, groups) = ds.normalize_users_groups(cfg, cloud.distro)
     for (name, members) in groups.items():
         cloud.distro.create_group(name, members)
     for (user, config) in users.items():
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index f07ba3fa..6b458d06 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -24,6 +24,7 @@
 from StringIO import StringIO
 
 import abc
+import itertools
 import os
 import re
 
@@ -187,8 +188,10 @@ class Distro(object):
             'gecos': "%s" % (self.default_user.title()),
             'sudo': "ALL=(ALL) NOPASSWD:ALL",
         }
-        if self.default_user_groups:
-            user_cfg['groups'] = _uniq_merge_sorted(self.default_user_groups)
+        def_groups = self.default_user_groups
+        if not def_groups:
+            def_groups = []
+        user_cfg['groups'] = util.uniq_merge_sorted(def_groups)
         return user_cfg
 
     def create_user(self, name, **kwargs):
@@ -397,39 +400,27 @@ def _get_arch_package_mirror_info(package_mirrors, arch):
     return default
 
 
-def _uniq_merge_sorted(*lists):
-    return sorted(_uniq_merge(*lists))
-
-
-def _uniq_merge(*lists):
-    combined_list = []
-    for a_list in lists:
-        if isinstance(a_list, (str, basestring)):
-            a_list = a_list.strip().split(",")
-        else:
-            a_list = [str(a) for a in a_list]
-        a_list = [a.strip() for a in a_list if a.strip()]
-        combined_list.extend(a_list)
-    uniq_list = []
-    for a in combined_list:
-        if a in uniq_list:
-            continue
-        else:
-            uniq_list.append(a)
-    return uniq_list
-
-
+# Normalizes a input group configuration
+# which can be a comma seperated list of
+# group names, or a list of group names
+# or a python dictionary of group names
+# to a list of members of that group.
+#
+# The output is a dictionary of group
+# names => members of that group which
+# is the standard form used in the rest
+# of cloud-init
 def _normalize_groups(grp_cfg):
     if isinstance(grp_cfg, (str, basestring, list)):
         c_grp_cfg = {}
-        for i in _uniq_merge(grp_cfg):
+        for i in util.uniq_merge(grp_cfg):
             c_grp_cfg[i] = []
         grp_cfg = c_grp_cfg
 
     groups = {}
     if isinstance(grp_cfg, (dict)):
         for (grp_name, grp_members) in grp_cfg.items():
-            groups[grp_name] = _uniq_merge_sorted(grp_members)
+            groups[grp_name] = util.uniq_merge_sorted(grp_members)
     else:
         raise TypeError(("Group config must be list, dict "
                          " or string types only and not %s") %
@@ -437,6 +428,21 @@ def _normalize_groups(grp_cfg):
     return groups
 
 
+# Normalizes a input group configuration
+# which can be a comma seperated list of
+# user names, or a list of string user names
+# or a list of dictionaries with components
+# that define the user config + 'name' (if
+# a 'name' field does not exist then the
+# default user is assumed to 'own' that
+# configuration.
+#
+# The output is a dictionary of user
+# names => user config which is the standard 
+# form used in the rest of cloud-init. Note
+# the default user will have a special config
+# entry 'default' which will be marked as true
+# all other users will be marked as false.
 def _normalize_users(u_cfg, def_user_cfg=None):
     if isinstance(u_cfg, (dict)):
         ad_ucfg = []
@@ -452,12 +458,12 @@ def _normalize_users(u_cfg, def_user_cfg=None):
                                  " for key %s") % (util.obj_name(v), k))
         u_cfg = ad_ucfg
     elif isinstance(u_cfg, (str, basestring)):
-        u_cfg = _uniq_merge_sorted(u_cfg)
+        u_cfg = util.uniq_merge_sorted(u_cfg)
 
     users = {}
     for user_config in u_cfg:
         if isinstance(user_config, (str, basestring, list)):
-            for u in _uniq_merge(user_config):
+            for u in util.uniq_merge(user_config):
                 if u and u not in users:
                     users[u] = {}
         elif isinstance(user_config, (dict)):
@@ -490,22 +496,59 @@ def _normalize_users(u_cfg, def_user_cfg=None):
 
     # Fixup the default user into the real
     # default user name and replace it...
+    def_user = None
     if users and 'default' in users:
         def_config = users.pop('default')
         if def_user_cfg:
+            # Pickup what the default 'real name' is
+            # and any groups that are provided by the
+            # default config
             def_user = def_user_cfg.pop('name')
             def_groups = def_user_cfg.pop('groups', [])
+            # Pickup any config + groups for that user name
+            # that we may have previously extracted
             parsed_config = users.pop(def_user, {})
-            users_groups = _uniq_merge_sorted(parsed_config.get('groups', []),
-                                              def_groups)
+            parsed_groups = parsed_config.get('groups', [])
+            # Now merge our extracted groups with
+            # anything the default config provided
+            users_groups = util.uniq_merge_sorted(parsed_groups, def_groups)
             parsed_config['groups'] = ",".join(users_groups)
+            # The real config for the default user is the
+            # combination of the default user config provided
+            # by the distro, the default user config provided
+            # by the above merging for the user 'default' and
+            # then the parsed config from the user's 'real name'
+            # which does not have to be 'default' (but could be)
             users[def_user] = util.mergemanydict([def_user_cfg,
                                                   def_config,
                                                   parsed_config])
 
+    # Ensure that only the default user that we
+    # found (if any) is actually marked as being
+    # the default user
+    if users:
+        for (uname, uconfig) in users.items():
+            if def_user and uname == def_user:
+                uconfig['default'] = True
+            else:
+                uconfig['default'] = False
+
     return users
 
 
+# Normalizes a set of user/users and group
+# dictionary configuration into a useable
+# format that the rest of cloud-init can
+# understand using the default user
+# provided by the input distrobution (if any)
+# to allow for mapping of the 'default' user.
+#
+# Output is a dictionary of group names -> [member] (list)
+# and a dictionary of user names -> user configuration (dict)
+#
+# If 'user' exists it will override
+# the 'users'[0] entry (if a list) otherwise it will
+# just become an entry in the returned dictionary (no override)
 def normalize_users_groups(cfg, distro):
     if not cfg:
         cfg = {}
@@ -547,6 +590,33 @@ def normalize_users_groups(cfg, distro):
     return (users, groups)
 
 
+# Given a user dictionary config it will
+# extract the default user name and user config
+# from that list and return that tuple or
+# return (None, None) if no default user is
+# found in the given input
+def extract_default(users, default_name=None, default_config=None):
+    if not users:
+        users = {}
+
+    def safe_find(entry):
+        config = entry[1]
+        if not config or 'default' not in config:
+            return False
+        else:
+            return config['default']
+
+    tmp_users = users.items()
+    tmp_users = dict(itertools.ifilter(safe_find, tmp_users))
+    if not tmp_users:
+        return (default_name, default_config)
+    else:
+        name = tmp_users.keys()[0]
+        config = tmp_users[name]
+        config.pop('default', None)
+        return (name, config)
+
+
 def fetch(name):
     locs = importer.find_module(name,
                                 ['', __name__],
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 94b17dfa..184b37a4 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -248,6 +248,36 @@ def read_conf(fname):
             raise
 
 
+# Merges X lists, and then keeps the
+# unique ones, but orders by sort order
+# instead of by the original order
+def uniq_merge_sorted(*lists):
+    return sorted(uniq_merge(*lists))
+
+
+# Merges X lists and then iterates over those
+# and only keeps the unique items (order preserving)
+# and returns that merged and uniqued list as the
+# final result.
+#
+# Note: if any entry is a string it will be
+# split on commas and empty entries will be
+# evicted and merged in accordingly.
+def uniq_merge(*lists):
+    combined_list = []
+    for a_list in lists:
+        if isinstance(a_list, (str, basestring)):
+            a_list = a_list.strip().split(",")
+            # Kickout the empty ones
+            a_list = [a for a in a_list if len(a)]
+        combined_list.extend(a_list)
+    uniq_list = []
+    for i in combined_list:
+        if i not in uniq_list:
+            uniq_list.append(i)
+    return uniq_list
+
+
 def clean_filename(fn):
     for (k, v) in FN_REPLACEMENTS.iteritems():
         fn = fn.replace(k, v)
-- 
cgit v1.2.3


From 7ab25c779433a614a6e4101ddfe852fc25f39c01 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Fri, 28 Sep 2012 14:06:22 -0700
Subject: Add a comment as to why distros can't be imported without being
 renamed due to previous usage of the attribute 'distros'

---
 cloudinit/config/cc_byobu.py                    | 4 ++++
 cloudinit/config/cc_set_passwords.py            | 4 ++++
 cloudinit/config/cc_ssh.py                      | 4 ++++
 cloudinit/config/cc_ssh_authkey_fingerprints.py | 8 ++++++--
 cloudinit/config/cc_ssh_import_id.py            | 4 ++++
 cloudinit/config/cc_users_groups.py             | 3 +++
 6 files changed, 25 insertions(+), 2 deletions(-)

(limited to 'cloudinit')

diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py
index e1ec5af5..e38fccdd 100644
--- a/cloudinit/config/cc_byobu.py
+++ b/cloudinit/config/cc_byobu.py
@@ -18,7 +18,11 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+# Ensure this is aliased to a name not 'distros'
+# since the module attribute 'distros'
+# is a list of distros that are supported, not a sub-module
 from cloudinit import distros as ds
+
 from cloudinit import util
 
 distros = ['ubuntu', 'debian']
diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py
index bb95f948..26c558ad 100644
--- a/cloudinit/config/cc_set_passwords.py
+++ b/cloudinit/config/cc_set_passwords.py
@@ -20,7 +20,11 @@
 
 import sys
 
+# Ensure this is aliased to a name not 'distros'
+# since the module attribute 'distros'
+# is a list of distros that are supported, not a sub-module
 from cloudinit import distros as ds
+
 from cloudinit import ssh_util
 from cloudinit import util
 
diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py
index c2ee4635..32e48c30 100644
--- a/cloudinit/config/cc_ssh.py
+++ b/cloudinit/config/cc_ssh.py
@@ -21,7 +21,11 @@
 import glob
 import os
 
+# Ensure this is aliased to a name not 'distros'
+# since the module attribute 'distros'
+# is a list of distros that are supported, not a sub-module
 from cloudinit import distros as ds
+
 from cloudinit import ssh_util
 from cloudinit import util
 
diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py
index 32214fba..8c9a8806 100644
--- a/cloudinit/config/cc_ssh_authkey_fingerprints.py
+++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py
@@ -21,7 +21,11 @@ import hashlib
 
 from prettytable import PrettyTable
 
-from cloudinit import distros
+# Ensure this is aliased to a name not 'distros'
+# since the module attribute 'distros'
+# is a list of distros that are supported, not a sub-module
+from cloudinit import distros as ds
+
 from cloudinit import ssh_util
 from cloudinit import util
 
@@ -94,7 +98,7 @@ def handle(name, cfg, cloud, log, _args):
 
     hash_meth = util.get_cfg_option_str(cfg, "authkey_hash", "md5")
     extract_func = ssh_util.extract_authorized_keys
-    (users, _groups) = distros.normalize_users_groups(cfg, cloud.distro)
+    (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro)
     for (user_name, _cfg) in users.items():
         (auth_key_fn, auth_key_entries) = extract_func(user_name, cloud.paths)
         _pprint_key_entries(user_name, auth_key_fn,
diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py
index a781cd7c..83af36e9 100644
--- a/cloudinit/config/cc_ssh_import_id.py
+++ b/cloudinit/config/cc_ssh_import_id.py
@@ -18,7 +18,11 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+# Ensure this is aliased to a name not 'distros'
+# since the module attribute 'distros'
+# is a list of distros that are supported, not a sub-module
 from cloudinit import distros as ds
+
 from cloudinit import util
 import pwd
 
diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py
index da587fb3..bf5b4581 100644
--- a/cloudinit/config/cc_users_groups.py
+++ b/cloudinit/config/cc_users_groups.py
@@ -16,6 +16,9 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+# Ensure this is aliased to a name not 'distros'
+# since the module attribute 'distros'
+# is a list of distros that are supported, not a sub-module
 from cloudinit import distros as ds
 
 from cloudinit.settings import PER_INSTANCE
-- 
cgit v1.2.3


From 5233b6edb70702476463b47c06cb02b3c7f74c51 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Fri, 28 Sep 2012 14:17:42 -0700
Subject: Make byobu more tolerant of the user not being located and warn when
 it is not found + only run the shell command when actual contents exist to
 run.

---
 cloudinit/config/cc_byobu.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

(limited to 'cloudinit')

diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py
index e38fccdd..92d428b7 100644
--- a/cloudinit/config/cc_byobu.py
+++ b/cloudinit/config/cc_byobu.py
@@ -62,16 +62,19 @@ def handle(name, cfg, cloud, log, args):
     shcmd = ""
     if mod_user:
         (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro)
-        (user, _user_config) = ds.extract_default(users, 'ubuntu')
-        shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst)
-        shcmd += " || X=$(($X+1)); "
+        (user, _user_config) = ds.extract_default(users)
+        if not user:
+            log.warn(("No default byobu user provided, "
+                      "can not launch %s for the default user"), bl_inst)
+        else:
+            shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst)
+            shcmd += " || X=$(($X+1)); "
     if mod_sys:
         shcmd += "echo \"%s\" | debconf-set-selections" % dc_val
         shcmd += " && dpkg-reconfigure byobu --frontend=noninteractive"
         shcmd += " || X=$(($X+1)); "
 
-    cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")]
-
-    log.debug("Setting byobu to %s", value)
-
-    util.subp(cmd, capture=False)
+    if len(shcmd):
+        cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")]
+        log.debug("Setting byobu to %s", value)
+        util.subp(cmd, capture=False)
-- 
cgit v1.2.3