sync with trunk

8 files changed, 193 insertions, 33 deletions

diff --git a/cloudinit/config/cc_migrator.py b/cloudinit/config/cc_migrator.py
new file mode 100644
index 00000000..58232fc9
--- /dev/null
+++ b/cloudinit/config/cc_migrator.py
@@ -0,0 +1,83 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2012 Yahoo! Inc.
+#
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import shutil
+
+from cloudinit import helpers
+from cloudinit import util
+
+from cloudinit.settings import PER_ALWAYS
+
+frequency = PER_ALWAYS
+
+
+def _migrate_canon_sems(cloud):
+    sem_path = cloud.paths.get_ipath('sem')
+    if not sem_path or not os.path.exists(sem_path):
+        return 0
+    am_adjusted = 0
+    for p in os.listdir(sem_path):
+        full_path = os.path.join(sem_path, p)
+        if os.path.isfile(full_path):
+            (name, ext) = os.path.splitext(p)
+            canon_name = helpers.canon_sem_name(name)
+            if canon_name != name:
+                new_path = os.path.join(sem_path, canon_name + ext)
+                shutil.move(full_path, new_path)
+                am_adjusted += 1
+    return am_adjusted
+
+
+def _migrate_legacy_sems(cloud, log):
+    sem_path = cloud.paths.get_ipath('sem')
+    if not sem_path or not os.path.exists(sem_path):
+        return
+    legacy_adjust = {
+        'apt-update-upgrade': [
+            'apt-configure',
+            'package-update-upgrade-install',
+        ],
+    }
+    sem_helper = helpers.FileSemaphores(sem_path)
+    for (mod_name, migrate_to) in legacy_adjust.items():
+        possibles = [mod_name, helpers.canon_sem_name(mod_name)]
+        old_exists = []
+        for p in os.listdir(sem_path):
+            (name, _ext) = os.path.splitext(p)
+            if name in possibles and os.path.isfile(p):
+                old_exists.append(p)
+        for p in old_exists:
+            util.del_file(os.path.join(sem_path, p))
+            (_name, freq) = os.path.splitext(p)
+            for m in migrate_to:
+                log.debug("Migrating %s => %s with the same frequency",
+                          p, m)
+                with sem_helper.lock(m, freq):
+                    pass
+
+
+def handle(name, cfg, cloud, log, _args):
+    do_migrate = util.get_cfg_option_str(cfg, "migrate", True)
+    if not util.translate_bool(do_migrate):
+        log.debug("Skipping module named %s, migration disabled", name)
+        return
+    sems_moved = _migrate_canon_sems(cloud)
+    log.debug("Migrated %s semaphore files to there canonicalized names",
+              sems_moved)
+    _migrate_legacy_sems(cloud, log)
diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py
index b0f27ebf..2b32fc94 100644
--- a/cloudinit/config/cc_set_hostname.py
+++ b/cloudinit/config/cc_set_hostname.py
@@ -27,9 +27,11 @@ def handle(name, cfg, cloud, log, _args):
                     " not setting the hostname in module %s"), name)
         return
 
-    (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud)
+    (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud)
     try:
-        log.debug("Setting hostname to %s", hostname)
-        cloud.distro.set_hostname(hostname)
+        log.debug("Setting the hostname to %s (%s)", fqdn, hostname)
+        cloud.distro.set_hostname(hostname, fqdn)
     except Exception:
-        util.logexc(log, "Failed to set hostname to %s", hostname)
+        util.logexc(log, "Failed to set the hostname to %s (%s)",
+                    fqdn, hostname)
+        raise
diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py
index 1d6679ea..52225cd8 100644
--- a/cloudinit/config/cc_update_hostname.py
+++ b/cloudinit/config/cc_update_hostname.py
@@ -32,10 +32,12 @@ def handle(name, cfg, cloud, log, _args):
                     " not updating the hostname in module %s"), name)
         return
 
-    (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud)
+    (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud)
     try:
         prev_fn = os.path.join(cloud.get_cpath('data'), "previous-hostname")
-        cloud.distro.update_hostname(hostname, prev_fn)
+        log.debug("Updating hostname to %s (%s)", fqdn, hostname)
+        cloud.distro.update_hostname(hostname, fqdn, prev_fn)
     except Exception:
-        util.logexc(log, "Failed to set the hostname to %s", hostname)
+        util.logexc(log, "Failed to update the hostname to %s (%s)",
+                    fqdn, hostname)
         raise
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 31724e2c..8a98e334 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -58,11 +58,11 @@ class Distro(object):
         return self._cfg.get(opt_name, default)
 
     @abc.abstractmethod
-    def set_hostname(self, hostname):
+    def set_hostname(self, hostname, fqdn=None):
         raise NotImplementedError()
 
     @abc.abstractmethod
-    def update_hostname(self, hostname, prev_hostname_fn):
+    def update_hostname(self, hostname, fqdn, prev_hostname_fn):
         raise NotImplementedError()
 
     @abc.abstractmethod
@@ -283,8 +283,10 @@ class Distro(object):
         # Ensure the dir is included and that
         # it actually exists as a directory
         sudoers_contents = ''
+        base_exists = False
         if os.path.exists(sudo_base):
             sudoers_contents = util.load_file(sudo_base)
+            base_exists = True
         found_include = False
         for line in sudoers_contents.splitlines():
             line = line.strip()
@@ -299,18 +301,24 @@ class Distro(object):
                 found_include = True
                 break
         if not found_include:
-            sudoers_contents += "\n#includedir %s\n" % (path)
             try:
-                if not os.path.exists(sudo_base):
+                if not base_exists:
+                    lines = [('# See sudoers(5) for more information'
+                              ' on "#include" directives:'), '',
+                             '# Added by cloud-init',
+                             "#includedir %s" % (path), '']
+                    sudoers_contents = "\n".join(lines)
                     util.write_file(sudo_base, sudoers_contents, 0440)
                 else:
-                    with open(sudo_base, 'a') as f:
-                        f.write(sudoers_contents)
-                LOG.debug("added '#includedir %s' to %s" % (path, sudo_base))
+                    lines = ['', '# Added by cloud-init',
+                             "#includedir %s" % (path), '']
+                    sudoers_contents = "\n".join(lines)
+                    util.append_file(sudo_base, sudoers_contents)
+                LOG.debug("Added '#includedir %s' to %s" % (path, sudo_base))
             except IOError as e:
                 util.logexc(LOG, "Failed to write %s" % sudo_base, e)
                 raise e
-        util.ensure_dir(path, 0755)
+        util.ensure_dir(path, 0750)
 
     def write_sudo_rules(self, user, rules, sudo_file=None):
         if not sudo_file:
@@ -423,12 +431,36 @@ def _get_arch_package_mirror_info(package_mirrors, arch):
 # is the standard form used in the rest
 # of cloud-init
 def _normalize_groups(grp_cfg):
-    if isinstance(grp_cfg, (str, basestring, list)):
+    if isinstance(grp_cfg, (str, basestring)):
+        grp_cfg = grp_cfg.strip().split(",")
+    if isinstance(grp_cfg, (list)):
         c_grp_cfg = {}
-        for i in util.uniq_merge(grp_cfg):
-            c_grp_cfg[i] = []
+        for i in grp_cfg:
+            if isinstance(i, (dict)):
+                for k, v in i.items():
+                    if k not in c_grp_cfg:
+                        if isinstance(v, (list)):
+                            c_grp_cfg[k] = list(v)
+                        elif isinstance(v, (basestring, str)):
+                            c_grp_cfg[k] = [v]
+                        else:
+                            raise TypeError("Bad group member type %s" %
+                                            util.obj_name(v))
+                    else:
+                        if isinstance(v, (list)):
+                            c_grp_cfg[k].extend(v)
+                        elif isinstance(v, (basestring, str)):
+                            c_grp_cfg[k].append(v)
+                        else:
+                            raise TypeError("Bad group member type %s" %
+                                            util.obj_name(v))
+            elif isinstance(i, (str, basestring)):
+                if i not in c_grp_cfg:
+                    c_grp_cfg[i] = []
+            else:
+                raise TypeError("Unknown group name type %s" %
+                                util.obj_name(i))
         grp_cfg = c_grp_cfg
-
     groups = {}
     if isinstance(grp_cfg, (dict)):
         for (grp_name, grp_members) in grp_cfg.items():
diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py
index cc7e53a0..ed4070b4 100644
--- a/cloudinit/distros/debian.py
+++ b/cloudinit/distros/debian.py
@@ -67,7 +67,7 @@ class Distro(distros.Distro):
         else:
             return distros.Distro._bring_up_interfaces(self, device_names)
 
-    def set_hostname(self, hostname):
+    def set_hostname(self, hostname, fqdn=None):
         self._write_hostname(hostname, "/etc/hostname")
         LOG.debug("Setting hostname to %s", hostname)
         util.subp(['hostname', hostname])
@@ -76,7 +76,7 @@ class Distro(distros.Distro):
         # "" gives trailing newline.
         util.write_file(out_fn, "%s\n" % str(hostname), 0644)
 
-    def update_hostname(self, hostname, prev_fn):
+    def update_hostname(self, hostname, fqdn, prev_fn):
         hostname_prev = self._read_hostname(prev_fn)
         hostname_in_etc = self._read_hostname("/etc/hostname")
         update_files = []
diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py
index bf3c18d2..e4c27216 100644
--- a/cloudinit/distros/rhel.py
+++ b/cloudinit/distros/rhel.py
@@ -146,8 +146,13 @@ class Distro(distros.Distro):
                 lines.insert(0, _make_header())
             util.write_file(fn, "\n".join(lines), 0644)
 
-    def set_hostname(self, hostname):
-        self._write_hostname(hostname, '/etc/sysconfig/network')
+    def set_hostname(self, hostname, fqdn=None):
+        # See: http://bit.ly/TwitgL
+        # Should be fqdn if we can use it
+        sysconfig_hostname = fqdn
+        if not sysconfig_hostname:
+            sysconfig_hostname = hostname
+        self._write_hostname(sysconfig_hostname, '/etc/sysconfig/network')
         LOG.debug("Setting hostname to %s", hostname)
         util.subp(['hostname', hostname])
 
@@ -165,28 +170,32 @@ class Distro(distros.Distro):
         }
         self._update_sysconfig_file(out_fn, host_cfg)
 
-    def update_hostname(self, hostname, prev_file):
+    def update_hostname(self, hostname, fqdn, prev_file):
+        # See: http://bit.ly/TwitgL
+        # Should be fqdn if we can use it
+        sysconfig_hostname = fqdn
+        if not sysconfig_hostname:
+            sysconfig_hostname = hostname
         hostname_prev = self._read_hostname(prev_file)
         hostname_in_sys = self._read_hostname("/etc/sysconfig/network")
         update_files = []
-        if not hostname_prev or hostname_prev != hostname:
+        if not hostname_prev or hostname_prev != sysconfig_hostname:
             update_files.append(prev_file)
         if (not hostname_in_sys or
             (hostname_in_sys == hostname_prev
-             and hostname_in_sys != hostname)):
+             and hostname_in_sys != sysconfig_hostname)):
             update_files.append("/etc/sysconfig/network")
         for fn in update_files:
             try:
-                self._write_hostname(hostname, fn)
+                self._write_hostname(sysconfig_hostname, fn)
             except:
                 util.logexc(LOG, "Failed to write hostname %s to %s",
-                            hostname, fn)
+                            sysconfig_hostname, fn)
         if (hostname_in_sys and hostname_prev and
             hostname_in_sys != hostname_prev):
             LOG.debug(("%s differs from /etc/sysconfig/network."
                         " Assuming user maintained hostname."), prev_file)
         if "/etc/sysconfig/network" in update_files:
-            # Only do this if we are running in non-adjusted root mode
             LOG.debug("Setting hostname to %s", hostname)
             util.subp(['hostname', hostname])
 
diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 985ce3e5..d26625a0 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -71,12 +71,17 @@ class FileLock(object):
         return "<%s using file %r>" % (util.obj_name(self), self.fn)
 
 
+def canon_sem_name(name):
+    return name.replace("-", "_")
+
+
 class FileSemaphores(object):
-    def __init__(self, sem_path):
+    def __init__(self, sem_path):   
         self.sem_path = sem_path
 
     @contextlib.contextmanager
     def lock(self, name, freq, clear_on_fail=False):
+        name = canon_sem_name(name)
         try:
             yield self._acquire(name, freq)
         except:
@@ -85,6 +90,7 @@ class FileSemaphores(object):
             raise
 
     def clear(self, name, freq):
+        name = canon_sem_name(name)
         sem_file = self._get_path(name, freq)
         try:
             util.del_file(sem_file)
@@ -119,6 +125,7 @@ class FileSemaphores(object):
     def has_run(self, name, freq):
         if not freq or freq == PER_ALWAYS:
             return False
+        name = canon_sem_name(name)
         sem_file = self._get_path(name, freq)
         # This isn't really a good atomic check
         # but it suffices for where and when cloudinit runs
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 4ed1a750..e0cf1cbe 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -36,6 +36,8 @@ from cloudinit.handlers import cloud_config as cc_part
 from cloudinit.handlers import shell_script as ss_part
 from cloudinit.handlers import upstart_job as up_part
 
+from cloudinit.sources import DataSourceNone
+
 from cloudinit import cloud
 from cloudinit import config
 from cloudinit import distros
@@ -58,8 +60,16 @@ class Init(object):
         self._cfg = None
         self._paths = None
         self._distro = None
-        # Created only when a fetch occurs
-        self.datasource = None
+        # Changed only when a fetch occurs
+        self.datasource = DataSourceNone.DataSourceNone({}, None, None)
+
+    def _reset(self, ds=False):
+        # Recreated on access
+        self._cfg = None
+        self._paths = None
+        self._distro = None
+        if ds:
+            self.datasource = DataSourceNone.DataSourceNone({}, None, None)
 
     @property
     def distro(self):
@@ -236,7 +246,7 @@ class Init(object):
         self.datasource = ds
         # Ensure we adjust our path members datasource
         # now that we have one (thus allowing ipath to be used)
-        self.paths.datasource = ds
+        self._reset()
         return ds
 
     def _get_instance_subdirs(self):
@@ -296,6 +306,10 @@ class Init(object):
         util.write_file(iid_fn, "%s\n" % iid)
         util.write_file(os.path.join(dp, 'previous-instance-id'),
                         "%s\n" % (previous_iid))
+        # Ensure needed components are regenerated
+        # after change of instance which may cause
+        # change of configuration
+        self._reset()
         return iid
 
     def fetch(self):
@@ -409,6 +423,17 @@ class Init(object):
             handlers.call_end(mod, data, frequency)
             called.append(mod)
 
+        # Perform post-consumption adjustments so that
+        # modules that run during the init stage reflect
+        # this consumed set.
+        #
+        # They will be recreated on future access...
+        self._reset()
+        # Note(harlowja): the 'active' datasource will have
+        # references to the previous config, distro, paths
+        # objects before the load of the userdata happened,
+        # this is expected.
+
 
 class Modules(object):
     def __init__(self, init, cfg_files=None):