pull in the rework of /var/lib/cloud.

- /var/lib/cloud is redesigned, and its layout now described in
  doc/var-lib-cloud.txt.
  The big plus point of this was to get instance specific data
  into /var/lib/cloud/instances, so that data could easily be purged.

  A symlink /var/lib/cloud/instance -> /var/lib/cloud/instances/<current_id>
  is maintained.

- Also, now run scripts in /var/lib/cloud/scripts/
   per-once
   per-boot
   per-instance

- bugs addressed:
  - LP: #704509

8 files changed, 217 insertions, 48 deletions

diff --git a/cloud-init-cfg.py b/cloud-init-cfg.py
index eb875182..326062a8 100755
--- a/cloud-init-cfg.py
+++ b/cloud-init-cfg.py
@@ -55,7 +55,7 @@ def main():
     log = logging.getLogger()
     log.info("cloud-init-cfg %s" % sys.argv[1:])
 
-    cfg_path = cloudinit.cloud_config
+    cfg_path = cloudinit.get_ipath_cur("cloud_config")
     cfg_env_name = cloudinit.cfg_env_name
     if os.environ.has_key(cfg_env_name):
         cfg_path = os.environ[cfg_env_name]
diff --git a/cloud-init.py b/cloud-init.py
index 28828648..bdfa9f01 100755
--- a/cloud-init.py
+++ b/cloud-init.py
@@ -56,6 +56,12 @@ def main():
     if cmd == "start-local":
         source_type = "local"
 
+    try:
+        cloudinit.initfs()
+    except Exception, e:
+        warn("failed to initfs, likely bad things to come: %s" % str(e))
+        
+
     cloudinit.logging_set_from_cfg_file()
     log = logging.getLogger()
     log.info(msg)
@@ -74,6 +80,9 @@ def main():
         sys.stderr.write("no instance data found in %s\n" % cmd)
         sys.exit(1)
 
+    # set this as the current instance
+    cloud.set_cur_instance()
+
     # store the metadata
     cloud.update_cache()
 
@@ -98,18 +107,12 @@ def main():
                 set_hostname, [ hostname, log ], False)
             cloud.sem_and_run("update_hostname", "always",
                 update_hostname, [ hostname, log ], False)
-    except:
+    except Exception, e:
+        util.logexc(log)
         warn("failed to set hostname\n")
 
     #print "user data is:" + cloud.get_user_data()
 
-    # set the defaults (like what ec2-set-defaults.py did)
-    try:
-        cloud.sem_and_run("set_defaults", "once-per-instance",
-            set_defaults,[ cloud ],False)
-    except:
-        warn("failed to set defaults\n")
-
     # finish, send the cloud-config event
     cloud.initctl_emit()
 
@@ -143,7 +146,7 @@ def set_hostname(hostname, log):
         log.error("failed to set_hostname")
 
 def update_hostname(hostname, log):
-    prev_file="%s/%s" % (cloudinit.datadir,"previous-hostname")
+    prev_file="%s/%s" % (cloudinit.get_cpath('datadir'),"previous-hostname")
     etc_file = "/etc/hostname"
 
     hostname_prev = None
diff --git a/cloudinit/CloudConfig/cc_runcmd.py b/cloudinit/CloudConfig/cc_runcmd.py
index 969f6394..afa7a441 100644
--- a/cloudinit/CloudConfig/cc_runcmd.py
+++ b/cloudinit/CloudConfig/cc_runcmd.py
@@ -21,8 +21,7 @@ import cloudinit.util as util
 def handle(name,cfg,cloud,log,args):
     if not cfg.has_key("runcmd"):
         return
-    outfile="%s/%s/runcmd" % \
-        (cloudinit.user_scripts_dir, cloud.get_instance_id())
+    outfile="%s/runcmd" % cloud.get_ipath('scripts')
 
     content="#!/bin/sh\n"
     escaped="%s%s%s%s" % ( "'", '\\', "'", "'" )
diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py
index 84826276..fc8fac5a 100644
--- a/cloudinit/DataSourceEc2.py
+++ b/cloudinit/DataSourceEc2.py
@@ -30,7 +30,7 @@ import errno
 
 class DataSourceEc2(DataSource.DataSource):
     api_ver  = '2009-04-04'
-    cachedir = cloudinit.cachedir + '/ec2'
+    seeddir = cloudinit.seeddir + '/ec2'
 
     def __init__(self):
         pass
@@ -40,10 +40,10 @@ class DataSourceEc2(DataSource.DataSource):
 
     def get_data(self):
         seedret={ }
-        if util.read_optional_seed(seedret,base=self.cachedir + "/"):
+        if util.read_optional_seed(seedret,base=self.seeddir+ "/"):
             self.userdata_raw = seedret['user-data']
             self.metadata = seedret['meta-data']
-            cloudinit.log.debug("using seeded ec2 data in %s" % self.cachedir)
+            cloudinit.log.debug("using seeded ec2 data in %s" % self.seeddir)
             return True
         
         try:
diff --git a/cloudinit/DataSourceNoCloud.py b/cloudinit/DataSourceNoCloud.py
index 78c9c9c8..2f6033a9 100644
--- a/cloudinit/DataSourceNoCloud.py
+++ b/cloudinit/DataSourceNoCloud.py
@@ -32,7 +32,7 @@ class DataSourceNoCloud(DataSource.DataSource):
     supported_seed_starts = ( "/" , "file://" )
     seed = None
     cmdline_id = "ds=nocloud"
-    seeddir = cloudinit.cachedir + '/nocloud'
+    seeddir = cloudinit.seeddir + '/nocloud'
 
     def __init__(self):
         pass
@@ -143,4 +143,4 @@ def parse_cmdline_data(ds_id,fill,cmdline=None):
 class DataSourceNoCloudNet(DataSourceNoCloud):
     cmdline_id = "ds=nocloud-net"
     supported_seed_starts = ( "http://", "https://", "ftp://" )
-    seeddir = cloudinit.cachedir + '/nocloud-net'
+    seeddir = cloudinit.seeddir + '/nocloud-net'
diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py
index be366d4c..296af051 100644
--- a/cloudinit/__init__.py
+++ b/cloudinit/__init__.py
@@ -18,26 +18,35 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-datadir = '/var/lib/cloud/data'
-semdir = '/var/lib/cloud/sem'
-pluginsdir = datadir + '/plugins'
-cachedir = datadir + '/cache'
-userdata_raw = datadir + '/user-data.txt'
-userdata = datadir + '/user-data.txt.i'
-user_scripts_dir = datadir + "/scripts"
-boothooks_dir = datadir + "/boothooks"
-cloud_config = datadir + '/cloud-config.txt'
-data_source_cache = cachedir + '/obj.pkl'
+varlibdir = '/var/lib/cloud'
+cur_instance_link = varlibdir + "/instance"
 system_config = '/etc/cloud/cloud.cfg'
+seeddir = varlibdir + "/seed"
 cfg_env_name = "CLOUD_CFG"
 
 def_log_file = '/var/log/cloud-init.log'
+def_log_user = "syslog"
+def_log_group = "adm"
+
 cfg_builtin = """
 log_cfgs: [ ]
 cloud_type: auto
 """
 logger_name = "cloudinit"
 
+pathmap = {
+   "handlers" : "/handlers",
+   "scripts" : "/scripts",
+   "sem" : "/sem",
+   "boothooks" : "/boothooks",
+   "userdata_raw" : "/user-data.txt",
+   "userdata" : "/user-data-raw.txt.i",
+   "obj_pkl" : "/obj.pkl",
+   "cloud_config" : "/cloud-config.txt",
+   "datadir" : "/data",
+   None : "",
+}
+
 import os
 from   configobj import ConfigObj
 
@@ -52,9 +61,10 @@ import util
 import logging
 import logging.config
 import StringIO
+import glob
 
 class NullHandler(logging.Handler):
-	def emit(self,record): pass
+    def emit(self,record): pass
 
 log = logging.getLogger(logger_name)
 log.addHandler(NullHandler())
@@ -104,7 +114,6 @@ class CloudInit:
         "all": ( "nocloud-net", "ec2" ),
         "local" : ( "nocloud", ),
     }
-
     cfg = None
     part_handlers = { }
     old_conffile = '/etc/ec2-init/ec2-config.cfg'
@@ -142,7 +151,11 @@ class CloudInit:
 
     def restore_from_cache(self):
         try:
-            f=open(data_source_cache, "rb")
+            # we try to restore from a current link and static path
+            # by using the instance link, if purge_cache was called
+            # the file wont exist
+            cache = get_ipath_cur('obj_pkl')
+            f=open(cache, "rb")
             data = cPickle.load(f)
             self.datasource = data
             return True
@@ -150,16 +163,17 @@ class CloudInit:
             return False
 
     def write_to_cache(self):
+        cache = self.get_ipath("obj_pkl")
         try:
-            os.makedirs(os.path.dirname(data_source_cache))
+            os.makedirs(os.path.dirname(cache))
         except OSError as e:
             if e.errno != errno.EEXIST:
                 return False
                 
         try:
-            f=open(data_source_cache, "wb")
+            f=open(cache, "wb")
             data = cPickle.dump(self.datasource,f)
-            os.chmod(data_source_cache,0400)
+            os.chmod(cache,0400)
             return True
         except:
             return False
@@ -182,6 +196,7 @@ class CloudInit:
             for ds in cfglist.split(','):
                 dslist.append(strip(ds).tolower())
             
+        log.debug("searching for data source in [%s]" % str(dslist))
         for ds in dslist:
             if ds not in self.datasource_map:
                 log.warn("data source %s not found in map" % ds)
@@ -200,6 +215,20 @@ class CloudInit:
         log.debug("did not find data source from %s" % dslist)
         raise DataSourceNotFoundException("Could not find data source")
 
+    def set_cur_instance(self):
+        try:
+            os.unlink(cur_instance_link)
+        except OSError, e:
+            if e.errno != errno.ENOENT: raise
+
+        os.symlink("./instances/%s" % self.get_instance_id(), cur_instance_link)
+        idir = self.get_ipath()
+        dlist = []
+        for d in [ "handlers", "scripts", "sem" ]:
+            dlist.append("%s/%s" % (idir, d))
+            
+        util.ensure_dirs(dlist)
+
     def get_userdata(self):
         return(self.datasource.get_userdata())
 
@@ -214,19 +243,20 @@ class CloudInit:
         self.store_userdata()
 
     def store_userdata(self):
-        util.write_file(userdata_raw, self.datasource.get_userdata_raw(), 0600)
-        util.write_file(userdata, self.datasource.get_userdata(), 0600)
+        util.write_file(self.get_ipath('userdata_raw'),
+            self.datasource.get_userdata_raw(), 0600)
+        util.write_file(self.get_ipath('userdata'),
+            self.datasource.get_userdata(), 0600)
 
     def initctl_emit(self):
+        cc_path = get_ipath_cur('cloud_config')
         subprocess.Popen(['initctl', 'emit', 'cloud-config',
-            '%s=%s' % (cfg_env_name,cloud_config)]).communicate()
+            '%s=%s' % (cfg_env_name,cc_path)]).communicate()
 
     def sem_getpath(self,name,freq):
-        freqtok = freq
         if freq == 'once-per-instance':
-            freqtok = self.datasource.get_instance_id()
-
-        return("%s/%s.%s" % (semdir,name,freqtok))
+            return("%s/%s" % (self.get_ipath("sem"),name))
+        return("%s/%s.%s" % (get_cpath("sem"), name, freq))
     
     def sem_has_run(self,name,freq):
         if freq == "always": return False
@@ -285,9 +315,40 @@ class CloudInit:
                 self.sem_clear(semname,freq)
             raise
 
+    # get_ipath : get the instance path for a name in pathmap
+    # (/var/lib/cloud/instances/<instance>/name)<name>)
+    def get_ipath(self, name=None):
+        return("%s/instances/%s%s" 
+               % (varlibdir,self.get_instance_id(), pathmap[name]))
+
     def consume_userdata(self):
         self.get_userdata()
         data = self
+
+        cdir = get_cpath("handlers")
+        idir = self.get_ipath("handlers")
+
+        # add the path to the plugins dir to the top of our list for import
+        # instance dir should be read before cloud-dir
+        sys.path.insert(0,cdir)
+        sys.path.insert(0,idir)
+
+        # add handlers in cdir
+        for fname in glob.glob("%s/*.py" % cdir):
+            if not os.path.isfile(fname): continue
+            modname = os.path.basename(fname)[0:-3]
+            try:
+                mod = __import__(modname)
+                lister = getattr(mod, "list_types")
+                handler = getattr(mod, "handle_part")
+                mtypes = lister()
+                for mtype in mtypes:
+                    self.part_handlers[mtype]=handler
+                log.debug("added handler for [%s] from %s" % (mtypes,fname))
+            except:
+                log.warn("failed to initialize handler in %s" % fname)
+                util.logexc(log)
+       
         # give callbacks opportunity to initialize
         for ctype, func in self.part_handlers.items():
             func(data, "__begin__",None,None)
@@ -304,16 +365,13 @@ class CloudInit:
             self.handlercount = 0
             return
 
-        # add the path to the plugins dir to the top of our list for import
-        if self.handlercount == 0:
-            sys.path.insert(0,pluginsdir)
-
         self.handlercount=self.handlercount+1
 
-        # write content to pluginsdir
+        # write content to instance's handlerdir
+        handlerdir = self.get_ipath("handler")
         modname  = 'part-handler-%03d' % self.handlercount
         modfname = modname + ".py"
-        util.write_file("%s/%s" % (pluginsdir,modfname), payload, 0600)
+        util.write_file("%s/%s" % (handlerdir,modfname), payload, 0600)
 
         try:
             mod = __import__(modname)
@@ -338,8 +396,9 @@ class CloudInit:
             return
 
         filename=filename.replace(os.sep,'_')
+        scriptsdir = get_ipath_cur('scripts')
         util.write_file("%s/%s/%s" % 
-            (user_scripts_dir,self.get_instance_id(),filename), payload, 0700)
+            (scriptsdir,self.get_instance_id(),filename), payload, 0700)
 
     def handle_upstart_job(self,data,ctype,filename,payload):
         if ctype == "__end__" or ctype == "__begin__": return
@@ -353,6 +412,7 @@ class CloudInit:
             self.cloud_config_str=""
             return
         if ctype == "__end__":
+            cloud_config = self.get_ipath("cloud_config")
             util.write_file(cloud_config, self.cloud_config_str, 0600)
 
             ## this could merge the cloud config with the system config
@@ -389,6 +449,7 @@ class CloudInit:
         elif start != 0:
             payload=payload[start:]
     
+        boothooks_dir = self.get_ipath("boothooks")
         filepath = "%s/%s" % (boothooks_dir,filename)
         util.write_file(filepath, payload, 0700)
         try:
@@ -418,14 +479,36 @@ class CloudInit:
         return(self.datasource.device_name_to_device(name))
 
 
+def initfs():
+    subds = [ 'scripts/per-instance', 'scripts/per-once', 'scripts/per-boot',
+              'seed', 'instances', 'handlers', 'sem', 'data' ]
+    dlist = [ ]
+    for subd in subds:
+        dlist.append("%s/%s" % (varlibdir, subd))
+    util.ensure_dirs(dlist)
+
+    fp = open(def_log_file,"ab")
+    fp.close()
+    util.chownbyname(def_log_file,def_log_user, def_log_group)
+        
+    
+
 def purge_cache():
     try:
-        os.unlink(data_source_cache)
+        os.unlink(cur_instance_link)
     except OSError as e:
         if e.errno != errno.ENOENT: return(False)
     except:
         return(False)
     return(True)
 
+def get_ipath_cur(name=None):
+    return("%s/instance/%s" % (varlibdir, pathmap[name]))
+
+# get_cpath : get the "clouddir" (/var/lib/cloud/<name>)
+# for a name in dirmap
+def get_cpath(name=None):
+    return("%s%s" % (varlibdir, pathmap[name]))
+
 class DataSourceNotFoundException(Exception):
     pass
diff --git a/cloudinit/util.py b/cloudinit/util.py
index d5ae2bec..a2291164 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -278,3 +278,31 @@ def read_cc_from_cmdline(cmdline=None):
         begin = cmdline.find(tag_begin, end + end_l)
 
     return('\n'.join(tokens))
+
+def ensure_dirs(dirlist, mode=0755):
+    fixmodes = []
+    for d in dirlist:
+        try:
+            if mode != None:
+                os.makedirs(d)
+            else:
+                os.makedirs(d, mode)
+        except OSError as e:
+            if e.errno != errno.EEXIST: raise
+            if mode != None: fixmodes.append(d)
+
+    for d in fixmodes:
+        os.chmod(d, mode)
+
+def chownbyname(fname,user=None,group=None):
+   uid = -1
+   gid = -1
+   if user == None and group == None: return
+   if user:
+      import pwd
+      uid = pwd.getpwnam(user).pw_uid
+   if group:
+      import grp
+      gid = grp.getgrnam(group).gr_gid
+
+   os.chown(fname,uid,gid)
diff --git a/doc/var-lib-cloud.txt b/doc/var-lib-cloud.txt
new file mode 100644
index 00000000..2a1acd2b
--- /dev/null
+++ b/doc/var-lib-cloud.txt
@@ -0,0 +1,56 @@
+/var/lib/cloud has the following structure:
+  - scripts/
+      per-instance/
+      per-boot/
+      per-once/
+
+    files in these directories will be run by 'run-parts' once per
+    instance, once per boot, and once per *ever*.
+
+  - seed/
+     <datasource>/
+        sys-user-data
+        user-data
+        meta-data
+
+     The 'seed/' directory allows you to seed a specific datasource
+     For example, to seed the 'nocloud' datasource you would need to
+     populate 
+       seed/nocloud/user-data
+       seed/nocloud/meta-data
+
+  - instance -> instances/i-abcde
+    This is a symlink to the current instance/<instance-id> directory
+    created/updated on boot
+  - instances/
+      i-abcdefgh/
+         scripts/ # all scripts in scripts are per-instance
+         sem/
+            config-puppet
+            config-ssh
+            set-hostname
+         cloud-config.txt
+         user-data.txt
+         user-data.txt.i
+         obj.pkl
+         handlers/
+         data/  # just a per-instance data location to be used
+          
+  - sem/
+      scripts.once
+    These are the cloud-specific semaphores.  The only thing that
+    would go here are files to mark that a "per-once" script
+    has run.
+
+  - handlers/
+      "persistent" handlers (not per-instance).  Same as handlers
+      from user-data, just will be cross-instance id
+
+  - data/
+      this is a persistent data location.  cloud-init won't really
+      use it, but something else (a handler or script could)
+
+to clear out the current instance's data as if to force a "new run" on reboot
+do:
+  ( cd /var/lib/cloud/instance && sudo rm -Rf * )
+