From 32eab86a04f4f6056b200e1cfac99ec0495cd5c0 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 19 Jan 2011 15:34:10 +0000
Subject: add doc about redesigned /var/lib/cloud

---
 doc/var-lib-cloud.txt | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 doc/var-lib-cloud.txt

diff --git a/doc/var-lib-cloud.txt b/doc/var-lib-cloud.txt
new file mode 100644
index 00000000..328edd01
--- /dev/null
+++ b/doc/var-lib-cloud.txt
@@ -0,0 +1,46 @@
+/var/lib/cloud has the following structure:
+  - scripts/
+      per-instance/
+      per-boot/
+      per-once/
+
+    files in these directories will be run by 'run-parts' once per
+    instance, once per boot, and once per *ever*.
+
+  - seed/
+     <datasource>/
+        sys-user-data
+        user-data
+        meta-data
+
+     The 'seed/' directory allows you to seed a specific datasource
+     For example, to seed the 'nocloud' datasource you would need to
+     populate 
+       seed/nocloud/user-data
+       seed/nocloud/meta-data
+
+  - instance -> instances/i-abcde
+    This is a symlink to the current instance/<instance-id> directory
+    created/updated on boot
+  - instances/
+      i-abcdefgh/
+         scripts/ # all scripts in scripts are per-instance
+         sem/
+            config-puppet
+            config-ssh
+            set-hostname
+         cloud-config.txt
+         user-data.txt
+         user-data.txt.i
+         obj.pkl
+  - sem/
+      scripts.once
+    These are the cloud-specific semaphores.  The only thing that
+    would go here are files to mark that a "per-once" script
+    has run.
+    
+
+to clear out the current instance's data as if to force a "new run" on reboot
+do:
+  ( cd /var/lib/cloud/instance && sudo rm -Rf * )
+
-- 
cgit v1.2.3


From ee03480e670c10d48a4825a458fac18aaf4fbec0 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 19 Jan 2011 18:35:07 +0000
Subject: initial /var/lib rework still lots to do. includes a fix for LP:
 #704509

LP: #704509
---
 cloud-init.py         | 17 ++++++----
 cloudinit/__init__.py | 93 ++++++++++++++++++++++++++++++++++++++++++++-------
 cloudinit/util.py     | 28 ++++++++++++++++
 doc/var-lib-cloud.txt |  7 +++-
 4 files changed, 125 insertions(+), 20 deletions(-)

diff --git a/cloud-init.py b/cloud-init.py
index 28828648..1278d2eb 100755
--- a/cloud-init.py
+++ b/cloud-init.py
@@ -56,6 +56,13 @@ def main():
     if cmd == "start-local":
         source_type = "local"
 
+    if cmd == "start-local":
+        try:
+            cloudinit.initfs()
+        except:
+            warn("failed to initfs, likely bad things to come")
+        
+
     cloudinit.logging_set_from_cfg_file()
     log = logging.getLogger()
     log.info(msg)
@@ -74,6 +81,9 @@ def main():
         sys.stderr.write("no instance data found in %s\n" % cmd)
         sys.exit(1)
 
+    # set this as the current instance
+    cloud.set_cur_instance()
+
     # store the metadata
     cloud.update_cache()
 
@@ -103,13 +113,6 @@ def main():
 
     #print "user data is:" + cloud.get_user_data()
 
-    # set the defaults (like what ec2-set-defaults.py did)
-    try:
-        cloud.sem_and_run("set_defaults", "once-per-instance",
-            set_defaults,[ cloud ],False)
-    except:
-        warn("failed to set defaults\n")
-
     # finish, send the cloud-config event
     cloud.initctl_emit()
 
diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py
index be366d4c..1de32f2e 100644
--- a/cloudinit/__init__.py
+++ b/cloudinit/__init__.py
@@ -18,9 +18,9 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+varlibdir = '/var/lib/cloud'
 datadir = '/var/lib/cloud/data'
 semdir = '/var/lib/cloud/sem'
-pluginsdir = datadir + '/plugins'
 cachedir = datadir + '/cache'
 userdata_raw = datadir + '/user-data.txt'
 userdata = datadir + '/user-data.txt.i'
@@ -32,6 +32,9 @@ system_config = '/etc/cloud/cloud.cfg'
 cfg_env_name = "CLOUD_CFG"
 
 def_log_file = '/var/log/cloud-init.log'
+def_log_user = "syslog"
+def_log_group = "adm"
+
 cfg_builtin = """
 log_cfgs: [ ]
 cloud_type: auto
@@ -52,9 +55,10 @@ import util
 import logging
 import logging.config
 import StringIO
+import glob
 
 class NullHandler(logging.Handler):
-	def emit(self,record): pass
+    def emit(self,record): pass
 
 log = logging.getLogger(logger_name)
 log.addHandler(NullHandler())
@@ -104,6 +108,12 @@ class CloudInit:
         "all": ( "nocloud-net", "ec2" ),
         "local" : ( "nocloud", ),
     }
+    dirmap = {
+       "handlers" : "/handlers",
+       "scripts" : "/scripts",
+       "sem" : "/sem",
+       None : "",
+    }
 
     cfg = None
     part_handlers = { }
@@ -200,6 +210,21 @@ class CloudInit:
         log.debug("did not find data source from %s" % dslist)
         raise DataSourceNotFoundException("Could not find data source")
 
+    def set_cur_instance(self):
+        lname = "%s/instance" % varlibdir
+        try:
+            os.unlink(lname)
+        except OSError, e:
+            if e.errno != errno.ENOENT: raise
+
+        os.symlink("./instances/%s" % self.get_instance_id(), lname)
+        idir = self.get_idir()
+        dlist = []
+        for d in [ "handlers", "scripts", "sem" ]:
+            dlist.append("%s/%s" % (idir, d))
+            
+        util.ensure_dirs(dlist)
+
     def get_userdata(self):
         return(self.datasource.get_userdata())
 
@@ -222,11 +247,9 @@ class CloudInit:
             '%s=%s' % (cfg_env_name,cloud_config)]).communicate()
 
     def sem_getpath(self,name,freq):
-        freqtok = freq
         if freq == 'once-per-instance':
-            freqtok = self.datasource.get_instance_id()
-
-        return("%s/%s.%s" % (semdir,name,freqtok))
+            return("%s/%s" % (self.get_idir("sem"),name))
+        return("%s/%s.%s" % (self.get_cdir("sem"), name, freq))
     
     def sem_has_run(self,name,freq):
         if freq == "always": return False
@@ -285,9 +308,45 @@ class CloudInit:
                 self.sem_clear(semname,freq)
             raise
 
+    # get_cdir : get the "clouddir" (/var/lib/cloud/<name>)
+    # for a name in dirmap
+    def get_idir(self, name=None):
+        return("%s/instances/%s%s" 
+               % (varlibdir,self.get_instance_id(), self.dirmap[name]))
+
+    # get_cdir : get the "clouddir" (/var/lib/cloud/<name>)
+    # for a name in dirmap
+    def get_cdir(self, name=None):
+        return("%s%s" % (varlibdir, self.dirmap[name]))
+
     def consume_userdata(self):
         self.get_userdata()
         data = self
+
+        cdir = self.get_cdir("handlers")
+        idir = self.get_idir("handlers")
+
+        # add the path to the plugins dir to the top of our list for import
+        # instance dir should be read before cloud-dir
+        sys.path.insert(0,cdir)
+        sys.path.insert(0,idir)
+
+        # add handlers in cdir
+        for fname in glob.glob("%s/*.py" % cdir):
+            if not os.path.isfile(fname): continue
+            modname = os.path.basename(fname)[0:-3]
+            try:
+                mod = __import__(modname)
+                lister = getattr(mod, "list_types")
+                handler = getattr(mod, "handle_part")
+                mtypes = lister()
+                for mtype in mtypes:
+                    self.part_handlers[mtype]=handler
+                log.debug("added handler for [%s] from %s" % (mtypes,fname))
+            except:
+                log.warn("failed to initialize handler in %s" % fname)
+                util.logexc(log)
+       
         # give callbacks opportunity to initialize
         for ctype, func in self.part_handlers.items():
             func(data, "__begin__",None,None)
@@ -304,16 +363,13 @@ class CloudInit:
             self.handlercount = 0
             return
 
-        # add the path to the plugins dir to the top of our list for import
-        if self.handlercount == 0:
-            sys.path.insert(0,pluginsdir)
-
         self.handlercount=self.handlercount+1
 
-        # write content to pluginsdir
+        # write content to instance's handlerdir
+        handlerdir = self.get_idir("handler")
         modname  = 'part-handler-%03d' % self.handlercount
         modfname = modname + ".py"
-        util.write_file("%s/%s" % (pluginsdir,modfname), payload, 0600)
+        util.write_file("%s/%s" % (handlerdir,modfname), payload, 0600)
 
         try:
             mod = __import__(modname)
@@ -418,6 +474,19 @@ class CloudInit:
         return(self.datasource.device_name_to_device(name))
 
 
+def initfs():
+    subds = [ 'scripts', 'seed', 'instances', 'handlers', 'sem' ]
+    dlist = [ ]
+    for subd in subds:
+        dlist.append("%s/%s" % (varlibdir, subd))
+    util.ensure_dirs(dlist)
+
+    fp = open(def_log_file,"ab")
+    fp.close()
+    util.chownbyname(def_log_file,def_log_user, def_log_group)
+        
+    
+
 def purge_cache():
     try:
         os.unlink(data_source_cache)
diff --git a/cloudinit/util.py b/cloudinit/util.py
index d5ae2bec..a2291164 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -278,3 +278,31 @@ def read_cc_from_cmdline(cmdline=None):
         begin = cmdline.find(tag_begin, end + end_l)
 
     return('\n'.join(tokens))
+
+def ensure_dirs(dirlist, mode=0755):
+    fixmodes = []
+    for d in dirlist:
+        try:
+            if mode != None:
+                os.makedirs(d)
+            else:
+                os.makedirs(d, mode)
+        except OSError as e:
+            if e.errno != errno.EEXIST: raise
+            if mode != None: fixmodes.append(d)
+
+    for d in fixmodes:
+        os.chmod(d, mode)
+
+def chownbyname(fname,user=None,group=None):
+   uid = -1
+   gid = -1
+   if user == None and group == None: return
+   if user:
+      import pwd
+      uid = pwd.getpwnam(user).pw_uid
+   if group:
+      import grp
+      gid = grp.getgrnam(group).gr_gid
+
+   os.chown(fname,uid,gid)
diff --git a/doc/var-lib-cloud.txt b/doc/var-lib-cloud.txt
index 328edd01..5cdcddbb 100644
--- a/doc/var-lib-cloud.txt
+++ b/doc/var-lib-cloud.txt
@@ -33,12 +33,17 @@
          user-data.txt
          user-data.txt.i
          obj.pkl
+         handlers/
   - sem/
       scripts.once
     These are the cloud-specific semaphores.  The only thing that
     would go here are files to mark that a "per-once" script
     has run.
-    
+
+  - handlers/
+      "persistent" handlers (not per-instance).  Same as handlers
+      from user-data, just will be cross-instance id
+
 
 to clear out the current instance's data as if to force a "new run" on reboot
 do:
-- 
cgit v1.2.3


From 9f4a828037e76ab6636f79a30eaa56fc6fddef52 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 19 Jan 2011 18:49:40 +0000
Subject: cloud-init.py: log exception on failure to set hostname

---
 cloud-init.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cloud-init.py b/cloud-init.py
index 1278d2eb..bda1ef8f 100755
--- a/cloud-init.py
+++ b/cloud-init.py
@@ -108,7 +108,8 @@ def main():
                 set_hostname, [ hostname, log ], False)
             cloud.sem_and_run("update_hostname", "always",
                 update_hostname, [ hostname, log ], False)
-    except:
+    except Exception, e:
+        util.logexc(log)
         warn("failed to set hostname\n")
 
     #print "user data is:" + cloud.get_user_data()
-- 
cgit v1.2.3


From 1aff955b2e322e55dd09036764fb5a2542efdcc1 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 19 Jan 2011 18:50:46 +0000
Subject: move boothooks and user-data into instance dir

---
 cloudinit/__init__.py | 38 ++++++++++++++++++++++----------------
 1 file changed, 22 insertions(+), 16 deletions(-)

diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py
index 1de32f2e..04da87e6 100644
--- a/cloudinit/__init__.py
+++ b/cloudinit/__init__.py
@@ -108,10 +108,13 @@ class CloudInit:
         "all": ( "nocloud-net", "ec2" ),
         "local" : ( "nocloud", ),
     }
-    dirmap = {
+    pathmap = {
        "handlers" : "/handlers",
        "scripts" : "/scripts",
        "sem" : "/sem",
+       "boothooks" : "/boothooks",
+       "userdata_raw" : "/user-data.txt",
+       "userdata" : "/user-data-raw.txt.i",
        None : "",
     }
 
@@ -218,7 +221,7 @@ class CloudInit:
             if e.errno != errno.ENOENT: raise
 
         os.symlink("./instances/%s" % self.get_instance_id(), lname)
-        idir = self.get_idir()
+        idir = self.get_ipath()
         dlist = []
         for d in [ "handlers", "scripts", "sem" ]:
             dlist.append("%s/%s" % (idir, d))
@@ -239,8 +242,10 @@ class CloudInit:
         self.store_userdata()
 
     def store_userdata(self):
-        util.write_file(userdata_raw, self.datasource.get_userdata_raw(), 0600)
-        util.write_file(userdata, self.datasource.get_userdata(), 0600)
+        util.write_file(self.get_ipath('userdata_raw'),
+            self.datasource.get_userdata_raw(), 0600)
+        util.write_file(self.get_ipath('userdata'),
+            self.datasource.get_userdata(), 0600)
 
     def initctl_emit(self):
         subprocess.Popen(['initctl', 'emit', 'cloud-config',
@@ -248,8 +253,8 @@ class CloudInit:
 
     def sem_getpath(self,name,freq):
         if freq == 'once-per-instance':
-            return("%s/%s" % (self.get_idir("sem"),name))
-        return("%s/%s.%s" % (self.get_cdir("sem"), name, freq))
+            return("%s/%s" % (self.get_ipath("sem"),name))
+        return("%s/%s.%s" % (self.get_cpath("sem"), name, freq))
     
     def sem_has_run(self,name,freq):
         if freq == "always": return False
@@ -308,23 +313,23 @@ class CloudInit:
                 self.sem_clear(semname,freq)
             raise
 
-    # get_cdir : get the "clouddir" (/var/lib/cloud/<name>)
-    # for a name in dirmap
-    def get_idir(self, name=None):
+    # get_ipath : get the instance path for a name in pathmap
+    # (/var/lib/cloud/instances/<instance>/name)<name>)
+    def get_ipath(self, name=None):
         return("%s/instances/%s%s" 
-               % (varlibdir,self.get_instance_id(), self.dirmap[name]))
+               % (varlibdir,self.get_instance_id(), self.pathmap[name]))
 
-    # get_cdir : get the "clouddir" (/var/lib/cloud/<name>)
+    # get_cpath : get the "clouddir" (/var/lib/cloud/<name>)
     # for a name in dirmap
-    def get_cdir(self, name=None):
-        return("%s%s" % (varlibdir, self.dirmap[name]))
+    def get_cpath(self, name=None):
+        return("%s%s" % (varlibdir, self.pathmap[name]))
 
     def consume_userdata(self):
         self.get_userdata()
         data = self
 
-        cdir = self.get_cdir("handlers")
-        idir = self.get_idir("handlers")
+        cdir = self.get_cpath("handlers")
+        idir = self.get_ipath("handlers")
 
         # add the path to the plugins dir to the top of our list for import
         # instance dir should be read before cloud-dir
@@ -366,7 +371,7 @@ class CloudInit:
         self.handlercount=self.handlercount+1
 
         # write content to instance's handlerdir
-        handlerdir = self.get_idir("handler")
+        handlerdir = self.get_ipath("handler")
         modname  = 'part-handler-%03d' % self.handlercount
         modfname = modname + ".py"
         util.write_file("%s/%s" % (handlerdir,modfname), payload, 0600)
@@ -445,6 +450,7 @@ class CloudInit:
         elif start != 0:
             payload=payload[start:]
     
+        boothooks_dir = self.get_ipath("boothooks")
         filepath = "%s/%s" % (boothooks_dir,filename)
         util.write_file(filepath, payload, 0700)
         try:
-- 
cgit v1.2.3


From a4edfa9ec71e901c53dc155d1ae910044a91cdf2 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 19 Jan 2011 18:54:15 +0000
Subject: improve log message on failure of initfs. run on both start-local and
 start

---
 cloud-init.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/cloud-init.py b/cloud-init.py
index bda1ef8f..32add699 100755
--- a/cloud-init.py
+++ b/cloud-init.py
@@ -56,11 +56,10 @@ def main():
     if cmd == "start-local":
         source_type = "local"
 
-    if cmd == "start-local":
-        try:
-            cloudinit.initfs()
-        except:
-            warn("failed to initfs, likely bad things to come")
+    try:
+        cloudinit.initfs()
+    except Exception, e:
+        warn("failed to initfs, likely bad things to come: %s" % str(e))
         
 
     cloudinit.logging_set_from_cfg_file()
-- 
cgit v1.2.3


From 01512fdad97b581cd8ac3da8758e704b060f3d03 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 19 Jan 2011 19:15:18 +0000
Subject: move cache to instance specific dir

---
 cloudinit/__init__.py | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py
index 04da87e6..32b91979 100644
--- a/cloudinit/__init__.py
+++ b/cloudinit/__init__.py
@@ -19,6 +19,7 @@
 #
 
 varlibdir = '/var/lib/cloud'
+cur_instance_link = varlibdir + "/instance"
 datadir = '/var/lib/cloud/data'
 semdir = '/var/lib/cloud/sem'
 cachedir = datadir + '/cache'
@@ -115,6 +116,7 @@ class CloudInit:
        "boothooks" : "/boothooks",
        "userdata_raw" : "/user-data.txt",
        "userdata" : "/user-data-raw.txt.i",
+       "obj_pkl" : "/obj.pkl",
        None : "",
     }
 
@@ -155,7 +157,11 @@ class CloudInit:
 
     def restore_from_cache(self):
         try:
-            f=open(data_source_cache, "rb")
+            # we try to restore from a current link and static path
+            # by using the instance link, if purge_cache was called
+            # the file wont exist
+            cache = "%s/%s" % (cur_instance_link, self.pathmap['obj_pkl'])
+            f=open(cache, "rb")
             data = cPickle.load(f)
             self.datasource = data
             return True
@@ -163,16 +169,17 @@ class CloudInit:
             return False
 
     def write_to_cache(self):
+        cache = self.get_ipath("obj_pkl")
         try:
-            os.makedirs(os.path.dirname(data_source_cache))
+            os.makedirs(os.path.dirname(cache))
         except OSError as e:
             if e.errno != errno.EEXIST:
                 return False
                 
         try:
-            f=open(data_source_cache, "wb")
+            f=open(cache, "wb")
             data = cPickle.dump(self.datasource,f)
-            os.chmod(data_source_cache,0400)
+            os.chmod(cache,0400)
             return True
         except:
             return False
@@ -195,6 +202,7 @@ class CloudInit:
             for ds in cfglist.split(','):
                 dslist.append(strip(ds).tolower())
             
+        log.debug("searching for data source in [%s]" % str(dslist))
         for ds in dslist:
             if ds not in self.datasource_map:
                 log.warn("data source %s not found in map" % ds)
@@ -214,13 +222,12 @@ class CloudInit:
         raise DataSourceNotFoundException("Could not find data source")
 
     def set_cur_instance(self):
-        lname = "%s/instance" % varlibdir
         try:
-            os.unlink(lname)
+            os.unlink(cur_instance_link)
         except OSError, e:
             if e.errno != errno.ENOENT: raise
 
-        os.symlink("./instances/%s" % self.get_instance_id(), lname)
+        os.symlink("./instances/%s" % self.get_instance_id(), cur_instance_link)
         idir = self.get_ipath()
         dlist = []
         for d in [ "handlers", "scripts", "sem" ]:
@@ -495,7 +502,7 @@ def initfs():
 
 def purge_cache():
     try:
-        os.unlink(data_source_cache)
+        os.unlink(cur_instance_link)
     except OSError as e:
         if e.errno != errno.ENOENT: return(False)
     except:
-- 
cgit v1.2.3


From 7434e3306d2526173c92ab57db30a0812cc47717 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 19 Jan 2011 19:46:15 +0000
Subject: convert 'cachedir' to 'seeddir', move cloud_config, scripts to
 instance

- cloud_config and scripts now live in instance directory
- cachedir is now more correctly named 'seeddir'
---
 cloud-init-cfg.py                  |  2 +-
 cloudinit/CloudConfig/cc_runcmd.py |  3 +--
 cloudinit/DataSourceEc2.py         |  6 ++---
 cloudinit/DataSourceNoCloud.py     |  4 ++--
 cloudinit/__init__.py              | 49 +++++++++++++++++++-------------------
 5 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/cloud-init-cfg.py b/cloud-init-cfg.py
index eb875182..326062a8 100755
--- a/cloud-init-cfg.py
+++ b/cloud-init-cfg.py
@@ -55,7 +55,7 @@ def main():
     log = logging.getLogger()
     log.info("cloud-init-cfg %s" % sys.argv[1:])
 
-    cfg_path = cloudinit.cloud_config
+    cfg_path = cloudinit.get_ipath_cur("cloud_config")
     cfg_env_name = cloudinit.cfg_env_name
     if os.environ.has_key(cfg_env_name):
         cfg_path = os.environ[cfg_env_name]
diff --git a/cloudinit/CloudConfig/cc_runcmd.py b/cloudinit/CloudConfig/cc_runcmd.py
index 969f6394..afa7a441 100644
--- a/cloudinit/CloudConfig/cc_runcmd.py
+++ b/cloudinit/CloudConfig/cc_runcmd.py
@@ -21,8 +21,7 @@ import cloudinit.util as util
 def handle(name,cfg,cloud,log,args):
     if not cfg.has_key("runcmd"):
         return
-    outfile="%s/%s/runcmd" % \
-        (cloudinit.user_scripts_dir, cloud.get_instance_id())
+    outfile="%s/runcmd" % cloud.get_ipath('scripts')
 
     content="#!/bin/sh\n"
     escaped="%s%s%s%s" % ( "'", '\\', "'", "'" )
diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py
index 84826276..fc8fac5a 100644
--- a/cloudinit/DataSourceEc2.py
+++ b/cloudinit/DataSourceEc2.py
@@ -30,7 +30,7 @@ import errno
 
 class DataSourceEc2(DataSource.DataSource):
     api_ver  = '2009-04-04'
-    cachedir = cloudinit.cachedir + '/ec2'
+    seeddir = cloudinit.seeddir + '/ec2'
 
     def __init__(self):
         pass
@@ -40,10 +40,10 @@ class DataSourceEc2(DataSource.DataSource):
 
     def get_data(self):
         seedret={ }
-        if util.read_optional_seed(seedret,base=self.cachedir + "/"):
+        if util.read_optional_seed(seedret,base=self.seeddir+ "/"):
             self.userdata_raw = seedret['user-data']
             self.metadata = seedret['meta-data']
-            cloudinit.log.debug("using seeded ec2 data in %s" % self.cachedir)
+            cloudinit.log.debug("using seeded ec2 data in %s" % self.seeddir)
             return True
         
         try:
diff --git a/cloudinit/DataSourceNoCloud.py b/cloudinit/DataSourceNoCloud.py
index 78c9c9c8..2f6033a9 100644
--- a/cloudinit/DataSourceNoCloud.py
+++ b/cloudinit/DataSourceNoCloud.py
@@ -32,7 +32,7 @@ class DataSourceNoCloud(DataSource.DataSource):
     supported_seed_starts = ( "/" , "file://" )
     seed = None
     cmdline_id = "ds=nocloud"
-    seeddir = cloudinit.cachedir + '/nocloud'
+    seeddir = cloudinit.seeddir + '/nocloud'
 
     def __init__(self):
         pass
@@ -143,4 +143,4 @@ def parse_cmdline_data(ds_id,fill,cmdline=None):
 class DataSourceNoCloudNet(DataSourceNoCloud):
     cmdline_id = "ds=nocloud-net"
     supported_seed_starts = ( "http://", "https://", "ftp://" )
-    seeddir = cloudinit.cachedir + '/nocloud-net'
+    seeddir = cloudinit.seeddir + '/nocloud-net'
diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py
index 32b91979..3bee2c0d 100644
--- a/cloudinit/__init__.py
+++ b/cloudinit/__init__.py
@@ -21,15 +21,8 @@
 varlibdir = '/var/lib/cloud'
 cur_instance_link = varlibdir + "/instance"
 datadir = '/var/lib/cloud/data'
-semdir = '/var/lib/cloud/sem'
-cachedir = datadir + '/cache'
-userdata_raw = datadir + '/user-data.txt'
-userdata = datadir + '/user-data.txt.i'
-user_scripts_dir = datadir + "/scripts"
-boothooks_dir = datadir + "/boothooks"
-cloud_config = datadir + '/cloud-config.txt'
-data_source_cache = cachedir + '/obj.pkl'
 system_config = '/etc/cloud/cloud.cfg'
+seeddir = varlibdir + "/seed"
 cfg_env_name = "CLOUD_CFG"
 
 def_log_file = '/var/log/cloud-init.log'
@@ -42,6 +35,18 @@ cloud_type: auto
 """
 logger_name = "cloudinit"
 
+pathmap = {
+   "handlers" : "/handlers",
+   "scripts" : "/scripts",
+   "sem" : "/sem",
+   "boothooks" : "/boothooks",
+   "userdata_raw" : "/user-data.txt",
+   "userdata" : "/user-data-raw.txt.i",
+   "obj_pkl" : "/obj.pkl",
+   "cloud_config" : "/cloud-config.txt",
+   None : "",
+}
+
 import os
 from   configobj import ConfigObj
 
@@ -109,17 +114,6 @@ class CloudInit:
         "all": ( "nocloud-net", "ec2" ),
         "local" : ( "nocloud", ),
     }
-    pathmap = {
-       "handlers" : "/handlers",
-       "scripts" : "/scripts",
-       "sem" : "/sem",
-       "boothooks" : "/boothooks",
-       "userdata_raw" : "/user-data.txt",
-       "userdata" : "/user-data-raw.txt.i",
-       "obj_pkl" : "/obj.pkl",
-       None : "",
-    }
-
     cfg = None
     part_handlers = { }
     old_conffile = '/etc/ec2-init/ec2-config.cfg'
@@ -160,7 +154,7 @@ class CloudInit:
             # we try to restore from a current link and static path
             # by using the instance link, if purge_cache was called
             # the file wont exist
-            cache = "%s/%s" % (cur_instance_link, self.pathmap['obj_pkl'])
+            cache = get_ipath_cur('obj_pkl')
             f=open(cache, "rb")
             data = cPickle.load(f)
             self.datasource = data
@@ -255,8 +249,9 @@ class CloudInit:
             self.datasource.get_userdata(), 0600)
 
     def initctl_emit(self):
+        cc_path = get_ipath_cur('cloud_config')
         subprocess.Popen(['initctl', 'emit', 'cloud-config',
-            '%s=%s' % (cfg_env_name,cloud_config)]).communicate()
+            '%s=%s' % (cfg_env_name,cc_path)]).communicate()
 
     def sem_getpath(self,name,freq):
         if freq == 'once-per-instance':
@@ -324,12 +319,12 @@ class CloudInit:
     # (/var/lib/cloud/instances/<instance>/name)<name>)
     def get_ipath(self, name=None):
         return("%s/instances/%s%s" 
-               % (varlibdir,self.get_instance_id(), self.pathmap[name]))
+               % (varlibdir,self.get_instance_id(), pathmap[name]))
 
     # get_cpath : get the "clouddir" (/var/lib/cloud/<name>)
     # for a name in dirmap
     def get_cpath(self, name=None):
-        return("%s%s" % (varlibdir, self.pathmap[name]))
+        return("%s%s" % (varlibdir, pathmap[name]))
 
     def consume_userdata(self):
         self.get_userdata()
@@ -406,8 +401,9 @@ class CloudInit:
             return
 
         filename=filename.replace(os.sep,'_')
+        scriptsdir = get_ipath_cur('scripts')
         util.write_file("%s/%s/%s" % 
-            (user_scripts_dir,self.get_instance_id(),filename), payload, 0700)
+            (scriptsdir,self.get_instance_id(),filename), payload, 0700)
 
     def handle_upstart_job(self,data,ctype,filename,payload):
         if ctype == "__end__" or ctype == "__begin__": return
@@ -421,6 +417,7 @@ class CloudInit:
             self.cloud_config_str=""
             return
         if ctype == "__end__":
+            cloud_config = self.get_ipath("cloud_config")
             util.write_file(cloud_config, self.cloud_config_str, 0600)
 
             ## this could merge the cloud config with the system config
@@ -509,5 +506,9 @@ def purge_cache():
         return(False)
     return(True)
 
+def get_ipath_cur(name=None):
+    return("%s/instance/%s" % (varlibdir, pathmap[name]))
+
+
 class DataSourceNotFoundException(Exception):
     pass
-- 
cgit v1.2.3


From aa100d108395640a1769158062f4123f135759f8 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 19 Jan 2011 19:50:26 +0000
Subject: make scripts sub-dirs for per-<item>

---
 cloudinit/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py
index 3bee2c0d..55350442 100644
--- a/cloudinit/__init__.py
+++ b/cloudinit/__init__.py
@@ -485,7 +485,8 @@ class CloudInit:
 
 
 def initfs():
-    subds = [ 'scripts', 'seed', 'instances', 'handlers', 'sem' ]
+    subds = [ 'scripts/per-instance', 'scripts/per-once', 'scripts/per-boot',
+              'seed', 'instances', 'handlers', 'sem' ]
     dlist = [ ]
     for subd in subds:
         dlist.append("%s/%s" % (varlibdir, subd))
-- 
cgit v1.2.3


From 27032a10e8c9234ee4444be206da7b568bfa872a Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 19 Jan 2011 20:27:37 +0000
Subject: write the previous-hostname file to persistent cloud/data

This change just uses a different facility for coming up with the path.
But, by design I'm chosing to put 'previous-hostname' in
 /var/lib/cloud/data/
rather than in
 /var/lib/cloud/instance/data/

The idea is that if the user:
 - started an instance
 - modified /etc/hostname
 - bundled instance (or create-image from it)
 - started new instance

They would expect their modified /etc/hostname to persist.  As such, the
previous-hostname file should be cross-instance data.

Bugs in this area include LP: #596993 and LP: #514492
---
 cloud-init.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloud-init.py b/cloud-init.py
index 32add699..bdfa9f01 100755
--- a/cloud-init.py
+++ b/cloud-init.py
@@ -146,7 +146,7 @@ def set_hostname(hostname, log):
         log.error("failed to set_hostname")
 
 def update_hostname(hostname, log):
-    prev_file="%s/%s" % (cloudinit.datadir,"previous-hostname")
+    prev_file="%s/%s" % (cloudinit.get_cpath('datadir'),"previous-hostname")
     etc_file = "/etc/hostname"
 
     hostname_prev = None
-- 
cgit v1.2.3


From 14aa0cac0d5e2b57dc94f2145fdbd3d494898019 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 19 Jan 2011 20:30:29 +0000
Subject: add 'data' entry in pathmap and move get_cpath to a static function

---
 cloudinit/__init__.py | 19 ++++++++++---------
 doc/var-lib-cloud.txt |  5 +++++
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py
index 55350442..8358085d 100644
--- a/cloudinit/__init__.py
+++ b/cloudinit/__init__.py
@@ -20,7 +20,6 @@
 
 varlibdir = '/var/lib/cloud'
 cur_instance_link = varlibdir + "/instance"
-datadir = '/var/lib/cloud/data'
 system_config = '/etc/cloud/cloud.cfg'
 seeddir = varlibdir + "/seed"
 cfg_env_name = "CLOUD_CFG"
@@ -44,6 +43,7 @@ pathmap = {
    "userdata" : "/user-data-raw.txt.i",
    "obj_pkl" : "/obj.pkl",
    "cloud_config" : "/cloud-config.txt",
+   "datadir" : "/data",
    None : "",
 }
 
@@ -256,7 +256,7 @@ class CloudInit:
     def sem_getpath(self,name,freq):
         if freq == 'once-per-instance':
             return("%s/%s" % (self.get_ipath("sem"),name))
-        return("%s/%s.%s" % (self.get_cpath("sem"), name, freq))
+        return("%s/%s.%s" % (get_cpath("sem"), name, freq))
     
     def sem_has_run(self,name,freq):
         if freq == "always": return False
@@ -321,16 +321,11 @@ class CloudInit:
         return("%s/instances/%s%s" 
                % (varlibdir,self.get_instance_id(), pathmap[name]))
 
-    # get_cpath : get the "clouddir" (/var/lib/cloud/<name>)
-    # for a name in dirmap
-    def get_cpath(self, name=None):
-        return("%s%s" % (varlibdir, pathmap[name]))
-
     def consume_userdata(self):
         self.get_userdata()
         data = self
 
-        cdir = self.get_cpath("handlers")
+        cdir = get_cpath("handlers")
         idir = self.get_ipath("handlers")
 
         # add the path to the plugins dir to the top of our list for import
@@ -486,7 +481,7 @@ class CloudInit:
 
 def initfs():
     subds = [ 'scripts/per-instance', 'scripts/per-once', 'scripts/per-boot',
-              'seed', 'instances', 'handlers', 'sem' ]
+              'seed', 'instances', 'handlers', 'sem', 'data' ]
     dlist = [ ]
     for subd in subds:
         dlist.append("%s/%s" % (varlibdir, subd))
@@ -510,6 +505,12 @@ def purge_cache():
 def get_ipath_cur(name=None):
     return("%s/instance/%s" % (varlibdir, pathmap[name]))
 
+# get_cpath : get the "clouddir" (/var/lib/cloud/<name>)
+# for a name in dirmap
+def get_cpath(self, name=None):
+    return("%s%s" % (varlibdir, pathmap[name]))
+
+
 
 class DataSourceNotFoundException(Exception):
     pass
diff --git a/doc/var-lib-cloud.txt b/doc/var-lib-cloud.txt
index 5cdcddbb..2a1acd2b 100644
--- a/doc/var-lib-cloud.txt
+++ b/doc/var-lib-cloud.txt
@@ -34,6 +34,8 @@
          user-data.txt.i
          obj.pkl
          handlers/
+         data/  # just a per-instance data location to be used
+          
   - sem/
       scripts.once
     These are the cloud-specific semaphores.  The only thing that
@@ -44,6 +46,9 @@
       "persistent" handlers (not per-instance).  Same as handlers
       from user-data, just will be cross-instance id
 
+  - data/
+      this is a persistent data location.  cloud-init won't really
+      use it, but something else (a handler or script could)
 
 to clear out the current instance's data as if to force a "new run" on reboot
 do:
-- 
cgit v1.2.3


From f2cb6fa249c0f9a17aa8efd27749860622fd8ae1 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 19 Jan 2011 20:41:59 +0000
Subject: fix bug in get_cpath

---
 cloudinit/__init__.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py
index 8358085d..296af051 100644
--- a/cloudinit/__init__.py
+++ b/cloudinit/__init__.py
@@ -507,10 +507,8 @@ def get_ipath_cur(name=None):
 
 # get_cpath : get the "clouddir" (/var/lib/cloud/<name>)
 # for a name in dirmap
-def get_cpath(self, name=None):
+def get_cpath(name=None):
     return("%s%s" % (varlibdir, pathmap[name]))
 
-
-
 class DataSourceNotFoundException(Exception):
     pass
-- 
cgit v1.2.3