From 32eab86a04f4f6056b200e1cfac99ec0495cd5c0 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 19 Jan 2011 15:34:10 +0000 Subject: add doc about redesigned /var/lib/cloud --- doc/var-lib-cloud.txt | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 doc/var-lib-cloud.txt (limited to 'doc') diff --git a/doc/var-lib-cloud.txt b/doc/var-lib-cloud.txt new file mode 100644 index 00000000..328edd01 --- /dev/null +++ b/doc/var-lib-cloud.txt @@ -0,0 +1,46 @@ +/var/lib/cloud has the following structure: + - scripts/ + per-instance/ + per-boot/ + per-once/ + + files in these directories will be run by 'run-parts' once per + instance, once per boot, and once per *ever*. + + - seed/ + / + sys-user-data + user-data + meta-data + + The 'seed/' directory allows you to seed a specific datasource + For example, to seed the 'nocloud' datasource you would need to + populate + seed/nocloud/user-data + seed/nocloud/meta-data + + - instance -> instances/i-abcde + This is a symlink to the current instance/ directory + created/updated on boot + - instances/ + i-abcdefgh/ + scripts/ # all scripts in scripts are per-instance + sem/ + config-puppet + config-ssh + set-hostname + cloud-config.txt + user-data.txt + user-data.txt.i + obj.pkl + - sem/ + scripts.once + These are the cloud-specific semaphores. The only thing that + would go here are files to mark that a "per-once" script + has run. + + +to clear out the current instance's data as if to force a "new run" on reboot +do: + ( cd /var/lib/cloud/instance && sudo rm -Rf * ) + -- cgit v1.2.3 From ee03480e670c10d48a4825a458fac18aaf4fbec0 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 19 Jan 2011 18:35:07 +0000 Subject: initial /var/lib rework still lots to do. includes a fix for LP: #704509 LP: #704509 --- cloud-init.py | 17 ++++++---- cloudinit/__init__.py | 93 ++++++++++++++++++++++++++++++++++++++++++++------- cloudinit/util.py | 28 ++++++++++++++++ doc/var-lib-cloud.txt | 7 +++- 4 files changed, 125 insertions(+), 20 deletions(-) (limited to 'doc') diff --git a/cloud-init.py b/cloud-init.py index 28828648..1278d2eb 100755 --- a/cloud-init.py +++ b/cloud-init.py @@ -56,6 +56,13 @@ def main(): if cmd == "start-local": source_type = "local" + if cmd == "start-local": + try: + cloudinit.initfs() + except: + warn("failed to initfs, likely bad things to come") + + cloudinit.logging_set_from_cfg_file() log = logging.getLogger() log.info(msg) @@ -74,6 +81,9 @@ def main(): sys.stderr.write("no instance data found in %s\n" % cmd) sys.exit(1) + # set this as the current instance + cloud.set_cur_instance() + # store the metadata cloud.update_cache() @@ -103,13 +113,6 @@ def main(): #print "user data is:" + cloud.get_user_data() - # set the defaults (like what ec2-set-defaults.py did) - try: - cloud.sem_and_run("set_defaults", "once-per-instance", - set_defaults,[ cloud ],False) - except: - warn("failed to set defaults\n") - # finish, send the cloud-config event cloud.initctl_emit() diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py index be366d4c..1de32f2e 100644 --- a/cloudinit/__init__.py +++ b/cloudinit/__init__.py @@ -18,9 +18,9 @@ # along with this program. If not, see . # +varlibdir = '/var/lib/cloud' datadir = '/var/lib/cloud/data' semdir = '/var/lib/cloud/sem' -pluginsdir = datadir + '/plugins' cachedir = datadir + '/cache' userdata_raw = datadir + '/user-data.txt' userdata = datadir + '/user-data.txt.i' @@ -32,6 +32,9 @@ system_config = '/etc/cloud/cloud.cfg' cfg_env_name = "CLOUD_CFG" def_log_file = '/var/log/cloud-init.log' +def_log_user = "syslog" +def_log_group = "adm" + cfg_builtin = """ log_cfgs: [ ] cloud_type: auto @@ -52,9 +55,10 @@ import util import logging import logging.config import StringIO +import glob class NullHandler(logging.Handler): - def emit(self,record): pass + def emit(self,record): pass log = logging.getLogger(logger_name) log.addHandler(NullHandler()) @@ -104,6 +108,12 @@ class CloudInit: "all": ( "nocloud-net", "ec2" ), "local" : ( "nocloud", ), } + dirmap = { + "handlers" : "/handlers", + "scripts" : "/scripts", + "sem" : "/sem", + None : "", + } cfg = None part_handlers = { } @@ -200,6 +210,21 @@ class CloudInit: log.debug("did not find data source from %s" % dslist) raise DataSourceNotFoundException("Could not find data source") + def set_cur_instance(self): + lname = "%s/instance" % varlibdir + try: + os.unlink(lname) + except OSError, e: + if e.errno != errno.ENOENT: raise + + os.symlink("./instances/%s" % self.get_instance_id(), lname) + idir = self.get_idir() + dlist = [] + for d in [ "handlers", "scripts", "sem" ]: + dlist.append("%s/%s" % (idir, d)) + + util.ensure_dirs(dlist) + def get_userdata(self): return(self.datasource.get_userdata()) @@ -222,11 +247,9 @@ class CloudInit: '%s=%s' % (cfg_env_name,cloud_config)]).communicate() def sem_getpath(self,name,freq): - freqtok = freq if freq == 'once-per-instance': - freqtok = self.datasource.get_instance_id() - - return("%s/%s.%s" % (semdir,name,freqtok)) + return("%s/%s" % (self.get_idir("sem"),name)) + return("%s/%s.%s" % (self.get_cdir("sem"), name, freq)) def sem_has_run(self,name,freq): if freq == "always": return False @@ -285,9 +308,45 @@ class CloudInit: self.sem_clear(semname,freq) raise + # get_cdir : get the "clouddir" (/var/lib/cloud/) + # for a name in dirmap + def get_idir(self, name=None): + return("%s/instances/%s%s" + % (varlibdir,self.get_instance_id(), self.dirmap[name])) + + # get_cdir : get the "clouddir" (/var/lib/cloud/) + # for a name in dirmap + def get_cdir(self, name=None): + return("%s%s" % (varlibdir, self.dirmap[name])) + def consume_userdata(self): self.get_userdata() data = self + + cdir = self.get_cdir("handlers") + idir = self.get_idir("handlers") + + # add the path to the plugins dir to the top of our list for import + # instance dir should be read before cloud-dir + sys.path.insert(0,cdir) + sys.path.insert(0,idir) + + # add handlers in cdir + for fname in glob.glob("%s/*.py" % cdir): + if not os.path.isfile(fname): continue + modname = os.path.basename(fname)[0:-3] + try: + mod = __import__(modname) + lister = getattr(mod, "list_types") + handler = getattr(mod, "handle_part") + mtypes = lister() + for mtype in mtypes: + self.part_handlers[mtype]=handler + log.debug("added handler for [%s] from %s" % (mtypes,fname)) + except: + log.warn("failed to initialize handler in %s" % fname) + util.logexc(log) + # give callbacks opportunity to initialize for ctype, func in self.part_handlers.items(): func(data, "__begin__",None,None) @@ -304,16 +363,13 @@ class CloudInit: self.handlercount = 0 return - # add the path to the plugins dir to the top of our list for import - if self.handlercount == 0: - sys.path.insert(0,pluginsdir) - self.handlercount=self.handlercount+1 - # write content to pluginsdir + # write content to instance's handlerdir + handlerdir = self.get_idir("handler") modname = 'part-handler-%03d' % self.handlercount modfname = modname + ".py" - util.write_file("%s/%s" % (pluginsdir,modfname), payload, 0600) + util.write_file("%s/%s" % (handlerdir,modfname), payload, 0600) try: mod = __import__(modname) @@ -418,6 +474,19 @@ class CloudInit: return(self.datasource.device_name_to_device(name)) +def initfs(): + subds = [ 'scripts', 'seed', 'instances', 'handlers', 'sem' ] + dlist = [ ] + for subd in subds: + dlist.append("%s/%s" % (varlibdir, subd)) + util.ensure_dirs(dlist) + + fp = open(def_log_file,"ab") + fp.close() + util.chownbyname(def_log_file,def_log_user, def_log_group) + + + def purge_cache(): try: os.unlink(data_source_cache) diff --git a/cloudinit/util.py b/cloudinit/util.py index d5ae2bec..a2291164 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -278,3 +278,31 @@ def read_cc_from_cmdline(cmdline=None): begin = cmdline.find(tag_begin, end + end_l) return('\n'.join(tokens)) + +def ensure_dirs(dirlist, mode=0755): + fixmodes = [] + for d in dirlist: + try: + if mode != None: + os.makedirs(d) + else: + os.makedirs(d, mode) + except OSError as e: + if e.errno != errno.EEXIST: raise + if mode != None: fixmodes.append(d) + + for d in fixmodes: + os.chmod(d, mode) + +def chownbyname(fname,user=None,group=None): + uid = -1 + gid = -1 + if user == None and group == None: return + if user: + import pwd + uid = pwd.getpwnam(user).pw_uid + if group: + import grp + gid = grp.getgrnam(group).gr_gid + + os.chown(fname,uid,gid) diff --git a/doc/var-lib-cloud.txt b/doc/var-lib-cloud.txt index 328edd01..5cdcddbb 100644 --- a/doc/var-lib-cloud.txt +++ b/doc/var-lib-cloud.txt @@ -33,12 +33,17 @@ user-data.txt user-data.txt.i obj.pkl + handlers/ - sem/ scripts.once These are the cloud-specific semaphores. The only thing that would go here are files to mark that a "per-once" script has run. - + + - handlers/ + "persistent" handlers (not per-instance). Same as handlers + from user-data, just will be cross-instance id + to clear out the current instance's data as if to force a "new run" on reboot do: -- cgit v1.2.3 From 14aa0cac0d5e2b57dc94f2145fdbd3d494898019 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 19 Jan 2011 20:30:29 +0000 Subject: add 'data' entry in pathmap and move get_cpath to a static function --- cloudinit/__init__.py | 19 ++++++++++--------- doc/var-lib-cloud.txt | 5 +++++ 2 files changed, 15 insertions(+), 9 deletions(-) (limited to 'doc') diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py index 55350442..8358085d 100644 --- a/cloudinit/__init__.py +++ b/cloudinit/__init__.py @@ -20,7 +20,6 @@ varlibdir = '/var/lib/cloud' cur_instance_link = varlibdir + "/instance" -datadir = '/var/lib/cloud/data' system_config = '/etc/cloud/cloud.cfg' seeddir = varlibdir + "/seed" cfg_env_name = "CLOUD_CFG" @@ -44,6 +43,7 @@ pathmap = { "userdata" : "/user-data-raw.txt.i", "obj_pkl" : "/obj.pkl", "cloud_config" : "/cloud-config.txt", + "datadir" : "/data", None : "", } @@ -256,7 +256,7 @@ class CloudInit: def sem_getpath(self,name,freq): if freq == 'once-per-instance': return("%s/%s" % (self.get_ipath("sem"),name)) - return("%s/%s.%s" % (self.get_cpath("sem"), name, freq)) + return("%s/%s.%s" % (get_cpath("sem"), name, freq)) def sem_has_run(self,name,freq): if freq == "always": return False @@ -321,16 +321,11 @@ class CloudInit: return("%s/instances/%s%s" % (varlibdir,self.get_instance_id(), pathmap[name])) - # get_cpath : get the "clouddir" (/var/lib/cloud/) - # for a name in dirmap - def get_cpath(self, name=None): - return("%s%s" % (varlibdir, pathmap[name])) - def consume_userdata(self): self.get_userdata() data = self - cdir = self.get_cpath("handlers") + cdir = get_cpath("handlers") idir = self.get_ipath("handlers") # add the path to the plugins dir to the top of our list for import @@ -486,7 +481,7 @@ class CloudInit: def initfs(): subds = [ 'scripts/per-instance', 'scripts/per-once', 'scripts/per-boot', - 'seed', 'instances', 'handlers', 'sem' ] + 'seed', 'instances', 'handlers', 'sem', 'data' ] dlist = [ ] for subd in subds: dlist.append("%s/%s" % (varlibdir, subd)) @@ -510,6 +505,12 @@ def purge_cache(): def get_ipath_cur(name=None): return("%s/instance/%s" % (varlibdir, pathmap[name])) +# get_cpath : get the "clouddir" (/var/lib/cloud/) +# for a name in dirmap +def get_cpath(self, name=None): + return("%s%s" % (varlibdir, pathmap[name])) + + class DataSourceNotFoundException(Exception): pass diff --git a/doc/var-lib-cloud.txt b/doc/var-lib-cloud.txt index 5cdcddbb..2a1acd2b 100644 --- a/doc/var-lib-cloud.txt +++ b/doc/var-lib-cloud.txt @@ -34,6 +34,8 @@ user-data.txt.i obj.pkl handlers/ + data/ # just a per-instance data location to be used + - sem/ scripts.once These are the cloud-specific semaphores. The only thing that @@ -44,6 +46,9 @@ "persistent" handlers (not per-instance). Same as handlers from user-data, just will be cross-instance id + - data/ + this is a persistent data location. cloud-init won't really + use it, but something else (a handler or script could) to clear out the current instance's data as if to force a "new run" on reboot do: -- cgit v1.2.3