diff options
41 files changed, 1747 insertions, 525 deletions
diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 00000000..8ddb3a07 --- /dev/null +++ b/ChangeLog @@ -0,0 +1,32 @@ +0.5.16: + - change permissions of /var/log/cloud-init.log to accomodate + syslog writing to it (LP: #704509) + - rework of /var/lib/cloud layout + - remove updates-check (LP: #653220) + - support resizing / on first boot (enabled by default) + - added support for running CloudConfig modules at cloud-init time + rather than cloud-config time, and the new 'cloud_init_modules' + entry in cloud.cfg to indicate which should run then. + The driving force behind this was to have the rsyslog module + able to run before rsyslog even runs so that a restart would + not be needed (rsyslog on ubuntu runs on 'filesystem') + - moved setting and updating of hostname to cloud_init_modules + this allows the user to easily disable these from running. + This also means: + - the semaphore name for 'set_hostname' and 'update_hostname' + changes to 'config_set_hostname' and 'config_update_hostname' + - added cloud-config option 'hostname' for setting hostname + - moved upstart/cloud-run-user-script.conf to upstart/cloud-final.conf + - cloud-final.conf now runs runs cloud-config modules similar + to cloud-config and cloud-init. + - LP: #653271 + - added writing of "boot-finished" to /var/lib/cloud/instance/boot-finished + this is the last thing done, indicating cloud-init is finished booting + - writes message to console with timestamp and uptime + - write ssh keys to console as one of the last things done + this is to ensure they don't get run off the 'get-console-ouptut' buffer + - user_scripts run via cloud-final and thus semaphore renamed from + user_scripts to config_user_scripts + - add support for redirecting output of cloud-init, cloud-config, cloud-final + via the config file, or user data config file + - add support for posting data about the instance to a url (phone_home) @@ -0,0 +1,23 @@ +- consider 'failsafe' DataSource + If all others fail, setting a default that + - sets the user password, writing it to console + - logs to console that this happened +- consider 'previous' DataSource + If no other data source is found, fall back to the 'previous' one + keep a indication of what instance id that is in /var/lib/cloud +- allow setting of user password and lock account +- move "user-scripts" upstart job to "final", possibly move its + contents to a managed script in /usr/lib/cloud +- "final" things + - messags to console + - failsafe warning + - ssh keys + - password (if set above) + - post system info home +- rewrite "cloud-init-query" + have DataSource and cloudinit expose explicit fields + - instance-id + - hostname + - mirror + - release + - ssh public keys diff --git a/cloud-init-cfg.py b/cloud-init-cfg.py index eb875182..442fc4d8 100755 --- a/cloud-init-cfg.py +++ b/cloud-init-cfg.py @@ -19,7 +19,7 @@ import sys import cloudinit -import cloudinit.CloudConfig +import cloudinit.CloudConfig as CC import logging import os import traceback @@ -35,11 +35,15 @@ def main(): # read cloud config jobs from config (builtin -> system) # and run all in order + modename = "config" + if len(sys.argv) < 2: Usage(sys.stderr) sys.exit(1) if sys.argv[1] == "all": name = "all" + if len(sys.argv) > 2: + modename = sys.argv[2] else: freq = None run_args = [] @@ -51,56 +55,37 @@ def main(): if len(sys.argv) > 3: run_args=sys.argv[3:] - cloudinit.logging_set_from_cfg_file() - log = logging.getLogger() - log.info("cloud-init-cfg %s" % sys.argv[1:]) - - cfg_path = cloudinit.cloud_config + cfg_path = cloudinit.get_ipath_cur("cloud_config") cfg_env_name = cloudinit.cfg_env_name if os.environ.has_key(cfg_env_name): cfg_path = os.environ[cfg_env_name] - cc = cloudinit.CloudConfig.CloudConfig(cfg_path) + cc = CC.CloudConfig(cfg_path) + + try: + (outfmt, errfmt) = CC.get_output_cfg(cc.cfg,modename) + CC.redirect_output(outfmt, errfmt) + except Exception, e: + err("Failed to get and set output config: %s\n" % e) + + cloudinit.logging_set_from_cfg(cc.cfg) + log = logging.getLogger() + log.info("cloud-init-cfg %s" % sys.argv[1:]) module_list = [ ] if name == "all": - # create 'module_list', an array of arrays - # where array[0] = config - # array[1] = freq - # array[2:] = arguemnts - if "cloud_config_modules" in cc.cfg: - for item in cc.cfg["cloud_config_modules"]: - if isinstance(item,str): - module_list.append((item,)) - elif isinstance(item,list): - module_list.append(item) - else: - fail("Failed to parse cloud_config_modules",log) - else: - fail("No cloud_config_modules found in config",log) + modlist_cfg_name = "cloud_%s_modules" % modename + print modlist_cfg_name + module_list = CC.read_cc_modules(cc.cfg,modlist_cfg_name) + if not len(module_list): + err("no modules to run in cloud_config [%s]" % modename,log) + sys.exit(0) else: module_list.append( [ name, freq ] + run_args ) - failures = [] - for cfg_mod in module_list: - name = cfg_mod[0] - freq = None - run_args = [ ] - if len(cfg_mod) > 1: - freq = cfg_mod[1] - if len(cfg_mod) > 2: - run_args = cfg_mod[2:] - - try: - log.debug("handling %s with freq=%s and args=%s" % - (name, freq, run_args )) - cc.handle(name, run_args, freq=freq) - except: - log.warn(traceback.format_exc()) - err("config handling of %s, %s, %s failed\n" % - (name,freq,run_args), log) - failures.append(name) - + failures = CC.run_cc_modules(cc,module_list,log) + if len(failures): + err("errors running cloud_config [%s]: %s" % (modename,failures), log) sys.exit(len(failures)) def err(msg,log=None): diff --git a/cloud-init-query.py b/cloud-init-query.py new file mode 100755 index 00000000..3e8c24ab --- /dev/null +++ b/cloud-init-query.py @@ -0,0 +1,55 @@ +#!/usr/bin/python +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import sys +import cloudinit +import cloudinit.CloudConfig +import logging +import os +import traceback + +def Usage(out = sys.stdout): + out.write("Usage: %s name\n" % sys.argv[0]) + +def main(): + # expect to be called with name of item to fetch + if len(sys.argv) != 2: + Usage(sys.stderr) + sys.exit(1) + + cc = cloudinit.CloudConfig.CloudConfig(cloudinit.cloud_config) + cloud_config = cc.cfg + data = { + 'user_data' : cc.cloud.get_userdata(), + 'user_data_raw' : cc.cloud.get_userdata_raw(), + 'instance_id' : cc.cloud.get_instance_id(), + } + + name = sys.argv[1].replace('-','_') + + if name not in data: + sys.stderr.write("unknown name '%s'. Known values are:\n %s\n" % + (sys.argv[1], ' '.join(data.keys()))) + sys.exit(1) + + print data[name] + sys.exit(0) + +if __name__ == '__main__': + main() diff --git a/cloud-init.py b/cloud-init.py index 11bf89af..adac1874 100755 --- a/cloud-init.py +++ b/cloud-init.py @@ -22,12 +22,13 @@ import sys import cloudinit import cloudinit.util as util +import cloudinit.CloudConfig as CC import time import logging import errno -def warn(str): - sys.stderr.write(str) +def warn(wstr): + sys.stderr.write(wstr) def main(): cmds = ( "start", "start-local" ) @@ -39,7 +40,7 @@ def main(): sys.stderr.write("bad command %s. use one of %s\n" % (cmd, cmds)) sys.exit(1) - now = time.strftime("%a, %d %b %Y %H:%M:%S %z") + now = time.strftime("%a, %d %b %Y %H:%M:%S %z",time.gmtime()) try: uptimef=open("/proc/uptime") uptime=uptimef.read().split(" ")[0] @@ -48,18 +49,31 @@ def main(): warn("unable to open /proc/uptime\n") uptime = "na" - msg = "cloud-init %s running: %s. up %s seconds" % (cmd, now, uptime) - sys.stderr.write(msg + "\n") - sys.stderr.flush() - source_type = "all" if cmd == "start-local": source_type = "local" - cloudinit.logging_set_from_cfg_file() + try: + cfg = cloudinit.get_base_cfg() + (outfmt, errfmt) = CC.get_output_cfg(cfg,"init") + CC.redirect_output(outfmt, errfmt) + except Exception, e: + warn("Failed to get and set output config: %s\n" % e) + + msg = "cloud-init %s running: %s. up %s seconds" % (cmd, now, uptime) + sys.stderr.write(msg + "\n") + sys.stderr.flush() + + cloudinit.logging_set_from_cfg(cfg) log = logging.getLogger() log.info(msg) + try: + cloudinit.initfs() + except Exception, e: + warn("failed to initfs, likely bad things to come: %s\n" % str(e)) + + # cache is not instance specific, so it has to be purged # but we want 'start' to benefit from a cache if # a previous start-local populated one @@ -74,6 +88,9 @@ def main(): sys.stderr.write("no instance data found in %s\n" % cmd) sys.exit(1) + # set this as the current instance + cloud.set_cur_instance() + # store the metadata cloud.update_cache() @@ -89,108 +106,35 @@ def main(): warn("consuming user data failed!\n") raise - try: - if util.get_cfg_option_bool(cloud.cfg,"preserve_hostname",False): - log.debug("preserve_hostname is set. not managing hostname") - else: - hostname = cloud.get_hostname() - cloud.sem_and_run("set_hostname", "once-per-instance", - set_hostname, [ hostname, log ], False) - cloud.sem_and_run("update_hostname", "always", - update_hostname, [ hostname, log ], False) - except: - warn("failed to set hostname\n") - - #print "user data is:" + cloud.get_user_data() - - # set the defaults (like what ec2-set-defaults.py did) - try: - cloud.sem_and_run("set_defaults", "once-per-instance", - set_defaults,[ cloud ],False) - except: - warn("failed to set defaults\n") - # finish, send the cloud-config event cloud.initctl_emit() - sys.exit(0) - -def set_defaults(cloud): - apply_locale(cloud.get_locale()) - -def apply_locale(locale): - subprocess.Popen(['locale-gen', locale]).communicate() - subprocess.Popen(['update-locale', locale]).communicate() - - util.render_to_file('default-locale', '/etc/default/locale', \ - { 'locale' : locale }) - -# read hostname from a 'hostname' file -# allow for comments and stripping line endings. -# if file doesn't exist, or no contents, return default -def read_hostname(filename, default=None): - try: - fp = open(filename,"r") - lines = fp.readlines() - fp.close() - for line in lines: - hpos = line.find("#") - if hpos != -1: - line = line[0:hpos] - line = line.rstrip() - if line: - return line - except IOError, e: - if e.errno == errno.ENOENT: pass - return default - -def set_hostname(hostname, log): - try: - subprocess.Popen(['hostname', hostname]).communicate() - util.write_file("/etc/hostname","%s\n" % hostname, 0644) - log.debug("populated /etc/hostname with %s on first boot", hostname) - except: - log.error("failed to set_hostname") - -def update_hostname(hostname, log): - prev_file="%s/%s" % (cloudinit.datadir,"previous-hostname") - etc_file = "/etc/hostname" - - hostname_prev = None - hostname_in_etc = None + cfg_path = cloudinit.get_ipath_cur("cloud_config") + cc = CC.CloudConfig(cfg_path, cloud) + # if the output config changed, update output and err try: - hostname_prev = read_hostname(prev_file) - except: - log.warn("Failed to open %s" % prev_file) - - try: - hostname_in_etc = read_hostname(etc_file) - except: - log.warn("Failed to open %s" % etc_file) - - update_files = [] - if not hostname_prev or hostname_prev != hostname: - update_files.append(prev_file) - - if (not hostname_in_etc or - (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): - update_files.append(etc_file) - - try: - for fname in update_files: - util.write_file(fname,"%s\n" % hostname, 0644) - log.debug("wrote %s to %s" % (hostname,fname)) - except: - log.warn("failed to write hostname to %s" % fname) - - if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: - log.debug("%s differs from %s. assuming user maintained" % - (prev_file,etc_file)) - - if etc_file in update_files: - log.debug("setting hostname to %s" % hostname) - subprocess.Popen(['hostname', hostname]).communicate() + outfmt_orig = outfmt + errfmt_orig = errfmt + (outfmt, errfmt) = CC.get_output_cfg(cc.cfg,"init") + if outfmt_orig != outfmt or errfmt_orig != errfmt: + warn("stdout, stderr changing to (%s,%s)" % (outfmt,errfmt)) + CC.redirect_output(outfmt, errfmt) + except Exception, e: + warn("Failed to get and set output config: %s\n" % e) + + module_list = CC.read_cc_modules(cc.cfg,"cloud_init_modules") + + failures = [] + if len(module_list): + failures = CC.run_cc_modules(cc,module_list,log) + else: + msg = "no cloud_init_modules to run" + sys.stderr.write(msg + "\n") + log.debug(msg) + sys.exit(0) + + sys.exit(len(failures)) if __name__ == '__main__': main() diff --git a/cloud.cfg b/cloud.cfg deleted file mode 100644 index 1f479dcc..00000000 --- a/cloud.cfg +++ /dev/null @@ -1,4 +0,0 @@ -cloud_type: auto -user: ubuntu -disable_root: 1 -preserve_hostname: False diff --git a/cloudinit/CloudConfig/__init__.py b/cloudinit/CloudConfig/__init__.py index 0a91059a..22ad63a6 100644 --- a/cloudinit/CloudConfig/__init__.py +++ b/cloudinit/CloudConfig/__init__.py @@ -21,16 +21,22 @@ import cloudinit import cloudinit.util as util import sys import traceback +import os +import subprocess per_instance="once-per-instance" per_always="always" +per_once="once" class CloudConfig(): cfgfile = None cfg = None - def __init__(self,cfgfile): - self.cloud = cloudinit.CloudInit() + def __init__(self,cfgfile, cloud=None): + if cloud == None: + self.cloud = cloudinit.CloudInit() + else: + self.cloud = cloud self.cfg = self.get_config_obj(cfgfile) self.cloud.get_data_source() @@ -38,6 +44,7 @@ class CloudConfig(): try: cfg = util.read_conf(cfgfile) except: + # TODO: this 'log' could/should be passed in cloudinit.log.critical("Failed loading of cloud config '%s'. Continuing with empty config\n" % cfgfile) cloudinit.log.debug(traceback.format_exc() + "\n") cfg = None @@ -58,3 +65,150 @@ class CloudConfig(): except: raise +# reads a cloudconfig module list, returns +# a 2 dimensional array suitable to pass to run_cc_modules +def read_cc_modules(cfg,name): + if name not in cfg: return([]) + module_list = [] + # create 'module_list', an array of arrays + # where array[0] = config + # array[1] = freq + # array[2:] = arguemnts + for item in cfg[name]: + if isinstance(item,str): + module_list.append((item,)) + elif isinstance(item,list): + module_list.append(item) + else: + raise TypeError("failed to read '%s' item in config") + return(module_list) + +def run_cc_modules(cc,module_list,log): + failures = [] + for cfg_mod in module_list: + name = cfg_mod[0] + freq = None + run_args = [ ] + if len(cfg_mod) > 1: + freq = cfg_mod[1] + if len(cfg_mod) > 2: + run_args = cfg_mod[2:] + + try: + log.debug("handling %s with freq=%s and args=%s" % + (name, freq, run_args )) + cc.handle(name, run_args, freq=freq) + except: + log.warn(traceback.format_exc()) + log.error("config handling of %s, %s, %s failed\n" % + (name,freq,run_args)) + failures.append(name) + + return(failures) + +# always returns well formated values +# cfg is expected to have an entry 'output' in it, which is a dictionary +# that includes entries for 'init', 'config', 'final' or 'all' +# init: /var/log/cloud.out +# config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ] +# final: +# output: "| logger -p" +# error: "> /dev/null" +# this returns the specific 'mode' entry, cleanly formatted, with value +# None if if none is given +def get_output_cfg(cfg, mode="init"): + ret = [ None, None ] + if not 'output' in cfg: return ret + + outcfg = cfg['output'] + if mode in outcfg: + modecfg = outcfg[mode] + else: + if 'all' not in outcfg: return ret + # if there is a 'all' item in the output list + # then it applies to all users of this (init, config, final) + modecfg = outcfg['all'] + + # if value is a string, it specifies stdout + if isinstance(modecfg,str): + ret = [ modecfg, None ] + + # if its a list, then we expect (stdout, stderr) + if isinstance(modecfg,list): + if len(modecfg) > 0: ret[0] = modecfg[0] + if len(modecfg) > 1: + ret[1] = modecfg[1] + + # if it is a dictionary, expect 'out' and 'error' + # items, which indicate out and error + if isinstance(modecfg, dict): + if 'output' in modecfg: + ret[0] = modecfg['output'] + if 'error' in modecfg: + ret[1] = modecfg['error'] + + # if err's entry == "&1", then make it same as stdout + # as in shell syntax of "echo foo >/dev/null 2>&1" + if ret[1] == "&1": ret[1] = ret[0] + + swlist = [ ">>", ">", "|" ] + for i in range(len(ret)): + if not ret[i]: continue + val = ret[i].lstrip() + found = False + for s in swlist: + if val.startswith(s): + val = "%s %s" % (s,val[len(s):].strip()) + found = True + break + if not found: + # default behavior is append + val = "%s %s" % ( ">>", val.strip()) + ret[i] = val + + return(ret) + + +# redirect_output(outfmt, errfmt, orig_out, orig_err) +# replace orig_out and orig_err with filehandles specified in outfmt or errfmt +# fmt can be: +# > FILEPATH +# >> FILEPATH +# | program [ arg1 [ arg2 [ ... ] ] ] +# +# with a '|', arguments are passed to shell, so one level of +# shell escape is required. +def redirect_output(outfmt,errfmt, o_out=sys.stdout, o_err=sys.stderr): + if outfmt: + (mode, arg) = outfmt.split(" ",1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) + new_fp = proc.stdin + else: + raise TypeError("invalid type for outfmt: %s" % outfmt) + + if o_out: + os.dup2(new_fp.fileno(), o_out.fileno()) + if errfmt == outfmt: + os.dup2(new_fp.fileno(), o_err.fileno()) + return + + if errfmt: + (mode, arg) = errfmt.split(" ",1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) + new_fp = proc.stdin + else: + raise TypeError("invalid type for outfmt: %s" % outfmt) + + if o_err: + os.dup2(new_fp.fileno(), o_err.fileno()) + return diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index 396c5e09..e918e8c8 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -25,20 +25,23 @@ def handle(name,cfg,cloud,log,args): update = util.get_cfg_option_bool(cfg, 'apt_update', False) upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) + release = get_release() + if cfg.has_key("apt_mirror"): + mirror = cfg["apt_mirror"] + else: + mirror = cloud.get_mirror() + if not util.get_cfg_option_bool(cfg, \ 'apt_preserve_sources_list', False): - if cfg.has_key("apt_mirror"): - mirror = cfg["apt_mirror"] - else: - mirror = cloud.get_mirror() - generate_sources_list(mirror) + generate_sources_list(release, mirror) old_mir = util.get_cfg_option_str(cfg,'apt_old_mirror', \ "archive.ubuntu.com/ubuntu") rename_apt_lists(old_mir, mirror) # process 'apt_sources' if cfg.has_key('apt_sources'): - errors = add_sources(cfg['apt_sources']) + errors = add_sources(cfg['apt_sources'], + { 'MIRROR' : mirror, 'RELEASE' : release } ) for e in errors: log.warn("Source Error: %s\n" % ':'.join(e)) @@ -96,17 +99,18 @@ def rename_apt_lists(omirror,new_mirror,lists_d="/var/lib/apt/lists"): for file in glob.glob("%s_*" % oprefix): os.rename(file,"%s%s" % (nprefix, file[olen:])) -def generate_sources_list(mirror): +def get_release(): stdout, stderr = subprocess.Popen(['lsb_release', '-cs'], stdout=subprocess.PIPE).communicate() - codename = stdout.strip() + return(stdout.strip()) +def generate_sources_list(codename, mirror): util.render_to_file('sources.list', '/etc/apt/sources.list', \ { 'mirror' : mirror, 'codename' : codename }) # srclist is a list of dictionaries, # each entry must have: 'source' # may have: key, ( keyid and keyserver) -def add_sources(srclist): +def add_sources(srclist, searchList={ }): elst = [] for ent in srclist: @@ -121,6 +125,8 @@ def add_sources(srclist): elst.append([source, "add-apt-repository failed"]) continue + source = util.render_string(source, searchList) + if not ent.has_key('filename'): ent['filename']='cloud_config_sources.list' diff --git a/cloudinit/CloudConfig/cc_final_message.py b/cloudinit/CloudConfig/cc_final_message.py new file mode 100644 index 00000000..4d72e409 --- /dev/null +++ b/cloudinit/CloudConfig/cc_final_message.py @@ -0,0 +1,55 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +from cloudinit.CloudConfig import per_always +import sys +from cloudinit import util, boot_finished +import time + +frequency = per_always + +final_message = "cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds" + +def handle(name,cfg,cloud,log,args): + if len(args) != 0: + msg_in = args[0] + else: + msg_in = util.get_cfg_option_str(cfg,"final_message",final_message) + + try: + uptimef=open("/proc/uptime") + uptime=uptimef.read().split(" ")[0] + uptimef.close() + except IOError as e: + log.warn("unable to open /proc/uptime\n") + uptime = "na" + + + try: + ts = time.strftime("%a, %d %b %Y %H:%M:%S %z",time.gmtime()) + except: + ts = "na" + + try: + subs = { 'UPTIME' : uptime, 'TIMESTAMP' : ts } + sys.stdout.write(util.render_string(msg_in, subs)) + except Exception as e: + log.warn("failed to render string to stdout: %s" % e) + + fp = open(boot_finished, "wb") + fp.write(uptime + "\n") + fp.close() diff --git a/cloudinit/CloudConfig/cc_keys_to_console.py b/cloudinit/CloudConfig/cc_keys_to_console.py new file mode 100644 index 00000000..47227b76 --- /dev/null +++ b/cloudinit/CloudConfig/cc_keys_to_console.py @@ -0,0 +1,31 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +from cloudinit.CloudConfig import per_instance +import subprocess + +frequency = per_instance + +def handle(name,cfg,cloud,log,args): + write_ssh_prog='/usr/lib/cloud-init/write-ssh-key-fingerprints' + try: + confp = open('/dev/console',"wb") + subprocess.call(write_ssh_prog,stdout=confp) + confp.close() + except: + log.warn("writing keys to console value") + raise diff --git a/cloudinit/CloudConfig/cc_locale.py b/cloudinit/CloudConfig/cc_locale.py new file mode 100644 index 00000000..c164b5ba --- /dev/null +++ b/cloudinit/CloudConfig/cc_locale.py @@ -0,0 +1,43 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit.util as util +import subprocess +import traceback + +def apply_locale(locale): + subprocess.Popen(['locale-gen', locale]).communicate() + subprocess.Popen(['update-locale', locale]).communicate() + + util.render_to_file('default-locale', '/etc/default/locale', \ + { 'locale' : locale }) + +def handle(name,cfg,cloud,log,args): + if len(args) != 0: + locale = args[0] + else: + locale = util.get_cfg_option_str(cfg,"locale",cloud.get_locale()) + + if not locale: return + + log.debug("setting locale to %s" % locale) + + try: + apply_locale(locale) + except Exception, e: + log.debug(traceback.format_exc(e)) + raise Exception("failed to apply locale %s" % locale) diff --git a/cloudinit/CloudConfig/cc_phone_home.py b/cloudinit/CloudConfig/cc_phone_home.py new file mode 100644 index 00000000..ee463757 --- /dev/null +++ b/cloudinit/CloudConfig/cc_phone_home.py @@ -0,0 +1,99 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +from cloudinit.CloudConfig import per_instance +import cloudinit.util as util +from time import sleep + +frequency = per_instance +post_list_all = [ 'pub_key_dsa', 'pub_key_rsa', 'instance_id', 'hostname' ] + +# phone_home: +# url: http://my.foo.bar/$INSTANCE/ +# post: all +# tries: 10 +# +# phone_home: +# url: http://my.foo.bar/$INSTANCE_ID/ +# post: [ pub_key_dsa, pub_key_rsa, instance_id +# +def handle(name,cfg,cloud,log,args): + if len(args) != 0: + ph_cfg = util.readconf(args[0]) + else: + if not 'phone_home' in cfg: return + ph_cfg = cfg['phone_home'] + + if 'url' not in ph_cfg: + log.warn("no 'url' token in phone_home") + return + + url = ph_cfg['url'] + post_list = ph_cfg.get('post', 'all') + tries = ph_cfg.get('tries',10) + try: + tries = int(tries) + except: + log.warn("tries is not an integer. using 10") + tries = 10 + + if post_list == "all": + post_list = post_list_all + + all_keys = { } + all_keys['instance_id'] = cloud.get_instance_id() + all_keys['hostname'] = cloud.get_hostname() + + pubkeys = { + 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', + 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', + } + + for n, path in pubkeys.iteritems(): + try: + fp = open(path, "rb") + all_keys[n] = fp.read() + all_keys[n] + fp.close() + except: + log.warn("%s: failed to open in phone_home" % path) + + submit_keys = { } + for k in post_list: + if k in all_keys: + submit_keys[k] = all_keys[k] + else: + submit_keys[k] = "N/A" + log.warn("requested key %s from 'post' list not available") + + url = util.render_string(url, { 'INSTANCE_ID' : all_keys['instance_id'] }) + + last_e = None + for i in range(0,tries): + try: + util.readurl(url, submit_keys) + log.debug("succeeded submit to %s on try %i" % (url, i+1)) + return + except Exception, e: + log.debug("failed to post to %s on try %i" % (url, i+1)) + last_e = e + sleep(3) + + log.warn("failed to post to %s in %i tries" % (url, tries)) + if last_e: raise(last_e) + + return diff --git a/cloudinit/CloudConfig/cc_resizefs.py b/cloudinit/CloudConfig/cc_resizefs.py new file mode 100644 index 00000000..11a10005 --- /dev/null +++ b/cloudinit/CloudConfig/cc_resizefs.py @@ -0,0 +1,54 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit.util as util +import subprocess +import traceback + +def handle(name,cfg,cloud,log,args): + if len(args) != 0: + resize_root = False + if str(value).lower() in [ 'true', '1', 'on', 'yes']: + resize_root = True + else: + resize_root = util.get_cfg_option_bool(cfg,"resize_rootfs",False) + + if not resize_root: return + + log.debug("resizing root filesystem on first boot") + + cmd = ['blkid', '-sTYPE', '-ovalue', '/dev/root'] + try: + (fstype,err) = util.subp(cmd) + except Exception, e: + log.warn("Failed to get filesystem type via %s" % cmd) + raise + + if fstype.startswith("ext"): + resize_cmd = [ 'resize2fs', '/dev/root' ] + elif fstype == "xfs": + resize_cmd = [ 'xfs_growfs', '/dev/root' ] + else: + log.debug("not resizing unknown filesystem %s" % fstype) + return + + try: + (out,err) = util.subp(resize_cmd) + except Exception, e: + log.warn("Failed to resize filesystem (%s,%s)" % cmd) + raise + diff --git a/cloudinit/CloudConfig/cc_rightscale_userdata.py b/cloudinit/CloudConfig/cc_rightscale_userdata.py new file mode 100644 index 00000000..a90e6d18 --- /dev/null +++ b/cloudinit/CloudConfig/cc_rightscale_userdata.py @@ -0,0 +1,73 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +## +## The purpose of this script is to allow cloud-init to consume +## rightscale style userdata. rightscale user data is key-value pairs +## in a url-query-string like format. +## +## for cloud-init support, there will be a key named +## 'CLOUD_INIT_REMOTE_HOOK'. +## +## This cloud-config module will +## - read the blob of data from raw user data, and parse it as key/value +## - for each key that is found, download the content to +## the local instance/scripts directory and set them executable. +## - the files in that directory will be run by the user-scripts module +## Therefore, this must run before that. +## +## +import cloudinit.util as util +from cloudinit.CloudConfig import per_once, per_always, per_instance +from cloudinit import get_ipath_cur +from urlparse import parse_qs + +frequency = per_instance +my_name = "cc_rightscale_userdata" +my_hookname = 'CLOUD_INIT_REMOTE_HOOK' + +def handle(name,cfg,cloud,log,args): + try: + ud = cloud.get_userdata_raw() + except: + log.warn("failed to get raw userdata in %s" % my_name) + return + + try: + mdict = parse_qs(cloud.get_userdata_raw()) + if not my_hookname in mdict: return + except: + log.warn("failed to urlparse.parse_qa(userdata_raw())") + raise + + scripts_d = get_ipath_cur('scripts') + i = 0 + errors = [ ] + first_e = None + for url in mdict[my_hookname]: + fname = "%s/rightscale-%02i" % (scripts_d,i) + i = i +1 + try: + content = util.readurl(url) + util.write_file(fname, content, mode=0700) + except Exception, e: + if not first_e: first_e = None + log.warn("%s failed to read %s: %s" % (my_name, url, e)) + + if first_e: + raise(e) diff --git a/cloudinit/CloudConfig/cc_rsyslog.py b/cloudinit/CloudConfig/cc_rsyslog.py new file mode 100644 index 00000000..3320dbb2 --- /dev/null +++ b/cloudinit/CloudConfig/cc_rsyslog.py @@ -0,0 +1,99 @@ +# vi: ts=4 expandtab syntax=python +# +# Copyright (C) 2009-2010 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit +import logging +import cloudinit.util as util +import subprocess +import traceback + +DEF_FILENAME = "20-cloud-config.conf" +DEF_DIR = "/etc/rsyslog.d" + +def handle(name,cfg,cloud,log,args): + # rsyslog: + # - "*.* @@192.158.1.1" + # - content: "*.* @@192.0.2.1:10514" + # - filename: 01-examplecom.conf + # content: | + # *.* @@syslogd.example.com + + # process 'rsyslog' + if not 'rsyslog' in cfg: return + + def_dir = cfg.get('rsyslog_dir', DEF_DIR) + def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) + + entries = cfg['rsyslog'] + + files = [ ] + elst = [ ] + for ent in cfg['rsyslog']: + if isinstance(ent,dict): + if not "content" in ent: + elst.append((ent, "no 'content' entry")) + continue + content = ent['content'] + filename = ent.get("filename", def_fname) + else: + content = ent + filename = def_fname + + if not filename.startswith("/"): + filename = "%s/%s" % (def_dir,filename) + + omode = "ab" + # truncate filename first time you see it + if filename not in files: + omode = "wb" + files.append(filename) + + try: + util.write_file(filename, content + "\n", omode=omode) + except Exception, e: + log.debug(traceback.format_exc(e)) + elst.append((content, "failed to write to %s" % filename)) + + # need to restart syslogd + restarted = False + try: + # if this config module is running at cloud-init time + # (before rsyslog is running) we don't actually have to + # restart syslog. + # + # upstart actually does what we want here, in that it doesn't + # start a service that wasn't running already on 'restart' + # it will also return failure on the attempt, so 'restarted' + # won't get set + log.debug("restarting rsyslog") + p = util.subp(['service', 'rsyslog', 'restart']) + restarted = True + + except Exception, e: + elst.append(("restart", str(e))) + + if restarted: + # this only needs to run if we *actually* restarted + # syslog above. + cloudinit.logging_set_from_cfg_file() + log = logging.getLogger() + log.debug("rsyslog configured %s" % files) + + for e in elst: + log.warn("rsyslog error: %s\n" % ':'.join(e)) + + return diff --git a/cloudinit/CloudConfig/cc_runcmd.py b/cloudinit/CloudConfig/cc_runcmd.py index 97d21900..afa7a441 100644 --- a/cloudinit/CloudConfig/cc_runcmd.py +++ b/cloudinit/CloudConfig/cc_runcmd.py @@ -21,7 +21,7 @@ import cloudinit.util as util def handle(name,cfg,cloud,log,args): if not cfg.has_key("runcmd"): return - outfile="%s/runcmd" % cloudinit.user_scripts_dir + outfile="%s/runcmd" % cloud.get_ipath('scripts') content="#!/bin/sh\n" escaped="%s%s%s%s" % ( "'", '\\', "'", "'" ) diff --git a/cloudinit/CloudConfig/cc_scripts_per_boot.py b/cloudinit/CloudConfig/cc_scripts_per_boot.py new file mode 100644 index 00000000..4e407fb7 --- /dev/null +++ b/cloudinit/CloudConfig/cc_scripts_per_boot.py @@ -0,0 +1,30 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit.util as util +from cloudinit.CloudConfig import per_once, per_always, per_instance +from cloudinit import get_cpath, get_ipath_cur + +frequency = per_always +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-boot") + +def handle(name,cfg,cloud,log,args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/CloudConfig/cc_scripts_per_instance.py b/cloudinit/CloudConfig/cc_scripts_per_instance.py new file mode 100644 index 00000000..22b41185 --- /dev/null +++ b/cloudinit/CloudConfig/cc_scripts_per_instance.py @@ -0,0 +1,30 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit.util as util +from cloudinit.CloudConfig import per_once, per_always, per_instance +from cloudinit import get_cpath, get_ipath_cur + +frequency = per_instance +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-instance") + +def handle(name,cfg,cloud,log,args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/CloudConfig/cc_scripts_per_once.py b/cloudinit/CloudConfig/cc_scripts_per_once.py new file mode 100644 index 00000000..9d752325 --- /dev/null +++ b/cloudinit/CloudConfig/cc_scripts_per_once.py @@ -0,0 +1,30 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit.util as util +from cloudinit.CloudConfig import per_once, per_always, per_instance +from cloudinit import get_cpath, get_ipath_cur + +frequency = per_once +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-once") + +def handle(name,cfg,cloud,log,args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/CloudConfig/cc_scripts_user.py b/cloudinit/CloudConfig/cc_scripts_user.py new file mode 100644 index 00000000..bafecd23 --- /dev/null +++ b/cloudinit/CloudConfig/cc_scripts_user.py @@ -0,0 +1,30 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit.util as util +from cloudinit.CloudConfig import per_once, per_always, per_instance +from cloudinit import get_cpath, get_ipath_cur + +frequency = per_instance +runparts_path = "%s/%s" % (get_ipath_cur(), "scripts") + +def handle(name,cfg,cloud,log,args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/CloudConfig/cc_set_hostname.py b/cloudinit/CloudConfig/cc_set_hostname.py new file mode 100644 index 00000000..34621e97 --- /dev/null +++ b/cloudinit/CloudConfig/cc_set_hostname.py @@ -0,0 +1,38 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit.util as util +import subprocess + +def handle(name,cfg,cloud,log,args): + if util.get_cfg_option_bool(cfg,"preserve_hostname",False): + log.debug("preserve_hostname is set. not setting hostname") + return(True) + + try: + hostname = util.get_cfg_option_str(cfg,"hostname",cloud.get_hostname()) + set_hostname(hostname, log) + except Exception, e: + util.logexc(log) + log.warn("failed to set hostname\n") + + return(True) + +def set_hostname(hostname, log): + subprocess.Popen(['hostname', hostname]).communicate() + util.write_file("/etc/hostname","%s\n" % hostname, 0644) + log.debug("populated /etc/hostname with %s on first boot", hostname) diff --git a/cloudinit/CloudConfig/cc_ssh.py b/cloudinit/CloudConfig/cc_ssh.py index 07527906..7b9ba5ab 100644 --- a/cloudinit/CloudConfig/cc_ssh.py +++ b/cloudinit/CloudConfig/cc_ssh.py @@ -60,18 +60,7 @@ def handle(name,cfg,cloud,log,args): send_ssh_keys_to_console() def send_ssh_keys_to_console(): - send_keys_sh = """ - { - echo - echo "#############################################################" - echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----" - ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key.pub - ssh-keygen -l -f /etc/ssh/ssh_host_dsa_key.pub - echo "-----END SSH HOST KEY FINGERPRINTS-----" - echo "#############################################################" - } | logger -p user.info -s -t "ec2" - """ - subprocess.call(('sh', '-c', send_keys_sh)) + subprocess.call(('/usr/lib/cloud-init/write-ssh-key-fingerprints',)) def apply_credentials(keys, user, disable_root): keys = set(keys) @@ -79,7 +68,7 @@ def apply_credentials(keys, user, disable_root): setup_user_keys(keys, user, '') if disable_root: - key_prefix = 'command="echo \'Please login as the %s user rather than root user.\';echo;sleep 10" ' % user + key_prefix = 'command="echo \'Please login as the user \\\"%s\\\" rather than the user \\\"root\\\".\';echo;sleep 10" ' % user else: key_prefix = '' diff --git a/cloudinit/CloudConfig/cc_ssh_import_id.py b/cloudinit/CloudConfig/cc_ssh_import_id.py index dd4d3184..bf1314be 100644 --- a/cloudinit/CloudConfig/cc_ssh_import_id.py +++ b/cloudinit/CloudConfig/cc_ssh_import_id.py @@ -31,7 +31,7 @@ def handle(name,cfg,cloud,log,args): if len(ids) == 0: return - cmd = [ "sudo", "-Hu", user, "ssh-import-lp-id" ] + ids + cmd = [ "sudo", "-Hu", user, "ssh-import-id" ] + ids log.debug("importing ssh ids. cmd = %s" % cmd) diff --git a/cloudinit/CloudConfig/cc_update_hostname.py b/cloudinit/CloudConfig/cc_update_hostname.py new file mode 100644 index 00000000..3663c0ab --- /dev/null +++ b/cloudinit/CloudConfig/cc_update_hostname.py @@ -0,0 +1,95 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit.util as util +import subprocess +import errno +from cloudinit.CloudConfig import per_always + +frequency = per_always + +def handle(name,cfg,cloud,log,args): + if util.get_cfg_option_bool(cfg,"preserve_hostname",False): + log.debug("preserve_hostname is set. not updating hostname") + return + + try: + hostname = util.get_cfg_option_str(cfg,"hostname",cloud.get_hostname()) + prev ="%s/%s" % (cloud.get_cpath('datadir'),"previous-hostname") + update_hostname(hostname, prev, log) + except Exception, e: + log.warn("failed to set hostname\n") + raise + +# read hostname from a 'hostname' file +# allow for comments and stripping line endings. +# if file doesn't exist, or no contents, return default +def read_hostname(filename, default=None): + try: + fp = open(filename,"r") + lines = fp.readlines() + fp.close() + for line in lines: + hpos = line.find("#") + if hpos != -1: + line = line[0:hpos] + line = line.rstrip() + if line: + return line + except IOError, e: + if e.errno != errno.ENOENT: raise + return default + +def update_hostname(hostname, prev_file, log): + etc_file = "/etc/hostname" + + hostname_prev = None + hostname_in_etc = None + + try: + hostname_prev = read_hostname(prev_file) + except Exception, e: + log.warn("Failed to open %s: %s" % (prev_file, e)) + + try: + hostname_in_etc = read_hostname(etc_file) + except: + log.warn("Failed to open %s" % etc_file) + + update_files = [] + if not hostname_prev or hostname_prev != hostname: + update_files.append(prev_file) + + if (not hostname_in_etc or + (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): + update_files.append(etc_file) + + try: + for fname in update_files: + util.write_file(fname,"%s\n" % hostname, 0644) + log.debug("wrote %s to %s" % (hostname,fname)) + except: + log.warn("failed to write hostname to %s" % fname) + + if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: + log.debug("%s differs from %s. assuming user maintained" % + (prev_file,etc_file)) + + if etc_file in update_files: + log.debug("setting hostname to %s" % hostname) + subprocess.Popen(['hostname', hostname]).communicate() + diff --git a/cloudinit/CloudConfig/cc_updates_check.py b/cloudinit/CloudConfig/cc_updates_check.py deleted file mode 100644 index aaa2e6b0..00000000 --- a/cloudinit/CloudConfig/cc_updates_check.py +++ /dev/null @@ -1,49 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# -# Author: Scott Moser <scott.moser@canonical.com> -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -import cloudinit.util as util -import cloudinit -import os -import time - -cronpre = "/etc/cron.d/cloudinit" - -def handle(name,cfg,cloud,log,args): - if not util.get_cfg_option_bool(cfg, 'updates-check', True): - return - build_info = "/etc/cloud/build.info" - if not os.path.isfile(build_info): - log.warn("no %s file" % build_info) - - avail="%s/%s" % ( cloudinit.datadir, "available.build" ) - cmd=( "uec-query-builds", "--system-suite", "--config", "%s" % build_info, - "--output", "%s" % avail, "is-update-available" ) - try: - util.subp(cmd) - except: - log.warn("failed to execute uec-query-build for updates check") - - # add a cron entry for this hour and this minute every day - try: - cron=open("%s-%s" % (cronpre, "updates") ,"w") - cron.write("%s root %s\n" % \ - (time.strftime("%M %H * * *"),' '.join(cmd))) - cron.close() - except: - log.warn("failed to enable cron update system check") - raise - diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py index 7a6c9dc9..1c0edc59 100644 --- a/cloudinit/DataSourceEc2.py +++ b/cloudinit/DataSourceEc2.py @@ -30,13 +30,7 @@ import errno class DataSourceEc2(DataSource.DataSource): api_ver = '2009-04-04' - cachedir = cloudinit.cachedir + '/ec2' - - location_locale_map = { - 'us' : 'en_US.UTF-8', - 'eu' : 'en_GB.UTF-8', - 'default' : 'en_US.UTF-8', - } + seeddir = cloudinit.seeddir + '/ec2' def __init__(self): pass @@ -46,10 +40,10 @@ class DataSourceEc2(DataSource.DataSource): def get_data(self): seedret={ } - if util.read_optional_seed(seedret,base=self.cachedir + "/"): + if util.read_optional_seed(seedret,base=self.seeddir+ "/"): self.userdata_raw = seedret['user-data'] self.metadata = seedret['meta-data'] - cloudinit.log.debug("using seeded ec2 data in %s" % self.cachedir) + cloudinit.log.debug("using seeded ec2 data in %s" % self.seeddir) return True try: @@ -71,13 +65,6 @@ class DataSourceEc2(DataSource.DataSource): def get_local_mirror(self): return(self.get_mirror_from_availability_zone()) - def get_locale(self): - az = self.metadata['placement']['availability-zone'] - if self.location_locale_map.has_key(az[0:2]): - return(self.location_locale_map[az[0:2]]) - else: - return(self.location_locale_map["default"]) - def get_mirror_from_availability_zone(self, availability_zone = None): # availability is like 'us-west-1b' or 'eu-west-1a' if availability_zone == None: @@ -121,7 +108,7 @@ class DataSourceEc2(DataSource.DataSource): cloudinit.log.warning("waiting for metadata service at %s\n" % url) cloudinit.log.warning(" %s [%02s/%s]: %s\n" % - (time.strftime("%H:%M:%S"), x+1, sleeps, reason)) + (time.strftime("%H:%M:%S",time.gmtime()), x+1, sleeps, reason)) time.sleep(sleeptime) cloudinit.log.critical("giving up on md after %i seconds\n" % diff --git a/cloudinit/DataSourceNoCloud.py b/cloudinit/DataSourceNoCloud.py index cd988d08..2f6033a9 100644 --- a/cloudinit/DataSourceNoCloud.py +++ b/cloudinit/DataSourceNoCloud.py @@ -32,7 +32,7 @@ class DataSourceNoCloud(DataSource.DataSource): supported_seed_starts = ( "/" , "file://" ) seed = None cmdline_id = "ds=nocloud" - seeddir = cloudinit.cachedir + '/nocloud' + seeddir = cloudinit.seeddir + '/nocloud' def __init__(self): pass @@ -108,16 +108,10 @@ class DataSourceNoCloud(DataSource.DataSource): # root=LABEL=uec-rootfs ro ds=nocloud def parse_cmdline_data(ds_id,fill,cmdline=None): if cmdline is None: - if 'DEBUG_PROC_CMDLINE' in os.environ: - cmdline = os.environ["DEBUG_PROC_CMDLINE"] - else: - cmdfp = open("/proc/cmdline") - cmdline = cmdfp.read().strip() - cmdfp.close() - cmdline = " %s " % cmdline.lower() - - if not ( " %s " % ds_id in cmdline or " %s;" % ds_id in cmdline ): - return False + cmdline = util.get_cmdline() + + if not ( " %s " % ds_id in cmdline or " %s;" % ds_id in cmdline ): + return False argline="" # cmdline can contain: @@ -149,4 +143,4 @@ def parse_cmdline_data(ds_id,fill,cmdline=None): class DataSourceNoCloudNet(DataSourceNoCloud): cmdline_id = "ds=nocloud-net" supported_seed_starts = ( "http://", "https://", "ftp://" ) - seeddir = cloudinit.cachedir + '/nocloud-net' + seeddir = cloudinit.seeddir + '/nocloud-net' diff --git a/cloudinit/UserDataHandler.py b/cloudinit/UserDataHandler.py index ab7d0bc8..fbb000fc 100644 --- a/cloudinit/UserDataHandler.py +++ b/cloudinit/UserDataHandler.py @@ -19,7 +19,9 @@ import email from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText - +from email.mime.base import MIMEBase +from email import encoders +import yaml starts_with_mappings={ '#include' : 'text/x-include-url', @@ -27,7 +29,8 @@ starts_with_mappings={ '#cloud-config' : 'text/cloud-config', '#upstart-job' : 'text/upstart-job', '#part-handler' : 'text/part-handler', - '#cloud-boothook' : 'text/cloud-boothook' + '#cloud-boothook' : 'text/cloud-boothook', + '#cloud-config-archive' : 'text/cloud-config-archive', } # if 'str' is compressed return decompressed otherwise return it @@ -43,12 +46,47 @@ def decomp_str(str): def do_include(str,parts): import urllib # is just a list of urls, one per line + # also support '#include <url here>' for line in str.splitlines(): if line == "#include": continue + if line.startswith("#include"): + line = line[len("#include"):].lstrip() if line.startswith("#"): continue content = urllib.urlopen(line).read() process_includes(email.message_from_string(decomp_str(content)),parts) +def explode_cc_archive(archive,parts): + for ent in yaml.load(archive): + # ent can be one of: + # dict { 'filename' : 'value' , 'content' : 'value', 'type' : 'value' } + # filename and type not be present + # or + # scalar(payload) + filename = 'part-%03d' % len(parts['content']) + def_type = "text/cloud-config" + if isinstance(ent,str): + content = ent + mtype = type_from_startswith(content,def_type) + else: + content = ent.get('content', '') + filename = ent.get('filename', filename) + mtype = ent.get('type', None) + if mtype == None: + mtype = type_from_startswith(payload,def_type) + + print "adding %s,%s" % (filename, mtype) + parts['content'].append(content) + parts['names'].append(filename) + parts['types'].append(mtype) + +def type_from_startswith(payload, default=None): + # slist is sorted longest first + slist = sorted(starts_with_mappings.keys(), key=lambda e: 0-len(e)) + for sstr in slist: + if payload.startswith(sstr): + return(starts_with_mappings[sstr]) + return default + def process_includes(msg,parts): # parts is a dictionary of arrays # parts['content'] @@ -67,10 +105,7 @@ def process_includes(msg,parts): ctype = None ctype_orig = part.get_content_type() if ctype_orig == "text/plain": - for str, gtype in starts_with_mappings.items(): - if payload.startswith(str): - ctype = gtype - break + ctype = type_from_startswith(payload) if ctype is None: ctype = ctype_orig @@ -79,6 +114,10 @@ def process_includes(msg,parts): do_include(payload,parts) continue + if ctype == "text/cloud-config-archive": + explode_cc_archive(payload,parts) + continue + filename = part.get_filename() if not filename: filename = 'part-%03d' % len(parts['content']) diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py index 6a59a23f..9c02ff8a 100644 --- a/cloudinit/__init__.py +++ b/cloudinit/__init__.py @@ -18,94 +18,35 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. # -datadir = '/var/lib/cloud/data' -semdir = '/var/lib/cloud/sem' -pluginsdir = datadir + '/plugins' -cachedir = datadir + '/cache' -userdata_raw = datadir + '/user-data.txt' -userdata = datadir + '/user-data.txt.i' -user_scripts_dir = datadir + "/scripts" -boothooks_dir = datadir + "/boothooks" -cloud_config = datadir + '/cloud-config.txt' -#cloud_config = '/tmp/cloud-config.txt' -data_source_cache = cachedir + '/obj.pkl' +varlibdir = '/var/lib/cloud' +cur_instance_link = varlibdir + "/instance" +boot_finished = cur_instance_link + "/boot-finished" system_config = '/etc/cloud/cloud.cfg' +seeddir = varlibdir + "/seed" cfg_env_name = "CLOUD_CFG" cfg_builtin = """ +log_cfgs: [ ] cloud_type: auto -user: ubuntu -disable_root: 1 - -cloud_config_modules: - - mounts - - ssh-import-id - - ssh - - grub-dpkg - - apt-update-upgrade - - puppet - - updates-check - - disable-ec2-metadata - - runcmd - - byobu - -log_cfg: built_in +def_log_file: /var/log/cloud-init.log +syslog_fix_perms: syslog:adm """ - -def_log_file = '/var/log/cloud-init.log' logger_name = "cloudinit" -built_in_log_base = """ -[loggers] -keys=root,cloudinit - -[handlers] -keys=consoleHandler,cloudLogHandler - -[formatters] -keys=simpleFormatter,arg0Formatter - -[logger_root] -level=DEBUG -handlers=consoleHandler,cloudLogHandler - -[logger_cloudinit] -level=DEBUG -qualname=cloudinit -handlers= -propagate=1 - -[handler_consoleHandler] -class=StreamHandler -level=WARNING -formatter=arg0Formatter -args=(sys.stderr,) - -[formatter_arg0Formatter] -format=%(asctime)s - %(filename)s[%(levelname)s]: %(message)s - -[formatter_simpleFormatter] -format=[CLOUDINIT] %(asctime)s - %(filename)s[%(levelname)s]: %(message)s -datefmt= - -""" - -built_in_log_clougLogHandlerLog=""" -[handler_cloudLogHandler] -class=FileHandler -level=DEBUG -formatter=simpleFormatter -args=('__CLOUDINIT_LOGGER_FILE__',) -""" - -built_in_log_cloudLogHandlerSyslog= """ -[handler_cloudLogHandler] -class=handlers.SysLogHandler -level=DEBUG -formatter=simpleFormatter -args=("/dev/log", handlers.SysLogHandler.LOG_USER) -""" - +pathmap = { + "handlers" : "/handlers", + "scripts" : "/scripts", + "sem" : "/sem", + "boothooks" : "/boothooks", + "userdata_raw" : "/user-data.txt", + "userdata" : "/user-data-raw.txt.i", + "obj_pkl" : "/obj.pkl", + "cloud_config" : "/cloud-config.txt", + "datadir" : "/data", + None : "", +} + +parsed_cfgs = { } import os from configobj import ConfigObj @@ -121,43 +62,44 @@ import util import logging import logging.config import StringIO +import glob class NullHandler(logging.Handler): - def emit(self,record): pass + def emit(self,record): pass log = logging.getLogger(logger_name) log.addHandler(NullHandler()) def logging_set_from_cfg_file(cfg_file=system_config): - logging_set_from_cfg(util.get_base_cfg(cfg_file,cfg_builtin)) + logging_set_from_cfg(util.get_base_cfg(cfg_file,cfg_builtin,parsed_cfgs)) + +def logging_set_from_cfg(cfg): + log_cfgs = [] + logcfg=util.get_cfg_option_str(cfg, "log_cfg", False) + if logcfg: + # if there is a 'logcfg' entry in the config, respect + # it, it is the old keyname + log_cfgs = [ logcfg ] + elif "log_cfgs" in cfg: + for cfg in cfg['log_cfgs']: + if isinstance(cfg,list): + log_cfgs.append('\n'.join(cfg)) + else: + log_cfgs.append() + + if not len(log_cfgs): + sys.stderr.write("Warning, no logging configured\n") + return -def logging_set_from_cfg(cfg, logfile=None): - if logfile is None: + for logcfg in log_cfgs: try: - open(def_log_file,"a").close() - logfile = def_log_file - except IOError as e: - if e.errno == errno.EACCES: - logfile = "/dev/null" - else: raise - - logcfg=util.get_cfg_option_str(cfg, "log_cfg", "built_in") - failsafe = "%s\n%s" % (built_in_log_base, built_in_log_clougLogHandlerLog) - builtin = False - if logcfg.lower() == "built_in": - logcfg = "%s\n%s" % (built_in_log_base, built_in_log_cloudLogHandlerSyslog) - builtin = True - - logcfg=logcfg.replace("__CLOUDINIT_LOGGER_FILE__",logfile) - try: - logging.config.fileConfig(StringIO.StringIO(logcfg)) - return - except: - if not builtin: - sys.stderr.write("Warning, setting config.fileConfig failed\n") + logging.config.fileConfig(StringIO.StringIO(logcfg)) + return + except: + pass + + raise Exception("no valid logging found\n") - failsafe=failsafe.replace("__CLOUDINIT_LOGGER_FILE__",logfile) - logging.config.fileConfig(StringIO.StringIO(failsafe)) import DataSourceEc2 import DataSourceNoCloud @@ -174,7 +116,6 @@ class CloudInit: "all": ( "nocloud-net", "ec2" ), "local" : ( "nocloud", ), } - cfg = None part_handlers = { } old_conffile = '/etc/ec2-init/ec2-config.cfg' @@ -196,7 +137,7 @@ class CloudInit: if self.cfg: return(self.cfg) - conf = util.get_base_cfg(system_config,cfg_builtin) + conf = util.get_base_cfg(self.sysconfig,cfg_builtin, parsed_cfgs) # support reading the old ConfigObj format file and merging # it into the yaml dictionary @@ -212,7 +153,11 @@ class CloudInit: def restore_from_cache(self): try: - f=open(data_source_cache, "rb") + # we try to restore from a current link and static path + # by using the instance link, if purge_cache was called + # the file wont exist + cache = get_ipath_cur('obj_pkl') + f=open(cache, "rb") data = cPickle.load(f) self.datasource = data return True @@ -220,16 +165,17 @@ class CloudInit: return False def write_to_cache(self): + cache = self.get_ipath("obj_pkl") try: - os.makedirs(os.path.dirname(data_source_cache)) + os.makedirs(os.path.dirname(cache)) except OSError as e: if e.errno != errno.EEXIST: return False try: - f=open(data_source_cache, "wb") + f=open(cache, "wb") data = cPickle.dump(self.datasource,f) - os.chmod(data_source_cache,0400) + os.chmod(cache,0400) return True except: return False @@ -252,6 +198,7 @@ class CloudInit: for ds in cfglist.split(','): dslist.append(strip(ds).tolower()) + log.debug("searching for data source in [%s]" % str(dslist)) for ds in dslist: if ds not in self.datasource_map: log.warn("data source %s not found in map" % ds) @@ -270,27 +217,48 @@ class CloudInit: log.debug("did not find data source from %s" % dslist) raise DataSourceNotFoundException("Could not find data source") + def set_cur_instance(self): + try: + os.unlink(cur_instance_link) + except OSError, e: + if e.errno != errno.ENOENT: raise + + os.symlink("./instances/%s" % self.get_instance_id(), cur_instance_link) + idir = self.get_ipath() + dlist = [] + for d in [ "handlers", "scripts", "sem" ]: + dlist.append("%s/%s" % (idir, d)) + + util.ensure_dirs(dlist) + def get_userdata(self): return(self.datasource.get_userdata()) + def get_userdata_raw(self): + return(self.datasource.get_userdata_raw()) + + def get_instance_id(self): + return(self.datasource.get_instance_id()) + def update_cache(self): self.write_to_cache() self.store_userdata() def store_userdata(self): - util.write_file(userdata_raw, self.datasource.get_userdata_raw(), 0600) - util.write_file(userdata, self.datasource.get_userdata(), 0600) + util.write_file(self.get_ipath('userdata_raw'), + self.datasource.get_userdata_raw(), 0600) + util.write_file(self.get_ipath('userdata'), + self.datasource.get_userdata(), 0600) def initctl_emit(self): + cc_path = get_ipath_cur('cloud_config') subprocess.Popen(['initctl', 'emit', 'cloud-config', - '%s=%s' % (cfg_env_name,cloud_config)]).communicate() + '%s=%s' % (cfg_env_name,cc_path)]).communicate() def sem_getpath(self,name,freq): - freqtok = freq if freq == 'once-per-instance': - freqtok = self.datasource.get_instance_id() - - return("%s/%s.%s" % (semdir,name,freqtok)) + return("%s/%s" % (self.get_ipath("sem"),name)) + return("%s/%s.%s" % (get_cpath("sem"), name, freq)) def sem_has_run(self,name,freq): if freq == "always": return False @@ -349,9 +317,40 @@ class CloudInit: self.sem_clear(semname,freq) raise + # get_ipath : get the instance path for a name in pathmap + # (/var/lib/cloud/instances/<instance>/name)<name>) + def get_ipath(self, name=None): + return("%s/instances/%s%s" + % (varlibdir,self.get_instance_id(), pathmap[name])) + def consume_userdata(self): self.get_userdata() data = self + + cdir = get_cpath("handlers") + idir = self.get_ipath("handlers") + + # add the path to the plugins dir to the top of our list for import + # instance dir should be read before cloud-dir + sys.path.insert(0,cdir) + sys.path.insert(0,idir) + + # add handlers in cdir + for fname in glob.glob("%s/*.py" % cdir): + if not os.path.isfile(fname): continue + modname = os.path.basename(fname)[0:-3] + try: + mod = __import__(modname) + lister = getattr(mod, "list_types") + handler = getattr(mod, "handle_part") + mtypes = lister() + for mtype in mtypes: + self.part_handlers[mtype]=handler + log.debug("added handler for [%s] from %s" % (mtypes,fname)) + except: + log.warn("failed to initialize handler in %s" % fname) + util.logexc(log) + # give callbacks opportunity to initialize for ctype, func in self.part_handlers.items(): func(data, "__begin__",None,None) @@ -368,16 +367,13 @@ class CloudInit: self.handlercount = 0 return - # add the path to the plugins dir to the top of our list for import - if self.handlercount == 0: - sys.path.insert(0,pluginsdir) - self.handlercount=self.handlercount+1 - # write content to pluginsdir + # write content to instance's handlerdir + handlerdir = self.get_ipath("handler") modname = 'part-handler-%03d' % self.handlercount modfname = modname + ".py" - util.write_file("%s/%s" % (pluginsdir,modfname), payload, 0600) + util.write_file("%s/%s" % (handlerdir,modfname), payload, 0600) try: mod = __import__(modname) @@ -402,7 +398,9 @@ class CloudInit: return filename=filename.replace(os.sep,'_') - util.write_file("%s/%s" % (user_scripts_dir,filename), payload, 0700) + scriptsdir = get_ipath_cur('scripts') + util.write_file("%s/%s/%s" % + (scriptsdir,self.get_instance_id(),filename), payload, 0700) def handle_upstart_job(self,data,ctype,filename,payload): if ctype == "__end__" or ctype == "__begin__": return @@ -416,6 +414,7 @@ class CloudInit: self.cloud_config_str="" return if ctype == "__end__": + cloud_config = self.get_ipath("cloud_config") util.write_file(cloud_config, self.cloud_config_str, 0600) ## this could merge the cloud config with the system config @@ -452,6 +451,7 @@ class CloudInit: elif start != 0: payload=payload[start:] + boothooks_dir = self.get_ipath("boothooks") filepath = "%s/%s" % (boothooks_dir,filename) util.write_file(filepath, payload, 0700) try: @@ -480,15 +480,58 @@ class CloudInit: def device_name_to_device(self,name): return(self.datasource.device_name_to_device(name)) + # I really don't know if this should be here or not, but + # I needed it in cc_update_hostname, where that code had a valid 'cloud' + # reference, but did not have a cloudinit handle + # (ie, no cloudinit.get_cpath()) + def get_cpath(self,name=None): + return(get_cpath(name)) + + +def initfs(): + subds = [ 'scripts/per-instance', 'scripts/per-once', 'scripts/per-boot', + 'seed', 'instances', 'handlers', 'sem', 'data' ] + dlist = [ ] + for subd in subds: + dlist.append("%s/%s" % (varlibdir, subd)) + util.ensure_dirs(dlist) + + cfg = util.get_base_cfg(system_config,cfg_builtin,parsed_cfgs) + log_file = None + if 'def_log_file' in cfg: + log_file = cfg['def_log_file'] + fp = open(log_file,"ab") + fp.close() + if log_file and 'syslog' in cfg: + perms = cfg['syslog'] + (u,g) = perms.split(':',1) + if u == "-1" or u == "None": u = None + if g == "-1" or g == "None": g = None + util.chownbyname(log_file, u, g) def purge_cache(): - try: - os.unlink(data_source_cache) - except OSError as e: - if e.errno != errno.ENOENT: return(False) - except: - return(False) + rmlist = ( boot_finished , cur_instance_link ) + for f in rmlist: + try: + os.unlink(f) + except OSError as e: + if e.errno == errno.ENOENT: continue + return(False) + except: + return(False) return(True) +# get_ipath_cur: get the current instance path for an item +def get_ipath_cur(name=None): + return("%s/instance/%s" % (varlibdir, pathmap[name])) + +# get_cpath : get the "clouddir" (/var/lib/cloud/<name>) +# for a name in dirmap +def get_cpath(name=None): + return("%s%s" % (varlibdir, pathmap[name])) + +def get_base_cfg(): + return(util.get_base_cfg(system_config,cfg_builtin,parsed_cfgs)) + class DataSourceNotFoundException(Exception): pass diff --git a/cloudinit/util.py b/cloudinit/util.py index c1b4fd2d..e958fc02 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -17,13 +17,16 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import yaml import os +import os.path import errno import subprocess from Cheetah.Template import Template import cloudinit import urllib2 +import urllib import logging import traceback +import re WARN = logging.WARN DEBUG = logging.DEBUG @@ -40,13 +43,32 @@ def read_conf(fname): return { } raise -def get_base_cfg(cfgfile,cfg_builtin=""): - syscfg = read_conf(cfgfile) +def get_base_cfg(cfgfile,cfg_builtin="", parsed_cfgs=None): + kerncfg = { } + syscfg = { } + if parsed_cfgs and cfgfile in parsed_cfgs: + return(parsed_cfgs[cfgfile]) + + contents = read_file_with_includes(cfgfile) + if contents: + syscfg = yaml.load(contents) + + kern_contents = read_cc_from_cmdline() + if kern_contents: + kerncfg = yaml.load(kern_contents) + + # kernel parameters override system config + combined = mergedict(kerncfg, syscfg) + if cfg_builtin: builtin = yaml.load(cfg_builtin) + fin = mergedict(combined,builtin) else: - return(syscfg) - return(mergedict(syscfg,builtin)) + fin = combined + + if parsed_cfgs != None: + parsed_cfgs[cfgfile] = fin + return(fin) def get_cfg_option_bool(yobj, key, default=False): if not yobj.has_key(key): return default @@ -106,6 +128,16 @@ def getkeybyid(keyid,keyserver): args=['sh', '-c', shcmd, "export-gpg-keyid", keyid, keyserver] return(subp(args)[0]) +def runparts(dirp, skip_no_exist=True): + if skip_no_exist and not os.path.isdir(dirp): return + + cmd = [ 'run-parts', '--regex', '.*', dirp ] + sp = subprocess.Popen(cmd) + sp.communicate() + if sp.returncode is not 0: + raise subprocess.CalledProcessError(sp.returncode,cmd) + return + def subp(args, input=None): s_in = None if input is not None: @@ -122,6 +154,10 @@ def render_to_file(template, outfile, searchList): f.write(t.respond()) f.close() +def render_string(template, searchList): + return(Template(template, searchList=[searchList]).respond()) + + # read_optional_seed # returns boolean indicating success or failure (presense of files) # if files are present, populates 'fill' dictionary with 'user-data' and @@ -168,3 +204,133 @@ def read_seeded(base="", ext="", timeout=2): def logexc(log,lvl=logging.DEBUG): log.log(lvl,traceback.format_exc()) + +class RecursiveInclude(Exception): + pass + +def read_file_with_includes(fname, rel = ".", stack=[], patt = None): + if not fname.startswith("/"): + fname = os.sep.join((rel, fname)) + + fname = os.path.realpath(fname) + + if fname in stack: + raise(RecursiveInclude("%s recursively included" % fname)) + if len(stack) > 10: + raise(RecursiveInclude("%s included, stack size = %i" % + (fname, len(stack)))) + + if patt == None: + patt = re.compile("^#(opt_include|include)[ \t].*$",re.MULTILINE) + + try: + fp = open(fname) + contents = fp.read() + fp.close() + except: + raise + + rel = os.path.dirname(fname) + stack.append(fname) + + cur = 0 + clen = len(contents) + while True: + match = patt.search(contents[cur:]) + if not match: break + loc = match.start() + cur + endl = match.end() + cur + + (key, cur_fname) = contents[loc:endl].split(None,2) + cur_fname = cur_fname.strip() + + try: + inc_contents = read_file_with_includes(cur_fname, rel, stack, patt) + except IOError, e: + if e.errno == errno.ENOENT and key == "#opt_include": + inc_contents = "" + else: + raise + contents = contents[0:loc] + inc_contents + contents[endl+1:] + cur = loc + len(inc_contents) + stack.pop() + return(contents) + +def get_cmdline(): + if 'DEBUG_PROC_CMDLINE' in os.environ: + cmdline = os.environ["DEBUG_PROC_CMDLINE"] + else: + try: + cmdfp = open("/proc/cmdline") + cmdline = cmdfp.read().strip() + cmdfp.close() + except: + cmdline = "" + return(cmdline) + +def read_cc_from_cmdline(cmdline=None): + # this should support reading cloud-config information from + # the kernel command line. It is intended to support content of the + # format: + # cc: <yaml content here> [end_cc] + # this would include: + # cc: ssh_import_id: [smoser, kirkland]\\n + # cc: ssh_import_id: [smoser, bob]\\nruncmd: [ [ ls, -l ], echo hi ] end_cc + # cc:ssh_import_id: [smoser] end_cc cc:runcmd: [ [ ls, -l ] ] end_cc + if cmdline is None: + cmdline = get_cmdline() + + tag_begin="cc:" + tag_end="end_cc" + begin_l = len(tag_begin) + end_l = len(tag_end) + clen = len(cmdline) + tokens = [ ] + begin = cmdline.find(tag_begin) + while begin >= 0: + end = cmdline.find(tag_end, begin + begin_l) + if end < 0: + end = clen + tokens.append(cmdline[begin+begin_l:end].lstrip().replace("\\n","\n")) + + begin = cmdline.find(tag_begin, end + end_l) + + return('\n'.join(tokens)) + +def ensure_dirs(dirlist, mode=0755): + fixmodes = [] + for d in dirlist: + try: + if mode != None: + os.makedirs(d) + else: + os.makedirs(d, mode) + except OSError as e: + if e.errno != errno.EEXIST: raise + if mode != None: fixmodes.append(d) + + for d in fixmodes: + os.chmod(d, mode) + +def chownbyname(fname,user=None,group=None): + uid = -1 + gid = -1 + if user == None and group == None: return + if user: + import pwd + uid = pwd.getpwnam(user).pw_uid + if group: + import grp + gid = grp.getgrnam(group).gr_gid + + os.chown(fname,uid,gid) + +def readurl(url, data=None): + if data is None: + req = urllib2.Request(url) + else: + encoded = urllib.urlencode(data) + req = urllib2.Request(url, encoded) + + response = urllib2.urlopen(req) + return(response.read()) diff --git a/config/cloud.cfg b/config/cloud.cfg new file mode 100644 index 00000000..2aa574c7 --- /dev/null +++ b/config/cloud.cfg @@ -0,0 +1,42 @@ +cloud: auto +user: ubuntu +disable_root: 1 +preserve_hostname: False + +cloud_init_modules: + - resizefs + - set_hostname + - update_hostname + - rsyslog + +cloud_config_modules: + - mounts + - ssh-import-id + - locale + - ssh + - grub-dpkg + - apt-update-upgrade + - puppet + - disable-ec2-metadata + - runcmd + - byobu + +cloud_final_modules: + - rightscale_userdata + - scripts-per-once + - scripts-per-boot + - scripts-per-instance + - scripts-user + - keys-to-console + - phone-home + - final-message + +## logging.cfg contains info on logging output for cloud-init +#include logging.cfg + +## dpkg-cloud-sources.cfg contains the values +## selected by dpkg configuration +#opt_include distro.cfg + +##local.cfg is for local overrides of any of the above +#opt_include local.cfg diff --git a/config/logging.cfg b/config/logging.cfg new file mode 100644 index 00000000..2e7ac2ed --- /dev/null +++ b/config/logging.cfg @@ -0,0 +1,57 @@ +## this yaml formated config file handles setting +## logger information. The values that are necessary to be set +## are seen at the bottom. The top '_log' are only used to remove +## redundency in a syslog and fallback-to-file case. +## +## The 'log_cfgs' entry defines a list of logger configs +## Each entry in the list is tried, and the first one that +## works is used. If a log_cfg list entry is an array, it will +## be joined with '\n'. +_log: + - &log_base | + [loggers] + keys=root,cloudinit + + [handlers] + keys=consoleHandler,cloudLogHandler + + [formatters] + keys=simpleFormatter,arg0Formatter + + [logger_root] + level=DEBUG + handlers=consoleHandler,cloudLogHandler + + [logger_cloudinit] + level=DEBUG + qualname=cloudinit + handlers= + propagate=1 + + [handler_consoleHandler] + class=StreamHandler + level=WARNING + formatter=arg0Formatter + args=(sys.stderr,) + + [formatter_arg0Formatter] + format=%(asctime)s - %(filename)s[%(levelname)s]: %(message)s + + [formatter_simpleFormatter] + format=[CLOUDINIT] %(filename)s[%(levelname)s]: %(message)s + - &log_file | + [handler_cloudLogHandler] + class=FileHandler + level=DEBUG + formatter=arg0Formatter + args=('/var/log/cloud-init.log',) + - &log_syslog | + [handler_cloudLogHandler] + class=handlers.SysLogHandler + level=DEBUG + formatter=simpleFormatter + args=("/dev/log", handlers.SysLogHandler.LOG_USER) + +log_cfgs: + - [ *log_base, *log_syslog ] + - [ *log_base, *log_file ] diff --git a/doc/examples/cloud-config-archive.txt b/doc/examples/cloud-config-archive.txt new file mode 100644 index 00000000..23b1024c --- /dev/null +++ b/doc/examples/cloud-config-archive.txt @@ -0,0 +1,16 @@ +#cloud-config-archive +- type: foo/wark + filename: bar + content: | + This is my payload + hello +- this is also payload +- | + multi line payload + here +- + type: text/upstart-job + filename: my-upstart.conf + content: | + whats this, yo? + diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 3a46a481..1ba51243 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -25,6 +25,9 @@ apt_mirror: http://us.archive.ubuntu.com/ubuntu/ apt_preserve_sources_list: true apt_sources: + - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" + keyid: F430BBA5 # GPG key ID published on a key server + filename: byobu-ppa.list # PPA shortcut: # * Setup correct apt sources.list line @@ -45,6 +48,12 @@ apt_sources: # See sources.list man page for more information about the format - source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted + # sources can use $MIRROR and $RELEASE and they will be replaced + # with the local mirror for this cloud, and the running release + # the entry below would be possibly turned into: + # - source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse + - source: deb $MIRROR $RELEASE multiverse + # this would have the same end effect as 'ppa:byobu/ppa' - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" keyid: F430BBA5 # GPG key ID published on a key server @@ -216,7 +225,7 @@ cloud_config_modules: # ssh_import_id: [ user1, user2 ] # ssh_import_id will feed the list in that variable to -# ssh-import-lp-id, so that public keys stored in launchpad +# ssh-import-id, so that public keys stored in launchpad # can easily be imported into the configured user # This can be a single string ('smoser') or a list ([smoser, kirkland]) ssh_import_id: [smoser] @@ -246,3 +255,72 @@ byobu_by_default: system # rather than as the 'ubuntu' user, then you must set this to false # default: true disable_root: false + +# set the locale to a given locale +# default: en_US.UTF-8 +locale: en_US.UTF-8 + +# add entries to rsyslog configuration +# The first occurance of a given filename will truncate. +# subsequent entries will append. +# if value is a scalar, its content is assumed to be 'content', and the +# default filename is used. +# if filename is not provided, it will default to 'rsylog_filename' +# if filename does not start with a '/', it will be put in 'rsyslog_dir' +# rsyslog_dir default: /etc/rsyslog.d +# rsyslog_filename default: 20-cloud-config.conf +rsyslog: + - ':syslogtag, isequal, "[CLOUDINIT]" /var/log/cloud-foo.log' + - content: "*.* @@192.0.2.1:10514" + - filename: 01-examplecom.conf + content: | + *.* @@syslogd.example.com + +# resize_rootfs should the / filesytem be resized on first boot +# this allows you to launch an instance with a larger disk / partition +# and have the instance automatically grow / to accomoddate it +# set to 'False' to disable +resize_rootfs: True + +# if hostname is set, cloud-init will set the system hostname +# appropriately to its value +# if not set, it will set hostname from the cloud metadata +# default: None + +# final_message +# default: cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds +# this message is written by cloud-final when the system is finished +# its first boot +final_message: "The system is finally up, after $UPTIME seconds" + +# configure where output will go +# 'output' entry is a dict with 'init', 'config', 'final' or 'all' +# entries. Each one defines where +# cloud-init, cloud-config, cloud-config-final or all output will go +# each entry in the dict can be a string, list or dict. +# if it is a string, it refers to stdout +# if it is a list, entry 0 is stdout, entry 1 is stderr +# if it is a dict, it is expected to have 'output' and 'error' fields +# default is to write to console only +# the special entry "&1" for an error means "same location as stdout" +# (Note, that '&1' has meaning in yaml, so it must be quoted) +output: + init: "> /var/log/my-cloud-init.log" + config: [ ">> /tmp/foo.out", "> /tmp/foo.err" ] + final: + output: "| tee /tmp/final.stdout | tee /tmp/bar.stdout" + error: "&1" + + +# phone_home: if this dictionary is present, then the phone_home +# cloud-config module will post specified data back to the given +# url +# default: none +# phone_home: +# url: http://my.foo.bar/$INSTANCE/ +# post: all +# tries: 10 +# +phone_home: + url: http://my.example.com/$INSTANCE_ID/ + post: [ pub_key_dsa, pub_key_rsa, instance_id ] diff --git a/doc/var-lib-cloud.txt b/doc/var-lib-cloud.txt new file mode 100644 index 00000000..0f96f267 --- /dev/null +++ b/doc/var-lib-cloud.txt @@ -0,0 +1,59 @@ +/var/lib/cloud has the following structure: + - scripts/ + per-instance/ + per-boot/ + per-once/ + + files in these directories will be run by 'run-parts' once per + instance, once per boot, and once per *ever*. + + - seed/ + <datasource>/ + sys-user-data + user-data + meta-data + + The 'seed/' directory allows you to seed a specific datasource + For example, to seed the 'nocloud' datasource you would need to + populate + seed/nocloud/user-data + seed/nocloud/meta-data + + - instance -> instances/i-abcde + This is a symlink to the current instance/<instance-id> directory + created/updated on boot + - instances/ + i-abcdefgh/ + scripts/ # all scripts in scripts are per-instance + sem/ + config-puppet + config-ssh + set-hostname + cloud-config.txt + user-data.txt + user-data.txt.i + obj.pkl + handlers/ + data/ # just a per-instance data location to be used + boot-finished + # this file indicates when "boot" is finished + # it is created by the 'final_message' cloud-config + + - sem/ + scripts.once + These are the cloud-specific semaphores. The only thing that + would go here are files to mark that a "per-once" script + has run. + + - handlers/ + "persistent" handlers (not per-instance). Same as handlers + from user-data, just will be cross-instance id + + - data/ + this is a persistent data location. cloud-init won't really + use it, but something else (a handler or script could) + +to clear out the current instance's data as if to force a "new run" on reboot +do: + ( cd /var/lib/cloud/instance && sudo rm -Rf * ) + @@ -35,14 +35,15 @@ setup(name='cloud-init', packages=['cloudinit', 'cloudinit.CloudConfig' ], scripts=['cloud-init.py', 'cloud-init-run-module.py', - 'cloud-init-cfg.py' + 'cloud-init-cfg.py', + 'cloud-init-query.py' ], - data_files=[('/etc/cloud', ['cloud.cfg']), + data_files=[('/etc/cloud', glob('config/*.cfg')), ('/etc/cloud/templates', glob('templates/*')), ('/etc/init', glob('upstart/*.conf')), ('/usr/share/cloud-init', []), - ('/usr/lib/cloud-init', - ['tools/uncloud-init','tools/write-mime-multipart']), + ('/usr/lib/cloud-init', + ['tools/uncloud-init', 'tools/write-ssh-key-fingerprints']), ('/usr/share/doc/cloud-init', filter(is_f,glob('doc/*'))), ('/usr/share/doc/cloud-init/examples', filter(is_f,glob('doc/examples/*'))), ('/usr/share/doc/cloud-init/examples/seed', filter(is_f,glob('doc/examples/seed/*'))), diff --git a/tools/write-mime-multipart b/tools/write-mime-multipart deleted file mode 100755 index 46032728..00000000 --- a/tools/write-mime-multipart +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/python -# largely taken from python examples -# http://docs.python.org/library/email-examples.html - -import os -import sys -import smtplib -# For guessing MIME type based on file name extension -import mimetypes - -from email import encoders -from email.message import Message -from email.mime.base import MIMEBase -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText -from optparse import OptionParser -import gzip - -COMMASPACE = ', ' - -starts_with_mappings={ - '#include' : 'text/x-include-url', - '#!' : 'text/x-shellscript', - '#cloud-config' : 'text/cloud-config', - '#upstart-job' : 'text/upstart-job', - '#part-handler' : 'text/part-handler', - '#cloud-boothook' : 'text/cloud-boothook' -} - -def get_type(fname,deftype): - f = file(fname,"rb") - line = f.readline() - f.close() - rtype = deftype - for str,mtype in starts_with_mappings.items(): - if line.startswith(str): - rtype = mtype - break - return(rtype) - -def main(): - outer = MIMEMultipart() - #outer['Subject'] = 'Contents of directory %s' % os.path.abspath(directory) - #outer['To'] = COMMASPACE.join(opts.recipients) - #outer['From'] = opts.sender - #outer.preamble = 'You will not see this in a MIME-aware mail reader.\n' - - parser = OptionParser() - - parser.add_option("-o", "--output", dest="output", - help="write output to FILE [default %default]", metavar="FILE", - default="-") - parser.add_option("-z", "--gzip", dest="compress", action="store_true", - help="compress output", default=False) - parser.add_option("-d", "--default", dest="deftype", - help="default mime type [default %default]", default="text/plain") - parser.add_option("--delim", dest="delim", - help="delimiter [default %default]", default=":") - - (options, args) = parser.parse_args() - - if (len(args)) < 1: - parser.error("Must give file list see '--help'") - - for arg in args: - t = arg.split(options.delim, 1) - path=t[0] - if len(t) > 1: - mtype = t[1] - else: - mtype = get_type(path,options.deftype) - - maintype, subtype = mtype.split('/', 1) - if maintype == 'text': - fp = open(path) - # Note: we should handle calculating the charset - msg = MIMEText(fp.read(), _subtype=subtype) - fp.close() - else: - fp = open(path, 'rb') - msg = MIMEBase(maintype, subtype) - msg.set_payload(fp.read()) - fp.close() - # Encode the payload using Base64 - encoders.encode_base64(msg) - - # Set the filename parameter - msg.add_header('Content-Disposition', 'attachment', - filename=os.path.basename(path)) - - outer.attach(msg) - - if options.output is "-": - ofile = sys.stdout - else: - ofile = file(options.output,"wb") - - if options.compress: - gfile = gzip.GzipFile(fileobj=ofile, filename = options.output ) - gfile.write(outer.as_string()) - gfile.close() - else: - ofile.write(outer.as_string()) - - ofile.close() - -if __name__ == '__main__': - main() diff --git a/tools/write-ssh-key-fingerprints b/tools/write-ssh-key-fingerprints new file mode 100755 index 00000000..9a081faa --- /dev/null +++ b/tools/write-ssh-key-fingerprints @@ -0,0 +1,10 @@ +#!/bin/sh +{ +echo +echo "#############################################################" +echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----" +ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key.pub +ssh-keygen -l -f /etc/ssh/ssh_host_dsa_key.pub +echo "-----END SSH HOST KEY FINGERPRINTS-----" +echo "#############################################################" +} | logger -p user.info -s -t "ec2" diff --git a/upstart/cloud-config.conf b/upstart/cloud-config.conf index 6649a99d..5edc58b9 100644 --- a/upstart/cloud-config.conf +++ b/upstart/cloud-config.conf @@ -5,4 +5,4 @@ start on (filesystem and started rsyslog) console output task -exec cloud-init-cfg all +exec cloud-init-cfg all config diff --git a/upstart/cloud-final.conf b/upstart/cloud-final.conf new file mode 100644 index 00000000..a04105a1 --- /dev/null +++ b/upstart/cloud-final.conf @@ -0,0 +1,10 @@ +# cloud-final.conf - run "final" jobs +# this runs around traditional "rc.local" time. +# and after all cloud-config jobs are run +description "execute cloud user/final scripts" + +start on (stopped rc RUNLEVEL=[2345] and stopped cloud-config) +console output +task + +exec cloud-init-cfg all final diff --git a/upstart/cloud-run-user-script.conf b/upstart/cloud-run-user-script.conf deleted file mode 100644 index e50006d4..00000000 --- a/upstart/cloud-run-user-script.conf +++ /dev/null @@ -1,14 +0,0 @@ -# cloud-run-user-script - runs user scripts found in user-data, that are -# stored in /var/lib/cloud/scripts by the initial cloudinit upstart job -description "execute cloud user scripts" - -start on (stopped rc RUNLEVEL=[2345] and stopped cloud-config) -console output -task - -script -sdir=/var/lib/cloud/data/scripts -[ -d "$sdir" ] || exit 0 -exec cloud-init-run-module once-per-instance user-scripts execute \ - run-parts --regex '.*' "$sdir" -end script |