diff options
Diffstat (limited to 'cloudinit')
24 files changed, 1283 insertions, 247 deletions
diff --git a/cloudinit/CloudConfig/__init__.py b/cloudinit/CloudConfig/__init__.py index 0a91059a..22ad63a6 100644 --- a/cloudinit/CloudConfig/__init__.py +++ b/cloudinit/CloudConfig/__init__.py @@ -21,16 +21,22 @@ import cloudinit import cloudinit.util as util import sys import traceback +import os +import subprocess per_instance="once-per-instance" per_always="always" +per_once="once" class CloudConfig(): cfgfile = None cfg = None - def __init__(self,cfgfile): - self.cloud = cloudinit.CloudInit() + def __init__(self,cfgfile, cloud=None): + if cloud == None: + self.cloud = cloudinit.CloudInit() + else: + self.cloud = cloud self.cfg = self.get_config_obj(cfgfile) self.cloud.get_data_source() @@ -38,6 +44,7 @@ class CloudConfig(): try: cfg = util.read_conf(cfgfile) except: + # TODO: this 'log' could/should be passed in cloudinit.log.critical("Failed loading of cloud config '%s'. Continuing with empty config\n" % cfgfile) cloudinit.log.debug(traceback.format_exc() + "\n") cfg = None @@ -58,3 +65,150 @@ class CloudConfig(): except: raise +# reads a cloudconfig module list, returns +# a 2 dimensional array suitable to pass to run_cc_modules +def read_cc_modules(cfg,name): + if name not in cfg: return([]) + module_list = [] + # create 'module_list', an array of arrays + # where array[0] = config + # array[1] = freq + # array[2:] = arguemnts + for item in cfg[name]: + if isinstance(item,str): + module_list.append((item,)) + elif isinstance(item,list): + module_list.append(item) + else: + raise TypeError("failed to read '%s' item in config") + return(module_list) + +def run_cc_modules(cc,module_list,log): + failures = [] + for cfg_mod in module_list: + name = cfg_mod[0] + freq = None + run_args = [ ] + if len(cfg_mod) > 1: + freq = cfg_mod[1] + if len(cfg_mod) > 2: + run_args = cfg_mod[2:] + + try: + log.debug("handling %s with freq=%s and args=%s" % + (name, freq, run_args )) + cc.handle(name, run_args, freq=freq) + except: + log.warn(traceback.format_exc()) + log.error("config handling of %s, %s, %s failed\n" % + (name,freq,run_args)) + failures.append(name) + + return(failures) + +# always returns well formated values +# cfg is expected to have an entry 'output' in it, which is a dictionary +# that includes entries for 'init', 'config', 'final' or 'all' +# init: /var/log/cloud.out +# config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ] +# final: +# output: "| logger -p" +# error: "> /dev/null" +# this returns the specific 'mode' entry, cleanly formatted, with value +# None if if none is given +def get_output_cfg(cfg, mode="init"): + ret = [ None, None ] + if not 'output' in cfg: return ret + + outcfg = cfg['output'] + if mode in outcfg: + modecfg = outcfg[mode] + else: + if 'all' not in outcfg: return ret + # if there is a 'all' item in the output list + # then it applies to all users of this (init, config, final) + modecfg = outcfg['all'] + + # if value is a string, it specifies stdout + if isinstance(modecfg,str): + ret = [ modecfg, None ] + + # if its a list, then we expect (stdout, stderr) + if isinstance(modecfg,list): + if len(modecfg) > 0: ret[0] = modecfg[0] + if len(modecfg) > 1: + ret[1] = modecfg[1] + + # if it is a dictionary, expect 'out' and 'error' + # items, which indicate out and error + if isinstance(modecfg, dict): + if 'output' in modecfg: + ret[0] = modecfg['output'] + if 'error' in modecfg: + ret[1] = modecfg['error'] + + # if err's entry == "&1", then make it same as stdout + # as in shell syntax of "echo foo >/dev/null 2>&1" + if ret[1] == "&1": ret[1] = ret[0] + + swlist = [ ">>", ">", "|" ] + for i in range(len(ret)): + if not ret[i]: continue + val = ret[i].lstrip() + found = False + for s in swlist: + if val.startswith(s): + val = "%s %s" % (s,val[len(s):].strip()) + found = True + break + if not found: + # default behavior is append + val = "%s %s" % ( ">>", val.strip()) + ret[i] = val + + return(ret) + + +# redirect_output(outfmt, errfmt, orig_out, orig_err) +# replace orig_out and orig_err with filehandles specified in outfmt or errfmt +# fmt can be: +# > FILEPATH +# >> FILEPATH +# | program [ arg1 [ arg2 [ ... ] ] ] +# +# with a '|', arguments are passed to shell, so one level of +# shell escape is required. +def redirect_output(outfmt,errfmt, o_out=sys.stdout, o_err=sys.stderr): + if outfmt: + (mode, arg) = outfmt.split(" ",1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) + new_fp = proc.stdin + else: + raise TypeError("invalid type for outfmt: %s" % outfmt) + + if o_out: + os.dup2(new_fp.fileno(), o_out.fileno()) + if errfmt == outfmt: + os.dup2(new_fp.fileno(), o_err.fileno()) + return + + if errfmt: + (mode, arg) = errfmt.split(" ",1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) + new_fp = proc.stdin + else: + raise TypeError("invalid type for outfmt: %s" % outfmt) + + if o_err: + os.dup2(new_fp.fileno(), o_err.fileno()) + return diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index 396c5e09..e918e8c8 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -25,20 +25,23 @@ def handle(name,cfg,cloud,log,args): update = util.get_cfg_option_bool(cfg, 'apt_update', False) upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) + release = get_release() + if cfg.has_key("apt_mirror"): + mirror = cfg["apt_mirror"] + else: + mirror = cloud.get_mirror() + if not util.get_cfg_option_bool(cfg, \ 'apt_preserve_sources_list', False): - if cfg.has_key("apt_mirror"): - mirror = cfg["apt_mirror"] - else: - mirror = cloud.get_mirror() - generate_sources_list(mirror) + generate_sources_list(release, mirror) old_mir = util.get_cfg_option_str(cfg,'apt_old_mirror', \ "archive.ubuntu.com/ubuntu") rename_apt_lists(old_mir, mirror) # process 'apt_sources' if cfg.has_key('apt_sources'): - errors = add_sources(cfg['apt_sources']) + errors = add_sources(cfg['apt_sources'], + { 'MIRROR' : mirror, 'RELEASE' : release } ) for e in errors: log.warn("Source Error: %s\n" % ':'.join(e)) @@ -96,17 +99,18 @@ def rename_apt_lists(omirror,new_mirror,lists_d="/var/lib/apt/lists"): for file in glob.glob("%s_*" % oprefix): os.rename(file,"%s%s" % (nprefix, file[olen:])) -def generate_sources_list(mirror): +def get_release(): stdout, stderr = subprocess.Popen(['lsb_release', '-cs'], stdout=subprocess.PIPE).communicate() - codename = stdout.strip() + return(stdout.strip()) +def generate_sources_list(codename, mirror): util.render_to_file('sources.list', '/etc/apt/sources.list', \ { 'mirror' : mirror, 'codename' : codename }) # srclist is a list of dictionaries, # each entry must have: 'source' # may have: key, ( keyid and keyserver) -def add_sources(srclist): +def add_sources(srclist, searchList={ }): elst = [] for ent in srclist: @@ -121,6 +125,8 @@ def add_sources(srclist): elst.append([source, "add-apt-repository failed"]) continue + source = util.render_string(source, searchList) + if not ent.has_key('filename'): ent['filename']='cloud_config_sources.list' diff --git a/cloudinit/CloudConfig/cc_final_message.py b/cloudinit/CloudConfig/cc_final_message.py new file mode 100644 index 00000000..4d72e409 --- /dev/null +++ b/cloudinit/CloudConfig/cc_final_message.py @@ -0,0 +1,55 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +from cloudinit.CloudConfig import per_always +import sys +from cloudinit import util, boot_finished +import time + +frequency = per_always + +final_message = "cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds" + +def handle(name,cfg,cloud,log,args): + if len(args) != 0: + msg_in = args[0] + else: + msg_in = util.get_cfg_option_str(cfg,"final_message",final_message) + + try: + uptimef=open("/proc/uptime") + uptime=uptimef.read().split(" ")[0] + uptimef.close() + except IOError as e: + log.warn("unable to open /proc/uptime\n") + uptime = "na" + + + try: + ts = time.strftime("%a, %d %b %Y %H:%M:%S %z",time.gmtime()) + except: + ts = "na" + + try: + subs = { 'UPTIME' : uptime, 'TIMESTAMP' : ts } + sys.stdout.write(util.render_string(msg_in, subs)) + except Exception as e: + log.warn("failed to render string to stdout: %s" % e) + + fp = open(boot_finished, "wb") + fp.write(uptime + "\n") + fp.close() diff --git a/cloudinit/CloudConfig/cc_keys_to_console.py b/cloudinit/CloudConfig/cc_keys_to_console.py new file mode 100644 index 00000000..47227b76 --- /dev/null +++ b/cloudinit/CloudConfig/cc_keys_to_console.py @@ -0,0 +1,31 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +from cloudinit.CloudConfig import per_instance +import subprocess + +frequency = per_instance + +def handle(name,cfg,cloud,log,args): + write_ssh_prog='/usr/lib/cloud-init/write-ssh-key-fingerprints' + try: + confp = open('/dev/console',"wb") + subprocess.call(write_ssh_prog,stdout=confp) + confp.close() + except: + log.warn("writing keys to console value") + raise diff --git a/cloudinit/CloudConfig/cc_locale.py b/cloudinit/CloudConfig/cc_locale.py new file mode 100644 index 00000000..c164b5ba --- /dev/null +++ b/cloudinit/CloudConfig/cc_locale.py @@ -0,0 +1,43 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit.util as util +import subprocess +import traceback + +def apply_locale(locale): + subprocess.Popen(['locale-gen', locale]).communicate() + subprocess.Popen(['update-locale', locale]).communicate() + + util.render_to_file('default-locale', '/etc/default/locale', \ + { 'locale' : locale }) + +def handle(name,cfg,cloud,log,args): + if len(args) != 0: + locale = args[0] + else: + locale = util.get_cfg_option_str(cfg,"locale",cloud.get_locale()) + + if not locale: return + + log.debug("setting locale to %s" % locale) + + try: + apply_locale(locale) + except Exception, e: + log.debug(traceback.format_exc(e)) + raise Exception("failed to apply locale %s" % locale) diff --git a/cloudinit/CloudConfig/cc_phone_home.py b/cloudinit/CloudConfig/cc_phone_home.py new file mode 100644 index 00000000..ee463757 --- /dev/null +++ b/cloudinit/CloudConfig/cc_phone_home.py @@ -0,0 +1,99 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +from cloudinit.CloudConfig import per_instance +import cloudinit.util as util +from time import sleep + +frequency = per_instance +post_list_all = [ 'pub_key_dsa', 'pub_key_rsa', 'instance_id', 'hostname' ] + +# phone_home: +# url: http://my.foo.bar/$INSTANCE/ +# post: all +# tries: 10 +# +# phone_home: +# url: http://my.foo.bar/$INSTANCE_ID/ +# post: [ pub_key_dsa, pub_key_rsa, instance_id +# +def handle(name,cfg,cloud,log,args): + if len(args) != 0: + ph_cfg = util.readconf(args[0]) + else: + if not 'phone_home' in cfg: return + ph_cfg = cfg['phone_home'] + + if 'url' not in ph_cfg: + log.warn("no 'url' token in phone_home") + return + + url = ph_cfg['url'] + post_list = ph_cfg.get('post', 'all') + tries = ph_cfg.get('tries',10) + try: + tries = int(tries) + except: + log.warn("tries is not an integer. using 10") + tries = 10 + + if post_list == "all": + post_list = post_list_all + + all_keys = { } + all_keys['instance_id'] = cloud.get_instance_id() + all_keys['hostname'] = cloud.get_hostname() + + pubkeys = { + 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', + 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', + } + + for n, path in pubkeys.iteritems(): + try: + fp = open(path, "rb") + all_keys[n] = fp.read() + all_keys[n] + fp.close() + except: + log.warn("%s: failed to open in phone_home" % path) + + submit_keys = { } + for k in post_list: + if k in all_keys: + submit_keys[k] = all_keys[k] + else: + submit_keys[k] = "N/A" + log.warn("requested key %s from 'post' list not available") + + url = util.render_string(url, { 'INSTANCE_ID' : all_keys['instance_id'] }) + + last_e = None + for i in range(0,tries): + try: + util.readurl(url, submit_keys) + log.debug("succeeded submit to %s on try %i" % (url, i+1)) + return + except Exception, e: + log.debug("failed to post to %s on try %i" % (url, i+1)) + last_e = e + sleep(3) + + log.warn("failed to post to %s in %i tries" % (url, tries)) + if last_e: raise(last_e) + + return diff --git a/cloudinit/CloudConfig/cc_resizefs.py b/cloudinit/CloudConfig/cc_resizefs.py new file mode 100644 index 00000000..11a10005 --- /dev/null +++ b/cloudinit/CloudConfig/cc_resizefs.py @@ -0,0 +1,54 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit.util as util +import subprocess +import traceback + +def handle(name,cfg,cloud,log,args): + if len(args) != 0: + resize_root = False + if str(value).lower() in [ 'true', '1', 'on', 'yes']: + resize_root = True + else: + resize_root = util.get_cfg_option_bool(cfg,"resize_rootfs",False) + + if not resize_root: return + + log.debug("resizing root filesystem on first boot") + + cmd = ['blkid', '-sTYPE', '-ovalue', '/dev/root'] + try: + (fstype,err) = util.subp(cmd) + except Exception, e: + log.warn("Failed to get filesystem type via %s" % cmd) + raise + + if fstype.startswith("ext"): + resize_cmd = [ 'resize2fs', '/dev/root' ] + elif fstype == "xfs": + resize_cmd = [ 'xfs_growfs', '/dev/root' ] + else: + log.debug("not resizing unknown filesystem %s" % fstype) + return + + try: + (out,err) = util.subp(resize_cmd) + except Exception, e: + log.warn("Failed to resize filesystem (%s,%s)" % cmd) + raise + diff --git a/cloudinit/CloudConfig/cc_rightscale_userdata.py b/cloudinit/CloudConfig/cc_rightscale_userdata.py new file mode 100644 index 00000000..a90e6d18 --- /dev/null +++ b/cloudinit/CloudConfig/cc_rightscale_userdata.py @@ -0,0 +1,73 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +## +## The purpose of this script is to allow cloud-init to consume +## rightscale style userdata. rightscale user data is key-value pairs +## in a url-query-string like format. +## +## for cloud-init support, there will be a key named +## 'CLOUD_INIT_REMOTE_HOOK'. +## +## This cloud-config module will +## - read the blob of data from raw user data, and parse it as key/value +## - for each key that is found, download the content to +## the local instance/scripts directory and set them executable. +## - the files in that directory will be run by the user-scripts module +## Therefore, this must run before that. +## +## +import cloudinit.util as util +from cloudinit.CloudConfig import per_once, per_always, per_instance +from cloudinit import get_ipath_cur +from urlparse import parse_qs + +frequency = per_instance +my_name = "cc_rightscale_userdata" +my_hookname = 'CLOUD_INIT_REMOTE_HOOK' + +def handle(name,cfg,cloud,log,args): + try: + ud = cloud.get_userdata_raw() + except: + log.warn("failed to get raw userdata in %s" % my_name) + return + + try: + mdict = parse_qs(cloud.get_userdata_raw()) + if not my_hookname in mdict: return + except: + log.warn("failed to urlparse.parse_qa(userdata_raw())") + raise + + scripts_d = get_ipath_cur('scripts') + i = 0 + errors = [ ] + first_e = None + for url in mdict[my_hookname]: + fname = "%s/rightscale-%02i" % (scripts_d,i) + i = i +1 + try: + content = util.readurl(url) + util.write_file(fname, content, mode=0700) + except Exception, e: + if not first_e: first_e = None + log.warn("%s failed to read %s: %s" % (my_name, url, e)) + + if first_e: + raise(e) diff --git a/cloudinit/CloudConfig/cc_rsyslog.py b/cloudinit/CloudConfig/cc_rsyslog.py new file mode 100644 index 00000000..3320dbb2 --- /dev/null +++ b/cloudinit/CloudConfig/cc_rsyslog.py @@ -0,0 +1,99 @@ +# vi: ts=4 expandtab syntax=python +# +# Copyright (C) 2009-2010 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit +import logging +import cloudinit.util as util +import subprocess +import traceback + +DEF_FILENAME = "20-cloud-config.conf" +DEF_DIR = "/etc/rsyslog.d" + +def handle(name,cfg,cloud,log,args): + # rsyslog: + # - "*.* @@192.158.1.1" + # - content: "*.* @@192.0.2.1:10514" + # - filename: 01-examplecom.conf + # content: | + # *.* @@syslogd.example.com + + # process 'rsyslog' + if not 'rsyslog' in cfg: return + + def_dir = cfg.get('rsyslog_dir', DEF_DIR) + def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) + + entries = cfg['rsyslog'] + + files = [ ] + elst = [ ] + for ent in cfg['rsyslog']: + if isinstance(ent,dict): + if not "content" in ent: + elst.append((ent, "no 'content' entry")) + continue + content = ent['content'] + filename = ent.get("filename", def_fname) + else: + content = ent + filename = def_fname + + if not filename.startswith("/"): + filename = "%s/%s" % (def_dir,filename) + + omode = "ab" + # truncate filename first time you see it + if filename not in files: + omode = "wb" + files.append(filename) + + try: + util.write_file(filename, content + "\n", omode=omode) + except Exception, e: + log.debug(traceback.format_exc(e)) + elst.append((content, "failed to write to %s" % filename)) + + # need to restart syslogd + restarted = False + try: + # if this config module is running at cloud-init time + # (before rsyslog is running) we don't actually have to + # restart syslog. + # + # upstart actually does what we want here, in that it doesn't + # start a service that wasn't running already on 'restart' + # it will also return failure on the attempt, so 'restarted' + # won't get set + log.debug("restarting rsyslog") + p = util.subp(['service', 'rsyslog', 'restart']) + restarted = True + + except Exception, e: + elst.append(("restart", str(e))) + + if restarted: + # this only needs to run if we *actually* restarted + # syslog above. + cloudinit.logging_set_from_cfg_file() + log = logging.getLogger() + log.debug("rsyslog configured %s" % files) + + for e in elst: + log.warn("rsyslog error: %s\n" % ':'.join(e)) + + return diff --git a/cloudinit/CloudConfig/cc_runcmd.py b/cloudinit/CloudConfig/cc_runcmd.py index 97d21900..afa7a441 100644 --- a/cloudinit/CloudConfig/cc_runcmd.py +++ b/cloudinit/CloudConfig/cc_runcmd.py @@ -21,7 +21,7 @@ import cloudinit.util as util def handle(name,cfg,cloud,log,args): if not cfg.has_key("runcmd"): return - outfile="%s/runcmd" % cloudinit.user_scripts_dir + outfile="%s/runcmd" % cloud.get_ipath('scripts') content="#!/bin/sh\n" escaped="%s%s%s%s" % ( "'", '\\', "'", "'" ) diff --git a/cloudinit/CloudConfig/cc_scripts_per_boot.py b/cloudinit/CloudConfig/cc_scripts_per_boot.py new file mode 100644 index 00000000..4e407fb7 --- /dev/null +++ b/cloudinit/CloudConfig/cc_scripts_per_boot.py @@ -0,0 +1,30 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit.util as util +from cloudinit.CloudConfig import per_once, per_always, per_instance +from cloudinit import get_cpath, get_ipath_cur + +frequency = per_always +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-boot") + +def handle(name,cfg,cloud,log,args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/CloudConfig/cc_scripts_per_instance.py b/cloudinit/CloudConfig/cc_scripts_per_instance.py new file mode 100644 index 00000000..22b41185 --- /dev/null +++ b/cloudinit/CloudConfig/cc_scripts_per_instance.py @@ -0,0 +1,30 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit.util as util +from cloudinit.CloudConfig import per_once, per_always, per_instance +from cloudinit import get_cpath, get_ipath_cur + +frequency = per_instance +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-instance") + +def handle(name,cfg,cloud,log,args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/CloudConfig/cc_scripts_per_once.py b/cloudinit/CloudConfig/cc_scripts_per_once.py new file mode 100644 index 00000000..9d752325 --- /dev/null +++ b/cloudinit/CloudConfig/cc_scripts_per_once.py @@ -0,0 +1,30 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit.util as util +from cloudinit.CloudConfig import per_once, per_always, per_instance +from cloudinit import get_cpath, get_ipath_cur + +frequency = per_once +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-once") + +def handle(name,cfg,cloud,log,args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/CloudConfig/cc_scripts_user.py b/cloudinit/CloudConfig/cc_scripts_user.py new file mode 100644 index 00000000..bafecd23 --- /dev/null +++ b/cloudinit/CloudConfig/cc_scripts_user.py @@ -0,0 +1,30 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit.util as util +from cloudinit.CloudConfig import per_once, per_always, per_instance +from cloudinit import get_cpath, get_ipath_cur + +frequency = per_instance +runparts_path = "%s/%s" % (get_ipath_cur(), "scripts") + +def handle(name,cfg,cloud,log,args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/CloudConfig/cc_set_hostname.py b/cloudinit/CloudConfig/cc_set_hostname.py new file mode 100644 index 00000000..34621e97 --- /dev/null +++ b/cloudinit/CloudConfig/cc_set_hostname.py @@ -0,0 +1,38 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit.util as util +import subprocess + +def handle(name,cfg,cloud,log,args): + if util.get_cfg_option_bool(cfg,"preserve_hostname",False): + log.debug("preserve_hostname is set. not setting hostname") + return(True) + + try: + hostname = util.get_cfg_option_str(cfg,"hostname",cloud.get_hostname()) + set_hostname(hostname, log) + except Exception, e: + util.logexc(log) + log.warn("failed to set hostname\n") + + return(True) + +def set_hostname(hostname, log): + subprocess.Popen(['hostname', hostname]).communicate() + util.write_file("/etc/hostname","%s\n" % hostname, 0644) + log.debug("populated /etc/hostname with %s on first boot", hostname) diff --git a/cloudinit/CloudConfig/cc_ssh.py b/cloudinit/CloudConfig/cc_ssh.py index 07527906..7b9ba5ab 100644 --- a/cloudinit/CloudConfig/cc_ssh.py +++ b/cloudinit/CloudConfig/cc_ssh.py @@ -60,18 +60,7 @@ def handle(name,cfg,cloud,log,args): send_ssh_keys_to_console() def send_ssh_keys_to_console(): - send_keys_sh = """ - { - echo - echo "#############################################################" - echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----" - ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key.pub - ssh-keygen -l -f /etc/ssh/ssh_host_dsa_key.pub - echo "-----END SSH HOST KEY FINGERPRINTS-----" - echo "#############################################################" - } | logger -p user.info -s -t "ec2" - """ - subprocess.call(('sh', '-c', send_keys_sh)) + subprocess.call(('/usr/lib/cloud-init/write-ssh-key-fingerprints',)) def apply_credentials(keys, user, disable_root): keys = set(keys) @@ -79,7 +68,7 @@ def apply_credentials(keys, user, disable_root): setup_user_keys(keys, user, '') if disable_root: - key_prefix = 'command="echo \'Please login as the %s user rather than root user.\';echo;sleep 10" ' % user + key_prefix = 'command="echo \'Please login as the user \\\"%s\\\" rather than the user \\\"root\\\".\';echo;sleep 10" ' % user else: key_prefix = '' diff --git a/cloudinit/CloudConfig/cc_ssh_import_id.py b/cloudinit/CloudConfig/cc_ssh_import_id.py index dd4d3184..bf1314be 100644 --- a/cloudinit/CloudConfig/cc_ssh_import_id.py +++ b/cloudinit/CloudConfig/cc_ssh_import_id.py @@ -31,7 +31,7 @@ def handle(name,cfg,cloud,log,args): if len(ids) == 0: return - cmd = [ "sudo", "-Hu", user, "ssh-import-lp-id" ] + ids + cmd = [ "sudo", "-Hu", user, "ssh-import-id" ] + ids log.debug("importing ssh ids. cmd = %s" % cmd) diff --git a/cloudinit/CloudConfig/cc_update_hostname.py b/cloudinit/CloudConfig/cc_update_hostname.py new file mode 100644 index 00000000..3663c0ab --- /dev/null +++ b/cloudinit/CloudConfig/cc_update_hostname.py @@ -0,0 +1,95 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import cloudinit.util as util +import subprocess +import errno +from cloudinit.CloudConfig import per_always + +frequency = per_always + +def handle(name,cfg,cloud,log,args): + if util.get_cfg_option_bool(cfg,"preserve_hostname",False): + log.debug("preserve_hostname is set. not updating hostname") + return + + try: + hostname = util.get_cfg_option_str(cfg,"hostname",cloud.get_hostname()) + prev ="%s/%s" % (cloud.get_cpath('datadir'),"previous-hostname") + update_hostname(hostname, prev, log) + except Exception, e: + log.warn("failed to set hostname\n") + raise + +# read hostname from a 'hostname' file +# allow for comments and stripping line endings. +# if file doesn't exist, or no contents, return default +def read_hostname(filename, default=None): + try: + fp = open(filename,"r") + lines = fp.readlines() + fp.close() + for line in lines: + hpos = line.find("#") + if hpos != -1: + line = line[0:hpos] + line = line.rstrip() + if line: + return line + except IOError, e: + if e.errno != errno.ENOENT: raise + return default + +def update_hostname(hostname, prev_file, log): + etc_file = "/etc/hostname" + + hostname_prev = None + hostname_in_etc = None + + try: + hostname_prev = read_hostname(prev_file) + except Exception, e: + log.warn("Failed to open %s: %s" % (prev_file, e)) + + try: + hostname_in_etc = read_hostname(etc_file) + except: + log.warn("Failed to open %s" % etc_file) + + update_files = [] + if not hostname_prev or hostname_prev != hostname: + update_files.append(prev_file) + + if (not hostname_in_etc or + (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): + update_files.append(etc_file) + + try: + for fname in update_files: + util.write_file(fname,"%s\n" % hostname, 0644) + log.debug("wrote %s to %s" % (hostname,fname)) + except: + log.warn("failed to write hostname to %s" % fname) + + if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: + log.debug("%s differs from %s. assuming user maintained" % + (prev_file,etc_file)) + + if etc_file in update_files: + log.debug("setting hostname to %s" % hostname) + subprocess.Popen(['hostname', hostname]).communicate() + diff --git a/cloudinit/CloudConfig/cc_updates_check.py b/cloudinit/CloudConfig/cc_updates_check.py deleted file mode 100644 index aaa2e6b0..00000000 --- a/cloudinit/CloudConfig/cc_updates_check.py +++ /dev/null @@ -1,49 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# -# Author: Scott Moser <scott.moser@canonical.com> -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -import cloudinit.util as util -import cloudinit -import os -import time - -cronpre = "/etc/cron.d/cloudinit" - -def handle(name,cfg,cloud,log,args): - if not util.get_cfg_option_bool(cfg, 'updates-check', True): - return - build_info = "/etc/cloud/build.info" - if not os.path.isfile(build_info): - log.warn("no %s file" % build_info) - - avail="%s/%s" % ( cloudinit.datadir, "available.build" ) - cmd=( "uec-query-builds", "--system-suite", "--config", "%s" % build_info, - "--output", "%s" % avail, "is-update-available" ) - try: - util.subp(cmd) - except: - log.warn("failed to execute uec-query-build for updates check") - - # add a cron entry for this hour and this minute every day - try: - cron=open("%s-%s" % (cronpre, "updates") ,"w") - cron.write("%s root %s\n" % \ - (time.strftime("%M %H * * *"),' '.join(cmd))) - cron.close() - except: - log.warn("failed to enable cron update system check") - raise - diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py index 7a6c9dc9..1c0edc59 100644 --- a/cloudinit/DataSourceEc2.py +++ b/cloudinit/DataSourceEc2.py @@ -30,13 +30,7 @@ import errno class DataSourceEc2(DataSource.DataSource): api_ver = '2009-04-04' - cachedir = cloudinit.cachedir + '/ec2' - - location_locale_map = { - 'us' : 'en_US.UTF-8', - 'eu' : 'en_GB.UTF-8', - 'default' : 'en_US.UTF-8', - } + seeddir = cloudinit.seeddir + '/ec2' def __init__(self): pass @@ -46,10 +40,10 @@ class DataSourceEc2(DataSource.DataSource): def get_data(self): seedret={ } - if util.read_optional_seed(seedret,base=self.cachedir + "/"): + if util.read_optional_seed(seedret,base=self.seeddir+ "/"): self.userdata_raw = seedret['user-data'] self.metadata = seedret['meta-data'] - cloudinit.log.debug("using seeded ec2 data in %s" % self.cachedir) + cloudinit.log.debug("using seeded ec2 data in %s" % self.seeddir) return True try: @@ -71,13 +65,6 @@ class DataSourceEc2(DataSource.DataSource): def get_local_mirror(self): return(self.get_mirror_from_availability_zone()) - def get_locale(self): - az = self.metadata['placement']['availability-zone'] - if self.location_locale_map.has_key(az[0:2]): - return(self.location_locale_map[az[0:2]]) - else: - return(self.location_locale_map["default"]) - def get_mirror_from_availability_zone(self, availability_zone = None): # availability is like 'us-west-1b' or 'eu-west-1a' if availability_zone == None: @@ -121,7 +108,7 @@ class DataSourceEc2(DataSource.DataSource): cloudinit.log.warning("waiting for metadata service at %s\n" % url) cloudinit.log.warning(" %s [%02s/%s]: %s\n" % - (time.strftime("%H:%M:%S"), x+1, sleeps, reason)) + (time.strftime("%H:%M:%S",time.gmtime()), x+1, sleeps, reason)) time.sleep(sleeptime) cloudinit.log.critical("giving up on md after %i seconds\n" % diff --git a/cloudinit/DataSourceNoCloud.py b/cloudinit/DataSourceNoCloud.py index cd988d08..2f6033a9 100644 --- a/cloudinit/DataSourceNoCloud.py +++ b/cloudinit/DataSourceNoCloud.py @@ -32,7 +32,7 @@ class DataSourceNoCloud(DataSource.DataSource): supported_seed_starts = ( "/" , "file://" ) seed = None cmdline_id = "ds=nocloud" - seeddir = cloudinit.cachedir + '/nocloud' + seeddir = cloudinit.seeddir + '/nocloud' def __init__(self): pass @@ -108,16 +108,10 @@ class DataSourceNoCloud(DataSource.DataSource): # root=LABEL=uec-rootfs ro ds=nocloud def parse_cmdline_data(ds_id,fill,cmdline=None): if cmdline is None: - if 'DEBUG_PROC_CMDLINE' in os.environ: - cmdline = os.environ["DEBUG_PROC_CMDLINE"] - else: - cmdfp = open("/proc/cmdline") - cmdline = cmdfp.read().strip() - cmdfp.close() - cmdline = " %s " % cmdline.lower() - - if not ( " %s " % ds_id in cmdline or " %s;" % ds_id in cmdline ): - return False + cmdline = util.get_cmdline() + + if not ( " %s " % ds_id in cmdline or " %s;" % ds_id in cmdline ): + return False argline="" # cmdline can contain: @@ -149,4 +143,4 @@ def parse_cmdline_data(ds_id,fill,cmdline=None): class DataSourceNoCloudNet(DataSourceNoCloud): cmdline_id = "ds=nocloud-net" supported_seed_starts = ( "http://", "https://", "ftp://" ) - seeddir = cloudinit.cachedir + '/nocloud-net' + seeddir = cloudinit.seeddir + '/nocloud-net' diff --git a/cloudinit/UserDataHandler.py b/cloudinit/UserDataHandler.py index ab7d0bc8..fbb000fc 100644 --- a/cloudinit/UserDataHandler.py +++ b/cloudinit/UserDataHandler.py @@ -19,7 +19,9 @@ import email from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText - +from email.mime.base import MIMEBase +from email import encoders +import yaml starts_with_mappings={ '#include' : 'text/x-include-url', @@ -27,7 +29,8 @@ starts_with_mappings={ '#cloud-config' : 'text/cloud-config', '#upstart-job' : 'text/upstart-job', '#part-handler' : 'text/part-handler', - '#cloud-boothook' : 'text/cloud-boothook' + '#cloud-boothook' : 'text/cloud-boothook', + '#cloud-config-archive' : 'text/cloud-config-archive', } # if 'str' is compressed return decompressed otherwise return it @@ -43,12 +46,47 @@ def decomp_str(str): def do_include(str,parts): import urllib # is just a list of urls, one per line + # also support '#include <url here>' for line in str.splitlines(): if line == "#include": continue + if line.startswith("#include"): + line = line[len("#include"):].lstrip() if line.startswith("#"): continue content = urllib.urlopen(line).read() process_includes(email.message_from_string(decomp_str(content)),parts) +def explode_cc_archive(archive,parts): + for ent in yaml.load(archive): + # ent can be one of: + # dict { 'filename' : 'value' , 'content' : 'value', 'type' : 'value' } + # filename and type not be present + # or + # scalar(payload) + filename = 'part-%03d' % len(parts['content']) + def_type = "text/cloud-config" + if isinstance(ent,str): + content = ent + mtype = type_from_startswith(content,def_type) + else: + content = ent.get('content', '') + filename = ent.get('filename', filename) + mtype = ent.get('type', None) + if mtype == None: + mtype = type_from_startswith(payload,def_type) + + print "adding %s,%s" % (filename, mtype) + parts['content'].append(content) + parts['names'].append(filename) + parts['types'].append(mtype) + +def type_from_startswith(payload, default=None): + # slist is sorted longest first + slist = sorted(starts_with_mappings.keys(), key=lambda e: 0-len(e)) + for sstr in slist: + if payload.startswith(sstr): + return(starts_with_mappings[sstr]) + return default + def process_includes(msg,parts): # parts is a dictionary of arrays # parts['content'] @@ -67,10 +105,7 @@ def process_includes(msg,parts): ctype = None ctype_orig = part.get_content_type() if ctype_orig == "text/plain": - for str, gtype in starts_with_mappings.items(): - if payload.startswith(str): - ctype = gtype - break + ctype = type_from_startswith(payload) if ctype is None: ctype = ctype_orig @@ -79,6 +114,10 @@ def process_includes(msg,parts): do_include(payload,parts) continue + if ctype == "text/cloud-config-archive": + explode_cc_archive(payload,parts) + continue + filename = part.get_filename() if not filename: filename = 'part-%03d' % len(parts['content']) diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py index 6a59a23f..9c02ff8a 100644 --- a/cloudinit/__init__.py +++ b/cloudinit/__init__.py @@ -18,94 +18,35 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. # -datadir = '/var/lib/cloud/data' -semdir = '/var/lib/cloud/sem' -pluginsdir = datadir + '/plugins' -cachedir = datadir + '/cache' -userdata_raw = datadir + '/user-data.txt' -userdata = datadir + '/user-data.txt.i' -user_scripts_dir = datadir + "/scripts" -boothooks_dir = datadir + "/boothooks" -cloud_config = datadir + '/cloud-config.txt' -#cloud_config = '/tmp/cloud-config.txt' -data_source_cache = cachedir + '/obj.pkl' +varlibdir = '/var/lib/cloud' +cur_instance_link = varlibdir + "/instance" +boot_finished = cur_instance_link + "/boot-finished" system_config = '/etc/cloud/cloud.cfg' +seeddir = varlibdir + "/seed" cfg_env_name = "CLOUD_CFG" cfg_builtin = """ +log_cfgs: [ ] cloud_type: auto -user: ubuntu -disable_root: 1 - -cloud_config_modules: - - mounts - - ssh-import-id - - ssh - - grub-dpkg - - apt-update-upgrade - - puppet - - updates-check - - disable-ec2-metadata - - runcmd - - byobu - -log_cfg: built_in +def_log_file: /var/log/cloud-init.log +syslog_fix_perms: syslog:adm """ - -def_log_file = '/var/log/cloud-init.log' logger_name = "cloudinit" -built_in_log_base = """ -[loggers] -keys=root,cloudinit - -[handlers] -keys=consoleHandler,cloudLogHandler - -[formatters] -keys=simpleFormatter,arg0Formatter - -[logger_root] -level=DEBUG -handlers=consoleHandler,cloudLogHandler - -[logger_cloudinit] -level=DEBUG -qualname=cloudinit -handlers= -propagate=1 - -[handler_consoleHandler] -class=StreamHandler -level=WARNING -formatter=arg0Formatter -args=(sys.stderr,) - -[formatter_arg0Formatter] -format=%(asctime)s - %(filename)s[%(levelname)s]: %(message)s - -[formatter_simpleFormatter] -format=[CLOUDINIT] %(asctime)s - %(filename)s[%(levelname)s]: %(message)s -datefmt= - -""" - -built_in_log_clougLogHandlerLog=""" -[handler_cloudLogHandler] -class=FileHandler -level=DEBUG -formatter=simpleFormatter -args=('__CLOUDINIT_LOGGER_FILE__',) -""" - -built_in_log_cloudLogHandlerSyslog= """ -[handler_cloudLogHandler] -class=handlers.SysLogHandler -level=DEBUG -formatter=simpleFormatter -args=("/dev/log", handlers.SysLogHandler.LOG_USER) -""" - +pathmap = { + "handlers" : "/handlers", + "scripts" : "/scripts", + "sem" : "/sem", + "boothooks" : "/boothooks", + "userdata_raw" : "/user-data.txt", + "userdata" : "/user-data-raw.txt.i", + "obj_pkl" : "/obj.pkl", + "cloud_config" : "/cloud-config.txt", + "datadir" : "/data", + None : "", +} + +parsed_cfgs = { } import os from configobj import ConfigObj @@ -121,43 +62,44 @@ import util import logging import logging.config import StringIO +import glob class NullHandler(logging.Handler): - def emit(self,record): pass + def emit(self,record): pass log = logging.getLogger(logger_name) log.addHandler(NullHandler()) def logging_set_from_cfg_file(cfg_file=system_config): - logging_set_from_cfg(util.get_base_cfg(cfg_file,cfg_builtin)) + logging_set_from_cfg(util.get_base_cfg(cfg_file,cfg_builtin,parsed_cfgs)) + +def logging_set_from_cfg(cfg): + log_cfgs = [] + logcfg=util.get_cfg_option_str(cfg, "log_cfg", False) + if logcfg: + # if there is a 'logcfg' entry in the config, respect + # it, it is the old keyname + log_cfgs = [ logcfg ] + elif "log_cfgs" in cfg: + for cfg in cfg['log_cfgs']: + if isinstance(cfg,list): + log_cfgs.append('\n'.join(cfg)) + else: + log_cfgs.append() + + if not len(log_cfgs): + sys.stderr.write("Warning, no logging configured\n") + return -def logging_set_from_cfg(cfg, logfile=None): - if logfile is None: + for logcfg in log_cfgs: try: - open(def_log_file,"a").close() - logfile = def_log_file - except IOError as e: - if e.errno == errno.EACCES: - logfile = "/dev/null" - else: raise - - logcfg=util.get_cfg_option_str(cfg, "log_cfg", "built_in") - failsafe = "%s\n%s" % (built_in_log_base, built_in_log_clougLogHandlerLog) - builtin = False - if logcfg.lower() == "built_in": - logcfg = "%s\n%s" % (built_in_log_base, built_in_log_cloudLogHandlerSyslog) - builtin = True - - logcfg=logcfg.replace("__CLOUDINIT_LOGGER_FILE__",logfile) - try: - logging.config.fileConfig(StringIO.StringIO(logcfg)) - return - except: - if not builtin: - sys.stderr.write("Warning, setting config.fileConfig failed\n") + logging.config.fileConfig(StringIO.StringIO(logcfg)) + return + except: + pass + + raise Exception("no valid logging found\n") - failsafe=failsafe.replace("__CLOUDINIT_LOGGER_FILE__",logfile) - logging.config.fileConfig(StringIO.StringIO(failsafe)) import DataSourceEc2 import DataSourceNoCloud @@ -174,7 +116,6 @@ class CloudInit: "all": ( "nocloud-net", "ec2" ), "local" : ( "nocloud", ), } - cfg = None part_handlers = { } old_conffile = '/etc/ec2-init/ec2-config.cfg' @@ -196,7 +137,7 @@ class CloudInit: if self.cfg: return(self.cfg) - conf = util.get_base_cfg(system_config,cfg_builtin) + conf = util.get_base_cfg(self.sysconfig,cfg_builtin, parsed_cfgs) # support reading the old ConfigObj format file and merging # it into the yaml dictionary @@ -212,7 +153,11 @@ class CloudInit: def restore_from_cache(self): try: - f=open(data_source_cache, "rb") + # we try to restore from a current link and static path + # by using the instance link, if purge_cache was called + # the file wont exist + cache = get_ipath_cur('obj_pkl') + f=open(cache, "rb") data = cPickle.load(f) self.datasource = data return True @@ -220,16 +165,17 @@ class CloudInit: return False def write_to_cache(self): + cache = self.get_ipath("obj_pkl") try: - os.makedirs(os.path.dirname(data_source_cache)) + os.makedirs(os.path.dirname(cache)) except OSError as e: if e.errno != errno.EEXIST: return False try: - f=open(data_source_cache, "wb") + f=open(cache, "wb") data = cPickle.dump(self.datasource,f) - os.chmod(data_source_cache,0400) + os.chmod(cache,0400) return True except: return False @@ -252,6 +198,7 @@ class CloudInit: for ds in cfglist.split(','): dslist.append(strip(ds).tolower()) + log.debug("searching for data source in [%s]" % str(dslist)) for ds in dslist: if ds not in self.datasource_map: log.warn("data source %s not found in map" % ds) @@ -270,27 +217,48 @@ class CloudInit: log.debug("did not find data source from %s" % dslist) raise DataSourceNotFoundException("Could not find data source") + def set_cur_instance(self): + try: + os.unlink(cur_instance_link) + except OSError, e: + if e.errno != errno.ENOENT: raise + + os.symlink("./instances/%s" % self.get_instance_id(), cur_instance_link) + idir = self.get_ipath() + dlist = [] + for d in [ "handlers", "scripts", "sem" ]: + dlist.append("%s/%s" % (idir, d)) + + util.ensure_dirs(dlist) + def get_userdata(self): return(self.datasource.get_userdata()) + def get_userdata_raw(self): + return(self.datasource.get_userdata_raw()) + + def get_instance_id(self): + return(self.datasource.get_instance_id()) + def update_cache(self): self.write_to_cache() self.store_userdata() def store_userdata(self): - util.write_file(userdata_raw, self.datasource.get_userdata_raw(), 0600) - util.write_file(userdata, self.datasource.get_userdata(), 0600) + util.write_file(self.get_ipath('userdata_raw'), + self.datasource.get_userdata_raw(), 0600) + util.write_file(self.get_ipath('userdata'), + self.datasource.get_userdata(), 0600) def initctl_emit(self): + cc_path = get_ipath_cur('cloud_config') subprocess.Popen(['initctl', 'emit', 'cloud-config', - '%s=%s' % (cfg_env_name,cloud_config)]).communicate() + '%s=%s' % (cfg_env_name,cc_path)]).communicate() def sem_getpath(self,name,freq): - freqtok = freq if freq == 'once-per-instance': - freqtok = self.datasource.get_instance_id() - - return("%s/%s.%s" % (semdir,name,freqtok)) + return("%s/%s" % (self.get_ipath("sem"),name)) + return("%s/%s.%s" % (get_cpath("sem"), name, freq)) def sem_has_run(self,name,freq): if freq == "always": return False @@ -349,9 +317,40 @@ class CloudInit: self.sem_clear(semname,freq) raise + # get_ipath : get the instance path for a name in pathmap + # (/var/lib/cloud/instances/<instance>/name)<name>) + def get_ipath(self, name=None): + return("%s/instances/%s%s" + % (varlibdir,self.get_instance_id(), pathmap[name])) + def consume_userdata(self): self.get_userdata() data = self + + cdir = get_cpath("handlers") + idir = self.get_ipath("handlers") + + # add the path to the plugins dir to the top of our list for import + # instance dir should be read before cloud-dir + sys.path.insert(0,cdir) + sys.path.insert(0,idir) + + # add handlers in cdir + for fname in glob.glob("%s/*.py" % cdir): + if not os.path.isfile(fname): continue + modname = os.path.basename(fname)[0:-3] + try: + mod = __import__(modname) + lister = getattr(mod, "list_types") + handler = getattr(mod, "handle_part") + mtypes = lister() + for mtype in mtypes: + self.part_handlers[mtype]=handler + log.debug("added handler for [%s] from %s" % (mtypes,fname)) + except: + log.warn("failed to initialize handler in %s" % fname) + util.logexc(log) + # give callbacks opportunity to initialize for ctype, func in self.part_handlers.items(): func(data, "__begin__",None,None) @@ -368,16 +367,13 @@ class CloudInit: self.handlercount = 0 return - # add the path to the plugins dir to the top of our list for import - if self.handlercount == 0: - sys.path.insert(0,pluginsdir) - self.handlercount=self.handlercount+1 - # write content to pluginsdir + # write content to instance's handlerdir + handlerdir = self.get_ipath("handler") modname = 'part-handler-%03d' % self.handlercount modfname = modname + ".py" - util.write_file("%s/%s" % (pluginsdir,modfname), payload, 0600) + util.write_file("%s/%s" % (handlerdir,modfname), payload, 0600) try: mod = __import__(modname) @@ -402,7 +398,9 @@ class CloudInit: return filename=filename.replace(os.sep,'_') - util.write_file("%s/%s" % (user_scripts_dir,filename), payload, 0700) + scriptsdir = get_ipath_cur('scripts') + util.write_file("%s/%s/%s" % + (scriptsdir,self.get_instance_id(),filename), payload, 0700) def handle_upstart_job(self,data,ctype,filename,payload): if ctype == "__end__" or ctype == "__begin__": return @@ -416,6 +414,7 @@ class CloudInit: self.cloud_config_str="" return if ctype == "__end__": + cloud_config = self.get_ipath("cloud_config") util.write_file(cloud_config, self.cloud_config_str, 0600) ## this could merge the cloud config with the system config @@ -452,6 +451,7 @@ class CloudInit: elif start != 0: payload=payload[start:] + boothooks_dir = self.get_ipath("boothooks") filepath = "%s/%s" % (boothooks_dir,filename) util.write_file(filepath, payload, 0700) try: @@ -480,15 +480,58 @@ class CloudInit: def device_name_to_device(self,name): return(self.datasource.device_name_to_device(name)) + # I really don't know if this should be here or not, but + # I needed it in cc_update_hostname, where that code had a valid 'cloud' + # reference, but did not have a cloudinit handle + # (ie, no cloudinit.get_cpath()) + def get_cpath(self,name=None): + return(get_cpath(name)) + + +def initfs(): + subds = [ 'scripts/per-instance', 'scripts/per-once', 'scripts/per-boot', + 'seed', 'instances', 'handlers', 'sem', 'data' ] + dlist = [ ] + for subd in subds: + dlist.append("%s/%s" % (varlibdir, subd)) + util.ensure_dirs(dlist) + + cfg = util.get_base_cfg(system_config,cfg_builtin,parsed_cfgs) + log_file = None + if 'def_log_file' in cfg: + log_file = cfg['def_log_file'] + fp = open(log_file,"ab") + fp.close() + if log_file and 'syslog' in cfg: + perms = cfg['syslog'] + (u,g) = perms.split(':',1) + if u == "-1" or u == "None": u = None + if g == "-1" or g == "None": g = None + util.chownbyname(log_file, u, g) def purge_cache(): - try: - os.unlink(data_source_cache) - except OSError as e: - if e.errno != errno.ENOENT: return(False) - except: - return(False) + rmlist = ( boot_finished , cur_instance_link ) + for f in rmlist: + try: + os.unlink(f) + except OSError as e: + if e.errno == errno.ENOENT: continue + return(False) + except: + return(False) return(True) +# get_ipath_cur: get the current instance path for an item +def get_ipath_cur(name=None): + return("%s/instance/%s" % (varlibdir, pathmap[name])) + +# get_cpath : get the "clouddir" (/var/lib/cloud/<name>) +# for a name in dirmap +def get_cpath(name=None): + return("%s%s" % (varlibdir, pathmap[name])) + +def get_base_cfg(): + return(util.get_base_cfg(system_config,cfg_builtin,parsed_cfgs)) + class DataSourceNotFoundException(Exception): pass diff --git a/cloudinit/util.py b/cloudinit/util.py index c1b4fd2d..e958fc02 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -17,13 +17,16 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import yaml import os +import os.path import errno import subprocess from Cheetah.Template import Template import cloudinit import urllib2 +import urllib import logging import traceback +import re WARN = logging.WARN DEBUG = logging.DEBUG @@ -40,13 +43,32 @@ def read_conf(fname): return { } raise -def get_base_cfg(cfgfile,cfg_builtin=""): - syscfg = read_conf(cfgfile) +def get_base_cfg(cfgfile,cfg_builtin="", parsed_cfgs=None): + kerncfg = { } + syscfg = { } + if parsed_cfgs and cfgfile in parsed_cfgs: + return(parsed_cfgs[cfgfile]) + + contents = read_file_with_includes(cfgfile) + if contents: + syscfg = yaml.load(contents) + + kern_contents = read_cc_from_cmdline() + if kern_contents: + kerncfg = yaml.load(kern_contents) + + # kernel parameters override system config + combined = mergedict(kerncfg, syscfg) + if cfg_builtin: builtin = yaml.load(cfg_builtin) + fin = mergedict(combined,builtin) else: - return(syscfg) - return(mergedict(syscfg,builtin)) + fin = combined + + if parsed_cfgs != None: + parsed_cfgs[cfgfile] = fin + return(fin) def get_cfg_option_bool(yobj, key, default=False): if not yobj.has_key(key): return default @@ -106,6 +128,16 @@ def getkeybyid(keyid,keyserver): args=['sh', '-c', shcmd, "export-gpg-keyid", keyid, keyserver] return(subp(args)[0]) +def runparts(dirp, skip_no_exist=True): + if skip_no_exist and not os.path.isdir(dirp): return + + cmd = [ 'run-parts', '--regex', '.*', dirp ] + sp = subprocess.Popen(cmd) + sp.communicate() + if sp.returncode is not 0: + raise subprocess.CalledProcessError(sp.returncode,cmd) + return + def subp(args, input=None): s_in = None if input is not None: @@ -122,6 +154,10 @@ def render_to_file(template, outfile, searchList): f.write(t.respond()) f.close() +def render_string(template, searchList): + return(Template(template, searchList=[searchList]).respond()) + + # read_optional_seed # returns boolean indicating success or failure (presense of files) # if files are present, populates 'fill' dictionary with 'user-data' and @@ -168,3 +204,133 @@ def read_seeded(base="", ext="", timeout=2): def logexc(log,lvl=logging.DEBUG): log.log(lvl,traceback.format_exc()) + +class RecursiveInclude(Exception): + pass + +def read_file_with_includes(fname, rel = ".", stack=[], patt = None): + if not fname.startswith("/"): + fname = os.sep.join((rel, fname)) + + fname = os.path.realpath(fname) + + if fname in stack: + raise(RecursiveInclude("%s recursively included" % fname)) + if len(stack) > 10: + raise(RecursiveInclude("%s included, stack size = %i" % + (fname, len(stack)))) + + if patt == None: + patt = re.compile("^#(opt_include|include)[ \t].*$",re.MULTILINE) + + try: + fp = open(fname) + contents = fp.read() + fp.close() + except: + raise + + rel = os.path.dirname(fname) + stack.append(fname) + + cur = 0 + clen = len(contents) + while True: + match = patt.search(contents[cur:]) + if not match: break + loc = match.start() + cur + endl = match.end() + cur + + (key, cur_fname) = contents[loc:endl].split(None,2) + cur_fname = cur_fname.strip() + + try: + inc_contents = read_file_with_includes(cur_fname, rel, stack, patt) + except IOError, e: + if e.errno == errno.ENOENT and key == "#opt_include": + inc_contents = "" + else: + raise + contents = contents[0:loc] + inc_contents + contents[endl+1:] + cur = loc + len(inc_contents) + stack.pop() + return(contents) + +def get_cmdline(): + if 'DEBUG_PROC_CMDLINE' in os.environ: + cmdline = os.environ["DEBUG_PROC_CMDLINE"] + else: + try: + cmdfp = open("/proc/cmdline") + cmdline = cmdfp.read().strip() + cmdfp.close() + except: + cmdline = "" + return(cmdline) + +def read_cc_from_cmdline(cmdline=None): + # this should support reading cloud-config information from + # the kernel command line. It is intended to support content of the + # format: + # cc: <yaml content here> [end_cc] + # this would include: + # cc: ssh_import_id: [smoser, kirkland]\\n + # cc: ssh_import_id: [smoser, bob]\\nruncmd: [ [ ls, -l ], echo hi ] end_cc + # cc:ssh_import_id: [smoser] end_cc cc:runcmd: [ [ ls, -l ] ] end_cc + if cmdline is None: + cmdline = get_cmdline() + + tag_begin="cc:" + tag_end="end_cc" + begin_l = len(tag_begin) + end_l = len(tag_end) + clen = len(cmdline) + tokens = [ ] + begin = cmdline.find(tag_begin) + while begin >= 0: + end = cmdline.find(tag_end, begin + begin_l) + if end < 0: + end = clen + tokens.append(cmdline[begin+begin_l:end].lstrip().replace("\\n","\n")) + + begin = cmdline.find(tag_begin, end + end_l) + + return('\n'.join(tokens)) + +def ensure_dirs(dirlist, mode=0755): + fixmodes = [] + for d in dirlist: + try: + if mode != None: + os.makedirs(d) + else: + os.makedirs(d, mode) + except OSError as e: + if e.errno != errno.EEXIST: raise + if mode != None: fixmodes.append(d) + + for d in fixmodes: + os.chmod(d, mode) + +def chownbyname(fname,user=None,group=None): + uid = -1 + gid = -1 + if user == None and group == None: return + if user: + import pwd + uid = pwd.getpwnam(user).pw_uid + if group: + import grp + gid = grp.getgrnam(group).gr_gid + + os.chown(fname,uid,gid) + +def readurl(url, data=None): + if data is None: + req = urllib2.Request(url) + else: + encoded = urllib.urlencode(data) + req = urllib2.Request(url, encoded) + + response = urllib2.urlopen(req) + return(response.read()) |