diff options
52 files changed, 2184 insertions, 1097 deletions
@@ -17,6 +17,10 @@ - support configuration of landscape-client via cloud-config (LP: #857366) - part-handlers now get base64 decoded content rather than 2xbase64 encoded in the payload parameter. (LP: #874342) + - add test case framework [Mike Milner] (LP: #890851) + - fix pylint warnings [Juerg Haefliger] (LP: #914739) + - add support for adding and deleting CA Certificates [Mike Milner] (LP: #915232) + - in ci-info lines, use '.' to indicate empty field for easier machine reading 0.6.2: - fix bug where update was not done unless update was explicitly set. It would not be run if 'upgrade' or packages were set to be installed diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..0fc6c46b --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ + +all: test + +pylint: + pylint cloudinit + +pyflakes: + pyflakes . + +test: + nosetests tests/unittests/ + +.PHONY: test pylint pyflakes + diff --git a/cloud-init-cfg.py b/cloud-init-cfg.py index 2930cd88..3a475c1c 100755 --- a/cloud-init-cfg.py +++ b/cloud-init-cfg.py @@ -2,8 +2,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -23,11 +25,12 @@ import cloudinit.util as util import cloudinit.CloudConfig as CC import logging import os -import traceback -def Usage(out = sys.stdout): + +def Usage(out=sys.stdout): out.write("Usage: %s name\n" % sys.argv[0]) - + + def main(): # expect to be called with # name [ freq [ args ] @@ -39,7 +42,7 @@ def main(): util.close_stdin() modename = "config" - + if len(sys.argv) < 2: Usage(sys.stderr) sys.exit(1) @@ -50,30 +53,30 @@ def main(): else: freq = None run_args = [] - name=sys.argv[1] + name = sys.argv[1] if len(sys.argv) > 2: freq = sys.argv[2] if freq == "None": freq = None if len(sys.argv) > 3: - run_args=sys.argv[3:] + run_args = sys.argv[3:] cfg_path = cloudinit.get_ipath_cur("cloud_config") cfg_env_name = cloudinit.cfg_env_name - if os.environ.has_key(cfg_env_name): + if cfg_env_name in os.environ: cfg_path = os.environ[cfg_env_name] - cloud = cloudinit.CloudInit(ds_deps=[]) # ds_deps=[], get only cached + cloud = cloudinit.CloudInit(ds_deps=[]) # ds_deps=[], get only cached try: cloud.get_data_source() except cloudinit.DataSourceNotFoundException as e: # there was no datasource found, theres nothing to do sys.exit(0) - cc = CC.CloudConfig(cfg_path,cloud) + cc = CC.CloudConfig(cfg_path, cloud) try: - (outfmt, errfmt) = CC.get_output_cfg(cc.cfg,modename) + (outfmt, errfmt) = CC.get_output_cfg(cc.cfg, modename) CC.redirect_output(outfmt, errfmt) except Exception as e: err("Failed to get and set output config: %s\n" % e) @@ -82,28 +85,30 @@ def main(): log = logging.getLogger() log.info("cloud-init-cfg %s" % sys.argv[1:]) - module_list = [ ] + module_list = [] if name == "all": modlist_cfg_name = "cloud_%s_modules" % modename - module_list = CC.read_cc_modules(cc.cfg,modlist_cfg_name) + module_list = CC.read_cc_modules(cc.cfg, modlist_cfg_name) if not len(module_list): - err("no modules to run in cloud_config [%s]" % modename,log) + err("no modules to run in cloud_config [%s]" % modename, log) sys.exit(0) else: - module_list.append( [ name, freq ] + run_args ) + module_list.append([name, freq] + run_args) - failures = CC.run_cc_modules(cc,module_list,log) + failures = CC.run_cc_modules(cc, module_list, log) if len(failures): - err("errors running cloud_config [%s]: %s" % (modename,failures), log) + err("errors running cloud_config [%s]: %s" % (modename, failures), log) sys.exit(len(failures)) -def err(msg,log=None): + +def err(msg, log=None): if log: log.error(msg) sys.stderr.write(msg + "\n") -def fail(msg,log=None): - err(msg,log) + +def fail(msg, log=None): + err(msg, log) sys.exit(1) if __name__ == '__main__': diff --git a/cloud-init-query.py b/cloud-init-query.py index 3e8c24ab..856cf462 100755 --- a/cloud-init-query.py +++ b/cloud-init-query.py @@ -2,8 +2,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -20,28 +22,27 @@ import sys import cloudinit import cloudinit.CloudConfig -import logging -import os -import traceback -def Usage(out = sys.stdout): + +def Usage(out=sys.stdout): out.write("Usage: %s name\n" % sys.argv[0]) - + + def main(): # expect to be called with name of item to fetch if len(sys.argv) != 2: Usage(sys.stderr) sys.exit(1) - cc = cloudinit.CloudConfig.CloudConfig(cloudinit.cloud_config) - cloud_config = cc.cfg + cfg_path = cloudinit.get_ipath_cur("cloud_config") + cc = cloudinit.CloudConfig.CloudConfig(cfg_path) data = { - 'user_data' : cc.cloud.get_userdata(), - 'user_data_raw' : cc.cloud.get_userdata_raw(), - 'instance_id' : cc.cloud.get_instance_id(), + 'user_data': cc.cloud.get_userdata(), + 'user_data_raw': cc.cloud.get_userdata_raw(), + 'instance_id': cc.cloud.get_instance_id(), } - name = sys.argv[1].replace('-','_') + name = sys.argv[1].replace('-', '_') if name not in data: sys.stderr.write("unknown name '%s'. Known values are:\n %s\n" % diff --git a/cloud-init.py b/cloud-init.py index bc673265..9d559020 100755 --- a/cloud-init.py +++ b/cloud-init.py @@ -2,8 +2,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -30,15 +32,17 @@ import logging import errno import os + def warn(wstr): sys.stderr.write("WARN:%s" % wstr) + def main(): util.close_stdin() - cmds = ( "start", "start-local" ) - deps = { "start" : ( ds.DEP_FILESYSTEM, ds.DEP_NETWORK ), - "start-local" : ( ds.DEP_FILESYSTEM, ) } + cmds = ("start", "start-local") + deps = {"start": (ds.DEP_FILESYSTEM, ds.DEP_NETWORK), + "start-local": (ds.DEP_FILESYSTEM, )} cmd = "" if len(sys.argv) > 1: @@ -54,14 +58,14 @@ def main(): sys.stderr.write("bad command %s. use one of %s\n" % (cmd, cmds)) sys.exit(1) - now = time.strftime("%a, %d %b %Y %H:%M:%S %z",time.gmtime()) + now = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) try: - uptimef=open("/proc/uptime") - uptime=uptimef.read().split(" ")[0] - uptimef.close() + uptimef = open("/proc/uptime") + uptime = uptimef.read().split(" ")[0] + uptimef.close() except IOError as e: - warn("unable to open /proc/uptime\n") - uptime = "na" + warn("unable to open /proc/uptime\n") + uptime = "na" try: cfg = cloudinit.get_base_cfg(cfg_path) @@ -74,7 +78,7 @@ def main(): raise try: - (outfmt, errfmt) = CC.get_output_cfg(cfg,"init") + (outfmt, errfmt) = CC.get_output_cfg(cfg, "init") CC.redirect_output(outfmt, errfmt) except Exception as e: warn("Failed to get and set output config: %s\n" % e) @@ -92,24 +96,24 @@ def main(): if cmd == "start": print netinfo.debug_info() - stop_files = ( cloudinit.get_ipath_cur("obj_pkl"), nonet_path ) + stop_files = (cloudinit.get_ipath_cur("obj_pkl"), nonet_path) # if starting as the network start, there are cases # where everything is already done for us, and it makes # most sense to exit early and silently for f in stop_files: try: - fp = open(f,"r") + fp = open(f, "r") fp.close() except: continue - + log.debug("no need for cloud-init start to run (%s)\n", f) sys.exit(0) elif cmd == "start-local": # cache is not instance specific, so it has to be purged # but we want 'start' to benefit from a cache if # a previous start-local populated one - manclean = util.get_cfg_option_bool(cfg, 'manual_cache_clean',False) + manclean = util.get_cfg_option_bool(cfg, 'manual_cache_clean', False) if manclean: log.debug("not purging cache, manual_cache_clean = True") cloudinit.purge_cache(not manclean) @@ -117,7 +121,8 @@ def main(): try: os.unlink(nonet_path) except OSError as e: - if e.errno != errno.ENOENT: raise + if e.errno != errno.ENOENT: + raise msg = "cloud-init %s running: %s. up %s seconds" % (cmd, now, uptime) sys.stderr.write(msg + "\n") @@ -146,7 +151,7 @@ def main(): # parse the user data (ec2-run-userdata.py) try: ran = cloud.sem_and_run("consume_userdata", cloudinit.per_instance, - cloud.consume_userdata,[cloudinit.per_instance],False) + cloud.consume_userdata, [cloudinit.per_instance], False) if not ran: cloud.consume_userdata(cloudinit.per_always) except: @@ -160,9 +165,9 @@ def main(): try: outfmt_orig = outfmt errfmt_orig = errfmt - (outfmt, errfmt) = CC.get_output_cfg(cc.cfg,"init") + (outfmt, errfmt) = CC.get_output_cfg(cc.cfg, "init") if outfmt_orig != outfmt or errfmt_orig != errfmt: - warn("stdout, stderr changing to (%s,%s)" % (outfmt,errfmt)) + warn("stdout, stderr changing to (%s,%s)" % (outfmt, errfmt)) CC.redirect_output(outfmt, errfmt) except Exception as e: warn("Failed to get and set output config: %s\n" % e) @@ -171,17 +176,17 @@ def main(): cc_path = cloudinit.get_ipath_cur('cloud_config') cc_ready = cc.cfg.get("cc_ready_cmd", ['initctl', 'emit', 'cloud-config', - '%s=%s' % (cloudinit.cfg_env_name, cc_path) ]) + '%s=%s' % (cloudinit.cfg_env_name, cc_path)]) if cc_ready: - if isinstance(cc_ready,str): - cc_ready = [ 'sh', '-c', cc_ready] + if isinstance(cc_ready, str): + cc_ready = ['sh', '-c', cc_ready] subprocess.Popen(cc_ready).communicate() - module_list = CC.read_cc_modules(cc.cfg,"cloud_init_modules") + module_list = CC.read_cc_modules(cc.cfg, "cloud_init_modules") failures = [] if len(module_list): - failures = CC.run_cc_modules(cc,module_list,log) + failures = CC.run_cc_modules(cc, module_list, log) else: msg = "no cloud_init_modules to run" sys.stderr.write(msg + "\n") diff --git a/cloudinit/CloudConfig/__init__.py b/cloudinit/CloudConfig/__init__.py index 82f422fc..a16bdde6 100644 --- a/cloudinit/CloudConfig/__init__.py +++ b/cloudinit/CloudConfig/__init__.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2008-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Chuck Short <chuck.short@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -16,6 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # + import yaml import cloudinit import cloudinit.util as util @@ -25,15 +28,16 @@ import os import subprocess import time -per_instance= cloudinit.per_instance +per_instance = cloudinit.per_instance per_always = cloudinit.per_always per_once = cloudinit.per_once + class CloudConfig(): cfgfile = None cfg = None - def __init__(self,cfgfile, cloud=None, ds_deps=[]): + def __init__(self, cfgfile, cloud=None, ds_deps=None): if cloud == None: self.cloud = cloudinit.CloudInit(ds_deps) self.cloud.get_data_source() @@ -41,62 +45,67 @@ class CloudConfig(): self.cloud = cloud self.cfg = self.get_config_obj(cfgfile) - def get_config_obj(self,cfgfile): + def get_config_obj(self, cfgfile): try: cfg = util.read_conf(cfgfile) except: # TODO: this 'log' could/should be passed in - cloudinit.log.critical("Failed loading of cloud config '%s'. Continuing with empty config\n" % cfgfile) + cloudinit.log.critical("Failed loading of cloud config '%s'. " + "Continuing with empty config\n" % cfgfile) cloudinit.log.debug(traceback.format_exc() + "\n") cfg = None - if cfg is None: cfg = { } + if cfg is None: + cfg = {} try: ds_cfg = self.cloud.datasource.get_config_obj() except: - ds_cfg = { } + ds_cfg = {} cfg = util.mergedict(cfg, ds_cfg) - return(util.mergedict(cfg,self.cloud.cfg)) + return(util.mergedict(cfg, self.cloud.cfg)) def handle(self, name, args, freq=None): try: - mod = __import__("cc_" + name.replace("-","_"),globals()) - def_freq = getattr(mod, "frequency",per_instance) + mod = __import__("cc_" + name.replace("-", "_"), globals()) + def_freq = getattr(mod, "frequency", per_instance) handler = getattr(mod, "handle") if not freq: freq = def_freq self.cloud.sem_and_run("config-" + name, freq, handler, - [ name, self.cfg, self.cloud, cloudinit.log, args ]) + [name, self.cfg, self.cloud, cloudinit.log, args]) except: raise + # reads a cloudconfig module list, returns # a 2 dimensional array suitable to pass to run_cc_modules -def read_cc_modules(cfg,name): - if name not in cfg: return([]) +def read_cc_modules(cfg, name): + if name not in cfg: + return([]) module_list = [] # create 'module_list', an array of arrays # where array[0] = config # array[1] = freq # array[2:] = arguemnts for item in cfg[name]: - if isinstance(item,str): + if isinstance(item, str): module_list.append((item,)) - elif isinstance(item,list): + elif isinstance(item, list): module_list.append(item) else: raise TypeError("failed to read '%s' item in config") return(module_list) - -def run_cc_modules(cc,module_list,log): + + +def run_cc_modules(cc, module_list, log): failures = [] for cfg_mod in module_list: name = cfg_mod[0] freq = None - run_args = [ ] + run_args = [] if len(cfg_mod) > 1: freq = cfg_mod[1] if len(cfg_mod) > 2: @@ -104,17 +113,18 @@ def run_cc_modules(cc,module_list,log): try: log.debug("handling %s with freq=%s and args=%s" % - (name, freq, run_args )) + (name, freq, run_args)) cc.handle(name, run_args, freq=freq) except: log.warn(traceback.format_exc()) log.error("config handling of %s, %s, %s failed\n" % - (name,freq,run_args)) + (name, freq, run_args)) failures.append(name) return(failures) -# always returns well formated values + +# always returns well formated values # cfg is expected to have an entry 'output' in it, which is a dictionary # that includes entries for 'init', 'config', 'final' or 'all' # init: /var/log/cloud.out @@ -125,25 +135,28 @@ def run_cc_modules(cc,module_list,log): # this returns the specific 'mode' entry, cleanly formatted, with value # None if if none is given def get_output_cfg(cfg, mode="init"): - ret = [ None, None ] - if not 'output' in cfg: return ret + ret = [None, None] + if not 'output' in cfg: + return ret outcfg = cfg['output'] if mode in outcfg: modecfg = outcfg[mode] else: - if 'all' not in outcfg: return ret + if 'all' not in outcfg: + return ret # if there is a 'all' item in the output list # then it applies to all users of this (init, config, final) modecfg = outcfg['all'] # if value is a string, it specifies stdout and stderr - if isinstance(modecfg,str): - ret = [ modecfg, modecfg ] + if isinstance(modecfg, str): + ret = [modecfg, modecfg] # if its a list, then we expect (stdout, stderr) - if isinstance(modecfg,list): - if len(modecfg) > 0: ret[0] = modecfg[0] + if isinstance(modecfg, list): + if len(modecfg) > 0: + ret[0] = modecfg[0] if len(modecfg) > 1: ret[1] = modecfg[1] @@ -157,26 +170,28 @@ def get_output_cfg(cfg, mode="init"): # if err's entry == "&1", then make it same as stdout # as in shell syntax of "echo foo >/dev/null 2>&1" - if ret[1] == "&1": ret[1] = ret[0] + if ret[1] == "&1": + ret[1] = ret[0] - swlist = [ ">>", ">", "|" ] + swlist = [">>", ">", "|"] for i in range(len(ret)): - if not ret[i]: continue + if not ret[i]: + continue val = ret[i].lstrip() found = False for s in swlist: if val.startswith(s): - val = "%s %s" % (s,val[len(s):].strip()) + val = "%s %s" % (s, val[len(s):].strip()) found = True break if not found: # default behavior is append - val = "%s %s" % ( ">>", val.strip()) + val = "%s %s" % (">>", val.strip()) ret[i] = val return(ret) - + # redirect_output(outfmt, errfmt, orig_out, orig_err) # replace orig_out and orig_err with filehandles specified in outfmt or errfmt # fmt can be: @@ -185,13 +200,14 @@ def get_output_cfg(cfg, mode="init"): # | program [ arg1 [ arg2 [ ... ] ] ] # # with a '|', arguments are passed to shell, so one level of -# shell escape is required. -def redirect_output(outfmt,errfmt, o_out=sys.stdout, o_err=sys.stderr): +# shell escape is required. +def redirect_output(outfmt, errfmt, o_out=sys.stdout, o_err=sys.stderr): if outfmt: - (mode, arg) = outfmt.split(" ",1) + (mode, arg) = outfmt.split(" ", 1) if mode == ">" or mode == ">>": owith = "ab" - if mode == ">": owith = "wb" + if mode == ">": + owith = "wb" new_fp = open(arg, owith) elif mode == "|": proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) @@ -206,10 +222,11 @@ def redirect_output(outfmt,errfmt, o_out=sys.stdout, o_err=sys.stderr): return if errfmt: - (mode, arg) = errfmt.split(" ",1) + (mode, arg) = errfmt.split(" ", 1) if mode == ">" or mode == ">>": owith = "ab" - if mode == ">": owith = "wb" + if mode == ">": + owith = "wb" new_fp = open(arg, owith) elif mode == "|": proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) @@ -221,30 +238,37 @@ def redirect_output(outfmt,errfmt, o_out=sys.stdout, o_err=sys.stderr): os.dup2(new_fp.fileno(), o_err.fileno()) return + def run_per_instance(name, func, args, clear_on_fail=False): - semfile = "%s/%s" % (cloudinit.get_ipath_cur("data"),name) - if os.path.exists(semfile): return + semfile = "%s/%s" % (cloudinit.get_ipath_cur("data"), name) + if os.path.exists(semfile): + return - util.write_file(semfile,str(time.time())) + util.write_file(semfile, str(time.time())) try: func(*args) except: - if clear_on_fail: os.unlink(semfile) + if clear_on_fail: + os.unlink(semfile) raise + # apt_get top level command (install, update...), and args to pass it -def apt_get(tlc,args=[]): - e=os.environ.copy() - e['DEBIAN_FRONTEND']='noninteractive' - cmd=[ 'apt-get', - '--option', 'Dpkg::Options::=--force-confold', '--assume-yes', - tlc ] +def apt_get(tlc, args=None): + if args is None: + args = [] + e = os.environ.copy() + e['DEBIAN_FRONTEND'] = 'noninteractive' + cmd = ['apt-get', '--option', 'Dpkg::Options::=--force-confold', + '--assume-yes', tlc] cmd.extend(args) - subprocess.check_call(cmd,env=e) + subprocess.check_call(cmd, env=e) + def update_package_sources(): run_per_instance("update-sources", apt_get, ("update",)) + def install_packages(pkglist): update_package_sources() - apt_get("install",pkglist) + apt_get("install", pkglist) diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py index a60512e1..a7049bce 100644 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ b/cloudinit/CloudConfig/cc_apt_update_upgrade.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,6 +17,7 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit.util as util import subprocess import traceback @@ -22,7 +25,8 @@ import os import glob import cloudinit.CloudConfig as cc -def handle(name,cfg,cloud,log,args): + +def handle(_name, cfg, cloud, log, _args): update = util.get_cfg_option_bool(cfg, 'apt_update', False) upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) @@ -35,18 +39,17 @@ def handle(name,cfg,cloud,log,args): if not util.get_cfg_option_bool(cfg, \ 'apt_preserve_sources_list', False): generate_sources_list(release, mirror) - old_mir = util.get_cfg_option_str(cfg,'apt_old_mirror', \ + old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', \ "archive.ubuntu.com/ubuntu") rename_apt_lists(old_mir, mirror) - # set up proxy proxy = cfg.get("apt_proxy", None) proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy" if proxy: try: contents = "Acquire::HTTP::Proxy \"%s\";\n" - with open(proxy_filename,"w") as fp: + with open(proxy_filename, "w") as fp: fp.write(contents % proxy) except Exception as e: log.warn("Failed to write proxy to %s" % proxy_filename) @@ -54,9 +57,9 @@ def handle(name,cfg,cloud,log,args): os.unlink(proxy_filename) # process 'apt_sources' - if cfg.has_key('apt_sources'): + if 'apt_sources' in cfg: errors = add_sources(cfg['apt_sources'], - { 'MIRROR' : mirror, 'RELEASE' : release } ) + {'MIRROR': mirror, 'RELEASE': release}) for e in errors: log.warn("Source Error: %s\n" % ':'.join(e)) @@ -69,9 +72,9 @@ def handle(name,cfg,cloud,log,args): log.error("Failed to run debconf-set-selections") log.debug(traceback.format_exc()) - pkglist = util.get_cfg_option_list_or_str(cfg,'packages',[]) + pkglist = util.get_cfg_option_list_or_str(cfg, 'packages', []) - errors = [ ] + errors = [] if update or len(pkglist) or upgrade: try: cc.update_package_sources() @@ -101,75 +104,90 @@ def handle(name,cfg,cloud,log,args): return(True) + def mirror2lists_fileprefix(mirror): - file=mirror + string = mirror # take of http:// or ftp:// - if file.endswith("/"): file=file[0:-1] - pos=file.find("://") + if string.endswith("/"): + string = string[0:-1] + pos = string.find("://") if pos >= 0: - file=file[pos+3:] - file=file.replace("/","_") - return file - -def rename_apt_lists(omirror,new_mirror,lists_d="/var/lib/apt/lists"): - - oprefix="%s/%s" % (lists_d,mirror2lists_fileprefix(omirror)) - nprefix="%s/%s" % (lists_d,mirror2lists_fileprefix(new_mirror)) - if(oprefix==nprefix): return - olen=len(oprefix) - for file in glob.glob("%s_*" % oprefix): - os.rename(file,"%s%s" % (nprefix, file[olen:])) + string = string[pos + 3:] + string = string.replace("/", "_") + return string + + +def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"): + oprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(omirror)) + nprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(new_mirror)) + if(oprefix == nprefix): + return + olen = len(oprefix) + for filename in glob.glob("%s_*" % oprefix): + os.rename(filename, "%s%s" % (nprefix, filename[olen:])) + def get_release(): - stdout, stderr = subprocess.Popen(['lsb_release', '-cs'], stdout=subprocess.PIPE).communicate() - return(stdout.strip()) + stdout, _stderr = subprocess.Popen(['lsb_release', '-cs'], + stdout=subprocess.PIPE).communicate() + return(str(stdout).strip()) + def generate_sources_list(codename, mirror): util.render_to_file('sources.list', '/etc/apt/sources.list', \ - { 'mirror' : mirror, 'codename' : codename }) + {'mirror': mirror, 'codename': codename}) + -# srclist is a list of dictionaries, -# each entry must have: 'source' -# may have: key, ( keyid and keyserver) -def add_sources(srclist, searchList={ }): +def add_sources(srclist, searchList=None): + """ + add entries in /etc/apt/sources.list.d for each abbreviated + sources.list entry in 'srclist'. When rendering template, also + include the values in dictionary searchList + """ + if searchList is None: + searchList = {} elst = [] for ent in srclist: - if not ent.has_key('source'): - elst.append([ "", "missing source" ]) + if 'source' not in ent: + elst.append(["", "missing source"]) continue - source=ent['source'] + source = ent['source'] if source.startswith("ppa:"): - try: util.subp(["add-apt-repository",source]) + try: + util.subp(["add-apt-repository", source]) except: elst.append([source, "add-apt-repository failed"]) continue source = util.render_string(source, searchList) - if not ent.has_key('filename'): - ent['filename']='cloud_config_sources.list' + if 'filename' not in ent: + ent['filename'] = 'cloud_config_sources.list' if not ent['filename'].startswith("/"): ent['filename'] = "%s/%s" % \ ("/etc/apt/sources.list.d/", ent['filename']) - if ( ent.has_key('keyid') and not ent.has_key('key') ): + if ('keyid' in ent and 'key' not in ent): ks = "keyserver.ubuntu.com" - if ent.has_key('keyserver'): ks = ent['keyserver'] + if 'keyserver' in ent: + ks = ent['keyserver'] try: ent['key'] = util.getkeybyid(ent['keyid'], ks) except: - elst.append([source,"failed to get key from %s" % ks]) + elst.append([source, "failed to get key from %s" % ks]) continue - if ent.has_key('key'): - try: util.subp(('apt-key', 'add', '-'), ent['key']) + if 'key' in ent: + try: + util.subp(('apt-key', 'add', '-'), ent['key']) except: elst.append([source, "failed add key"]) - try: util.write_file(ent['filename'], source + "\n", omode="ab") + try: + util.write_file(ent['filename'], source + "\n", omode="ab") except: elst.append([source, "failed write to file %s" % ent['filename']]) @@ -187,10 +205,10 @@ def find_apt_mirror(cloud, cfg): } mirror = None - cfg_mirror = cfg.get("apt_mirror",None) + cfg_mirror = cfg.get("apt_mirror", None) if cfg_mirror: mirror = cfg["apt_mirror"] - elif cfg.has_key("apt_mirror_search"): + elif "apt_mirror_search" in cfg: mirror = util.search_for_mirror(cfg['apt_mirror_search']) else: if cloud: @@ -202,19 +220,20 @@ def find_apt_mirror(cloud, cfg): if not mirror and cloud: # if we have a fqdn, then search its domain portion first - ( hostname, fqdn ) = util.get_hostname_fqdn(cfg, cloud) + (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) mydom = ".".join(fqdn.split(".")[1:]) if mydom: doms.append(".%s" % mydom) - doms.extend((".localdomain", "",)) + if not mirror: + doms.extend((".localdomain", "",)) - mirror_list = [] - mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro ) - for post in doms: - mirror_list.append(mirrorfmt % post) + mirror_list = [] + mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) + for post in doms: + mirror_list.append(mirrorfmt % post) - mirror = util.search_for_mirror(mirror_list) + mirror = util.search_for_mirror(mirror_list) if not mirror: mirror = defaults[distro] diff --git a/cloudinit/CloudConfig/cc_bootcmd.py b/cloudinit/CloudConfig/cc_bootcmd.py index 5a9e4356..f584da02 100644 --- a/cloudinit/CloudConfig/cc_bootcmd.py +++ b/cloudinit/CloudConfig/cc_bootcmd.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -18,11 +20,13 @@ import cloudinit.util as util import subprocess import tempfile +import os from cloudinit.CloudConfig import per_always frequency = per_always -def handle(name,cfg,cloud,log,args): - if not cfg.has_key("bootcmd"): + +def handle(_name, cfg, cloud, log, _args): + if "bootcmd" not in cfg: return try: @@ -33,10 +37,10 @@ def handle(name,cfg,cloud,log,args): except: log.warn("failed to shellify bootcmd") raise - + try: - env=os.environ.copy() - env['INSTANCE_ID']=cloud.get_instance_id() + env = os.environ.copy() + env['INSTANCE_ID'] = cloud.get_instance_id() subprocess.check_call(['/bin/sh'], env=env, stdin=tmpf) tmpf.close() except: diff --git a/cloudinit/CloudConfig/cc_byobu.py b/cloudinit/CloudConfig/cc_byobu.py index 406a1f67..e821b261 100644 --- a/cloudinit/CloudConfig/cc_byobu.py +++ b/cloudinit/CloudConfig/cc_byobu.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,23 +17,26 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit.util as util import subprocess import traceback -def handle(name,cfg,cloud,log,args): + +def handle(_name, cfg, _cloud, log, args): if len(args) != 0: value = args[0] else: - value = util.get_cfg_option_str(cfg,"byobu_by_default","") + value = util.get_cfg_option_str(cfg, "byobu_by_default", "") - if not value: return + if not value: + return if value == "user" or value == "system": value = "enable-%s" % value - valid = ( "enable-user", "enable-system", "enable", - "disable-user", "disable-system", "disable" ) + valid = ("enable-user", "enable-system", "enable", + "disable-user", "disable-system", "disable") if not value in valid: log.warn("Unknown value %s for byobu_by_default" % value) @@ -50,7 +55,7 @@ def handle(name,cfg,cloud,log,args): shcmd = "" if mod_user: - user = util.get_cfg_option_str(cfg,"user","ubuntu") + user = util.get_cfg_option_str(cfg, "user", "ubuntu") shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst) shcmd += " || X=$(($X+1)); " if mod_sys: @@ -58,7 +63,7 @@ def handle(name,cfg,cloud,log,args): shcmd += " && dpkg-reconfigure byobu --frontend=noninteractive" shcmd += " || X=$(($X+1)); " - cmd = [ "/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X" ) ] + cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] log.debug("setting byobu to %s" % value) @@ -66,7 +71,7 @@ def handle(name,cfg,cloud,log,args): subprocess.check_call(cmd) except subprocess.CalledProcessError as e: log.debug(traceback.format_exc(e)) - raise Exception("Cmd returned %s: %s" % ( e.returncode, cmd)) + raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) except OSError as e: log.debug(traceback.format_exc(e)) - raise Exception("Cmd failed to execute: %s" % ( cmd )) + raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/CloudConfig/cc_ca_certs.py b/cloudinit/CloudConfig/cc_ca_certs.py new file mode 100644 index 00000000..c18821f9 --- /dev/null +++ b/cloudinit/CloudConfig/cc_ca_certs.py @@ -0,0 +1,88 @@ +# vi: ts=4 expandtab +# +# Author: Mike Milner <mike.milner@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import os +from subprocess import check_call +from cloudinit.util import (write_file, get_cfg_option_list_or_str, + delete_dir_contents) + +CA_CERT_PATH = "/usr/share/ca-certificates/" +CA_CERT_FILENAME = "cloud-init-ca-certs.crt" +CA_CERT_CONFIG = "/etc/ca-certificates.conf" +CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" + + +def update_ca_certs(): + """ + Updates the CA certificate cache on the current machine. + """ + check_call(["update-ca-certificates"]) + + +def add_ca_certs(certs): + """ + Adds certificates to the system. To actually apply the new certificates + you must also call L{update_ca_certs}. + + @param certs: A list of certificate strings. + """ + if certs: + cert_file_contents = "\n".join(certs) + cert_file_fullpath = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) + write_file(cert_file_fullpath, cert_file_contents, mode=0644) + # Append cert filename to CA_CERT_CONFIG file. + write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="a") + + +def remove_default_ca_certs(): + """ + Removes all default trusted CA certificates from the system. To actually + apply the change you must also call L{update_ca_certs}. + """ + delete_dir_contents(CA_CERT_PATH) + delete_dir_contents(CA_CERT_SYSTEM_PATH) + write_file(CA_CERT_CONFIG, "", mode=0644) + + +def handle(_name, cfg, _cloud, log, _args): + """ + Call to handle ca-cert sections in cloud-config file. + + @param name: The module name "ca-cert" from cloud.cfg + @param cfg: A nested dict containing the entire cloud config contents. + @param cloud: The L{CloudInit} object in use. + @param log: Pre-initialized Python logger object to use for logging. + @param args: Any module arguments from cloud.cfg + """ + # If there isn't a ca-certs section in the configuration don't do anything + if "ca-certs" not in cfg: + return + ca_cert_cfg = cfg['ca-certs'] + + # If there is a remove-defaults option set to true, remove the system + # default trusted CA certs first. + if ca_cert_cfg.get("remove-defaults", False): + log.debug("removing default certificates") + remove_default_ca_certs() + + # If we are given any new trusted CA certs to add, add them. + if "trusted" in ca_cert_cfg: + trusted_certs = get_cfg_option_list_or_str(ca_cert_cfg, "trusted") + if trusted_certs: + log.debug("adding %d certificates" % len(trusted_certs)) + add_ca_certs(trusted_certs) + + # Update the system with the new cert configuration. + update_ca_certs() diff --git a/cloudinit/CloudConfig/cc_chef.py b/cloudinit/CloudConfig/cc_chef.py index 14960a3c..090aaa37 100644 --- a/cloudinit/CloudConfig/cc_chef.py +++ b/cloudinit/CloudConfig/cc_chef.py @@ -1,7 +1,10 @@ # vi: ts=4 expandtab # +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# # Author: Avishai Ish-Shalom <avishai@fewbytes.com> # Author: Mike Moulton <mike@meltmedia.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -14,21 +17,20 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import os -import pwd -import socket import subprocess import json -import StringIO -import ConfigParser import cloudinit.CloudConfig as cc import cloudinit.util as util ruby_version_default = "1.8" -def handle(name,cfg,cloud,log,args): + +def handle(_name, cfg, cloud, log, _args): # If there isn't a chef key in the configuration don't do anything - if not cfg.has_key('chef'): return + if 'chef' not in cfg: + return chef_cfg = cfg['chef'] # ensure the chef directories we use exist @@ -38,35 +40,36 @@ def handle(name,cfg,cloud,log,args): # set the validation key based on the presence of either 'validation_key' # or 'validation_cert'. In the case where both exist, 'validation_key' # takes precedence - if chef_cfg.has_key('validation_key') or chef_cfg.has_key('validation_cert'): + if ('validation_key' in chef_cfg or 'validation_cert' in chef_cfg): validation_key = util.get_cfg_option_str(chef_cfg, 'validation_key', chef_cfg['validation_cert']) with open('/etc/chef/validation.pem', 'w') as validation_key_fh: validation_key_fh.write(validation_key) - validation_name = chef_cfg.get('validation_name','chef-validator') # create the chef config from template util.render_to_file('chef_client.rb', '/etc/chef/client.rb', - {'server_url': chef_cfg['server_url'], - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', - cloud.datasource.get_instance_id()), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - '_default'), - 'validation_name': chef_cfg['validation_name']}) + {'server_url': chef_cfg['server_url'], + 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', + cloud.datasource.get_instance_id()), + 'environment': util.get_cfg_option_str(chef_cfg, 'environment', + '_default'), + 'validation_name': chef_cfg['validation_name']}) # set the firstboot json with open('/etc/chef/firstboot.json', 'w') as firstboot_json_fh: initial_json = {} - if chef_cfg.has_key('run_list'): + if 'run_list' in chef_cfg: initial_json['run_list'] = chef_cfg['run_list'] - if chef_cfg.has_key('initial_attributes'): + if 'initial_attributes' in chef_cfg: initial_attributes = chef_cfg['initial_attributes'] - for k in initial_attributes.keys(): initial_json[k] = initial_attributes[k] + for k in initial_attributes.keys(): + initial_json[k] = initial_attributes[k] firstboot_json_fh.write(json.dumps(initial_json)) # If chef is not installed, we install chef based on 'install_type' if not os.path.isfile('/usr/bin/chef-client'): - install_type = util.get_cfg_option_str(chef_cfg, 'install_type', 'packages') + install_type = util.get_cfg_option_str(chef_cfg, 'install_type', + 'packages') if install_type == "gems": # this will install and run the chef-client from gems chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) @@ -75,37 +78,42 @@ def handle(name,cfg,cloud,log,args): install_chef_from_gems(ruby_version, chef_version) # and finally, run chef-client log.debug('running chef-client') - subprocess.check_call(['/usr/bin/chef-client', '-d', '-i', '1800', '-s', '20']) + subprocess.check_call(['/usr/bin/chef-client', '-d', '-i', '1800', + '-s', '20']) else: # this will install and run the chef-client from packages cc.install_packages(('chef',)) + def get_ruby_packages(version): # return a list of packages needed to install ruby at version - pkgs = [ 'ruby%s' % version, 'ruby%s-dev' % version ] + pkgs = ['ruby%s' % version, 'ruby%s-dev' % version] if version == "1.8": pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8')) return(pkgs) -def install_chef_from_gems(ruby_version, chef_version = None): + +def install_chef_from_gems(ruby_version, chef_version=None): cc.install_packages(get_ruby_packages(ruby_version)) if not os.path.exists('/usr/bin/gem'): - os.symlink('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') + os.symlink('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') if not os.path.exists('/usr/bin/ruby'): - os.symlink('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') + os.symlink('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') if chef_version: - subprocess.check_call(['/usr/bin/gem','install','chef', + subprocess.check_call(['/usr/bin/gem', 'install', 'chef', '-v %s' % chef_version, '--no-ri', - '--no-rdoc','--bindir','/usr/bin','-q']) + '--no-rdoc', '--bindir', '/usr/bin', '-q']) else: - subprocess.check_call(['/usr/bin/gem','install','chef', - '--no-ri','--no-rdoc','--bindir', - '/usr/bin','-q']) + subprocess.check_call(['/usr/bin/gem', 'install', 'chef', + '--no-ri', '--no-rdoc', '--bindir', + '/usr/bin', '-q']) + def ensure_dir(d): if not os.path.exists(d): os.makedirs(d) + def mkdirs(dirs): for d in dirs: ensure_dir(d) diff --git a/cloudinit/CloudConfig/cc_disable_ec2_metadata.py b/cloudinit/CloudConfig/cc_disable_ec2_metadata.py index 5cd1d1c1..6b31ea8e 100644 --- a/cloudinit/CloudConfig/cc_disable_ec2_metadata.py +++ b/cloudinit/CloudConfig/cc_disable_ec2_metadata.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -21,7 +23,8 @@ from cloudinit.CloudConfig import per_always frequency = per_always -def handle(name,cfg,cloud,log,args): + +def handle(_name, cfg, _cloud, _log, _args): if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False): - fwall="route add -host 169.254.169.254 reject" + fwall = "route add -host 169.254.169.254 reject" subprocess.call(fwall.split(' ')) diff --git a/cloudinit/CloudConfig/cc_final_message.py b/cloudinit/CloudConfig/cc_final_message.py index fbdb5c94..abb4ca32 100644 --- a/cloudinit/CloudConfig/cc_final_message.py +++ b/cloudinit/CloudConfig/cc_final_message.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,6 +17,7 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + from cloudinit.CloudConfig import per_always import sys from cloudinit import util, boot_finished @@ -24,28 +27,28 @@ frequency = per_always final_message = "cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds" -def handle(name,cfg,cloud,log,args): + +def handle(_name, cfg, _cloud, log, args): if len(args) != 0: msg_in = args[0] else: - msg_in = util.get_cfg_option_str(cfg,"final_message",final_message) + msg_in = util.get_cfg_option_str(cfg, "final_message", final_message) try: - uptimef=open("/proc/uptime") - uptime=uptimef.read().split(" ")[0] + uptimef = open("/proc/uptime") + uptime = uptimef.read().split(" ")[0] uptimef.close() except IOError as e: log.warn("unable to open /proc/uptime\n") uptime = "na" - try: - ts = time.strftime("%a, %d %b %Y %H:%M:%S %z",time.gmtime()) + ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) except: ts = "na" try: - subs = { 'UPTIME' : uptime, 'TIMESTAMP' : ts } + subs = {'UPTIME': uptime, 'TIMESTAMP': ts} sys.stdout.write("%s\n" % util.render_string(msg_in, subs)) except Exception as e: log.warn("failed to render string to stdout: %s" % e) diff --git a/cloudinit/CloudConfig/cc_foo.py b/cloudinit/CloudConfig/cc_foo.py index aa95578f..35ec3fa7 100644 --- a/cloudinit/CloudConfig/cc_foo.py +++ b/cloudinit/CloudConfig/cc_foo.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,10 +17,13 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import cloudinit -import cloudinit.util as util + +#import cloudinit +#import cloudinit.util as util from cloudinit.CloudConfig import per_instance frequency = per_instance -def handle(name,cfg,cloud,log,args): - print "hi" + + +def handle(_name, _cfg, _cloud, _log, _args): + print "hi" diff --git a/cloudinit/CloudConfig/cc_grub_dpkg.py b/cloudinit/CloudConfig/cc_grub_dpkg.py index b26e90e8..9f3a7eaf 100644 --- a/cloudinit/CloudConfig/cc_grub_dpkg.py +++ b/cloudinit/CloudConfig/cc_grub_dpkg.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,42 +17,45 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit.util as util -import subprocess import traceback import os -def handle(name,cfg,cloud,log,args): - - idevs=None - idevs_empty=None + +def handle(_name, cfg, _cloud, log, _args): + idevs = None + idevs_empty = None if "grub-dpkg" in cfg: - idevs=util.get_cfg_option_str(cfg["grub-dpkg"], - "grub-pc/install_devices",None) - idevs_empty=util.get_cfg_option_str(cfg["grub-dpkg"], - "grub-pc/install_devices_empty",None) - - if (( os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda") ) or - ( os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda") )): - if idevs == None: idevs="" - if idevs_empty == None: idevs_empty="true" + idevs = util.get_cfg_option_str(cfg["grub-dpkg"], + "grub-pc/install_devices", None) + idevs_empty = util.get_cfg_option_str(cfg["grub-dpkg"], + "grub-pc/install_devices_empty", None) + + if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or + (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): + if idevs == None: + idevs = "" + if idevs_empty == None: + idevs_empty = "true" else: - if idevs_empty == None: idevs_empty="false" + if idevs_empty == None: + idevs_empty = "false" if idevs == None: idevs = "/dev/sda" - for dev in ( "/dev/sda", "/dev/vda", "/dev/sda1", "/dev/vda1"): + for dev in ("/dev/sda", "/dev/vda", "/dev/sda1", "/dev/vda1"): if os.path.exists(dev): idevs = dev break - + # now idevs and idevs_empty are set to determined values # or, those set by user dconf_sel = "grub-pc grub-pc/install_devices string %s\n" % idevs + \ "grub-pc grub-pc/install_devices_empty boolean %s\n" % idevs_empty log.debug("setting grub debconf-set-selections with '%s','%s'" % - (idevs,idevs_empty)) + (idevs, idevs_empty)) try: util.subp(('debconf-set-selections'), dconf_sel) diff --git a/cloudinit/CloudConfig/cc_keys_to_console.py b/cloudinit/CloudConfig/cc_keys_to_console.py index 79dd2ea9..73a477c0 100644 --- a/cloudinit/CloudConfig/cc_keys_to_console.py +++ b/cloudinit/CloudConfig/cc_keys_to_console.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,18 +17,22 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + from cloudinit.CloudConfig import per_instance import cloudinit.util as util import subprocess frequency = per_instance -def handle(name,cfg,cloud,log,args): - cmd = [ '/usr/lib/cloud-init/write-ssh-key-fingerprints' ] - fp_blacklist = util.get_cfg_option_list_or_str(cfg, "ssh_fp_console_blacklist", []) - key_blacklist = util.get_cfg_option_list_or_str(cfg, "ssh_key_console_blacklist", ["ssh-dss"]) + +def handle(_name, cfg, _cloud, log, _args): + cmd = ['/usr/lib/cloud-init/write-ssh-key-fingerprints'] + fp_blacklist = util.get_cfg_option_list_or_str(cfg, + "ssh_fp_console_blacklist", []) + key_blacklist = util.get_cfg_option_list_or_str(cfg, + "ssh_key_console_blacklist", ["ssh-dss"]) try: - confp = open('/dev/console',"wb") + confp = open('/dev/console', "wb") cmd.append(','.join(fp_blacklist)) cmd.append(','.join(key_blacklist)) subprocess.call(cmd, stdout=confp) diff --git a/cloudinit/CloudConfig/cc_landscape.py b/cloudinit/CloudConfig/cc_landscape.py index e3e4f7ba..596fd67e 100644 --- a/cloudinit/CloudConfig/cc_landscape.py +++ b/cloudinit/CloudConfig/cc_landscape.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,6 +17,7 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + from cloudinit.CloudConfig import per_instance from configobj import ConfigObj @@ -23,7 +26,7 @@ frequency = per_instance lsc_client_cfg_file = "/etc/landscape/client.conf" # defaults taken from stock client.conf in landscape-client 11.07.1.1-0ubuntu2 -lsc_builtincfg = { +lsc_builtincfg = { 'client': { 'log_level': "info", 'url': "https://landscape.canonical.com/message-system", @@ -32,14 +35,15 @@ lsc_builtincfg = { } } -def handle(name,cfg,cloud,log,args): + +def handle(_name, cfg, _cloud, log, _args): """ Basically turn a top level 'landscape' entry with a 'client' dict and render it to ConfigObj format under '[client]' section in /etc/landscape/client.conf """ - ls_cloudcfg = cfg.get("landscape", { }) + ls_cloudcfg = cfg.get("landscape", {}) if not isinstance(ls_cloudcfg, dict): raise(Exception("'landscape' existed in config, but not a dict")) @@ -51,6 +55,7 @@ def handle(name,cfg,cloud,log,args): log.debug("updated %s" % lsc_client_cfg_file) + def mergeTogether(objs): """ merge together ConfigObj objects or things that ConfigObj() will take in diff --git a/cloudinit/CloudConfig/cc_locale.py b/cloudinit/CloudConfig/cc_locale.py index 8542f641..2bb22fdb 100644 --- a/cloudinit/CloudConfig/cc_locale.py +++ b/cloudinit/CloudConfig/cc_locale.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,29 +17,33 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit.util as util import os.path import subprocess import traceback + def apply_locale(locale, cfgfile): if os.path.exists('/usr/sbin/locale-gen'): subprocess.Popen(['locale-gen', locale]).communicate() if os.path.exists('/usr/sbin/update-locale'): subprocess.Popen(['update-locale', locale]).communicate() - util.render_to_file('default-locale', cfgfile, { 'locale' : locale }) + util.render_to_file('default-locale', cfgfile, {'locale': locale}) + -def handle(name,cfg,cloud,log,args): +def handle(_name, cfg, cloud, log, args): if len(args) != 0: locale = args[0] else: - locale = util.get_cfg_option_str(cfg,"locale",cloud.get_locale()) + locale = util.get_cfg_option_str(cfg, "locale", cloud.get_locale()) locale_cfgfile = util.get_cfg_option_str(cfg, "locale_configfile", "/etc/default/locale") - if not locale: return + if not locale: + return log.debug("setting locale to %s" % locale) diff --git a/cloudinit/CloudConfig/cc_mcollective.py b/cloudinit/CloudConfig/cc_mcollective.py index c7912aa4..a2a6230c 100644 --- a/cloudinit/CloudConfig/cc_mcollective.py +++ b/cloudinit/CloudConfig/cc_mcollective.py @@ -1,9 +1,11 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Marc Cluet <marc.cluet@canonical.com> # Based on code by Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -16,11 +18,9 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import os -import pwd -import socket import subprocess -import fileinput import StringIO import ConfigParser import cloudinit.CloudConfig as cc @@ -29,49 +29,59 @@ import cloudinit.util as util pubcert_file = "/etc/mcollective/ssl/server-public.pem" pricert_file = "/etc/mcollective/ssl/server-private.pem" + # Our fake header section class FakeSecHead(object): def __init__(self, fp): self.fp = fp self.sechead = '[nullsection]\n' + def readline(self): if self.sechead: - try: return self.sechead - finally: self.sechead = None - else: return self.fp.readline() + try: + return self.sechead + finally: + self.sechead = None + else: + return self.fp.readline() -def handle(name,cfg,cloud,log,args): + +def handle(_name, cfg, _cloud, _log, _args): # If there isn't a mcollective key in the configuration don't do anything - if not cfg.has_key('mcollective'): return + if 'mcollective' not in cfg: + return mcollective_cfg = cfg['mcollective'] # Start by installing the mcollective package ... cc.install_packages(("mcollective",)) # ... and then update the mcollective configuration - if mcollective_cfg.has_key('conf'): + if 'conf' in mcollective_cfg: # Create object for reading server.cfg values mcollective_config = ConfigParser.ConfigParser() - # Read server.cfg values from original file in order to be able to mix the rest up - mcollective_config.readfp(FakeSecHead(open('/etc/mcollective/server.cfg'))) + # Read server.cfg values from original file in order to be able to mix + # the rest up + mcollective_config.readfp(FakeSecHead(open('/etc/mcollective/' + 'server.cfg'))) for cfg_name, cfg in mcollective_cfg['conf'].iteritems(): if cfg_name == 'public-cert': util.write_file(pubcert_file, cfg, mode=0644) mcollective_config.set(cfg_name, 'plugin.ssl_server_public', pubcert_file) - mcollective_config.set(cfg_name,'securityprovider','ssl') + mcollective_config.set(cfg_name, 'securityprovider', 'ssl') elif cfg_name == 'private-cert': util.write_file(pricert_file, cfg, mode=0600) mcollective_config.set(cfg_name, 'plugin.ssl_server_private', pricert_file) - mcollective_config.set(cfg_name,'securityprovider','ssl') + mcollective_config.set(cfg_name, 'securityprovider', 'ssl') else: # Iterate throug the config items, we'll use ConfigParser.set # to overwrite or create new items as needed for o, v in cfg.iteritems(): - mcollective_config.set(cfg_name,o,v) + mcollective_config.set(cfg_name, o, v) # We got all our config as wanted we'll rename # the previous server.cfg and create our new one - os.rename('/etc/mcollective/server.cfg','/etc/mcollective/server.cfg.old') + os.rename('/etc/mcollective/server.cfg', + '/etc/mcollective/server.cfg.old') outputfile = StringIO.StringIO() mcollective_config.write(outputfile) # Now we got the whole file, write to disk except first line @@ -81,9 +91,9 @@ def handle(name,cfg,cloud,log,args): # search and replace of '=' with ':' could be problematic though. # this most likely needs fixing. util.write_file('/etc/mcollective/server.cfg', - outputfile.getvalue().replace('[nullsection]\n','').replace(' =',':'), + outputfile.getvalue().replace('[nullsection]\n', '').replace(' =', + ':'), mode=0644) # Start mcollective subprocess.check_call(['service', 'mcollective', 'start']) - diff --git a/cloudinit/CloudConfig/cc_mounts.py b/cloudinit/CloudConfig/cc_mounts.py index db382f04..6cdd74e8 100644 --- a/cloudinit/CloudConfig/cc_mounts.py +++ b/cloudinit/CloudConfig/cc_mounts.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,33 +17,36 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit.util as util import os import re -import string +from string import whitespace # pylint: disable=W0402 + def is_mdname(name): # return true if this is a metadata service name - if name in [ "ami", "root", "swap" ]: + if name in ["ami", "root", "swap"]: return True # names 'ephemeral0' or 'ephemeral1' # 'ebs[0-9]' appears when '--block-device-mapping sdf=snap-d4d90bbc' - for enumname in ( "ephemeral", "ebs" ): + for enumname in ("ephemeral", "ebs"): if name.startswith(enumname) and name.find(":") == -1: return True return False -def handle(name,cfg,cloud,log,args): + +def handle(_name, cfg, cloud, log, _args): # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno - defvals = [ None, None, "auto", "defaults,nobootwait", "0", "2" ] + defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] defvals = cfg.get("mount_default_fields", defvals) # these are our default set of mounts - defmnts = [ [ "ephemeral0", "/mnt", "auto", defvals[3], "0", "2" ], - [ "swap", "none", "swap", "sw", "0", "0" ] ] + defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"], + ["swap", "none", "swap", "sw", "0", "0"]] - cfgmnt = [ ] - if cfg.has_key("mounts"): + cfgmnt = [] + if "mounts" in cfg: cfgmnt = cfg["mounts"] # shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 @@ -50,7 +55,8 @@ def handle(name,cfg,cloud,log,args): for i in range(len(cfgmnt)): # skip something that wasn't a list - if not isinstance(cfgmnt[i],list): continue + if not isinstance(cfgmnt[i], list): + continue # workaround, allow user to specify 'ephemeral' # rather than more ec2 correct 'ephemeral0' @@ -75,7 +81,7 @@ def handle(name,cfg,cloud,log,args): # but do not convert None to 'None' (LP: #898365) for j in range(len(cfgmnt[i])): if isinstance(cfgmnt[i][j], int): - cfgmnt[i][j]=str(cfgmnt[i][j]) + cfgmnt[i][j] = str(cfgmnt[i][j]) for i in range(len(cfgmnt)): # fill in values with defaults from defvals above @@ -93,12 +99,12 @@ def handle(name,cfg,cloud,log,args): if cfgmnt[j][0] == cfgmnt[i][0]: cfgmnt[j][1] = None - # for each of the "default" mounts, add them only if no other # entry has the same device name for defmnt in defmnts: devname = cloud.device_name_to_device(defmnt[0]) - if devname is None: continue + if devname is None: + continue if devname.startswith("/"): defmnt[0] = devname else: @@ -109,54 +115,65 @@ def handle(name,cfg,cloud,log,args): if cfgm[0] == defmnt[0]: cfgmnt_has = True break - - if cfgmnt_has: continue - cfgmnt.append(defmnt) + if cfgmnt_has: + continue + cfgmnt.append(defmnt) # now, each entry in the cfgmnt list has all fstab values # if the second field is None (not the string, the value) we skip it - actlist = filter(lambda x: x[1] is not None, cfgmnt) + actlist = [x for x in cfgmnt if x[1] is not None] - if len(actlist) == 0: return + if len(actlist) == 0: + return - comment="comment=cloudconfig" - cc_lines = [ ] + comment = "comment=cloudconfig" + cc_lines = [] needswap = False - dirs = [ ] + dirs = [] for line in actlist: # write 'comment' in the fs_mntops, entry, claiming this - line[3]="%s,comment=cloudconfig" % line[3] - if line[2] == "swap": needswap = True - if line[1].startswith("/"): dirs.append(line[1]) + line[3] = "%s,comment=cloudconfig" % line[3] + if line[2] == "swap": + needswap = True + if line[1].startswith("/"): + dirs.append(line[1]) cc_lines.append('\t'.join(line)) - fstab_lines = [ ] - fstab=open("/etc/fstab","r+") - ws = re.compile("[%s]+" % string.whitespace) + fstab_lines = [] + fstab = open("/etc/fstab", "r+") + ws = re.compile("[%s]+" % whitespace) for line in fstab.read().splitlines(): try: toks = ws.split(line) - if toks[3].find(comment) != -1: continue + if toks[3].find(comment) != -1: + continue except: pass fstab_lines.append(line) fstab_lines.extend(cc_lines) - + fstab.seek(0) fstab.write("%s\n" % '\n'.join(fstab_lines)) fstab.truncate() fstab.close() if needswap: - try: util.subp(("swapon", "-a")) - except: log.warn("Failed to enable swap") + try: + util.subp(("swapon", "-a")) + except: + log.warn("Failed to enable swap") for d in dirs: - if os.path.exists(d): continue - try: os.makedirs(d) - except: log.warn("Failed to make '%s' config-mount\n",d) + if os.path.exists(d): + continue + try: + os.makedirs(d) + except: + log.warn("Failed to make '%s' config-mount\n", d) - try: util.subp(("mount","-a")) - except: log.warn("'mount -a' failed") + try: + util.subp(("mount", "-a")) + except: + log.warn("'mount -a' failed") diff --git a/cloudinit/CloudConfig/cc_phone_home.py b/cloudinit/CloudConfig/cc_phone_home.py index f291e1d4..a7ff74e1 100644 --- a/cloudinit/CloudConfig/cc_phone_home.py +++ b/cloudinit/CloudConfig/cc_phone_home.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -20,7 +22,9 @@ import cloudinit.util as util from time import sleep frequency = per_instance -post_list_all = [ 'pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', 'hostname' ] +post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', + 'hostname'] + # phone_home: # url: http://my.foo.bar/$INSTANCE/ @@ -30,12 +34,13 @@ post_list_all = [ 'pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', # phone_home: # url: http://my.foo.bar/$INSTANCE_ID/ # post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id -# -def handle(name,cfg,cloud,log,args): +# +def handle(_name, cfg, cloud, log, args): if len(args) != 0: - ph_cfg = util.readconf(args[0]) + ph_cfg = util.read_conf(args[0]) else: - if not 'phone_home' in cfg: return + if not 'phone_home' in cfg: + return ph_cfg = cfg['phone_home'] if 'url' not in ph_cfg: @@ -44,7 +49,7 @@ def handle(name,cfg,cloud,log,args): url = ph_cfg['url'] post_list = ph_cfg.get('post', 'all') - tries = ph_cfg.get('tries',10) + tries = ph_cfg.get('tries', 10) try: tries = int(tries) except: @@ -54,7 +59,7 @@ def handle(name,cfg,cloud,log,args): if post_list == "all": post_list = post_list_all - all_keys = { } + all_keys = {} all_keys['instance_id'] = cloud.get_instance_id() all_keys['hostname'] = cloud.get_hostname() @@ -68,12 +73,11 @@ def handle(name,cfg,cloud,log,args): try: fp = open(path, "rb") all_keys[n] = fp.read() - all_keys[n] fp.close() except: log.warn("%s: failed to open in phone_home" % path) - submit_keys = { } + submit_keys = {} for k in post_list: if k in all_keys: submit_keys[k] = all_keys[k] @@ -81,20 +85,22 @@ def handle(name,cfg,cloud,log,args): submit_keys[k] = "N/A" log.warn("requested key %s from 'post' list not available") - url = util.render_string(url, { 'INSTANCE_ID' : all_keys['instance_id'] }) + url = util.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) - last_e = None - for i in range(0,tries): + null_exc = object() + last_e = null_exc + for i in range(0, tries): try: util.readurl(url, submit_keys) - log.debug("succeeded submit to %s on try %i" % (url, i+1)) + log.debug("succeeded submit to %s on try %i" % (url, i + 1)) return except Exception as e: - log.debug("failed to post to %s on try %i" % (url, i+1)) + log.debug("failed to post to %s on try %i" % (url, i + 1)) last_e = e sleep(3) log.warn("failed to post to %s in %i tries" % (url, tries)) - if last_e: raise(last_e) - + if last_e is not null_exc: + raise(last_e) + return diff --git a/cloudinit/CloudConfig/cc_puppet.py b/cloudinit/CloudConfig/cc_puppet.py index c635d1f5..6fc475f6 100644 --- a/cloudinit/CloudConfig/cc_puppet.py +++ b/cloudinit/CloudConfig/cc_puppet.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,6 +17,7 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import os import os.path import pwd @@ -25,21 +28,25 @@ import ConfigParser import cloudinit.CloudConfig as cc import cloudinit.util as util -def handle(name,cfg,cloud,log,args): + +def handle(_name, cfg, cloud, log, _args): # If there isn't a puppet key in the configuration don't do anything - if not cfg.has_key('puppet'): return + if 'puppet' not in cfg: + return puppet_cfg = cfg['puppet'] # Start by installing the puppet package ... cc.install_packages(("puppet",)) # ... and then update the puppet configuration - if puppet_cfg.has_key('conf'): + if 'conf' in puppet_cfg: # Add all sections from the conf object to puppet.conf puppet_conf_fh = open('/etc/puppet/puppet.conf', 'r') # Create object for reading puppet.conf values puppet_config = ConfigParser.ConfigParser() - # Read puppet.conf values from original file in order to be able to mix the rest up - puppet_config.readfp(StringIO.StringIO(''.join(i.lstrip() for i in puppet_conf_fh.readlines()))) + # Read puppet.conf values from original file in order to be able to + # mix the rest up + puppet_config.readfp(StringIO.StringIO(''.join(i.lstrip() for i in + puppet_conf_fh.readlines()))) # Close original file, no longer needed puppet_conf_fh.close() for cfg_name, cfg in puppet_cfg['conf'].iteritems(): @@ -63,7 +70,8 @@ def handle(name,cfg,cloud,log,args): util.restorecon_if_possible('/var/lib/puppet', recursive=True) else: #puppet_conf_fh.write("\n[%s]\n" % (cfg_name)) - # If puppet.conf already has this section we don't want to write it again + # If puppet.conf already has this section we don't want to + # write it again if puppet_config.has_section(cfg_name) == False: puppet_config.add_section(cfg_name) # Iterate throug the config items, we'll use ConfigParser.set @@ -77,11 +85,11 @@ def handle(name,cfg,cloud,log,args): cloud.datasource.get_instance_id()) # certname needs to be downcase v = v.lower() - puppet_config.set(cfg_name,o,v) + puppet_config.set(cfg_name, o, v) #puppet_conf_fh.write("%s=%s\n" % (o, v)) # We got all our config as wanted we'll rename # the previous puppet.conf and create our new one - os.rename('/etc/puppet/puppet.conf','/etc/puppet/puppet.conf.old') + os.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') with open('/etc/puppet/puppet.conf', 'wb') as configfile: puppet_config.write(configfile) util.restorecon_if_possible('/etc/puppet/puppet.conf') @@ -98,4 +106,3 @@ def handle(name,cfg,cloud,log,args): log.warn("Do not know how to enable puppet service on this system") # Start puppetd subprocess.check_call(['service', 'puppet', 'start']) - diff --git a/cloudinit/CloudConfig/cc_resizefs.py b/cloudinit/CloudConfig/cc_resizefs.py index 883c269b..0186d4d2 100644 --- a/cloudinit/CloudConfig/cc_resizefs.py +++ b/cloudinit/CloudConfig/cc_resizefs.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,32 +17,37 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit.util as util import subprocess -import traceback import os import stat import tempfile +from cloudinit.CloudConfig import per_always + +frequency = per_always + -def handle(name,cfg,cloud,log,args): +def handle(_name, cfg, _cloud, log, args): if len(args) != 0: resize_root = False - if str(value).lower() in [ 'true', '1', 'on', 'yes']: + if str(args[0]).lower() in ['true', '1', 'on', 'yes']: resize_root = True else: - resize_root = util.get_cfg_option_bool(cfg,"resize_rootfs",True) + resize_root = util.get_cfg_option_bool(cfg, "resize_rootfs", True) - if not resize_root: return + if not resize_root: + return # this really only uses the filename from mktemp, then we mknod into it (fd, devpth) = tempfile.mkstemp() os.unlink(devpth) os.close(fd) - + try: - st_dev=os.stat("/").st_dev - dev=os.makedev(os.major(st_dev),os.minor(st_dev)) - os.mknod(devpth, 0400 | stat.S_IFBLK, dev) + st_dev = os.stat("/").st_dev + dev = os.makedev(os.major(st_dev), os.minor(st_dev)) + os.mknod(devpth, 0400 | stat.S_IFBLK, dev) except: if util.islxc(): log.debug("inside lxc, ignoring mknod failure in resizefs") @@ -48,9 +55,9 @@ def handle(name,cfg,cloud,log,args): log.warn("Failed to make device node to resize /") raise - cmd = [ 'blkid', '-c', '/dev/null', '-sTYPE', '-ovalue', devpth ] + cmd = ['blkid', '-c', '/dev/null', '-sTYPE', '-ovalue', devpth] try: - (fstype,err) = util.subp(cmd) + (fstype, _err) = util.subp(cmd) except subprocess.CalledProcessError as e: log.warn("Failed to get filesystem type of maj=%s, min=%s via: %s" % (os.major(st_dev), os.minor(st_dev), cmd)) @@ -58,20 +65,20 @@ def handle(name,cfg,cloud,log,args): os.unlink(devpth) raise - log.debug("resizing root filesystem (type=%s, maj=%i, min=%i)" % - (fstype.rstrip("\n"), os.major(st_dev), os.minor(st_dev))) + log.debug("resizing root filesystem (type=%s, maj=%i, min=%i)" % + (str(fstype).rstrip("\n"), os.major(st_dev), os.minor(st_dev))) - if fstype.startswith("ext"): - resize_cmd = [ 'resize2fs', devpth ] + if str(fstype).startswith("ext"): + resize_cmd = ['resize2fs', devpth] elif fstype == "xfs": - resize_cmd = [ 'xfs_growfs', devpth ] + resize_cmd = ['xfs_growfs', devpth] else: os.unlink(devpth) log.debug("not resizing unknown filesystem %s" % fstype) return try: - (out,err) = util.subp(resize_cmd) + util.subp(resize_cmd) except subprocess.CalledProcessError as e: log.warn("Failed to resize filesystem (%s)" % resize_cmd) log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) diff --git a/cloudinit/CloudConfig/cc_rightscale_userdata.py b/cloudinit/CloudConfig/cc_rightscale_userdata.py index 4875acbe..5ed0848f 100644 --- a/cloudinit/CloudConfig/cc_rightscale_userdata.py +++ b/cloudinit/CloudConfig/cc_rightscale_userdata.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -24,16 +26,17 @@ ## for cloud-init support, there will be a key named ## 'CLOUD_INIT_REMOTE_HOOK'. ## -## This cloud-config module will +## This cloud-config module will ## - read the blob of data from raw user data, and parse it as key/value -## - for each key that is found, download the content to +## - for each key that is found, download the content to ## the local instance/scripts directory and set them executable. ## - the files in that directory will be run by the user-scripts module ## Therefore, this must run before that. ## ## + import cloudinit.util as util -from cloudinit.CloudConfig import per_once, per_always, per_instance +from cloudinit.CloudConfig import per_instance from cloudinit import get_ipath_cur from urlparse import parse_qs @@ -41,7 +44,8 @@ frequency = per_instance my_name = "cc_rightscale_userdata" my_hookname = 'CLOUD_INIT_REMOTE_HOOK' -def handle(name,cfg,cloud,log,args): + +def handle(_name, _cfg, cloud, log, _args): try: ud = cloud.get_userdata_raw() except: @@ -49,25 +53,26 @@ def handle(name,cfg,cloud,log,args): return try: - mdict = parse_qs(cloud.get_userdata_raw()) - if not my_hookname in mdict: return + mdict = parse_qs(ud) + if not my_hookname in mdict: + return except: log.warn("failed to urlparse.parse_qa(userdata_raw())") raise scripts_d = get_ipath_cur('scripts') i = 0 - errors = [ ] first_e = None for url in mdict[my_hookname]: - fname = "%s/rightscale-%02i" % (scripts_d,i) - i = i +1 + fname = "%s/rightscale-%02i" % (scripts_d, i) + i = i + 1 try: content = util.readurl(url) util.write_file(fname, content, mode=0700) except Exception as e: - if not first_e: first_e = None + if not first_e: + first_e = None log.warn("%s failed to read %s: %s" % (my_name, url, e)) - + if first_e: raise(e) diff --git a/cloudinit/CloudConfig/cc_rsyslog.py b/cloudinit/CloudConfig/cc_rsyslog.py index a8ee8476..ac7f2c74 100644 --- a/cloudinit/CloudConfig/cc_rsyslog.py +++ b/cloudinit/CloudConfig/cc_rsyslog.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab syntax=python # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,16 +17,17 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit import logging import cloudinit.util as util -import subprocess import traceback DEF_FILENAME = "20-cloud-config.conf" DEF_DIR = "/etc/rsyslog.d" -def handle(name,cfg,cloud,log,args): + +def handle(_name, cfg, _cloud, log, _args): # rsyslog: # - "*.* @@192.158.1.1" # - content: "*.* @@192.0.2.1:10514" @@ -33,17 +36,16 @@ def handle(name,cfg,cloud,log,args): # *.* @@syslogd.example.com # process 'rsyslog' - if not 'rsyslog' in cfg: return + if not 'rsyslog' in cfg: + return def_dir = cfg.get('rsyslog_dir', DEF_DIR) def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) - entries = cfg['rsyslog'] - - files = [ ] - elst = [ ] + files = [] + elst = [] for ent in cfg['rsyslog']: - if isinstance(ent,dict): + if isinstance(ent, dict): if not "content" in ent: elst.append((ent, "no 'content' entry")) continue @@ -54,7 +56,7 @@ def handle(name,cfg,cloud,log,args): filename = def_fname if not filename.startswith("/"): - filename = "%s/%s" % (def_dir,filename) + filename = "%s/%s" % (def_dir, filename) omode = "ab" # truncate filename first time you see it @@ -71,7 +73,7 @@ def handle(name,cfg,cloud,log,args): # need to restart syslogd restarted = False try: - # if this config module is running at cloud-init time + # if this config module is running at cloud-init time # (before rsyslog is running) we don't actually have to # restart syslog. # @@ -80,12 +82,12 @@ def handle(name,cfg,cloud,log,args): # it will also return failure on the attempt, so 'restarted' # won't get set log.debug("restarting rsyslog") - p = util.subp(['service', 'rsyslog', 'restart']) + util.subp(['service', 'rsyslog', 'restart']) restarted = True except Exception as e: elst.append(("restart", str(e))) - + if restarted: # this only needs to run if we *actually* restarted # syslog above. diff --git a/cloudinit/CloudConfig/cc_runcmd.py b/cloudinit/CloudConfig/cc_runcmd.py index f030980a..f7e8c671 100644 --- a/cloudinit/CloudConfig/cc_runcmd.py +++ b/cloudinit/CloudConfig/cc_runcmd.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,15 +17,16 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import cloudinit + import cloudinit.util as util -def handle(name,cfg,cloud,log,args): - if not cfg.has_key("runcmd"): + +def handle(_name, cfg, cloud, log, _args): + if "runcmd" not in cfg: return - outfile="%s/runcmd" % cloud.get_ipath('scripts') + outfile = "%s/runcmd" % cloud.get_ipath('scripts') try: content = util.shellify(cfg["runcmd"]) - util.write_file(outfile,content,0700) + util.write_file(outfile, content, 0700) except: log.warn("failed to open %s for runcmd" % outfile) diff --git a/cloudinit/CloudConfig/cc_scripts_per_boot.py b/cloudinit/CloudConfig/cc_scripts_per_boot.py index 4e407fb7..41a74754 100644 --- a/cloudinit/CloudConfig/cc_scripts_per_boot.py +++ b/cloudinit/CloudConfig/cc_scripts_per_boot.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,14 +17,16 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit.util as util -from cloudinit.CloudConfig import per_once, per_always, per_instance -from cloudinit import get_cpath, get_ipath_cur +from cloudinit.CloudConfig import per_always +from cloudinit import get_cpath frequency = per_always runparts_path = "%s/%s" % (get_cpath(), "scripts/per-boot") -def handle(name,cfg,cloud,log,args): + +def handle(_name, _cfg, _cloud, log, _args): try: util.runparts(runparts_path) except: diff --git a/cloudinit/CloudConfig/cc_scripts_per_instance.py b/cloudinit/CloudConfig/cc_scripts_per_instance.py index 22b41185..a2981eab 100644 --- a/cloudinit/CloudConfig/cc_scripts_per_instance.py +++ b/cloudinit/CloudConfig/cc_scripts_per_instance.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,14 +17,16 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit.util as util -from cloudinit.CloudConfig import per_once, per_always, per_instance -from cloudinit import get_cpath, get_ipath_cur +from cloudinit.CloudConfig import per_instance +from cloudinit import get_cpath frequency = per_instance runparts_path = "%s/%s" % (get_cpath(), "scripts/per-instance") -def handle(name,cfg,cloud,log,args): + +def handle(_name, _cfg, _cloud, log, _args): try: util.runparts(runparts_path) except: diff --git a/cloudinit/CloudConfig/cc_scripts_per_once.py b/cloudinit/CloudConfig/cc_scripts_per_once.py index 9d752325..a69151da 100644 --- a/cloudinit/CloudConfig/cc_scripts_per_once.py +++ b/cloudinit/CloudConfig/cc_scripts_per_once.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,14 +17,16 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit.util as util -from cloudinit.CloudConfig import per_once, per_always, per_instance -from cloudinit import get_cpath, get_ipath_cur +from cloudinit.CloudConfig import per_once +from cloudinit import get_cpath frequency = per_once runparts_path = "%s/%s" % (get_cpath(), "scripts/per-once") -def handle(name,cfg,cloud,log,args): + +def handle(_name, _cfg, _cloud, log, _args): try: util.runparts(runparts_path) except: diff --git a/cloudinit/CloudConfig/cc_scripts_user.py b/cloudinit/CloudConfig/cc_scripts_user.py index bafecd23..933aa4e0 100644 --- a/cloudinit/CloudConfig/cc_scripts_user.py +++ b/cloudinit/CloudConfig/cc_scripts_user.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,14 +17,16 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit.util as util -from cloudinit.CloudConfig import per_once, per_always, per_instance -from cloudinit import get_cpath, get_ipath_cur +from cloudinit.CloudConfig import per_instance +from cloudinit import get_ipath_cur frequency = per_instance runparts_path = "%s/%s" % (get_ipath_cur(), "scripts") -def handle(name,cfg,cloud,log,args): + +def handle(_name, _cfg, _cloud, log, _args): try: util.runparts(runparts_path) except: diff --git a/cloudinit/CloudConfig/cc_set_hostname.py b/cloudinit/CloudConfig/cc_set_hostname.py index bc190049..acea74d9 100644 --- a/cloudinit/CloudConfig/cc_set_hostname.py +++ b/cloudinit/CloudConfig/cc_set_hostname.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,24 +17,26 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit.util as util -import subprocess -def handle(name,cfg,cloud,log,args): - if util.get_cfg_option_bool(cfg,"preserve_hostname",False): + +def handle(_name, cfg, cloud, log, _args): + if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug("preserve_hostname is set. not setting hostname") return(True) - ( hostname, fqdn ) = util.get_hostname_fqdn(cfg, cloud) + (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) try: set_hostname(hostname, log) - except Exception as e: + except Exception: util.logexc(log) log.warn("failed to set hostname to %s\n", hostname) return(True) + def set_hostname(hostname, log): util.subp(['hostname', hostname]) - util.write_file("/etc/hostname","%s\n" % hostname, 0644) + util.write_file("/etc/hostname", "%s\n" % hostname, 0644) log.debug("populated /etc/hostname with %s on first boot", hostname) diff --git a/cloudinit/CloudConfig/cc_set_passwords.py b/cloudinit/CloudConfig/cc_set_passwords.py index edfbaeeb..9d0bbdb8 100644 --- a/cloudinit/CloudConfig/cc_set_passwords.py +++ b/cloudinit/CloudConfig/cc_set_passwords.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,20 +17,21 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit.util as util -import subprocess import sys import random -import string +from string import letters, digits # pylint: disable=W0402 + -def handle(name,cfg,cloud,log,args): +def handle(_name, cfg, _cloud, log, args): if len(args) != 0: # if run from command line, and give args, wipe the chpasswd['list'] password = args[0] if 'chpasswd' in cfg and 'list' in cfg['chpasswd']: del cfg['chpasswd']['list'] else: - password = util.get_cfg_option_str(cfg,"password",None) + password = util.get_cfg_option_str(cfg, "password", None) expire = True pw_auth = "no" @@ -37,11 +40,11 @@ def handle(name,cfg,cloud,log,args): if 'chpasswd' in cfg: chfg = cfg['chpasswd'] - plist = util.get_cfg_option_str(chfg,'list',plist) - expire = util.get_cfg_option_bool(chfg,'expire', expire) + plist = util.get_cfg_option_str(chfg, 'list', plist) + expire = util.get_cfg_option_bool(chfg, 'expire', expire) if not plist and password: - user = util.get_cfg_option_str(cfg,"user","ubuntu") + user = util.get_cfg_option_str(cfg, "user", "ubuntu") plist = "%s:%s" % (user, password) errors = [] @@ -50,13 +53,13 @@ def handle(name,cfg,cloud,log,args): randlist = [] users = [] for line in plist.splitlines(): - u,p = line.split(':',1) + u, p = line.split(':', 1) if p == "R" or p == "RANDOM": p = rand_user_password() - randlist.append("%s:%s" % (u,p)) - plist_in.append("%s:%s" % (u,p)) + randlist.append("%s:%s" % (u, p)) + plist_in.append("%s:%s" % (u, p)) users.append(u) - + ch_in = '\n'.join(plist_in) try: util.subp(['chpasswd'], ch_in) @@ -67,36 +70,36 @@ def handle(name,cfg,cloud,log,args): if len(randlist): sys.stdout.write("%s\n%s\n" % ("Set the following passwords\n", - '\n'.join(randlist) )) + '\n'.join(randlist))) if expire: - enum=len(errors) + enum = len(errors) for u in users: try: util.subp(['passwd', '--expire', u]) except Exception as e: errors.append(e) - log.warn("failed to expire account for %s" % u ) + log.warn("failed to expire account for %s" % u) if enum == len(errors): log.debug("expired passwords for: %s" % u) if 'ssh_pwauth' in cfg: val = str(cfg['ssh_pwauth']).lower() - if val in ( "true", "1", "yes"): - pw_auth="yes" - change_pwauth=True - elif val in ( "false", "0", "no"): - pw_auth="no" - change_pwauth=True + if val in ("true", "1", "yes"): + pw_auth = "yes" + change_pwauth = True + elif val in ("false", "0", "no"): + pw_auth = "no" + change_pwauth = True else: - change_pwauth=False - + change_pwauth = False + if change_pwauth: pa_s = "\(#*\)\(PasswordAuthentication[[:space:]]\+\)\(yes\|no\)" msg = "set PasswordAuthentication to '%s'" % pw_auth try: - cmd = [ 'sed', '-i', 's,%s,\\2%s,' % (pa_s, pw_auth), - '/etc/ssh/sshd_config' ] + cmd = ['sed', '-i', 's,%s,\\2%s,' % (pa_s, pw_auth), + '/etc/ssh/sshd_config'] util.subp(cmd) log.debug(msg) except Exception as e: @@ -104,7 +107,8 @@ def handle(name,cfg,cloud,log,args): errors.append(e) try: - p = util.subp(['service', cfg.get('ssh_svcname', 'ssh'), 'restart']) + p = util.subp(['service', cfg.get('ssh_svcname', 'ssh'), + 'restart']) log.debug("restarted sshd") except: log.warn("restart of ssh failed") @@ -114,11 +118,12 @@ def handle(name,cfg,cloud,log,args): return -def rand_str(strlen=32, select_from=string.letters+string.digits): - return("".join([random.choice(select_from) for x in range(0, strlen)])) -def rand_user_password(pwlen=9): - selfrom=(string.letters.translate(None,'loLOI') + - string.digits.translate(None,'01')) - return(rand_str(pwlen,select_from=selfrom)) +def rand_str(strlen=32, select_from=letters + digits): + return("".join([random.choice(select_from) for _x in range(0, strlen)])) + +def rand_user_password(pwlen=9): + selfrom = (letters.translate(None, 'loLOI') + + digits.translate(None, '01')) + return(rand_str(pwlen, select_from=selfrom)) diff --git a/cloudinit/CloudConfig/cc_ssh.py b/cloudinit/CloudConfig/cc_ssh.py index 9f5dc567..48eb58bc 100644 --- a/cloudinit/CloudConfig/cc_ssh.py +++ b/cloudinit/CloudConfig/cc_ssh.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,55 +17,57 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit.util as util import cloudinit.SshUtil as sshutil import os import glob import subprocess -DISABLE_ROOT_OPTS="no-port-forwarding,no-agent-forwarding,no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" rather than the user \\\"root\\\".\';echo;sleep 10\"" - +DISABLE_ROOT_OPTS = "no-port-forwarding,no-agent-forwarding," \ +"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " \ +"rather than the user \\\"root\\\".\';echo;sleep 10\"" -global_log = None -def handle(name,cfg,cloud,log,args): - global global_log - global_log = log +def handle(_name, cfg, cloud, log, _args): # remove the static keys from the pristine image if cfg.get("ssh_deletekeys", True): for f in glob.glob("/etc/ssh/ssh_host_*key*"): - try: os.unlink(f) - except: pass + try: + os.unlink(f) + except: + pass - if cfg.has_key("ssh_keys"): + if "ssh_keys" in cfg: # if there are keys in cloud-config, use them key2file = { - "rsa_private" : ("/etc/ssh/ssh_host_rsa_key", 0600), - "rsa_public" : ("/etc/ssh/ssh_host_rsa_key.pub", 0644), - "dsa_private" : ("/etc/ssh/ssh_host_dsa_key", 0600), - "dsa_public" : ("/etc/ssh/ssh_host_dsa_key.pub", 0644), - "ecdsa_private" : ("/etc/ssh/ssh_host_ecdsa_key", 0600), - "ecdsa_public" : ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), + "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), + "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), + "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), + "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), + "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), + "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), } - for key,val in cfg["ssh_keys"].items(): - if key2file.has_key(key): - util.write_file(key2file[key][0],val,key2file[key][1]) + for key, val in cfg["ssh_keys"].items(): + if key in key2file: + util.write_file(key2file[key][0], val, key2file[key][1]) - priv2pub = { 'rsa_private':'rsa_public', 'dsa_private':'dsa_public', - 'ecdsa_private': 'ecdsa_public', } + priv2pub = {'rsa_private': 'rsa_public', 'dsa_private': 'dsa_public', + 'ecdsa_private': 'ecdsa_public', } cmd = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' - for priv,pub in priv2pub.iteritems(): - if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: continue - pair=(key2file[priv][0], key2file[pub][0]) + for priv, pub in priv2pub.iteritems(): + if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: + continue + pair = (key2file[priv][0], key2file[pub][0]) subprocess.call(('sh', '-xc', cmd % pair)) log.debug("generated %s from %s" % pair) else: # if not, generate them for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', - ['rsa', 'dsa', 'ecdsa']): + ['rsa', 'dsa', 'ecdsa']): keyfile = '/etc/ssh/ssh_host_%s_key' % keytype if not os.path.exists(keyfile): subprocess.call(['ssh-keygen', '-t', keytype, '-N', '', @@ -72,30 +76,31 @@ def handle(name,cfg,cloud,log,args): util.restorecon_if_possible('/etc/ssh', recursive=True) try: - user = util.get_cfg_option_str(cfg,'user') + user = util.get_cfg_option_str(cfg, 'user') disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", DISABLE_ROOT_OPTS) keys = cloud.get_public_ssh_keys() - if cfg.has_key("ssh_authorized_keys"): + if "ssh_authorized_keys" in cfg: cfgkeys = cfg["ssh_authorized_keys"] keys.extend(cfgkeys) - apply_credentials(keys,user,disable_root, disable_root_opts) + apply_credentials(keys, user, disable_root, disable_root_opts, log) except: util.logexc(log) log.warn("applying credentials failed!\n") -def apply_credentials(keys, user, disable_root, disable_root_opts=DISABLE_ROOT_OPTS, log=global_log): + +def apply_credentials(keys, user, disable_root, + disable_root_opts=DISABLE_ROOT_OPTS, log=None): keys = set(keys) if user: sshutil.setup_user_keys(keys, user, '', log) - + if disable_root: key_prefix = disable_root_opts.replace('$USER', user) else: key_prefix = '' sshutil.setup_user_keys(keys, 'root', key_prefix, log) - diff --git a/cloudinit/CloudConfig/cc_ssh_import_id.py b/cloudinit/CloudConfig/cc_ssh_import_id.py index bf1314be..bbf5bd83 100644 --- a/cloudinit/CloudConfig/cc_ssh_import_id.py +++ b/cloudinit/CloudConfig/cc_ssh_import_id.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,23 +17,26 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit.util as util import subprocess import traceback -def handle(name,cfg,cloud,log,args): + +def handle(_name, cfg, _cloud, log, args): if len(args) != 0: user = args[0] - ids = [ ] + ids = [] if len(args) > 1: ids = args[1:] else: - user = util.get_cfg_option_str(cfg,"user","ubuntu") - ids = util.get_cfg_option_list_or_str(cfg,"ssh_import_id",[]) + user = util.get_cfg_option_str(cfg, "user", "ubuntu") + ids = util.get_cfg_option_list_or_str(cfg, "ssh_import_id", []) - if len(ids) == 0: return + if len(ids) == 0: + return - cmd = [ "sudo", "-Hu", user, "ssh-import-id" ] + ids + cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids log.debug("importing ssh ids. cmd = %s" % cmd) @@ -39,7 +44,7 @@ def handle(name,cfg,cloud,log,args): subprocess.check_call(cmd) except subprocess.CalledProcessError as e: log.debug(traceback.format_exc(e)) - raise Exception("Cmd returned %s: %s" % ( e.returncode, cmd)) + raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) except OSError as e: log.debug(traceback.format_exc(e)) - raise Exception("Cmd failed to execute: %s" % ( cmd )) + raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/CloudConfig/cc_timezone.py b/cloudinit/CloudConfig/cc_timezone.py index e8a453f9..e5c9901b 100644 --- a/cloudinit/CloudConfig/cc_timezone.py +++ b/cloudinit/CloudConfig/cc_timezone.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,31 +17,33 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + from cloudinit.CloudConfig import per_instance from cloudinit import util -import subprocess import os.path import shutil frequency = per_instance tz_base = "/usr/share/zoneinfo" -def handle(name,cfg,cloud,log,args): + +def handle(_name, cfg, _cloud, log, args): if len(args) != 0: timezone = args[0] else: - timezone = util.get_cfg_option_str(cfg,"timezone",False) + timezone = util.get_cfg_option_str(cfg, "timezone", False) - if not timezone: return + if not timezone: + return - tz_file = "%s/%s" % (tz_base , timezone) + tz_file = "%s/%s" % (tz_base, timezone) if not os.path.isfile(tz_file): log.debug("Invalid timezone %s" % tz_file) raise Exception("Invalid timezone %s" % tz_file) try: - fp=open("/etc/timezone","wb") + fp = open("/etc/timezone", "wb") fp.write("%s\n" % timezone) fp.close() except: @@ -58,6 +62,6 @@ def handle(name,cfg,cloud,log,args): except: log.debug("failed to copy %s to /etc/localtime" % tz_file) raise - + log.debug("set timezone to %s" % timezone) return diff --git a/cloudinit/CloudConfig/cc_update_etc_hosts.py b/cloudinit/CloudConfig/cc_update_etc_hosts.py index 1dc9c1e9..572e6750 100644 --- a/cloudinit/CloudConfig/cc_update_etc_hosts.py +++ b/cloudinit/CloudConfig/cc_update_etc_hosts.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,16 +17,18 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit.util as util from cloudinit.CloudConfig import per_always import StringIO frequency = per_always -def handle(name,cfg,cloud,log,args): - ( hostname, fqdn ) = util.get_hostname_fqdn(cfg, cloud) - manage_hosts = util.get_cfg_option_bool(cfg,"manage_etc_hosts", False) +def handle(_name, cfg, cloud, log, _args): + (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + + manage_hosts = util.get_cfg_option_bool(cfg, "manage_etc_hosts", False) if manage_hosts in ("True", "true", True, "template"): # render from template file try: @@ -32,9 +36,9 @@ def handle(name,cfg,cloud,log,args): log.info("manage_etc_hosts was set, but no hostname found") return - util.render_to_file('hosts', '/etc/hosts', \ - { 'hostname' : hostname, 'fqdn' : fqdn }) - except Exception as e: + util.render_to_file('hosts', '/etc/hosts', + {'hostname': hostname, 'fqdn': fqdn}) + except Exception: log.warn("failed to update /etc/hosts") raise elif manage_hosts == "localhost": @@ -48,37 +52,36 @@ def handle(name,cfg,cloud,log,args): log.debug("not managing /etc/hosts") -def update_etc_hosts(hostname, fqdn, log): - with open('/etc/hosts', 'r') as etchosts: - header = "# Added by cloud-init\n" - hosts_line = "127.0.1.1\t%s %s\n" % (fqdn, hostname) - need_write = False - need_change = True - new_etchosts = StringIO.StringIO() - for line in etchosts: - split_line = [s.strip() for s in line.split()] - if len(split_line) < 2: +def update_etc_hosts(hostname, fqdn, _log): + with open('/etc/hosts', 'r') as etchosts: + header = "# Added by cloud-init\n" + hosts_line = "127.0.1.1\t%s %s\n" % (fqdn, hostname) + need_write = False + need_change = True + new_etchosts = StringIO.StringIO() + for line in etchosts: + split_line = [s.strip() for s in line.split()] + if len(split_line) < 2: new_etchosts.write(line) continue - if line == header: + if line == header: continue - ip, hosts = split_line[0], split_line[1:] - if ip == "127.0.1.1": - if sorted([hostname, fqdn]) == sorted(hosts): - need_change = False - if need_change == True: - line = "%s%s" % (header, hosts_line) - need_change = False - need_write = True - new_etchosts.write(line) - etchosts.close() - if need_change == True: - new_etchosts.write("%s%s" % (header, hosts_line)) - need_write = True - if need_write == True: - new_etcfile = open ('/etc/hosts','wb') - new_etcfile.write(new_etchosts.getvalue()) - new_etcfile.close() - new_etchosts.close() - return - + ip, hosts = split_line[0], split_line[1:] + if ip == "127.0.1.1": + if sorted([hostname, fqdn]) == sorted(hosts): + need_change = False + if need_change == True: + line = "%s%s" % (header, hosts_line) + need_change = False + need_write = True + new_etchosts.write(line) + etchosts.close() + if need_change == True: + new_etchosts.write("%s%s" % (header, hosts_line)) + need_write = True + if need_write == True: + new_etcfile = open('/etc/hosts', 'wb') + new_etcfile.write(new_etchosts.getvalue()) + new_etcfile.close() + new_etchosts.close() + return diff --git a/cloudinit/CloudConfig/cc_update_hostname.py b/cloudinit/CloudConfig/cc_update_hostname.py index 3f55c73b..b9d1919a 100644 --- a/cloudinit/CloudConfig/cc_update_hostname.py +++ b/cloudinit/CloudConfig/cc_update_hostname.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,6 +17,7 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import cloudinit.util as util import subprocess import errno @@ -22,25 +25,27 @@ from cloudinit.CloudConfig import per_always frequency = per_always -def handle(name,cfg,cloud,log,args): - if util.get_cfg_option_bool(cfg,"preserve_hostname",False): + +def handle(_name, cfg, cloud, log, _args): + if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug("preserve_hostname is set. not updating hostname") return - ( hostname, fqdn ) = util.get_hostname_fqdn(cfg, cloud) + (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) try: - prev ="%s/%s" % (cloud.get_cpath('data'),"previous-hostname") + prev = "%s/%s" % (cloud.get_cpath('data'), "previous-hostname") update_hostname(hostname, prev, log) - except Exception as e: + except Exception: log.warn("failed to set hostname\n") raise + # read hostname from a 'hostname' file # allow for comments and stripping line endings. # if file doesn't exist, or no contents, return default def read_hostname(filename, default=None): try: - fp = open(filename,"r") + fp = open(filename, "r") lines = fp.readlines() fp.close() for line in lines: @@ -51,9 +56,11 @@ def read_hostname(filename, default=None): if line: return line except IOError as e: - if e.errno != errno.ENOENT: raise + if e.errno != errno.ENOENT: + raise return default - + + def update_hostname(hostname, prev_file, log): etc_file = "/etc/hostname" @@ -74,22 +81,21 @@ def update_hostname(hostname, prev_file, log): if not hostname_prev or hostname_prev != hostname: update_files.append(prev_file) - if (not hostname_in_etc or + if (not hostname_in_etc or (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): update_files.append(etc_file) try: for fname in update_files: - util.write_file(fname,"%s\n" % hostname, 0644) - log.debug("wrote %s to %s" % (hostname,fname)) + util.write_file(fname, "%s\n" % hostname, 0644) + log.debug("wrote %s to %s" % (hostname, fname)) except: log.warn("failed to write hostname to %s" % fname) if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: log.debug("%s differs from %s. assuming user maintained" % - (prev_file,etc_file)) + (prev_file, etc_file)) if etc_file in update_files: log.debug("setting hostname to %s" % hostname) subprocess.Popen(['hostname', hostname]).communicate() - diff --git a/cloudinit/DataSource.py b/cloudinit/DataSource.py index 06061a2e..f38e3b20 100644 --- a/cloudinit/DataSource.py +++ b/cloudinit/DataSource.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Hafliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -20,22 +22,23 @@ DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" -import UserDataHandler as ud +import cloudinit.UserDataHandler as ud import cloudinit.util as util import socket + class DataSource: userdata = None metadata = None userdata_raw = None cfgname = "" - # system config (passed in from cloudinit, + # system config (passed in from cloudinit, # cloud-config before input from the DataSource) - sys_cfg = { } + sys_cfg = {} # datasource config, the cloud-config['datasource']['__name__'] - ds_cfg = { } # datasource config + ds_cfg = {} # datasource config - def __init__(self,sys_cfg=None): + def __init__(self, sys_cfg=None): if not self.cfgname: name = str(self.__class__).split(".")[-1] if name.startswith("DataSource"): @@ -45,7 +48,7 @@ class DataSource: self.sys_cfg = sys_cfg self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, - ("datasource",self.cfgname),self.ds_cfg) + ("datasource", self.cfgname), self.ds_cfg) def get_userdata(self): if self.userdata == None: @@ -55,26 +58,26 @@ class DataSource: def get_userdata_raw(self): return(self.userdata_raw) - # the data sources' config_obj is a cloud-config formated # object that came to it from ways other than cloud-config # because cloud-config content would be handled elsewhere def get_config_obj(self): - return({ }) + return({}) def get_public_ssh_keys(self): keys = [] - if not self.metadata.has_key('public-keys'): return([]) + if 'public-keys' not in self.metadata: + return([]) if isinstance(self.metadata['public-keys'], str): - return([self.metadata['public-keys'],]) - - for keyname, klist in self.metadata['public-keys'].items(): + return([self.metadata['public-keys'], ]) + + for _keyname, klist in self.metadata['public-keys'].items(): # lp:506332 uec metadata service responds with # data that makes boto populate a string for 'klist' rather # than a list. - if isinstance(klist,str): - klist = [ klist ] + if isinstance(klist, str): + klist = [klist] for pkey in klist: # there is an empty string at the end of the keylist, trim it if pkey: @@ -82,7 +85,7 @@ class DataSource: return(keys) - def device_name_to_device(self, name): + def device_name_to_device(self, _name): # translate a 'name' to a device # the primary function at this point is on ec2 # to consult metadata service, that has @@ -103,7 +106,7 @@ class DataSource: def get_hostname(self, fqdn=False): defdomain = "localdomain" - defhost = "localhost" + defhost = "localhost" domain = defdomain if not 'local-hostname' in self.metadata: @@ -119,19 +122,18 @@ class DataSource: fqdn = util.get_fqdn_from_hosts(hostname) if fqdn and fqdn.find(".") > 0: - toks = fqdn.split(".") + toks = str(fqdn).split(".") elif hostname: - toks = [ hostname, defdomain ] + toks = [hostname, defdomain] else: - toks = [ defhost, defdomain ] - + toks = [defhost, defdomain] else: # if there is an ipv4 address in 'local-hostname', then # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx lhost = self.metadata['local-hostname'] if is_ipv4(lhost): - toks = "ip-%s" % lhost.replace(".","-") + toks = "ip-%s" % lhost.replace(".", "-") else: toks = lhost.split(".") @@ -142,10 +144,11 @@ class DataSource: hostname = toks[0] if fqdn: - return "%s.%s" % (hostname,domain) + return "%s.%s" % (hostname, domain) else: return hostname + # return a list of classes that have the same depends as 'depends' # iterate through cfg_list, loading "DataSourceCollections" modules # and calling their "get_datasource_list". @@ -157,11 +160,14 @@ class DataSource: # ie, pkglist=[ "foo", "" ] # will first try to load foo.DataSource<item> # then DataSource<item> -def list_sources(cfg_list, depends, pkglist=[]): +def list_sources(cfg_list, depends, pkglist=None): + if pkglist is None: + pkglist = [] retlist = [] for ds_coll in cfg_list: for pkg in pkglist: - if pkg: pkg="%s." % pkg + if pkg: + pkg = "%s." % pkg try: mod = __import__("%sDataSource%s" % (pkg, ds_coll)) if pkg: @@ -173,15 +179,16 @@ def list_sources(cfg_list, depends, pkglist=[]): raise return(retlist) + # depends is a list of dependencies (DEP_FILESYSTEM) # dslist is a list of 2 item lists -# dslist = [ +# dslist = [ # ( class, ( depends-that-this-class-needs ) ) # } # it returns a list of 'class' that matched these deps exactly # it is a helper function for DataSourceCollections def list_from_depends(depends, dslist): - retlist = [ ] + retlist = [] depset = set(depends) for elem in dslist: (cls, deps) = elem @@ -197,8 +204,8 @@ def is_ipv4(instr): return False try: - r = filter(lambda x: int(x) < 256 and x > 0, toks) + toks = [x for x in toks if (int(x) < 256 and int(x) > 0)] except: return False - return True + return (len(toks) == 4) diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py index aee10ffa..06635746 100644 --- a/cloudinit/DataSourceEc2.py +++ b/cloudinit/DataSourceEc2.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Hafliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -16,42 +18,44 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import DataSource +import cloudinit.DataSource as DataSource -from cloudinit import seeddir, log +from cloudinit import seeddir as base_seeddir +from cloudinit import log import cloudinit.util as util import socket import urllib2 import time -import sys import boto.utils as boto_utils import os.path -import errno -import urlparse + class DataSourceEc2(DataSource.DataSource): - api_ver = '2009-04-04' - seeddir = seeddir + '/ec2' + api_ver = '2009-04-04' + seeddir = base_seeddir + '/ec2' metadata_address = "http://169.254.169.254" def __str__(self): return("DataSourceEc2") def get_data(self): - seedret={ } - if util.read_optional_seed(seedret,base=self.seeddir+ "/"): + seedret = {} + if util.read_optional_seed(seedret, base=self.seeddir + "/"): self.userdata_raw = seedret['user-data'] self.metadata = seedret['meta-data'] log.debug("using seeded ec2 data in %s" % self.seeddir) return True - + try: if not self.wait_for_metadata_service(): return False start = time.time() - self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, None, self.metadata_address) - self.metadata = boto_utils.get_instance_metadata(self.api_ver, self.metadata_address) - log.debug("crawl of metadata service took %ds" % (time.time()-start)) + self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, + None, self.metadata_address) + self.metadata = boto_utils.get_instance_metadata(self.api_ver, + self.metadata_address) + log.debug("crawl of metadata service took %ds" % (time.time() - + start)) return True except Exception as e: print e @@ -66,7 +70,7 @@ class DataSourceEc2(DataSource.DataSource): def get_local_mirror(self): return(self.get_mirror_from_availability_zone()) - def get_mirror_from_availability_zone(self, availability_zone = None): + def get_mirror_from_availability_zone(self, availability_zone=None): # availability is like 'us-west-1b' or 'eu-west-1a' if availability_zone == None: availability_zone = self.get_availability_zone() @@ -77,7 +81,7 @@ class DataSourceEc2(DataSource.DataSource): return fallback try: - host="%s.ec2.archive.ubuntu.com" % availability_zone[:-1] + host = "%s.ec2.archive.ubuntu.com" % availability_zone[:-1] socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM) return 'http://%s/ubuntu/' % host except: @@ -87,12 +91,12 @@ class DataSourceEc2(DataSource.DataSource): mcfg = self.ds_cfg if not hasattr(mcfg, "get"): - mcfg = {} + mcfg = {} max_wait = 120 try: - max_wait = int(mcfg.get("max_wait",max_wait)) - except Exception as e: + max_wait = int(mcfg.get("max_wait", max_wait)) + except Exception: util.logexc(log) log.warn("Failed to get max wait. using %s" % max_wait) @@ -101,8 +105,8 @@ class DataSourceEc2(DataSource.DataSource): timeout = 50 try: - timeout = int(mcfg.get("timeout",timeout)) - except Exception as e: + timeout = int(mcfg.get("timeout", timeout)) + except Exception: util.logexc(log) log.warn("Failed to get timeout, using %s" % timeout) @@ -122,8 +126,8 @@ class DataSourceEc2(DataSource.DataSource): log.warn("Empty metadata url list! using default list") mdurls = def_mdurls - urls = [ ] - url2base = { False: False } + urls = [] + url2base = {False: False} for url in mdurls: cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver) urls.append(cur) @@ -137,7 +141,7 @@ class DataSourceEc2(DataSource.DataSource): log.debug("Using metadata source: '%s'" % url2base[url]) else: log.critical("giving up on md after %i seconds\n" % - int(time.time()-starttime)) + int(time.time() - starttime)) self.metadata_address = url2base[url] return (bool(url)) @@ -146,7 +150,7 @@ class DataSourceEc2(DataSource.DataSource): # consult metadata service, that has # ephemeral0: sdb # and return 'sdb' for input 'ephemeral0' - if not self.metadata.has_key('block-device-mapping'): + if 'block-device-mapping' not in self.metadata: return(None) found = None @@ -166,22 +170,23 @@ class DataSourceEc2(DataSource.DataSource): # when the kernel named them 'vda' or 'xvda' # we want to return the correct value for what will actually # exist in this instance - mappings = { "sd": ("vd", "xvd") } + mappings = {"sd": ("vd", "xvd")} ofound = found short = os.path.basename(found) - + if not found.startswith("/"): - found="/dev/%s" % found + found = "/dev/%s" % found if os.path.exists(found): return(found) for nfrom, tlist in mappings.items(): - if not short.startswith(nfrom): continue + if not short.startswith(nfrom): + continue for nto in tlist: cand = "/dev/%s%s" % (nto, short[len(nfrom):]) if os.path.exists(cand): - log.debug("remapped device name %s => %s" % (found,cand)) + log.debug("remapped device name %s => %s" % (found, cand)) return(cand) # on t1.micro, ephemeral0 will appear in block-device-mapping from @@ -195,14 +200,16 @@ class DataSourceEc2(DataSource.DataSource): def is_vpc(self): # per comment in LP: #615545 - ph="public-hostname"; p4="public-ipv4" + ph = "public-hostname" + p4 = "public-ipv4" if ((ph not in self.metadata or self.metadata[ph] == "") and (p4 not in self.metadata or self.metadata[p4] == "")): return True return False -def wait_for_metadata_service(urls, max_wait=None, timeout=None, status_cb=None): +def wait_for_metadata_service(urls, max_wait=None, timeout=None, + status_cb=None): """ urls: a list of urls to try max_wait: roughly the maximum time to wait before giving up @@ -210,7 +217,7 @@ def wait_for_metadata_service(urls, max_wait=None, timeout=None, status_cb=None) be tried once and given the timeout provided. timeout: the timeout provided to urllib2.urlopen status_cb: call method with string message when a url is not available - + the idea of this routine is to wait for the EC2 metdata service to come up. On both Eucalyptus and EC2 we have seen the case where the instance hit the MD before the MD service was up. EC2 seems @@ -229,18 +236,20 @@ def wait_for_metadata_service(urls, max_wait=None, timeout=None, status_cb=None) starttime = time.time() sleeptime = 1 - timeout_orig = timeout + + def nullstatus_cb(msg): + return if status_cb == None: - def status_cb(msg): return + status_cb = nullstatus_cb def timeup(max_wait, starttime): return((max_wait <= 0 or max_wait == None) or - (time.time()-starttime > max_wait)) + (time.time() - starttime > max_wait)) loop_n = 0 while True: - sleeptime=int(loop_n/5)+1 + sleeptime = int(loop_n / 5) + 1 for url in urls: now = time.time() if loop_n != 0: @@ -268,7 +277,8 @@ def wait_for_metadata_service(urls, max_wait=None, timeout=None, status_cb=None) if log: status_cb("'%s' failed [%s/%ss]: %s" % - (url, int(time.time()-starttime), max_wait, reason)) + (url, int(time.time() - starttime), max_wait, + reason)) if timeup(max_wait, starttime): break @@ -279,10 +289,11 @@ def wait_for_metadata_service(urls, max_wait=None, timeout=None, status_cb=None) return False -datasources = [ - ( DataSourceEc2, ( DataSource.DEP_FILESYSTEM , DataSource.DEP_NETWORK ) ), +datasources = [ + (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), ] + # return a list of data sources that match this set of dependencies def get_datasource_list(depends): return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/DataSourceNoCloud.py b/cloudinit/DataSourceNoCloud.py index 956b1a5e..fa64f2e5 100644 --- a/cloudinit/DataSourceNoCloud.py +++ b/cloudinit/DataSourceNoCloud.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Hafliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -16,36 +18,34 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import DataSource +import cloudinit.DataSource as DataSource -from cloudinit import seeddir, log +from cloudinit import seeddir as base_seeddir +from cloudinit import log import cloudinit.util as util -import sys -import os.path -import os -import errno + class DataSourceNoCloud(DataSource.DataSource): metadata = None userdata = None userdata_raw = None - supported_seed_starts = ( "/" , "file://" ) + supported_seed_starts = ("/", "file://") seed = None cmdline_id = "ds=nocloud" - seeddir = seeddir + '/nocloud' + seeddir = base_seeddir + '/nocloud' def __str__(self): - mstr="DataSourceNoCloud" + mstr = "DataSourceNoCloud" mstr = mstr + " [seed=%s]" % self.seed return(mstr) def get_data(self): - defaults = { - "instance-id" : "nocloud" + defaults = { + "instance-id": "nocloud" } - found = [ ] - md = { } + found = [] + md = {} ud = "" try: @@ -57,9 +57,9 @@ class DataSourceNoCloud(DataSource.DataSource): return False # check to see if the seeddir has data. - seedret={ } - if util.read_optional_seed(seedret,base=self.seeddir + "/"): - md = util.mergedict(md,seedret['meta-data']) + seedret = {} + if util.read_optional_seed(seedret, base=self.seeddir + "/"): + md = util.mergedict(md, seedret['meta-data']) ud = seedret['user-data'] found.append(self.seeddir) log.debug("using seeded cache data in %s" % self.seeddir) @@ -76,7 +76,7 @@ class DataSourceNoCloud(DataSource.DataSource): seedfound = False for proto in self.supported_seed_starts: if seedfrom.startswith(proto): - seedfound=proto + seedfound = proto break if not seedfound: log.debug("seed from %s not supported by %s" % @@ -85,69 +85,75 @@ class DataSourceNoCloud(DataSource.DataSource): # this could throw errors, but the user told us to do it # so if errors are raised, let them raise - (md_seed,ud) = util.read_seeded(seedfrom, timeout=None) + (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) log.debug("using seeded cache data from %s" % seedfrom) # values in the command line override those from the seed - md = util.mergedict(md,md_seed) + md = util.mergedict(md, md_seed) found.append(seedfrom) - md = util.mergedict(md,defaults) + md = util.mergedict(md, defaults) self.seed = ",".join(found) - self.metadata = md; + self.metadata = md self.userdata_raw = ud return True + # returns true or false indicating if cmdline indicated # that this module should be used # example cmdline: # root=LABEL=uec-rootfs ro ds=nocloud -def parse_cmdline_data(ds_id,fill,cmdline=None): +def parse_cmdline_data(ds_id, fill, cmdline=None): if cmdline is None: cmdline = util.get_cmdline() cmdline = " %s " % cmdline - if not ( " %s " % ds_id in cmdline or " %s;" % ds_id in cmdline ): + if not (" %s " % ds_id in cmdline or " %s;" % ds_id in cmdline): return False - argline="" + argline = "" # cmdline can contain: # ds=nocloud[;key=val;key=val] for tok in cmdline.split(): - if tok.startswith(ds_id): argline=tok.split("=",1) - + if tok.startswith(ds_id): + argline = tok.split("=", 1) + # argline array is now 'nocloud' followed optionally by # a ';' and then key=value pairs also terminated with ';' - tmp=argline[1].split(";") + tmp = argline[1].split(";") if len(tmp) > 1: - kvpairs=tmp[1:] + kvpairs = tmp[1:] else: - kvpairs=() + kvpairs = () # short2long mapping to save cmdline typing - s2l = { "h" : "local-hostname", "i" : "instance-id", "s" : "seedfrom" } + s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"} for item in kvpairs: try: - (k,v) = item.split("=",1) + (k, v) = item.split("=", 1) except: - k=item - v=None - if k in s2l: k=s2l[k] - fill[k]=v + k = item + v = None + if k in s2l: + k = s2l[k] + fill[k] = v return(True) + class DataSourceNoCloudNet(DataSourceNoCloud): cmdline_id = "ds=nocloud-net" - supported_seed_starts = ( "http://", "https://", "ftp://" ) - seeddir = seeddir + '/nocloud-net' + supported_seed_starts = ("http://", "https://", "ftp://") + seeddir = base_seeddir + '/nocloud-net' + datasources = ( - ( DataSourceNoCloud, ( DataSource.DEP_FILESYSTEM, ) ), - ( DataSourceNoCloudNet, - ( DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK ) ), + (DataSourceNoCloud, (DataSource.DEP_FILESYSTEM, )), + (DataSourceNoCloudNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), ) + # return a list of data sources that match this set of dependencies def get_datasource_list(depends): return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/DataSourceOVF.py b/cloudinit/DataSourceOVF.py index a9089d70..1f2b622e 100644 --- a/cloudinit/DataSourceOVF.py +++ b/cloudinit/DataSourceOVF.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Hafliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -16,58 +18,59 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import DataSource +import cloudinit.DataSource as DataSource -from cloudinit import seeddir, log +from cloudinit import seeddir as base_seeddir +from cloudinit import log import cloudinit.util as util -import sys import os.path import os -import errno from xml.dom import minidom -from xml.dom import Node import base64 import re import tempfile import subprocess + class DataSourceOVF(DataSource.DataSource): seed = None - seeddir = seeddir + '/ovf' + seeddir = base_seeddir + '/ovf' environment = None - cfg = { } + cfg = {} userdata_raw = None metadata = None - supported_seed_starts = ( "/" , "file://" ) + supported_seed_starts = ("/", "file://") def __str__(self): - mstr="DataSourceOVF" + mstr = "DataSourceOVF" mstr = mstr + " [seed=%s]" % self.seed return(mstr) def get_data(self): - found = [ ] - md = { } + found = [] + md = {} ud = "" - defaults = { - "instance-id" : "iid-dsovf" + defaults = { + "instance-id": "iid-dsovf" } - (seedfile, contents) = get_ovf_env(seeddir) + (seedfile, contents) = get_ovf_env(base_seeddir) if seedfile: # found a seed dir - seed = "%s/%s" % (seeddir,seedfile) + seed = "%s/%s" % (base_seeddir, seedfile) (md, ud, cfg) = read_ovf_environment(contents) self.environment = contents found.append(seed) else: - np = { 'iso' : transport_iso9660, - 'vmware-guestd' : transport_vmware_guestd, } + np = {'iso': transport_iso9660, + 'vmware-guestd': transport_vmware_guestd, } + name = None for name, transfunc in np.iteritems(): - (contents, dev, fname) = transfunc() - if contents: break + (contents, _dev, _fname) = transfunc() + if contents: + break if contents: (md, ud, cfg) = read_ovf_environment(contents) @@ -90,45 +93,48 @@ class DataSourceOVF(DataSource.DataSource): (seedfrom, self.__class__)) return False - (md_seed,ud) = util.read_seeded(seedfrom, timeout=None) + (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) log.debug("using seeded cache data from %s" % seedfrom) - md = util.mergedict(md,md_seed) + md = util.mergedict(md, md_seed) found.append(seedfrom) - - md = util.mergedict(md,defaults) + md = util.mergedict(md, defaults) self.seed = ",".join(found) - self.metadata = md; + self.metadata = md self.userdata_raw = ud self.cfg = cfg return True def get_public_ssh_keys(self): - if not 'public-keys' in self.metadata: return([]) - return([self.metadata['public-keys'],]) - + if not 'public-keys' in self.metadata: + return([]) + return([self.metadata['public-keys'], ]) + # the data sources' config_obj is a cloud-config formated # object that came to it from ways other than cloud-config # because cloud-config content would be handled elsewhere def get_config_obj(self): return(self.cfg) + class DataSourceOVFNet(DataSourceOVF): - seeddir = seeddir + '/ovf-net' - supported_seed_starts = ( "http://", "https://", "ftp://" ) + seeddir = base_seeddir + '/ovf-net' + supported_seed_starts = ("http://", "https://", "ftp://") + # this will return a dict with some content # meta-data, user-data def read_ovf_environment(contents): props = getProperties(contents) - md = { } - cfg = { } + md = {} + cfg = {} ud = "" - cfg_props = [ 'password', ] - md_props = [ 'seedfrom', 'local-hostname', 'public-keys', 'instance-id' ] + cfg_props = ['password', ] + md_props = ['seedfrom', 'local-hostname', 'public-keys', 'instance-id'] for prop, val in props.iteritems(): - if prop == 'hostname': prop = "local-hostname" + if prop == 'hostname': + prop = "local-hostname" if prop in md_props: md[prop] = val elif prop in cfg_props: @@ -139,31 +145,32 @@ def read_ovf_environment(contents): except: ud = val return(md, ud, cfg) - + # returns tuple of filename (in 'dirname', and the contents of the file) # on "not found", returns 'None' for filename and False for contents def get_ovf_env(dirname): - env_names = ("ovf-env.xml", "ovf_env.xml", "OVF_ENV.XML", "OVF-ENV.XML" ) + env_names = ("ovf-env.xml", "ovf_env.xml", "OVF_ENV.XML", "OVF-ENV.XML") for fname in env_names: - if os.path.isfile("%s/%s" % (dirname,fname)): - fp = open("%s/%s" % (dirname,fname)) + if os.path.isfile("%s/%s" % (dirname, fname)): + fp = open("%s/%s" % (dirname, fname)) contents = fp.read() fp.close() - return(fname,contents) - return(None,False) + return(fname, contents) + return(None, False) + # transport functions take no input and return # a 3 tuple of content, path, filename def transport_iso9660(require_iso=False): - # default_regex matches values in + # default_regex matches values in # /lib/udev/rules.d/60-cdrom_id.rules # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end" envname = "CLOUD_INIT_CDROM_DEV_REGEX" default_regex = "^(sr[0-9]+|hd[a-z]|xvd.*)" - devname_regex = os.environ.get(envname,default_regex) + devname_regex = os.environ.get(envname, default_regex) cdmatch = re.compile(devname_regex) # go through mounts to see if it was already mounted @@ -171,19 +178,20 @@ def transport_iso9660(require_iso=False): mounts = fp.readlines() fp.close() - mounted = { } + mounted = {} for mpline in mounts: - (dev,mp,fstype,opts,freq,passno) = mpline.split() - mounted[dev]=(dev,fstype,mp,False) - mp = mp.replace("\\040"," ") - if fstype != "iso9660" and require_iso: continue + (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() + mounted[dev] = (dev, fstype, mp, False) + mp = mp.replace("\\040", " ") + if fstype != "iso9660" and require_iso: + continue - if cdmatch.match(dev[5:]) == None: # take off '/dev/' + if cdmatch.match(dev[5:]) == None: # take off '/dev/' continue - - (fname,contents) = get_ovf_env(mp) + + (fname, contents) = get_ovf_env(mp) if contents is not False: - return(contents,dev,fname) + return(contents, dev, fname) tmpd = None dvnull = None @@ -203,7 +211,8 @@ def transport_iso9660(require_iso=False): fp.read(512) fp.close() except: - if fp: fp.close() + if fp: + fp.close() continue if tmpd is None: @@ -214,20 +223,21 @@ def transport_iso9660(require_iso=False): except: pass - cmd = [ "mount", "-o", "ro", fullp, tmpd ] - if require_iso: cmd.extend(('-t','iso9660')) + cmd = ["mount", "-o", "ro", fullp, tmpd] + if require_iso: + cmd.extend(('-t', 'iso9660')) rc = subprocess.call(cmd, stderr=dvnull, stdout=dvnull, stdin=dvnull) if rc: continue - (fname,contents) = get_ovf_env(tmpd) + (fname, contents) = get_ovf_env(tmpd) subprocess.call(["umount", tmpd]) if contents is not False: os.rmdir(tmpd) - return(contents,fullp,fname) + return(contents, fullp, fname) if tmpd: os.rmdir(tmpd) @@ -237,11 +247,13 @@ def transport_iso9660(require_iso=False): return(False, None, None) + def transport_vmware_guestd(): - # http://blogs.vmware.com/vapp/2009/07/selfconfiguration-and-the-ovf-environment.html + # http://blogs.vmware.com/vapp/2009/07/ \ + # selfconfiguration-and-the-ovf-environment.html # try: # cmd = ['vmware-guestd', '--cmd', 'info-get guestinfo.ovfEnv'] - # (out,err) = subp(cmd) + # (out, err) = subp(cmd) # return(out, 'guestinfo.ovfEnv', 'vmware-guestd') # except: # # would need to error check here and see why this failed @@ -250,13 +262,16 @@ def transport_vmware_guestd(): return(False, None, None) -def findChild(node,filter_func): +def findChild(node, filter_func): ret = [] - if not node.hasChildNodes(): return ret + if not node.hasChildNodes(): + return ret for child in node.childNodes: - if filter_func(child): ret.append(child) + if filter_func(child): + ret.append(child) return(ret) + def getProperties(environString): dom = minidom.parseString(environString) if dom.documentElement.localName != "Environment": @@ -267,7 +282,7 @@ def getProperties(environString): envNsURI = "http://schemas.dmtf.org/ovf/environment/1" - # could also check here that elem.namespaceURI == + # could also check here that elem.namespaceURI == # "http://schemas.dmtf.org/ovf/environment/1" propSections = findChild(dom.documentElement, lambda n: n.localName == "PropertySection") @@ -275,37 +290,43 @@ def getProperties(environString): if len(propSections) == 0: raise Exception("No 'PropertySection's") - props = { } + props = {} propElems = findChild(propSections[0], lambda n: n.localName == "Property") for elem in propElems: - key = elem.attributes.getNamedItemNS(envNsURI,"key").value - val = elem.attributes.getNamedItemNS(envNsURI,"value").value + key = elem.attributes.getNamedItemNS(envNsURI, "key").value + val = elem.attributes.getNamedItemNS(envNsURI, "value").value props[key] = val return(props) + datasources = ( - ( DataSourceOVF, ( DataSource.DEP_FILESYSTEM, ) ), - ( DataSourceOVFNet, - ( DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK ) ), + (DataSourceOVF, (DataSource.DEP_FILESYSTEM, )), + (DataSourceOVFNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), ) + # return a list of data sources that match this set of dependencies def get_datasource_list(depends): return(DataSource.list_from_depends(depends, datasources)) + if __name__ == "__main__": - import sys - envStr = open(sys.argv[1]).read() - props = getProperties(envStr) - import pprint - pprint.pprint(props) - - md, ud, cfg = read_ovf_environment(envStr) - print "=== md ===" - pprint.pprint(md) - print "=== ud ===" - pprint.pprint(ud) - print "=== cfg ===" - pprint.pprint(cfg) + def main(): + import sys + envStr = open(sys.argv[1]).read() + props = getProperties(envStr) + import pprint + pprint.pprint(props) + + md, ud, cfg = read_ovf_environment(envStr) + print "=== md ===" + pprint.pprint(md) + print "=== ud ===" + pprint.pprint(ud) + print "=== cfg ===" + pprint.pprint(cfg) + + main() diff --git a/cloudinit/SshUtil.py b/cloudinit/SshUtil.py index fdd3bb27..22f4f2fc 100644 --- a/cloudinit/SshUtil.py +++ b/cloudinit/SshUtil.py @@ -1,9 +1,29 @@ #!/usr/bin/python +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Hafliger <juerg.haefliger@hp.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. import os import os.path import cloudinit.util as util + class AuthKeyEntry(): # lines are options, keytype, base64-encoded key, comment # man page says the following which I did not understand: @@ -18,13 +38,13 @@ class AuthKeyEntry(): line_in = "" def __init__(self, line, def_opt=None): - line=line.rstrip("\n\r") + line = line.rstrip("\n\r") self.line_in = line if line.startswith("#") or line.strip() == "": self.is_comment = True else: ent = line.strip() - toks = ent.split(None,3) + toks = ent.split(None, 3) if len(toks) == 1: self.base64 = toks[0] elif len(toks) == 2: @@ -37,7 +57,7 @@ class AuthKeyEntry(): quoted = False # taken from auth_rsa_key_allowed in auth-rsa.c try: - while (i < len(ent) and + while (i < len(ent) and ((quoted) or (ent[i] not in (" ", "\t")))): curc = ent[i] nextc = ent[i + 1] @@ -46,15 +66,15 @@ class AuthKeyEntry(): elif curc == '"': quoted = not quoted i = i + 1 - except IndexError as e: + except IndexError: self.is_comment = True - return() + return try: self.options = ent[0:i] (self.keytype, self.base64, self.comment) = \ - ent[i+1:].split(None,3) - except ValueError as e: + ent[i + 1:].split(None, 3) + except ValueError: # we did not understand this line self.is_comment = True @@ -64,30 +84,33 @@ class AuthKeyEntry(): return def debug(self): - print("line_in=%s\ncomment: %s\noptions=%s\nkeytype=%s\nbase64=%s\ncomment=%s\n" % - (self.line_in, self.is_comment, self.options, self.keytype, self.base64, self.comment)), + print("line_in=%s\ncomment: %s\noptions=%s\nkeytype=%s\nbase64=%s\n" + "comment=%s\n" % (self.line_in, self.is_comment, self.options, + self.keytype, self.base64, self.comment)), + def __repr__(self): if self.is_comment: return(self.line_in) else: - toks = [ ] + toks = [] for e in (self.options, self.keytype, self.base64, self.comment): if e: toks.append(e) - + return(' '.join(toks)) - + + def update_authorized_keys(fname, keys): # keys is a list of AuthKeyEntries # key_prefix is the prefix (options) to prepend try: fp = open(fname, "r") - lines = fp.readlines() # lines have carriage return + lines = fp.readlines() # lines have carriage return fp.close() - except IOError as e: - lines = [ ] + except IOError: + lines = [] - ka_stats = { } # keys_added status + ka_stats = {} # keys_added status for k in keys: ka_stats[k] = False @@ -95,7 +118,7 @@ def update_authorized_keys(fname, keys): for key in keys: to_add.append(key) - for i in range(0,len(lines)): + for i in range(0, len(lines)): ent = AuthKeyEntry(lines[i]) for k in keys: if k.base64 == ent.base64 and not k.is_comment: @@ -115,7 +138,7 @@ def update_authorized_keys(fname, keys): else: return('\n'.join(lines) + "\n") - + def setup_user_keys(keys, user, key_prefix, log=None): import pwd saved_umask = os.umask(077) @@ -129,11 +152,11 @@ def setup_user_keys(keys, user, key_prefix, log=None): try: ssh_cfg = parse_ssh_config() - akeys = ssh_cfg.get("AuthorizedKeysFile","%h/.ssh/authorized_keys") + akeys = ssh_cfg.get("AuthorizedKeysFile", "%h/.ssh/authorized_keys") akeys = akeys.replace("%h", pwent.pw_dir) akeys = akeys.replace("%u", user) authorized_keys = akeys - except Exception as e: + except Exception: authorized_keys = '%s/.ssh/authorized_keys' % pwent.pw_dir if log: util.logexc(log) @@ -151,46 +174,52 @@ def setup_user_keys(keys, user, key_prefix, log=None): os.umask(saved_umask) -if __name__ == "__main__": - import sys - # usage: orig_file, new_keys, [key_prefix] - # prints out merged, where 'new_keys' will trump old - ## example - ## ### begin authorized_keys ### - # ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEA28CDAGtxSucHezSKqwh1wAs39xdeZTSVmmyMcKDI5Njnd1d/Uhgj/awxP0Whep8eRSm6F+Xgwi0pH1KNPCszPvq+03K+yi3YkYkQIkVBhctK6AP/UmlVQTVmjJdEvgtrppFTjCzf16q0BT0mXX5YFV3csgm8cJn7UveKHkYjJp8= smoser-work - # ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA3I7VUf2l5gSn5uavROsc5HRDpZdQueUq5ozemNSj8T7enqKHOEaFoU2VoPgGEWC9RyzSQVeyD6s7APMcE82EtmW4skVEgEGSbDc1pvxzxtchBj78hJP6Cf5TCMFSXw+Fz5rF1dR23QDbN1mkHs7adr8GW4kSWqU7Q7NDwfIrJJtO7Hi42GyXtvEONHbiRPOe8stqUly7MvUoN+5kfjBM8Qqpfl2+FNhTYWpMfYdPUnE7u536WqzFmsaqJctz3gBxH9Ex7dFtrxR4qiqEr9Qtlu3xGn7Bw07/+i1D+ey3ONkZLN+LQ714cgj8fRS4Hj29SCmXp5Kt5/82cD/VN3NtHw== smoser@brickies - # ### end authorized_keys ### - # - # ### begin new_keys ### - # ssh-rsa nonmatch smoser@newhost - # ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEA28CDAGtxSucHezSKqwh1wAs39xdeZTSVmmyMcKDI5Njnd1d/Uhgj/awxP0Whep8eRSm6F+Xgwi0pH1KNPCszPvq+03K+yi3YkYkQIkVBhctK6AP/UmlVQTVmjJdEvgtrppFTjCzf16q0BT0mXX5YFV3csgm8cJn7UveKHkYjJp8= new_comment - # ### end new_keys ### - # - # Then run as: - # program authorized_keys new_keys 'no-port-forwarding,command=\"echo hi world;\"' - def_prefix = None - orig_key_file = sys.argv[1] - new_key_file = sys.argv[2] - if len(sys.argv) > 3: - def_prefix = sys.argv[3] - fp = open(new_key_file) - - newkeys = [ ] - for line in fp.readlines(): - newkeys.append(AuthKeyEntry(line, def_prefix)) - - fp.close() - print update_authorized_keys(orig_key_file, newkeys) def parse_ssh_config(fname="/etc/ssh/sshd_config"): - ret = { } - fp=open(fname) + ret = {} + fp = open(fname) for l in fp.readlines(): l = l.strip() if not l or l.startswith("#"): continue - key,val = l.split(None,1) - ret[key]=val + key, val = l.split(None, 1) + ret[key] = val fp.close() return(ret) +if __name__ == "__main__": + def main(): + import sys + # usage: orig_file, new_keys, [key_prefix] + # prints out merged, where 'new_keys' will trump old + ## example + ## ### begin auth_keys ### + # ssh-rsa AAAAB3NzaC1xxxxxxxxxV3csgm8cJn7UveKHkYjJp8= smoser-work + # ssh-rsa AAAAB3NzaC1xxxxxxxxxCmXp5Kt5/82cD/VN3NtHw== smoser@brickies + # ### end authorized_keys ### + # + # ### begin new_keys ### + # ssh-rsa nonmatch smoser@newhost + # ssh-rsa AAAAB3NzaC1xxxxxxxxxV3csgm8cJn7UveKHkYjJp8= new_comment + # ### end new_keys ### + # + # Then run as: + # program auth_keys new_keys \ + # 'no-port-forwarding,command=\"echo hi world;\"' + def_prefix = None + orig_key_file = sys.argv[1] + new_key_file = sys.argv[2] + if len(sys.argv) > 3: + def_prefix = sys.argv[3] + fp = open(new_key_file) + + newkeys = [] + for line in fp.readlines(): + newkeys.append(AuthKeyEntry(line, def_prefix)) + + fp.close() + print update_authorized_keys(orig_key_file, newkeys) + + main() + +# vi: ts=4 expandtab diff --git a/cloudinit/UserDataHandler.py b/cloudinit/UserDataHandler.py index 541ee87a..93d1d36a 100644 --- a/cloudinit/UserDataHandler.py +++ b/cloudinit/UserDataHandler.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Hafliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,39 +17,41 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import email from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.mime.base import MIMEBase -from email import encoders import yaml import cloudinit import cloudinit.util as util import hashlib -import os import urllib -starts_with_mappings={ - '#include' : 'text/x-include-url', - '#include-once' : 'text/x-include-once-url', - '#!' : 'text/x-shellscript', - '#cloud-config' : 'text/cloud-config', - '#upstart-job' : 'text/upstart-job', - '#part-handler' : 'text/part-handler', - '#cloud-boothook' : 'text/cloud-boothook', - '#cloud-config-archive' : 'text/cloud-config-archive', + +starts_with_mappings = { + '#include': 'text/x-include-url', + '#include-once': 'text/x-include-once-url', + '#!': 'text/x-shellscript', + '#cloud-config': 'text/cloud-config', + '#upstart-job': 'text/upstart-job', + '#part-handler': 'text/part-handler', + '#cloud-boothook': 'text/cloud-boothook', + '#cloud-config-archive': 'text/cloud-config-archive', } -# if 'str' is compressed return decompressed otherwise return it -def decomp_str(str): + +# if 'string' is compressed return decompressed otherwise return it +def decomp_str(string): import StringIO import gzip try: - uncomp = gzip.GzipFile(None,"rb",1,StringIO.StringIO(str)).read() + uncomp = gzip.GzipFile(None, "rb", 1, StringIO.StringIO(string)).read() return(uncomp) except: - return(str) + return(string) + def do_include(content, appendmsg): import os @@ -55,7 +59,8 @@ def do_include(content, appendmsg): # also support '#include <url here>' includeonce = False for line in content.splitlines(): - if line == "#include": continue + if line == "#include": + continue if line == "#include-once": includeonce = True continue @@ -64,10 +69,11 @@ def do_include(content, appendmsg): includeonce = True elif line.startswith("#include"): line = line[len("#include"):].lstrip() - if line.startswith("#"): continue + if line.startswith("#"): + continue # urls cannot not have leading or trailing white space - msum = hashlib.md5() + msum = hashlib.md5() # pylint: disable=E1101 msum.update(line.strip()) includeonce_filename = "%s/urlcache/%s" % ( cloudinit.get_ipath_cur("data"), msum.hexdigest()) @@ -79,7 +85,7 @@ def do_include(content, appendmsg): content = urllib.urlopen(line).read() if includeonce: util.write_file(includeonce_filename, content, mode=0600) - except Exception as e: + except Exception: raise process_includes(message_from_string(decomp_str(content)), appendmsg) @@ -88,14 +94,14 @@ def do_include(content, appendmsg): def explode_cc_archive(archive, appendmsg): for ent in yaml.load(archive): # ent can be one of: - # dict { 'filename' : 'value' , 'content' : 'value', 'type' : 'value' } + # dict { 'filename' : 'value', 'content' : 'value', 'type' : 'value' } # filename and type not be present # or # scalar(payload) - + def_type = "text/cloud-config" - if isinstance(ent,str): - ent = { 'content': ent } + if isinstance(ent, str): + ent = {'content': ent} content = ent.get('content', '') mtype = ent.get('type', None) @@ -118,7 +124,7 @@ def explode_cc_archive(archive, appendmsg): continue msg.add_header(header, ent['header']) - _attach_part(appendmsg,msg) + _attach_part(appendmsg, msg) def multi_part_count(outermsg, newcount=None): @@ -135,6 +141,7 @@ def multi_part_count(outermsg, newcount=None): return(int(outermsg.get('Number-Attachments', 0))) + def _attach_part(outermsg, part): """ Attach an part to an outer message. outermsg must be a MIMEMultipart. @@ -143,18 +150,20 @@ def _attach_part(outermsg, part): cur = multi_part_count(outermsg) if not part.get_filename(None): part.add_header('Content-Disposition', 'attachment', - filename = 'part-%03d' % (cur+1)) + filename='part-%03d' % (cur + 1)) outermsg.attach(part) - multi_part_count(outermsg, cur+1) - + multi_part_count(outermsg, cur + 1) + + def type_from_startswith(payload, default=None): # slist is sorted longest first - slist = sorted(starts_with_mappings.keys(), key=lambda e: 0-len(e)) + slist = sorted(starts_with_mappings.keys(), key=lambda e: 0 - len(e)) for sstr in slist: if payload.startswith(sstr): return(starts_with_mappings[sstr]) return default + def process_includes(msg, appendmsg=None): if appendmsg == None: appendmsg = MIMEMultipart() @@ -190,32 +199,36 @@ def process_includes(msg, appendmsg=None): _attach_part(appendmsg, part) -def message_from_string(data, headers={}): + +def message_from_string(data, headers=None): + if headers is None: + headers = {} if "mime-version:" in data[0:4096].lower(): - was_mime = True msg = email.message_from_string(data) - for (key,val) in headers.items(): + for (key, val) in headers.items(): if key in msg: - msg.replace_header(key,val) + msg.replace_header(key, val) else: msg[key] = val else: - was_mime = False - mtype = headers.get("Content-Type","text/plain") + mtype = headers.get("Content-Type", "text/plain") maintype, subtype = mtype.split("/", 1) msg = MIMEBase(maintype, subtype, *headers) msg.set_payload(data) return(msg) + # this is heavily wasteful, reads through userdata string input def preprocess_userdata(data): newmsg = MIMEMultipart() process_includes(message_from_string(decomp_str(data)), newmsg) return(newmsg.as_string()) -# callback is a function that will be called with (data, content_type, filename, payload) -def walk_userdata(istr, callback, data = None): + +# callback is a function that will be called with (data, content_type, +# filename, payload) +def walk_userdata(istr, callback, data=None): partnum = 0 for part in message_from_string(istr).walk(): # multipart/* are just containers @@ -232,12 +245,16 @@ def walk_userdata(istr, callback, data = None): callback(data, ctype, filename, part.get_payload(decode=True)) - partnum = partnum+1 + partnum = partnum + 1 + if __name__ == "__main__": - import sys - data = decomp_str(file(sys.argv[1]).read()) - newmsg = MIMEMultipart() - process_includes(message_from_string(data), newmsg) - print newmsg - print "#found %s parts" % multi_part_count(newmsg) + def main(): + import sys + data = decomp_str(file(sys.argv[1]).read()) + newmsg = MIMEMultipart() + process_includes(message_from_string(data), newmsg) + print newmsg + print "#found %s parts" % multi_part_count(newmsg) + + main() diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py index d01d443d..7a34e053 100644 --- a/cloudinit/__init__.py +++ b/cloudinit/__init__.py @@ -2,8 +2,10 @@ # # Common code for the EC2 initialisation scripts in Ubuntu # Copyright (C) 2008-2009 Canonical Ltd +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Soren Hansen <soren@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -26,34 +28,33 @@ seeddir = varlibdir + "/seed" cfg_env_name = "CLOUD_CFG" cfg_builtin = """ -log_cfgs: [ ] -datasource_list: [ "NoCloud", "OVF", "Ec2" ] +log_cfgs: [] +datasource_list: ["NoCloud", "OVF", "Ec2"] def_log_file: /var/log/cloud-init.log syslog_fix_perms: syslog:adm """ logger_name = "cloudinit" pathmap = { - "handlers" : "/handlers", - "scripts" : "/scripts", - "sem" : "/sem", - "boothooks" : "/boothooks", - "userdata_raw" : "/user-data.txt", - "userdata" : "/user-data.txt.i", - "obj_pkl" : "/obj.pkl", - "cloud_config" : "/cloud-config.txt", - "data" : "/data", - None : "", + "handlers": "/handlers", + "scripts": "/scripts", + "sem": "/sem", + "boothooks": "/boothooks", + "userdata_raw": "/user-data.txt", + "userdata": "/user-data.txt.i", + "obj_pkl": "/obj.pkl", + "cloud_config": "/cloud-config.txt", + "data": "/data", + None: "", } -per_instance="once-per-instance" -per_always="always" -per_once="once" +per_instance = "once-per-instance" +per_always = "always" +per_once = "once" -parsed_cfgs = { } +parsed_cfgs = {} import os -from configobj import ConfigObj import cPickle import sys @@ -62,32 +63,38 @@ import errno import pwd import subprocess import yaml -import util import logging import logging.config import StringIO import glob import traceback +import cloudinit.util as util + + class NullHandler(logging.Handler): - def emit(self,record): pass + def emit(self, record): + pass + log = logging.getLogger(logger_name) log.addHandler(NullHandler()) + def logging_set_from_cfg_file(cfg_file=system_config): - logging_set_from_cfg(util.get_base_cfg(cfg_file,cfg_builtin,parsed_cfgs)) + logging_set_from_cfg(util.get_base_cfg(cfg_file, cfg_builtin, parsed_cfgs)) + def logging_set_from_cfg(cfg): log_cfgs = [] - logcfg=util.get_cfg_option_str(cfg, "log_cfg", False) + logcfg = util.get_cfg_option_str(cfg, "log_cfg", False) if logcfg: # if there is a 'logcfg' entry in the config, respect # it, it is the old keyname - log_cfgs = [ logcfg ] + log_cfgs = [logcfg] elif "log_cfgs" in cfg: for cfg in cfg['log_cfgs']: - if isinstance(cfg,list): + if isinstance(cfg, list): log_cfgs.append('\n'.join(cfg)) else: log_cfgs.append() @@ -106,38 +113,41 @@ def logging_set_from_cfg(cfg): raise Exception("no valid logging found\n") -import DataSource -import UserDataHandler +import cloudinit.DataSource as DataSource +import cloudinit.UserDataHandler as UserDataHandler + class CloudInit: cfg = None - part_handlers = { } + part_handlers = {} old_conffile = '/etc/ec2-init/ec2-config.cfg' - ds_deps = [ DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK ] + ds_deps = [DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK] datasource = None + cloud_config_str = '' + datasource_name = '' - builtin_handlers = [ ] + builtin_handlers = [] - def __init__(self, ds_deps = None, sysconfig=system_config): + def __init__(self, ds_deps=None, sysconfig=system_config): self.builtin_handlers = [ - [ 'text/x-shellscript', self.handle_user_script, per_always ], - [ 'text/cloud-config', self.handle_cloud_config, per_always ], - [ 'text/upstart-job', self.handle_upstart_job, per_instance ], - [ 'text/cloud-boothook', self.handle_cloud_boothook, per_always ], + ['text/x-shellscript', self.handle_user_script, per_always], + ['text/cloud-config', self.handle_cloud_config, per_always], + ['text/upstart-job', self.handle_upstart_job, per_instance], + ['text/cloud-boothook', self.handle_cloud_boothook, per_always], ] if ds_deps != None: self.ds_deps = ds_deps - self.sysconfig=sysconfig - self.cfg=self.read_cfg() + self.sysconfig = sysconfig + self.cfg = self.read_cfg() def read_cfg(self): if self.cfg: return(self.cfg) try: - conf = util.get_base_cfg(self.sysconfig,cfg_builtin, parsed_cfgs) - except Exception as e: + conf = util.get_base_cfg(self.sysconfig, cfg_builtin, parsed_cfgs) + except Exception: conf = get_builtin_cfg() # support reading the old ConfigObj format file and merging @@ -145,8 +155,9 @@ class CloudInit: try: from configobj import ConfigObj oldcfg = ConfigObj(self.old_conffile) - if oldcfg is None: oldcfg = { } - conf = util.mergedict(conf,oldcfg) + if oldcfg is None: + oldcfg = {} + conf = util.mergedict(conf, oldcfg) except: pass @@ -158,7 +169,7 @@ class CloudInit: # by using the instance link, if purge_cache was called # the file wont exist cache = get_ipath_cur('obj_pkl') - f=open(cache, "rb") + f = open(cache, "rb") data = cPickle.load(f) f.close() self.datasource = data @@ -173,25 +184,27 @@ class CloudInit: except OSError as e: if e.errno != errno.EEXIST: return False - + try: - f=open(cache, "wb") - data = cPickle.dump(self.datasource,f) + f = open(cache, "wb") + cPickle.dump(self.datasource, f) f.close() - os.chmod(cache,0400) + os.chmod(cache, 0400) except: raise - + def get_data_source(self): - if self.datasource is not None: return True + if self.datasource is not None: + return True if self.restore_from_cache(): log.debug("restored from cache type %s" % self.datasource) return True - cfglist=self.cfg['datasource_list'] + cfglist = self.cfg['datasource_list'] dslist = list_sources(cfglist, self.ds_deps) - dsnames = map(lambda f: f.__name__, dslist) + dsnames = [f.__name__ for f in dslist] + log.debug("searching for data source in %s" % dsnames) for cls in dslist: ds = cls.__name__ @@ -203,9 +216,8 @@ class CloudInit: log.debug("found data source %s" % ds) return True except Exception as e: - log.warn("get_data of %s raised %s" % (ds,e)) + log.warn("get_data of %s raised %s" % (ds, e)) util.logexc(log) - pass msg = "Did not find data source. searched classes: %s" % dsnames log.debug(msg) raise DataSourceNotFoundException(msg) @@ -214,18 +226,19 @@ class CloudInit: try: os.unlink(cur_instance_link) except OSError as e: - if e.errno != errno.ENOENT: raise + if e.errno != errno.ENOENT: + raise iid = self.get_instance_id() os.symlink("./instances/%s" % iid, cur_instance_link) idir = self.get_ipath() dlist = [] - for d in [ "handlers", "scripts", "sem" ]: + for d in ["handlers", "scripts", "sem"]: dlist.append("%s/%s" % (idir, d)) - + util.ensure_dirs(dlist) - ds = "%s: %s\n" % ( self.datasource.__class__, str(self.datasource) ) + ds = "%s: %s\n" % (self.datasource.__class__, str(self.datasource)) dp = self.get_cpath('data') util.write_file("%s/%s" % (idir, 'datasource'), ds) util.write_file("%s/%s" % (dp, 'previous-datasource'), ds) @@ -250,66 +263,69 @@ class CloudInit: util.write_file(self.get_ipath('userdata'), self.datasource.get_userdata(), 0600) - def sem_getpath(self,name,freq): + def sem_getpath(self, name, freq): if freq == 'once-per-instance': - return("%s/%s" % (self.get_ipath("sem"),name)) + return("%s/%s" % (self.get_ipath("sem"), name)) return("%s/%s.%s" % (get_cpath("sem"), name, freq)) - - def sem_has_run(self,name,freq): - if freq == per_always: return False - semfile = self.sem_getpath(name,freq) + + def sem_has_run(self, name, freq): + if freq == per_always: + return False + semfile = self.sem_getpath(name, freq) if os.path.exists(semfile): return True return False - - def sem_acquire(self,name,freq): + + def sem_acquire(self, name, freq): from time import time - semfile = self.sem_getpath(name,freq) - + semfile = self.sem_getpath(name, freq) + try: os.makedirs(os.path.dirname(semfile)) except OSError as e: if e.errno != errno.EEXIST: raise e - + if os.path.exists(semfile) and freq != per_always: return False - + # race condition try: - f = open(semfile,"w") + f = open(semfile, "w") f.write("%s\n" % str(time())) f.close() except: return(False) return(True) - - def sem_clear(self,name,freq): - semfile = self.sem_getpath(name,freq) + + def sem_clear(self, name, freq): + semfile = self.sem_getpath(name, freq) try: os.unlink(semfile) except OSError as e: if e.errno != errno.ENOENT: return False - + return True # acquire lock on 'name' for given 'freq' # if that does not exist, then call 'func' with given 'args' # if 'clear_on_fail' is True and func throws an exception # then remove the lock (so it would run again) - def sem_and_run(self,semname,freq,func,args=[],clear_on_fail=False): - if self.sem_has_run(semname,freq): + def sem_and_run(self, semname, freq, func, args=None, clear_on_fail=False): + if args is None: + args = [] + if self.sem_has_run(semname, freq): log.debug("%s already ran %s", semname, freq) return False try: - if not self.sem_acquire(semname,freq): + if not self.sem_acquire(semname, freq): raise Exception("Failed to acquire lock on %s" % semname) func(*args) except: if clear_on_fail: - self.sem_clear(semname,freq) + self.sem_clear(semname, freq) raise return True @@ -317,8 +333,8 @@ class CloudInit: # get_ipath : get the instance path for a name in pathmap # (/var/lib/cloud/instances/<instance>/name)<name>) def get_ipath(self, name=None): - return("%s/instances/%s%s" - % (varlibdir,self.get_instance_id(), pathmap[name])) + return("%s/instances/%s%s" + % (varlibdir, self.get_instance_id(), pathmap[name])) def consume_userdata(self, frequency=per_instance): self.get_userdata() @@ -329,18 +345,20 @@ class CloudInit: # add the path to the plugins dir to the top of our list for import # instance dir should be read before cloud-dir - sys.path.insert(0,cdir) - sys.path.insert(0,idir) + sys.path.insert(0, cdir) + sys.path.insert(0, idir) - part_handlers = { } + part_handlers = {} # add handlers in cdir for fname in glob.glob("%s/*.py" % cdir): - if not os.path.isfile(fname): continue + if not os.path.isfile(fname): + continue modname = os.path.basename(fname)[0:-3] try: mod = __import__(modname) handler_register(mod, part_handlers, data, frequency) - log.debug("added handler for [%s] from %s" % (mod.list_types(), fname)) + log.debug("added handler for [%s] from %s" % (mod.list_types(), + fname)) except: log.warn("failed to initialize handler in %s" % fname) util.logexc(log) @@ -353,44 +371,46 @@ class CloudInit: part_handlers, data, frequency) # walk the data - pdata = { 'handlers': part_handlers, 'handlerdir': idir, - 'data' : data, 'frequency': frequency } + pdata = {'handlers': part_handlers, 'handlerdir': idir, + 'data': data, 'frequency': frequency} UserDataHandler.walk_userdata(self.get_userdata(), - partwalker_callback, data = pdata) + partwalker_callback, data=pdata) # give callbacks opportunity to finalize - called = [ ] - for (mtype, mod) in part_handlers.iteritems(): + called = [] + for (_mtype, mod) in part_handlers.iteritems(): if mod in called: continue handler_call_end(mod, data, frequency) - def handle_user_script(self,data,ctype,filename,payload, frequency): - if ctype == "__end__": return + def handle_user_script(self, _data, ctype, filename, payload, _frequency): + if ctype == "__end__": + return if ctype == "__begin__": # maybe delete existing things here return - filename=filename.replace(os.sep,'_') + filename = filename.replace(os.sep, '_') scriptsdir = get_ipath_cur('scripts') - util.write_file("%s/%s" % - (scriptsdir,filename), util.dos2unix(payload), 0700) + util.write_file("%s/%s" % + (scriptsdir, filename), util.dos2unix(payload), 0700) - def handle_upstart_job(self,data,ctype,filename,payload, frequency): + def handle_upstart_job(self, _data, ctype, filename, payload, frequency): # upstart jobs are only written on the first boot if frequency != per_instance: return - if ctype == "__end__" or ctype == "__begin__": return + if ctype == "__end__" or ctype == "__begin__": + return if not filename.endswith(".conf"): - filename=filename+".conf" + filename = filename + ".conf" - util.write_file("%s/%s" % ("/etc/init",filename), + util.write_file("%s/%s" % ("/etc/init", filename), util.dos2unix(payload), 0644) - def handle_cloud_config(self,data,ctype,filename,payload, frequency): + def handle_cloud_config(self, _data, ctype, filename, payload, _frequency): if ctype == "__begin__": - self.cloud_config_str="" + self.cloud_config_str = "" return if ctype == "__end__": cloud_config = self.get_ipath("cloud_config") @@ -401,37 +421,40 @@ class CloudInit: ## as CloudConfig does that also, merging it with this cfg ## # ccfg = yaml.load(self.cloud_config_str) - # if ccfg is None: ccfg = { } + # if ccfg is None: ccfg = {} # self.cfg = util.mergedict(ccfg, self.cfg) return - self.cloud_config_str+="\n#%s\n%s" % (filename,payload) + self.cloud_config_str += "\n#%s\n%s" % (filename, payload) - def handle_cloud_boothook(self,data,ctype,filename,payload, frequency): - if ctype == "__end__": return - if ctype == "__begin__": return + def handle_cloud_boothook(self, _data, ctype, filename, payload, + _frequency): + if ctype == "__end__": + return + if ctype == "__begin__": + return - filename=filename.replace(os.sep,'_') + filename = filename.replace(os.sep, '_') payload = util.dos2unix(payload) - prefix="#cloud-boothook" + prefix = "#cloud-boothook" start = 0 if payload.startswith(prefix): start = len(prefix) + 1 - + boothooks_dir = self.get_ipath("boothooks") - filepath = "%s/%s" % (boothooks_dir,filename) + filepath = "%s/%s" % (boothooks_dir, filename) util.write_file(filepath, payload[start:], 0700) try: - env=os.environ.copy() - env['INSTANCE_ID']= self.datasource.get_instance_id() - ret = subprocess.check_call([filepath], env=env) + env = os.environ.copy() + env['INSTANCE_ID'] = self.datasource.get_instance_id() + subprocess.check_call([filepath], env=env) except subprocess.CalledProcessError as e: log.error("boothooks script %s returned %i" % - (filepath,e.returncode)) + (filepath, e.returncode)) except Exception as e: log.error("boothooks unknown exception %s when running %s" % - (e,filepath)) + (e, filepath)) def get_public_ssh_keys(self): return(self.datasource.get_public_ssh_keys()) @@ -445,71 +468,84 @@ class CloudInit: def get_hostname(self, fqdn=False): return(self.datasource.get_hostname(fqdn=fqdn)) - def device_name_to_device(self,name): + def device_name_to_device(self, name): return(self.datasource.device_name_to_device(name)) # I really don't know if this should be here or not, but # I needed it in cc_update_hostname, where that code had a valid 'cloud' # reference, but did not have a cloudinit handle # (ie, no cloudinit.get_cpath()) - def get_cpath(self,name=None): + def get_cpath(self, name=None): return(get_cpath(name)) def initfs(): - subds = [ 'scripts/per-instance', 'scripts/per-once', 'scripts/per-boot', - 'seed', 'instances', 'handlers', 'sem', 'data' ] - dlist = [ ] + subds = ['scripts/per-instance', 'scripts/per-once', 'scripts/per-boot', + 'seed', 'instances', 'handlers', 'sem', 'data'] + dlist = [] for subd in subds: dlist.append("%s/%s" % (varlibdir, subd)) util.ensure_dirs(dlist) - cfg = util.get_base_cfg(system_config,cfg_builtin,parsed_cfgs) + cfg = util.get_base_cfg(system_config, cfg_builtin, parsed_cfgs) log_file = util.get_cfg_option_str(cfg, 'def_log_file', None) perms = util.get_cfg_option_str(cfg, 'syslog_fix_perms', None) if log_file: - fp = open(log_file,"ab") + fp = open(log_file, "ab") fp.close() if log_file and perms: - (u,g) = perms.split(':',1) - if u == "-1" or u == "None": u = None - if g == "-1" or g == "None": g = None + (u, g) = perms.split(':', 1) + if u == "-1" or u == "None": + u = None + if g == "-1" or g == "None": + g = None util.chownbyname(log_file, u, g) + def purge_cache(rmcur=True): - rmlist = [ boot_finished ] - if rmcur: rmlist.append(cur_instance_link) + rmlist = [boot_finished] + if rmcur: + rmlist.append(cur_instance_link) for f in rmlist: try: os.unlink(f) except OSError as e: - if e.errno == errno.ENOENT: continue + if e.errno == errno.ENOENT: + continue return(False) except: return(False) return(True) + # get_ipath_cur: get the current instance path for an item def get_ipath_cur(name=None): return("%s/%s%s" % (varlibdir, "instance", pathmap[name])) + # get_cpath : get the "clouddir" (/var/lib/cloud/<name>) # for a name in dirmap def get_cpath(name=None): return("%s%s" % (varlibdir, pathmap[name])) + def get_base_cfg(cfg_path=None): - if cfg_path is None: cfg_path = system_config - return(util.get_base_cfg(cfg_path,cfg_builtin,parsed_cfgs)) + if cfg_path is None: + cfg_path = system_config + return(util.get_base_cfg(cfg_path, cfg_builtin, parsed_cfgs)) + def get_builtin_cfg(): return(yaml.load(cfg_builtin)) + class DataSourceNotFoundException(Exception): pass + def list_sources(cfg_list, depends): - return(DataSource.list_sources(cfg_list,depends, ["cloudinit", "" ])) + return(DataSource.list_sources(cfg_list, depends, ["cloudinit", ""])) + def handler_register(mod, part_handlers, data, frequency=per_instance): if not hasattr(mod, "handler_version"): @@ -521,27 +557,30 @@ def handler_register(mod, part_handlers, data, frequency=per_instance): handler_call_begin(mod, data, frequency) return(mod) + def handler_call_begin(mod, data, frequency): handler_handle_part(mod, data, "__begin__", None, None, frequency) + def handler_call_end(mod, data, frequency): handler_handle_part(mod, data, "__end__", None, None, frequency) + def handler_handle_part(mod, data, ctype, filename, payload, frequency): # only add the handler if the module should run modfreq = getattr(mod, "frequency", per_instance) - if not ( modfreq == per_always or - ( frequency == per_instance and modfreq == per_instance)): + if not (modfreq == per_always or + (frequency == per_instance and modfreq == per_instance)): return if mod.handler_version == 1: mod.handle_part(data, ctype, filename, payload) else: mod.handle_part(data, ctype, filename, payload, frequency) -def partwalker_handle_handler(pdata, ctype, filename, payload): +def partwalker_handle_handler(pdata, _ctype, _filename, payload): curcount = pdata['handlercount'] - modname = 'part-handler-%03d' % curcount + modname = 'part-handler-%03d' % curcount frequency = pdata['frequency'] modfname = modname + ".py" @@ -557,6 +596,7 @@ def partwalker_handle_handler(pdata, ctype, filename, payload): traceback.print_exc(file=sys.stderr) return + def partwalker_callback(pdata, ctype, filename, payload): # data here is the part_handlers array and then the data to pass through if ctype == "text/part-handler": @@ -569,18 +609,20 @@ def partwalker_callback(pdata, ctype, filename, payload): handler_handle_part(pdata['handlers'][ctype], pdata['data'], ctype, filename, payload, pdata['frequency']) + class InternalPartHandler: freq = per_instance - mtypes = [ ] + mtypes = [] handler_version = 1 handler = None - def __init__(self, handler, mtypes, frequency, version = 2): + + def __init__(self, handler, mtypes, frequency, version=2): self.handler = handler self.mtypes = mtypes self.frequency = frequency self.handler_version = version - def __repr__(): + def __repr__(self): return("InternalPartHandler: [%s]" % self.mtypes) def list_types(self): diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index 3ba922e8..80223256 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -1,15 +1,37 @@ +#!/usr/bin/python +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + import subprocess -def netdev_info(): - fields = ( "hwaddr", "addr", "bcast", "mask" ) - ifcfg_out = subprocess.check_output(["ifconfig", "-a"]) - devs = { } + +def netdev_info(empty=""): + fields = ("hwaddr", "addr", "bcast", "mask") + ifcfg_out = str(subprocess.check_output(["ifconfig", "-a"])) + devs = {} for line in ifcfg_out.splitlines(): if len(line) == 0: continue if line[0] not in ("\t", " "): curdev = line.split()[0] - devs[curdev] = { "up": False } + devs[curdev] = {"up": False} for field in fields: devs[curdev][field] = "" toks = line.lower().strip().split() @@ -23,25 +45,33 @@ def netdev_info(): for i in range(len(toks)): if toks[i] == "hwaddr": try: - devs[curdev]["hwaddr"] = toks[i+1] - except IndexError as e: + devs[curdev]["hwaddr"] = toks[i + 1] + except IndexError: pass for field in ("addr", "bcast", "mask"): target = "%s%s" % (field, fieldpost) - if devs[curdev].get(target,""): + if devs[curdev].get(target, ""): continue if toks[i] == "%s:" % field: try: - devs[curdev][target] = toks[i+1] - except IndexError as e: + devs[curdev][target] = toks[i + 1] + except IndexError: pass elif toks[i].startswith("%s:" % field): - devs[curdev][target] = toks[i][len(field)+1:] + devs[curdev][target] = toks[i][len(field) + 1:] + + if empty != "": + for (devname, dev) in devs.iteritems(): + for field in dev: + if dev[field] == "": + dev[field] = empty + return(devs) + def route_info(): - route_out = subprocess.check_output(["route", "-n"]) - routes = [ ] + route_out = str(subprocess.check_output(["route", "-n"])) + routes = [] for line in route_out.splitlines()[1:]: if not line: continue @@ -51,33 +81,36 @@ def route_info(): routes.append(toks) return(routes) + def getgateway(): for r in route_info(): if r[3].find("G") >= 0: - return("%s[%s]" % (r[1],r[7])) + return("%s[%s]" % (r[1], r[7])) return(None) + def debug_info(pre="ci-info: "): - lines = [ ] + lines = [] try: - netdev = netdev_info() - except Exception as e: + netdev = netdev_info(empty=".") + except Exception: lines.append("netdev_info failed!") netdev = {} for (dev, d) in netdev.iteritems(): lines.append("%s%-6s: %i %-15s %-15s %s" % - (pre, dev, d["up"], d["addr"],d["mask"], d["hwaddr"])) + (pre, dev, d["up"], d["addr"], d["mask"], d["hwaddr"])) try: - routes = route_info() - except Exception as e: - lines.append("route_info failed") - routes = [] + routes = route_info() + except Exception: + lines.append("route_info failed") + routes = [] n = 0 for r in routes: - lines.append("%sroute-%d: %-15s %-15s %-15s %-6s %s" % - (pre, n, r[0], r[1], r[2], r[7], r[3])) - n = n+1 + lines.append("%sroute-%d: %-15s %-15s %-15s %-6s %s" % + (pre, n, r[0], r[1], r[2], r[7], r[3])) + n = n + 1 return('\n'.join(lines)) + if __name__ == '__main__': - print debug_info() + print debug_info() diff --git a/cloudinit/util.py b/cloudinit/util.py index de95ec79..e6489648 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Hafliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -15,9 +17,11 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + import yaml import os import os.path +import shutil import errno import subprocess from Cheetah.Template import Template @@ -37,20 +41,22 @@ try: except ImportError: HAVE_LIBSELINUX = False + def read_conf(fname): try: - stream = open(fname,"r") - conf = yaml.load(stream) - stream.close() - return conf + stream = open(fname, "r") + conf = yaml.load(stream) + stream.close() + return conf except IOError as e: if e.errno == errno.ENOENT: - return { } + return {} raise -def get_base_cfg(cfgfile,cfg_builtin="", parsed_cfgs=None): - kerncfg = { } - syscfg = { } + +def get_base_cfg(cfgfile, cfg_builtin="", parsed_cfgs=None): + kerncfg = {} + syscfg = {} if parsed_cfgs and cfgfile in parsed_cfgs: return(parsed_cfgs[cfgfile]) @@ -65,7 +71,7 @@ def get_base_cfg(cfgfile,cfg_builtin="", parsed_cfgs=None): if cfg_builtin: builtin = yaml.load(cfg_builtin) - fin = mergedict(combined,builtin) + fin = mergedict(combined, builtin) else: fin = combined @@ -73,106 +79,164 @@ def get_base_cfg(cfgfile,cfg_builtin="", parsed_cfgs=None): parsed_cfgs[cfgfile] = fin return(fin) + def get_cfg_option_bool(yobj, key, default=False): - if not yobj.has_key(key): return default + if key not in yobj: + return default val = yobj[key] - if val is True: return True - if str(val).lower() in [ 'true', '1', 'on', 'yes']: + if val is True: + return True + if str(val).lower() in ['true', '1', 'on', 'yes']: return True return False + def get_cfg_option_str(yobj, key, default=None): - if not yobj.has_key(key): return default + if key not in yobj: + return default return yobj[key] + def get_cfg_option_list_or_str(yobj, key, default=None): - if not yobj.has_key(key): return default - if yobj[key] is None: return [] - if isinstance(yobj[key],list): return yobj[key] - return([yobj[key]]) + """ + Gets the C{key} config option from C{yobj} as a list of strings. If the + key is present as a single string it will be returned as a list with one + string arg. + + @param yobj: The configuration object. + @param key: The configuration key to get. + @param default: The default to return if key is not found. + @return: The configuration option as a list of strings or default if key + is not found. + """ + if not key in yobj: + return default + if yobj[key] is None: + return [] + if isinstance(yobj[key], list): + return yobj[key] + return [yobj[key]] + # get a cfg entry by its path array # for f['a']['b']: get_cfg_by_path(mycfg,('a','b')) -def get_cfg_by_path(yobj,keyp,default=None): +def get_cfg_by_path(yobj, keyp, default=None): cur = yobj for tok in keyp: - if tok not in cur: return(default) + if tok not in cur: + return(default) cur = cur[tok] return(cur) -# merge values from cand into source -# if src has a key, cand will not override -def mergedict(src,cand): - if isinstance(src,dict) and isinstance(cand,dict): - for k,v in cand.iteritems(): + +def mergedict(src, cand): + """ + Merge values from C{cand} into C{src}. If C{src} has a key C{cand} will + not override. Nested dictionaries are merged recursively. + """ + if isinstance(src, dict) and isinstance(cand, dict): + for k, v in cand.iteritems(): if k not in src: src[k] = v else: - src[k] = mergedict(src[k],v) + src[k] = mergedict(src[k], v) return src -def write_file(file,content,mode=0644,omode="wb"): - try: - os.makedirs(os.path.dirname(file)) - except OSError as e: - if e.errno != errno.EEXIST: - raise e - f=open(file,omode) - if mode != None: - os.chmod(file,mode) - f.write(content) - f.close() - restorecon_if_possible(file) +def delete_dir_contents(dirname): + """ + Deletes all contents of a directory without deleting the directory itself. + + @param dirname: The directory whose contents should be deleted. + """ + for node in os.listdir(dirname): + node_fullpath = os.path.join(dirname, node) + if os.path.isdir(node_fullpath): + shutil.rmtree(node_fullpath) + else: + os.unlink(node_fullpath) + + +def write_file(filename, content, mode=0644, omode="wb"): + """ + Writes a file with the given content and sets the file mode as specified. + Resotres the SELinux context if possible. + + @param filename: The full path of the file to write. + @param content: The content to write to the file. + @param mode: The filesystem mode to set on the file. + @param omode: The open mode used when opening the file (r, rb, a, etc.) + """ + try: + os.makedirs(os.path.dirname(filename)) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + + f = open(filename, omode) + if mode is not None: + os.chmod(filename, mode) + f.write(content) + f.close() + restorecon_if_possible(filename) + def restorecon_if_possible(path, recursive=False): if HAVE_LIBSELINUX and selinux.is_selinux_enabled(): selinux.restorecon(path, recursive=recursive) + # get keyid from keyserver -def getkeybyid(keyid,keyserver): - shcmd=""" - k=${1} ks=${2}; - exec 2>/dev/null - [ -n "$k" ] || exit 1; - armour=$(gpg --list-keys --armour "${k}") - if [ -z "${armour}" ]; then - gpg --keyserver ${ks} --recv $k >/dev/null && - armour=$(gpg --export --armour "${k}") && - gpg --batch --yes --delete-keys "${k}" - fi - [ -n "${armour}" ] && echo "${armour}" - """ - args=['sh', '-c', shcmd, "export-gpg-keyid", keyid, keyserver] - return(subp(args)[0]) +def getkeybyid(keyid, keyserver): + shcmd = """ + k=${1} ks=${2}; + exec 2>/dev/null + [ -n "$k" ] || exit 1; + armour=$(gpg --list-keys --armour "${k}") + if [ -z "${armour}" ]; then + gpg --keyserver ${ks} --recv $k >/dev/null && + armour=$(gpg --export --armour "${k}") && + gpg --batch --yes --delete-keys "${k}" + fi + [ -n "${armour}" ] && echo "${armour}" + """ + args = ['sh', '-c', shcmd, "export-gpg-keyid", keyid, keyserver] + return(subp(args)[0]) + def runparts(dirp, skip_no_exist=True): - if skip_no_exist and not os.path.isdir(dirp): return - + if skip_no_exist and not os.path.isdir(dirp): + return + # per bug 857926, Fedora's run-parts will exit failure on empty dir - if os.path.isdir(dirp) and os.listdir(dirp) == []: return + if os.path.isdir(dirp) and os.listdir(dirp) == []: + return - cmd = [ 'run-parts', '--regex', '.*', dirp ] + cmd = ['run-parts', '--regex', '.*', dirp] sp = subprocess.Popen(cmd) sp.communicate() if sp.returncode is not 0: - raise subprocess.CalledProcessError(sp.returncode,cmd) + raise subprocess.CalledProcessError(sp.returncode, cmd) return -def subp(args, input=None): + +def subp(args, input_=None): sp = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - out,err = sp.communicate(input) + out, err = sp.communicate(input_) if sp.returncode is not 0: - raise subprocess.CalledProcessError(sp.returncode,args, (out,err)) - return(out,err) + raise subprocess.CalledProcessError(sp.returncode, args, (out, err)) + return(out, err) + def render_to_file(template, outfile, searchList): - t = Template(file='/etc/cloud/templates/%s.tmpl' % template, searchList=[searchList]) + t = Template(file='/etc/cloud/templates/%s.tmpl' % template, + searchList=[searchList]) f = open(outfile, 'w') f.write(t.respond()) f.close() + def render_string(template, searchList): return(Template(template, searchList=[searchList]).respond()) @@ -181,22 +245,22 @@ def render_string(template, searchList): # returns boolean indicating success or failure (presense of files) # if files are present, populates 'fill' dictionary with 'user-data' and # 'meta-data' entries -def read_optional_seed(fill,base="",ext="", timeout=5): +def read_optional_seed(fill, base="", ext="", timeout=5): try: - (md,ud) = read_seeded(base,ext,timeout) - fill['user-data']= ud - fill['meta-data']= md + (md, ud) = read_seeded(base, ext, timeout) + fill['user-data'] = ud + fill['meta-data'] = md return True except OSError, e: if e.errno == errno.ENOENT: return False raise - + # raise OSError with enoent if not found def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): if base.startswith("/"): - base="file://%s" % base + base = "file://%s" % base # default retries for file is 0. for network is 10 if base.startswith("file://"): @@ -209,20 +273,22 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): ud_url = "%s%s%s" % (base, "user-data", ext) md_url = "%s%s%s" % (base, "meta-data", ext) - raise_err = None - for attempt in range(0,retries+1): + no_exc = object() + raise_err = no_exc + for attempt in range(0, retries + 1): try: md_str = readurl(md_url, timeout=timeout) ud = readurl(ud_url, timeout=timeout) md = yaml.load(md_str) - - return(md,ud) + + return(md, ud) except urllib2.HTTPError as e: raise_err = e except urllib2.URLError as e: raise_err = e - if isinstance(e.reason,OSError) and e.reason.errno == errno.ENOENT: - raise_err = e.reason + if (isinstance(e.reason, OSError) and + e.reason.errno == errno.ENOENT): + raise_err = e.reason if attempt == retries: break @@ -232,13 +298,18 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): raise(raise_err) -def logexc(log,lvl=logging.DEBUG): - log.log(lvl,traceback.format_exc()) + +def logexc(log, lvl=logging.DEBUG): + log.log(lvl, traceback.format_exc()) + class RecursiveInclude(Exception): pass -def read_file_with_includes(fname, rel = ".", stack=[], patt = None): + +def read_file_with_includes(fname, rel=".", stack=None, patt=None): + if stack is None: + stack = [] if not fname.startswith("/"): fname = os.sep.join((rel, fname)) @@ -251,7 +322,7 @@ def read_file_with_includes(fname, rel = ".", stack=[], patt = None): (fname, len(stack)))) if patt == None: - patt = re.compile("^#(opt_include|include)[ \t].*$",re.MULTILINE) + patt = re.compile("^#(opt_include|include)[ \t].*$", re.MULTILINE) try: fp = open(fname) @@ -264,14 +335,14 @@ def read_file_with_includes(fname, rel = ".", stack=[], patt = None): stack.append(fname) cur = 0 - clen = len(contents) while True: match = patt.search(contents[cur:]) - if not match: break + if not match: + break loc = match.start() + cur endl = match.end() + cur - (key, cur_fname) = contents[loc:endl].split(None,2) + (key, cur_fname) = contents[loc:endl].split(None, 2) cur_fname = cur_fname.strip() try: @@ -281,43 +352,47 @@ def read_file_with_includes(fname, rel = ".", stack=[], patt = None): inc_contents = "" else: raise - contents = contents[0:loc] + inc_contents + contents[endl+1:] - cur = loc + len(inc_contents) + contents = contents[0:loc] + inc_contents + contents[endl + 1:] + cur = loc + len(inc_contents) stack.pop() return(contents) + def read_conf_d(confd): # get reverse sorted list (later trumps newer) - confs = sorted(os.listdir(confd),reverse=True) - + confs = sorted(os.listdir(confd), reverse=True) + # remove anything not ending in '.cfg' - confs = filter(lambda f: f.endswith(".cfg"), confs) + confs = [f for f in confs if f.endswith(".cfg")] # remove anything not a file - confs = filter(lambda f: os.path.isfile("%s/%s" % (confd,f)),confs) + confs = [f for f in confs if os.path.isfile("%s/%s" % (confd, f))] - cfg = { } + cfg = {} for conf in confs: - cfg = mergedict(cfg,read_conf("%s/%s" % (confd,conf))) + cfg = mergedict(cfg, read_conf("%s/%s" % (confd, conf))) return(cfg) + def read_conf_with_confd(cfgfile): cfg = read_conf(cfgfile) confd = False if "conf_d" in cfg: if cfg['conf_d'] is not None: confd = cfg['conf_d'] - if not isinstance(confd,str): - raise Exception("cfgfile %s contains 'conf_d' with non-string" % cfgfile) + if not isinstance(confd, str): + raise Exception("cfgfile %s contains 'conf_d' " + "with non-string" % cfgfile) elif os.path.isdir("%s.d" % cfgfile): confd = "%s.d" % cfgfile - if not confd: return(cfg) + if not confd: + return(cfg) confd_cfg = read_conf_d(confd) - return(mergedict(confd_cfg,cfg)) + return(mergedict(confd_cfg, cfg)) def get_cmdline(): @@ -331,7 +406,8 @@ def get_cmdline(): except: cmdline = "" return(cmdline) - + + def read_cc_from_cmdline(cmdline=None): # this should support reading cloud-config information from # the kernel command line. It is intended to support content of the @@ -344,23 +420,25 @@ def read_cc_from_cmdline(cmdline=None): if cmdline is None: cmdline = get_cmdline() - tag_begin="cc:" - tag_end="end_cc" + tag_begin = "cc:" + tag_end = "end_cc" begin_l = len(tag_begin) end_l = len(tag_end) clen = len(cmdline) - tokens = [ ] + tokens = [] begin = cmdline.find(tag_begin) while begin >= 0: end = cmdline.find(tag_end, begin + begin_l) if end < 0: end = clen - tokens.append(cmdline[begin+begin_l:end].lstrip().replace("\\n","\n")) - + tokens.append(cmdline[begin + begin_l:end].lstrip().replace("\\n", + "\n")) + begin = cmdline.find(tag_begin, end + end_l) return('\n'.join(tokens)) + def ensure_dirs(dirlist, mode=0755): fixmodes = [] for d in dirlist: @@ -370,27 +448,32 @@ def ensure_dirs(dirlist, mode=0755): else: os.makedirs(d, mode) except OSError as e: - if e.errno != errno.EEXIST: raise - if mode != None: fixmodes.append(d) + if e.errno != errno.EEXIST: + raise + if mode != None: + fixmodes.append(d) for d in fixmodes: os.chmod(d, mode) -def chownbyname(fname,user=None,group=None): - uid = -1 - gid = -1 - if user == None and group == None: return - if user: - import pwd - uid = pwd.getpwnam(user).pw_uid - if group: - import grp - gid = grp.getgrnam(group).gr_gid - os.chown(fname,uid,gid) +def chownbyname(fname, user=None, group=None): + uid = -1 + gid = -1 + if user == None and group == None: + return + if user: + import pwd + uid = pwd.getpwnam(user).pw_uid + if group: + import grp + gid = grp.getgrnam(group).gr_gid + + os.chown(fname, uid, gid) + def readurl(url, data=None, timeout=None): - openargs = { } + openargs = {} if timeout != None: openargs['timeout'] = timeout @@ -403,47 +486,51 @@ def readurl(url, data=None, timeout=None): response = urllib2.urlopen(req, **openargs) return(response.read()) + # shellify, takes a list of commands # for each entry in the list # if it is an array, shell protect it (with single ticks) # if it is a string, do nothing def shellify(cmdlist): - content="#!/bin/sh\n" - escaped="%s%s%s%s" % ( "'", '\\', "'", "'" ) + content = "#!/bin/sh\n" + escaped = "%s%s%s%s" % ("'", '\\', "'", "'") for args in cmdlist: # if the item is a list, wrap all items in single tick # if its not, then just write it directly - if isinstance(args,list): - fixed = [ ] + if isinstance(args, list): + fixed = [] for f in args: - fixed.append("'%s'" % str(f).replace("'",escaped)) - content="%s%s\n" % ( content, ' '.join(fixed) ) + fixed.append("'%s'" % str(f).replace("'", escaped)) + content = "%s%s\n" % (content, ' '.join(fixed)) else: - content="%s%s\n" % ( content, str(args) ) + content = "%s%s\n" % (content, str(args)) return content -def dos2unix(input): + +def dos2unix(string): # find first end of line - pos = input.find('\n') - if pos <= 0 or input[pos-1] != '\r': return(input) - return(input.replace('\r\n','\n')) + pos = string.find('\n') + if pos <= 0 or string[pos - 1] != '\r': + return(string) + return(string.replace('\r\n', '\n')) + def islxc(): # is this host running lxc? try: with open("/proc/1/cgroup") as f: - if f.read() == "/": + if f.read() == "/": return True except IOError as e: if e.errno != errno.ENOENT: raise try: - # try to run a program named 'lxc-is-container'. if it returns true, then - # we're inside a container. otherwise, no + # try to run a program named 'lxc-is-container'. if it returns true, + # then we're inside a container. otherwise, no sp = subprocess.Popen(['lxc-is-container'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out,err = sp.communicate(None) + sp.communicate(None) return(sp.returncode == 0) except OSError as e: if e.errno != errno.ENOENT: @@ -451,13 +538,14 @@ def islxc(): return False + def get_hostname_fqdn(cfg, cloud): # return the hostname and fqdn from 'cfg'. If not found in cfg, # then fall back to data from cloud if "fqdn" in cfg: # user specified a fqdn. Default hostname then is based off that fqdn = cfg['fqdn'] - hostname = get_cfg_option_str(cfg,"hostname",fqdn.split('.')[0]) + hostname = get_cfg_option_str(cfg, "hostname", fqdn.split('.')[0]) else: if "hostname" in cfg and cfg['hostname'].find('.') > 0: # user specified hostname, and it had '.' in it @@ -465,7 +553,7 @@ def get_hostname_fqdn(cfg, cloud): fqdn = cfg['hostname'] hostname = cfg['hostname'][:fqdn.find('.')] else: - # no fqdn set, get fqdn from cloud. + # no fqdn set, get fqdn from cloud. # get hostname from cfg if available otherwise cloud fqdn = cloud.get_hostname(fqdn=True) if "hostname" in cfg: @@ -474,9 +562,10 @@ def get_hostname_fqdn(cfg, cloud): hostname = cloud.get_hostname() return(hostname, fqdn) + def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): # this parses /etc/hosts to get a fqdn. It should return the same - # result as 'hostname -f <hostname>' if /etc/hosts.conf + # result as 'hostname -f <hostname>' if /etc/hosts.conf # did not have did not have 'bind' in the order attribute fqdn = None try: @@ -502,38 +591,42 @@ def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): return fqdn + def is_resolvable(name): """ determine if a url is resolvable, return a boolean """ try: socket.getaddrinfo(name, None) return True - except socket.gaierror as e: + except socket.gaierror: return False + def is_resolvable_url(url): """ determine if this url is resolvable (existing or ip) """ return(is_resolvable(urlparse.urlparse(url).hostname)) + def search_for_mirror(candidates): """ Search through a list of mirror urls for one that works """ for cand in candidates: try: if is_resolvable_url(cand): return cand - except Exception as e: + except Exception: raise return None + def close_stdin(): - """ - reopen stdin as /dev/null so even subprocesses or other os level things get - /dev/null as input. - - if _CLOUD_INIT_SAVE_STDIN is set in environment to a non empty or '0' value - then input will not be closed (only useful potentially for debugging). - """ - if os.environ.get("_CLOUD_INIT_SAVE_STDIN") in ("", "0", False): - return - with open(os.devnull) as fp: - os.dup2(fp.fileno(), sys.stdin.fileno()) + """ + reopen stdin as /dev/null so even subprocesses or other os level things get + /dev/null as input. + + if _CLOUD_INIT_SAVE_STDIN is set in environment to a non empty or '0' value + then input will not be closed (only useful potentially for debugging). + """ + if os.environ.get("_CLOUD_INIT_SAVE_STDIN") in ("", "0", False): + return + with open(os.devnull) as fp: + os.dup2(fp.fileno(), sys.stdin.fileno()) diff --git a/config/cloud.cfg b/config/cloud.cfg index 45538955..25d02cee 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -9,6 +9,7 @@ cloud_init_modules: - set_hostname - update_hostname - update_etc_hosts + - ca-certs - rsyslog - ssh diff --git a/debian.trunk/control b/debian.trunk/control index 76d0db0d..eef3cd1d 100644 --- a/debian.trunk/control +++ b/debian.trunk/control @@ -5,6 +5,10 @@ Maintainer: Scott Moser <smoser@ubuntu.com> Build-Depends: cdbs, debhelper (>= 5.0.38), python (>= 2.6.6-3~), + python-nose, + pyflakes, + pylint, + python-mocker, XS-Python-Version: all Standards-Version: 3.9.1 diff --git a/doc/examples/cloud-config-ca-certs.txt b/doc/examples/cloud-config-ca-certs.txt new file mode 100644 index 00000000..5e9115a0 --- /dev/null +++ b/doc/examples/cloud-config-ca-certs.txt @@ -0,0 +1,31 @@ +#cloud-config +# +# This is an example file to configure an instance's trusted CA certificates +# system-wide for SSL/TLS trust establishment when the instance boots for the +# first time. +# +# Make sure that this file is valid yaml before starting instances. +# It should be passed as user-data when starting the instance. + +ca-certs: + # If present and set to True, the 'remove-defaults' parameter will remove + # all the default trusted CA certificates that are normally shipped with + # Ubuntu. + # This is mainly for paranoid admins - most users will not need this + # functionality. + remove-defaults: true + + # If present, the 'trusted' parameter should contain a certificate (or list + # of certificates) to add to the system as trusted CA certificates. + # Pay close attention to the YAML multiline list syntax. The example shown + # here is for a list of multiline certificates. + trusted: + - | + -----BEGIN CERTIFICATE----- + YOUR-ORGS-TRUSTED-CA-CERT-HERE + -----END CERTIFICATE----- + - | + -----BEGIN CERTIFICATE----- + YOUR-ORGS-TRUSTED-CA-CERT-HERE + -----END CERTIFICATE----- + diff --git a/tests/unittests/test_handler_ca_certs.py b/tests/unittests/test_handler_ca_certs.py new file mode 100644 index 00000000..d6513b5b --- /dev/null +++ b/tests/unittests/test_handler_ca_certs.py @@ -0,0 +1,178 @@ +from mocker import MockerTestCase + +from cloudinit.util import write_file, delete_dir_contents +from cloudinit.CloudConfig.cc_ca_certs import ( + handle, update_ca_certs, add_ca_certs, remove_default_ca_certs) +from logging import getLogger + + +class TestNoConfig(MockerTestCase): + def setUp(self): + super(TestNoConfig, self).setUp() + self.name = "ca-certs" + self.cloud_init = None + self.log = getLogger("TestNoConfig") + self.args = [] + + def test_no_config(self): + """ + Test that nothing is done if no ca-certs configuration is provided. + """ + config = {"unknown-key": "value"} + + self.mocker.replace(write_file, passthrough=False) + self.mocker.replace(update_ca_certs, passthrough=False) + self.mocker.replay() + + handle(self.name, config, self.cloud_init, self.log, self.args) + + +class TestConfig(MockerTestCase): + def setUp(self): + super(TestConfig, self).setUp() + self.name = "ca-certs" + self.cloud_init = None + self.log = getLogger("TestNoConfig") + self.args = [] + + # Mock out the functions that actually modify the system + self.mock_add = self.mocker.replace(add_ca_certs, passthrough=False) + self.mock_update = self.mocker.replace(update_ca_certs, + passthrough=False) + self.mock_remove = self.mocker.replace(remove_default_ca_certs, + passthrough=False) + # Order must be correct + self.mocker.order() + + def test_no_trusted_list(self): + """ + Test that no certificates are written if the 'trusted' key is not + present. + """ + config = {"ca-certs": {}} + + # No functions should be called + self.mock_update() + self.mocker.replay() + + handle(self.name, config, self.cloud_init, self.log, self.args) + + def test_empty_trusted_list(self): + """Test that no certificate are written if 'trusted' list is empty""" + config = {"ca-certs": {"trusted": []}} + + # No functions should be called + self.mock_update() + self.mocker.replay() + + handle(self.name, config, self.cloud_init, self.log, self.args) + + def test_single_trusted(self): + """Test that a single cert gets passed to add_ca_certs""" + config = {"ca-certs": {"trusted": ["CERT1"]}} + + self.mock_add(["CERT1"]) + self.mock_update() + self.mocker.replay() + + handle(self.name, config, self.cloud_init, self.log, self.args) + + def test_multiple_trusted(self): + """Test that multiple certs get passed to add_ca_certs""" + config = {"ca-certs": {"trusted": ["CERT1", "CERT2"]}} + + self.mock_add(["CERT1", "CERT2"]) + self.mock_update() + self.mocker.replay() + + handle(self.name, config, self.cloud_init, self.log, self.args) + + def test_remove_default_ca_certs(self): + """Test remove_defaults works as expected""" + config = {"ca-certs": {"remove-defaults": True}} + + self.mock_remove() + self.mock_update() + self.mocker.replay() + + handle(self.name, config, self.cloud_init, self.log, self.args) + + def test_no_remove_defaults_if_false(self): + """Test remove_defaults is not called when config value is False""" + config = {"ca-certs": {"remove-defaults": False}} + + self.mock_update() + self.mocker.replay() + + handle(self.name, config, self.cloud_init, self.log, self.args) + + def test_correct_order_for_remove_then_add(self): + """Test remove_defaults is not called when config value is False""" + config = {"ca-certs": {"remove-defaults": True, "trusted": ["CERT1"]}} + + self.mock_remove() + self.mock_add(["CERT1"]) + self.mock_update() + self.mocker.replay() + + handle(self.name, config, self.cloud_init, self.log, self.args) + + +class TestAddCaCerts(MockerTestCase): + def test_no_certs_in_list(self): + """Test that no certificate are written if not provided.""" + self.mocker.replace(write_file, passthrough=False) + self.mocker.replay() + + add_ca_certs([]) + + def test_single_cert(self): + """Test adding a single certificate to the trusted CAs""" + cert = "CERT1\nLINE2\nLINE3" + + mock_write = self.mocker.replace(write_file, passthrough=False) + mock_write("/usr/share/ca-certificates/cloud-init-ca-certs.crt", + cert, mode=0644) + mock_write("/etc/ca-certificates.conf", + "\ncloud-init-ca-certs.crt", omode="a") + self.mocker.replay() + + add_ca_certs([cert]) + + def test_multiple_certs(self): + """Test adding multiple certificates to the trusted CAs""" + certs = ["CERT1\nLINE2\nLINE3", "CERT2\nLINE2\nLINE3"] + expected_cert_file = "\n".join(certs) + + mock_write = self.mocker.replace(write_file, passthrough=False) + mock_write("/usr/share/ca-certificates/cloud-init-ca-certs.crt", + expected_cert_file, mode=0644) + mock_write("/etc/ca-certificates.conf", + "\ncloud-init-ca-certs.crt", omode="a") + self.mocker.replay() + + add_ca_certs(certs) + + +class TestUpdateCaCerts(MockerTestCase): + def test_commands(self): + mock_check_call = self.mocker.replace("subprocess.check_call", + passthrough=False) + mock_check_call(["update-ca-certificates"]) + self.mocker.replay() + + update_ca_certs() + + +class TestRemoveDefaultCaCerts(MockerTestCase): + def test_commands(self): + mock_delete_dir_contents = self.mocker.replace(delete_dir_contents, + passthrough=False) + mock_write = self.mocker.replace(write_file, passthrough=False) + + mock_delete_dir_contents("/usr/share/ca-certificates/") + mock_delete_dir_contents("/etc/ssl/certs/") + mock_write("/etc/ca-certificates.conf", "", mode=0644) + self.mocker.replay() + + remove_default_ca_certs() diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py new file mode 100644 index 00000000..d8da8bc9 --- /dev/null +++ b/tests/unittests/test_util.py @@ -0,0 +1,250 @@ +from unittest import TestCase +from mocker import MockerTestCase +from tempfile import mkdtemp +from shutil import rmtree +import os +import stat + +from cloudinit.util import (mergedict, get_cfg_option_list_or_str, write_file, + delete_dir_contents) + + +class TestMergeDict(TestCase): + def test_simple_merge(self): + """Test simple non-conflict merge.""" + source = {"key1": "value1"} + candidate = {"key2": "value2"} + result = mergedict(source, candidate) + self.assertEqual({"key1": "value1", "key2": "value2"}, result) + + def test_nested_merge(self): + """Test nested merge.""" + source = {"key1": {"key1.1": "value1.1"}} + candidate = {"key1": {"key1.2": "value1.2"}} + result = mergedict(source, candidate) + self.assertEqual( + {"key1": {"key1.1": "value1.1", "key1.2": "value1.2"}}, result) + + def test_merge_does_not_override(self): + """Test that candidate doesn't override source.""" + source = {"key1": "value1", "key2": "value2"} + candidate = {"key2": "value2", "key2": "NEW VALUE"} + result = mergedict(source, candidate) + self.assertEqual(source, result) + + def test_empty_candidate(self): + """Test empty candidate doesn't change source.""" + source = {"key": "value"} + candidate = {} + result = mergedict(source, candidate) + self.assertEqual(source, result) + + def test_empty_source(self): + """Test empty source is replaced by candidate.""" + source = {} + candidate = {"key": "value"} + result = mergedict(source, candidate) + self.assertEqual(candidate, result) + + def test_non_dict_candidate(self): + """Test non-dict candidate is discarded.""" + source = {"key": "value"} + candidate = "not a dict" + result = mergedict(source, candidate) + self.assertEqual(source, result) + + def test_non_dict_source(self): + """Test non-dict source is not modified with a dict candidate.""" + source = "not a dict" + candidate = {"key": "value"} + result = mergedict(source, candidate) + self.assertEqual(source, result) + + def test_neither_dict(self): + """Test if neither candidate or source is dict source wins.""" + source = "source" + candidate = "candidate" + result = mergedict(source, candidate) + self.assertEqual(source, result) + + +class TestGetCfgOptionListOrStr(TestCase): + def test_not_found_no_default(self): + """None is returned if key is not found and no default given.""" + config = {} + result = get_cfg_option_list_or_str(config, "key") + self.assertIsNone(result) + + def test_not_found_with_default(self): + """Default is returned if key is not found.""" + config = {} + result = get_cfg_option_list_or_str(config, "key", default=["DEFAULT"]) + self.assertEqual(["DEFAULT"], result) + + def test_found_with_default(self): + """Default is not returned if key is found.""" + config = {"key": ["value1"]} + result = get_cfg_option_list_or_str(config, "key", default=["DEFAULT"]) + self.assertEqual(["value1"], result) + + def test_found_convert_to_list(self): + """Single string is converted to one element list.""" + config = {"key": "value1"} + result = get_cfg_option_list_or_str(config, "key") + self.assertEqual(["value1"], result) + + def test_value_is_none(self): + """If value is None empty list is returned.""" + config = {"key": None} + result = get_cfg_option_list_or_str(config, "key") + self.assertEqual([], result) + + +class TestWriteFile(MockerTestCase): + def setUp(self): + super(TestWriteFile, self).setUp() + # Make a temp directoy for tests to use. + self.tmp = mkdtemp(prefix="unittest_") + + def tearDown(self): + super(TestWriteFile, self).tearDown() + # Clean up temp directory + rmtree(self.tmp) + + def test_basic_usage(self): + """Verify basic usage with default args.""" + path = os.path.join(self.tmp, "NewFile.txt") + contents = "Hey there" + + write_file(path, contents) + + self.assertTrue(os.path.exists(path)) + self.assertTrue(os.path.isfile(path)) + with open(path) as f: + create_contents = f.read() + self.assertEqual(contents, create_contents) + file_stat = os.stat(path) + self.assertEqual(0644, stat.S_IMODE(file_stat.st_mode)) + + def test_dir_is_created_if_required(self): + """Verifiy that directories are created is required.""" + dirname = os.path.join(self.tmp, "subdir") + path = os.path.join(dirname, "NewFile.txt") + contents = "Hey there" + + write_file(path, contents) + + self.assertTrue(os.path.isdir(dirname)) + self.assertTrue(os.path.isfile(path)) + + def test_custom_mode(self): + """Verify custom mode works properly.""" + path = os.path.join(self.tmp, "NewFile.txt") + contents = "Hey there" + + write_file(path, contents, mode=0666) + + self.assertTrue(os.path.exists(path)) + self.assertTrue(os.path.isfile(path)) + file_stat = os.stat(path) + self.assertEqual(0666, stat.S_IMODE(file_stat.st_mode)) + + def test_custom_omode(self): + """Verify custom omode works properly.""" + path = os.path.join(self.tmp, "NewFile.txt") + contents = "Hey there" + + # Create file first with basic content + with open(path, "wb") as f: + f.write("LINE1\n") + write_file(path, contents, omode="a") + + self.assertTrue(os.path.exists(path)) + self.assertTrue(os.path.isfile(path)) + with open(path) as f: + create_contents = f.read() + self.assertEqual("LINE1\nHey there", create_contents) + + def test_restorecon_if_possible_is_called(self): + """Make sure the restorecon_if_possible is called correctly.""" + path = os.path.join(self.tmp, "NewFile.txt") + contents = "Hey there" + + # Mock out the restorecon_if_possible call to test if it's called. + mock_restorecon = self.mocker.replace( + "cloudinit.util.restorecon_if_possible", passthrough=False) + mock_restorecon(path) + self.mocker.replay() + + write_file(path, contents) + + +class TestDeleteDirContents(TestCase): + def setUp(self): + super(TestDeleteDirContents, self).setUp() + # Make a temp directoy for tests to use. + self.tmp = mkdtemp(prefix="unittest_") + + def tearDown(self): + super(TestDeleteDirContents, self).tearDown() + # Clean up temp directory + rmtree(self.tmp) + + def assertDirEmpty(self, dirname): + self.assertEqual([], os.listdir(dirname)) + + def test_does_not_delete_dir(self): + """Ensure directory itself is not deleted.""" + delete_dir_contents(self.tmp) + + self.assertTrue(os.path.isdir(self.tmp)) + self.assertDirEmpty(self.tmp) + + def test_deletes_files(self): + """Single file should be deleted.""" + with open(os.path.join(self.tmp, "new_file.txt"), "wb") as f: + f.write("DELETE ME") + + delete_dir_contents(self.tmp) + + self.assertDirEmpty(self.tmp) + + def test_deletes_empty_dirs(self): + """Empty directories should be deleted.""" + os.mkdir(os.path.join(self.tmp, "new_dir")) + + delete_dir_contents(self.tmp) + + self.assertDirEmpty(self.tmp) + + def test_deletes_nested_dirs(self): + """Nested directories should be deleted.""" + os.mkdir(os.path.join(self.tmp, "new_dir")) + os.mkdir(os.path.join(self.tmp, "new_dir", "new_subdir")) + + delete_dir_contents(self.tmp) + + self.assertDirEmpty(self.tmp) + + def test_deletes_non_empty_dirs(self): + """Non-empty directories should be deleted.""" + os.mkdir(os.path.join(self.tmp, "new_dir")) + f_name = os.path.join(self.tmp, "new_dir", "new_file.txt") + with open(f_name, "wb") as f: + f.write("DELETE ME") + + delete_dir_contents(self.tmp) + + self.assertDirEmpty(self.tmp) + + def test_deletes_symlinks(self): + """Symlinks should be deleted.""" + file_name = os.path.join(self.tmp, "new_file.txt") + link_name = os.path.join(self.tmp, "new_file_link.txt") + with open(file_name, "wb") as f: + f.write("DELETE ME") + os.symlink(file_name, link_name) + + delete_dir_contents(self.tmp) + + self.assertDirEmpty(self.tmp) diff --git a/tools/run-pylint b/tools/run-pylint new file mode 100755 index 00000000..e271c3d5 --- /dev/null +++ b/tools/run-pylint @@ -0,0 +1,49 @@ +#!/bin/bash + +def_files='cloud*.py cloudinit/*.py cloudinit/CloudConfig/*.py' + +if [ $# -eq 0 ]; then + files=( ) + for f in $def_files; do + [ -f "$f" ] || { echo "failed, $f not a file" 1>&2; exit 1; } + files[${#files[@]}]=${f} + done +else + files=( "$@" ); +fi + +cmd=( + pylint + --reports=n + --include-ids=y + --max-line-length=79 + + --disable=R + --disable=I + + --disable=W0142 # Used * or ** magic + --disable=W0511 # TODO/FIXME note + --disable=W0702 # No exception type(s) specified + --disable=W0703 # Catch "Exception" + + --disable=C0103 # Invalid name + --disable=C0111 # Missing docstring + + "${files[@]}" +) + +echo -e "\nRunning pylint:" +echo "${cmd[@]}" +"${cmd[@]}" + +cmd=( + pep8 + + --ignore=E501 # Line too long (these are caught by pylint above) + + "${files[@]}" +) + +echo -e "\nRunning pep8:" +echo "${cmd[@]}" +"${cmd[@]}" |