From 87ebd91af8c230b8f1deed8e734297e40397eea0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 12:31:49 -0700 Subject: Move binaries to an actual binary dir. --- bin/cloud-init-cfg.py | 115 ++++++++++++++++++++++++ bin/cloud-init-query.py | 56 ++++++++++++ bin/cloud-init.py | 229 ++++++++++++++++++++++++++++++++++++++++++++++++ cloud-init-cfg.py | 115 ------------------------ cloud-init-query.py | 56 ------------ cloud-init.py | 229 ------------------------------------------------ 6 files changed, 400 insertions(+), 400 deletions(-) create mode 100755 bin/cloud-init-cfg.py create mode 100755 bin/cloud-init-query.py create mode 100755 bin/cloud-init.py delete mode 100755 cloud-init-cfg.py delete mode 100755 cloud-init-query.py delete mode 100755 cloud-init.py diff --git a/bin/cloud-init-cfg.py b/bin/cloud-init-cfg.py new file mode 100755 index 00000000..3a475c1c --- /dev/null +++ b/bin/cloud-init-cfg.py @@ -0,0 +1,115 @@ +#!/usr/bin/python +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys +import cloudinit +import cloudinit.util as util +import cloudinit.CloudConfig as CC +import logging +import os + + +def Usage(out=sys.stdout): + out.write("Usage: %s name\n" % sys.argv[0]) + + +def main(): + # expect to be called with + # name [ freq [ args ] + # run the cloud-config job 'name' at with given args + # or + # read cloud config jobs from config (builtin -> system) + # and run all in order + + util.close_stdin() + + modename = "config" + + if len(sys.argv) < 2: + Usage(sys.stderr) + sys.exit(1) + if sys.argv[1] == "all": + name = "all" + if len(sys.argv) > 2: + modename = sys.argv[2] + else: + freq = None + run_args = [] + name = sys.argv[1] + if len(sys.argv) > 2: + freq = sys.argv[2] + if freq == "None": + freq = None + if len(sys.argv) > 3: + run_args = sys.argv[3:] + + cfg_path = cloudinit.get_ipath_cur("cloud_config") + cfg_env_name = cloudinit.cfg_env_name + if cfg_env_name in os.environ: + cfg_path = os.environ[cfg_env_name] + + cloud = cloudinit.CloudInit(ds_deps=[]) # ds_deps=[], get only cached + try: + cloud.get_data_source() + except cloudinit.DataSourceNotFoundException as e: + # there was no datasource found, theres nothing to do + sys.exit(0) + + cc = CC.CloudConfig(cfg_path, cloud) + + try: + (outfmt, errfmt) = CC.get_output_cfg(cc.cfg, modename) + CC.redirect_output(outfmt, errfmt) + except Exception as e: + err("Failed to get and set output config: %s\n" % e) + + cloudinit.logging_set_from_cfg(cc.cfg) + log = logging.getLogger() + log.info("cloud-init-cfg %s" % sys.argv[1:]) + + module_list = [] + if name == "all": + modlist_cfg_name = "cloud_%s_modules" % modename + module_list = CC.read_cc_modules(cc.cfg, modlist_cfg_name) + if not len(module_list): + err("no modules to run in cloud_config [%s]" % modename, log) + sys.exit(0) + else: + module_list.append([name, freq] + run_args) + + failures = CC.run_cc_modules(cc, module_list, log) + if len(failures): + err("errors running cloud_config [%s]: %s" % (modename, failures), log) + sys.exit(len(failures)) + + +def err(msg, log=None): + if log: + log.error(msg) + sys.stderr.write(msg + "\n") + + +def fail(msg, log=None): + err(msg, log) + sys.exit(1) + +if __name__ == '__main__': + main() diff --git a/bin/cloud-init-query.py b/bin/cloud-init-query.py new file mode 100755 index 00000000..856cf462 --- /dev/null +++ b/bin/cloud-init-query.py @@ -0,0 +1,56 @@ +#!/usr/bin/python +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys +import cloudinit +import cloudinit.CloudConfig + + +def Usage(out=sys.stdout): + out.write("Usage: %s name\n" % sys.argv[0]) + + +def main(): + # expect to be called with name of item to fetch + if len(sys.argv) != 2: + Usage(sys.stderr) + sys.exit(1) + + cfg_path = cloudinit.get_ipath_cur("cloud_config") + cc = cloudinit.CloudConfig.CloudConfig(cfg_path) + data = { + 'user_data': cc.cloud.get_userdata(), + 'user_data_raw': cc.cloud.get_userdata_raw(), + 'instance_id': cc.cloud.get_instance_id(), + } + + name = sys.argv[1].replace('-', '_') + + if name not in data: + sys.stderr.write("unknown name '%s'. Known values are:\n %s\n" % + (sys.argv[1], ' '.join(data.keys()))) + sys.exit(1) + + print data[name] + sys.exit(0) + +if __name__ == '__main__': + main() diff --git a/bin/cloud-init.py b/bin/cloud-init.py new file mode 100755 index 00000000..2acea3f8 --- /dev/null +++ b/bin/cloud-init.py @@ -0,0 +1,229 @@ +#!/usr/bin/python +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import subprocess +import sys + +import cloudinit +import cloudinit.util as util +import cloudinit.CloudConfig as CC +import cloudinit.DataSource as ds +import cloudinit.netinfo as netinfo +import time +import traceback +import logging +import errno +import os + + +def warn(wstr): + sys.stderr.write("WARN:%s" % wstr) + + +def main(): + util.close_stdin() + + cmds = ("start", "start-local") + deps = {"start": (ds.DEP_FILESYSTEM, ds.DEP_NETWORK), + "start-local": (ds.DEP_FILESYSTEM, )} + + cmd = "" + if len(sys.argv) > 1: + cmd = sys.argv[1] + + cfg_path = None + if len(sys.argv) > 2: + # this is really for debugging only + # but you can invoke on development system with ./config/cloud.cfg + cfg_path = sys.argv[2] + + if not cmd in cmds: + sys.stderr.write("bad command %s. use one of %s\n" % (cmd, cmds)) + sys.exit(1) + + now = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) + try: + uptimef = open("/proc/uptime") + uptime = uptimef.read().split(" ")[0] + uptimef.close() + except IOError as e: + warn("unable to open /proc/uptime\n") + uptime = "na" + + cmdline_msg = None + cmdline_exc = None + if cmd == "start": + target = "%s.d/%s" % (cloudinit.system_config, + "91_kernel_cmdline_url.cfg") + if os.path.exists(target): + cmdline_msg = "cmdline: %s existed" % target + else: + cmdline = util.get_cmdline() + try: + (key, url, content) = cloudinit.get_cmdline_url( + cmdline=cmdline) + if key and content: + util.write_file(target, content, mode=0600) + cmdline_msg = ("cmdline: wrote %s from %s, %s" % + (target, key, url)) + elif key: + cmdline_msg = ("cmdline: %s, %s had no cloud-config" % + (key, url)) + except Exception: + cmdline_exc = ("cmdline: '%s' raised exception\n%s" % + (cmdline, traceback.format_exc())) + warn(cmdline_exc) + + try: + cfg = cloudinit.get_base_cfg(cfg_path) + except Exception as e: + warn("Failed to get base config. falling back to builtin: %s\n" % e) + try: + cfg = cloudinit.get_builtin_cfg() + except Exception as e: + warn("Unable to load builtin config\n") + raise + + try: + (outfmt, errfmt) = CC.get_output_cfg(cfg, "init") + CC.redirect_output(outfmt, errfmt) + except Exception as e: + warn("Failed to get and set output config: %s\n" % e) + + cloudinit.logging_set_from_cfg(cfg) + log = logging.getLogger() + + if cmdline_exc: + log.debug(cmdline_exc) + elif cmdline_msg: + log.debug(cmdline_msg) + + try: + cloudinit.initfs() + except Exception as e: + warn("failed to initfs, likely bad things to come: %s\n" % str(e)) + + nonet_path = "%s/%s" % (cloudinit.get_cpath("data"), "no-net") + + if cmd == "start": + print netinfo.debug_info() + + stop_files = (cloudinit.get_ipath_cur("obj_pkl"), nonet_path) + # if starting as the network start, there are cases + # where everything is already done for us, and it makes + # most sense to exit early and silently + for f in stop_files: + try: + fp = open(f, "r") + fp.close() + except: + continue + + log.debug("no need for cloud-init start to run (%s)\n", f) + sys.exit(0) + elif cmd == "start-local": + # cache is not instance specific, so it has to be purged + # but we want 'start' to benefit from a cache if + # a previous start-local populated one + manclean = util.get_cfg_option_bool(cfg, 'manual_cache_clean', False) + if manclean: + log.debug("not purging cache, manual_cache_clean = True") + cloudinit.purge_cache(not manclean) + + try: + os.unlink(nonet_path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + msg = "cloud-init %s running: %s. up %s seconds" % (cmd, now, uptime) + sys.stderr.write(msg + "\n") + sys.stderr.flush() + + log.info(msg) + + cloud = cloudinit.CloudInit(ds_deps=deps[cmd]) + + try: + cloud.get_data_source() + except cloudinit.DataSourceNotFoundException as e: + sys.stderr.write("no instance data found in %s\n" % cmd) + sys.exit(0) + + # set this as the current instance + cloud.set_cur_instance() + + # store the metadata + cloud.update_cache() + + msg = "found data source: %s" % cloud.datasource + sys.stderr.write(msg + "\n") + log.debug(msg) + + # parse the user data (ec2-run-userdata.py) + try: + ran = cloud.sem_and_run("consume_userdata", cloudinit.per_instance, + cloud.consume_userdata, [cloudinit.per_instance], False) + if not ran: + cloud.consume_userdata(cloudinit.per_always) + except: + warn("consuming user data failed!\n") + raise + + cfg_path = cloudinit.get_ipath_cur("cloud_config") + cc = CC.CloudConfig(cfg_path, cloud) + + # if the output config changed, update output and err + try: + outfmt_orig = outfmt + errfmt_orig = errfmt + (outfmt, errfmt) = CC.get_output_cfg(cc.cfg, "init") + if outfmt_orig != outfmt or errfmt_orig != errfmt: + warn("stdout, stderr changing to (%s,%s)" % (outfmt, errfmt)) + CC.redirect_output(outfmt, errfmt) + except Exception as e: + warn("Failed to get and set output config: %s\n" % e) + + # send the cloud-config ready event + cc_path = cloudinit.get_ipath_cur('cloud_config') + cc_ready = cc.cfg.get("cc_ready_cmd", + ['initctl', 'emit', 'cloud-config', + '%s=%s' % (cloudinit.cfg_env_name, cc_path)]) + if cc_ready: + if isinstance(cc_ready, str): + cc_ready = ['sh', '-c', cc_ready] + subprocess.Popen(cc_ready).communicate() + + module_list = CC.read_cc_modules(cc.cfg, "cloud_init_modules") + + failures = [] + if len(module_list): + failures = CC.run_cc_modules(cc, module_list, log) + else: + msg = "no cloud_init_modules to run" + sys.stderr.write(msg + "\n") + log.debug(msg) + sys.exit(0) + + sys.exit(len(failures)) + +if __name__ == '__main__': + main() diff --git a/cloud-init-cfg.py b/cloud-init-cfg.py deleted file mode 100755 index 3a475c1c..00000000 --- a/cloud-init-cfg.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/python -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import sys -import cloudinit -import cloudinit.util as util -import cloudinit.CloudConfig as CC -import logging -import os - - -def Usage(out=sys.stdout): - out.write("Usage: %s name\n" % sys.argv[0]) - - -def main(): - # expect to be called with - # name [ freq [ args ] - # run the cloud-config job 'name' at with given args - # or - # read cloud config jobs from config (builtin -> system) - # and run all in order - - util.close_stdin() - - modename = "config" - - if len(sys.argv) < 2: - Usage(sys.stderr) - sys.exit(1) - if sys.argv[1] == "all": - name = "all" - if len(sys.argv) > 2: - modename = sys.argv[2] - else: - freq = None - run_args = [] - name = sys.argv[1] - if len(sys.argv) > 2: - freq = sys.argv[2] - if freq == "None": - freq = None - if len(sys.argv) > 3: - run_args = sys.argv[3:] - - cfg_path = cloudinit.get_ipath_cur("cloud_config") - cfg_env_name = cloudinit.cfg_env_name - if cfg_env_name in os.environ: - cfg_path = os.environ[cfg_env_name] - - cloud = cloudinit.CloudInit(ds_deps=[]) # ds_deps=[], get only cached - try: - cloud.get_data_source() - except cloudinit.DataSourceNotFoundException as e: - # there was no datasource found, theres nothing to do - sys.exit(0) - - cc = CC.CloudConfig(cfg_path, cloud) - - try: - (outfmt, errfmt) = CC.get_output_cfg(cc.cfg, modename) - CC.redirect_output(outfmt, errfmt) - except Exception as e: - err("Failed to get and set output config: %s\n" % e) - - cloudinit.logging_set_from_cfg(cc.cfg) - log = logging.getLogger() - log.info("cloud-init-cfg %s" % sys.argv[1:]) - - module_list = [] - if name == "all": - modlist_cfg_name = "cloud_%s_modules" % modename - module_list = CC.read_cc_modules(cc.cfg, modlist_cfg_name) - if not len(module_list): - err("no modules to run in cloud_config [%s]" % modename, log) - sys.exit(0) - else: - module_list.append([name, freq] + run_args) - - failures = CC.run_cc_modules(cc, module_list, log) - if len(failures): - err("errors running cloud_config [%s]: %s" % (modename, failures), log) - sys.exit(len(failures)) - - -def err(msg, log=None): - if log: - log.error(msg) - sys.stderr.write(msg + "\n") - - -def fail(msg, log=None): - err(msg, log) - sys.exit(1) - -if __name__ == '__main__': - main() diff --git a/cloud-init-query.py b/cloud-init-query.py deleted file mode 100755 index 856cf462..00000000 --- a/cloud-init-query.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/python -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import sys -import cloudinit -import cloudinit.CloudConfig - - -def Usage(out=sys.stdout): - out.write("Usage: %s name\n" % sys.argv[0]) - - -def main(): - # expect to be called with name of item to fetch - if len(sys.argv) != 2: - Usage(sys.stderr) - sys.exit(1) - - cfg_path = cloudinit.get_ipath_cur("cloud_config") - cc = cloudinit.CloudConfig.CloudConfig(cfg_path) - data = { - 'user_data': cc.cloud.get_userdata(), - 'user_data_raw': cc.cloud.get_userdata_raw(), - 'instance_id': cc.cloud.get_instance_id(), - } - - name = sys.argv[1].replace('-', '_') - - if name not in data: - sys.stderr.write("unknown name '%s'. Known values are:\n %s\n" % - (sys.argv[1], ' '.join(data.keys()))) - sys.exit(1) - - print data[name] - sys.exit(0) - -if __name__ == '__main__': - main() diff --git a/cloud-init.py b/cloud-init.py deleted file mode 100755 index 2acea3f8..00000000 --- a/cloud-init.py +++ /dev/null @@ -1,229 +0,0 @@ -#!/usr/bin/python -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import subprocess -import sys - -import cloudinit -import cloudinit.util as util -import cloudinit.CloudConfig as CC -import cloudinit.DataSource as ds -import cloudinit.netinfo as netinfo -import time -import traceback -import logging -import errno -import os - - -def warn(wstr): - sys.stderr.write("WARN:%s" % wstr) - - -def main(): - util.close_stdin() - - cmds = ("start", "start-local") - deps = {"start": (ds.DEP_FILESYSTEM, ds.DEP_NETWORK), - "start-local": (ds.DEP_FILESYSTEM, )} - - cmd = "" - if len(sys.argv) > 1: - cmd = sys.argv[1] - - cfg_path = None - if len(sys.argv) > 2: - # this is really for debugging only - # but you can invoke on development system with ./config/cloud.cfg - cfg_path = sys.argv[2] - - if not cmd in cmds: - sys.stderr.write("bad command %s. use one of %s\n" % (cmd, cmds)) - sys.exit(1) - - now = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) - try: - uptimef = open("/proc/uptime") - uptime = uptimef.read().split(" ")[0] - uptimef.close() - except IOError as e: - warn("unable to open /proc/uptime\n") - uptime = "na" - - cmdline_msg = None - cmdline_exc = None - if cmd == "start": - target = "%s.d/%s" % (cloudinit.system_config, - "91_kernel_cmdline_url.cfg") - if os.path.exists(target): - cmdline_msg = "cmdline: %s existed" % target - else: - cmdline = util.get_cmdline() - try: - (key, url, content) = cloudinit.get_cmdline_url( - cmdline=cmdline) - if key and content: - util.write_file(target, content, mode=0600) - cmdline_msg = ("cmdline: wrote %s from %s, %s" % - (target, key, url)) - elif key: - cmdline_msg = ("cmdline: %s, %s had no cloud-config" % - (key, url)) - except Exception: - cmdline_exc = ("cmdline: '%s' raised exception\n%s" % - (cmdline, traceback.format_exc())) - warn(cmdline_exc) - - try: - cfg = cloudinit.get_base_cfg(cfg_path) - except Exception as e: - warn("Failed to get base config. falling back to builtin: %s\n" % e) - try: - cfg = cloudinit.get_builtin_cfg() - except Exception as e: - warn("Unable to load builtin config\n") - raise - - try: - (outfmt, errfmt) = CC.get_output_cfg(cfg, "init") - CC.redirect_output(outfmt, errfmt) - except Exception as e: - warn("Failed to get and set output config: %s\n" % e) - - cloudinit.logging_set_from_cfg(cfg) - log = logging.getLogger() - - if cmdline_exc: - log.debug(cmdline_exc) - elif cmdline_msg: - log.debug(cmdline_msg) - - try: - cloudinit.initfs() - except Exception as e: - warn("failed to initfs, likely bad things to come: %s\n" % str(e)) - - nonet_path = "%s/%s" % (cloudinit.get_cpath("data"), "no-net") - - if cmd == "start": - print netinfo.debug_info() - - stop_files = (cloudinit.get_ipath_cur("obj_pkl"), nonet_path) - # if starting as the network start, there are cases - # where everything is already done for us, and it makes - # most sense to exit early and silently - for f in stop_files: - try: - fp = open(f, "r") - fp.close() - except: - continue - - log.debug("no need for cloud-init start to run (%s)\n", f) - sys.exit(0) - elif cmd == "start-local": - # cache is not instance specific, so it has to be purged - # but we want 'start' to benefit from a cache if - # a previous start-local populated one - manclean = util.get_cfg_option_bool(cfg, 'manual_cache_clean', False) - if manclean: - log.debug("not purging cache, manual_cache_clean = True") - cloudinit.purge_cache(not manclean) - - try: - os.unlink(nonet_path) - except OSError as e: - if e.errno != errno.ENOENT: - raise - - msg = "cloud-init %s running: %s. up %s seconds" % (cmd, now, uptime) - sys.stderr.write(msg + "\n") - sys.stderr.flush() - - log.info(msg) - - cloud = cloudinit.CloudInit(ds_deps=deps[cmd]) - - try: - cloud.get_data_source() - except cloudinit.DataSourceNotFoundException as e: - sys.stderr.write("no instance data found in %s\n" % cmd) - sys.exit(0) - - # set this as the current instance - cloud.set_cur_instance() - - # store the metadata - cloud.update_cache() - - msg = "found data source: %s" % cloud.datasource - sys.stderr.write(msg + "\n") - log.debug(msg) - - # parse the user data (ec2-run-userdata.py) - try: - ran = cloud.sem_and_run("consume_userdata", cloudinit.per_instance, - cloud.consume_userdata, [cloudinit.per_instance], False) - if not ran: - cloud.consume_userdata(cloudinit.per_always) - except: - warn("consuming user data failed!\n") - raise - - cfg_path = cloudinit.get_ipath_cur("cloud_config") - cc = CC.CloudConfig(cfg_path, cloud) - - # if the output config changed, update output and err - try: - outfmt_orig = outfmt - errfmt_orig = errfmt - (outfmt, errfmt) = CC.get_output_cfg(cc.cfg, "init") - if outfmt_orig != outfmt or errfmt_orig != errfmt: - warn("stdout, stderr changing to (%s,%s)" % (outfmt, errfmt)) - CC.redirect_output(outfmt, errfmt) - except Exception as e: - warn("Failed to get and set output config: %s\n" % e) - - # send the cloud-config ready event - cc_path = cloudinit.get_ipath_cur('cloud_config') - cc_ready = cc.cfg.get("cc_ready_cmd", - ['initctl', 'emit', 'cloud-config', - '%s=%s' % (cloudinit.cfg_env_name, cc_path)]) - if cc_ready: - if isinstance(cc_ready, str): - cc_ready = ['sh', '-c', cc_ready] - subprocess.Popen(cc_ready).communicate() - - module_list = CC.read_cc_modules(cc.cfg, "cloud_init_modules") - - failures = [] - if len(module_list): - failures = CC.run_cc_modules(cc, module_list, log) - else: - msg = "no cloud_init_modules to run" - sys.stderr.write(msg + "\n") - log.debug(msg) - sys.exit(0) - - sys.exit(len(failures)) - -if __name__ == '__main__': - main() -- cgit v1.2.3 From 8900f9cba622eeaf3810003c5a6ff7522312277b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 12:42:38 -0700 Subject: 1. Adding some new helper files that split off file inclusion, templating, importing, constant usage. 1. Move all datasources to a new sources directory 1. Rename some files to be more consistent with python file/module naming. --- cloudinit/CloudConfig/__init__.py | 274 ------------------ cloudinit/CloudConfig/cc_apt_pipelining.py | 53 ---- cloudinit/CloudConfig/cc_apt_update_upgrade.py | 241 ---------------- cloudinit/CloudConfig/cc_bootcmd.py | 48 ---- cloudinit/CloudConfig/cc_byobu.py | 77 ----- cloudinit/CloudConfig/cc_ca_certs.py | 90 ------ cloudinit/CloudConfig/cc_chef.py | 119 -------- cloudinit/CloudConfig/cc_disable_ec2_metadata.py | 30 -- cloudinit/CloudConfig/cc_final_message.py | 58 ---- cloudinit/CloudConfig/cc_foo.py | 29 -- cloudinit/CloudConfig/cc_grub_dpkg.py | 64 ----- cloudinit/CloudConfig/cc_keys_to_console.py | 42 --- cloudinit/CloudConfig/cc_landscape.py | 75 ----- cloudinit/CloudConfig/cc_locale.py | 54 ---- cloudinit/CloudConfig/cc_mcollective.py | 99 ------- cloudinit/CloudConfig/cc_mounts.py | 179 ------------ cloudinit/CloudConfig/cc_phone_home.py | 106 ------- cloudinit/CloudConfig/cc_puppet.py | 108 ------- cloudinit/CloudConfig/cc_resizefs.py | 108 ------- cloudinit/CloudConfig/cc_rightscale_userdata.py | 78 ----- cloudinit/CloudConfig/cc_rsyslog.py | 101 ------- cloudinit/CloudConfig/cc_runcmd.py | 32 --- cloudinit/CloudConfig/cc_salt_minion.py | 56 ---- cloudinit/CloudConfig/cc_scripts_per_boot.py | 34 --- cloudinit/CloudConfig/cc_scripts_per_instance.py | 34 --- cloudinit/CloudConfig/cc_scripts_per_once.py | 34 --- cloudinit/CloudConfig/cc_scripts_user.py | 34 --- cloudinit/CloudConfig/cc_set_hostname.py | 42 --- cloudinit/CloudConfig/cc_set_passwords.py | 129 --------- cloudinit/CloudConfig/cc_ssh.py | 106 ------- cloudinit/CloudConfig/cc_ssh_import_id.py | 50 ---- cloudinit/CloudConfig/cc_timezone.py | 67 ----- cloudinit/CloudConfig/cc_update_etc_hosts.py | 87 ------ cloudinit/CloudConfig/cc_update_hostname.py | 101 ------- cloudinit/DataSource.py | 214 -------------- cloudinit/DataSourceCloudStack.py | 92 ------ cloudinit/DataSourceConfigDrive.py | 231 --------------- cloudinit/DataSourceEc2.py | 217 -------------- cloudinit/DataSourceMAAS.py | 345 ----------------------- cloudinit/DataSourceNoCloud.py | 232 --------------- cloudinit/DataSourceOVF.py | 332 ---------------------- cloudinit/SshUtil.py | 227 --------------- cloudinit/UserDataHandler.py | 262 ----------------- cloudinit/constants.py | 37 +++ cloudinit/handlers/DataSource.py | 214 ++++++++++++++ cloudinit/handlers/DataSourceCloudStack.py | 92 ++++++ cloudinit/handlers/DataSourceConfigDrive.py | 231 +++++++++++++++ cloudinit/handlers/DataSourceEc2.py | 217 ++++++++++++++ cloudinit/handlers/DataSourceMAAS.py | 345 +++++++++++++++++++++++ cloudinit/handlers/DataSourceNoCloud.py | 232 +++++++++++++++ cloudinit/handlers/DataSourceOVF.py | 332 ++++++++++++++++++++++ cloudinit/handlers/__init__.py | 274 ++++++++++++++++++ cloudinit/handlers/cc_apt_pipelining.py | 53 ++++ cloudinit/handlers/cc_apt_update_upgrade.py | 241 ++++++++++++++++ cloudinit/handlers/cc_bootcmd.py | 48 ++++ cloudinit/handlers/cc_byobu.py | 77 +++++ cloudinit/handlers/cc_ca_certs.py | 90 ++++++ cloudinit/handlers/cc_chef.py | 119 ++++++++ cloudinit/handlers/cc_disable_ec2_metadata.py | 30 ++ cloudinit/handlers/cc_final_message.py | 58 ++++ cloudinit/handlers/cc_foo.py | 29 ++ cloudinit/handlers/cc_grub_dpkg.py | 64 +++++ cloudinit/handlers/cc_keys_to_console.py | 42 +++ cloudinit/handlers/cc_landscape.py | 75 +++++ cloudinit/handlers/cc_locale.py | 54 ++++ cloudinit/handlers/cc_mcollective.py | 99 +++++++ cloudinit/handlers/cc_mounts.py | 179 ++++++++++++ cloudinit/handlers/cc_phone_home.py | 106 +++++++ cloudinit/handlers/cc_puppet.py | 108 +++++++ cloudinit/handlers/cc_resizefs.py | 108 +++++++ cloudinit/handlers/cc_rightscale_userdata.py | 78 +++++ cloudinit/handlers/cc_rsyslog.py | 101 +++++++ cloudinit/handlers/cc_runcmd.py | 32 +++ cloudinit/handlers/cc_salt_minion.py | 56 ++++ cloudinit/handlers/cc_scripts_per_boot.py | 34 +++ cloudinit/handlers/cc_scripts_per_instance.py | 34 +++ cloudinit/handlers/cc_scripts_per_once.py | 34 +++ cloudinit/handlers/cc_scripts_user.py | 34 +++ cloudinit/handlers/cc_set_hostname.py | 42 +++ cloudinit/handlers/cc_set_passwords.py | 129 +++++++++ cloudinit/handlers/cc_ssh.py | 106 +++++++ cloudinit/handlers/cc_ssh_import_id.py | 50 ++++ cloudinit/handlers/cc_timezone.py | 67 +++++ cloudinit/handlers/cc_update_etc_hosts.py | 87 ++++++ cloudinit/handlers/cc_update_hostname.py | 101 +++++++ cloudinit/importer.py | 11 + cloudinit/includer.py | 65 +++++ cloudinit/log.py | 94 ++++++ cloudinit/ssh_util.py | 227 +++++++++++++++ cloudinit/templater.py | 17 ++ cloudinit/user_data.py | 262 +++++++++++++++++ 91 files changed, 5215 insertions(+), 4991 deletions(-) delete mode 100644 cloudinit/CloudConfig/__init__.py delete mode 100644 cloudinit/CloudConfig/cc_apt_pipelining.py delete mode 100644 cloudinit/CloudConfig/cc_apt_update_upgrade.py delete mode 100644 cloudinit/CloudConfig/cc_bootcmd.py delete mode 100644 cloudinit/CloudConfig/cc_byobu.py delete mode 100644 cloudinit/CloudConfig/cc_ca_certs.py delete mode 100644 cloudinit/CloudConfig/cc_chef.py delete mode 100644 cloudinit/CloudConfig/cc_disable_ec2_metadata.py delete mode 100644 cloudinit/CloudConfig/cc_final_message.py delete mode 100644 cloudinit/CloudConfig/cc_foo.py delete mode 100644 cloudinit/CloudConfig/cc_grub_dpkg.py delete mode 100644 cloudinit/CloudConfig/cc_keys_to_console.py delete mode 100644 cloudinit/CloudConfig/cc_landscape.py delete mode 100644 cloudinit/CloudConfig/cc_locale.py delete mode 100644 cloudinit/CloudConfig/cc_mcollective.py delete mode 100644 cloudinit/CloudConfig/cc_mounts.py delete mode 100644 cloudinit/CloudConfig/cc_phone_home.py delete mode 100644 cloudinit/CloudConfig/cc_puppet.py delete mode 100644 cloudinit/CloudConfig/cc_resizefs.py delete mode 100644 cloudinit/CloudConfig/cc_rightscale_userdata.py delete mode 100644 cloudinit/CloudConfig/cc_rsyslog.py delete mode 100644 cloudinit/CloudConfig/cc_runcmd.py delete mode 100644 cloudinit/CloudConfig/cc_salt_minion.py delete mode 100644 cloudinit/CloudConfig/cc_scripts_per_boot.py delete mode 100644 cloudinit/CloudConfig/cc_scripts_per_instance.py delete mode 100644 cloudinit/CloudConfig/cc_scripts_per_once.py delete mode 100644 cloudinit/CloudConfig/cc_scripts_user.py delete mode 100644 cloudinit/CloudConfig/cc_set_hostname.py delete mode 100644 cloudinit/CloudConfig/cc_set_passwords.py delete mode 100644 cloudinit/CloudConfig/cc_ssh.py delete mode 100644 cloudinit/CloudConfig/cc_ssh_import_id.py delete mode 100644 cloudinit/CloudConfig/cc_timezone.py delete mode 100644 cloudinit/CloudConfig/cc_update_etc_hosts.py delete mode 100644 cloudinit/CloudConfig/cc_update_hostname.py delete mode 100644 cloudinit/DataSource.py delete mode 100644 cloudinit/DataSourceCloudStack.py delete mode 100644 cloudinit/DataSourceConfigDrive.py delete mode 100644 cloudinit/DataSourceEc2.py delete mode 100644 cloudinit/DataSourceMAAS.py delete mode 100644 cloudinit/DataSourceNoCloud.py delete mode 100644 cloudinit/DataSourceOVF.py delete mode 100644 cloudinit/SshUtil.py delete mode 100644 cloudinit/UserDataHandler.py create mode 100644 cloudinit/constants.py create mode 100644 cloudinit/handlers/DataSource.py create mode 100644 cloudinit/handlers/DataSourceCloudStack.py create mode 100644 cloudinit/handlers/DataSourceConfigDrive.py create mode 100644 cloudinit/handlers/DataSourceEc2.py create mode 100644 cloudinit/handlers/DataSourceMAAS.py create mode 100644 cloudinit/handlers/DataSourceNoCloud.py create mode 100644 cloudinit/handlers/DataSourceOVF.py create mode 100644 cloudinit/handlers/__init__.py create mode 100644 cloudinit/handlers/cc_apt_pipelining.py create mode 100644 cloudinit/handlers/cc_apt_update_upgrade.py create mode 100644 cloudinit/handlers/cc_bootcmd.py create mode 100644 cloudinit/handlers/cc_byobu.py create mode 100644 cloudinit/handlers/cc_ca_certs.py create mode 100644 cloudinit/handlers/cc_chef.py create mode 100644 cloudinit/handlers/cc_disable_ec2_metadata.py create mode 100644 cloudinit/handlers/cc_final_message.py create mode 100644 cloudinit/handlers/cc_foo.py create mode 100644 cloudinit/handlers/cc_grub_dpkg.py create mode 100644 cloudinit/handlers/cc_keys_to_console.py create mode 100644 cloudinit/handlers/cc_landscape.py create mode 100644 cloudinit/handlers/cc_locale.py create mode 100644 cloudinit/handlers/cc_mcollective.py create mode 100644 cloudinit/handlers/cc_mounts.py create mode 100644 cloudinit/handlers/cc_phone_home.py create mode 100644 cloudinit/handlers/cc_puppet.py create mode 100644 cloudinit/handlers/cc_resizefs.py create mode 100644 cloudinit/handlers/cc_rightscale_userdata.py create mode 100644 cloudinit/handlers/cc_rsyslog.py create mode 100644 cloudinit/handlers/cc_runcmd.py create mode 100644 cloudinit/handlers/cc_salt_minion.py create mode 100644 cloudinit/handlers/cc_scripts_per_boot.py create mode 100644 cloudinit/handlers/cc_scripts_per_instance.py create mode 100644 cloudinit/handlers/cc_scripts_per_once.py create mode 100644 cloudinit/handlers/cc_scripts_user.py create mode 100644 cloudinit/handlers/cc_set_hostname.py create mode 100644 cloudinit/handlers/cc_set_passwords.py create mode 100644 cloudinit/handlers/cc_ssh.py create mode 100644 cloudinit/handlers/cc_ssh_import_id.py create mode 100644 cloudinit/handlers/cc_timezone.py create mode 100644 cloudinit/handlers/cc_update_etc_hosts.py create mode 100644 cloudinit/handlers/cc_update_hostname.py create mode 100644 cloudinit/importer.py create mode 100644 cloudinit/includer.py create mode 100644 cloudinit/log.py create mode 100644 cloudinit/ssh_util.py create mode 100644 cloudinit/templater.py create mode 100644 cloudinit/user_data.py diff --git a/cloudinit/CloudConfig/__init__.py b/cloudinit/CloudConfig/__init__.py deleted file mode 100644 index a16bdde6..00000000 --- a/cloudinit/CloudConfig/__init__.py +++ /dev/null @@ -1,274 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2008-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Chuck Short -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -import yaml -import cloudinit -import cloudinit.util as util -import sys -import traceback -import os -import subprocess -import time - -per_instance = cloudinit.per_instance -per_always = cloudinit.per_always -per_once = cloudinit.per_once - - -class CloudConfig(): - cfgfile = None - cfg = None - - def __init__(self, cfgfile, cloud=None, ds_deps=None): - if cloud == None: - self.cloud = cloudinit.CloudInit(ds_deps) - self.cloud.get_data_source() - else: - self.cloud = cloud - self.cfg = self.get_config_obj(cfgfile) - - def get_config_obj(self, cfgfile): - try: - cfg = util.read_conf(cfgfile) - except: - # TODO: this 'log' could/should be passed in - cloudinit.log.critical("Failed loading of cloud config '%s'. " - "Continuing with empty config\n" % cfgfile) - cloudinit.log.debug(traceback.format_exc() + "\n") - cfg = None - if cfg is None: - cfg = {} - - try: - ds_cfg = self.cloud.datasource.get_config_obj() - except: - ds_cfg = {} - - cfg = util.mergedict(cfg, ds_cfg) - return(util.mergedict(cfg, self.cloud.cfg)) - - def handle(self, name, args, freq=None): - try: - mod = __import__("cc_" + name.replace("-", "_"), globals()) - def_freq = getattr(mod, "frequency", per_instance) - handler = getattr(mod, "handle") - - if not freq: - freq = def_freq - - self.cloud.sem_and_run("config-" + name, freq, handler, - [name, self.cfg, self.cloud, cloudinit.log, args]) - except: - raise - - -# reads a cloudconfig module list, returns -# a 2 dimensional array suitable to pass to run_cc_modules -def read_cc_modules(cfg, name): - if name not in cfg: - return([]) - module_list = [] - # create 'module_list', an array of arrays - # where array[0] = config - # array[1] = freq - # array[2:] = arguemnts - for item in cfg[name]: - if isinstance(item, str): - module_list.append((item,)) - elif isinstance(item, list): - module_list.append(item) - else: - raise TypeError("failed to read '%s' item in config") - return(module_list) - - -def run_cc_modules(cc, module_list, log): - failures = [] - for cfg_mod in module_list: - name = cfg_mod[0] - freq = None - run_args = [] - if len(cfg_mod) > 1: - freq = cfg_mod[1] - if len(cfg_mod) > 2: - run_args = cfg_mod[2:] - - try: - log.debug("handling %s with freq=%s and args=%s" % - (name, freq, run_args)) - cc.handle(name, run_args, freq=freq) - except: - log.warn(traceback.format_exc()) - log.error("config handling of %s, %s, %s failed\n" % - (name, freq, run_args)) - failures.append(name) - - return(failures) - - -# always returns well formated values -# cfg is expected to have an entry 'output' in it, which is a dictionary -# that includes entries for 'init', 'config', 'final' or 'all' -# init: /var/log/cloud.out -# config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ] -# final: -# output: "| logger -p" -# error: "> /dev/null" -# this returns the specific 'mode' entry, cleanly formatted, with value -# None if if none is given -def get_output_cfg(cfg, mode="init"): - ret = [None, None] - if not 'output' in cfg: - return ret - - outcfg = cfg['output'] - if mode in outcfg: - modecfg = outcfg[mode] - else: - if 'all' not in outcfg: - return ret - # if there is a 'all' item in the output list - # then it applies to all users of this (init, config, final) - modecfg = outcfg['all'] - - # if value is a string, it specifies stdout and stderr - if isinstance(modecfg, str): - ret = [modecfg, modecfg] - - # if its a list, then we expect (stdout, stderr) - if isinstance(modecfg, list): - if len(modecfg) > 0: - ret[0] = modecfg[0] - if len(modecfg) > 1: - ret[1] = modecfg[1] - - # if it is a dictionary, expect 'out' and 'error' - # items, which indicate out and error - if isinstance(modecfg, dict): - if 'output' in modecfg: - ret[0] = modecfg['output'] - if 'error' in modecfg: - ret[1] = modecfg['error'] - - # if err's entry == "&1", then make it same as stdout - # as in shell syntax of "echo foo >/dev/null 2>&1" - if ret[1] == "&1": - ret[1] = ret[0] - - swlist = [">>", ">", "|"] - for i in range(len(ret)): - if not ret[i]: - continue - val = ret[i].lstrip() - found = False - for s in swlist: - if val.startswith(s): - val = "%s %s" % (s, val[len(s):].strip()) - found = True - break - if not found: - # default behavior is append - val = "%s %s" % (">>", val.strip()) - ret[i] = val - - return(ret) - - -# redirect_output(outfmt, errfmt, orig_out, orig_err) -# replace orig_out and orig_err with filehandles specified in outfmt or errfmt -# fmt can be: -# > FILEPATH -# >> FILEPATH -# | program [ arg1 [ arg2 [ ... ] ] ] -# -# with a '|', arguments are passed to shell, so one level of -# shell escape is required. -def redirect_output(outfmt, errfmt, o_out=sys.stdout, o_err=sys.stderr): - if outfmt: - (mode, arg) = outfmt.split(" ", 1) - if mode == ">" or mode == ">>": - owith = "ab" - if mode == ">": - owith = "wb" - new_fp = open(arg, owith) - elif mode == "|": - proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin - else: - raise TypeError("invalid type for outfmt: %s" % outfmt) - - if o_out: - os.dup2(new_fp.fileno(), o_out.fileno()) - if errfmt == outfmt: - os.dup2(new_fp.fileno(), o_err.fileno()) - return - - if errfmt: - (mode, arg) = errfmt.split(" ", 1) - if mode == ">" or mode == ">>": - owith = "ab" - if mode == ">": - owith = "wb" - new_fp = open(arg, owith) - elif mode == "|": - proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin - else: - raise TypeError("invalid type for outfmt: %s" % outfmt) - - if o_err: - os.dup2(new_fp.fileno(), o_err.fileno()) - return - - -def run_per_instance(name, func, args, clear_on_fail=False): - semfile = "%s/%s" % (cloudinit.get_ipath_cur("data"), name) - if os.path.exists(semfile): - return - - util.write_file(semfile, str(time.time())) - try: - func(*args) - except: - if clear_on_fail: - os.unlink(semfile) - raise - - -# apt_get top level command (install, update...), and args to pass it -def apt_get(tlc, args=None): - if args is None: - args = [] - e = os.environ.copy() - e['DEBIAN_FRONTEND'] = 'noninteractive' - cmd = ['apt-get', '--option', 'Dpkg::Options::=--force-confold', - '--assume-yes', tlc] - cmd.extend(args) - subprocess.check_call(cmd, env=e) - - -def update_package_sources(): - run_per_instance("update-sources", apt_get, ("update",)) - - -def install_packages(pkglist): - update_package_sources() - apt_get("install", pkglist) diff --git a/cloudinit/CloudConfig/cc_apt_pipelining.py b/cloudinit/CloudConfig/cc_apt_pipelining.py deleted file mode 100644 index 0286a9ae..00000000 --- a/cloudinit/CloudConfig/cc_apt_pipelining.py +++ /dev/null @@ -1,53 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# -# Author: Ben Howard -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance - -frequency = per_instance -default_file = "/etc/apt/apt.conf.d/90cloud-init-pipelining" - - -def handle(_name, cfg, _cloud, log, _args): - - apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) - apt_pipe_value = str(apt_pipe_value).lower() - - if apt_pipe_value == "false": - write_apt_snippet("0", log) - - elif apt_pipe_value in ("none", "unchanged", "os"): - return - - elif apt_pipe_value in str(range(0, 6)): - write_apt_snippet(apt_pipe_value, log) - - else: - log.warn("Invalid option for apt_pipeling: %s" % apt_pipe_value) - - -def write_apt_snippet(setting, log, f_name=default_file): - """ Writes f_name with apt pipeline depth 'setting' """ - - acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' - file_contents = ("//Written by cloud-init per 'apt_pipelining'\n" - + (acquire_pipeline_depth % setting)) - - util.write_file(f_name, file_contents) - - log.debug("Wrote %s with APT pipeline setting" % f_name) diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py deleted file mode 100644 index a7049bce..00000000 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ /dev/null @@ -1,241 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import traceback -import os -import glob -import cloudinit.CloudConfig as cc - - -def handle(_name, cfg, cloud, log, _args): - update = util.get_cfg_option_bool(cfg, 'apt_update', False) - upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) - - release = get_release() - - mirror = find_apt_mirror(cloud, cfg) - - log.debug("selected mirror at: %s" % mirror) - - if not util.get_cfg_option_bool(cfg, \ - 'apt_preserve_sources_list', False): - generate_sources_list(release, mirror) - old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', \ - "archive.ubuntu.com/ubuntu") - rename_apt_lists(old_mir, mirror) - - # set up proxy - proxy = cfg.get("apt_proxy", None) - proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy" - if proxy: - try: - contents = "Acquire::HTTP::Proxy \"%s\";\n" - with open(proxy_filename, "w") as fp: - fp.write(contents % proxy) - except Exception as e: - log.warn("Failed to write proxy to %s" % proxy_filename) - elif os.path.isfile(proxy_filename): - os.unlink(proxy_filename) - - # process 'apt_sources' - if 'apt_sources' in cfg: - errors = add_sources(cfg['apt_sources'], - {'MIRROR': mirror, 'RELEASE': release}) - for e in errors: - log.warn("Source Error: %s\n" % ':'.join(e)) - - dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False) - if dconf_sel: - log.debug("setting debconf selections per cloud config") - try: - util.subp(('debconf-set-selections', '-'), dconf_sel) - except: - log.error("Failed to run debconf-set-selections") - log.debug(traceback.format_exc()) - - pkglist = util.get_cfg_option_list_or_str(cfg, 'packages', []) - - errors = [] - if update or len(pkglist) or upgrade: - try: - cc.update_package_sources() - except subprocess.CalledProcessError as e: - log.warn("apt-get update failed") - log.debug(traceback.format_exc()) - errors.append(e) - - if upgrade: - try: - cc.apt_get("upgrade") - except subprocess.CalledProcessError as e: - log.warn("apt upgrade failed") - log.debug(traceback.format_exc()) - errors.append(e) - - if len(pkglist): - try: - cc.install_packages(pkglist) - except subprocess.CalledProcessError as e: - log.warn("Failed to install packages: %s " % pkglist) - log.debug(traceback.format_exc()) - errors.append(e) - - if len(errors): - raise errors[0] - - return(True) - - -def mirror2lists_fileprefix(mirror): - string = mirror - # take of http:// or ftp:// - if string.endswith("/"): - string = string[0:-1] - pos = string.find("://") - if pos >= 0: - string = string[pos + 3:] - string = string.replace("/", "_") - return string - - -def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"): - oprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(omirror)) - nprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(new_mirror)) - if(oprefix == nprefix): - return - olen = len(oprefix) - for filename in glob.glob("%s_*" % oprefix): - os.rename(filename, "%s%s" % (nprefix, filename[olen:])) - - -def get_release(): - stdout, _stderr = subprocess.Popen(['lsb_release', '-cs'], - stdout=subprocess.PIPE).communicate() - return(str(stdout).strip()) - - -def generate_sources_list(codename, mirror): - util.render_to_file('sources.list', '/etc/apt/sources.list', \ - {'mirror': mirror, 'codename': codename}) - - -def add_sources(srclist, searchList=None): - """ - add entries in /etc/apt/sources.list.d for each abbreviated - sources.list entry in 'srclist'. When rendering template, also - include the values in dictionary searchList - """ - if searchList is None: - searchList = {} - elst = [] - - for ent in srclist: - if 'source' not in ent: - elst.append(["", "missing source"]) - continue - - source = ent['source'] - if source.startswith("ppa:"): - try: - util.subp(["add-apt-repository", source]) - except: - elst.append([source, "add-apt-repository failed"]) - continue - - source = util.render_string(source, searchList) - - if 'filename' not in ent: - ent['filename'] = 'cloud_config_sources.list' - - if not ent['filename'].startswith("/"): - ent['filename'] = "%s/%s" % \ - ("/etc/apt/sources.list.d/", ent['filename']) - - if ('keyid' in ent and 'key' not in ent): - ks = "keyserver.ubuntu.com" - if 'keyserver' in ent: - ks = ent['keyserver'] - try: - ent['key'] = util.getkeybyid(ent['keyid'], ks) - except: - elst.append([source, "failed to get key from %s" % ks]) - continue - - if 'key' in ent: - try: - util.subp(('apt-key', 'add', '-'), ent['key']) - except: - elst.append([source, "failed add key"]) - - try: - util.write_file(ent['filename'], source + "\n", omode="ab") - except: - elst.append([source, "failed write to file %s" % ent['filename']]) - - return(elst) - - -def find_apt_mirror(cloud, cfg): - """ find an apt_mirror given the cloud and cfg provided """ - - # TODO: distro and defaults should be configurable - distro = "ubuntu" - defaults = { - 'ubuntu': "http://archive.ubuntu.com/ubuntu", - 'debian': "http://archive.debian.org/debian", - } - mirror = None - - cfg_mirror = cfg.get("apt_mirror", None) - if cfg_mirror: - mirror = cfg["apt_mirror"] - elif "apt_mirror_search" in cfg: - mirror = util.search_for_mirror(cfg['apt_mirror_search']) - else: - if cloud: - mirror = cloud.get_mirror() - - mydom = "" - - doms = [] - - if not mirror and cloud: - # if we have a fqdn, then search its domain portion first - (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - mydom = ".".join(fqdn.split(".")[1:]) - if mydom: - doms.append(".%s" % mydom) - - if not mirror: - doms.extend((".localdomain", "",)) - - mirror_list = [] - mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) - for post in doms: - mirror_list.append(mirrorfmt % post) - - mirror = util.search_for_mirror(mirror_list) - - if not mirror: - mirror = defaults[distro] - - return mirror diff --git a/cloudinit/CloudConfig/cc_bootcmd.py b/cloudinit/CloudConfig/cc_bootcmd.py deleted file mode 100644 index f584da02..00000000 --- a/cloudinit/CloudConfig/cc_bootcmd.py +++ /dev/null @@ -1,48 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import cloudinit.util as util -import subprocess -import tempfile -import os -from cloudinit.CloudConfig import per_always -frequency = per_always - - -def handle(_name, cfg, cloud, log, _args): - if "bootcmd" not in cfg: - return - - try: - content = util.shellify(cfg["bootcmd"]) - tmpf = tempfile.TemporaryFile() - tmpf.write(content) - tmpf.seek(0) - except: - log.warn("failed to shellify bootcmd") - raise - - try: - env = os.environ.copy() - env['INSTANCE_ID'] = cloud.get_instance_id() - subprocess.check_call(['/bin/sh'], env=env, stdin=tmpf) - tmpf.close() - except: - log.warn("failed to run commands from bootcmd") - raise diff --git a/cloudinit/CloudConfig/cc_byobu.py b/cloudinit/CloudConfig/cc_byobu.py deleted file mode 100644 index e821b261..00000000 --- a/cloudinit/CloudConfig/cc_byobu.py +++ /dev/null @@ -1,77 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import traceback - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - value = args[0] - else: - value = util.get_cfg_option_str(cfg, "byobu_by_default", "") - - if not value: - return - - if value == "user" or value == "system": - value = "enable-%s" % value - - valid = ("enable-user", "enable-system", "enable", - "disable-user", "disable-system", "disable") - if not value in valid: - log.warn("Unknown value %s for byobu_by_default" % value) - - mod_user = value.endswith("-user") - mod_sys = value.endswith("-system") - if value.startswith("enable"): - bl_inst = "install" - dc_val = "byobu byobu/launch-by-default boolean true" - mod_sys = True - else: - if value == "disable": - mod_user = True - mod_sys = True - bl_inst = "uninstall" - dc_val = "byobu byobu/launch-by-default boolean false" - - shcmd = "" - if mod_user: - user = util.get_cfg_option_str(cfg, "user", "ubuntu") - shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst) - shcmd += " || X=$(($X+1)); " - if mod_sys: - shcmd += "echo \"%s\" | debconf-set-selections" % dc_val - shcmd += " && dpkg-reconfigure byobu --frontend=noninteractive" - shcmd += " || X=$(($X+1)); " - - cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] - - log.debug("setting byobu to %s" % value) - - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) - except OSError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/CloudConfig/cc_ca_certs.py b/cloudinit/CloudConfig/cc_ca_certs.py deleted file mode 100644 index 3af6238a..00000000 --- a/cloudinit/CloudConfig/cc_ca_certs.py +++ /dev/null @@ -1,90 +0,0 @@ -# vi: ts=4 expandtab -# -# Author: Mike Milner -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import os -from subprocess import check_call -from cloudinit.util import (write_file, get_cfg_option_list_or_str, - delete_dir_contents, subp) - -CA_CERT_PATH = "/usr/share/ca-certificates/" -CA_CERT_FILENAME = "cloud-init-ca-certs.crt" -CA_CERT_CONFIG = "/etc/ca-certificates.conf" -CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" - - -def update_ca_certs(): - """ - Updates the CA certificate cache on the current machine. - """ - check_call(["update-ca-certificates"]) - - -def add_ca_certs(certs): - """ - Adds certificates to the system. To actually apply the new certificates - you must also call L{update_ca_certs}. - - @param certs: A list of certificate strings. - """ - if certs: - cert_file_contents = "\n".join(certs) - cert_file_fullpath = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) - write_file(cert_file_fullpath, cert_file_contents, mode=0644) - # Append cert filename to CA_CERT_CONFIG file. - write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="a") - - -def remove_default_ca_certs(): - """ - Removes all default trusted CA certificates from the system. To actually - apply the change you must also call L{update_ca_certs}. - """ - delete_dir_contents(CA_CERT_PATH) - delete_dir_contents(CA_CERT_SYSTEM_PATH) - write_file(CA_CERT_CONFIG, "", mode=0644) - debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" - subp(('debconf-set-selections', '-'), debconf_sel) - - -def handle(_name, cfg, _cloud, log, _args): - """ - Call to handle ca-cert sections in cloud-config file. - - @param name: The module name "ca-cert" from cloud.cfg - @param cfg: A nested dict containing the entire cloud config contents. - @param cloud: The L{CloudInit} object in use. - @param log: Pre-initialized Python logger object to use for logging. - @param args: Any module arguments from cloud.cfg - """ - # If there isn't a ca-certs section in the configuration don't do anything - if "ca-certs" not in cfg: - return - ca_cert_cfg = cfg['ca-certs'] - - # If there is a remove-defaults option set to true, remove the system - # default trusted CA certs first. - if ca_cert_cfg.get("remove-defaults", False): - log.debug("removing default certificates") - remove_default_ca_certs() - - # If we are given any new trusted CA certs to add, add them. - if "trusted" in ca_cert_cfg: - trusted_certs = get_cfg_option_list_or_str(ca_cert_cfg, "trusted") - if trusted_certs: - log.debug("adding %d certificates" % len(trusted_certs)) - add_ca_certs(trusted_certs) - - # Update the system with the new cert configuration. - update_ca_certs() diff --git a/cloudinit/CloudConfig/cc_chef.py b/cloudinit/CloudConfig/cc_chef.py deleted file mode 100644 index 941e04fe..00000000 --- a/cloudinit/CloudConfig/cc_chef.py +++ /dev/null @@ -1,119 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Avishai Ish-Shalom -# Author: Mike Moulton -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import subprocess -import json -import cloudinit.CloudConfig as cc -import cloudinit.util as util - -ruby_version_default = "1.8" - - -def handle(_name, cfg, cloud, log, _args): - # If there isn't a chef key in the configuration don't do anything - if 'chef' not in cfg: - return - chef_cfg = cfg['chef'] - - # ensure the chef directories we use exist - mkdirs(['/etc/chef', '/var/log/chef', '/var/lib/chef', - '/var/cache/chef', '/var/backups/chef', '/var/run/chef']) - - # set the validation key based on the presence of either 'validation_key' - # or 'validation_cert'. In the case where both exist, 'validation_key' - # takes precedence - for key in ('validation_key', 'validation_cert'): - if key in chef_cfg and chef_cfg[key]: - with open('/etc/chef/validation.pem', 'w') as validation_key_fh: - validation_key_fh.write(chef_cfg[key]) - break - - # create the chef config from template - util.render_to_file('chef_client.rb', '/etc/chef/client.rb', - {'server_url': chef_cfg['server_url'], - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', - cloud.datasource.get_instance_id()), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - '_default'), - 'validation_name': chef_cfg['validation_name']}) - - # set the firstboot json - with open('/etc/chef/firstboot.json', 'w') as firstboot_json_fh: - initial_json = {} - if 'run_list' in chef_cfg: - initial_json['run_list'] = chef_cfg['run_list'] - if 'initial_attributes' in chef_cfg: - initial_attributes = chef_cfg['initial_attributes'] - for k in initial_attributes.keys(): - initial_json[k] = initial_attributes[k] - firstboot_json_fh.write(json.dumps(initial_json)) - - # If chef is not installed, we install chef based on 'install_type' - if not os.path.isfile('/usr/bin/chef-client'): - install_type = util.get_cfg_option_str(chef_cfg, 'install_type', - 'packages') - if install_type == "gems": - # this will install and run the chef-client from gems - chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) - ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', - ruby_version_default) - install_chef_from_gems(ruby_version, chef_version) - # and finally, run chef-client - log.debug('running chef-client') - subprocess.check_call(['/usr/bin/chef-client', '-d', '-i', '1800', - '-s', '20']) - else: - # this will install and run the chef-client from packages - cc.install_packages(('chef',)) - - -def get_ruby_packages(version): - # return a list of packages needed to install ruby at version - pkgs = ['ruby%s' % version, 'ruby%s-dev' % version] - if version == "1.8": - pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8')) - return(pkgs) - - -def install_chef_from_gems(ruby_version, chef_version=None): - cc.install_packages(get_ruby_packages(ruby_version)) - if not os.path.exists('/usr/bin/gem'): - os.symlink('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') - if not os.path.exists('/usr/bin/ruby'): - os.symlink('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') - if chef_version: - subprocess.check_call(['/usr/bin/gem', 'install', 'chef', - '-v %s' % chef_version, '--no-ri', - '--no-rdoc', '--bindir', '/usr/bin', '-q']) - else: - subprocess.check_call(['/usr/bin/gem', 'install', 'chef', - '--no-ri', '--no-rdoc', '--bindir', - '/usr/bin', '-q']) - - -def ensure_dir(d): - if not os.path.exists(d): - os.makedirs(d) - - -def mkdirs(dirs): - for d in dirs: - ensure_dir(d) diff --git a/cloudinit/CloudConfig/cc_disable_ec2_metadata.py b/cloudinit/CloudConfig/cc_disable_ec2_metadata.py deleted file mode 100644 index 6b31ea8e..00000000 --- a/cloudinit/CloudConfig/cc_disable_ec2_metadata.py +++ /dev/null @@ -1,30 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import cloudinit.util as util -import subprocess -from cloudinit.CloudConfig import per_always - -frequency = per_always - - -def handle(_name, cfg, _cloud, _log, _args): - if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False): - fwall = "route add -host 169.254.169.254 reject" - subprocess.call(fwall.split(' ')) diff --git a/cloudinit/CloudConfig/cc_final_message.py b/cloudinit/CloudConfig/cc_final_message.py deleted file mode 100644 index abb4ca32..00000000 --- a/cloudinit/CloudConfig/cc_final_message.py +++ /dev/null @@ -1,58 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.CloudConfig import per_always -import sys -from cloudinit import util, boot_finished -import time - -frequency = per_always - -final_message = "cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds" - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - msg_in = args[0] - else: - msg_in = util.get_cfg_option_str(cfg, "final_message", final_message) - - try: - uptimef = open("/proc/uptime") - uptime = uptimef.read().split(" ")[0] - uptimef.close() - except IOError as e: - log.warn("unable to open /proc/uptime\n") - uptime = "na" - - try: - ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) - except: - ts = "na" - - try: - subs = {'UPTIME': uptime, 'TIMESTAMP': ts} - sys.stdout.write("%s\n" % util.render_string(msg_in, subs)) - except Exception as e: - log.warn("failed to render string to stdout: %s" % e) - - fp = open(boot_finished, "wb") - fp.write(uptime + "\n") - fp.close() diff --git a/cloudinit/CloudConfig/cc_foo.py b/cloudinit/CloudConfig/cc_foo.py deleted file mode 100644 index 35ec3fa7..00000000 --- a/cloudinit/CloudConfig/cc_foo.py +++ /dev/null @@ -1,29 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -#import cloudinit -#import cloudinit.util as util -from cloudinit.CloudConfig import per_instance - -frequency = per_instance - - -def handle(_name, _cfg, _cloud, _log, _args): - print "hi" diff --git a/cloudinit/CloudConfig/cc_grub_dpkg.py b/cloudinit/CloudConfig/cc_grub_dpkg.py deleted file mode 100644 index 9f3a7eaf..00000000 --- a/cloudinit/CloudConfig/cc_grub_dpkg.py +++ /dev/null @@ -1,64 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import traceback -import os - - -def handle(_name, cfg, _cloud, log, _args): - idevs = None - idevs_empty = None - - if "grub-dpkg" in cfg: - idevs = util.get_cfg_option_str(cfg["grub-dpkg"], - "grub-pc/install_devices", None) - idevs_empty = util.get_cfg_option_str(cfg["grub-dpkg"], - "grub-pc/install_devices_empty", None) - - if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): - if idevs == None: - idevs = "" - if idevs_empty == None: - idevs_empty = "true" - else: - if idevs_empty == None: - idevs_empty = "false" - if idevs == None: - idevs = "/dev/sda" - for dev in ("/dev/sda", "/dev/vda", "/dev/sda1", "/dev/vda1"): - if os.path.exists(dev): - idevs = dev - break - - # now idevs and idevs_empty are set to determined values - # or, those set by user - - dconf_sel = "grub-pc grub-pc/install_devices string %s\n" % idevs + \ - "grub-pc grub-pc/install_devices_empty boolean %s\n" % idevs_empty - log.debug("setting grub debconf-set-selections with '%s','%s'" % - (idevs, idevs_empty)) - - try: - util.subp(('debconf-set-selections'), dconf_sel) - except: - log.error("Failed to run debconf-set-selections for grub-dpkg") - log.debug(traceback.format_exc()) diff --git a/cloudinit/CloudConfig/cc_keys_to_console.py b/cloudinit/CloudConfig/cc_keys_to_console.py deleted file mode 100644 index 73a477c0..00000000 --- a/cloudinit/CloudConfig/cc_keys_to_console.py +++ /dev/null @@ -1,42 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.CloudConfig import per_instance -import cloudinit.util as util -import subprocess - -frequency = per_instance - - -def handle(_name, cfg, _cloud, log, _args): - cmd = ['/usr/lib/cloud-init/write-ssh-key-fingerprints'] - fp_blacklist = util.get_cfg_option_list_or_str(cfg, - "ssh_fp_console_blacklist", []) - key_blacklist = util.get_cfg_option_list_or_str(cfg, - "ssh_key_console_blacklist", ["ssh-dss"]) - try: - confp = open('/dev/console', "wb") - cmd.append(','.join(fp_blacklist)) - cmd.append(','.join(key_blacklist)) - subprocess.call(cmd, stdout=confp) - confp.close() - except: - log.warn("writing keys to console value") - raise diff --git a/cloudinit/CloudConfig/cc_landscape.py b/cloudinit/CloudConfig/cc_landscape.py deleted file mode 100644 index a4113cbe..00000000 --- a/cloudinit/CloudConfig/cc_landscape.py +++ /dev/null @@ -1,75 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import os.path -from cloudinit.CloudConfig import per_instance -from configobj import ConfigObj - -frequency = per_instance - -lsc_client_cfg_file = "/etc/landscape/client.conf" - -# defaults taken from stock client.conf in landscape-client 11.07.1.1-0ubuntu2 -lsc_builtincfg = { - 'client': { - 'log_level': "info", - 'url': "https://landscape.canonical.com/message-system", - 'ping_url': "http://landscape.canonical.com/ping", - 'data_path': "/var/lib/landscape/client", - } -} - - -def handle(_name, cfg, _cloud, log, _args): - """ - Basically turn a top level 'landscape' entry with a 'client' dict - and render it to ConfigObj format under '[client]' section in - /etc/landscape/client.conf - """ - - ls_cloudcfg = cfg.get("landscape", {}) - - if not isinstance(ls_cloudcfg, dict): - raise(Exception("'landscape' existed in config, but not a dict")) - - merged = mergeTogether([lsc_builtincfg, lsc_client_cfg_file, ls_cloudcfg]) - - if not os.path.isdir(os.path.dirname(lsc_client_cfg_file)): - os.makedirs(os.path.dirname(lsc_client_cfg_file)) - - with open(lsc_client_cfg_file, "w") as fp: - merged.write(fp) - - log.debug("updated %s" % lsc_client_cfg_file) - - -def mergeTogether(objs): - """ - merge together ConfigObj objects or things that ConfigObj() will take in - later entries override earlier - """ - cfg = ConfigObj({}) - for obj in objs: - if isinstance(obj, ConfigObj): - cfg.merge(obj) - else: - cfg.merge(ConfigObj(obj)) - return cfg diff --git a/cloudinit/CloudConfig/cc_locale.py b/cloudinit/CloudConfig/cc_locale.py deleted file mode 100644 index 2bb22fdb..00000000 --- a/cloudinit/CloudConfig/cc_locale.py +++ /dev/null @@ -1,54 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import os.path -import subprocess -import traceback - - -def apply_locale(locale, cfgfile): - if os.path.exists('/usr/sbin/locale-gen'): - subprocess.Popen(['locale-gen', locale]).communicate() - if os.path.exists('/usr/sbin/update-locale'): - subprocess.Popen(['update-locale', locale]).communicate() - - util.render_to_file('default-locale', cfgfile, {'locale': locale}) - - -def handle(_name, cfg, cloud, log, args): - if len(args) != 0: - locale = args[0] - else: - locale = util.get_cfg_option_str(cfg, "locale", cloud.get_locale()) - - locale_cfgfile = util.get_cfg_option_str(cfg, "locale_configfile", - "/etc/default/locale") - - if not locale: - return - - log.debug("setting locale to %s" % locale) - - try: - apply_locale(locale, locale_cfgfile) - except Exception as e: - log.debug(traceback.format_exc(e)) - raise Exception("failed to apply locale %s" % locale) diff --git a/cloudinit/CloudConfig/cc_mcollective.py b/cloudinit/CloudConfig/cc_mcollective.py deleted file mode 100644 index a2a6230c..00000000 --- a/cloudinit/CloudConfig/cc_mcollective.py +++ /dev/null @@ -1,99 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Marc Cluet -# Based on code by Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import subprocess -import StringIO -import ConfigParser -import cloudinit.CloudConfig as cc -import cloudinit.util as util - -pubcert_file = "/etc/mcollective/ssl/server-public.pem" -pricert_file = "/etc/mcollective/ssl/server-private.pem" - - -# Our fake header section -class FakeSecHead(object): - def __init__(self, fp): - self.fp = fp - self.sechead = '[nullsection]\n' - - def readline(self): - if self.sechead: - try: - return self.sechead - finally: - self.sechead = None - else: - return self.fp.readline() - - -def handle(_name, cfg, _cloud, _log, _args): - # If there isn't a mcollective key in the configuration don't do anything - if 'mcollective' not in cfg: - return - mcollective_cfg = cfg['mcollective'] - # Start by installing the mcollective package ... - cc.install_packages(("mcollective",)) - - # ... and then update the mcollective configuration - if 'conf' in mcollective_cfg: - # Create object for reading server.cfg values - mcollective_config = ConfigParser.ConfigParser() - # Read server.cfg values from original file in order to be able to mix - # the rest up - mcollective_config.readfp(FakeSecHead(open('/etc/mcollective/' - 'server.cfg'))) - for cfg_name, cfg in mcollective_cfg['conf'].iteritems(): - if cfg_name == 'public-cert': - util.write_file(pubcert_file, cfg, mode=0644) - mcollective_config.set(cfg_name, - 'plugin.ssl_server_public', pubcert_file) - mcollective_config.set(cfg_name, 'securityprovider', 'ssl') - elif cfg_name == 'private-cert': - util.write_file(pricert_file, cfg, mode=0600) - mcollective_config.set(cfg_name, - 'plugin.ssl_server_private', pricert_file) - mcollective_config.set(cfg_name, 'securityprovider', 'ssl') - else: - # Iterate throug the config items, we'll use ConfigParser.set - # to overwrite or create new items as needed - for o, v in cfg.iteritems(): - mcollective_config.set(cfg_name, o, v) - # We got all our config as wanted we'll rename - # the previous server.cfg and create our new one - os.rename('/etc/mcollective/server.cfg', - '/etc/mcollective/server.cfg.old') - outputfile = StringIO.StringIO() - mcollective_config.write(outputfile) - # Now we got the whole file, write to disk except first line - # Note below, that we've just used ConfigParser because it generally - # works. Below, we remove the initial 'nullsection' header - # and then change 'key = value' to 'key: value'. The global - # search and replace of '=' with ':' could be problematic though. - # this most likely needs fixing. - util.write_file('/etc/mcollective/server.cfg', - outputfile.getvalue().replace('[nullsection]\n', '').replace(' =', - ':'), - mode=0644) - - # Start mcollective - subprocess.check_call(['service', 'mcollective', 'start']) diff --git a/cloudinit/CloudConfig/cc_mounts.py b/cloudinit/CloudConfig/cc_mounts.py deleted file mode 100644 index 6cdd74e8..00000000 --- a/cloudinit/CloudConfig/cc_mounts.py +++ /dev/null @@ -1,179 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import os -import re -from string import whitespace # pylint: disable=W0402 - - -def is_mdname(name): - # return true if this is a metadata service name - if name in ["ami", "root", "swap"]: - return True - # names 'ephemeral0' or 'ephemeral1' - # 'ebs[0-9]' appears when '--block-device-mapping sdf=snap-d4d90bbc' - for enumname in ("ephemeral", "ebs"): - if name.startswith(enumname) and name.find(":") == -1: - return True - return False - - -def handle(_name, cfg, cloud, log, _args): - # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno - defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] - defvals = cfg.get("mount_default_fields", defvals) - - # these are our default set of mounts - defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"], - ["swap", "none", "swap", "sw", "0", "0"]] - - cfgmnt = [] - if "mounts" in cfg: - cfgmnt = cfg["mounts"] - - # shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 - shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" - shortname = re.compile(shortname_filter) - - for i in range(len(cfgmnt)): - # skip something that wasn't a list - if not isinstance(cfgmnt[i], list): - continue - - # workaround, allow user to specify 'ephemeral' - # rather than more ec2 correct 'ephemeral0' - if cfgmnt[i][0] == "ephemeral": - cfgmnt[i][0] = "ephemeral0" - - if is_mdname(cfgmnt[i][0]): - newname = cloud.device_name_to_device(cfgmnt[i][0]) - if not newname: - log.debug("ignoring nonexistant named mount %s" % cfgmnt[i][0]) - cfgmnt[i][1] = None - else: - if newname.startswith("/"): - cfgmnt[i][0] = newname - else: - cfgmnt[i][0] = "/dev/%s" % newname - else: - if shortname.match(cfgmnt[i][0]): - cfgmnt[i][0] = "/dev/%s" % cfgmnt[i][0] - - # in case the user did not quote a field (likely fs-freq, fs_passno) - # but do not convert None to 'None' (LP: #898365) - for j in range(len(cfgmnt[i])): - if isinstance(cfgmnt[i][j], int): - cfgmnt[i][j] = str(cfgmnt[i][j]) - - for i in range(len(cfgmnt)): - # fill in values with defaults from defvals above - for j in range(len(defvals)): - if len(cfgmnt[i]) <= j: - cfgmnt[i].append(defvals[j]) - elif cfgmnt[i][j] is None: - cfgmnt[i][j] = defvals[j] - - # if the second entry in the list is 'None' this - # clears all previous entries of that same 'fs_spec' - # (fs_spec is the first field in /etc/fstab, ie, that device) - if cfgmnt[i][1] is None: - for j in range(i): - if cfgmnt[j][0] == cfgmnt[i][0]: - cfgmnt[j][1] = None - - # for each of the "default" mounts, add them only if no other - # entry has the same device name - for defmnt in defmnts: - devname = cloud.device_name_to_device(defmnt[0]) - if devname is None: - continue - if devname.startswith("/"): - defmnt[0] = devname - else: - defmnt[0] = "/dev/%s" % devname - - cfgmnt_has = False - for cfgm in cfgmnt: - if cfgm[0] == defmnt[0]: - cfgmnt_has = True - break - - if cfgmnt_has: - continue - cfgmnt.append(defmnt) - - # now, each entry in the cfgmnt list has all fstab values - # if the second field is None (not the string, the value) we skip it - actlist = [x for x in cfgmnt if x[1] is not None] - - if len(actlist) == 0: - return - - comment = "comment=cloudconfig" - cc_lines = [] - needswap = False - dirs = [] - for line in actlist: - # write 'comment' in the fs_mntops, entry, claiming this - line[3] = "%s,comment=cloudconfig" % line[3] - if line[2] == "swap": - needswap = True - if line[1].startswith("/"): - dirs.append(line[1]) - cc_lines.append('\t'.join(line)) - - fstab_lines = [] - fstab = open("/etc/fstab", "r+") - ws = re.compile("[%s]+" % whitespace) - for line in fstab.read().splitlines(): - try: - toks = ws.split(line) - if toks[3].find(comment) != -1: - continue - except: - pass - fstab_lines.append(line) - - fstab_lines.extend(cc_lines) - - fstab.seek(0) - fstab.write("%s\n" % '\n'.join(fstab_lines)) - fstab.truncate() - fstab.close() - - if needswap: - try: - util.subp(("swapon", "-a")) - except: - log.warn("Failed to enable swap") - - for d in dirs: - if os.path.exists(d): - continue - try: - os.makedirs(d) - except: - log.warn("Failed to make '%s' config-mount\n", d) - - try: - util.subp(("mount", "-a")) - except: - log.warn("'mount -a' failed") diff --git a/cloudinit/CloudConfig/cc_phone_home.py b/cloudinit/CloudConfig/cc_phone_home.py deleted file mode 100644 index a7ff74e1..00000000 --- a/cloudinit/CloudConfig/cc_phone_home.py +++ /dev/null @@ -1,106 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -from cloudinit.CloudConfig import per_instance -import cloudinit.util as util -from time import sleep - -frequency = per_instance -post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', - 'hostname'] - - -# phone_home: -# url: http://my.foo.bar/$INSTANCE/ -# post: all -# tries: 10 -# -# phone_home: -# url: http://my.foo.bar/$INSTANCE_ID/ -# post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id -# -def handle(_name, cfg, cloud, log, args): - if len(args) != 0: - ph_cfg = util.read_conf(args[0]) - else: - if not 'phone_home' in cfg: - return - ph_cfg = cfg['phone_home'] - - if 'url' not in ph_cfg: - log.warn("no 'url' token in phone_home") - return - - url = ph_cfg['url'] - post_list = ph_cfg.get('post', 'all') - tries = ph_cfg.get('tries', 10) - try: - tries = int(tries) - except: - log.warn("tries is not an integer. using 10") - tries = 10 - - if post_list == "all": - post_list = post_list_all - - all_keys = {} - all_keys['instance_id'] = cloud.get_instance_id() - all_keys['hostname'] = cloud.get_hostname() - - pubkeys = { - 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', - 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', - 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', - } - - for n, path in pubkeys.iteritems(): - try: - fp = open(path, "rb") - all_keys[n] = fp.read() - fp.close() - except: - log.warn("%s: failed to open in phone_home" % path) - - submit_keys = {} - for k in post_list: - if k in all_keys: - submit_keys[k] = all_keys[k] - else: - submit_keys[k] = "N/A" - log.warn("requested key %s from 'post' list not available") - - url = util.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) - - null_exc = object() - last_e = null_exc - for i in range(0, tries): - try: - util.readurl(url, submit_keys) - log.debug("succeeded submit to %s on try %i" % (url, i + 1)) - return - except Exception as e: - log.debug("failed to post to %s on try %i" % (url, i + 1)) - last_e = e - sleep(3) - - log.warn("failed to post to %s in %i tries" % (url, tries)) - if last_e is not null_exc: - raise(last_e) - - return diff --git a/cloudinit/CloudConfig/cc_puppet.py b/cloudinit/CloudConfig/cc_puppet.py deleted file mode 100644 index 6fc475f6..00000000 --- a/cloudinit/CloudConfig/cc_puppet.py +++ /dev/null @@ -1,108 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import os.path -import pwd -import socket -import subprocess -import StringIO -import ConfigParser -import cloudinit.CloudConfig as cc -import cloudinit.util as util - - -def handle(_name, cfg, cloud, log, _args): - # If there isn't a puppet key in the configuration don't do anything - if 'puppet' not in cfg: - return - puppet_cfg = cfg['puppet'] - # Start by installing the puppet package ... - cc.install_packages(("puppet",)) - - # ... and then update the puppet configuration - if 'conf' in puppet_cfg: - # Add all sections from the conf object to puppet.conf - puppet_conf_fh = open('/etc/puppet/puppet.conf', 'r') - # Create object for reading puppet.conf values - puppet_config = ConfigParser.ConfigParser() - # Read puppet.conf values from original file in order to be able to - # mix the rest up - puppet_config.readfp(StringIO.StringIO(''.join(i.lstrip() for i in - puppet_conf_fh.readlines()))) - # Close original file, no longer needed - puppet_conf_fh.close() - for cfg_name, cfg in puppet_cfg['conf'].iteritems(): - # ca_cert configuration is a special case - # Dump the puppetmaster ca certificate in the correct place - if cfg_name == 'ca_cert': - # Puppet ssl sub-directory isn't created yet - # Create it with the proper permissions and ownership - os.makedirs('/var/lib/puppet/ssl') - os.chmod('/var/lib/puppet/ssl', 0771) - os.chown('/var/lib/puppet/ssl', - pwd.getpwnam('puppet').pw_uid, 0) - os.makedirs('/var/lib/puppet/ssl/certs/') - os.chown('/var/lib/puppet/ssl/certs/', - pwd.getpwnam('puppet').pw_uid, 0) - ca_fh = open('/var/lib/puppet/ssl/certs/ca.pem', 'w') - ca_fh.write(cfg) - ca_fh.close() - os.chown('/var/lib/puppet/ssl/certs/ca.pem', - pwd.getpwnam('puppet').pw_uid, 0) - util.restorecon_if_possible('/var/lib/puppet', recursive=True) - else: - #puppet_conf_fh.write("\n[%s]\n" % (cfg_name)) - # If puppet.conf already has this section we don't want to - # write it again - if puppet_config.has_section(cfg_name) == False: - puppet_config.add_section(cfg_name) - # Iterate throug the config items, we'll use ConfigParser.set - # to overwrite or create new items as needed - for o, v in cfg.iteritems(): - if o == 'certname': - # Expand %f as the fqdn - v = v.replace("%f", socket.getfqdn()) - # Expand %i as the instance id - v = v.replace("%i", - cloud.datasource.get_instance_id()) - # certname needs to be downcase - v = v.lower() - puppet_config.set(cfg_name, o, v) - #puppet_conf_fh.write("%s=%s\n" % (o, v)) - # We got all our config as wanted we'll rename - # the previous puppet.conf and create our new one - os.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') - with open('/etc/puppet/puppet.conf', 'wb') as configfile: - puppet_config.write(configfile) - util.restorecon_if_possible('/etc/puppet/puppet.conf') - # Set puppet to automatically start - if os.path.exists('/etc/default/puppet'): - subprocess.check_call(['sed', '-i', - '-e', 's/^START=.*/START=yes/', - '/etc/default/puppet']) - elif os.path.exists('/bin/systemctl'): - subprocess.check_call(['/bin/systemctl', 'enable', 'puppet.service']) - elif os.path.exists('/sbin/chkconfig'): - subprocess.check_call(['/sbin/chkconfig', 'puppet', 'on']) - else: - log.warn("Do not know how to enable puppet service on this system") - # Start puppetd - subprocess.check_call(['service', 'puppet', 'start']) diff --git a/cloudinit/CloudConfig/cc_resizefs.py b/cloudinit/CloudConfig/cc_resizefs.py deleted file mode 100644 index 2dc66def..00000000 --- a/cloudinit/CloudConfig/cc_resizefs.py +++ /dev/null @@ -1,108 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import os -import stat -import sys -import time -import tempfile -from cloudinit.CloudConfig import per_always - -frequency = per_always - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - resize_root = False - if str(args[0]).lower() in ['true', '1', 'on', 'yes']: - resize_root = True - else: - resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) - - if str(resize_root).lower() in ['false', '0']: - return - - # we use mktemp rather than mkstemp because early in boot nothing - # else should be able to race us for this, and we need to mknod. - devpth = tempfile.mktemp(prefix="cloudinit.resizefs.", dir="/run") - - try: - st_dev = os.stat("/").st_dev - dev = os.makedev(os.major(st_dev), os.minor(st_dev)) - os.mknod(devpth, 0400 | stat.S_IFBLK, dev) - except: - if util.is_container(): - log.debug("inside container, ignoring mknod failure in resizefs") - return - log.warn("Failed to make device node to resize /") - raise - - cmd = ['blkid', '-c', '/dev/null', '-sTYPE', '-ovalue', devpth] - try: - (fstype, _err) = util.subp(cmd) - except subprocess.CalledProcessError as e: - log.warn("Failed to get filesystem type of maj=%s, min=%s via: %s" % - (os.major(st_dev), os.minor(st_dev), cmd)) - log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) - os.unlink(devpth) - raise - - if str(fstype).startswith("ext"): - resize_cmd = ['resize2fs', devpth] - elif fstype == "xfs": - resize_cmd = ['xfs_growfs', devpth] - else: - os.unlink(devpth) - log.debug("not resizing unknown filesystem %s" % fstype) - return - - if resize_root == "noblock": - fid = os.fork() - if fid == 0: - try: - do_resize(resize_cmd, devpth, log) - os._exit(0) # pylint: disable=W0212 - except Exception as exc: - sys.stderr.write("Failed: %s" % exc) - os._exit(1) # pylint: disable=W0212 - else: - do_resize(resize_cmd, devpth, log) - - log.debug("resizing root filesystem (type=%s, maj=%i, min=%i, val=%s)" % - (str(fstype).rstrip("\n"), os.major(st_dev), os.minor(st_dev), - resize_root)) - - return - - -def do_resize(resize_cmd, devpth, log): - try: - start = time.time() - util.subp(resize_cmd) - except subprocess.CalledProcessError as e: - log.warn("Failed to resize filesystem (%s)" % resize_cmd) - log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) - os.unlink(devpth) - raise - - os.unlink(devpth) - log.debug("resize took %s seconds" % (time.time() - start)) diff --git a/cloudinit/CloudConfig/cc_rightscale_userdata.py b/cloudinit/CloudConfig/cc_rightscale_userdata.py deleted file mode 100644 index 5ed0848f..00000000 --- a/cloudinit/CloudConfig/cc_rightscale_userdata.py +++ /dev/null @@ -1,78 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -## -## The purpose of this script is to allow cloud-init to consume -## rightscale style userdata. rightscale user data is key-value pairs -## in a url-query-string like format. -## -## for cloud-init support, there will be a key named -## 'CLOUD_INIT_REMOTE_HOOK'. -## -## This cloud-config module will -## - read the blob of data from raw user data, and parse it as key/value -## - for each key that is found, download the content to -## the local instance/scripts directory and set them executable. -## - the files in that directory will be run by the user-scripts module -## Therefore, this must run before that. -## -## - -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_ipath_cur -from urlparse import parse_qs - -frequency = per_instance -my_name = "cc_rightscale_userdata" -my_hookname = 'CLOUD_INIT_REMOTE_HOOK' - - -def handle(_name, _cfg, cloud, log, _args): - try: - ud = cloud.get_userdata_raw() - except: - log.warn("failed to get raw userdata in %s" % my_name) - return - - try: - mdict = parse_qs(ud) - if not my_hookname in mdict: - return - except: - log.warn("failed to urlparse.parse_qa(userdata_raw())") - raise - - scripts_d = get_ipath_cur('scripts') - i = 0 - first_e = None - for url in mdict[my_hookname]: - fname = "%s/rightscale-%02i" % (scripts_d, i) - i = i + 1 - try: - content = util.readurl(url) - util.write_file(fname, content, mode=0700) - except Exception as e: - if not first_e: - first_e = None - log.warn("%s failed to read %s: %s" % (my_name, url, e)) - - if first_e: - raise(e) diff --git a/cloudinit/CloudConfig/cc_rsyslog.py b/cloudinit/CloudConfig/cc_rsyslog.py deleted file mode 100644 index ac7f2c74..00000000 --- a/cloudinit/CloudConfig/cc_rsyslog.py +++ /dev/null @@ -1,101 +0,0 @@ -# vi: ts=4 expandtab syntax=python -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit -import logging -import cloudinit.util as util -import traceback - -DEF_FILENAME = "20-cloud-config.conf" -DEF_DIR = "/etc/rsyslog.d" - - -def handle(_name, cfg, _cloud, log, _args): - # rsyslog: - # - "*.* @@192.158.1.1" - # - content: "*.* @@192.0.2.1:10514" - # - filename: 01-examplecom.conf - # content: | - # *.* @@syslogd.example.com - - # process 'rsyslog' - if not 'rsyslog' in cfg: - return - - def_dir = cfg.get('rsyslog_dir', DEF_DIR) - def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) - - files = [] - elst = [] - for ent in cfg['rsyslog']: - if isinstance(ent, dict): - if not "content" in ent: - elst.append((ent, "no 'content' entry")) - continue - content = ent['content'] - filename = ent.get("filename", def_fname) - else: - content = ent - filename = def_fname - - if not filename.startswith("/"): - filename = "%s/%s" % (def_dir, filename) - - omode = "ab" - # truncate filename first time you see it - if filename not in files: - omode = "wb" - files.append(filename) - - try: - util.write_file(filename, content + "\n", omode=omode) - except Exception as e: - log.debug(traceback.format_exc(e)) - elst.append((content, "failed to write to %s" % filename)) - - # need to restart syslogd - restarted = False - try: - # if this config module is running at cloud-init time - # (before rsyslog is running) we don't actually have to - # restart syslog. - # - # upstart actually does what we want here, in that it doesn't - # start a service that wasn't running already on 'restart' - # it will also return failure on the attempt, so 'restarted' - # won't get set - log.debug("restarting rsyslog") - util.subp(['service', 'rsyslog', 'restart']) - restarted = True - - except Exception as e: - elst.append(("restart", str(e))) - - if restarted: - # this only needs to run if we *actually* restarted - # syslog above. - cloudinit.logging_set_from_cfg_file() - log = logging.getLogger() - log.debug("rsyslog configured %s" % files) - - for e in elst: - log.warn("rsyslog error: %s\n" % ':'.join(e)) - - return diff --git a/cloudinit/CloudConfig/cc_runcmd.py b/cloudinit/CloudConfig/cc_runcmd.py deleted file mode 100644 index f7e8c671..00000000 --- a/cloudinit/CloudConfig/cc_runcmd.py +++ /dev/null @@ -1,32 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util - - -def handle(_name, cfg, cloud, log, _args): - if "runcmd" not in cfg: - return - outfile = "%s/runcmd" % cloud.get_ipath('scripts') - try: - content = util.shellify(cfg["runcmd"]) - util.write_file(outfile, content, 0700) - except: - log.warn("failed to open %s for runcmd" % outfile) diff --git a/cloudinit/CloudConfig/cc_salt_minion.py b/cloudinit/CloudConfig/cc_salt_minion.py deleted file mode 100644 index 1a3b5039..00000000 --- a/cloudinit/CloudConfig/cc_salt_minion.py +++ /dev/null @@ -1,56 +0,0 @@ -# vi: ts=4 expandtab -# -# Author: Jeff Bauer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import os.path -import subprocess -import cloudinit.CloudConfig as cc -import yaml - - -def handle(_name, cfg, _cloud, _log, _args): - # If there isn't a salt key in the configuration don't do anything - if 'salt_minion' not in cfg: - return - salt_cfg = cfg['salt_minion'] - # Start by installing the salt package ... - cc.install_packages(("salt",)) - config_dir = '/etc/salt' - if not os.path.isdir(config_dir): - os.makedirs(config_dir) - # ... and then update the salt configuration - if 'conf' in salt_cfg: - # Add all sections from the conf object to /etc/salt/minion - minion_config = os.path.join(config_dir, 'minion') - yaml.dump(salt_cfg['conf'], - file(minion_config, 'w'), - default_flow_style=False) - # ... copy the key pair if specified - if 'public_key' in salt_cfg and 'private_key' in salt_cfg: - pki_dir = '/etc/salt/pki' - cumask = os.umask(077) - if not os.path.isdir(pki_dir): - os.makedirs(pki_dir) - pub_name = os.path.join(pki_dir, 'minion.pub') - pem_name = os.path.join(pki_dir, 'minion.pem') - with open(pub_name, 'w') as f: - f.write(salt_cfg['public_key']) - with open(pem_name, 'w') as f: - f.write(salt_cfg['private_key']) - os.umask(cumask) - - # Start salt-minion - subprocess.check_call(['service', 'salt-minion', 'start']) diff --git a/cloudinit/CloudConfig/cc_scripts_per_boot.py b/cloudinit/CloudConfig/cc_scripts_per_boot.py deleted file mode 100644 index 41a74754..00000000 --- a/cloudinit/CloudConfig/cc_scripts_per_boot.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_always -from cloudinit import get_cpath - -frequency = per_always -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-boot") - - -def handle(_name, _cfg, _cloud, log, _args): - try: - util.runparts(runparts_path) - except: - log.warn("failed to run-parts in %s" % runparts_path) - raise diff --git a/cloudinit/CloudConfig/cc_scripts_per_instance.py b/cloudinit/CloudConfig/cc_scripts_per_instance.py deleted file mode 100644 index a2981eab..00000000 --- a/cloudinit/CloudConfig/cc_scripts_per_instance.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_cpath - -frequency = per_instance -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-instance") - - -def handle(_name, _cfg, _cloud, log, _args): - try: - util.runparts(runparts_path) - except: - log.warn("failed to run-parts in %s" % runparts_path) - raise diff --git a/cloudinit/CloudConfig/cc_scripts_per_once.py b/cloudinit/CloudConfig/cc_scripts_per_once.py deleted file mode 100644 index a69151da..00000000 --- a/cloudinit/CloudConfig/cc_scripts_per_once.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_once -from cloudinit import get_cpath - -frequency = per_once -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-once") - - -def handle(_name, _cfg, _cloud, log, _args): - try: - util.runparts(runparts_path) - except: - log.warn("failed to run-parts in %s" % runparts_path) - raise diff --git a/cloudinit/CloudConfig/cc_scripts_user.py b/cloudinit/CloudConfig/cc_scripts_user.py deleted file mode 100644 index 933aa4e0..00000000 --- a/cloudinit/CloudConfig/cc_scripts_user.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_ipath_cur - -frequency = per_instance -runparts_path = "%s/%s" % (get_ipath_cur(), "scripts") - - -def handle(_name, _cfg, _cloud, log, _args): - try: - util.runparts(runparts_path) - except: - log.warn("failed to run-parts in %s" % runparts_path) - raise diff --git a/cloudinit/CloudConfig/cc_set_hostname.py b/cloudinit/CloudConfig/cc_set_hostname.py deleted file mode 100644 index acea74d9..00000000 --- a/cloudinit/CloudConfig/cc_set_hostname.py +++ /dev/null @@ -1,42 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util - - -def handle(_name, cfg, cloud, log, _args): - if util.get_cfg_option_bool(cfg, "preserve_hostname", False): - log.debug("preserve_hostname is set. not setting hostname") - return(True) - - (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) - try: - set_hostname(hostname, log) - except Exception: - util.logexc(log) - log.warn("failed to set hostname to %s\n", hostname) - - return(True) - - -def set_hostname(hostname, log): - util.subp(['hostname', hostname]) - util.write_file("/etc/hostname", "%s\n" % hostname, 0644) - log.debug("populated /etc/hostname with %s on first boot", hostname) diff --git a/cloudinit/CloudConfig/cc_set_passwords.py b/cloudinit/CloudConfig/cc_set_passwords.py deleted file mode 100644 index 9d0bbdb8..00000000 --- a/cloudinit/CloudConfig/cc_set_passwords.py +++ /dev/null @@ -1,129 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import sys -import random -from string import letters, digits # pylint: disable=W0402 - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - # if run from command line, and give args, wipe the chpasswd['list'] - password = args[0] - if 'chpasswd' in cfg and 'list' in cfg['chpasswd']: - del cfg['chpasswd']['list'] - else: - password = util.get_cfg_option_str(cfg, "password", None) - - expire = True - pw_auth = "no" - change_pwauth = False - plist = None - - if 'chpasswd' in cfg: - chfg = cfg['chpasswd'] - plist = util.get_cfg_option_str(chfg, 'list', plist) - expire = util.get_cfg_option_bool(chfg, 'expire', expire) - - if not plist and password: - user = util.get_cfg_option_str(cfg, "user", "ubuntu") - plist = "%s:%s" % (user, password) - - errors = [] - if plist: - plist_in = [] - randlist = [] - users = [] - for line in plist.splitlines(): - u, p = line.split(':', 1) - if p == "R" or p == "RANDOM": - p = rand_user_password() - randlist.append("%s:%s" % (u, p)) - plist_in.append("%s:%s" % (u, p)) - users.append(u) - - ch_in = '\n'.join(plist_in) - try: - util.subp(['chpasswd'], ch_in) - log.debug("changed password for %s:" % users) - except Exception as e: - errors.append(e) - log.warn("failed to set passwords with chpasswd: %s" % e) - - if len(randlist): - sys.stdout.write("%s\n%s\n" % ("Set the following passwords\n", - '\n'.join(randlist))) - - if expire: - enum = len(errors) - for u in users: - try: - util.subp(['passwd', '--expire', u]) - except Exception as e: - errors.append(e) - log.warn("failed to expire account for %s" % u) - if enum == len(errors): - log.debug("expired passwords for: %s" % u) - - if 'ssh_pwauth' in cfg: - val = str(cfg['ssh_pwauth']).lower() - if val in ("true", "1", "yes"): - pw_auth = "yes" - change_pwauth = True - elif val in ("false", "0", "no"): - pw_auth = "no" - change_pwauth = True - else: - change_pwauth = False - - if change_pwauth: - pa_s = "\(#*\)\(PasswordAuthentication[[:space:]]\+\)\(yes\|no\)" - msg = "set PasswordAuthentication to '%s'" % pw_auth - try: - cmd = ['sed', '-i', 's,%s,\\2%s,' % (pa_s, pw_auth), - '/etc/ssh/sshd_config'] - util.subp(cmd) - log.debug(msg) - except Exception as e: - log.warn("failed %s" % msg) - errors.append(e) - - try: - p = util.subp(['service', cfg.get('ssh_svcname', 'ssh'), - 'restart']) - log.debug("restarted sshd") - except: - log.warn("restart of ssh failed") - - if len(errors): - raise(errors[0]) - - return - - -def rand_str(strlen=32, select_from=letters + digits): - return("".join([random.choice(select_from) for _x in range(0, strlen)])) - - -def rand_user_password(pwlen=9): - selfrom = (letters.translate(None, 'loLOI') + - digits.translate(None, '01')) - return(rand_str(pwlen, select_from=selfrom)) diff --git a/cloudinit/CloudConfig/cc_ssh.py b/cloudinit/CloudConfig/cc_ssh.py deleted file mode 100644 index 48eb58bc..00000000 --- a/cloudinit/CloudConfig/cc_ssh.py +++ /dev/null @@ -1,106 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import cloudinit.SshUtil as sshutil -import os -import glob -import subprocess - -DISABLE_ROOT_OPTS = "no-port-forwarding,no-agent-forwarding," \ -"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " \ -"rather than the user \\\"root\\\".\';echo;sleep 10\"" - - -def handle(_name, cfg, cloud, log, _args): - - # remove the static keys from the pristine image - if cfg.get("ssh_deletekeys", True): - for f in glob.glob("/etc/ssh/ssh_host_*key*"): - try: - os.unlink(f) - except: - pass - - if "ssh_keys" in cfg: - # if there are keys in cloud-config, use them - key2file = { - "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), - "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), - "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), - "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), - "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), - "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), - } - - for key, val in cfg["ssh_keys"].items(): - if key in key2file: - util.write_file(key2file[key][0], val, key2file[key][1]) - - priv2pub = {'rsa_private': 'rsa_public', 'dsa_private': 'dsa_public', - 'ecdsa_private': 'ecdsa_public', } - - cmd = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' - for priv, pub in priv2pub.iteritems(): - if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: - continue - pair = (key2file[priv][0], key2file[pub][0]) - subprocess.call(('sh', '-xc', cmd % pair)) - log.debug("generated %s from %s" % pair) - else: - # if not, generate them - for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', - ['rsa', 'dsa', 'ecdsa']): - keyfile = '/etc/ssh/ssh_host_%s_key' % keytype - if not os.path.exists(keyfile): - subprocess.call(['ssh-keygen', '-t', keytype, '-N', '', - '-f', keyfile]) - - util.restorecon_if_possible('/etc/ssh', recursive=True) - - try: - user = util.get_cfg_option_str(cfg, 'user') - disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) - disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", - DISABLE_ROOT_OPTS) - keys = cloud.get_public_ssh_keys() - - if "ssh_authorized_keys" in cfg: - cfgkeys = cfg["ssh_authorized_keys"] - keys.extend(cfgkeys) - - apply_credentials(keys, user, disable_root, disable_root_opts, log) - except: - util.logexc(log) - log.warn("applying credentials failed!\n") - - -def apply_credentials(keys, user, disable_root, - disable_root_opts=DISABLE_ROOT_OPTS, log=None): - keys = set(keys) - if user: - sshutil.setup_user_keys(keys, user, '', log) - - if disable_root: - key_prefix = disable_root_opts.replace('$USER', user) - else: - key_prefix = '' - - sshutil.setup_user_keys(keys, 'root', key_prefix, log) diff --git a/cloudinit/CloudConfig/cc_ssh_import_id.py b/cloudinit/CloudConfig/cc_ssh_import_id.py deleted file mode 100644 index bbf5bd83..00000000 --- a/cloudinit/CloudConfig/cc_ssh_import_id.py +++ /dev/null @@ -1,50 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import traceback - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - user = args[0] - ids = [] - if len(args) > 1: - ids = args[1:] - else: - user = util.get_cfg_option_str(cfg, "user", "ubuntu") - ids = util.get_cfg_option_list_or_str(cfg, "ssh_import_id", []) - - if len(ids) == 0: - return - - cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids - - log.debug("importing ssh ids. cmd = %s" % cmd) - - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) - except OSError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/CloudConfig/cc_timezone.py b/cloudinit/CloudConfig/cc_timezone.py deleted file mode 100644 index e5c9901b..00000000 --- a/cloudinit/CloudConfig/cc_timezone.py +++ /dev/null @@ -1,67 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.CloudConfig import per_instance -from cloudinit import util -import os.path -import shutil - -frequency = per_instance -tz_base = "/usr/share/zoneinfo" - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - timezone = args[0] - else: - timezone = util.get_cfg_option_str(cfg, "timezone", False) - - if not timezone: - return - - tz_file = "%s/%s" % (tz_base, timezone) - - if not os.path.isfile(tz_file): - log.debug("Invalid timezone %s" % tz_file) - raise Exception("Invalid timezone %s" % tz_file) - - try: - fp = open("/etc/timezone", "wb") - fp.write("%s\n" % timezone) - fp.close() - except: - log.debug("failed to write to /etc/timezone") - raise - if os.path.exists("/etc/sysconfig/clock"): - try: - with open("/etc/sysconfig/clock", "w") as fp: - fp.write('ZONE="%s"\n' % timezone) - except: - log.debug("failed to write to /etc/sysconfig/clock") - raise - - try: - shutil.copy(tz_file, "/etc/localtime") - except: - log.debug("failed to copy %s to /etc/localtime" % tz_file) - raise - - log.debug("set timezone to %s" % timezone) - return diff --git a/cloudinit/CloudConfig/cc_update_etc_hosts.py b/cloudinit/CloudConfig/cc_update_etc_hosts.py deleted file mode 100644 index 6ad2fca8..00000000 --- a/cloudinit/CloudConfig/cc_update_etc_hosts.py +++ /dev/null @@ -1,87 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_always -import StringIO - -frequency = per_always - - -def handle(_name, cfg, cloud, log, _args): - (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - - manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) - if manage_hosts in ("True", "true", True, "template"): - # render from template file - try: - if not hostname: - log.info("manage_etc_hosts was set, but no hostname found") - return - - util.render_to_file('hosts', '/etc/hosts', - {'hostname': hostname, 'fqdn': fqdn}) - except Exception: - log.warn("failed to update /etc/hosts") - raise - elif manage_hosts == "localhost": - log.debug("managing 127.0.1.1 in /etc/hosts") - update_etc_hosts(hostname, fqdn, log) - return - else: - if manage_hosts not in ("False", False): - log.warn("Unknown value for manage_etc_hosts. Assuming False") - else: - log.debug("not managing /etc/hosts") - - -def update_etc_hosts(hostname, fqdn, _log): - with open('/etc/hosts', 'r') as etchosts: - header = "# Added by cloud-init\n" - hosts_line = "127.0.1.1\t%s %s\n" % (fqdn, hostname) - need_write = False - need_change = True - new_etchosts = StringIO.StringIO() - for line in etchosts: - split_line = [s.strip() for s in line.split()] - if len(split_line) < 2: - new_etchosts.write(line) - continue - if line == header: - continue - ip, hosts = split_line[0], split_line[1:] - if ip == "127.0.1.1": - if sorted([hostname, fqdn]) == sorted(hosts): - need_change = False - if need_change == True: - line = "%s%s" % (header, hosts_line) - need_change = False - need_write = True - new_etchosts.write(line) - etchosts.close() - if need_change == True: - new_etchosts.write("%s%s" % (header, hosts_line)) - need_write = True - if need_write == True: - new_etcfile = open('/etc/hosts', 'wb') - new_etcfile.write(new_etchosts.getvalue()) - new_etcfile.close() - new_etchosts.close() - return diff --git a/cloudinit/CloudConfig/cc_update_hostname.py b/cloudinit/CloudConfig/cc_update_hostname.py deleted file mode 100644 index b9d1919a..00000000 --- a/cloudinit/CloudConfig/cc_update_hostname.py +++ /dev/null @@ -1,101 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import errno -from cloudinit.CloudConfig import per_always - -frequency = per_always - - -def handle(_name, cfg, cloud, log, _args): - if util.get_cfg_option_bool(cfg, "preserve_hostname", False): - log.debug("preserve_hostname is set. not updating hostname") - return - - (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) - try: - prev = "%s/%s" % (cloud.get_cpath('data'), "previous-hostname") - update_hostname(hostname, prev, log) - except Exception: - log.warn("failed to set hostname\n") - raise - - -# read hostname from a 'hostname' file -# allow for comments and stripping line endings. -# if file doesn't exist, or no contents, return default -def read_hostname(filename, default=None): - try: - fp = open(filename, "r") - lines = fp.readlines() - fp.close() - for line in lines: - hpos = line.find("#") - if hpos != -1: - line = line[0:hpos] - line = line.rstrip() - if line: - return line - except IOError as e: - if e.errno != errno.ENOENT: - raise - return default - - -def update_hostname(hostname, prev_file, log): - etc_file = "/etc/hostname" - - hostname_prev = None - hostname_in_etc = None - - try: - hostname_prev = read_hostname(prev_file) - except Exception as e: - log.warn("Failed to open %s: %s" % (prev_file, e)) - - try: - hostname_in_etc = read_hostname(etc_file) - except: - log.warn("Failed to open %s" % etc_file) - - update_files = [] - if not hostname_prev or hostname_prev != hostname: - update_files.append(prev_file) - - if (not hostname_in_etc or - (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): - update_files.append(etc_file) - - try: - for fname in update_files: - util.write_file(fname, "%s\n" % hostname, 0644) - log.debug("wrote %s to %s" % (hostname, fname)) - except: - log.warn("failed to write hostname to %s" % fname) - - if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: - log.debug("%s differs from %s. assuming user maintained" % - (prev_file, etc_file)) - - if etc_file in update_files: - log.debug("setting hostname to %s" % hostname) - subprocess.Popen(['hostname', hostname]).communicate() diff --git a/cloudinit/DataSource.py b/cloudinit/DataSource.py deleted file mode 100644 index e2a9150d..00000000 --- a/cloudinit/DataSource.py +++ /dev/null @@ -1,214 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -DEP_FILESYSTEM = "FILESYSTEM" -DEP_NETWORK = "NETWORK" - -import cloudinit.UserDataHandler as ud -import cloudinit.util as util -import socket - - -class DataSource: - userdata = None - metadata = None - userdata_raw = None - cfgname = "" - # system config (passed in from cloudinit, - # cloud-config before input from the DataSource) - sys_cfg = {} - # datasource config, the cloud-config['datasource']['__name__'] - ds_cfg = {} # datasource config - - def __init__(self, sys_cfg=None): - if not self.cfgname: - name = str(self.__class__).split(".")[-1] - if name.startswith("DataSource"): - name = name[len("DataSource"):] - self.cfgname = name - if sys_cfg: - self.sys_cfg = sys_cfg - - self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, - ("datasource", self.cfgname), self.ds_cfg) - - def get_userdata(self): - if self.userdata == None: - self.userdata = ud.preprocess_userdata(self.userdata_raw) - return self.userdata - - def get_userdata_raw(self): - return(self.userdata_raw) - - # the data sources' config_obj is a cloud-config formated - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return({}) - - def get_public_ssh_keys(self): - keys = [] - if 'public-keys' not in self.metadata: - return([]) - - if isinstance(self.metadata['public-keys'], str): - return(str(self.metadata['public-keys']).splitlines()) - - if isinstance(self.metadata['public-keys'], list): - return(self.metadata['public-keys']) - - for _keyname, klist in self.metadata['public-keys'].items(): - # lp:506332 uec metadata service responds with - # data that makes boto populate a string for 'klist' rather - # than a list. - if isinstance(klist, str): - klist = [klist] - for pkey in klist: - # there is an empty string at the end of the keylist, trim it - if pkey: - keys.append(pkey) - - return(keys) - - def device_name_to_device(self, _name): - # translate a 'name' to a device - # the primary function at this point is on ec2 - # to consult metadata service, that has - # ephemeral0: sdb - # and return 'sdb' for input 'ephemeral0' - return(None) - - def get_locale(self): - return('en_US.UTF-8') - - def get_local_mirror(self): - return None - - def get_instance_id(self): - if 'instance-id' not in self.metadata: - return "iid-datasource" - return(self.metadata['instance-id']) - - def get_hostname(self, fqdn=False): - defdomain = "localdomain" - defhost = "localhost" - - domain = defdomain - if not 'local-hostname' in self.metadata: - - # this is somewhat questionable really. - # the cloud datasource was asked for a hostname - # and didn't have one. raising error might be more appropriate - # but instead, basically look up the existing hostname - toks = [] - - hostname = socket.gethostname() - - fqdn = util.get_fqdn_from_hosts(hostname) - - if fqdn and fqdn.find(".") > 0: - toks = str(fqdn).split(".") - elif hostname: - toks = [hostname, defdomain] - else: - toks = [defhost, defdomain] - - else: - # if there is an ipv4 address in 'local-hostname', then - # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx - lhost = self.metadata['local-hostname'] - if is_ipv4(lhost): - toks = "ip-%s" % lhost.replace(".", "-") - else: - toks = lhost.split(".") - - if len(toks) > 1: - hostname = toks[0] - domain = '.'.join(toks[1:]) - else: - hostname = toks[0] - - if fqdn: - return "%s.%s" % (hostname, domain) - else: - return hostname - - -# return a list of classes that have the same depends as 'depends' -# iterate through cfg_list, loading "DataSourceCollections" modules -# and calling their "get_datasource_list". -# return an ordered list of classes that match -# -# - modules must be named "DataSource", where 'item' is an entry -# in cfg_list -# - if pkglist is given, it will iterate try loading from that package -# ie, pkglist=[ "foo", "" ] -# will first try to load foo.DataSource -# then DataSource -def list_sources(cfg_list, depends, pkglist=None): - if pkglist is None: - pkglist = [] - retlist = [] - for ds_coll in cfg_list: - for pkg in pkglist: - if pkg: - pkg = "%s." % pkg - try: - mod = __import__("%sDataSource%s" % (pkg, ds_coll)) - if pkg: - mod = getattr(mod, "DataSource%s" % ds_coll) - lister = getattr(mod, "get_datasource_list") - retlist.extend(lister(depends)) - break - except: - raise - return(retlist) - - -# depends is a list of dependencies (DEP_FILESYSTEM) -# dslist is a list of 2 item lists -# dslist = [ -# ( class, ( depends-that-this-class-needs ) ) -# } -# it returns a list of 'class' that matched these deps exactly -# it is a helper function for DataSourceCollections -def list_from_depends(depends, dslist): - retlist = [] - depset = set(depends) - for elem in dslist: - (cls, deps) = elem - if depset == set(deps): - retlist.append(cls) - return(retlist) - - -def is_ipv4(instr): - """ determine if input string is a ipv4 address. return boolean""" - toks = instr.split('.') - if len(toks) != 4: - return False - - try: - toks = [x for x in toks if (int(x) < 256 and int(x) > 0)] - except: - return False - - return (len(toks) == 4) diff --git a/cloudinit/DataSourceCloudStack.py b/cloudinit/DataSourceCloudStack.py deleted file mode 100644 index 5afdf7b6..00000000 --- a/cloudinit/DataSourceCloudStack.py +++ /dev/null @@ -1,92 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Cosmin Luta -# -# Author: Cosmin Luta -# Author: Scott Moser -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -from socket import inet_ntoa -import time -import boto.utils as boto_utils -from struct import pack - - -class DataSourceCloudStack(DataSource.DataSource): - api_ver = 'latest' - seeddir = base_seeddir + '/cs' - metadata_address = None - - def __init__(self, sys_cfg=None): - DataSource.DataSource.__init__(self, sys_cfg) - # Cloudstack has its metadata/userdata URLs located at - # http:///latest/ - self.metadata_address = "http://%s/" % self.get_default_gateway() - - def get_default_gateway(self): - """ Returns the default gateway ip address in the dotted format - """ - with open("/proc/net/route", "r") as f: - for line in f.readlines(): - items = line.split("\t") - if items[1] == "00000000": - # found the default route, get the gateway - gw = inet_ntoa(pack(" -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import os.path -import os -import json -import subprocess - -DEFAULT_IID = "iid-dsconfigdrive" - - -class DataSourceConfigDrive(DataSource.DataSource): - seed = None - seeddir = base_seeddir + '/config_drive' - cfg = {} - userdata_raw = None - metadata = None - dsmode = "local" - - def __str__(self): - mstr = "DataSourceConfigDrive[%s]" % self.dsmode - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) - - def get_data(self): - found = None - md = {} - ud = "" - - defaults = {"instance-id": DEFAULT_IID, "dsmode": "pass"} - - if os.path.isdir(self.seeddir): - try: - (md, ud) = read_config_drive_dir(self.seeddir) - found = self.seeddir - except nonConfigDriveDir: - pass - - if not found: - dev = cfg_drive_device() - if dev: - try: - (md, ud) = util.mount_callback_umount(dev, - read_config_drive_dir) - found = dev - except (nonConfigDriveDir, util.mountFailedError): - pass - - if not found: - return False - - if 'dsconfig' in md: - self.cfg = md['dscfg'] - - md = util.mergedict(md, defaults) - - # update interfaces and ifup only on the local datasource - # this way the DataSourceConfigDriveNet doesn't do it also. - if 'network-interfaces' in md and self.dsmode == "local": - if md['dsmode'] == "pass": - log.info("updating network interfaces from configdrive") - else: - log.debug("updating network interfaces from configdrive") - - util.write_file("/etc/network/interfaces", - md['network-interfaces']) - try: - (out, err) = util.subp(['ifup', '--all']) - if len(out) or len(err): - log.warn("ifup --all had stderr: %s" % err) - - except subprocess.CalledProcessError as exc: - log.warn("ifup --all failed: %s" % (exc.output[1])) - - self.seed = found - self.metadata = md - self.userdata_raw = ud - - if md['dsmode'] == self.dsmode: - return True - - log.debug("%s: not claiming datasource, dsmode=%s" % - (self, md['dsmode'])) - return False - - def get_public_ssh_keys(self): - if not 'public-keys' in self.metadata: - return([]) - return(self.metadata['public-keys']) - - # the data sources' config_obj is a cloud-config formated - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return(self.cfg) - - -class DataSourceConfigDriveNet(DataSourceConfigDrive): - dsmode = "net" - - -class nonConfigDriveDir(Exception): - pass - - -def cfg_drive_device(): - """ get the config drive device. return a string like '/dev/vdb' - or None (if there is no non-root device attached). This does not - check the contents, only reports that if there *were* a config_drive - attached, it would be this device. - per config_drive documentation, this is - "associated as the last available disk on the instance" - """ - - if 'CLOUD_INIT_CONFIG_DRIVE_DEVICE' in os.environ: - return(os.environ['CLOUD_INIT_CONFIG_DRIVE_DEVICE']) - - # we are looking for a raw block device (sda, not sda1) with a vfat - # filesystem on it. - - letters = "abcdefghijklmnopqrstuvwxyz" - devs = util.find_devs_with("TYPE=vfat") - - # filter out anything not ending in a letter (ignore partitions) - devs = [f for f in devs if f[-1] in letters] - - # sort them in reverse so "last" device is first - devs.sort(reverse=True) - - if len(devs): - return(devs[0]) - - return(None) - - -def read_config_drive_dir(source_dir): - """ - read_config_drive_dir(source_dir): - read source_dir, and return a tuple with metadata dict and user-data - string populated. If not a valid dir, raise a nonConfigDriveDir - """ - md = {} - ud = "" - - flist = ("etc/network/interfaces", "root/.ssh/authorized_keys", "meta.js") - found = [f for f in flist if os.path.isfile("%s/%s" % (source_dir, f))] - keydata = "" - - if len(found) == 0: - raise nonConfigDriveDir("%s: %s" % (source_dir, "no files found")) - - if "etc/network/interfaces" in found: - with open("%s/%s" % (source_dir, "/etc/network/interfaces")) as fp: - md['network-interfaces'] = fp.read() - - if "root/.ssh/authorized_keys" in found: - with open("%s/%s" % (source_dir, "root/.ssh/authorized_keys")) as fp: - keydata = fp.read() - - meta_js = {} - - if "meta.js" in found: - content = '' - with open("%s/%s" % (source_dir, "meta.js")) as fp: - content = fp.read() - md['meta_js'] = content - try: - meta_js = json.loads(content) - except ValueError: - raise nonConfigDriveDir("%s: %s" % - (source_dir, "invalid json in meta.js")) - - keydata = meta_js.get('public-keys', keydata) - - if keydata: - lines = keydata.splitlines() - md['public-keys'] = [l for l in lines - if len(l) and not l.startswith("#")] - - for copy in ('dsmode', 'instance-id', 'dscfg'): - if copy in meta_js: - md[copy] = meta_js[copy] - - if 'user-data' in meta_js: - ud = meta_js['user-data'] - - return(md, ud) - -datasources = ( - (DataSourceConfigDrive, (DataSource.DEP_FILESYSTEM, )), - (DataSourceConfigDriveNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -) - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - -if __name__ == "__main__": - def main(): - import sys - import pprint - print cfg_drive_device() - (md, ud) = read_config_drive_dir(sys.argv[1]) - print "=== md ===" - pprint.pprint(md) - print "=== ud ===" - print(ud) - - main() - -# vi: ts=4 expandtab diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py deleted file mode 100644 index 7051ecda..00000000 --- a/cloudinit/DataSourceEc2.py +++ /dev/null @@ -1,217 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import socket -import time -import boto.utils as boto_utils -import os.path - - -class DataSourceEc2(DataSource.DataSource): - api_ver = '2009-04-04' - seeddir = base_seeddir + '/ec2' - metadata_address = "http://169.254.169.254" - - def __str__(self): - return("DataSourceEc2") - - def get_data(self): - seedret = {} - if util.read_optional_seed(seedret, base=self.seeddir + "/"): - self.userdata_raw = seedret['user-data'] - self.metadata = seedret['meta-data'] - log.debug("using seeded ec2 data in %s" % self.seeddir) - return True - - try: - if not self.wait_for_metadata_service(): - return False - start = time.time() - self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, - None, self.metadata_address) - self.metadata = boto_utils.get_instance_metadata(self.api_ver, - self.metadata_address) - log.debug("crawl of metadata service took %ds" % (time.time() - - start)) - return True - except Exception as e: - print e - return False - - def get_instance_id(self): - return(self.metadata['instance-id']) - - def get_availability_zone(self): - return(self.metadata['placement']['availability-zone']) - - def get_local_mirror(self): - return(self.get_mirror_from_availability_zone()) - - def get_mirror_from_availability_zone(self, availability_zone=None): - # availability is like 'us-west-1b' or 'eu-west-1a' - if availability_zone == None: - availability_zone = self.get_availability_zone() - - fallback = None - - if self.is_vpc(): - return fallback - - try: - host = "%s.ec2.archive.ubuntu.com" % availability_zone[:-1] - socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM) - return 'http://%s/ubuntu/' % host - except: - return fallback - - def wait_for_metadata_service(self): - mcfg = self.ds_cfg - - if not hasattr(mcfg, "get"): - mcfg = {} - - max_wait = 120 - try: - max_wait = int(mcfg.get("max_wait", max_wait)) - except Exception: - util.logexc(log) - log.warn("Failed to get max wait. using %s" % max_wait) - - if max_wait == 0: - return False - - timeout = 50 - try: - timeout = int(mcfg.get("timeout", timeout)) - except Exception: - util.logexc(log) - log.warn("Failed to get timeout, using %s" % timeout) - - def_mdurls = ["http://169.254.169.254", "http://instance-data:8773"] - mdurls = mcfg.get("metadata_urls", def_mdurls) - - # Remove addresses from the list that wont resolve. - filtered = [x for x in mdurls if util.is_resolvable_url(x)] - - if set(filtered) != set(mdurls): - log.debug("removed the following from metadata urls: %s" % - list((set(mdurls) - set(filtered)))) - - if len(filtered): - mdurls = filtered - else: - log.warn("Empty metadata url list! using default list") - mdurls = def_mdurls - - urls = [] - url2base = {False: False} - for url in mdurls: - cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver) - urls.append(cur) - url2base[cur] = url - - starttime = time.time() - url = util.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=log.warn) - - if url: - log.debug("Using metadata source: '%s'" % url2base[url]) - else: - log.critical("giving up on md after %i seconds\n" % - int(time.time() - starttime)) - - self.metadata_address = url2base[url] - return (bool(url)) - - def device_name_to_device(self, name): - # consult metadata service, that has - # ephemeral0: sdb - # and return 'sdb' for input 'ephemeral0' - if 'block-device-mapping' not in self.metadata: - return(None) - - found = None - for entname, device in self.metadata['block-device-mapping'].items(): - if entname == name: - found = device - break - # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0' - if entname == "ephemeral" and name == "ephemeral0": - found = device - if found == None: - log.debug("unable to convert %s to a device" % name) - return None - - # LP: #611137 - # the metadata service may believe that devices are named 'sda' - # when the kernel named them 'vda' or 'xvda' - # we want to return the correct value for what will actually - # exist in this instance - mappings = {"sd": ("vd", "xvd")} - ofound = found - short = os.path.basename(found) - - if not found.startswith("/"): - found = "/dev/%s" % found - - if os.path.exists(found): - return(found) - - for nfrom, tlist in mappings.items(): - if not short.startswith(nfrom): - continue - for nto in tlist: - cand = "/dev/%s%s" % (nto, short[len(nfrom):]) - if os.path.exists(cand): - log.debug("remapped device name %s => %s" % (found, cand)) - return(cand) - - # on t1.micro, ephemeral0 will appear in block-device-mapping from - # metadata, but it will not exist on disk (and never will) - # at this pint, we've verified that the path did not exist - # in the special case of 'ephemeral0' return None to avoid bogus - # fstab entry (LP: #744019) - if name == "ephemeral0": - return None - return ofound - - def is_vpc(self): - # per comment in LP: #615545 - ph = "public-hostname" - p4 = "public-ipv4" - if ((ph not in self.metadata or self.metadata[ph] == "") and - (p4 not in self.metadata or self.metadata[p4] == "")): - return True - return False - - -datasources = [ - (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -] - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/DataSourceMAAS.py b/cloudinit/DataSourceMAAS.py deleted file mode 100644 index 61a0038f..00000000 --- a/cloudinit/DataSourceMAAS.py +++ /dev/null @@ -1,345 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# -# Author: Scott Moser -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import errno -import oauth.oauth as oauth -import os.path -import urllib2 -import time - - -MD_VERSION = "2012-03-01" - - -class DataSourceMAAS(DataSource.DataSource): - """ - DataSourceMAAS reads instance information from MAAS. - Given a config metadata_url, and oauth tokens, it expects to find - files under the root named: - instance-id - user-data - hostname - """ - seeddir = base_seeddir + '/maas' - baseurl = None - - def __str__(self): - return("DataSourceMAAS[%s]" % self.baseurl) - - def get_data(self): - mcfg = self.ds_cfg - - try: - (userdata, metadata) = read_maas_seed_dir(self.seeddir) - self.userdata_raw = userdata - self.metadata = metadata - self.baseurl = self.seeddir - return True - except MAASSeedDirNone: - pass - except MAASSeedDirMalformed as exc: - log.warn("%s was malformed: %s\n" % (self.seeddir, exc)) - raise - - try: - # if there is no metadata_url, then we're not configured - url = mcfg.get('metadata_url', None) - if url == None: - return False - - if not self.wait_for_metadata_service(url): - return False - - self.baseurl = url - - (userdata, metadata) = read_maas_seed_url(self.baseurl, - self.md_headers) - self.userdata_raw = userdata - self.metadata = metadata - return True - except Exception: - util.logexc(log) - return False - - def md_headers(self, url): - mcfg = self.ds_cfg - - # if we are missing token_key, token_secret or consumer_key - # then just do non-authed requests - for required in ('token_key', 'token_secret', 'consumer_key'): - if required not in mcfg: - return({}) - - consumer_secret = mcfg.get('consumer_secret', "") - - return(oauth_headers(url=url, consumer_key=mcfg['consumer_key'], - token_key=mcfg['token_key'], token_secret=mcfg['token_secret'], - consumer_secret=consumer_secret)) - - def wait_for_metadata_service(self, url): - mcfg = self.ds_cfg - - max_wait = 120 - try: - max_wait = int(mcfg.get("max_wait", max_wait)) - except Exception: - util.logexc(log) - log.warn("Failed to get max wait. using %s" % max_wait) - - if max_wait == 0: - return False - - timeout = 50 - try: - timeout = int(mcfg.get("timeout", timeout)) - except Exception: - util.logexc(log) - log.warn("Failed to get timeout, using %s" % timeout) - - starttime = time.time() - check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) - url = util.wait_for_url(urls=[check_url], max_wait=max_wait, - timeout=timeout, status_cb=log.warn, - headers_cb=self.md_headers) - - if url: - log.debug("Using metadata source: '%s'" % url) - else: - log.critical("giving up on md after %i seconds\n" % - int(time.time() - starttime)) - - return (bool(url)) - - -def read_maas_seed_dir(seed_d): - """ - Return user-data and metadata for a maas seed dir in seed_d. - Expected format of seed_d are the following files: - * instance-id - * local-hostname - * user-data - """ - files = ('local-hostname', 'instance-id', 'user-data', 'public-keys') - md = {} - - if not os.path.isdir(seed_d): - raise MAASSeedDirNone("%s: not a directory") - - for fname in files: - try: - with open(os.path.join(seed_d, fname)) as fp: - md[fname] = fp.read() - fp.close() - except IOError as e: - if e.errno != errno.ENOENT: - raise - - return(check_seed_contents(md, seed_d)) - - -def read_maas_seed_url(seed_url, header_cb=None, timeout=None, - version=MD_VERSION): - """ - Read the maas datasource at seed_url. - header_cb is a method that should return a headers dictionary that will - be given to urllib2.Request() - - Expected format of seed_url is are the following files: - * //meta-data/instance-id - * //meta-data/local-hostname - * //user-data - """ - files = ('meta-data/local-hostname', - 'meta-data/instance-id', - 'meta-data/public-keys', - 'user-data') - - base_url = "%s/%s" % (seed_url, version) - md = {} - for fname in files: - url = "%s/%s" % (base_url, fname) - if header_cb: - headers = header_cb(url) - else: - headers = {} - - try: - req = urllib2.Request(url, data=None, headers=headers) - resp = urllib2.urlopen(req, timeout=timeout) - md[os.path.basename(fname)] = resp.read() - except urllib2.HTTPError as e: - if e.code != 404: - raise - - return(check_seed_contents(md, seed_url)) - - -def check_seed_contents(content, seed): - """Validate if content is Is the content a dict that is valid as a - return for a datasource. - Either return a (userdata, metadata) tuple or - Raise MAASSeedDirMalformed or MAASSeedDirNone - """ - md_required = ('instance-id', 'local-hostname') - found = content.keys() - - if len(content) == 0: - raise MAASSeedDirNone("%s: no data files found" % seed) - - missing = [k for k in md_required if k not in found] - if len(missing): - raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing)) - - userdata = content.get('user-data', "") - md = {} - for (key, val) in content.iteritems(): - if key == 'user-data': - continue - md[key] = val - - return(userdata, md) - - -def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): - consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) - token = oauth.OAuthToken(token_key, token_secret) - params = { - 'oauth_version': "1.0", - 'oauth_nonce': oauth.generate_nonce(), - 'oauth_timestamp': int(time.time()), - 'oauth_token': token.key, - 'oauth_consumer_key': consumer.key, - } - req = oauth.OAuthRequest(http_url=url, parameters=params) - req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(), - consumer, token) - return(req.to_header()) - - -class MAASSeedDirNone(Exception): - pass - - -class MAASSeedDirMalformed(Exception): - pass - - -datasources = [ - (DataSourceMAAS, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -] - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - - -if __name__ == "__main__": - def main(): - """ - Call with single argument of directory or http or https url. - If url is given additional arguments are allowed, which will be - interpreted as consumer_key, token_key, token_secret, consumer_secret - """ - import argparse - import pprint - - parser = argparse.ArgumentParser(description='Interact with MAAS DS') - parser.add_argument("--config", metavar="file", - help="specify DS config file", default=None) - parser.add_argument("--ckey", metavar="key", - help="the consumer key to auth with", default=None) - parser.add_argument("--tkey", metavar="key", - help="the token key to auth with", default=None) - parser.add_argument("--csec", metavar="secret", - help="the consumer secret (likely '')", default="") - parser.add_argument("--tsec", metavar="secret", - help="the token secret to auth with", default=None) - parser.add_argument("--apiver", metavar="version", - help="the apiver to use ("" can be used)", default=MD_VERSION) - - subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") - subcmds.add_parser('crawl', help="crawl the datasource") - subcmds.add_parser('get', help="do a single GET of provided url") - subcmds.add_parser('check-seed', help="read andn verify seed at url") - - parser.add_argument("url", help="the data source to query") - - args = parser.parse_args() - - creds = {'consumer_key': args.ckey, 'token_key': args.tkey, - 'token_secret': args.tsec, 'consumer_secret': args.csec} - - if args.config: - import yaml - with open(args.config) as fp: - cfg = yaml.load(fp) - if 'datasource' in cfg: - cfg = cfg['datasource']['MAAS'] - for key in creds.keys(): - if key in cfg and creds[key] == None: - creds[key] = cfg[key] - - def geturl(url, headers_cb): - req = urllib2.Request(url, data=None, headers=headers_cb(url)) - return(urllib2.urlopen(req).read()) - - def printurl(url, headers_cb): - print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) - - def crawl(url, headers_cb=None): - if url.endswith("/"): - for line in geturl(url, headers_cb).splitlines(): - if line.endswith("/"): - crawl("%s%s" % (url, line), headers_cb) - else: - printurl("%s%s" % (url, line), headers_cb) - else: - printurl(url, headers_cb) - - def my_headers(url): - headers = {} - if creds.get('consumer_key', None) != None: - headers = oauth_headers(url, **creds) - return headers - - if args.subcmd == "check-seed": - if args.url.startswith("http"): - (userdata, metadata) = read_maas_seed_url(args.url, - header_cb=my_headers, version=args.apiver) - else: - (userdata, metadata) = read_maas_seed_url(args.url) - print "=== userdata ===" - print userdata - print "=== metadata ===" - pprint.pprint(metadata) - - elif args.subcmd == "get": - printurl(args.url, my_headers) - - elif args.subcmd == "crawl": - if not args.url.endswith("/"): - args.url = "%s/" % args.url - crawl(args.url, my_headers) - - main() diff --git a/cloudinit/DataSourceNoCloud.py b/cloudinit/DataSourceNoCloud.py deleted file mode 100644 index e8c56b8f..00000000 --- a/cloudinit/DataSourceNoCloud.py +++ /dev/null @@ -1,232 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import errno -import subprocess - - -class DataSourceNoCloud(DataSource.DataSource): - metadata = None - userdata = None - userdata_raw = None - supported_seed_starts = ("/", "file://") - dsmode = "local" - seed = None - cmdline_id = "ds=nocloud" - seeddir = base_seeddir + '/nocloud' - - def __str__(self): - mstr = "DataSourceNoCloud" - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) - - def get_data(self): - defaults = { - "instance-id": "nocloud", "dsmode": self.dsmode - } - - found = [] - md = {} - ud = "" - - try: - # parse the kernel command line, getting data passed in - if parse_cmdline_data(self.cmdline_id, md): - found.append("cmdline") - except: - util.logexc(log) - return False - - # check to see if the seeddir has data. - seedret = {} - if util.read_optional_seed(seedret, base=self.seeddir + "/"): - md = util.mergedict(md, seedret['meta-data']) - ud = seedret['user-data'] - found.append(self.seeddir) - log.debug("using seeded cache data in %s" % self.seeddir) - - # if the datasource config had a 'seedfrom' entry, then that takes - # precedence over a 'seedfrom' that was found in a filesystem - # but not over external medi - if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']: - found.append("ds_config") - md["seedfrom"] = self.ds_cfg['seedfrom'] - - fslist = util.find_devs_with("TYPE=vfat") - fslist.extend(util.find_devs_with("TYPE=iso9660")) - - label_list = util.find_devs_with("LABEL=cidata") - devlist = list(set(fslist) & set(label_list)) - devlist.sort(reverse=True) - - for dev in devlist: - try: - (newmd, newud) = util.mount_callback_umount(dev, - util.read_seeded) - md = util.mergedict(newmd, md) - ud = newud - - # for seed from a device, the default mode is 'net'. - # that is more likely to be what is desired. - # If they want dsmode of local, then they must - # specify that. - if 'dsmode' not in md: - md['dsmode'] = "net" - - log.debug("using data from %s" % dev) - found.append(dev) - break - except OSError, e: - if e.errno != errno.ENOENT: - raise - except util.mountFailedError: - log.warn("Failed to mount %s when looking for seed" % dev) - - # there was no indication on kernel cmdline or data - # in the seeddir suggesting this handler should be used. - if len(found) == 0: - return False - - seeded_interfaces = None - - # the special argument "seedfrom" indicates we should - # attempt to seed the userdata / metadata from its value - # its primarily value is in allowing the user to type less - # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg - if "seedfrom" in md: - seedfrom = md["seedfrom"] - seedfound = False - for proto in self.supported_seed_starts: - if seedfrom.startswith(proto): - seedfound = proto - break - if not seedfound: - log.debug("seed from %s not supported by %s" % - (seedfrom, self.__class__)) - return False - - if 'network-interfaces' in md: - seeded_interfaces = self.dsmode - - # this could throw errors, but the user told us to do it - # so if errors are raised, let them raise - (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) - log.debug("using seeded cache data from %s" % seedfrom) - - # values in the command line override those from the seed - md = util.mergedict(md, md_seed) - found.append(seedfrom) - - md = util.mergedict(md, defaults) - - # update the network-interfaces if metadata had 'network-interfaces' - # entry and this is the local datasource, or 'seedfrom' was used - # and the source of the seed was self.dsmode - # ('local' for NoCloud, 'net' for NoCloudNet') - if ('network-interfaces' in md and - (self.dsmode in ("local", seeded_interfaces))): - log.info("updating network interfaces from nocloud") - - util.write_file("/etc/network/interfaces", - md['network-interfaces']) - try: - (out, err) = util.subp(['ifup', '--all']) - if len(out) or len(err): - log.warn("ifup --all had stderr: %s" % err) - - except subprocess.CalledProcessError as exc: - log.warn("ifup --all failed: %s" % (exc.output[1])) - - self.seed = ",".join(found) - self.metadata = md - self.userdata_raw = ud - - if md['dsmode'] == self.dsmode: - return True - - log.debug("%s: not claiming datasource, dsmode=%s" % - (self, md['dsmode'])) - return False - - -# returns true or false indicating if cmdline indicated -# that this module should be used -# example cmdline: -# root=LABEL=uec-rootfs ro ds=nocloud -def parse_cmdline_data(ds_id, fill, cmdline=None): - if cmdline is None: - cmdline = util.get_cmdline() - cmdline = " %s " % cmdline - - if not (" %s " % ds_id in cmdline or " %s;" % ds_id in cmdline): - return False - - argline = "" - # cmdline can contain: - # ds=nocloud[;key=val;key=val] - for tok in cmdline.split(): - if tok.startswith(ds_id): - argline = tok.split("=", 1) - - # argline array is now 'nocloud' followed optionally by - # a ';' and then key=value pairs also terminated with ';' - tmp = argline[1].split(";") - if len(tmp) > 1: - kvpairs = tmp[1:] - else: - kvpairs = () - - # short2long mapping to save cmdline typing - s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"} - for item in kvpairs: - try: - (k, v) = item.split("=", 1) - except: - k = item - v = None - if k in s2l: - k = s2l[k] - fill[k] = v - - return(True) - - -class DataSourceNoCloudNet(DataSourceNoCloud): - cmdline_id = "ds=nocloud-net" - supported_seed_starts = ("http://", "https://", "ftp://") - seeddir = base_seeddir + '/nocloud-net' - dsmode = "net" - - -datasources = ( - (DataSourceNoCloud, (DataSource.DEP_FILESYSTEM, )), - (DataSourceNoCloudNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -) - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/DataSourceOVF.py b/cloudinit/DataSourceOVF.py deleted file mode 100644 index a0b1b518..00000000 --- a/cloudinit/DataSourceOVF.py +++ /dev/null @@ -1,332 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import os.path -import os -from xml.dom import minidom -import base64 -import re -import tempfile -import subprocess - - -class DataSourceOVF(DataSource.DataSource): - seed = None - seeddir = base_seeddir + '/ovf' - environment = None - cfg = {} - userdata_raw = None - metadata = None - supported_seed_starts = ("/", "file://") - - def __str__(self): - mstr = "DataSourceOVF" - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) - - def get_data(self): - found = [] - md = {} - ud = "" - - defaults = { - "instance-id": "iid-dsovf" - } - - (seedfile, contents) = get_ovf_env(base_seeddir) - if seedfile: - # found a seed dir - seed = "%s/%s" % (base_seeddir, seedfile) - (md, ud, cfg) = read_ovf_environment(contents) - self.environment = contents - - found.append(seed) - else: - np = {'iso': transport_iso9660, - 'vmware-guestd': transport_vmware_guestd, } - name = None - for name, transfunc in np.iteritems(): - (contents, _dev, _fname) = transfunc() - if contents: - break - - if contents: - (md, ud, cfg) = read_ovf_environment(contents) - self.environment = contents - found.append(name) - - # There was no OVF transports found - if len(found) == 0: - return False - - if 'seedfrom' in md and md['seedfrom']: - seedfrom = md['seedfrom'] - seedfound = False - for proto in self.supported_seed_starts: - if seedfrom.startswith(proto): - seedfound = proto - break - if not seedfound: - log.debug("seed from %s not supported by %s" % - (seedfrom, self.__class__)) - return False - - (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) - log.debug("using seeded cache data from %s" % seedfrom) - - md = util.mergedict(md, md_seed) - found.append(seedfrom) - - md = util.mergedict(md, defaults) - self.seed = ",".join(found) - self.metadata = md - self.userdata_raw = ud - self.cfg = cfg - return True - - def get_public_ssh_keys(self): - if not 'public-keys' in self.metadata: - return([]) - return([self.metadata['public-keys'], ]) - - # the data sources' config_obj is a cloud-config formated - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return(self.cfg) - - -class DataSourceOVFNet(DataSourceOVF): - seeddir = base_seeddir + '/ovf-net' - supported_seed_starts = ("http://", "https://", "ftp://") - - -# this will return a dict with some content -# meta-data, user-data -def read_ovf_environment(contents): - props = getProperties(contents) - md = {} - cfg = {} - ud = "" - cfg_props = ['password', ] - md_props = ['seedfrom', 'local-hostname', 'public-keys', 'instance-id'] - for prop, val in props.iteritems(): - if prop == 'hostname': - prop = "local-hostname" - if prop in md_props: - md[prop] = val - elif prop in cfg_props: - cfg[prop] = val - elif prop == "user-data": - try: - ud = base64.decodestring(val) - except: - ud = val - return(md, ud, cfg) - - -# returns tuple of filename (in 'dirname', and the contents of the file) -# on "not found", returns 'None' for filename and False for contents -def get_ovf_env(dirname): - env_names = ("ovf-env.xml", "ovf_env.xml", "OVF_ENV.XML", "OVF-ENV.XML") - for fname in env_names: - if os.path.isfile("%s/%s" % (dirname, fname)): - fp = open("%s/%s" % (dirname, fname)) - contents = fp.read() - fp.close() - return(fname, contents) - return(None, False) - - -# transport functions take no input and return -# a 3 tuple of content, path, filename -def transport_iso9660(require_iso=True): - - # default_regex matches values in - # /lib/udev/rules.d/60-cdrom_id.rules - # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end" - envname = "CLOUD_INIT_CDROM_DEV_REGEX" - default_regex = "^(sr[0-9]+|hd[a-z]|xvd.*)" - - devname_regex = os.environ.get(envname, default_regex) - cdmatch = re.compile(devname_regex) - - # go through mounts to see if it was already mounted - fp = open("/proc/mounts") - mounts = fp.readlines() - fp.close() - - mounted = {} - for mpline in mounts: - (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() - mounted[dev] = (dev, fstype, mp, False) - mp = mp.replace("\\040", " ") - if fstype != "iso9660" and require_iso: - continue - - if cdmatch.match(dev[5:]) == None: # take off '/dev/' - continue - - (fname, contents) = get_ovf_env(mp) - if contents is not False: - return(contents, dev, fname) - - tmpd = None - dvnull = None - - devs = os.listdir("/dev/") - devs.sort() - - for dev in devs: - fullp = "/dev/%s" % dev - - if fullp in mounted or not cdmatch.match(dev) or os.path.isdir(fullp): - continue - - fp = None - try: - fp = open(fullp, "rb") - fp.read(512) - fp.close() - except: - if fp: - fp.close() - continue - - if tmpd is None: - tmpd = tempfile.mkdtemp() - if dvnull is None: - try: - dvnull = open("/dev/null") - except: - pass - - cmd = ["mount", "-o", "ro", fullp, tmpd] - if require_iso: - cmd.extend(('-t', 'iso9660')) - - rc = subprocess.call(cmd, stderr=dvnull, stdout=dvnull, stdin=dvnull) - if rc: - continue - - (fname, contents) = get_ovf_env(tmpd) - - subprocess.call(["umount", tmpd]) - - if contents is not False: - os.rmdir(tmpd) - return(contents, fullp, fname) - - if tmpd: - os.rmdir(tmpd) - - if dvnull: - dvnull.close() - - return(False, None, None) - - -def transport_vmware_guestd(): - # http://blogs.vmware.com/vapp/2009/07/ \ - # selfconfiguration-and-the-ovf-environment.html - # try: - # cmd = ['vmware-guestd', '--cmd', 'info-get guestinfo.ovfEnv'] - # (out, err) = subp(cmd) - # return(out, 'guestinfo.ovfEnv', 'vmware-guestd') - # except: - # # would need to error check here and see why this failed - # # to know if log/error should be raised - # return(False, None, None) - return(False, None, None) - - -def findChild(node, filter_func): - ret = [] - if not node.hasChildNodes(): - return ret - for child in node.childNodes: - if filter_func(child): - ret.append(child) - return(ret) - - -def getProperties(environString): - dom = minidom.parseString(environString) - if dom.documentElement.localName != "Environment": - raise Exception("No Environment Node") - - if not dom.documentElement.hasChildNodes(): - raise Exception("No Child Nodes") - - envNsURI = "http://schemas.dmtf.org/ovf/environment/1" - - # could also check here that elem.namespaceURI == - # "http://schemas.dmtf.org/ovf/environment/1" - propSections = findChild(dom.documentElement, - lambda n: n.localName == "PropertySection") - - if len(propSections) == 0: - raise Exception("No 'PropertySection's") - - props = {} - propElems = findChild(propSections[0], lambda n: n.localName == "Property") - - for elem in propElems: - key = elem.attributes.getNamedItemNS(envNsURI, "key").value - val = elem.attributes.getNamedItemNS(envNsURI, "value").value - props[key] = val - - return(props) - - -datasources = ( - (DataSourceOVF, (DataSource.DEP_FILESYSTEM, )), - (DataSourceOVFNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -) - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - - -if __name__ == "__main__": - def main(): - import sys - envStr = open(sys.argv[1]).read() - props = getProperties(envStr) - import pprint - pprint.pprint(props) - - md, ud, cfg = read_ovf_environment(envStr) - print "=== md ===" - pprint.pprint(md) - print "=== ud ===" - pprint.pprint(ud) - print "=== cfg ===" - pprint.pprint(cfg) - - main() diff --git a/cloudinit/SshUtil.py b/cloudinit/SshUtil.py deleted file mode 100644 index a081fbe8..00000000 --- a/cloudinit/SshUtil.py +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/python -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import os.path -import cloudinit.util as util - - -class AuthKeyEntry(): - # lines are options, keytype, base64-encoded key, comment - # man page says the following which I did not understand: - # The options field is optional; its presence is determined by whether - # the line starts with a number or not (the options field never starts - # with a number) - options = None - keytype = None - base64 = None - comment = None - is_comment = False - line_in = "" - - def __init__(self, line, def_opt=None): - line = line.rstrip("\n\r") - self.line_in = line - if line.startswith("#") or line.strip() == "": - self.is_comment = True - else: - ent = line.strip() - toks = ent.split(None, 3) - if len(toks) == 1: - self.base64 = toks[0] - elif len(toks) == 2: - (self.base64, self.comment) = toks - elif len(toks) == 3: - (self.keytype, self.base64, self.comment) = toks - elif len(toks) == 4: - i = 0 - ent = line.strip() - quoted = False - # taken from auth_rsa_key_allowed in auth-rsa.c - try: - while (i < len(ent) and - ((quoted) or (ent[i] not in (" ", "\t")))): - curc = ent[i] - nextc = ent[i + 1] - if curc == "\\" and nextc == '"': - i = i + 1 - elif curc == '"': - quoted = not quoted - i = i + 1 - except IndexError: - self.is_comment = True - return - - try: - self.options = ent[0:i] - (self.keytype, self.base64, self.comment) = \ - ent[i + 1:].split(None, 3) - except ValueError: - # we did not understand this line - self.is_comment = True - - if self.options == None and def_opt: - self.options = def_opt - - return - - def debug(self): - print("line_in=%s\ncomment: %s\noptions=%s\nkeytype=%s\nbase64=%s\n" - "comment=%s\n" % (self.line_in, self.is_comment, self.options, - self.keytype, self.base64, self.comment)), - - def __repr__(self): - if self.is_comment: - return(self.line_in) - else: - toks = [] - for e in (self.options, self.keytype, self.base64, self.comment): - if e: - toks.append(e) - - return(' '.join(toks)) - - -def update_authorized_keys(fname, keys): - # keys is a list of AuthKeyEntries - # key_prefix is the prefix (options) to prepend - try: - fp = open(fname, "r") - lines = fp.readlines() # lines have carriage return - fp.close() - except IOError: - lines = [] - - ka_stats = {} # keys_added status - for k in keys: - ka_stats[k] = False - - to_add = [] - for key in keys: - to_add.append(key) - - for i in range(0, len(lines)): - ent = AuthKeyEntry(lines[i]) - for k in keys: - if k.base64 == ent.base64 and not k.is_comment: - ent = k - try: - to_add.remove(k) - except ValueError: - pass - lines[i] = str(ent) - - # now append any entries we did not match above - for key in to_add: - lines.append(str(key)) - - if len(lines) == 0: - return("") - else: - return('\n'.join(lines) + "\n") - - -def setup_user_keys(keys, user, key_prefix, log=None): - import pwd - saved_umask = os.umask(077) - - pwent = pwd.getpwnam(user) - - ssh_dir = '%s/.ssh' % pwent.pw_dir - if not os.path.exists(ssh_dir): - os.mkdir(ssh_dir) - os.chown(ssh_dir, pwent.pw_uid, pwent.pw_gid) - - try: - ssh_cfg = parse_ssh_config() - akeys = ssh_cfg.get("AuthorizedKeysFile", "%h/.ssh/authorized_keys") - akeys = akeys.replace("%h", pwent.pw_dir) - akeys = akeys.replace("%u", user) - if not akeys.startswith('/'): - akeys = os.path.join(pwent.pw_dir, akeys) - authorized_keys = akeys - except Exception: - authorized_keys = '%s/.ssh/authorized_keys' % pwent.pw_dir - if log: - util.logexc(log) - - key_entries = [] - for k in keys: - ke = AuthKeyEntry(k, def_opt=key_prefix) - key_entries.append(ke) - - content = update_authorized_keys(authorized_keys, key_entries) - util.write_file(authorized_keys, content, 0600) - - os.chown(authorized_keys, pwent.pw_uid, pwent.pw_gid) - util.restorecon_if_possible(ssh_dir, recursive=True) - - os.umask(saved_umask) - - -def parse_ssh_config(fname="/etc/ssh/sshd_config"): - ret = {} - fp = open(fname) - for l in fp.readlines(): - l = l.strip() - if not l or l.startswith("#"): - continue - key, val = l.split(None, 1) - ret[key] = val - fp.close() - return(ret) - -if __name__ == "__main__": - def main(): - import sys - # usage: orig_file, new_keys, [key_prefix] - # prints out merged, where 'new_keys' will trump old - ## example - ## ### begin auth_keys ### - # ssh-rsa AAAAB3NzaC1xxxxxxxxxV3csgm8cJn7UveKHkYjJp8= smoser-work - # ssh-rsa AAAAB3NzaC1xxxxxxxxxCmXp5Kt5/82cD/VN3NtHw== smoser@brickies - # ### end authorized_keys ### - # - # ### begin new_keys ### - # ssh-rsa nonmatch smoser@newhost - # ssh-rsa AAAAB3NzaC1xxxxxxxxxV3csgm8cJn7UveKHkYjJp8= new_comment - # ### end new_keys ### - # - # Then run as: - # program auth_keys new_keys \ - # 'no-port-forwarding,command=\"echo hi world;\"' - def_prefix = None - orig_key_file = sys.argv[1] - new_key_file = sys.argv[2] - if len(sys.argv) > 3: - def_prefix = sys.argv[3] - fp = open(new_key_file) - - newkeys = [] - for line in fp.readlines(): - newkeys.append(AuthKeyEntry(line, def_prefix)) - - fp.close() - print update_authorized_keys(orig_key_file, newkeys) - - main() - -# vi: ts=4 expandtab diff --git a/cloudinit/UserDataHandler.py b/cloudinit/UserDataHandler.py deleted file mode 100644 index ec914480..00000000 --- a/cloudinit/UserDataHandler.py +++ /dev/null @@ -1,262 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import email - -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText -from email.mime.base import MIMEBase -import yaml -import cloudinit -import cloudinit.util as util -import hashlib -import urllib - - -starts_with_mappings = { - '#include': 'text/x-include-url', - '#include-once': 'text/x-include-once-url', - '#!': 'text/x-shellscript', - '#cloud-config': 'text/cloud-config', - '#upstart-job': 'text/upstart-job', - '#part-handler': 'text/part-handler', - '#cloud-boothook': 'text/cloud-boothook', - '#cloud-config-archive': 'text/cloud-config-archive', -} - - -# if 'string' is compressed return decompressed otherwise return it -def decomp_str(string): - import StringIO - import gzip - try: - uncomp = gzip.GzipFile(None, "rb", 1, StringIO.StringIO(string)).read() - return(uncomp) - except: - return(string) - - -def do_include(content, appendmsg): - import os - # is just a list of urls, one per line - # also support '#include ' - includeonce = False - for line in content.splitlines(): - if line == "#include": - continue - if line == "#include-once": - includeonce = True - continue - if line.startswith("#include-once"): - line = line[len("#include-once"):].lstrip() - includeonce = True - elif line.startswith("#include"): - line = line[len("#include"):].lstrip() - if line.startswith("#"): - continue - if line.strip() == "": - continue - - # urls cannot not have leading or trailing white space - msum = hashlib.md5() # pylint: disable=E1101 - msum.update(line.strip()) - includeonce_filename = "%s/urlcache/%s" % ( - cloudinit.get_ipath_cur("data"), msum.hexdigest()) - try: - if includeonce and os.path.isfile(includeonce_filename): - with open(includeonce_filename, "r") as fp: - content = fp.read() - else: - content = urllib.urlopen(line).read() - if includeonce: - util.write_file(includeonce_filename, content, mode=0600) - except Exception: - raise - - process_includes(message_from_string(decomp_str(content)), appendmsg) - - -def explode_cc_archive(archive, appendmsg): - for ent in yaml.load(archive): - # ent can be one of: - # dict { 'filename' : 'value', 'content' : 'value', 'type' : 'value' } - # filename and type not be present - # or - # scalar(payload) - - def_type = "text/cloud-config" - if isinstance(ent, str): - ent = {'content': ent} - - content = ent.get('content', '') - mtype = ent.get('type', None) - if mtype == None: - mtype = type_from_startswith(content, def_type) - - maintype, subtype = mtype.split('/', 1) - if maintype == "text": - msg = MIMEText(content, _subtype=subtype) - else: - msg = MIMEBase(maintype, subtype) - msg.set_payload(content) - - if 'filename' in ent: - msg.add_header('Content-Disposition', 'attachment', - filename=ent['filename']) - - for header in ent.keys(): - if header in ('content', 'filename', 'type'): - continue - msg.add_header(header, ent['header']) - - _attach_part(appendmsg, msg) - - -def multi_part_count(outermsg, newcount=None): - """ - Return the number of attachments to this MIMEMultipart by looking - at its 'Number-Attachments' header. - """ - nfield = 'Number-Attachments' - if nfield not in outermsg: - outermsg[nfield] = "0" - - if newcount != None: - outermsg.replace_header(nfield, str(newcount)) - - return(int(outermsg.get('Number-Attachments', 0))) - - -def _attach_part(outermsg, part): - """ - Attach an part to an outer message. outermsg must be a MIMEMultipart. - Modifies a header in outermsg to keep track of number of attachments. - """ - cur = multi_part_count(outermsg) - if not part.get_filename(None): - part.add_header('Content-Disposition', 'attachment', - filename='part-%03d' % (cur + 1)) - outermsg.attach(part) - multi_part_count(outermsg, cur + 1) - - -def type_from_startswith(payload, default=None): - # slist is sorted longest first - slist = sorted(starts_with_mappings.keys(), key=lambda e: 0 - len(e)) - for sstr in slist: - if payload.startswith(sstr): - return(starts_with_mappings[sstr]) - return default - - -def process_includes(msg, appendmsg=None): - if appendmsg == None: - appendmsg = MIMEMultipart() - - for part in msg.walk(): - # multipart/* are just containers - if part.get_content_maintype() == 'multipart': - continue - - ctype = None - ctype_orig = part.get_content_type() - - payload = part.get_payload(decode=True) - - if ctype_orig in ("text/plain", "text/x-not-multipart"): - ctype = type_from_startswith(payload) - - if ctype is None: - ctype = ctype_orig - - if ctype in ('text/x-include-url', 'text/x-include-once-url'): - do_include(payload, appendmsg) - continue - - if ctype == "text/cloud-config-archive": - explode_cc_archive(payload, appendmsg) - continue - - if 'Content-Type' in msg: - msg.replace_header('Content-Type', ctype) - else: - msg['Content-Type'] = ctype - - _attach_part(appendmsg, part) - - -def message_from_string(data, headers=None): - if headers is None: - headers = {} - if "mime-version:" in data[0:4096].lower(): - msg = email.message_from_string(data) - for (key, val) in headers.items(): - if key in msg: - msg.replace_header(key, val) - else: - msg[key] = val - else: - mtype = headers.get("Content-Type", "text/x-not-multipart") - maintype, subtype = mtype.split("/", 1) - msg = MIMEBase(maintype, subtype, *headers) - msg.set_payload(data) - - return(msg) - - -# this is heavily wasteful, reads through userdata string input -def preprocess_userdata(data): - newmsg = MIMEMultipart() - process_includes(message_from_string(decomp_str(data)), newmsg) - return(newmsg.as_string()) - - -# callback is a function that will be called with (data, content_type, -# filename, payload) -def walk_userdata(istr, callback, data=None): - partnum = 0 - for part in message_from_string(istr).walk(): - # multipart/* are just containers - if part.get_content_maintype() == 'multipart': - continue - - ctype = part.get_content_type() - if ctype is None: - ctype = 'application/octet-stream' - - filename = part.get_filename() - if not filename: - filename = 'part-%03d' % partnum - - callback(data, ctype, filename, part.get_payload(decode=True)) - - partnum = partnum + 1 - - -if __name__ == "__main__": - def main(): - import sys - data = decomp_str(file(sys.argv[1]).read()) - newmsg = MIMEMultipart() - process_includes(message_from_string(data), newmsg) - print newmsg - print "#found %s parts" % multi_part_count(newmsg) - - main() diff --git a/cloudinit/constants.py b/cloudinit/constants.py new file mode 100644 index 00000000..7bc90f27 --- /dev/null +++ b/cloudinit/constants.py @@ -0,0 +1,37 @@ +import os + +VAR_LIB_DIR = '/var/lib/cloud' +CUR_INSTANCE_LINK = os.path.join(VAR_LIB_DIR, "instance") +BOOT_FINISHED = os.path.join(CUR_INSTANCE_LINK, "boot-finished") +SEED_DIR = os.path.join(VAR_LIB_DIR, "seed") + +CFG_ENV_NAME = "CLOUD_CFG" +CLOUD_CONFIG = '/etc/cloud/cloud.cfg' + +CFG_BUILTIN = { + 'datasource_list': ['NoCloud', + 'ConfigDrive', + 'OVF', + 'MAAS', + 'Ec2', + 'CloudStack'], + 'def_log_file': '/var/log/cloud-init.log', + 'log_cfgs': [], + 'syslog_fix_perms': 'syslog:adm' +} + +PATH_MAP = { + "handlers": "handlers", + "scripts": "scripts", + "sem": "sem", + "boothooks": "boothooks", + "userdata_raw": "user-data.txt", + "userdata": "user-data.txt.i", + "obj_pkl": "obj.pkl", + "cloud_config": "cloud-config.txt", + "data": "data", +} + +PER_INSTANCE = "once-per-instance" +PER_ALWAYS = "always" +PER_ONCE = "once" diff --git a/cloudinit/handlers/DataSource.py b/cloudinit/handlers/DataSource.py new file mode 100644 index 00000000..e2a9150d --- /dev/null +++ b/cloudinit/handlers/DataSource.py @@ -0,0 +1,214 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +DEP_FILESYSTEM = "FILESYSTEM" +DEP_NETWORK = "NETWORK" + +import cloudinit.UserDataHandler as ud +import cloudinit.util as util +import socket + + +class DataSource: + userdata = None + metadata = None + userdata_raw = None + cfgname = "" + # system config (passed in from cloudinit, + # cloud-config before input from the DataSource) + sys_cfg = {} + # datasource config, the cloud-config['datasource']['__name__'] + ds_cfg = {} # datasource config + + def __init__(self, sys_cfg=None): + if not self.cfgname: + name = str(self.__class__).split(".")[-1] + if name.startswith("DataSource"): + name = name[len("DataSource"):] + self.cfgname = name + if sys_cfg: + self.sys_cfg = sys_cfg + + self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, + ("datasource", self.cfgname), self.ds_cfg) + + def get_userdata(self): + if self.userdata == None: + self.userdata = ud.preprocess_userdata(self.userdata_raw) + return self.userdata + + def get_userdata_raw(self): + return(self.userdata_raw) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return({}) + + def get_public_ssh_keys(self): + keys = [] + if 'public-keys' not in self.metadata: + return([]) + + if isinstance(self.metadata['public-keys'], str): + return(str(self.metadata['public-keys']).splitlines()) + + if isinstance(self.metadata['public-keys'], list): + return(self.metadata['public-keys']) + + for _keyname, klist in self.metadata['public-keys'].items(): + # lp:506332 uec metadata service responds with + # data that makes boto populate a string for 'klist' rather + # than a list. + if isinstance(klist, str): + klist = [klist] + for pkey in klist: + # there is an empty string at the end of the keylist, trim it + if pkey: + keys.append(pkey) + + return(keys) + + def device_name_to_device(self, _name): + # translate a 'name' to a device + # the primary function at this point is on ec2 + # to consult metadata service, that has + # ephemeral0: sdb + # and return 'sdb' for input 'ephemeral0' + return(None) + + def get_locale(self): + return('en_US.UTF-8') + + def get_local_mirror(self): + return None + + def get_instance_id(self): + if 'instance-id' not in self.metadata: + return "iid-datasource" + return(self.metadata['instance-id']) + + def get_hostname(self, fqdn=False): + defdomain = "localdomain" + defhost = "localhost" + + domain = defdomain + if not 'local-hostname' in self.metadata: + + # this is somewhat questionable really. + # the cloud datasource was asked for a hostname + # and didn't have one. raising error might be more appropriate + # but instead, basically look up the existing hostname + toks = [] + + hostname = socket.gethostname() + + fqdn = util.get_fqdn_from_hosts(hostname) + + if fqdn and fqdn.find(".") > 0: + toks = str(fqdn).split(".") + elif hostname: + toks = [hostname, defdomain] + else: + toks = [defhost, defdomain] + + else: + # if there is an ipv4 address in 'local-hostname', then + # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx + lhost = self.metadata['local-hostname'] + if is_ipv4(lhost): + toks = "ip-%s" % lhost.replace(".", "-") + else: + toks = lhost.split(".") + + if len(toks) > 1: + hostname = toks[0] + domain = '.'.join(toks[1:]) + else: + hostname = toks[0] + + if fqdn: + return "%s.%s" % (hostname, domain) + else: + return hostname + + +# return a list of classes that have the same depends as 'depends' +# iterate through cfg_list, loading "DataSourceCollections" modules +# and calling their "get_datasource_list". +# return an ordered list of classes that match +# +# - modules must be named "DataSource", where 'item' is an entry +# in cfg_list +# - if pkglist is given, it will iterate try loading from that package +# ie, pkglist=[ "foo", "" ] +# will first try to load foo.DataSource +# then DataSource +def list_sources(cfg_list, depends, pkglist=None): + if pkglist is None: + pkglist = [] + retlist = [] + for ds_coll in cfg_list: + for pkg in pkglist: + if pkg: + pkg = "%s." % pkg + try: + mod = __import__("%sDataSource%s" % (pkg, ds_coll)) + if pkg: + mod = getattr(mod, "DataSource%s" % ds_coll) + lister = getattr(mod, "get_datasource_list") + retlist.extend(lister(depends)) + break + except: + raise + return(retlist) + + +# depends is a list of dependencies (DEP_FILESYSTEM) +# dslist is a list of 2 item lists +# dslist = [ +# ( class, ( depends-that-this-class-needs ) ) +# } +# it returns a list of 'class' that matched these deps exactly +# it is a helper function for DataSourceCollections +def list_from_depends(depends, dslist): + retlist = [] + depset = set(depends) + for elem in dslist: + (cls, deps) = elem + if depset == set(deps): + retlist.append(cls) + return(retlist) + + +def is_ipv4(instr): + """ determine if input string is a ipv4 address. return boolean""" + toks = instr.split('.') + if len(toks) != 4: + return False + + try: + toks = [x for x in toks if (int(x) < 256 and int(x) > 0)] + except: + return False + + return (len(toks) == 4) diff --git a/cloudinit/handlers/DataSourceCloudStack.py b/cloudinit/handlers/DataSourceCloudStack.py new file mode 100644 index 00000000..5afdf7b6 --- /dev/null +++ b/cloudinit/handlers/DataSourceCloudStack.py @@ -0,0 +1,92 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Cosmin Luta +# +# Author: Cosmin Luta +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +from socket import inet_ntoa +import time +import boto.utils as boto_utils +from struct import pack + + +class DataSourceCloudStack(DataSource.DataSource): + api_ver = 'latest' + seeddir = base_seeddir + '/cs' + metadata_address = None + + def __init__(self, sys_cfg=None): + DataSource.DataSource.__init__(self, sys_cfg) + # Cloudstack has its metadata/userdata URLs located at + # http:///latest/ + self.metadata_address = "http://%s/" % self.get_default_gateway() + + def get_default_gateway(self): + """ Returns the default gateway ip address in the dotted format + """ + with open("/proc/net/route", "r") as f: + for line in f.readlines(): + items = line.split("\t") + if items[1] == "00000000": + # found the default route, get the gateway + gw = inet_ntoa(pack(" +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import os.path +import os +import json +import subprocess + +DEFAULT_IID = "iid-dsconfigdrive" + + +class DataSourceConfigDrive(DataSource.DataSource): + seed = None + seeddir = base_seeddir + '/config_drive' + cfg = {} + userdata_raw = None + metadata = None + dsmode = "local" + + def __str__(self): + mstr = "DataSourceConfigDrive[%s]" % self.dsmode + mstr = mstr + " [seed=%s]" % self.seed + return(mstr) + + def get_data(self): + found = None + md = {} + ud = "" + + defaults = {"instance-id": DEFAULT_IID, "dsmode": "pass"} + + if os.path.isdir(self.seeddir): + try: + (md, ud) = read_config_drive_dir(self.seeddir) + found = self.seeddir + except nonConfigDriveDir: + pass + + if not found: + dev = cfg_drive_device() + if dev: + try: + (md, ud) = util.mount_callback_umount(dev, + read_config_drive_dir) + found = dev + except (nonConfigDriveDir, util.mountFailedError): + pass + + if not found: + return False + + if 'dsconfig' in md: + self.cfg = md['dscfg'] + + md = util.mergedict(md, defaults) + + # update interfaces and ifup only on the local datasource + # this way the DataSourceConfigDriveNet doesn't do it also. + if 'network-interfaces' in md and self.dsmode == "local": + if md['dsmode'] == "pass": + log.info("updating network interfaces from configdrive") + else: + log.debug("updating network interfaces from configdrive") + + util.write_file("/etc/network/interfaces", + md['network-interfaces']) + try: + (out, err) = util.subp(['ifup', '--all']) + if len(out) or len(err): + log.warn("ifup --all had stderr: %s" % err) + + except subprocess.CalledProcessError as exc: + log.warn("ifup --all failed: %s" % (exc.output[1])) + + self.seed = found + self.metadata = md + self.userdata_raw = ud + + if md['dsmode'] == self.dsmode: + return True + + log.debug("%s: not claiming datasource, dsmode=%s" % + (self, md['dsmode'])) + return False + + def get_public_ssh_keys(self): + if not 'public-keys' in self.metadata: + return([]) + return(self.metadata['public-keys']) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return(self.cfg) + + +class DataSourceConfigDriveNet(DataSourceConfigDrive): + dsmode = "net" + + +class nonConfigDriveDir(Exception): + pass + + +def cfg_drive_device(): + """ get the config drive device. return a string like '/dev/vdb' + or None (if there is no non-root device attached). This does not + check the contents, only reports that if there *were* a config_drive + attached, it would be this device. + per config_drive documentation, this is + "associated as the last available disk on the instance" + """ + + if 'CLOUD_INIT_CONFIG_DRIVE_DEVICE' in os.environ: + return(os.environ['CLOUD_INIT_CONFIG_DRIVE_DEVICE']) + + # we are looking for a raw block device (sda, not sda1) with a vfat + # filesystem on it. + + letters = "abcdefghijklmnopqrstuvwxyz" + devs = util.find_devs_with("TYPE=vfat") + + # filter out anything not ending in a letter (ignore partitions) + devs = [f for f in devs if f[-1] in letters] + + # sort them in reverse so "last" device is first + devs.sort(reverse=True) + + if len(devs): + return(devs[0]) + + return(None) + + +def read_config_drive_dir(source_dir): + """ + read_config_drive_dir(source_dir): + read source_dir, and return a tuple with metadata dict and user-data + string populated. If not a valid dir, raise a nonConfigDriveDir + """ + md = {} + ud = "" + + flist = ("etc/network/interfaces", "root/.ssh/authorized_keys", "meta.js") + found = [f for f in flist if os.path.isfile("%s/%s" % (source_dir, f))] + keydata = "" + + if len(found) == 0: + raise nonConfigDriveDir("%s: %s" % (source_dir, "no files found")) + + if "etc/network/interfaces" in found: + with open("%s/%s" % (source_dir, "/etc/network/interfaces")) as fp: + md['network-interfaces'] = fp.read() + + if "root/.ssh/authorized_keys" in found: + with open("%s/%s" % (source_dir, "root/.ssh/authorized_keys")) as fp: + keydata = fp.read() + + meta_js = {} + + if "meta.js" in found: + content = '' + with open("%s/%s" % (source_dir, "meta.js")) as fp: + content = fp.read() + md['meta_js'] = content + try: + meta_js = json.loads(content) + except ValueError: + raise nonConfigDriveDir("%s: %s" % + (source_dir, "invalid json in meta.js")) + + keydata = meta_js.get('public-keys', keydata) + + if keydata: + lines = keydata.splitlines() + md['public-keys'] = [l for l in lines + if len(l) and not l.startswith("#")] + + for copy in ('dsmode', 'instance-id', 'dscfg'): + if copy in meta_js: + md[copy] = meta_js[copy] + + if 'user-data' in meta_js: + ud = meta_js['user-data'] + + return(md, ud) + +datasources = ( + (DataSourceConfigDrive, (DataSource.DEP_FILESYSTEM, )), + (DataSourceConfigDriveNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +) + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) + +if __name__ == "__main__": + def main(): + import sys + import pprint + print cfg_drive_device() + (md, ud) = read_config_drive_dir(sys.argv[1]) + print "=== md ===" + pprint.pprint(md) + print "=== ud ===" + print(ud) + + main() + +# vi: ts=4 expandtab diff --git a/cloudinit/handlers/DataSourceEc2.py b/cloudinit/handlers/DataSourceEc2.py new file mode 100644 index 00000000..7051ecda --- /dev/null +++ b/cloudinit/handlers/DataSourceEc2.py @@ -0,0 +1,217 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import socket +import time +import boto.utils as boto_utils +import os.path + + +class DataSourceEc2(DataSource.DataSource): + api_ver = '2009-04-04' + seeddir = base_seeddir + '/ec2' + metadata_address = "http://169.254.169.254" + + def __str__(self): + return("DataSourceEc2") + + def get_data(self): + seedret = {} + if util.read_optional_seed(seedret, base=self.seeddir + "/"): + self.userdata_raw = seedret['user-data'] + self.metadata = seedret['meta-data'] + log.debug("using seeded ec2 data in %s" % self.seeddir) + return True + + try: + if not self.wait_for_metadata_service(): + return False + start = time.time() + self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, + None, self.metadata_address) + self.metadata = boto_utils.get_instance_metadata(self.api_ver, + self.metadata_address) + log.debug("crawl of metadata service took %ds" % (time.time() - + start)) + return True + except Exception as e: + print e + return False + + def get_instance_id(self): + return(self.metadata['instance-id']) + + def get_availability_zone(self): + return(self.metadata['placement']['availability-zone']) + + def get_local_mirror(self): + return(self.get_mirror_from_availability_zone()) + + def get_mirror_from_availability_zone(self, availability_zone=None): + # availability is like 'us-west-1b' or 'eu-west-1a' + if availability_zone == None: + availability_zone = self.get_availability_zone() + + fallback = None + + if self.is_vpc(): + return fallback + + try: + host = "%s.ec2.archive.ubuntu.com" % availability_zone[:-1] + socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM) + return 'http://%s/ubuntu/' % host + except: + return fallback + + def wait_for_metadata_service(self): + mcfg = self.ds_cfg + + if not hasattr(mcfg, "get"): + mcfg = {} + + max_wait = 120 + try: + max_wait = int(mcfg.get("max_wait", max_wait)) + except Exception: + util.logexc(log) + log.warn("Failed to get max wait. using %s" % max_wait) + + if max_wait == 0: + return False + + timeout = 50 + try: + timeout = int(mcfg.get("timeout", timeout)) + except Exception: + util.logexc(log) + log.warn("Failed to get timeout, using %s" % timeout) + + def_mdurls = ["http://169.254.169.254", "http://instance-data:8773"] + mdurls = mcfg.get("metadata_urls", def_mdurls) + + # Remove addresses from the list that wont resolve. + filtered = [x for x in mdurls if util.is_resolvable_url(x)] + + if set(filtered) != set(mdurls): + log.debug("removed the following from metadata urls: %s" % + list((set(mdurls) - set(filtered)))) + + if len(filtered): + mdurls = filtered + else: + log.warn("Empty metadata url list! using default list") + mdurls = def_mdurls + + urls = [] + url2base = {False: False} + for url in mdurls: + cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver) + urls.append(cur) + url2base[cur] = url + + starttime = time.time() + url = util.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, status_cb=log.warn) + + if url: + log.debug("Using metadata source: '%s'" % url2base[url]) + else: + log.critical("giving up on md after %i seconds\n" % + int(time.time() - starttime)) + + self.metadata_address = url2base[url] + return (bool(url)) + + def device_name_to_device(self, name): + # consult metadata service, that has + # ephemeral0: sdb + # and return 'sdb' for input 'ephemeral0' + if 'block-device-mapping' not in self.metadata: + return(None) + + found = None + for entname, device in self.metadata['block-device-mapping'].items(): + if entname == name: + found = device + break + # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0' + if entname == "ephemeral" and name == "ephemeral0": + found = device + if found == None: + log.debug("unable to convert %s to a device" % name) + return None + + # LP: #611137 + # the metadata service may believe that devices are named 'sda' + # when the kernel named them 'vda' or 'xvda' + # we want to return the correct value for what will actually + # exist in this instance + mappings = {"sd": ("vd", "xvd")} + ofound = found + short = os.path.basename(found) + + if not found.startswith("/"): + found = "/dev/%s" % found + + if os.path.exists(found): + return(found) + + for nfrom, tlist in mappings.items(): + if not short.startswith(nfrom): + continue + for nto in tlist: + cand = "/dev/%s%s" % (nto, short[len(nfrom):]) + if os.path.exists(cand): + log.debug("remapped device name %s => %s" % (found, cand)) + return(cand) + + # on t1.micro, ephemeral0 will appear in block-device-mapping from + # metadata, but it will not exist on disk (and never will) + # at this pint, we've verified that the path did not exist + # in the special case of 'ephemeral0' return None to avoid bogus + # fstab entry (LP: #744019) + if name == "ephemeral0": + return None + return ofound + + def is_vpc(self): + # per comment in LP: #615545 + ph = "public-hostname" + p4 = "public-ipv4" + if ((ph not in self.metadata or self.metadata[ph] == "") and + (p4 not in self.metadata or self.metadata[p4] == "")): + return True + return False + + +datasources = [ + (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +] + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/handlers/DataSourceMAAS.py b/cloudinit/handlers/DataSourceMAAS.py new file mode 100644 index 00000000..61a0038f --- /dev/null +++ b/cloudinit/handlers/DataSourceMAAS.py @@ -0,0 +1,345 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import errno +import oauth.oauth as oauth +import os.path +import urllib2 +import time + + +MD_VERSION = "2012-03-01" + + +class DataSourceMAAS(DataSource.DataSource): + """ + DataSourceMAAS reads instance information from MAAS. + Given a config metadata_url, and oauth tokens, it expects to find + files under the root named: + instance-id + user-data + hostname + """ + seeddir = base_seeddir + '/maas' + baseurl = None + + def __str__(self): + return("DataSourceMAAS[%s]" % self.baseurl) + + def get_data(self): + mcfg = self.ds_cfg + + try: + (userdata, metadata) = read_maas_seed_dir(self.seeddir) + self.userdata_raw = userdata + self.metadata = metadata + self.baseurl = self.seeddir + return True + except MAASSeedDirNone: + pass + except MAASSeedDirMalformed as exc: + log.warn("%s was malformed: %s\n" % (self.seeddir, exc)) + raise + + try: + # if there is no metadata_url, then we're not configured + url = mcfg.get('metadata_url', None) + if url == None: + return False + + if not self.wait_for_metadata_service(url): + return False + + self.baseurl = url + + (userdata, metadata) = read_maas_seed_url(self.baseurl, + self.md_headers) + self.userdata_raw = userdata + self.metadata = metadata + return True + except Exception: + util.logexc(log) + return False + + def md_headers(self, url): + mcfg = self.ds_cfg + + # if we are missing token_key, token_secret or consumer_key + # then just do non-authed requests + for required in ('token_key', 'token_secret', 'consumer_key'): + if required not in mcfg: + return({}) + + consumer_secret = mcfg.get('consumer_secret', "") + + return(oauth_headers(url=url, consumer_key=mcfg['consumer_key'], + token_key=mcfg['token_key'], token_secret=mcfg['token_secret'], + consumer_secret=consumer_secret)) + + def wait_for_metadata_service(self, url): + mcfg = self.ds_cfg + + max_wait = 120 + try: + max_wait = int(mcfg.get("max_wait", max_wait)) + except Exception: + util.logexc(log) + log.warn("Failed to get max wait. using %s" % max_wait) + + if max_wait == 0: + return False + + timeout = 50 + try: + timeout = int(mcfg.get("timeout", timeout)) + except Exception: + util.logexc(log) + log.warn("Failed to get timeout, using %s" % timeout) + + starttime = time.time() + check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) + url = util.wait_for_url(urls=[check_url], max_wait=max_wait, + timeout=timeout, status_cb=log.warn, + headers_cb=self.md_headers) + + if url: + log.debug("Using metadata source: '%s'" % url) + else: + log.critical("giving up on md after %i seconds\n" % + int(time.time() - starttime)) + + return (bool(url)) + + +def read_maas_seed_dir(seed_d): + """ + Return user-data and metadata for a maas seed dir in seed_d. + Expected format of seed_d are the following files: + * instance-id + * local-hostname + * user-data + """ + files = ('local-hostname', 'instance-id', 'user-data', 'public-keys') + md = {} + + if not os.path.isdir(seed_d): + raise MAASSeedDirNone("%s: not a directory") + + for fname in files: + try: + with open(os.path.join(seed_d, fname)) as fp: + md[fname] = fp.read() + fp.close() + except IOError as e: + if e.errno != errno.ENOENT: + raise + + return(check_seed_contents(md, seed_d)) + + +def read_maas_seed_url(seed_url, header_cb=None, timeout=None, + version=MD_VERSION): + """ + Read the maas datasource at seed_url. + header_cb is a method that should return a headers dictionary that will + be given to urllib2.Request() + + Expected format of seed_url is are the following files: + * //meta-data/instance-id + * //meta-data/local-hostname + * //user-data + """ + files = ('meta-data/local-hostname', + 'meta-data/instance-id', + 'meta-data/public-keys', + 'user-data') + + base_url = "%s/%s" % (seed_url, version) + md = {} + for fname in files: + url = "%s/%s" % (base_url, fname) + if header_cb: + headers = header_cb(url) + else: + headers = {} + + try: + req = urllib2.Request(url, data=None, headers=headers) + resp = urllib2.urlopen(req, timeout=timeout) + md[os.path.basename(fname)] = resp.read() + except urllib2.HTTPError as e: + if e.code != 404: + raise + + return(check_seed_contents(md, seed_url)) + + +def check_seed_contents(content, seed): + """Validate if content is Is the content a dict that is valid as a + return for a datasource. + Either return a (userdata, metadata) tuple or + Raise MAASSeedDirMalformed or MAASSeedDirNone + """ + md_required = ('instance-id', 'local-hostname') + found = content.keys() + + if len(content) == 0: + raise MAASSeedDirNone("%s: no data files found" % seed) + + missing = [k for k in md_required if k not in found] + if len(missing): + raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing)) + + userdata = content.get('user-data', "") + md = {} + for (key, val) in content.iteritems(): + if key == 'user-data': + continue + md[key] = val + + return(userdata, md) + + +def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): + consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) + token = oauth.OAuthToken(token_key, token_secret) + params = { + 'oauth_version': "1.0", + 'oauth_nonce': oauth.generate_nonce(), + 'oauth_timestamp': int(time.time()), + 'oauth_token': token.key, + 'oauth_consumer_key': consumer.key, + } + req = oauth.OAuthRequest(http_url=url, parameters=params) + req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(), + consumer, token) + return(req.to_header()) + + +class MAASSeedDirNone(Exception): + pass + + +class MAASSeedDirMalformed(Exception): + pass + + +datasources = [ + (DataSourceMAAS, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +] + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) + + +if __name__ == "__main__": + def main(): + """ + Call with single argument of directory or http or https url. + If url is given additional arguments are allowed, which will be + interpreted as consumer_key, token_key, token_secret, consumer_secret + """ + import argparse + import pprint + + parser = argparse.ArgumentParser(description='Interact with MAAS DS') + parser.add_argument("--config", metavar="file", + help="specify DS config file", default=None) + parser.add_argument("--ckey", metavar="key", + help="the consumer key to auth with", default=None) + parser.add_argument("--tkey", metavar="key", + help="the token key to auth with", default=None) + parser.add_argument("--csec", metavar="secret", + help="the consumer secret (likely '')", default="") + parser.add_argument("--tsec", metavar="secret", + help="the token secret to auth with", default=None) + parser.add_argument("--apiver", metavar="version", + help="the apiver to use ("" can be used)", default=MD_VERSION) + + subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") + subcmds.add_parser('crawl', help="crawl the datasource") + subcmds.add_parser('get', help="do a single GET of provided url") + subcmds.add_parser('check-seed', help="read andn verify seed at url") + + parser.add_argument("url", help="the data source to query") + + args = parser.parse_args() + + creds = {'consumer_key': args.ckey, 'token_key': args.tkey, + 'token_secret': args.tsec, 'consumer_secret': args.csec} + + if args.config: + import yaml + with open(args.config) as fp: + cfg = yaml.load(fp) + if 'datasource' in cfg: + cfg = cfg['datasource']['MAAS'] + for key in creds.keys(): + if key in cfg and creds[key] == None: + creds[key] = cfg[key] + + def geturl(url, headers_cb): + req = urllib2.Request(url, data=None, headers=headers_cb(url)) + return(urllib2.urlopen(req).read()) + + def printurl(url, headers_cb): + print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) + + def crawl(url, headers_cb=None): + if url.endswith("/"): + for line in geturl(url, headers_cb).splitlines(): + if line.endswith("/"): + crawl("%s%s" % (url, line), headers_cb) + else: + printurl("%s%s" % (url, line), headers_cb) + else: + printurl(url, headers_cb) + + def my_headers(url): + headers = {} + if creds.get('consumer_key', None) != None: + headers = oauth_headers(url, **creds) + return headers + + if args.subcmd == "check-seed": + if args.url.startswith("http"): + (userdata, metadata) = read_maas_seed_url(args.url, + header_cb=my_headers, version=args.apiver) + else: + (userdata, metadata) = read_maas_seed_url(args.url) + print "=== userdata ===" + print userdata + print "=== metadata ===" + pprint.pprint(metadata) + + elif args.subcmd == "get": + printurl(args.url, my_headers) + + elif args.subcmd == "crawl": + if not args.url.endswith("/"): + args.url = "%s/" % args.url + crawl(args.url, my_headers) + + main() diff --git a/cloudinit/handlers/DataSourceNoCloud.py b/cloudinit/handlers/DataSourceNoCloud.py new file mode 100644 index 00000000..e8c56b8f --- /dev/null +++ b/cloudinit/handlers/DataSourceNoCloud.py @@ -0,0 +1,232 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import errno +import subprocess + + +class DataSourceNoCloud(DataSource.DataSource): + metadata = None + userdata = None + userdata_raw = None + supported_seed_starts = ("/", "file://") + dsmode = "local" + seed = None + cmdline_id = "ds=nocloud" + seeddir = base_seeddir + '/nocloud' + + def __str__(self): + mstr = "DataSourceNoCloud" + mstr = mstr + " [seed=%s]" % self.seed + return(mstr) + + def get_data(self): + defaults = { + "instance-id": "nocloud", "dsmode": self.dsmode + } + + found = [] + md = {} + ud = "" + + try: + # parse the kernel command line, getting data passed in + if parse_cmdline_data(self.cmdline_id, md): + found.append("cmdline") + except: + util.logexc(log) + return False + + # check to see if the seeddir has data. + seedret = {} + if util.read_optional_seed(seedret, base=self.seeddir + "/"): + md = util.mergedict(md, seedret['meta-data']) + ud = seedret['user-data'] + found.append(self.seeddir) + log.debug("using seeded cache data in %s" % self.seeddir) + + # if the datasource config had a 'seedfrom' entry, then that takes + # precedence over a 'seedfrom' that was found in a filesystem + # but not over external medi + if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']: + found.append("ds_config") + md["seedfrom"] = self.ds_cfg['seedfrom'] + + fslist = util.find_devs_with("TYPE=vfat") + fslist.extend(util.find_devs_with("TYPE=iso9660")) + + label_list = util.find_devs_with("LABEL=cidata") + devlist = list(set(fslist) & set(label_list)) + devlist.sort(reverse=True) + + for dev in devlist: + try: + (newmd, newud) = util.mount_callback_umount(dev, + util.read_seeded) + md = util.mergedict(newmd, md) + ud = newud + + # for seed from a device, the default mode is 'net'. + # that is more likely to be what is desired. + # If they want dsmode of local, then they must + # specify that. + if 'dsmode' not in md: + md['dsmode'] = "net" + + log.debug("using data from %s" % dev) + found.append(dev) + break + except OSError, e: + if e.errno != errno.ENOENT: + raise + except util.mountFailedError: + log.warn("Failed to mount %s when looking for seed" % dev) + + # there was no indication on kernel cmdline or data + # in the seeddir suggesting this handler should be used. + if len(found) == 0: + return False + + seeded_interfaces = None + + # the special argument "seedfrom" indicates we should + # attempt to seed the userdata / metadata from its value + # its primarily value is in allowing the user to type less + # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg + if "seedfrom" in md: + seedfrom = md["seedfrom"] + seedfound = False + for proto in self.supported_seed_starts: + if seedfrom.startswith(proto): + seedfound = proto + break + if not seedfound: + log.debug("seed from %s not supported by %s" % + (seedfrom, self.__class__)) + return False + + if 'network-interfaces' in md: + seeded_interfaces = self.dsmode + + # this could throw errors, but the user told us to do it + # so if errors are raised, let them raise + (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) + log.debug("using seeded cache data from %s" % seedfrom) + + # values in the command line override those from the seed + md = util.mergedict(md, md_seed) + found.append(seedfrom) + + md = util.mergedict(md, defaults) + + # update the network-interfaces if metadata had 'network-interfaces' + # entry and this is the local datasource, or 'seedfrom' was used + # and the source of the seed was self.dsmode + # ('local' for NoCloud, 'net' for NoCloudNet') + if ('network-interfaces' in md and + (self.dsmode in ("local", seeded_interfaces))): + log.info("updating network interfaces from nocloud") + + util.write_file("/etc/network/interfaces", + md['network-interfaces']) + try: + (out, err) = util.subp(['ifup', '--all']) + if len(out) or len(err): + log.warn("ifup --all had stderr: %s" % err) + + except subprocess.CalledProcessError as exc: + log.warn("ifup --all failed: %s" % (exc.output[1])) + + self.seed = ",".join(found) + self.metadata = md + self.userdata_raw = ud + + if md['dsmode'] == self.dsmode: + return True + + log.debug("%s: not claiming datasource, dsmode=%s" % + (self, md['dsmode'])) + return False + + +# returns true or false indicating if cmdline indicated +# that this module should be used +# example cmdline: +# root=LABEL=uec-rootfs ro ds=nocloud +def parse_cmdline_data(ds_id, fill, cmdline=None): + if cmdline is None: + cmdline = util.get_cmdline() + cmdline = " %s " % cmdline + + if not (" %s " % ds_id in cmdline or " %s;" % ds_id in cmdline): + return False + + argline = "" + # cmdline can contain: + # ds=nocloud[;key=val;key=val] + for tok in cmdline.split(): + if tok.startswith(ds_id): + argline = tok.split("=", 1) + + # argline array is now 'nocloud' followed optionally by + # a ';' and then key=value pairs also terminated with ';' + tmp = argline[1].split(";") + if len(tmp) > 1: + kvpairs = tmp[1:] + else: + kvpairs = () + + # short2long mapping to save cmdline typing + s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"} + for item in kvpairs: + try: + (k, v) = item.split("=", 1) + except: + k = item + v = None + if k in s2l: + k = s2l[k] + fill[k] = v + + return(True) + + +class DataSourceNoCloudNet(DataSourceNoCloud): + cmdline_id = "ds=nocloud-net" + supported_seed_starts = ("http://", "https://", "ftp://") + seeddir = base_seeddir + '/nocloud-net' + dsmode = "net" + + +datasources = ( + (DataSourceNoCloud, (DataSource.DEP_FILESYSTEM, )), + (DataSourceNoCloudNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +) + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/handlers/DataSourceOVF.py b/cloudinit/handlers/DataSourceOVF.py new file mode 100644 index 00000000..a0b1b518 --- /dev/null +++ b/cloudinit/handlers/DataSourceOVF.py @@ -0,0 +1,332 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import os.path +import os +from xml.dom import minidom +import base64 +import re +import tempfile +import subprocess + + +class DataSourceOVF(DataSource.DataSource): + seed = None + seeddir = base_seeddir + '/ovf' + environment = None + cfg = {} + userdata_raw = None + metadata = None + supported_seed_starts = ("/", "file://") + + def __str__(self): + mstr = "DataSourceOVF" + mstr = mstr + " [seed=%s]" % self.seed + return(mstr) + + def get_data(self): + found = [] + md = {} + ud = "" + + defaults = { + "instance-id": "iid-dsovf" + } + + (seedfile, contents) = get_ovf_env(base_seeddir) + if seedfile: + # found a seed dir + seed = "%s/%s" % (base_seeddir, seedfile) + (md, ud, cfg) = read_ovf_environment(contents) + self.environment = contents + + found.append(seed) + else: + np = {'iso': transport_iso9660, + 'vmware-guestd': transport_vmware_guestd, } + name = None + for name, transfunc in np.iteritems(): + (contents, _dev, _fname) = transfunc() + if contents: + break + + if contents: + (md, ud, cfg) = read_ovf_environment(contents) + self.environment = contents + found.append(name) + + # There was no OVF transports found + if len(found) == 0: + return False + + if 'seedfrom' in md and md['seedfrom']: + seedfrom = md['seedfrom'] + seedfound = False + for proto in self.supported_seed_starts: + if seedfrom.startswith(proto): + seedfound = proto + break + if not seedfound: + log.debug("seed from %s not supported by %s" % + (seedfrom, self.__class__)) + return False + + (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) + log.debug("using seeded cache data from %s" % seedfrom) + + md = util.mergedict(md, md_seed) + found.append(seedfrom) + + md = util.mergedict(md, defaults) + self.seed = ",".join(found) + self.metadata = md + self.userdata_raw = ud + self.cfg = cfg + return True + + def get_public_ssh_keys(self): + if not 'public-keys' in self.metadata: + return([]) + return([self.metadata['public-keys'], ]) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return(self.cfg) + + +class DataSourceOVFNet(DataSourceOVF): + seeddir = base_seeddir + '/ovf-net' + supported_seed_starts = ("http://", "https://", "ftp://") + + +# this will return a dict with some content +# meta-data, user-data +def read_ovf_environment(contents): + props = getProperties(contents) + md = {} + cfg = {} + ud = "" + cfg_props = ['password', ] + md_props = ['seedfrom', 'local-hostname', 'public-keys', 'instance-id'] + for prop, val in props.iteritems(): + if prop == 'hostname': + prop = "local-hostname" + if prop in md_props: + md[prop] = val + elif prop in cfg_props: + cfg[prop] = val + elif prop == "user-data": + try: + ud = base64.decodestring(val) + except: + ud = val + return(md, ud, cfg) + + +# returns tuple of filename (in 'dirname', and the contents of the file) +# on "not found", returns 'None' for filename and False for contents +def get_ovf_env(dirname): + env_names = ("ovf-env.xml", "ovf_env.xml", "OVF_ENV.XML", "OVF-ENV.XML") + for fname in env_names: + if os.path.isfile("%s/%s" % (dirname, fname)): + fp = open("%s/%s" % (dirname, fname)) + contents = fp.read() + fp.close() + return(fname, contents) + return(None, False) + + +# transport functions take no input and return +# a 3 tuple of content, path, filename +def transport_iso9660(require_iso=True): + + # default_regex matches values in + # /lib/udev/rules.d/60-cdrom_id.rules + # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end" + envname = "CLOUD_INIT_CDROM_DEV_REGEX" + default_regex = "^(sr[0-9]+|hd[a-z]|xvd.*)" + + devname_regex = os.environ.get(envname, default_regex) + cdmatch = re.compile(devname_regex) + + # go through mounts to see if it was already mounted + fp = open("/proc/mounts") + mounts = fp.readlines() + fp.close() + + mounted = {} + for mpline in mounts: + (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() + mounted[dev] = (dev, fstype, mp, False) + mp = mp.replace("\\040", " ") + if fstype != "iso9660" and require_iso: + continue + + if cdmatch.match(dev[5:]) == None: # take off '/dev/' + continue + + (fname, contents) = get_ovf_env(mp) + if contents is not False: + return(contents, dev, fname) + + tmpd = None + dvnull = None + + devs = os.listdir("/dev/") + devs.sort() + + for dev in devs: + fullp = "/dev/%s" % dev + + if fullp in mounted or not cdmatch.match(dev) or os.path.isdir(fullp): + continue + + fp = None + try: + fp = open(fullp, "rb") + fp.read(512) + fp.close() + except: + if fp: + fp.close() + continue + + if tmpd is None: + tmpd = tempfile.mkdtemp() + if dvnull is None: + try: + dvnull = open("/dev/null") + except: + pass + + cmd = ["mount", "-o", "ro", fullp, tmpd] + if require_iso: + cmd.extend(('-t', 'iso9660')) + + rc = subprocess.call(cmd, stderr=dvnull, stdout=dvnull, stdin=dvnull) + if rc: + continue + + (fname, contents) = get_ovf_env(tmpd) + + subprocess.call(["umount", tmpd]) + + if contents is not False: + os.rmdir(tmpd) + return(contents, fullp, fname) + + if tmpd: + os.rmdir(tmpd) + + if dvnull: + dvnull.close() + + return(False, None, None) + + +def transport_vmware_guestd(): + # http://blogs.vmware.com/vapp/2009/07/ \ + # selfconfiguration-and-the-ovf-environment.html + # try: + # cmd = ['vmware-guestd', '--cmd', 'info-get guestinfo.ovfEnv'] + # (out, err) = subp(cmd) + # return(out, 'guestinfo.ovfEnv', 'vmware-guestd') + # except: + # # would need to error check here and see why this failed + # # to know if log/error should be raised + # return(False, None, None) + return(False, None, None) + + +def findChild(node, filter_func): + ret = [] + if not node.hasChildNodes(): + return ret + for child in node.childNodes: + if filter_func(child): + ret.append(child) + return(ret) + + +def getProperties(environString): + dom = minidom.parseString(environString) + if dom.documentElement.localName != "Environment": + raise Exception("No Environment Node") + + if not dom.documentElement.hasChildNodes(): + raise Exception("No Child Nodes") + + envNsURI = "http://schemas.dmtf.org/ovf/environment/1" + + # could also check here that elem.namespaceURI == + # "http://schemas.dmtf.org/ovf/environment/1" + propSections = findChild(dom.documentElement, + lambda n: n.localName == "PropertySection") + + if len(propSections) == 0: + raise Exception("No 'PropertySection's") + + props = {} + propElems = findChild(propSections[0], lambda n: n.localName == "Property") + + for elem in propElems: + key = elem.attributes.getNamedItemNS(envNsURI, "key").value + val = elem.attributes.getNamedItemNS(envNsURI, "value").value + props[key] = val + + return(props) + + +datasources = ( + (DataSourceOVF, (DataSource.DEP_FILESYSTEM, )), + (DataSourceOVFNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +) + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) + + +if __name__ == "__main__": + def main(): + import sys + envStr = open(sys.argv[1]).read() + props = getProperties(envStr) + import pprint + pprint.pprint(props) + + md, ud, cfg = read_ovf_environment(envStr) + print "=== md ===" + pprint.pprint(md) + print "=== ud ===" + pprint.pprint(ud) + print "=== cfg ===" + pprint.pprint(cfg) + + main() diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py new file mode 100644 index 00000000..a16bdde6 --- /dev/null +++ b/cloudinit/handlers/__init__.py @@ -0,0 +1,274 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2008-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Chuck Short +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import yaml +import cloudinit +import cloudinit.util as util +import sys +import traceback +import os +import subprocess +import time + +per_instance = cloudinit.per_instance +per_always = cloudinit.per_always +per_once = cloudinit.per_once + + +class CloudConfig(): + cfgfile = None + cfg = None + + def __init__(self, cfgfile, cloud=None, ds_deps=None): + if cloud == None: + self.cloud = cloudinit.CloudInit(ds_deps) + self.cloud.get_data_source() + else: + self.cloud = cloud + self.cfg = self.get_config_obj(cfgfile) + + def get_config_obj(self, cfgfile): + try: + cfg = util.read_conf(cfgfile) + except: + # TODO: this 'log' could/should be passed in + cloudinit.log.critical("Failed loading of cloud config '%s'. " + "Continuing with empty config\n" % cfgfile) + cloudinit.log.debug(traceback.format_exc() + "\n") + cfg = None + if cfg is None: + cfg = {} + + try: + ds_cfg = self.cloud.datasource.get_config_obj() + except: + ds_cfg = {} + + cfg = util.mergedict(cfg, ds_cfg) + return(util.mergedict(cfg, self.cloud.cfg)) + + def handle(self, name, args, freq=None): + try: + mod = __import__("cc_" + name.replace("-", "_"), globals()) + def_freq = getattr(mod, "frequency", per_instance) + handler = getattr(mod, "handle") + + if not freq: + freq = def_freq + + self.cloud.sem_and_run("config-" + name, freq, handler, + [name, self.cfg, self.cloud, cloudinit.log, args]) + except: + raise + + +# reads a cloudconfig module list, returns +# a 2 dimensional array suitable to pass to run_cc_modules +def read_cc_modules(cfg, name): + if name not in cfg: + return([]) + module_list = [] + # create 'module_list', an array of arrays + # where array[0] = config + # array[1] = freq + # array[2:] = arguemnts + for item in cfg[name]: + if isinstance(item, str): + module_list.append((item,)) + elif isinstance(item, list): + module_list.append(item) + else: + raise TypeError("failed to read '%s' item in config") + return(module_list) + + +def run_cc_modules(cc, module_list, log): + failures = [] + for cfg_mod in module_list: + name = cfg_mod[0] + freq = None + run_args = [] + if len(cfg_mod) > 1: + freq = cfg_mod[1] + if len(cfg_mod) > 2: + run_args = cfg_mod[2:] + + try: + log.debug("handling %s with freq=%s and args=%s" % + (name, freq, run_args)) + cc.handle(name, run_args, freq=freq) + except: + log.warn(traceback.format_exc()) + log.error("config handling of %s, %s, %s failed\n" % + (name, freq, run_args)) + failures.append(name) + + return(failures) + + +# always returns well formated values +# cfg is expected to have an entry 'output' in it, which is a dictionary +# that includes entries for 'init', 'config', 'final' or 'all' +# init: /var/log/cloud.out +# config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ] +# final: +# output: "| logger -p" +# error: "> /dev/null" +# this returns the specific 'mode' entry, cleanly formatted, with value +# None if if none is given +def get_output_cfg(cfg, mode="init"): + ret = [None, None] + if not 'output' in cfg: + return ret + + outcfg = cfg['output'] + if mode in outcfg: + modecfg = outcfg[mode] + else: + if 'all' not in outcfg: + return ret + # if there is a 'all' item in the output list + # then it applies to all users of this (init, config, final) + modecfg = outcfg['all'] + + # if value is a string, it specifies stdout and stderr + if isinstance(modecfg, str): + ret = [modecfg, modecfg] + + # if its a list, then we expect (stdout, stderr) + if isinstance(modecfg, list): + if len(modecfg) > 0: + ret[0] = modecfg[0] + if len(modecfg) > 1: + ret[1] = modecfg[1] + + # if it is a dictionary, expect 'out' and 'error' + # items, which indicate out and error + if isinstance(modecfg, dict): + if 'output' in modecfg: + ret[0] = modecfg['output'] + if 'error' in modecfg: + ret[1] = modecfg['error'] + + # if err's entry == "&1", then make it same as stdout + # as in shell syntax of "echo foo >/dev/null 2>&1" + if ret[1] == "&1": + ret[1] = ret[0] + + swlist = [">>", ">", "|"] + for i in range(len(ret)): + if not ret[i]: + continue + val = ret[i].lstrip() + found = False + for s in swlist: + if val.startswith(s): + val = "%s %s" % (s, val[len(s):].strip()) + found = True + break + if not found: + # default behavior is append + val = "%s %s" % (">>", val.strip()) + ret[i] = val + + return(ret) + + +# redirect_output(outfmt, errfmt, orig_out, orig_err) +# replace orig_out and orig_err with filehandles specified in outfmt or errfmt +# fmt can be: +# > FILEPATH +# >> FILEPATH +# | program [ arg1 [ arg2 [ ... ] ] ] +# +# with a '|', arguments are passed to shell, so one level of +# shell escape is required. +def redirect_output(outfmt, errfmt, o_out=sys.stdout, o_err=sys.stderr): + if outfmt: + (mode, arg) = outfmt.split(" ", 1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": + owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) + new_fp = proc.stdin + else: + raise TypeError("invalid type for outfmt: %s" % outfmt) + + if o_out: + os.dup2(new_fp.fileno(), o_out.fileno()) + if errfmt == outfmt: + os.dup2(new_fp.fileno(), o_err.fileno()) + return + + if errfmt: + (mode, arg) = errfmt.split(" ", 1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": + owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) + new_fp = proc.stdin + else: + raise TypeError("invalid type for outfmt: %s" % outfmt) + + if o_err: + os.dup2(new_fp.fileno(), o_err.fileno()) + return + + +def run_per_instance(name, func, args, clear_on_fail=False): + semfile = "%s/%s" % (cloudinit.get_ipath_cur("data"), name) + if os.path.exists(semfile): + return + + util.write_file(semfile, str(time.time())) + try: + func(*args) + except: + if clear_on_fail: + os.unlink(semfile) + raise + + +# apt_get top level command (install, update...), and args to pass it +def apt_get(tlc, args=None): + if args is None: + args = [] + e = os.environ.copy() + e['DEBIAN_FRONTEND'] = 'noninteractive' + cmd = ['apt-get', '--option', 'Dpkg::Options::=--force-confold', + '--assume-yes', tlc] + cmd.extend(args) + subprocess.check_call(cmd, env=e) + + +def update_package_sources(): + run_per_instance("update-sources", apt_get, ("update",)) + + +def install_packages(pkglist): + update_package_sources() + apt_get("install", pkglist) diff --git a/cloudinit/handlers/cc_apt_pipelining.py b/cloudinit/handlers/cc_apt_pipelining.py new file mode 100644 index 00000000..0286a9ae --- /dev/null +++ b/cloudinit/handlers/cc_apt_pipelining.py @@ -0,0 +1,53 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Ben Howard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance + +frequency = per_instance +default_file = "/etc/apt/apt.conf.d/90cloud-init-pipelining" + + +def handle(_name, cfg, _cloud, log, _args): + + apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) + apt_pipe_value = str(apt_pipe_value).lower() + + if apt_pipe_value == "false": + write_apt_snippet("0", log) + + elif apt_pipe_value in ("none", "unchanged", "os"): + return + + elif apt_pipe_value in str(range(0, 6)): + write_apt_snippet(apt_pipe_value, log) + + else: + log.warn("Invalid option for apt_pipeling: %s" % apt_pipe_value) + + +def write_apt_snippet(setting, log, f_name=default_file): + """ Writes f_name with apt pipeline depth 'setting' """ + + acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' + file_contents = ("//Written by cloud-init per 'apt_pipelining'\n" + + (acquire_pipeline_depth % setting)) + + util.write_file(f_name, file_contents) + + log.debug("Wrote %s with APT pipeline setting" % f_name) diff --git a/cloudinit/handlers/cc_apt_update_upgrade.py b/cloudinit/handlers/cc_apt_update_upgrade.py new file mode 100644 index 00000000..a7049bce --- /dev/null +++ b/cloudinit/handlers/cc_apt_update_upgrade.py @@ -0,0 +1,241 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import traceback +import os +import glob +import cloudinit.CloudConfig as cc + + +def handle(_name, cfg, cloud, log, _args): + update = util.get_cfg_option_bool(cfg, 'apt_update', False) + upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) + + release = get_release() + + mirror = find_apt_mirror(cloud, cfg) + + log.debug("selected mirror at: %s" % mirror) + + if not util.get_cfg_option_bool(cfg, \ + 'apt_preserve_sources_list', False): + generate_sources_list(release, mirror) + old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', \ + "archive.ubuntu.com/ubuntu") + rename_apt_lists(old_mir, mirror) + + # set up proxy + proxy = cfg.get("apt_proxy", None) + proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy" + if proxy: + try: + contents = "Acquire::HTTP::Proxy \"%s\";\n" + with open(proxy_filename, "w") as fp: + fp.write(contents % proxy) + except Exception as e: + log.warn("Failed to write proxy to %s" % proxy_filename) + elif os.path.isfile(proxy_filename): + os.unlink(proxy_filename) + + # process 'apt_sources' + if 'apt_sources' in cfg: + errors = add_sources(cfg['apt_sources'], + {'MIRROR': mirror, 'RELEASE': release}) + for e in errors: + log.warn("Source Error: %s\n" % ':'.join(e)) + + dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False) + if dconf_sel: + log.debug("setting debconf selections per cloud config") + try: + util.subp(('debconf-set-selections', '-'), dconf_sel) + except: + log.error("Failed to run debconf-set-selections") + log.debug(traceback.format_exc()) + + pkglist = util.get_cfg_option_list_or_str(cfg, 'packages', []) + + errors = [] + if update or len(pkglist) or upgrade: + try: + cc.update_package_sources() + except subprocess.CalledProcessError as e: + log.warn("apt-get update failed") + log.debug(traceback.format_exc()) + errors.append(e) + + if upgrade: + try: + cc.apt_get("upgrade") + except subprocess.CalledProcessError as e: + log.warn("apt upgrade failed") + log.debug(traceback.format_exc()) + errors.append(e) + + if len(pkglist): + try: + cc.install_packages(pkglist) + except subprocess.CalledProcessError as e: + log.warn("Failed to install packages: %s " % pkglist) + log.debug(traceback.format_exc()) + errors.append(e) + + if len(errors): + raise errors[0] + + return(True) + + +def mirror2lists_fileprefix(mirror): + string = mirror + # take of http:// or ftp:// + if string.endswith("/"): + string = string[0:-1] + pos = string.find("://") + if pos >= 0: + string = string[pos + 3:] + string = string.replace("/", "_") + return string + + +def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"): + oprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(omirror)) + nprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(new_mirror)) + if(oprefix == nprefix): + return + olen = len(oprefix) + for filename in glob.glob("%s_*" % oprefix): + os.rename(filename, "%s%s" % (nprefix, filename[olen:])) + + +def get_release(): + stdout, _stderr = subprocess.Popen(['lsb_release', '-cs'], + stdout=subprocess.PIPE).communicate() + return(str(stdout).strip()) + + +def generate_sources_list(codename, mirror): + util.render_to_file('sources.list', '/etc/apt/sources.list', \ + {'mirror': mirror, 'codename': codename}) + + +def add_sources(srclist, searchList=None): + """ + add entries in /etc/apt/sources.list.d for each abbreviated + sources.list entry in 'srclist'. When rendering template, also + include the values in dictionary searchList + """ + if searchList is None: + searchList = {} + elst = [] + + for ent in srclist: + if 'source' not in ent: + elst.append(["", "missing source"]) + continue + + source = ent['source'] + if source.startswith("ppa:"): + try: + util.subp(["add-apt-repository", source]) + except: + elst.append([source, "add-apt-repository failed"]) + continue + + source = util.render_string(source, searchList) + + if 'filename' not in ent: + ent['filename'] = 'cloud_config_sources.list' + + if not ent['filename'].startswith("/"): + ent['filename'] = "%s/%s" % \ + ("/etc/apt/sources.list.d/", ent['filename']) + + if ('keyid' in ent and 'key' not in ent): + ks = "keyserver.ubuntu.com" + if 'keyserver' in ent: + ks = ent['keyserver'] + try: + ent['key'] = util.getkeybyid(ent['keyid'], ks) + except: + elst.append([source, "failed to get key from %s" % ks]) + continue + + if 'key' in ent: + try: + util.subp(('apt-key', 'add', '-'), ent['key']) + except: + elst.append([source, "failed add key"]) + + try: + util.write_file(ent['filename'], source + "\n", omode="ab") + except: + elst.append([source, "failed write to file %s" % ent['filename']]) + + return(elst) + + +def find_apt_mirror(cloud, cfg): + """ find an apt_mirror given the cloud and cfg provided """ + + # TODO: distro and defaults should be configurable + distro = "ubuntu" + defaults = { + 'ubuntu': "http://archive.ubuntu.com/ubuntu", + 'debian': "http://archive.debian.org/debian", + } + mirror = None + + cfg_mirror = cfg.get("apt_mirror", None) + if cfg_mirror: + mirror = cfg["apt_mirror"] + elif "apt_mirror_search" in cfg: + mirror = util.search_for_mirror(cfg['apt_mirror_search']) + else: + if cloud: + mirror = cloud.get_mirror() + + mydom = "" + + doms = [] + + if not mirror and cloud: + # if we have a fqdn, then search its domain portion first + (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + mydom = ".".join(fqdn.split(".")[1:]) + if mydom: + doms.append(".%s" % mydom) + + if not mirror: + doms.extend((".localdomain", "",)) + + mirror_list = [] + mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) + for post in doms: + mirror_list.append(mirrorfmt % post) + + mirror = util.search_for_mirror(mirror_list) + + if not mirror: + mirror = defaults[distro] + + return mirror diff --git a/cloudinit/handlers/cc_bootcmd.py b/cloudinit/handlers/cc_bootcmd.py new file mode 100644 index 00000000..f584da02 --- /dev/null +++ b/cloudinit/handlers/cc_bootcmd.py @@ -0,0 +1,48 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import cloudinit.util as util +import subprocess +import tempfile +import os +from cloudinit.CloudConfig import per_always +frequency = per_always + + +def handle(_name, cfg, cloud, log, _args): + if "bootcmd" not in cfg: + return + + try: + content = util.shellify(cfg["bootcmd"]) + tmpf = tempfile.TemporaryFile() + tmpf.write(content) + tmpf.seek(0) + except: + log.warn("failed to shellify bootcmd") + raise + + try: + env = os.environ.copy() + env['INSTANCE_ID'] = cloud.get_instance_id() + subprocess.check_call(['/bin/sh'], env=env, stdin=tmpf) + tmpf.close() + except: + log.warn("failed to run commands from bootcmd") + raise diff --git a/cloudinit/handlers/cc_byobu.py b/cloudinit/handlers/cc_byobu.py new file mode 100644 index 00000000..e821b261 --- /dev/null +++ b/cloudinit/handlers/cc_byobu.py @@ -0,0 +1,77 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import traceback + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + value = args[0] + else: + value = util.get_cfg_option_str(cfg, "byobu_by_default", "") + + if not value: + return + + if value == "user" or value == "system": + value = "enable-%s" % value + + valid = ("enable-user", "enable-system", "enable", + "disable-user", "disable-system", "disable") + if not value in valid: + log.warn("Unknown value %s for byobu_by_default" % value) + + mod_user = value.endswith("-user") + mod_sys = value.endswith("-system") + if value.startswith("enable"): + bl_inst = "install" + dc_val = "byobu byobu/launch-by-default boolean true" + mod_sys = True + else: + if value == "disable": + mod_user = True + mod_sys = True + bl_inst = "uninstall" + dc_val = "byobu byobu/launch-by-default boolean false" + + shcmd = "" + if mod_user: + user = util.get_cfg_option_str(cfg, "user", "ubuntu") + shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst) + shcmd += " || X=$(($X+1)); " + if mod_sys: + shcmd += "echo \"%s\" | debconf-set-selections" % dc_val + shcmd += " && dpkg-reconfigure byobu --frontend=noninteractive" + shcmd += " || X=$(($X+1)); " + + cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] + + log.debug("setting byobu to %s" % value) + + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) + except OSError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/handlers/cc_ca_certs.py b/cloudinit/handlers/cc_ca_certs.py new file mode 100644 index 00000000..3af6238a --- /dev/null +++ b/cloudinit/handlers/cc_ca_certs.py @@ -0,0 +1,90 @@ +# vi: ts=4 expandtab +# +# Author: Mike Milner +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import os +from subprocess import check_call +from cloudinit.util import (write_file, get_cfg_option_list_or_str, + delete_dir_contents, subp) + +CA_CERT_PATH = "/usr/share/ca-certificates/" +CA_CERT_FILENAME = "cloud-init-ca-certs.crt" +CA_CERT_CONFIG = "/etc/ca-certificates.conf" +CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" + + +def update_ca_certs(): + """ + Updates the CA certificate cache on the current machine. + """ + check_call(["update-ca-certificates"]) + + +def add_ca_certs(certs): + """ + Adds certificates to the system. To actually apply the new certificates + you must also call L{update_ca_certs}. + + @param certs: A list of certificate strings. + """ + if certs: + cert_file_contents = "\n".join(certs) + cert_file_fullpath = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) + write_file(cert_file_fullpath, cert_file_contents, mode=0644) + # Append cert filename to CA_CERT_CONFIG file. + write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="a") + + +def remove_default_ca_certs(): + """ + Removes all default trusted CA certificates from the system. To actually + apply the change you must also call L{update_ca_certs}. + """ + delete_dir_contents(CA_CERT_PATH) + delete_dir_contents(CA_CERT_SYSTEM_PATH) + write_file(CA_CERT_CONFIG, "", mode=0644) + debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" + subp(('debconf-set-selections', '-'), debconf_sel) + + +def handle(_name, cfg, _cloud, log, _args): + """ + Call to handle ca-cert sections in cloud-config file. + + @param name: The module name "ca-cert" from cloud.cfg + @param cfg: A nested dict containing the entire cloud config contents. + @param cloud: The L{CloudInit} object in use. + @param log: Pre-initialized Python logger object to use for logging. + @param args: Any module arguments from cloud.cfg + """ + # If there isn't a ca-certs section in the configuration don't do anything + if "ca-certs" not in cfg: + return + ca_cert_cfg = cfg['ca-certs'] + + # If there is a remove-defaults option set to true, remove the system + # default trusted CA certs first. + if ca_cert_cfg.get("remove-defaults", False): + log.debug("removing default certificates") + remove_default_ca_certs() + + # If we are given any new trusted CA certs to add, add them. + if "trusted" in ca_cert_cfg: + trusted_certs = get_cfg_option_list_or_str(ca_cert_cfg, "trusted") + if trusted_certs: + log.debug("adding %d certificates" % len(trusted_certs)) + add_ca_certs(trusted_certs) + + # Update the system with the new cert configuration. + update_ca_certs() diff --git a/cloudinit/handlers/cc_chef.py b/cloudinit/handlers/cc_chef.py new file mode 100644 index 00000000..941e04fe --- /dev/null +++ b/cloudinit/handlers/cc_chef.py @@ -0,0 +1,119 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Avishai Ish-Shalom +# Author: Mike Moulton +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import subprocess +import json +import cloudinit.CloudConfig as cc +import cloudinit.util as util + +ruby_version_default = "1.8" + + +def handle(_name, cfg, cloud, log, _args): + # If there isn't a chef key in the configuration don't do anything + if 'chef' not in cfg: + return + chef_cfg = cfg['chef'] + + # ensure the chef directories we use exist + mkdirs(['/etc/chef', '/var/log/chef', '/var/lib/chef', + '/var/cache/chef', '/var/backups/chef', '/var/run/chef']) + + # set the validation key based on the presence of either 'validation_key' + # or 'validation_cert'. In the case where both exist, 'validation_key' + # takes precedence + for key in ('validation_key', 'validation_cert'): + if key in chef_cfg and chef_cfg[key]: + with open('/etc/chef/validation.pem', 'w') as validation_key_fh: + validation_key_fh.write(chef_cfg[key]) + break + + # create the chef config from template + util.render_to_file('chef_client.rb', '/etc/chef/client.rb', + {'server_url': chef_cfg['server_url'], + 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', + cloud.datasource.get_instance_id()), + 'environment': util.get_cfg_option_str(chef_cfg, 'environment', + '_default'), + 'validation_name': chef_cfg['validation_name']}) + + # set the firstboot json + with open('/etc/chef/firstboot.json', 'w') as firstboot_json_fh: + initial_json = {} + if 'run_list' in chef_cfg: + initial_json['run_list'] = chef_cfg['run_list'] + if 'initial_attributes' in chef_cfg: + initial_attributes = chef_cfg['initial_attributes'] + for k in initial_attributes.keys(): + initial_json[k] = initial_attributes[k] + firstboot_json_fh.write(json.dumps(initial_json)) + + # If chef is not installed, we install chef based on 'install_type' + if not os.path.isfile('/usr/bin/chef-client'): + install_type = util.get_cfg_option_str(chef_cfg, 'install_type', + 'packages') + if install_type == "gems": + # this will install and run the chef-client from gems + chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) + ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', + ruby_version_default) + install_chef_from_gems(ruby_version, chef_version) + # and finally, run chef-client + log.debug('running chef-client') + subprocess.check_call(['/usr/bin/chef-client', '-d', '-i', '1800', + '-s', '20']) + else: + # this will install and run the chef-client from packages + cc.install_packages(('chef',)) + + +def get_ruby_packages(version): + # return a list of packages needed to install ruby at version + pkgs = ['ruby%s' % version, 'ruby%s-dev' % version] + if version == "1.8": + pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8')) + return(pkgs) + + +def install_chef_from_gems(ruby_version, chef_version=None): + cc.install_packages(get_ruby_packages(ruby_version)) + if not os.path.exists('/usr/bin/gem'): + os.symlink('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') + if not os.path.exists('/usr/bin/ruby'): + os.symlink('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') + if chef_version: + subprocess.check_call(['/usr/bin/gem', 'install', 'chef', + '-v %s' % chef_version, '--no-ri', + '--no-rdoc', '--bindir', '/usr/bin', '-q']) + else: + subprocess.check_call(['/usr/bin/gem', 'install', 'chef', + '--no-ri', '--no-rdoc', '--bindir', + '/usr/bin', '-q']) + + +def ensure_dir(d): + if not os.path.exists(d): + os.makedirs(d) + + +def mkdirs(dirs): + for d in dirs: + ensure_dir(d) diff --git a/cloudinit/handlers/cc_disable_ec2_metadata.py b/cloudinit/handlers/cc_disable_ec2_metadata.py new file mode 100644 index 00000000..6b31ea8e --- /dev/null +++ b/cloudinit/handlers/cc_disable_ec2_metadata.py @@ -0,0 +1,30 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import cloudinit.util as util +import subprocess +from cloudinit.CloudConfig import per_always + +frequency = per_always + + +def handle(_name, cfg, _cloud, _log, _args): + if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False): + fwall = "route add -host 169.254.169.254 reject" + subprocess.call(fwall.split(' ')) diff --git a/cloudinit/handlers/cc_final_message.py b/cloudinit/handlers/cc_final_message.py new file mode 100644 index 00000000..abb4ca32 --- /dev/null +++ b/cloudinit/handlers/cc_final_message.py @@ -0,0 +1,58 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.CloudConfig import per_always +import sys +from cloudinit import util, boot_finished +import time + +frequency = per_always + +final_message = "cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds" + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + msg_in = args[0] + else: + msg_in = util.get_cfg_option_str(cfg, "final_message", final_message) + + try: + uptimef = open("/proc/uptime") + uptime = uptimef.read().split(" ")[0] + uptimef.close() + except IOError as e: + log.warn("unable to open /proc/uptime\n") + uptime = "na" + + try: + ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) + except: + ts = "na" + + try: + subs = {'UPTIME': uptime, 'TIMESTAMP': ts} + sys.stdout.write("%s\n" % util.render_string(msg_in, subs)) + except Exception as e: + log.warn("failed to render string to stdout: %s" % e) + + fp = open(boot_finished, "wb") + fp.write(uptime + "\n") + fp.close() diff --git a/cloudinit/handlers/cc_foo.py b/cloudinit/handlers/cc_foo.py new file mode 100644 index 00000000..35ec3fa7 --- /dev/null +++ b/cloudinit/handlers/cc_foo.py @@ -0,0 +1,29 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#import cloudinit +#import cloudinit.util as util +from cloudinit.CloudConfig import per_instance + +frequency = per_instance + + +def handle(_name, _cfg, _cloud, _log, _args): + print "hi" diff --git a/cloudinit/handlers/cc_grub_dpkg.py b/cloudinit/handlers/cc_grub_dpkg.py new file mode 100644 index 00000000..9f3a7eaf --- /dev/null +++ b/cloudinit/handlers/cc_grub_dpkg.py @@ -0,0 +1,64 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import traceback +import os + + +def handle(_name, cfg, _cloud, log, _args): + idevs = None + idevs_empty = None + + if "grub-dpkg" in cfg: + idevs = util.get_cfg_option_str(cfg["grub-dpkg"], + "grub-pc/install_devices", None) + idevs_empty = util.get_cfg_option_str(cfg["grub-dpkg"], + "grub-pc/install_devices_empty", None) + + if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or + (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): + if idevs == None: + idevs = "" + if idevs_empty == None: + idevs_empty = "true" + else: + if idevs_empty == None: + idevs_empty = "false" + if idevs == None: + idevs = "/dev/sda" + for dev in ("/dev/sda", "/dev/vda", "/dev/sda1", "/dev/vda1"): + if os.path.exists(dev): + idevs = dev + break + + # now idevs and idevs_empty are set to determined values + # or, those set by user + + dconf_sel = "grub-pc grub-pc/install_devices string %s\n" % idevs + \ + "grub-pc grub-pc/install_devices_empty boolean %s\n" % idevs_empty + log.debug("setting grub debconf-set-selections with '%s','%s'" % + (idevs, idevs_empty)) + + try: + util.subp(('debconf-set-selections'), dconf_sel) + except: + log.error("Failed to run debconf-set-selections for grub-dpkg") + log.debug(traceback.format_exc()) diff --git a/cloudinit/handlers/cc_keys_to_console.py b/cloudinit/handlers/cc_keys_to_console.py new file mode 100644 index 00000000..73a477c0 --- /dev/null +++ b/cloudinit/handlers/cc_keys_to_console.py @@ -0,0 +1,42 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.CloudConfig import per_instance +import cloudinit.util as util +import subprocess + +frequency = per_instance + + +def handle(_name, cfg, _cloud, log, _args): + cmd = ['/usr/lib/cloud-init/write-ssh-key-fingerprints'] + fp_blacklist = util.get_cfg_option_list_or_str(cfg, + "ssh_fp_console_blacklist", []) + key_blacklist = util.get_cfg_option_list_or_str(cfg, + "ssh_key_console_blacklist", ["ssh-dss"]) + try: + confp = open('/dev/console', "wb") + cmd.append(','.join(fp_blacklist)) + cmd.append(','.join(key_blacklist)) + subprocess.call(cmd, stdout=confp) + confp.close() + except: + log.warn("writing keys to console value") + raise diff --git a/cloudinit/handlers/cc_landscape.py b/cloudinit/handlers/cc_landscape.py new file mode 100644 index 00000000..a4113cbe --- /dev/null +++ b/cloudinit/handlers/cc_landscape.py @@ -0,0 +1,75 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +from cloudinit.CloudConfig import per_instance +from configobj import ConfigObj + +frequency = per_instance + +lsc_client_cfg_file = "/etc/landscape/client.conf" + +# defaults taken from stock client.conf in landscape-client 11.07.1.1-0ubuntu2 +lsc_builtincfg = { + 'client': { + 'log_level': "info", + 'url': "https://landscape.canonical.com/message-system", + 'ping_url': "http://landscape.canonical.com/ping", + 'data_path': "/var/lib/landscape/client", + } +} + + +def handle(_name, cfg, _cloud, log, _args): + """ + Basically turn a top level 'landscape' entry with a 'client' dict + and render it to ConfigObj format under '[client]' section in + /etc/landscape/client.conf + """ + + ls_cloudcfg = cfg.get("landscape", {}) + + if not isinstance(ls_cloudcfg, dict): + raise(Exception("'landscape' existed in config, but not a dict")) + + merged = mergeTogether([lsc_builtincfg, lsc_client_cfg_file, ls_cloudcfg]) + + if not os.path.isdir(os.path.dirname(lsc_client_cfg_file)): + os.makedirs(os.path.dirname(lsc_client_cfg_file)) + + with open(lsc_client_cfg_file, "w") as fp: + merged.write(fp) + + log.debug("updated %s" % lsc_client_cfg_file) + + +def mergeTogether(objs): + """ + merge together ConfigObj objects or things that ConfigObj() will take in + later entries override earlier + """ + cfg = ConfigObj({}) + for obj in objs: + if isinstance(obj, ConfigObj): + cfg.merge(obj) + else: + cfg.merge(ConfigObj(obj)) + return cfg diff --git a/cloudinit/handlers/cc_locale.py b/cloudinit/handlers/cc_locale.py new file mode 100644 index 00000000..2bb22fdb --- /dev/null +++ b/cloudinit/handlers/cc_locale.py @@ -0,0 +1,54 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import os.path +import subprocess +import traceback + + +def apply_locale(locale, cfgfile): + if os.path.exists('/usr/sbin/locale-gen'): + subprocess.Popen(['locale-gen', locale]).communicate() + if os.path.exists('/usr/sbin/update-locale'): + subprocess.Popen(['update-locale', locale]).communicate() + + util.render_to_file('default-locale', cfgfile, {'locale': locale}) + + +def handle(_name, cfg, cloud, log, args): + if len(args) != 0: + locale = args[0] + else: + locale = util.get_cfg_option_str(cfg, "locale", cloud.get_locale()) + + locale_cfgfile = util.get_cfg_option_str(cfg, "locale_configfile", + "/etc/default/locale") + + if not locale: + return + + log.debug("setting locale to %s" % locale) + + try: + apply_locale(locale, locale_cfgfile) + except Exception as e: + log.debug(traceback.format_exc(e)) + raise Exception("failed to apply locale %s" % locale) diff --git a/cloudinit/handlers/cc_mcollective.py b/cloudinit/handlers/cc_mcollective.py new file mode 100644 index 00000000..a2a6230c --- /dev/null +++ b/cloudinit/handlers/cc_mcollective.py @@ -0,0 +1,99 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Marc Cluet +# Based on code by Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import subprocess +import StringIO +import ConfigParser +import cloudinit.CloudConfig as cc +import cloudinit.util as util + +pubcert_file = "/etc/mcollective/ssl/server-public.pem" +pricert_file = "/etc/mcollective/ssl/server-private.pem" + + +# Our fake header section +class FakeSecHead(object): + def __init__(self, fp): + self.fp = fp + self.sechead = '[nullsection]\n' + + def readline(self): + if self.sechead: + try: + return self.sechead + finally: + self.sechead = None + else: + return self.fp.readline() + + +def handle(_name, cfg, _cloud, _log, _args): + # If there isn't a mcollective key in the configuration don't do anything + if 'mcollective' not in cfg: + return + mcollective_cfg = cfg['mcollective'] + # Start by installing the mcollective package ... + cc.install_packages(("mcollective",)) + + # ... and then update the mcollective configuration + if 'conf' in mcollective_cfg: + # Create object for reading server.cfg values + mcollective_config = ConfigParser.ConfigParser() + # Read server.cfg values from original file in order to be able to mix + # the rest up + mcollective_config.readfp(FakeSecHead(open('/etc/mcollective/' + 'server.cfg'))) + for cfg_name, cfg in mcollective_cfg['conf'].iteritems(): + if cfg_name == 'public-cert': + util.write_file(pubcert_file, cfg, mode=0644) + mcollective_config.set(cfg_name, + 'plugin.ssl_server_public', pubcert_file) + mcollective_config.set(cfg_name, 'securityprovider', 'ssl') + elif cfg_name == 'private-cert': + util.write_file(pricert_file, cfg, mode=0600) + mcollective_config.set(cfg_name, + 'plugin.ssl_server_private', pricert_file) + mcollective_config.set(cfg_name, 'securityprovider', 'ssl') + else: + # Iterate throug the config items, we'll use ConfigParser.set + # to overwrite or create new items as needed + for o, v in cfg.iteritems(): + mcollective_config.set(cfg_name, o, v) + # We got all our config as wanted we'll rename + # the previous server.cfg and create our new one + os.rename('/etc/mcollective/server.cfg', + '/etc/mcollective/server.cfg.old') + outputfile = StringIO.StringIO() + mcollective_config.write(outputfile) + # Now we got the whole file, write to disk except first line + # Note below, that we've just used ConfigParser because it generally + # works. Below, we remove the initial 'nullsection' header + # and then change 'key = value' to 'key: value'. The global + # search and replace of '=' with ':' could be problematic though. + # this most likely needs fixing. + util.write_file('/etc/mcollective/server.cfg', + outputfile.getvalue().replace('[nullsection]\n', '').replace(' =', + ':'), + mode=0644) + + # Start mcollective + subprocess.check_call(['service', 'mcollective', 'start']) diff --git a/cloudinit/handlers/cc_mounts.py b/cloudinit/handlers/cc_mounts.py new file mode 100644 index 00000000..6cdd74e8 --- /dev/null +++ b/cloudinit/handlers/cc_mounts.py @@ -0,0 +1,179 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import os +import re +from string import whitespace # pylint: disable=W0402 + + +def is_mdname(name): + # return true if this is a metadata service name + if name in ["ami", "root", "swap"]: + return True + # names 'ephemeral0' or 'ephemeral1' + # 'ebs[0-9]' appears when '--block-device-mapping sdf=snap-d4d90bbc' + for enumname in ("ephemeral", "ebs"): + if name.startswith(enumname) and name.find(":") == -1: + return True + return False + + +def handle(_name, cfg, cloud, log, _args): + # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno + defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] + defvals = cfg.get("mount_default_fields", defvals) + + # these are our default set of mounts + defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"], + ["swap", "none", "swap", "sw", "0", "0"]] + + cfgmnt = [] + if "mounts" in cfg: + cfgmnt = cfg["mounts"] + + # shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 + shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" + shortname = re.compile(shortname_filter) + + for i in range(len(cfgmnt)): + # skip something that wasn't a list + if not isinstance(cfgmnt[i], list): + continue + + # workaround, allow user to specify 'ephemeral' + # rather than more ec2 correct 'ephemeral0' + if cfgmnt[i][0] == "ephemeral": + cfgmnt[i][0] = "ephemeral0" + + if is_mdname(cfgmnt[i][0]): + newname = cloud.device_name_to_device(cfgmnt[i][0]) + if not newname: + log.debug("ignoring nonexistant named mount %s" % cfgmnt[i][0]) + cfgmnt[i][1] = None + else: + if newname.startswith("/"): + cfgmnt[i][0] = newname + else: + cfgmnt[i][0] = "/dev/%s" % newname + else: + if shortname.match(cfgmnt[i][0]): + cfgmnt[i][0] = "/dev/%s" % cfgmnt[i][0] + + # in case the user did not quote a field (likely fs-freq, fs_passno) + # but do not convert None to 'None' (LP: #898365) + for j in range(len(cfgmnt[i])): + if isinstance(cfgmnt[i][j], int): + cfgmnt[i][j] = str(cfgmnt[i][j]) + + for i in range(len(cfgmnt)): + # fill in values with defaults from defvals above + for j in range(len(defvals)): + if len(cfgmnt[i]) <= j: + cfgmnt[i].append(defvals[j]) + elif cfgmnt[i][j] is None: + cfgmnt[i][j] = defvals[j] + + # if the second entry in the list is 'None' this + # clears all previous entries of that same 'fs_spec' + # (fs_spec is the first field in /etc/fstab, ie, that device) + if cfgmnt[i][1] is None: + for j in range(i): + if cfgmnt[j][0] == cfgmnt[i][0]: + cfgmnt[j][1] = None + + # for each of the "default" mounts, add them only if no other + # entry has the same device name + for defmnt in defmnts: + devname = cloud.device_name_to_device(defmnt[0]) + if devname is None: + continue + if devname.startswith("/"): + defmnt[0] = devname + else: + defmnt[0] = "/dev/%s" % devname + + cfgmnt_has = False + for cfgm in cfgmnt: + if cfgm[0] == defmnt[0]: + cfgmnt_has = True + break + + if cfgmnt_has: + continue + cfgmnt.append(defmnt) + + # now, each entry in the cfgmnt list has all fstab values + # if the second field is None (not the string, the value) we skip it + actlist = [x for x in cfgmnt if x[1] is not None] + + if len(actlist) == 0: + return + + comment = "comment=cloudconfig" + cc_lines = [] + needswap = False + dirs = [] + for line in actlist: + # write 'comment' in the fs_mntops, entry, claiming this + line[3] = "%s,comment=cloudconfig" % line[3] + if line[2] == "swap": + needswap = True + if line[1].startswith("/"): + dirs.append(line[1]) + cc_lines.append('\t'.join(line)) + + fstab_lines = [] + fstab = open("/etc/fstab", "r+") + ws = re.compile("[%s]+" % whitespace) + for line in fstab.read().splitlines(): + try: + toks = ws.split(line) + if toks[3].find(comment) != -1: + continue + except: + pass + fstab_lines.append(line) + + fstab_lines.extend(cc_lines) + + fstab.seek(0) + fstab.write("%s\n" % '\n'.join(fstab_lines)) + fstab.truncate() + fstab.close() + + if needswap: + try: + util.subp(("swapon", "-a")) + except: + log.warn("Failed to enable swap") + + for d in dirs: + if os.path.exists(d): + continue + try: + os.makedirs(d) + except: + log.warn("Failed to make '%s' config-mount\n", d) + + try: + util.subp(("mount", "-a")) + except: + log.warn("'mount -a' failed") diff --git a/cloudinit/handlers/cc_phone_home.py b/cloudinit/handlers/cc_phone_home.py new file mode 100644 index 00000000..a7ff74e1 --- /dev/null +++ b/cloudinit/handlers/cc_phone_home.py @@ -0,0 +1,106 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +from cloudinit.CloudConfig import per_instance +import cloudinit.util as util +from time import sleep + +frequency = per_instance +post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', + 'hostname'] + + +# phone_home: +# url: http://my.foo.bar/$INSTANCE/ +# post: all +# tries: 10 +# +# phone_home: +# url: http://my.foo.bar/$INSTANCE_ID/ +# post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id +# +def handle(_name, cfg, cloud, log, args): + if len(args) != 0: + ph_cfg = util.read_conf(args[0]) + else: + if not 'phone_home' in cfg: + return + ph_cfg = cfg['phone_home'] + + if 'url' not in ph_cfg: + log.warn("no 'url' token in phone_home") + return + + url = ph_cfg['url'] + post_list = ph_cfg.get('post', 'all') + tries = ph_cfg.get('tries', 10) + try: + tries = int(tries) + except: + log.warn("tries is not an integer. using 10") + tries = 10 + + if post_list == "all": + post_list = post_list_all + + all_keys = {} + all_keys['instance_id'] = cloud.get_instance_id() + all_keys['hostname'] = cloud.get_hostname() + + pubkeys = { + 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', + 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', + 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', + } + + for n, path in pubkeys.iteritems(): + try: + fp = open(path, "rb") + all_keys[n] = fp.read() + fp.close() + except: + log.warn("%s: failed to open in phone_home" % path) + + submit_keys = {} + for k in post_list: + if k in all_keys: + submit_keys[k] = all_keys[k] + else: + submit_keys[k] = "N/A" + log.warn("requested key %s from 'post' list not available") + + url = util.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) + + null_exc = object() + last_e = null_exc + for i in range(0, tries): + try: + util.readurl(url, submit_keys) + log.debug("succeeded submit to %s on try %i" % (url, i + 1)) + return + except Exception as e: + log.debug("failed to post to %s on try %i" % (url, i + 1)) + last_e = e + sleep(3) + + log.warn("failed to post to %s in %i tries" % (url, tries)) + if last_e is not null_exc: + raise(last_e) + + return diff --git a/cloudinit/handlers/cc_puppet.py b/cloudinit/handlers/cc_puppet.py new file mode 100644 index 00000000..6fc475f6 --- /dev/null +++ b/cloudinit/handlers/cc_puppet.py @@ -0,0 +1,108 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +import pwd +import socket +import subprocess +import StringIO +import ConfigParser +import cloudinit.CloudConfig as cc +import cloudinit.util as util + + +def handle(_name, cfg, cloud, log, _args): + # If there isn't a puppet key in the configuration don't do anything + if 'puppet' not in cfg: + return + puppet_cfg = cfg['puppet'] + # Start by installing the puppet package ... + cc.install_packages(("puppet",)) + + # ... and then update the puppet configuration + if 'conf' in puppet_cfg: + # Add all sections from the conf object to puppet.conf + puppet_conf_fh = open('/etc/puppet/puppet.conf', 'r') + # Create object for reading puppet.conf values + puppet_config = ConfigParser.ConfigParser() + # Read puppet.conf values from original file in order to be able to + # mix the rest up + puppet_config.readfp(StringIO.StringIO(''.join(i.lstrip() for i in + puppet_conf_fh.readlines()))) + # Close original file, no longer needed + puppet_conf_fh.close() + for cfg_name, cfg in puppet_cfg['conf'].iteritems(): + # ca_cert configuration is a special case + # Dump the puppetmaster ca certificate in the correct place + if cfg_name == 'ca_cert': + # Puppet ssl sub-directory isn't created yet + # Create it with the proper permissions and ownership + os.makedirs('/var/lib/puppet/ssl') + os.chmod('/var/lib/puppet/ssl', 0771) + os.chown('/var/lib/puppet/ssl', + pwd.getpwnam('puppet').pw_uid, 0) + os.makedirs('/var/lib/puppet/ssl/certs/') + os.chown('/var/lib/puppet/ssl/certs/', + pwd.getpwnam('puppet').pw_uid, 0) + ca_fh = open('/var/lib/puppet/ssl/certs/ca.pem', 'w') + ca_fh.write(cfg) + ca_fh.close() + os.chown('/var/lib/puppet/ssl/certs/ca.pem', + pwd.getpwnam('puppet').pw_uid, 0) + util.restorecon_if_possible('/var/lib/puppet', recursive=True) + else: + #puppet_conf_fh.write("\n[%s]\n" % (cfg_name)) + # If puppet.conf already has this section we don't want to + # write it again + if puppet_config.has_section(cfg_name) == False: + puppet_config.add_section(cfg_name) + # Iterate throug the config items, we'll use ConfigParser.set + # to overwrite or create new items as needed + for o, v in cfg.iteritems(): + if o == 'certname': + # Expand %f as the fqdn + v = v.replace("%f", socket.getfqdn()) + # Expand %i as the instance id + v = v.replace("%i", + cloud.datasource.get_instance_id()) + # certname needs to be downcase + v = v.lower() + puppet_config.set(cfg_name, o, v) + #puppet_conf_fh.write("%s=%s\n" % (o, v)) + # We got all our config as wanted we'll rename + # the previous puppet.conf and create our new one + os.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') + with open('/etc/puppet/puppet.conf', 'wb') as configfile: + puppet_config.write(configfile) + util.restorecon_if_possible('/etc/puppet/puppet.conf') + # Set puppet to automatically start + if os.path.exists('/etc/default/puppet'): + subprocess.check_call(['sed', '-i', + '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet']) + elif os.path.exists('/bin/systemctl'): + subprocess.check_call(['/bin/systemctl', 'enable', 'puppet.service']) + elif os.path.exists('/sbin/chkconfig'): + subprocess.check_call(['/sbin/chkconfig', 'puppet', 'on']) + else: + log.warn("Do not know how to enable puppet service on this system") + # Start puppetd + subprocess.check_call(['service', 'puppet', 'start']) diff --git a/cloudinit/handlers/cc_resizefs.py b/cloudinit/handlers/cc_resizefs.py new file mode 100644 index 00000000..2dc66def --- /dev/null +++ b/cloudinit/handlers/cc_resizefs.py @@ -0,0 +1,108 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import os +import stat +import sys +import time +import tempfile +from cloudinit.CloudConfig import per_always + +frequency = per_always + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + resize_root = False + if str(args[0]).lower() in ['true', '1', 'on', 'yes']: + resize_root = True + else: + resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) + + if str(resize_root).lower() in ['false', '0']: + return + + # we use mktemp rather than mkstemp because early in boot nothing + # else should be able to race us for this, and we need to mknod. + devpth = tempfile.mktemp(prefix="cloudinit.resizefs.", dir="/run") + + try: + st_dev = os.stat("/").st_dev + dev = os.makedev(os.major(st_dev), os.minor(st_dev)) + os.mknod(devpth, 0400 | stat.S_IFBLK, dev) + except: + if util.is_container(): + log.debug("inside container, ignoring mknod failure in resizefs") + return + log.warn("Failed to make device node to resize /") + raise + + cmd = ['blkid', '-c', '/dev/null', '-sTYPE', '-ovalue', devpth] + try: + (fstype, _err) = util.subp(cmd) + except subprocess.CalledProcessError as e: + log.warn("Failed to get filesystem type of maj=%s, min=%s via: %s" % + (os.major(st_dev), os.minor(st_dev), cmd)) + log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) + os.unlink(devpth) + raise + + if str(fstype).startswith("ext"): + resize_cmd = ['resize2fs', devpth] + elif fstype == "xfs": + resize_cmd = ['xfs_growfs', devpth] + else: + os.unlink(devpth) + log.debug("not resizing unknown filesystem %s" % fstype) + return + + if resize_root == "noblock": + fid = os.fork() + if fid == 0: + try: + do_resize(resize_cmd, devpth, log) + os._exit(0) # pylint: disable=W0212 + except Exception as exc: + sys.stderr.write("Failed: %s" % exc) + os._exit(1) # pylint: disable=W0212 + else: + do_resize(resize_cmd, devpth, log) + + log.debug("resizing root filesystem (type=%s, maj=%i, min=%i, val=%s)" % + (str(fstype).rstrip("\n"), os.major(st_dev), os.minor(st_dev), + resize_root)) + + return + + +def do_resize(resize_cmd, devpth, log): + try: + start = time.time() + util.subp(resize_cmd) + except subprocess.CalledProcessError as e: + log.warn("Failed to resize filesystem (%s)" % resize_cmd) + log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) + os.unlink(devpth) + raise + + os.unlink(devpth) + log.debug("resize took %s seconds" % (time.time() - start)) diff --git a/cloudinit/handlers/cc_rightscale_userdata.py b/cloudinit/handlers/cc_rightscale_userdata.py new file mode 100644 index 00000000..5ed0848f --- /dev/null +++ b/cloudinit/handlers/cc_rightscale_userdata.py @@ -0,0 +1,78 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +## +## The purpose of this script is to allow cloud-init to consume +## rightscale style userdata. rightscale user data is key-value pairs +## in a url-query-string like format. +## +## for cloud-init support, there will be a key named +## 'CLOUD_INIT_REMOTE_HOOK'. +## +## This cloud-config module will +## - read the blob of data from raw user data, and parse it as key/value +## - for each key that is found, download the content to +## the local instance/scripts directory and set them executable. +## - the files in that directory will be run by the user-scripts module +## Therefore, this must run before that. +## +## + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance +from cloudinit import get_ipath_cur +from urlparse import parse_qs + +frequency = per_instance +my_name = "cc_rightscale_userdata" +my_hookname = 'CLOUD_INIT_REMOTE_HOOK' + + +def handle(_name, _cfg, cloud, log, _args): + try: + ud = cloud.get_userdata_raw() + except: + log.warn("failed to get raw userdata in %s" % my_name) + return + + try: + mdict = parse_qs(ud) + if not my_hookname in mdict: + return + except: + log.warn("failed to urlparse.parse_qa(userdata_raw())") + raise + + scripts_d = get_ipath_cur('scripts') + i = 0 + first_e = None + for url in mdict[my_hookname]: + fname = "%s/rightscale-%02i" % (scripts_d, i) + i = i + 1 + try: + content = util.readurl(url) + util.write_file(fname, content, mode=0700) + except Exception as e: + if not first_e: + first_e = None + log.warn("%s failed to read %s: %s" % (my_name, url, e)) + + if first_e: + raise(e) diff --git a/cloudinit/handlers/cc_rsyslog.py b/cloudinit/handlers/cc_rsyslog.py new file mode 100644 index 00000000..ac7f2c74 --- /dev/null +++ b/cloudinit/handlers/cc_rsyslog.py @@ -0,0 +1,101 @@ +# vi: ts=4 expandtab syntax=python +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit +import logging +import cloudinit.util as util +import traceback + +DEF_FILENAME = "20-cloud-config.conf" +DEF_DIR = "/etc/rsyslog.d" + + +def handle(_name, cfg, _cloud, log, _args): + # rsyslog: + # - "*.* @@192.158.1.1" + # - content: "*.* @@192.0.2.1:10514" + # - filename: 01-examplecom.conf + # content: | + # *.* @@syslogd.example.com + + # process 'rsyslog' + if not 'rsyslog' in cfg: + return + + def_dir = cfg.get('rsyslog_dir', DEF_DIR) + def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) + + files = [] + elst = [] + for ent in cfg['rsyslog']: + if isinstance(ent, dict): + if not "content" in ent: + elst.append((ent, "no 'content' entry")) + continue + content = ent['content'] + filename = ent.get("filename", def_fname) + else: + content = ent + filename = def_fname + + if not filename.startswith("/"): + filename = "%s/%s" % (def_dir, filename) + + omode = "ab" + # truncate filename first time you see it + if filename not in files: + omode = "wb" + files.append(filename) + + try: + util.write_file(filename, content + "\n", omode=omode) + except Exception as e: + log.debug(traceback.format_exc(e)) + elst.append((content, "failed to write to %s" % filename)) + + # need to restart syslogd + restarted = False + try: + # if this config module is running at cloud-init time + # (before rsyslog is running) we don't actually have to + # restart syslog. + # + # upstart actually does what we want here, in that it doesn't + # start a service that wasn't running already on 'restart' + # it will also return failure on the attempt, so 'restarted' + # won't get set + log.debug("restarting rsyslog") + util.subp(['service', 'rsyslog', 'restart']) + restarted = True + + except Exception as e: + elst.append(("restart", str(e))) + + if restarted: + # this only needs to run if we *actually* restarted + # syslog above. + cloudinit.logging_set_from_cfg_file() + log = logging.getLogger() + log.debug("rsyslog configured %s" % files) + + for e in elst: + log.warn("rsyslog error: %s\n" % ':'.join(e)) + + return diff --git a/cloudinit/handlers/cc_runcmd.py b/cloudinit/handlers/cc_runcmd.py new file mode 100644 index 00000000..f7e8c671 --- /dev/null +++ b/cloudinit/handlers/cc_runcmd.py @@ -0,0 +1,32 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util + + +def handle(_name, cfg, cloud, log, _args): + if "runcmd" not in cfg: + return + outfile = "%s/runcmd" % cloud.get_ipath('scripts') + try: + content = util.shellify(cfg["runcmd"]) + util.write_file(outfile, content, 0700) + except: + log.warn("failed to open %s for runcmd" % outfile) diff --git a/cloudinit/handlers/cc_salt_minion.py b/cloudinit/handlers/cc_salt_minion.py new file mode 100644 index 00000000..1a3b5039 --- /dev/null +++ b/cloudinit/handlers/cc_salt_minion.py @@ -0,0 +1,56 @@ +# vi: ts=4 expandtab +# +# Author: Jeff Bauer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +import subprocess +import cloudinit.CloudConfig as cc +import yaml + + +def handle(_name, cfg, _cloud, _log, _args): + # If there isn't a salt key in the configuration don't do anything + if 'salt_minion' not in cfg: + return + salt_cfg = cfg['salt_minion'] + # Start by installing the salt package ... + cc.install_packages(("salt",)) + config_dir = '/etc/salt' + if not os.path.isdir(config_dir): + os.makedirs(config_dir) + # ... and then update the salt configuration + if 'conf' in salt_cfg: + # Add all sections from the conf object to /etc/salt/minion + minion_config = os.path.join(config_dir, 'minion') + yaml.dump(salt_cfg['conf'], + file(minion_config, 'w'), + default_flow_style=False) + # ... copy the key pair if specified + if 'public_key' in salt_cfg and 'private_key' in salt_cfg: + pki_dir = '/etc/salt/pki' + cumask = os.umask(077) + if not os.path.isdir(pki_dir): + os.makedirs(pki_dir) + pub_name = os.path.join(pki_dir, 'minion.pub') + pem_name = os.path.join(pki_dir, 'minion.pem') + with open(pub_name, 'w') as f: + f.write(salt_cfg['public_key']) + with open(pem_name, 'w') as f: + f.write(salt_cfg['private_key']) + os.umask(cumask) + + # Start salt-minion + subprocess.check_call(['service', 'salt-minion', 'start']) diff --git a/cloudinit/handlers/cc_scripts_per_boot.py b/cloudinit/handlers/cc_scripts_per_boot.py new file mode 100644 index 00000000..41a74754 --- /dev/null +++ b/cloudinit/handlers/cc_scripts_per_boot.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_always +from cloudinit import get_cpath + +frequency = per_always +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-boot") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/handlers/cc_scripts_per_instance.py b/cloudinit/handlers/cc_scripts_per_instance.py new file mode 100644 index 00000000..a2981eab --- /dev/null +++ b/cloudinit/handlers/cc_scripts_per_instance.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance +from cloudinit import get_cpath + +frequency = per_instance +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-instance") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/handlers/cc_scripts_per_once.py b/cloudinit/handlers/cc_scripts_per_once.py new file mode 100644 index 00000000..a69151da --- /dev/null +++ b/cloudinit/handlers/cc_scripts_per_once.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_once +from cloudinit import get_cpath + +frequency = per_once +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-once") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/handlers/cc_scripts_user.py b/cloudinit/handlers/cc_scripts_user.py new file mode 100644 index 00000000..933aa4e0 --- /dev/null +++ b/cloudinit/handlers/cc_scripts_user.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance +from cloudinit import get_ipath_cur + +frequency = per_instance +runparts_path = "%s/%s" % (get_ipath_cur(), "scripts") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/handlers/cc_set_hostname.py b/cloudinit/handlers/cc_set_hostname.py new file mode 100644 index 00000000..acea74d9 --- /dev/null +++ b/cloudinit/handlers/cc_set_hostname.py @@ -0,0 +1,42 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util + + +def handle(_name, cfg, cloud, log, _args): + if util.get_cfg_option_bool(cfg, "preserve_hostname", False): + log.debug("preserve_hostname is set. not setting hostname") + return(True) + + (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) + try: + set_hostname(hostname, log) + except Exception: + util.logexc(log) + log.warn("failed to set hostname to %s\n", hostname) + + return(True) + + +def set_hostname(hostname, log): + util.subp(['hostname', hostname]) + util.write_file("/etc/hostname", "%s\n" % hostname, 0644) + log.debug("populated /etc/hostname with %s on first boot", hostname) diff --git a/cloudinit/handlers/cc_set_passwords.py b/cloudinit/handlers/cc_set_passwords.py new file mode 100644 index 00000000..9d0bbdb8 --- /dev/null +++ b/cloudinit/handlers/cc_set_passwords.py @@ -0,0 +1,129 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import sys +import random +from string import letters, digits # pylint: disable=W0402 + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + # if run from command line, and give args, wipe the chpasswd['list'] + password = args[0] + if 'chpasswd' in cfg and 'list' in cfg['chpasswd']: + del cfg['chpasswd']['list'] + else: + password = util.get_cfg_option_str(cfg, "password", None) + + expire = True + pw_auth = "no" + change_pwauth = False + plist = None + + if 'chpasswd' in cfg: + chfg = cfg['chpasswd'] + plist = util.get_cfg_option_str(chfg, 'list', plist) + expire = util.get_cfg_option_bool(chfg, 'expire', expire) + + if not plist and password: + user = util.get_cfg_option_str(cfg, "user", "ubuntu") + plist = "%s:%s" % (user, password) + + errors = [] + if plist: + plist_in = [] + randlist = [] + users = [] + for line in plist.splitlines(): + u, p = line.split(':', 1) + if p == "R" or p == "RANDOM": + p = rand_user_password() + randlist.append("%s:%s" % (u, p)) + plist_in.append("%s:%s" % (u, p)) + users.append(u) + + ch_in = '\n'.join(plist_in) + try: + util.subp(['chpasswd'], ch_in) + log.debug("changed password for %s:" % users) + except Exception as e: + errors.append(e) + log.warn("failed to set passwords with chpasswd: %s" % e) + + if len(randlist): + sys.stdout.write("%s\n%s\n" % ("Set the following passwords\n", + '\n'.join(randlist))) + + if expire: + enum = len(errors) + for u in users: + try: + util.subp(['passwd', '--expire', u]) + except Exception as e: + errors.append(e) + log.warn("failed to expire account for %s" % u) + if enum == len(errors): + log.debug("expired passwords for: %s" % u) + + if 'ssh_pwauth' in cfg: + val = str(cfg['ssh_pwauth']).lower() + if val in ("true", "1", "yes"): + pw_auth = "yes" + change_pwauth = True + elif val in ("false", "0", "no"): + pw_auth = "no" + change_pwauth = True + else: + change_pwauth = False + + if change_pwauth: + pa_s = "\(#*\)\(PasswordAuthentication[[:space:]]\+\)\(yes\|no\)" + msg = "set PasswordAuthentication to '%s'" % pw_auth + try: + cmd = ['sed', '-i', 's,%s,\\2%s,' % (pa_s, pw_auth), + '/etc/ssh/sshd_config'] + util.subp(cmd) + log.debug(msg) + except Exception as e: + log.warn("failed %s" % msg) + errors.append(e) + + try: + p = util.subp(['service', cfg.get('ssh_svcname', 'ssh'), + 'restart']) + log.debug("restarted sshd") + except: + log.warn("restart of ssh failed") + + if len(errors): + raise(errors[0]) + + return + + +def rand_str(strlen=32, select_from=letters + digits): + return("".join([random.choice(select_from) for _x in range(0, strlen)])) + + +def rand_user_password(pwlen=9): + selfrom = (letters.translate(None, 'loLOI') + + digits.translate(None, '01')) + return(rand_str(pwlen, select_from=selfrom)) diff --git a/cloudinit/handlers/cc_ssh.py b/cloudinit/handlers/cc_ssh.py new file mode 100644 index 00000000..48eb58bc --- /dev/null +++ b/cloudinit/handlers/cc_ssh.py @@ -0,0 +1,106 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import cloudinit.SshUtil as sshutil +import os +import glob +import subprocess + +DISABLE_ROOT_OPTS = "no-port-forwarding,no-agent-forwarding," \ +"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " \ +"rather than the user \\\"root\\\".\';echo;sleep 10\"" + + +def handle(_name, cfg, cloud, log, _args): + + # remove the static keys from the pristine image + if cfg.get("ssh_deletekeys", True): + for f in glob.glob("/etc/ssh/ssh_host_*key*"): + try: + os.unlink(f) + except: + pass + + if "ssh_keys" in cfg: + # if there are keys in cloud-config, use them + key2file = { + "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), + "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), + "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), + "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), + "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), + "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), + } + + for key, val in cfg["ssh_keys"].items(): + if key in key2file: + util.write_file(key2file[key][0], val, key2file[key][1]) + + priv2pub = {'rsa_private': 'rsa_public', 'dsa_private': 'dsa_public', + 'ecdsa_private': 'ecdsa_public', } + + cmd = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' + for priv, pub in priv2pub.iteritems(): + if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: + continue + pair = (key2file[priv][0], key2file[pub][0]) + subprocess.call(('sh', '-xc', cmd % pair)) + log.debug("generated %s from %s" % pair) + else: + # if not, generate them + for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', + ['rsa', 'dsa', 'ecdsa']): + keyfile = '/etc/ssh/ssh_host_%s_key' % keytype + if not os.path.exists(keyfile): + subprocess.call(['ssh-keygen', '-t', keytype, '-N', '', + '-f', keyfile]) + + util.restorecon_if_possible('/etc/ssh', recursive=True) + + try: + user = util.get_cfg_option_str(cfg, 'user') + disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) + disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", + DISABLE_ROOT_OPTS) + keys = cloud.get_public_ssh_keys() + + if "ssh_authorized_keys" in cfg: + cfgkeys = cfg["ssh_authorized_keys"] + keys.extend(cfgkeys) + + apply_credentials(keys, user, disable_root, disable_root_opts, log) + except: + util.logexc(log) + log.warn("applying credentials failed!\n") + + +def apply_credentials(keys, user, disable_root, + disable_root_opts=DISABLE_ROOT_OPTS, log=None): + keys = set(keys) + if user: + sshutil.setup_user_keys(keys, user, '', log) + + if disable_root: + key_prefix = disable_root_opts.replace('$USER', user) + else: + key_prefix = '' + + sshutil.setup_user_keys(keys, 'root', key_prefix, log) diff --git a/cloudinit/handlers/cc_ssh_import_id.py b/cloudinit/handlers/cc_ssh_import_id.py new file mode 100644 index 00000000..bbf5bd83 --- /dev/null +++ b/cloudinit/handlers/cc_ssh_import_id.py @@ -0,0 +1,50 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import traceback + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + user = args[0] + ids = [] + if len(args) > 1: + ids = args[1:] + else: + user = util.get_cfg_option_str(cfg, "user", "ubuntu") + ids = util.get_cfg_option_list_or_str(cfg, "ssh_import_id", []) + + if len(ids) == 0: + return + + cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids + + log.debug("importing ssh ids. cmd = %s" % cmd) + + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) + except OSError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/handlers/cc_timezone.py b/cloudinit/handlers/cc_timezone.py new file mode 100644 index 00000000..e5c9901b --- /dev/null +++ b/cloudinit/handlers/cc_timezone.py @@ -0,0 +1,67 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.CloudConfig import per_instance +from cloudinit import util +import os.path +import shutil + +frequency = per_instance +tz_base = "/usr/share/zoneinfo" + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + timezone = args[0] + else: + timezone = util.get_cfg_option_str(cfg, "timezone", False) + + if not timezone: + return + + tz_file = "%s/%s" % (tz_base, timezone) + + if not os.path.isfile(tz_file): + log.debug("Invalid timezone %s" % tz_file) + raise Exception("Invalid timezone %s" % tz_file) + + try: + fp = open("/etc/timezone", "wb") + fp.write("%s\n" % timezone) + fp.close() + except: + log.debug("failed to write to /etc/timezone") + raise + if os.path.exists("/etc/sysconfig/clock"): + try: + with open("/etc/sysconfig/clock", "w") as fp: + fp.write('ZONE="%s"\n' % timezone) + except: + log.debug("failed to write to /etc/sysconfig/clock") + raise + + try: + shutil.copy(tz_file, "/etc/localtime") + except: + log.debug("failed to copy %s to /etc/localtime" % tz_file) + raise + + log.debug("set timezone to %s" % timezone) + return diff --git a/cloudinit/handlers/cc_update_etc_hosts.py b/cloudinit/handlers/cc_update_etc_hosts.py new file mode 100644 index 00000000..6ad2fca8 --- /dev/null +++ b/cloudinit/handlers/cc_update_etc_hosts.py @@ -0,0 +1,87 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_always +import StringIO + +frequency = per_always + + +def handle(_name, cfg, cloud, log, _args): + (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + + manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) + if manage_hosts in ("True", "true", True, "template"): + # render from template file + try: + if not hostname: + log.info("manage_etc_hosts was set, but no hostname found") + return + + util.render_to_file('hosts', '/etc/hosts', + {'hostname': hostname, 'fqdn': fqdn}) + except Exception: + log.warn("failed to update /etc/hosts") + raise + elif manage_hosts == "localhost": + log.debug("managing 127.0.1.1 in /etc/hosts") + update_etc_hosts(hostname, fqdn, log) + return + else: + if manage_hosts not in ("False", False): + log.warn("Unknown value for manage_etc_hosts. Assuming False") + else: + log.debug("not managing /etc/hosts") + + +def update_etc_hosts(hostname, fqdn, _log): + with open('/etc/hosts', 'r') as etchosts: + header = "# Added by cloud-init\n" + hosts_line = "127.0.1.1\t%s %s\n" % (fqdn, hostname) + need_write = False + need_change = True + new_etchosts = StringIO.StringIO() + for line in etchosts: + split_line = [s.strip() for s in line.split()] + if len(split_line) < 2: + new_etchosts.write(line) + continue + if line == header: + continue + ip, hosts = split_line[0], split_line[1:] + if ip == "127.0.1.1": + if sorted([hostname, fqdn]) == sorted(hosts): + need_change = False + if need_change == True: + line = "%s%s" % (header, hosts_line) + need_change = False + need_write = True + new_etchosts.write(line) + etchosts.close() + if need_change == True: + new_etchosts.write("%s%s" % (header, hosts_line)) + need_write = True + if need_write == True: + new_etcfile = open('/etc/hosts', 'wb') + new_etcfile.write(new_etchosts.getvalue()) + new_etcfile.close() + new_etchosts.close() + return diff --git a/cloudinit/handlers/cc_update_hostname.py b/cloudinit/handlers/cc_update_hostname.py new file mode 100644 index 00000000..b9d1919a --- /dev/null +++ b/cloudinit/handlers/cc_update_hostname.py @@ -0,0 +1,101 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import errno +from cloudinit.CloudConfig import per_always + +frequency = per_always + + +def handle(_name, cfg, cloud, log, _args): + if util.get_cfg_option_bool(cfg, "preserve_hostname", False): + log.debug("preserve_hostname is set. not updating hostname") + return + + (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) + try: + prev = "%s/%s" % (cloud.get_cpath('data'), "previous-hostname") + update_hostname(hostname, prev, log) + except Exception: + log.warn("failed to set hostname\n") + raise + + +# read hostname from a 'hostname' file +# allow for comments and stripping line endings. +# if file doesn't exist, or no contents, return default +def read_hostname(filename, default=None): + try: + fp = open(filename, "r") + lines = fp.readlines() + fp.close() + for line in lines: + hpos = line.find("#") + if hpos != -1: + line = line[0:hpos] + line = line.rstrip() + if line: + return line + except IOError as e: + if e.errno != errno.ENOENT: + raise + return default + + +def update_hostname(hostname, prev_file, log): + etc_file = "/etc/hostname" + + hostname_prev = None + hostname_in_etc = None + + try: + hostname_prev = read_hostname(prev_file) + except Exception as e: + log.warn("Failed to open %s: %s" % (prev_file, e)) + + try: + hostname_in_etc = read_hostname(etc_file) + except: + log.warn("Failed to open %s" % etc_file) + + update_files = [] + if not hostname_prev or hostname_prev != hostname: + update_files.append(prev_file) + + if (not hostname_in_etc or + (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): + update_files.append(etc_file) + + try: + for fname in update_files: + util.write_file(fname, "%s\n" % hostname, 0644) + log.debug("wrote %s to %s" % (hostname, fname)) + except: + log.warn("failed to write hostname to %s" % fname) + + if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: + log.debug("%s differs from %s. assuming user maintained" % + (prev_file, etc_file)) + + if etc_file in update_files: + log.debug("setting hostname to %s" % hostname) + subprocess.Popen(['hostname', hostname]).communicate() diff --git a/cloudinit/importer.py b/cloudinit/importer.py new file mode 100644 index 00000000..deea0956 --- /dev/null +++ b/cloudinit/importer.py @@ -0,0 +1,11 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +import sys + + +def import_module(module_name): + try: + __import__(module_name) + return sys.modules.get(module_name, None) + except ImportError as err: + raise RuntimeError('Could not load module %s: %s' % (module_name, err)) diff --git a/cloudinit/includer.py b/cloudinit/includer.py new file mode 100644 index 00000000..d1022c5a --- /dev/null +++ b/cloudinit/includer.py @@ -0,0 +1,65 @@ +import os +import re + +from cloudinit import downloader as down +from cloudinit import exceptions as excp +from cloudinit import log as logging +from cloudinit import shell as sh + +INCLUDE_PATT = re.compile("^#(opt_include|include)[ \t](.*)$", re.MULTILINE) +OPT_PATS = ['opt_include'] + +LOG = logging.getLogger(__name__) + + +class Includer(object): + + def __init__(self, root_fn, stack_limit=10): + self.root_fn = root_fn + self.stack_limit = stack_limit + + def _read_file(self, fname): + return sh.read_file(fname) + + def _read(self, fname, stack, rel): + if len(stack) >= self.stack_limit: + raise excp.StackExceeded("Stack limit of %s reached while including %s" % (self.stack_limit, fname)) + + canon_fname = self._canon_name(fname, rel) + if canon_fname in stack: + raise excp.RecursiveInclude("File %s recursively included" % (canon_fname)) + + stack.add(canon_fname) + new_rel = os.path.dirname(canon_fname) + contents = self._read_file(canon_fname) + + def include_cb(match): + is_optional = (match.group(1).lower() in OPT_PATS) + fn = match.group(2).strip() + if not fn: + # Should we die?? + return match.group(0) + else: + try: + LOG.debug("Including file %s", fn) + return self._read(fn, stack, new_rel) + except IOError: + if is_optional: + return '' + else: + raise + + adjusted_contents = INCLUDE_PATT.sub(include_cb, contents) + stack.remove(fname) + return adjusted_contents + + def _canon_name(self, fname, rel): + fname = fname.strip() + if not fname.startswith("/"): + fname = os.path.sep.join([rel, fname]) + return os.path.realpath(fname) + + def read(self, relative_to="."): + stack = set() + return self._read(self.root_fn, stack, rel=relative_to) + diff --git a/cloudinit/log.py b/cloudinit/log.py new file mode 100644 index 00000000..4d270045 --- /dev/null +++ b/cloudinit/log.py @@ -0,0 +1,94 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +import logging +import logging.handlers +import sys + +# Logging levels for easy access +CRITICAL = logging.CRITICAL +FATAL = logging.FATAL +ERROR = logging.ERROR +WARNING = logging.WARNING +WARN = logging.WARN +INFO = logging.INFO +DEBUG = logging.DEBUG +NOTSET = logging.NOTSET + +# File log rotation settings +ROTATE_AMOUNT = 10 # Only keep the past 9 + 1 active +ROTATE_SIZE = 10 * 1024 * 1024 # 10 MB + + +class ConsoleFormatter(logging.Formatter): + + def _get_mini_level(self, record): + if record.levelno in [INFO, NOTSET] or not record.levelname: + return '' + lvlname = record.levelname + return lvlname[0].upper() + ": " + + def format(self, record): + record.message = record.getMessage() + rdict = dict(record.__dict__) + rdict['minilevelname'] = self._get_mini_level(record) + # Skipping exception info for the console... + return self._fmt % (rdict) + + +def setupLogging(level, filename=None, filelevel=logging.DEBUG): + root = getLogger() + consolelg = logging.StreamHandler(sys.stdout) + consolelg.setFormatter(ConsoleFormatter('%(minilevelname)s%(message)s')) + consolelg.setLevel(level) + root.addHandler(consolelg) + if filename: + filelg = logging.handlers.RotatingFileHandler(filename, maxBytes=ROTATE_SIZE, backupCount=ROTATE_AMOUNT) + filelg.setFormatter(logging.Formatter('%(levelname)s: @%(name)s : %(message)s')) + filelg.setLevel(filelevel) + root.addHandler(filelg) + root.setLevel(level) + + +def logging_set_from_cfg(cfg): + log_cfgs = [] + logcfg = util.get_cfg_option_str(cfg, "log_cfg", False) + if logcfg: + # if there is a 'logcfg' entry in the config, respect + # it, it is the old keyname + log_cfgs = [logcfg] + elif "log_cfgs" in cfg: + for cfg in cfg['log_cfgs']: + if isinstance(cfg, list): + log_cfgs.append('\n'.join(cfg)) + else: + log_cfgs.append() + + if not len(log_cfgs): + sys.stderr.write("Warning, no logging configured\n") + return + + for logcfg in log_cfgs: + try: + logging.config.fileConfig(StringIO.StringIO(logcfg)) + return + except: + pass + + raise Exception("no valid logging found\n") + + +def getLogger(name='cloudinit'): + return logging.getLogger(name) + + +# Fixes this annoyance... +# No handlers could be found for logger XXX annoying output... +try: + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + +logger = getLogger() +logger.addHandler(NullHandler()) diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py new file mode 100644 index 00000000..a081fbe8 --- /dev/null +++ b/cloudinit/ssh_util.py @@ -0,0 +1,227 @@ +#!/usr/bin/python +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +import cloudinit.util as util + + +class AuthKeyEntry(): + # lines are options, keytype, base64-encoded key, comment + # man page says the following which I did not understand: + # The options field is optional; its presence is determined by whether + # the line starts with a number or not (the options field never starts + # with a number) + options = None + keytype = None + base64 = None + comment = None + is_comment = False + line_in = "" + + def __init__(self, line, def_opt=None): + line = line.rstrip("\n\r") + self.line_in = line + if line.startswith("#") or line.strip() == "": + self.is_comment = True + else: + ent = line.strip() + toks = ent.split(None, 3) + if len(toks) == 1: + self.base64 = toks[0] + elif len(toks) == 2: + (self.base64, self.comment) = toks + elif len(toks) == 3: + (self.keytype, self.base64, self.comment) = toks + elif len(toks) == 4: + i = 0 + ent = line.strip() + quoted = False + # taken from auth_rsa_key_allowed in auth-rsa.c + try: + while (i < len(ent) and + ((quoted) or (ent[i] not in (" ", "\t")))): + curc = ent[i] + nextc = ent[i + 1] + if curc == "\\" and nextc == '"': + i = i + 1 + elif curc == '"': + quoted = not quoted + i = i + 1 + except IndexError: + self.is_comment = True + return + + try: + self.options = ent[0:i] + (self.keytype, self.base64, self.comment) = \ + ent[i + 1:].split(None, 3) + except ValueError: + # we did not understand this line + self.is_comment = True + + if self.options == None and def_opt: + self.options = def_opt + + return + + def debug(self): + print("line_in=%s\ncomment: %s\noptions=%s\nkeytype=%s\nbase64=%s\n" + "comment=%s\n" % (self.line_in, self.is_comment, self.options, + self.keytype, self.base64, self.comment)), + + def __repr__(self): + if self.is_comment: + return(self.line_in) + else: + toks = [] + for e in (self.options, self.keytype, self.base64, self.comment): + if e: + toks.append(e) + + return(' '.join(toks)) + + +def update_authorized_keys(fname, keys): + # keys is a list of AuthKeyEntries + # key_prefix is the prefix (options) to prepend + try: + fp = open(fname, "r") + lines = fp.readlines() # lines have carriage return + fp.close() + except IOError: + lines = [] + + ka_stats = {} # keys_added status + for k in keys: + ka_stats[k] = False + + to_add = [] + for key in keys: + to_add.append(key) + + for i in range(0, len(lines)): + ent = AuthKeyEntry(lines[i]) + for k in keys: + if k.base64 == ent.base64 and not k.is_comment: + ent = k + try: + to_add.remove(k) + except ValueError: + pass + lines[i] = str(ent) + + # now append any entries we did not match above + for key in to_add: + lines.append(str(key)) + + if len(lines) == 0: + return("") + else: + return('\n'.join(lines) + "\n") + + +def setup_user_keys(keys, user, key_prefix, log=None): + import pwd + saved_umask = os.umask(077) + + pwent = pwd.getpwnam(user) + + ssh_dir = '%s/.ssh' % pwent.pw_dir + if not os.path.exists(ssh_dir): + os.mkdir(ssh_dir) + os.chown(ssh_dir, pwent.pw_uid, pwent.pw_gid) + + try: + ssh_cfg = parse_ssh_config() + akeys = ssh_cfg.get("AuthorizedKeysFile", "%h/.ssh/authorized_keys") + akeys = akeys.replace("%h", pwent.pw_dir) + akeys = akeys.replace("%u", user) + if not akeys.startswith('/'): + akeys = os.path.join(pwent.pw_dir, akeys) + authorized_keys = akeys + except Exception: + authorized_keys = '%s/.ssh/authorized_keys' % pwent.pw_dir + if log: + util.logexc(log) + + key_entries = [] + for k in keys: + ke = AuthKeyEntry(k, def_opt=key_prefix) + key_entries.append(ke) + + content = update_authorized_keys(authorized_keys, key_entries) + util.write_file(authorized_keys, content, 0600) + + os.chown(authorized_keys, pwent.pw_uid, pwent.pw_gid) + util.restorecon_if_possible(ssh_dir, recursive=True) + + os.umask(saved_umask) + + +def parse_ssh_config(fname="/etc/ssh/sshd_config"): + ret = {} + fp = open(fname) + for l in fp.readlines(): + l = l.strip() + if not l or l.startswith("#"): + continue + key, val = l.split(None, 1) + ret[key] = val + fp.close() + return(ret) + +if __name__ == "__main__": + def main(): + import sys + # usage: orig_file, new_keys, [key_prefix] + # prints out merged, where 'new_keys' will trump old + ## example + ## ### begin auth_keys ### + # ssh-rsa AAAAB3NzaC1xxxxxxxxxV3csgm8cJn7UveKHkYjJp8= smoser-work + # ssh-rsa AAAAB3NzaC1xxxxxxxxxCmXp5Kt5/82cD/VN3NtHw== smoser@brickies + # ### end authorized_keys ### + # + # ### begin new_keys ### + # ssh-rsa nonmatch smoser@newhost + # ssh-rsa AAAAB3NzaC1xxxxxxxxxV3csgm8cJn7UveKHkYjJp8= new_comment + # ### end new_keys ### + # + # Then run as: + # program auth_keys new_keys \ + # 'no-port-forwarding,command=\"echo hi world;\"' + def_prefix = None + orig_key_file = sys.argv[1] + new_key_file = sys.argv[2] + if len(sys.argv) > 3: + def_prefix = sys.argv[3] + fp = open(new_key_file) + + newkeys = [] + for line in fp.readlines(): + newkeys.append(AuthKeyEntry(line, def_prefix)) + + fp.close() + print update_authorized_keys(orig_key_file, newkeys) + + main() + +# vi: ts=4 expandtab diff --git a/cloudinit/templater.py b/cloudinit/templater.py new file mode 100644 index 00000000..b6b320ab --- /dev/null +++ b/cloudinit/templater.py @@ -0,0 +1,17 @@ +import os + +from Cheetah.Template import Template + +from cloudinit import util + +TEMPLATE_DIR = '/etc/cloud/templates/' + + +def render_to_file(template, outfile, searchList): + contents = Template(file=os.path.join(TEMPLATE_DIR, template), + searchList=[searchList]).respond() + util.write_file(outfile, contents) + + +def render_string(template, searchList): + return Template(template, searchList=[searchList]).respond() diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py new file mode 100644 index 00000000..ec914480 --- /dev/null +++ b/cloudinit/user_data.py @@ -0,0 +1,262 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import email + +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.mime.base import MIMEBase +import yaml +import cloudinit +import cloudinit.util as util +import hashlib +import urllib + + +starts_with_mappings = { + '#include': 'text/x-include-url', + '#include-once': 'text/x-include-once-url', + '#!': 'text/x-shellscript', + '#cloud-config': 'text/cloud-config', + '#upstart-job': 'text/upstart-job', + '#part-handler': 'text/part-handler', + '#cloud-boothook': 'text/cloud-boothook', + '#cloud-config-archive': 'text/cloud-config-archive', +} + + +# if 'string' is compressed return decompressed otherwise return it +def decomp_str(string): + import StringIO + import gzip + try: + uncomp = gzip.GzipFile(None, "rb", 1, StringIO.StringIO(string)).read() + return(uncomp) + except: + return(string) + + +def do_include(content, appendmsg): + import os + # is just a list of urls, one per line + # also support '#include ' + includeonce = False + for line in content.splitlines(): + if line == "#include": + continue + if line == "#include-once": + includeonce = True + continue + if line.startswith("#include-once"): + line = line[len("#include-once"):].lstrip() + includeonce = True + elif line.startswith("#include"): + line = line[len("#include"):].lstrip() + if line.startswith("#"): + continue + if line.strip() == "": + continue + + # urls cannot not have leading or trailing white space + msum = hashlib.md5() # pylint: disable=E1101 + msum.update(line.strip()) + includeonce_filename = "%s/urlcache/%s" % ( + cloudinit.get_ipath_cur("data"), msum.hexdigest()) + try: + if includeonce and os.path.isfile(includeonce_filename): + with open(includeonce_filename, "r") as fp: + content = fp.read() + else: + content = urllib.urlopen(line).read() + if includeonce: + util.write_file(includeonce_filename, content, mode=0600) + except Exception: + raise + + process_includes(message_from_string(decomp_str(content)), appendmsg) + + +def explode_cc_archive(archive, appendmsg): + for ent in yaml.load(archive): + # ent can be one of: + # dict { 'filename' : 'value', 'content' : 'value', 'type' : 'value' } + # filename and type not be present + # or + # scalar(payload) + + def_type = "text/cloud-config" + if isinstance(ent, str): + ent = {'content': ent} + + content = ent.get('content', '') + mtype = ent.get('type', None) + if mtype == None: + mtype = type_from_startswith(content, def_type) + + maintype, subtype = mtype.split('/', 1) + if maintype == "text": + msg = MIMEText(content, _subtype=subtype) + else: + msg = MIMEBase(maintype, subtype) + msg.set_payload(content) + + if 'filename' in ent: + msg.add_header('Content-Disposition', 'attachment', + filename=ent['filename']) + + for header in ent.keys(): + if header in ('content', 'filename', 'type'): + continue + msg.add_header(header, ent['header']) + + _attach_part(appendmsg, msg) + + +def multi_part_count(outermsg, newcount=None): + """ + Return the number of attachments to this MIMEMultipart by looking + at its 'Number-Attachments' header. + """ + nfield = 'Number-Attachments' + if nfield not in outermsg: + outermsg[nfield] = "0" + + if newcount != None: + outermsg.replace_header(nfield, str(newcount)) + + return(int(outermsg.get('Number-Attachments', 0))) + + +def _attach_part(outermsg, part): + """ + Attach an part to an outer message. outermsg must be a MIMEMultipart. + Modifies a header in outermsg to keep track of number of attachments. + """ + cur = multi_part_count(outermsg) + if not part.get_filename(None): + part.add_header('Content-Disposition', 'attachment', + filename='part-%03d' % (cur + 1)) + outermsg.attach(part) + multi_part_count(outermsg, cur + 1) + + +def type_from_startswith(payload, default=None): + # slist is sorted longest first + slist = sorted(starts_with_mappings.keys(), key=lambda e: 0 - len(e)) + for sstr in slist: + if payload.startswith(sstr): + return(starts_with_mappings[sstr]) + return default + + +def process_includes(msg, appendmsg=None): + if appendmsg == None: + appendmsg = MIMEMultipart() + + for part in msg.walk(): + # multipart/* are just containers + if part.get_content_maintype() == 'multipart': + continue + + ctype = None + ctype_orig = part.get_content_type() + + payload = part.get_payload(decode=True) + + if ctype_orig in ("text/plain", "text/x-not-multipart"): + ctype = type_from_startswith(payload) + + if ctype is None: + ctype = ctype_orig + + if ctype in ('text/x-include-url', 'text/x-include-once-url'): + do_include(payload, appendmsg) + continue + + if ctype == "text/cloud-config-archive": + explode_cc_archive(payload, appendmsg) + continue + + if 'Content-Type' in msg: + msg.replace_header('Content-Type', ctype) + else: + msg['Content-Type'] = ctype + + _attach_part(appendmsg, part) + + +def message_from_string(data, headers=None): + if headers is None: + headers = {} + if "mime-version:" in data[0:4096].lower(): + msg = email.message_from_string(data) + for (key, val) in headers.items(): + if key in msg: + msg.replace_header(key, val) + else: + msg[key] = val + else: + mtype = headers.get("Content-Type", "text/x-not-multipart") + maintype, subtype = mtype.split("/", 1) + msg = MIMEBase(maintype, subtype, *headers) + msg.set_payload(data) + + return(msg) + + +# this is heavily wasteful, reads through userdata string input +def preprocess_userdata(data): + newmsg = MIMEMultipart() + process_includes(message_from_string(decomp_str(data)), newmsg) + return(newmsg.as_string()) + + +# callback is a function that will be called with (data, content_type, +# filename, payload) +def walk_userdata(istr, callback, data=None): + partnum = 0 + for part in message_from_string(istr).walk(): + # multipart/* are just containers + if part.get_content_maintype() == 'multipart': + continue + + ctype = part.get_content_type() + if ctype is None: + ctype = 'application/octet-stream' + + filename = part.get_filename() + if not filename: + filename = 'part-%03d' % partnum + + callback(data, ctype, filename, part.get_payload(decode=True)) + + partnum = partnum + 1 + + +if __name__ == "__main__": + def main(): + import sys + data = decomp_str(file(sys.argv[1]).read()) + newmsg = MIMEMultipart() + process_includes(message_from_string(data), newmsg) + print newmsg + print "#found %s parts" % multi_part_count(newmsg) + + main() -- cgit v1.2.3 From 869402301c9793cece24a9357ee3c13dcdafb6e2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 12:45:28 -0700 Subject: Darn it. Those shouldn't be there! --- cloudinit/handlers/DataSource.py | 214 ----------------- cloudinit/handlers/DataSourceCloudStack.py | 92 -------- cloudinit/handlers/DataSourceConfigDrive.py | 231 ------------------- cloudinit/handlers/DataSourceEc2.py | 217 ----------------- cloudinit/handlers/DataSourceMAAS.py | 345 ---------------------------- cloudinit/handlers/DataSourceNoCloud.py | 232 ------------------- cloudinit/handlers/DataSourceOVF.py | 332 -------------------------- cloudinit/sources/DataSource.py | 214 +++++++++++++++++ cloudinit/sources/DataSourceCloudStack.py | 92 ++++++++ cloudinit/sources/DataSourceConfigDrive.py | 231 +++++++++++++++++++ cloudinit/sources/DataSourceEc2.py | 217 +++++++++++++++++ cloudinit/sources/DataSourceMAAS.py | 345 ++++++++++++++++++++++++++++ cloudinit/sources/DataSourceNoCloud.py | 232 +++++++++++++++++++ cloudinit/sources/DataSourceOVF.py | 332 ++++++++++++++++++++++++++ cloudinit/sources/__init__.py | 0 15 files changed, 1663 insertions(+), 1663 deletions(-) delete mode 100644 cloudinit/handlers/DataSource.py delete mode 100644 cloudinit/handlers/DataSourceCloudStack.py delete mode 100644 cloudinit/handlers/DataSourceConfigDrive.py delete mode 100644 cloudinit/handlers/DataSourceEc2.py delete mode 100644 cloudinit/handlers/DataSourceMAAS.py delete mode 100644 cloudinit/handlers/DataSourceNoCloud.py delete mode 100644 cloudinit/handlers/DataSourceOVF.py create mode 100644 cloudinit/sources/DataSource.py create mode 100644 cloudinit/sources/DataSourceCloudStack.py create mode 100644 cloudinit/sources/DataSourceConfigDrive.py create mode 100644 cloudinit/sources/DataSourceEc2.py create mode 100644 cloudinit/sources/DataSourceMAAS.py create mode 100644 cloudinit/sources/DataSourceNoCloud.py create mode 100644 cloudinit/sources/DataSourceOVF.py create mode 100644 cloudinit/sources/__init__.py diff --git a/cloudinit/handlers/DataSource.py b/cloudinit/handlers/DataSource.py deleted file mode 100644 index e2a9150d..00000000 --- a/cloudinit/handlers/DataSource.py +++ /dev/null @@ -1,214 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -DEP_FILESYSTEM = "FILESYSTEM" -DEP_NETWORK = "NETWORK" - -import cloudinit.UserDataHandler as ud -import cloudinit.util as util -import socket - - -class DataSource: - userdata = None - metadata = None - userdata_raw = None - cfgname = "" - # system config (passed in from cloudinit, - # cloud-config before input from the DataSource) - sys_cfg = {} - # datasource config, the cloud-config['datasource']['__name__'] - ds_cfg = {} # datasource config - - def __init__(self, sys_cfg=None): - if not self.cfgname: - name = str(self.__class__).split(".")[-1] - if name.startswith("DataSource"): - name = name[len("DataSource"):] - self.cfgname = name - if sys_cfg: - self.sys_cfg = sys_cfg - - self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, - ("datasource", self.cfgname), self.ds_cfg) - - def get_userdata(self): - if self.userdata == None: - self.userdata = ud.preprocess_userdata(self.userdata_raw) - return self.userdata - - def get_userdata_raw(self): - return(self.userdata_raw) - - # the data sources' config_obj is a cloud-config formated - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return({}) - - def get_public_ssh_keys(self): - keys = [] - if 'public-keys' not in self.metadata: - return([]) - - if isinstance(self.metadata['public-keys'], str): - return(str(self.metadata['public-keys']).splitlines()) - - if isinstance(self.metadata['public-keys'], list): - return(self.metadata['public-keys']) - - for _keyname, klist in self.metadata['public-keys'].items(): - # lp:506332 uec metadata service responds with - # data that makes boto populate a string for 'klist' rather - # than a list. - if isinstance(klist, str): - klist = [klist] - for pkey in klist: - # there is an empty string at the end of the keylist, trim it - if pkey: - keys.append(pkey) - - return(keys) - - def device_name_to_device(self, _name): - # translate a 'name' to a device - # the primary function at this point is on ec2 - # to consult metadata service, that has - # ephemeral0: sdb - # and return 'sdb' for input 'ephemeral0' - return(None) - - def get_locale(self): - return('en_US.UTF-8') - - def get_local_mirror(self): - return None - - def get_instance_id(self): - if 'instance-id' not in self.metadata: - return "iid-datasource" - return(self.metadata['instance-id']) - - def get_hostname(self, fqdn=False): - defdomain = "localdomain" - defhost = "localhost" - - domain = defdomain - if not 'local-hostname' in self.metadata: - - # this is somewhat questionable really. - # the cloud datasource was asked for a hostname - # and didn't have one. raising error might be more appropriate - # but instead, basically look up the existing hostname - toks = [] - - hostname = socket.gethostname() - - fqdn = util.get_fqdn_from_hosts(hostname) - - if fqdn and fqdn.find(".") > 0: - toks = str(fqdn).split(".") - elif hostname: - toks = [hostname, defdomain] - else: - toks = [defhost, defdomain] - - else: - # if there is an ipv4 address in 'local-hostname', then - # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx - lhost = self.metadata['local-hostname'] - if is_ipv4(lhost): - toks = "ip-%s" % lhost.replace(".", "-") - else: - toks = lhost.split(".") - - if len(toks) > 1: - hostname = toks[0] - domain = '.'.join(toks[1:]) - else: - hostname = toks[0] - - if fqdn: - return "%s.%s" % (hostname, domain) - else: - return hostname - - -# return a list of classes that have the same depends as 'depends' -# iterate through cfg_list, loading "DataSourceCollections" modules -# and calling their "get_datasource_list". -# return an ordered list of classes that match -# -# - modules must be named "DataSource", where 'item' is an entry -# in cfg_list -# - if pkglist is given, it will iterate try loading from that package -# ie, pkglist=[ "foo", "" ] -# will first try to load foo.DataSource -# then DataSource -def list_sources(cfg_list, depends, pkglist=None): - if pkglist is None: - pkglist = [] - retlist = [] - for ds_coll in cfg_list: - for pkg in pkglist: - if pkg: - pkg = "%s." % pkg - try: - mod = __import__("%sDataSource%s" % (pkg, ds_coll)) - if pkg: - mod = getattr(mod, "DataSource%s" % ds_coll) - lister = getattr(mod, "get_datasource_list") - retlist.extend(lister(depends)) - break - except: - raise - return(retlist) - - -# depends is a list of dependencies (DEP_FILESYSTEM) -# dslist is a list of 2 item lists -# dslist = [ -# ( class, ( depends-that-this-class-needs ) ) -# } -# it returns a list of 'class' that matched these deps exactly -# it is a helper function for DataSourceCollections -def list_from_depends(depends, dslist): - retlist = [] - depset = set(depends) - for elem in dslist: - (cls, deps) = elem - if depset == set(deps): - retlist.append(cls) - return(retlist) - - -def is_ipv4(instr): - """ determine if input string is a ipv4 address. return boolean""" - toks = instr.split('.') - if len(toks) != 4: - return False - - try: - toks = [x for x in toks if (int(x) < 256 and int(x) > 0)] - except: - return False - - return (len(toks) == 4) diff --git a/cloudinit/handlers/DataSourceCloudStack.py b/cloudinit/handlers/DataSourceCloudStack.py deleted file mode 100644 index 5afdf7b6..00000000 --- a/cloudinit/handlers/DataSourceCloudStack.py +++ /dev/null @@ -1,92 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Cosmin Luta -# -# Author: Cosmin Luta -# Author: Scott Moser -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -from socket import inet_ntoa -import time -import boto.utils as boto_utils -from struct import pack - - -class DataSourceCloudStack(DataSource.DataSource): - api_ver = 'latest' - seeddir = base_seeddir + '/cs' - metadata_address = None - - def __init__(self, sys_cfg=None): - DataSource.DataSource.__init__(self, sys_cfg) - # Cloudstack has its metadata/userdata URLs located at - # http:///latest/ - self.metadata_address = "http://%s/" % self.get_default_gateway() - - def get_default_gateway(self): - """ Returns the default gateway ip address in the dotted format - """ - with open("/proc/net/route", "r") as f: - for line in f.readlines(): - items = line.split("\t") - if items[1] == "00000000": - # found the default route, get the gateway - gw = inet_ntoa(pack(" -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import os.path -import os -import json -import subprocess - -DEFAULT_IID = "iid-dsconfigdrive" - - -class DataSourceConfigDrive(DataSource.DataSource): - seed = None - seeddir = base_seeddir + '/config_drive' - cfg = {} - userdata_raw = None - metadata = None - dsmode = "local" - - def __str__(self): - mstr = "DataSourceConfigDrive[%s]" % self.dsmode - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) - - def get_data(self): - found = None - md = {} - ud = "" - - defaults = {"instance-id": DEFAULT_IID, "dsmode": "pass"} - - if os.path.isdir(self.seeddir): - try: - (md, ud) = read_config_drive_dir(self.seeddir) - found = self.seeddir - except nonConfigDriveDir: - pass - - if not found: - dev = cfg_drive_device() - if dev: - try: - (md, ud) = util.mount_callback_umount(dev, - read_config_drive_dir) - found = dev - except (nonConfigDriveDir, util.mountFailedError): - pass - - if not found: - return False - - if 'dsconfig' in md: - self.cfg = md['dscfg'] - - md = util.mergedict(md, defaults) - - # update interfaces and ifup only on the local datasource - # this way the DataSourceConfigDriveNet doesn't do it also. - if 'network-interfaces' in md and self.dsmode == "local": - if md['dsmode'] == "pass": - log.info("updating network interfaces from configdrive") - else: - log.debug("updating network interfaces from configdrive") - - util.write_file("/etc/network/interfaces", - md['network-interfaces']) - try: - (out, err) = util.subp(['ifup', '--all']) - if len(out) or len(err): - log.warn("ifup --all had stderr: %s" % err) - - except subprocess.CalledProcessError as exc: - log.warn("ifup --all failed: %s" % (exc.output[1])) - - self.seed = found - self.metadata = md - self.userdata_raw = ud - - if md['dsmode'] == self.dsmode: - return True - - log.debug("%s: not claiming datasource, dsmode=%s" % - (self, md['dsmode'])) - return False - - def get_public_ssh_keys(self): - if not 'public-keys' in self.metadata: - return([]) - return(self.metadata['public-keys']) - - # the data sources' config_obj is a cloud-config formated - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return(self.cfg) - - -class DataSourceConfigDriveNet(DataSourceConfigDrive): - dsmode = "net" - - -class nonConfigDriveDir(Exception): - pass - - -def cfg_drive_device(): - """ get the config drive device. return a string like '/dev/vdb' - or None (if there is no non-root device attached). This does not - check the contents, only reports that if there *were* a config_drive - attached, it would be this device. - per config_drive documentation, this is - "associated as the last available disk on the instance" - """ - - if 'CLOUD_INIT_CONFIG_DRIVE_DEVICE' in os.environ: - return(os.environ['CLOUD_INIT_CONFIG_DRIVE_DEVICE']) - - # we are looking for a raw block device (sda, not sda1) with a vfat - # filesystem on it. - - letters = "abcdefghijklmnopqrstuvwxyz" - devs = util.find_devs_with("TYPE=vfat") - - # filter out anything not ending in a letter (ignore partitions) - devs = [f for f in devs if f[-1] in letters] - - # sort them in reverse so "last" device is first - devs.sort(reverse=True) - - if len(devs): - return(devs[0]) - - return(None) - - -def read_config_drive_dir(source_dir): - """ - read_config_drive_dir(source_dir): - read source_dir, and return a tuple with metadata dict and user-data - string populated. If not a valid dir, raise a nonConfigDriveDir - """ - md = {} - ud = "" - - flist = ("etc/network/interfaces", "root/.ssh/authorized_keys", "meta.js") - found = [f for f in flist if os.path.isfile("%s/%s" % (source_dir, f))] - keydata = "" - - if len(found) == 0: - raise nonConfigDriveDir("%s: %s" % (source_dir, "no files found")) - - if "etc/network/interfaces" in found: - with open("%s/%s" % (source_dir, "/etc/network/interfaces")) as fp: - md['network-interfaces'] = fp.read() - - if "root/.ssh/authorized_keys" in found: - with open("%s/%s" % (source_dir, "root/.ssh/authorized_keys")) as fp: - keydata = fp.read() - - meta_js = {} - - if "meta.js" in found: - content = '' - with open("%s/%s" % (source_dir, "meta.js")) as fp: - content = fp.read() - md['meta_js'] = content - try: - meta_js = json.loads(content) - except ValueError: - raise nonConfigDriveDir("%s: %s" % - (source_dir, "invalid json in meta.js")) - - keydata = meta_js.get('public-keys', keydata) - - if keydata: - lines = keydata.splitlines() - md['public-keys'] = [l for l in lines - if len(l) and not l.startswith("#")] - - for copy in ('dsmode', 'instance-id', 'dscfg'): - if copy in meta_js: - md[copy] = meta_js[copy] - - if 'user-data' in meta_js: - ud = meta_js['user-data'] - - return(md, ud) - -datasources = ( - (DataSourceConfigDrive, (DataSource.DEP_FILESYSTEM, )), - (DataSourceConfigDriveNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -) - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - -if __name__ == "__main__": - def main(): - import sys - import pprint - print cfg_drive_device() - (md, ud) = read_config_drive_dir(sys.argv[1]) - print "=== md ===" - pprint.pprint(md) - print "=== ud ===" - print(ud) - - main() - -# vi: ts=4 expandtab diff --git a/cloudinit/handlers/DataSourceEc2.py b/cloudinit/handlers/DataSourceEc2.py deleted file mode 100644 index 7051ecda..00000000 --- a/cloudinit/handlers/DataSourceEc2.py +++ /dev/null @@ -1,217 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import socket -import time -import boto.utils as boto_utils -import os.path - - -class DataSourceEc2(DataSource.DataSource): - api_ver = '2009-04-04' - seeddir = base_seeddir + '/ec2' - metadata_address = "http://169.254.169.254" - - def __str__(self): - return("DataSourceEc2") - - def get_data(self): - seedret = {} - if util.read_optional_seed(seedret, base=self.seeddir + "/"): - self.userdata_raw = seedret['user-data'] - self.metadata = seedret['meta-data'] - log.debug("using seeded ec2 data in %s" % self.seeddir) - return True - - try: - if not self.wait_for_metadata_service(): - return False - start = time.time() - self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, - None, self.metadata_address) - self.metadata = boto_utils.get_instance_metadata(self.api_ver, - self.metadata_address) - log.debug("crawl of metadata service took %ds" % (time.time() - - start)) - return True - except Exception as e: - print e - return False - - def get_instance_id(self): - return(self.metadata['instance-id']) - - def get_availability_zone(self): - return(self.metadata['placement']['availability-zone']) - - def get_local_mirror(self): - return(self.get_mirror_from_availability_zone()) - - def get_mirror_from_availability_zone(self, availability_zone=None): - # availability is like 'us-west-1b' or 'eu-west-1a' - if availability_zone == None: - availability_zone = self.get_availability_zone() - - fallback = None - - if self.is_vpc(): - return fallback - - try: - host = "%s.ec2.archive.ubuntu.com" % availability_zone[:-1] - socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM) - return 'http://%s/ubuntu/' % host - except: - return fallback - - def wait_for_metadata_service(self): - mcfg = self.ds_cfg - - if not hasattr(mcfg, "get"): - mcfg = {} - - max_wait = 120 - try: - max_wait = int(mcfg.get("max_wait", max_wait)) - except Exception: - util.logexc(log) - log.warn("Failed to get max wait. using %s" % max_wait) - - if max_wait == 0: - return False - - timeout = 50 - try: - timeout = int(mcfg.get("timeout", timeout)) - except Exception: - util.logexc(log) - log.warn("Failed to get timeout, using %s" % timeout) - - def_mdurls = ["http://169.254.169.254", "http://instance-data:8773"] - mdurls = mcfg.get("metadata_urls", def_mdurls) - - # Remove addresses from the list that wont resolve. - filtered = [x for x in mdurls if util.is_resolvable_url(x)] - - if set(filtered) != set(mdurls): - log.debug("removed the following from metadata urls: %s" % - list((set(mdurls) - set(filtered)))) - - if len(filtered): - mdurls = filtered - else: - log.warn("Empty metadata url list! using default list") - mdurls = def_mdurls - - urls = [] - url2base = {False: False} - for url in mdurls: - cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver) - urls.append(cur) - url2base[cur] = url - - starttime = time.time() - url = util.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=log.warn) - - if url: - log.debug("Using metadata source: '%s'" % url2base[url]) - else: - log.critical("giving up on md after %i seconds\n" % - int(time.time() - starttime)) - - self.metadata_address = url2base[url] - return (bool(url)) - - def device_name_to_device(self, name): - # consult metadata service, that has - # ephemeral0: sdb - # and return 'sdb' for input 'ephemeral0' - if 'block-device-mapping' not in self.metadata: - return(None) - - found = None - for entname, device in self.metadata['block-device-mapping'].items(): - if entname == name: - found = device - break - # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0' - if entname == "ephemeral" and name == "ephemeral0": - found = device - if found == None: - log.debug("unable to convert %s to a device" % name) - return None - - # LP: #611137 - # the metadata service may believe that devices are named 'sda' - # when the kernel named them 'vda' or 'xvda' - # we want to return the correct value for what will actually - # exist in this instance - mappings = {"sd": ("vd", "xvd")} - ofound = found - short = os.path.basename(found) - - if not found.startswith("/"): - found = "/dev/%s" % found - - if os.path.exists(found): - return(found) - - for nfrom, tlist in mappings.items(): - if not short.startswith(nfrom): - continue - for nto in tlist: - cand = "/dev/%s%s" % (nto, short[len(nfrom):]) - if os.path.exists(cand): - log.debug("remapped device name %s => %s" % (found, cand)) - return(cand) - - # on t1.micro, ephemeral0 will appear in block-device-mapping from - # metadata, but it will not exist on disk (and never will) - # at this pint, we've verified that the path did not exist - # in the special case of 'ephemeral0' return None to avoid bogus - # fstab entry (LP: #744019) - if name == "ephemeral0": - return None - return ofound - - def is_vpc(self): - # per comment in LP: #615545 - ph = "public-hostname" - p4 = "public-ipv4" - if ((ph not in self.metadata or self.metadata[ph] == "") and - (p4 not in self.metadata or self.metadata[p4] == "")): - return True - return False - - -datasources = [ - (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -] - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/handlers/DataSourceMAAS.py b/cloudinit/handlers/DataSourceMAAS.py deleted file mode 100644 index 61a0038f..00000000 --- a/cloudinit/handlers/DataSourceMAAS.py +++ /dev/null @@ -1,345 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# -# Author: Scott Moser -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import errno -import oauth.oauth as oauth -import os.path -import urllib2 -import time - - -MD_VERSION = "2012-03-01" - - -class DataSourceMAAS(DataSource.DataSource): - """ - DataSourceMAAS reads instance information from MAAS. - Given a config metadata_url, and oauth tokens, it expects to find - files under the root named: - instance-id - user-data - hostname - """ - seeddir = base_seeddir + '/maas' - baseurl = None - - def __str__(self): - return("DataSourceMAAS[%s]" % self.baseurl) - - def get_data(self): - mcfg = self.ds_cfg - - try: - (userdata, metadata) = read_maas_seed_dir(self.seeddir) - self.userdata_raw = userdata - self.metadata = metadata - self.baseurl = self.seeddir - return True - except MAASSeedDirNone: - pass - except MAASSeedDirMalformed as exc: - log.warn("%s was malformed: %s\n" % (self.seeddir, exc)) - raise - - try: - # if there is no metadata_url, then we're not configured - url = mcfg.get('metadata_url', None) - if url == None: - return False - - if not self.wait_for_metadata_service(url): - return False - - self.baseurl = url - - (userdata, metadata) = read_maas_seed_url(self.baseurl, - self.md_headers) - self.userdata_raw = userdata - self.metadata = metadata - return True - except Exception: - util.logexc(log) - return False - - def md_headers(self, url): - mcfg = self.ds_cfg - - # if we are missing token_key, token_secret or consumer_key - # then just do non-authed requests - for required in ('token_key', 'token_secret', 'consumer_key'): - if required not in mcfg: - return({}) - - consumer_secret = mcfg.get('consumer_secret', "") - - return(oauth_headers(url=url, consumer_key=mcfg['consumer_key'], - token_key=mcfg['token_key'], token_secret=mcfg['token_secret'], - consumer_secret=consumer_secret)) - - def wait_for_metadata_service(self, url): - mcfg = self.ds_cfg - - max_wait = 120 - try: - max_wait = int(mcfg.get("max_wait", max_wait)) - except Exception: - util.logexc(log) - log.warn("Failed to get max wait. using %s" % max_wait) - - if max_wait == 0: - return False - - timeout = 50 - try: - timeout = int(mcfg.get("timeout", timeout)) - except Exception: - util.logexc(log) - log.warn("Failed to get timeout, using %s" % timeout) - - starttime = time.time() - check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) - url = util.wait_for_url(urls=[check_url], max_wait=max_wait, - timeout=timeout, status_cb=log.warn, - headers_cb=self.md_headers) - - if url: - log.debug("Using metadata source: '%s'" % url) - else: - log.critical("giving up on md after %i seconds\n" % - int(time.time() - starttime)) - - return (bool(url)) - - -def read_maas_seed_dir(seed_d): - """ - Return user-data and metadata for a maas seed dir in seed_d. - Expected format of seed_d are the following files: - * instance-id - * local-hostname - * user-data - """ - files = ('local-hostname', 'instance-id', 'user-data', 'public-keys') - md = {} - - if not os.path.isdir(seed_d): - raise MAASSeedDirNone("%s: not a directory") - - for fname in files: - try: - with open(os.path.join(seed_d, fname)) as fp: - md[fname] = fp.read() - fp.close() - except IOError as e: - if e.errno != errno.ENOENT: - raise - - return(check_seed_contents(md, seed_d)) - - -def read_maas_seed_url(seed_url, header_cb=None, timeout=None, - version=MD_VERSION): - """ - Read the maas datasource at seed_url. - header_cb is a method that should return a headers dictionary that will - be given to urllib2.Request() - - Expected format of seed_url is are the following files: - * //meta-data/instance-id - * //meta-data/local-hostname - * //user-data - """ - files = ('meta-data/local-hostname', - 'meta-data/instance-id', - 'meta-data/public-keys', - 'user-data') - - base_url = "%s/%s" % (seed_url, version) - md = {} - for fname in files: - url = "%s/%s" % (base_url, fname) - if header_cb: - headers = header_cb(url) - else: - headers = {} - - try: - req = urllib2.Request(url, data=None, headers=headers) - resp = urllib2.urlopen(req, timeout=timeout) - md[os.path.basename(fname)] = resp.read() - except urllib2.HTTPError as e: - if e.code != 404: - raise - - return(check_seed_contents(md, seed_url)) - - -def check_seed_contents(content, seed): - """Validate if content is Is the content a dict that is valid as a - return for a datasource. - Either return a (userdata, metadata) tuple or - Raise MAASSeedDirMalformed or MAASSeedDirNone - """ - md_required = ('instance-id', 'local-hostname') - found = content.keys() - - if len(content) == 0: - raise MAASSeedDirNone("%s: no data files found" % seed) - - missing = [k for k in md_required if k not in found] - if len(missing): - raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing)) - - userdata = content.get('user-data', "") - md = {} - for (key, val) in content.iteritems(): - if key == 'user-data': - continue - md[key] = val - - return(userdata, md) - - -def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): - consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) - token = oauth.OAuthToken(token_key, token_secret) - params = { - 'oauth_version': "1.0", - 'oauth_nonce': oauth.generate_nonce(), - 'oauth_timestamp': int(time.time()), - 'oauth_token': token.key, - 'oauth_consumer_key': consumer.key, - } - req = oauth.OAuthRequest(http_url=url, parameters=params) - req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(), - consumer, token) - return(req.to_header()) - - -class MAASSeedDirNone(Exception): - pass - - -class MAASSeedDirMalformed(Exception): - pass - - -datasources = [ - (DataSourceMAAS, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -] - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - - -if __name__ == "__main__": - def main(): - """ - Call with single argument of directory or http or https url. - If url is given additional arguments are allowed, which will be - interpreted as consumer_key, token_key, token_secret, consumer_secret - """ - import argparse - import pprint - - parser = argparse.ArgumentParser(description='Interact with MAAS DS') - parser.add_argument("--config", metavar="file", - help="specify DS config file", default=None) - parser.add_argument("--ckey", metavar="key", - help="the consumer key to auth with", default=None) - parser.add_argument("--tkey", metavar="key", - help="the token key to auth with", default=None) - parser.add_argument("--csec", metavar="secret", - help="the consumer secret (likely '')", default="") - parser.add_argument("--tsec", metavar="secret", - help="the token secret to auth with", default=None) - parser.add_argument("--apiver", metavar="version", - help="the apiver to use ("" can be used)", default=MD_VERSION) - - subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") - subcmds.add_parser('crawl', help="crawl the datasource") - subcmds.add_parser('get', help="do a single GET of provided url") - subcmds.add_parser('check-seed', help="read andn verify seed at url") - - parser.add_argument("url", help="the data source to query") - - args = parser.parse_args() - - creds = {'consumer_key': args.ckey, 'token_key': args.tkey, - 'token_secret': args.tsec, 'consumer_secret': args.csec} - - if args.config: - import yaml - with open(args.config) as fp: - cfg = yaml.load(fp) - if 'datasource' in cfg: - cfg = cfg['datasource']['MAAS'] - for key in creds.keys(): - if key in cfg and creds[key] == None: - creds[key] = cfg[key] - - def geturl(url, headers_cb): - req = urllib2.Request(url, data=None, headers=headers_cb(url)) - return(urllib2.urlopen(req).read()) - - def printurl(url, headers_cb): - print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) - - def crawl(url, headers_cb=None): - if url.endswith("/"): - for line in geturl(url, headers_cb).splitlines(): - if line.endswith("/"): - crawl("%s%s" % (url, line), headers_cb) - else: - printurl("%s%s" % (url, line), headers_cb) - else: - printurl(url, headers_cb) - - def my_headers(url): - headers = {} - if creds.get('consumer_key', None) != None: - headers = oauth_headers(url, **creds) - return headers - - if args.subcmd == "check-seed": - if args.url.startswith("http"): - (userdata, metadata) = read_maas_seed_url(args.url, - header_cb=my_headers, version=args.apiver) - else: - (userdata, metadata) = read_maas_seed_url(args.url) - print "=== userdata ===" - print userdata - print "=== metadata ===" - pprint.pprint(metadata) - - elif args.subcmd == "get": - printurl(args.url, my_headers) - - elif args.subcmd == "crawl": - if not args.url.endswith("/"): - args.url = "%s/" % args.url - crawl(args.url, my_headers) - - main() diff --git a/cloudinit/handlers/DataSourceNoCloud.py b/cloudinit/handlers/DataSourceNoCloud.py deleted file mode 100644 index e8c56b8f..00000000 --- a/cloudinit/handlers/DataSourceNoCloud.py +++ /dev/null @@ -1,232 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import errno -import subprocess - - -class DataSourceNoCloud(DataSource.DataSource): - metadata = None - userdata = None - userdata_raw = None - supported_seed_starts = ("/", "file://") - dsmode = "local" - seed = None - cmdline_id = "ds=nocloud" - seeddir = base_seeddir + '/nocloud' - - def __str__(self): - mstr = "DataSourceNoCloud" - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) - - def get_data(self): - defaults = { - "instance-id": "nocloud", "dsmode": self.dsmode - } - - found = [] - md = {} - ud = "" - - try: - # parse the kernel command line, getting data passed in - if parse_cmdline_data(self.cmdline_id, md): - found.append("cmdline") - except: - util.logexc(log) - return False - - # check to see if the seeddir has data. - seedret = {} - if util.read_optional_seed(seedret, base=self.seeddir + "/"): - md = util.mergedict(md, seedret['meta-data']) - ud = seedret['user-data'] - found.append(self.seeddir) - log.debug("using seeded cache data in %s" % self.seeddir) - - # if the datasource config had a 'seedfrom' entry, then that takes - # precedence over a 'seedfrom' that was found in a filesystem - # but not over external medi - if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']: - found.append("ds_config") - md["seedfrom"] = self.ds_cfg['seedfrom'] - - fslist = util.find_devs_with("TYPE=vfat") - fslist.extend(util.find_devs_with("TYPE=iso9660")) - - label_list = util.find_devs_with("LABEL=cidata") - devlist = list(set(fslist) & set(label_list)) - devlist.sort(reverse=True) - - for dev in devlist: - try: - (newmd, newud) = util.mount_callback_umount(dev, - util.read_seeded) - md = util.mergedict(newmd, md) - ud = newud - - # for seed from a device, the default mode is 'net'. - # that is more likely to be what is desired. - # If they want dsmode of local, then they must - # specify that. - if 'dsmode' not in md: - md['dsmode'] = "net" - - log.debug("using data from %s" % dev) - found.append(dev) - break - except OSError, e: - if e.errno != errno.ENOENT: - raise - except util.mountFailedError: - log.warn("Failed to mount %s when looking for seed" % dev) - - # there was no indication on kernel cmdline or data - # in the seeddir suggesting this handler should be used. - if len(found) == 0: - return False - - seeded_interfaces = None - - # the special argument "seedfrom" indicates we should - # attempt to seed the userdata / metadata from its value - # its primarily value is in allowing the user to type less - # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg - if "seedfrom" in md: - seedfrom = md["seedfrom"] - seedfound = False - for proto in self.supported_seed_starts: - if seedfrom.startswith(proto): - seedfound = proto - break - if not seedfound: - log.debug("seed from %s not supported by %s" % - (seedfrom, self.__class__)) - return False - - if 'network-interfaces' in md: - seeded_interfaces = self.dsmode - - # this could throw errors, but the user told us to do it - # so if errors are raised, let them raise - (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) - log.debug("using seeded cache data from %s" % seedfrom) - - # values in the command line override those from the seed - md = util.mergedict(md, md_seed) - found.append(seedfrom) - - md = util.mergedict(md, defaults) - - # update the network-interfaces if metadata had 'network-interfaces' - # entry and this is the local datasource, or 'seedfrom' was used - # and the source of the seed was self.dsmode - # ('local' for NoCloud, 'net' for NoCloudNet') - if ('network-interfaces' in md and - (self.dsmode in ("local", seeded_interfaces))): - log.info("updating network interfaces from nocloud") - - util.write_file("/etc/network/interfaces", - md['network-interfaces']) - try: - (out, err) = util.subp(['ifup', '--all']) - if len(out) or len(err): - log.warn("ifup --all had stderr: %s" % err) - - except subprocess.CalledProcessError as exc: - log.warn("ifup --all failed: %s" % (exc.output[1])) - - self.seed = ",".join(found) - self.metadata = md - self.userdata_raw = ud - - if md['dsmode'] == self.dsmode: - return True - - log.debug("%s: not claiming datasource, dsmode=%s" % - (self, md['dsmode'])) - return False - - -# returns true or false indicating if cmdline indicated -# that this module should be used -# example cmdline: -# root=LABEL=uec-rootfs ro ds=nocloud -def parse_cmdline_data(ds_id, fill, cmdline=None): - if cmdline is None: - cmdline = util.get_cmdline() - cmdline = " %s " % cmdline - - if not (" %s " % ds_id in cmdline or " %s;" % ds_id in cmdline): - return False - - argline = "" - # cmdline can contain: - # ds=nocloud[;key=val;key=val] - for tok in cmdline.split(): - if tok.startswith(ds_id): - argline = tok.split("=", 1) - - # argline array is now 'nocloud' followed optionally by - # a ';' and then key=value pairs also terminated with ';' - tmp = argline[1].split(";") - if len(tmp) > 1: - kvpairs = tmp[1:] - else: - kvpairs = () - - # short2long mapping to save cmdline typing - s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"} - for item in kvpairs: - try: - (k, v) = item.split("=", 1) - except: - k = item - v = None - if k in s2l: - k = s2l[k] - fill[k] = v - - return(True) - - -class DataSourceNoCloudNet(DataSourceNoCloud): - cmdline_id = "ds=nocloud-net" - supported_seed_starts = ("http://", "https://", "ftp://") - seeddir = base_seeddir + '/nocloud-net' - dsmode = "net" - - -datasources = ( - (DataSourceNoCloud, (DataSource.DEP_FILESYSTEM, )), - (DataSourceNoCloudNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -) - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/handlers/DataSourceOVF.py b/cloudinit/handlers/DataSourceOVF.py deleted file mode 100644 index a0b1b518..00000000 --- a/cloudinit/handlers/DataSourceOVF.py +++ /dev/null @@ -1,332 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import os.path -import os -from xml.dom import minidom -import base64 -import re -import tempfile -import subprocess - - -class DataSourceOVF(DataSource.DataSource): - seed = None - seeddir = base_seeddir + '/ovf' - environment = None - cfg = {} - userdata_raw = None - metadata = None - supported_seed_starts = ("/", "file://") - - def __str__(self): - mstr = "DataSourceOVF" - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) - - def get_data(self): - found = [] - md = {} - ud = "" - - defaults = { - "instance-id": "iid-dsovf" - } - - (seedfile, contents) = get_ovf_env(base_seeddir) - if seedfile: - # found a seed dir - seed = "%s/%s" % (base_seeddir, seedfile) - (md, ud, cfg) = read_ovf_environment(contents) - self.environment = contents - - found.append(seed) - else: - np = {'iso': transport_iso9660, - 'vmware-guestd': transport_vmware_guestd, } - name = None - for name, transfunc in np.iteritems(): - (contents, _dev, _fname) = transfunc() - if contents: - break - - if contents: - (md, ud, cfg) = read_ovf_environment(contents) - self.environment = contents - found.append(name) - - # There was no OVF transports found - if len(found) == 0: - return False - - if 'seedfrom' in md and md['seedfrom']: - seedfrom = md['seedfrom'] - seedfound = False - for proto in self.supported_seed_starts: - if seedfrom.startswith(proto): - seedfound = proto - break - if not seedfound: - log.debug("seed from %s not supported by %s" % - (seedfrom, self.__class__)) - return False - - (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) - log.debug("using seeded cache data from %s" % seedfrom) - - md = util.mergedict(md, md_seed) - found.append(seedfrom) - - md = util.mergedict(md, defaults) - self.seed = ",".join(found) - self.metadata = md - self.userdata_raw = ud - self.cfg = cfg - return True - - def get_public_ssh_keys(self): - if not 'public-keys' in self.metadata: - return([]) - return([self.metadata['public-keys'], ]) - - # the data sources' config_obj is a cloud-config formated - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return(self.cfg) - - -class DataSourceOVFNet(DataSourceOVF): - seeddir = base_seeddir + '/ovf-net' - supported_seed_starts = ("http://", "https://", "ftp://") - - -# this will return a dict with some content -# meta-data, user-data -def read_ovf_environment(contents): - props = getProperties(contents) - md = {} - cfg = {} - ud = "" - cfg_props = ['password', ] - md_props = ['seedfrom', 'local-hostname', 'public-keys', 'instance-id'] - for prop, val in props.iteritems(): - if prop == 'hostname': - prop = "local-hostname" - if prop in md_props: - md[prop] = val - elif prop in cfg_props: - cfg[prop] = val - elif prop == "user-data": - try: - ud = base64.decodestring(val) - except: - ud = val - return(md, ud, cfg) - - -# returns tuple of filename (in 'dirname', and the contents of the file) -# on "not found", returns 'None' for filename and False for contents -def get_ovf_env(dirname): - env_names = ("ovf-env.xml", "ovf_env.xml", "OVF_ENV.XML", "OVF-ENV.XML") - for fname in env_names: - if os.path.isfile("%s/%s" % (dirname, fname)): - fp = open("%s/%s" % (dirname, fname)) - contents = fp.read() - fp.close() - return(fname, contents) - return(None, False) - - -# transport functions take no input and return -# a 3 tuple of content, path, filename -def transport_iso9660(require_iso=True): - - # default_regex matches values in - # /lib/udev/rules.d/60-cdrom_id.rules - # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end" - envname = "CLOUD_INIT_CDROM_DEV_REGEX" - default_regex = "^(sr[0-9]+|hd[a-z]|xvd.*)" - - devname_regex = os.environ.get(envname, default_regex) - cdmatch = re.compile(devname_regex) - - # go through mounts to see if it was already mounted - fp = open("/proc/mounts") - mounts = fp.readlines() - fp.close() - - mounted = {} - for mpline in mounts: - (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() - mounted[dev] = (dev, fstype, mp, False) - mp = mp.replace("\\040", " ") - if fstype != "iso9660" and require_iso: - continue - - if cdmatch.match(dev[5:]) == None: # take off '/dev/' - continue - - (fname, contents) = get_ovf_env(mp) - if contents is not False: - return(contents, dev, fname) - - tmpd = None - dvnull = None - - devs = os.listdir("/dev/") - devs.sort() - - for dev in devs: - fullp = "/dev/%s" % dev - - if fullp in mounted or not cdmatch.match(dev) or os.path.isdir(fullp): - continue - - fp = None - try: - fp = open(fullp, "rb") - fp.read(512) - fp.close() - except: - if fp: - fp.close() - continue - - if tmpd is None: - tmpd = tempfile.mkdtemp() - if dvnull is None: - try: - dvnull = open("/dev/null") - except: - pass - - cmd = ["mount", "-o", "ro", fullp, tmpd] - if require_iso: - cmd.extend(('-t', 'iso9660')) - - rc = subprocess.call(cmd, stderr=dvnull, stdout=dvnull, stdin=dvnull) - if rc: - continue - - (fname, contents) = get_ovf_env(tmpd) - - subprocess.call(["umount", tmpd]) - - if contents is not False: - os.rmdir(tmpd) - return(contents, fullp, fname) - - if tmpd: - os.rmdir(tmpd) - - if dvnull: - dvnull.close() - - return(False, None, None) - - -def transport_vmware_guestd(): - # http://blogs.vmware.com/vapp/2009/07/ \ - # selfconfiguration-and-the-ovf-environment.html - # try: - # cmd = ['vmware-guestd', '--cmd', 'info-get guestinfo.ovfEnv'] - # (out, err) = subp(cmd) - # return(out, 'guestinfo.ovfEnv', 'vmware-guestd') - # except: - # # would need to error check here and see why this failed - # # to know if log/error should be raised - # return(False, None, None) - return(False, None, None) - - -def findChild(node, filter_func): - ret = [] - if not node.hasChildNodes(): - return ret - for child in node.childNodes: - if filter_func(child): - ret.append(child) - return(ret) - - -def getProperties(environString): - dom = minidom.parseString(environString) - if dom.documentElement.localName != "Environment": - raise Exception("No Environment Node") - - if not dom.documentElement.hasChildNodes(): - raise Exception("No Child Nodes") - - envNsURI = "http://schemas.dmtf.org/ovf/environment/1" - - # could also check here that elem.namespaceURI == - # "http://schemas.dmtf.org/ovf/environment/1" - propSections = findChild(dom.documentElement, - lambda n: n.localName == "PropertySection") - - if len(propSections) == 0: - raise Exception("No 'PropertySection's") - - props = {} - propElems = findChild(propSections[0], lambda n: n.localName == "Property") - - for elem in propElems: - key = elem.attributes.getNamedItemNS(envNsURI, "key").value - val = elem.attributes.getNamedItemNS(envNsURI, "value").value - props[key] = val - - return(props) - - -datasources = ( - (DataSourceOVF, (DataSource.DEP_FILESYSTEM, )), - (DataSourceOVFNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -) - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - - -if __name__ == "__main__": - def main(): - import sys - envStr = open(sys.argv[1]).read() - props = getProperties(envStr) - import pprint - pprint.pprint(props) - - md, ud, cfg = read_ovf_environment(envStr) - print "=== md ===" - pprint.pprint(md) - print "=== ud ===" - pprint.pprint(ud) - print "=== cfg ===" - pprint.pprint(cfg) - - main() diff --git a/cloudinit/sources/DataSource.py b/cloudinit/sources/DataSource.py new file mode 100644 index 00000000..e2a9150d --- /dev/null +++ b/cloudinit/sources/DataSource.py @@ -0,0 +1,214 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +DEP_FILESYSTEM = "FILESYSTEM" +DEP_NETWORK = "NETWORK" + +import cloudinit.UserDataHandler as ud +import cloudinit.util as util +import socket + + +class DataSource: + userdata = None + metadata = None + userdata_raw = None + cfgname = "" + # system config (passed in from cloudinit, + # cloud-config before input from the DataSource) + sys_cfg = {} + # datasource config, the cloud-config['datasource']['__name__'] + ds_cfg = {} # datasource config + + def __init__(self, sys_cfg=None): + if not self.cfgname: + name = str(self.__class__).split(".")[-1] + if name.startswith("DataSource"): + name = name[len("DataSource"):] + self.cfgname = name + if sys_cfg: + self.sys_cfg = sys_cfg + + self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, + ("datasource", self.cfgname), self.ds_cfg) + + def get_userdata(self): + if self.userdata == None: + self.userdata = ud.preprocess_userdata(self.userdata_raw) + return self.userdata + + def get_userdata_raw(self): + return(self.userdata_raw) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return({}) + + def get_public_ssh_keys(self): + keys = [] + if 'public-keys' not in self.metadata: + return([]) + + if isinstance(self.metadata['public-keys'], str): + return(str(self.metadata['public-keys']).splitlines()) + + if isinstance(self.metadata['public-keys'], list): + return(self.metadata['public-keys']) + + for _keyname, klist in self.metadata['public-keys'].items(): + # lp:506332 uec metadata service responds with + # data that makes boto populate a string for 'klist' rather + # than a list. + if isinstance(klist, str): + klist = [klist] + for pkey in klist: + # there is an empty string at the end of the keylist, trim it + if pkey: + keys.append(pkey) + + return(keys) + + def device_name_to_device(self, _name): + # translate a 'name' to a device + # the primary function at this point is on ec2 + # to consult metadata service, that has + # ephemeral0: sdb + # and return 'sdb' for input 'ephemeral0' + return(None) + + def get_locale(self): + return('en_US.UTF-8') + + def get_local_mirror(self): + return None + + def get_instance_id(self): + if 'instance-id' not in self.metadata: + return "iid-datasource" + return(self.metadata['instance-id']) + + def get_hostname(self, fqdn=False): + defdomain = "localdomain" + defhost = "localhost" + + domain = defdomain + if not 'local-hostname' in self.metadata: + + # this is somewhat questionable really. + # the cloud datasource was asked for a hostname + # and didn't have one. raising error might be more appropriate + # but instead, basically look up the existing hostname + toks = [] + + hostname = socket.gethostname() + + fqdn = util.get_fqdn_from_hosts(hostname) + + if fqdn and fqdn.find(".") > 0: + toks = str(fqdn).split(".") + elif hostname: + toks = [hostname, defdomain] + else: + toks = [defhost, defdomain] + + else: + # if there is an ipv4 address in 'local-hostname', then + # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx + lhost = self.metadata['local-hostname'] + if is_ipv4(lhost): + toks = "ip-%s" % lhost.replace(".", "-") + else: + toks = lhost.split(".") + + if len(toks) > 1: + hostname = toks[0] + domain = '.'.join(toks[1:]) + else: + hostname = toks[0] + + if fqdn: + return "%s.%s" % (hostname, domain) + else: + return hostname + + +# return a list of classes that have the same depends as 'depends' +# iterate through cfg_list, loading "DataSourceCollections" modules +# and calling their "get_datasource_list". +# return an ordered list of classes that match +# +# - modules must be named "DataSource", where 'item' is an entry +# in cfg_list +# - if pkglist is given, it will iterate try loading from that package +# ie, pkglist=[ "foo", "" ] +# will first try to load foo.DataSource +# then DataSource +def list_sources(cfg_list, depends, pkglist=None): + if pkglist is None: + pkglist = [] + retlist = [] + for ds_coll in cfg_list: + for pkg in pkglist: + if pkg: + pkg = "%s." % pkg + try: + mod = __import__("%sDataSource%s" % (pkg, ds_coll)) + if pkg: + mod = getattr(mod, "DataSource%s" % ds_coll) + lister = getattr(mod, "get_datasource_list") + retlist.extend(lister(depends)) + break + except: + raise + return(retlist) + + +# depends is a list of dependencies (DEP_FILESYSTEM) +# dslist is a list of 2 item lists +# dslist = [ +# ( class, ( depends-that-this-class-needs ) ) +# } +# it returns a list of 'class' that matched these deps exactly +# it is a helper function for DataSourceCollections +def list_from_depends(depends, dslist): + retlist = [] + depset = set(depends) + for elem in dslist: + (cls, deps) = elem + if depset == set(deps): + retlist.append(cls) + return(retlist) + + +def is_ipv4(instr): + """ determine if input string is a ipv4 address. return boolean""" + toks = instr.split('.') + if len(toks) != 4: + return False + + try: + toks = [x for x in toks if (int(x) < 256 and int(x) > 0)] + except: + return False + + return (len(toks) == 4) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py new file mode 100644 index 00000000..5afdf7b6 --- /dev/null +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -0,0 +1,92 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Cosmin Luta +# +# Author: Cosmin Luta +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +from socket import inet_ntoa +import time +import boto.utils as boto_utils +from struct import pack + + +class DataSourceCloudStack(DataSource.DataSource): + api_ver = 'latest' + seeddir = base_seeddir + '/cs' + metadata_address = None + + def __init__(self, sys_cfg=None): + DataSource.DataSource.__init__(self, sys_cfg) + # Cloudstack has its metadata/userdata URLs located at + # http:///latest/ + self.metadata_address = "http://%s/" % self.get_default_gateway() + + def get_default_gateway(self): + """ Returns the default gateway ip address in the dotted format + """ + with open("/proc/net/route", "r") as f: + for line in f.readlines(): + items = line.split("\t") + if items[1] == "00000000": + # found the default route, get the gateway + gw = inet_ntoa(pack(" +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import os.path +import os +import json +import subprocess + +DEFAULT_IID = "iid-dsconfigdrive" + + +class DataSourceConfigDrive(DataSource.DataSource): + seed = None + seeddir = base_seeddir + '/config_drive' + cfg = {} + userdata_raw = None + metadata = None + dsmode = "local" + + def __str__(self): + mstr = "DataSourceConfigDrive[%s]" % self.dsmode + mstr = mstr + " [seed=%s]" % self.seed + return(mstr) + + def get_data(self): + found = None + md = {} + ud = "" + + defaults = {"instance-id": DEFAULT_IID, "dsmode": "pass"} + + if os.path.isdir(self.seeddir): + try: + (md, ud) = read_config_drive_dir(self.seeddir) + found = self.seeddir + except nonConfigDriveDir: + pass + + if not found: + dev = cfg_drive_device() + if dev: + try: + (md, ud) = util.mount_callback_umount(dev, + read_config_drive_dir) + found = dev + except (nonConfigDriveDir, util.mountFailedError): + pass + + if not found: + return False + + if 'dsconfig' in md: + self.cfg = md['dscfg'] + + md = util.mergedict(md, defaults) + + # update interfaces and ifup only on the local datasource + # this way the DataSourceConfigDriveNet doesn't do it also. + if 'network-interfaces' in md and self.dsmode == "local": + if md['dsmode'] == "pass": + log.info("updating network interfaces from configdrive") + else: + log.debug("updating network interfaces from configdrive") + + util.write_file("/etc/network/interfaces", + md['network-interfaces']) + try: + (out, err) = util.subp(['ifup', '--all']) + if len(out) or len(err): + log.warn("ifup --all had stderr: %s" % err) + + except subprocess.CalledProcessError as exc: + log.warn("ifup --all failed: %s" % (exc.output[1])) + + self.seed = found + self.metadata = md + self.userdata_raw = ud + + if md['dsmode'] == self.dsmode: + return True + + log.debug("%s: not claiming datasource, dsmode=%s" % + (self, md['dsmode'])) + return False + + def get_public_ssh_keys(self): + if not 'public-keys' in self.metadata: + return([]) + return(self.metadata['public-keys']) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return(self.cfg) + + +class DataSourceConfigDriveNet(DataSourceConfigDrive): + dsmode = "net" + + +class nonConfigDriveDir(Exception): + pass + + +def cfg_drive_device(): + """ get the config drive device. return a string like '/dev/vdb' + or None (if there is no non-root device attached). This does not + check the contents, only reports that if there *were* a config_drive + attached, it would be this device. + per config_drive documentation, this is + "associated as the last available disk on the instance" + """ + + if 'CLOUD_INIT_CONFIG_DRIVE_DEVICE' in os.environ: + return(os.environ['CLOUD_INIT_CONFIG_DRIVE_DEVICE']) + + # we are looking for a raw block device (sda, not sda1) with a vfat + # filesystem on it. + + letters = "abcdefghijklmnopqrstuvwxyz" + devs = util.find_devs_with("TYPE=vfat") + + # filter out anything not ending in a letter (ignore partitions) + devs = [f for f in devs if f[-1] in letters] + + # sort them in reverse so "last" device is first + devs.sort(reverse=True) + + if len(devs): + return(devs[0]) + + return(None) + + +def read_config_drive_dir(source_dir): + """ + read_config_drive_dir(source_dir): + read source_dir, and return a tuple with metadata dict and user-data + string populated. If not a valid dir, raise a nonConfigDriveDir + """ + md = {} + ud = "" + + flist = ("etc/network/interfaces", "root/.ssh/authorized_keys", "meta.js") + found = [f for f in flist if os.path.isfile("%s/%s" % (source_dir, f))] + keydata = "" + + if len(found) == 0: + raise nonConfigDriveDir("%s: %s" % (source_dir, "no files found")) + + if "etc/network/interfaces" in found: + with open("%s/%s" % (source_dir, "/etc/network/interfaces")) as fp: + md['network-interfaces'] = fp.read() + + if "root/.ssh/authorized_keys" in found: + with open("%s/%s" % (source_dir, "root/.ssh/authorized_keys")) as fp: + keydata = fp.read() + + meta_js = {} + + if "meta.js" in found: + content = '' + with open("%s/%s" % (source_dir, "meta.js")) as fp: + content = fp.read() + md['meta_js'] = content + try: + meta_js = json.loads(content) + except ValueError: + raise nonConfigDriveDir("%s: %s" % + (source_dir, "invalid json in meta.js")) + + keydata = meta_js.get('public-keys', keydata) + + if keydata: + lines = keydata.splitlines() + md['public-keys'] = [l for l in lines + if len(l) and not l.startswith("#")] + + for copy in ('dsmode', 'instance-id', 'dscfg'): + if copy in meta_js: + md[copy] = meta_js[copy] + + if 'user-data' in meta_js: + ud = meta_js['user-data'] + + return(md, ud) + +datasources = ( + (DataSourceConfigDrive, (DataSource.DEP_FILESYSTEM, )), + (DataSourceConfigDriveNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +) + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) + +if __name__ == "__main__": + def main(): + import sys + import pprint + print cfg_drive_device() + (md, ud) = read_config_drive_dir(sys.argv[1]) + print "=== md ===" + pprint.pprint(md) + print "=== ud ===" + print(ud) + + main() + +# vi: ts=4 expandtab diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py new file mode 100644 index 00000000..7051ecda --- /dev/null +++ b/cloudinit/sources/DataSourceEc2.py @@ -0,0 +1,217 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import socket +import time +import boto.utils as boto_utils +import os.path + + +class DataSourceEc2(DataSource.DataSource): + api_ver = '2009-04-04' + seeddir = base_seeddir + '/ec2' + metadata_address = "http://169.254.169.254" + + def __str__(self): + return("DataSourceEc2") + + def get_data(self): + seedret = {} + if util.read_optional_seed(seedret, base=self.seeddir + "/"): + self.userdata_raw = seedret['user-data'] + self.metadata = seedret['meta-data'] + log.debug("using seeded ec2 data in %s" % self.seeddir) + return True + + try: + if not self.wait_for_metadata_service(): + return False + start = time.time() + self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, + None, self.metadata_address) + self.metadata = boto_utils.get_instance_metadata(self.api_ver, + self.metadata_address) + log.debug("crawl of metadata service took %ds" % (time.time() - + start)) + return True + except Exception as e: + print e + return False + + def get_instance_id(self): + return(self.metadata['instance-id']) + + def get_availability_zone(self): + return(self.metadata['placement']['availability-zone']) + + def get_local_mirror(self): + return(self.get_mirror_from_availability_zone()) + + def get_mirror_from_availability_zone(self, availability_zone=None): + # availability is like 'us-west-1b' or 'eu-west-1a' + if availability_zone == None: + availability_zone = self.get_availability_zone() + + fallback = None + + if self.is_vpc(): + return fallback + + try: + host = "%s.ec2.archive.ubuntu.com" % availability_zone[:-1] + socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM) + return 'http://%s/ubuntu/' % host + except: + return fallback + + def wait_for_metadata_service(self): + mcfg = self.ds_cfg + + if not hasattr(mcfg, "get"): + mcfg = {} + + max_wait = 120 + try: + max_wait = int(mcfg.get("max_wait", max_wait)) + except Exception: + util.logexc(log) + log.warn("Failed to get max wait. using %s" % max_wait) + + if max_wait == 0: + return False + + timeout = 50 + try: + timeout = int(mcfg.get("timeout", timeout)) + except Exception: + util.logexc(log) + log.warn("Failed to get timeout, using %s" % timeout) + + def_mdurls = ["http://169.254.169.254", "http://instance-data:8773"] + mdurls = mcfg.get("metadata_urls", def_mdurls) + + # Remove addresses from the list that wont resolve. + filtered = [x for x in mdurls if util.is_resolvable_url(x)] + + if set(filtered) != set(mdurls): + log.debug("removed the following from metadata urls: %s" % + list((set(mdurls) - set(filtered)))) + + if len(filtered): + mdurls = filtered + else: + log.warn("Empty metadata url list! using default list") + mdurls = def_mdurls + + urls = [] + url2base = {False: False} + for url in mdurls: + cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver) + urls.append(cur) + url2base[cur] = url + + starttime = time.time() + url = util.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, status_cb=log.warn) + + if url: + log.debug("Using metadata source: '%s'" % url2base[url]) + else: + log.critical("giving up on md after %i seconds\n" % + int(time.time() - starttime)) + + self.metadata_address = url2base[url] + return (bool(url)) + + def device_name_to_device(self, name): + # consult metadata service, that has + # ephemeral0: sdb + # and return 'sdb' for input 'ephemeral0' + if 'block-device-mapping' not in self.metadata: + return(None) + + found = None + for entname, device in self.metadata['block-device-mapping'].items(): + if entname == name: + found = device + break + # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0' + if entname == "ephemeral" and name == "ephemeral0": + found = device + if found == None: + log.debug("unable to convert %s to a device" % name) + return None + + # LP: #611137 + # the metadata service may believe that devices are named 'sda' + # when the kernel named them 'vda' or 'xvda' + # we want to return the correct value for what will actually + # exist in this instance + mappings = {"sd": ("vd", "xvd")} + ofound = found + short = os.path.basename(found) + + if not found.startswith("/"): + found = "/dev/%s" % found + + if os.path.exists(found): + return(found) + + for nfrom, tlist in mappings.items(): + if not short.startswith(nfrom): + continue + for nto in tlist: + cand = "/dev/%s%s" % (nto, short[len(nfrom):]) + if os.path.exists(cand): + log.debug("remapped device name %s => %s" % (found, cand)) + return(cand) + + # on t1.micro, ephemeral0 will appear in block-device-mapping from + # metadata, but it will not exist on disk (and never will) + # at this pint, we've verified that the path did not exist + # in the special case of 'ephemeral0' return None to avoid bogus + # fstab entry (LP: #744019) + if name == "ephemeral0": + return None + return ofound + + def is_vpc(self): + # per comment in LP: #615545 + ph = "public-hostname" + p4 = "public-ipv4" + if ((ph not in self.metadata or self.metadata[ph] == "") and + (p4 not in self.metadata or self.metadata[p4] == "")): + return True + return False + + +datasources = [ + (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +] + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py new file mode 100644 index 00000000..61a0038f --- /dev/null +++ b/cloudinit/sources/DataSourceMAAS.py @@ -0,0 +1,345 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import errno +import oauth.oauth as oauth +import os.path +import urllib2 +import time + + +MD_VERSION = "2012-03-01" + + +class DataSourceMAAS(DataSource.DataSource): + """ + DataSourceMAAS reads instance information from MAAS. + Given a config metadata_url, and oauth tokens, it expects to find + files under the root named: + instance-id + user-data + hostname + """ + seeddir = base_seeddir + '/maas' + baseurl = None + + def __str__(self): + return("DataSourceMAAS[%s]" % self.baseurl) + + def get_data(self): + mcfg = self.ds_cfg + + try: + (userdata, metadata) = read_maas_seed_dir(self.seeddir) + self.userdata_raw = userdata + self.metadata = metadata + self.baseurl = self.seeddir + return True + except MAASSeedDirNone: + pass + except MAASSeedDirMalformed as exc: + log.warn("%s was malformed: %s\n" % (self.seeddir, exc)) + raise + + try: + # if there is no metadata_url, then we're not configured + url = mcfg.get('metadata_url', None) + if url == None: + return False + + if not self.wait_for_metadata_service(url): + return False + + self.baseurl = url + + (userdata, metadata) = read_maas_seed_url(self.baseurl, + self.md_headers) + self.userdata_raw = userdata + self.metadata = metadata + return True + except Exception: + util.logexc(log) + return False + + def md_headers(self, url): + mcfg = self.ds_cfg + + # if we are missing token_key, token_secret or consumer_key + # then just do non-authed requests + for required in ('token_key', 'token_secret', 'consumer_key'): + if required not in mcfg: + return({}) + + consumer_secret = mcfg.get('consumer_secret', "") + + return(oauth_headers(url=url, consumer_key=mcfg['consumer_key'], + token_key=mcfg['token_key'], token_secret=mcfg['token_secret'], + consumer_secret=consumer_secret)) + + def wait_for_metadata_service(self, url): + mcfg = self.ds_cfg + + max_wait = 120 + try: + max_wait = int(mcfg.get("max_wait", max_wait)) + except Exception: + util.logexc(log) + log.warn("Failed to get max wait. using %s" % max_wait) + + if max_wait == 0: + return False + + timeout = 50 + try: + timeout = int(mcfg.get("timeout", timeout)) + except Exception: + util.logexc(log) + log.warn("Failed to get timeout, using %s" % timeout) + + starttime = time.time() + check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) + url = util.wait_for_url(urls=[check_url], max_wait=max_wait, + timeout=timeout, status_cb=log.warn, + headers_cb=self.md_headers) + + if url: + log.debug("Using metadata source: '%s'" % url) + else: + log.critical("giving up on md after %i seconds\n" % + int(time.time() - starttime)) + + return (bool(url)) + + +def read_maas_seed_dir(seed_d): + """ + Return user-data and metadata for a maas seed dir in seed_d. + Expected format of seed_d are the following files: + * instance-id + * local-hostname + * user-data + """ + files = ('local-hostname', 'instance-id', 'user-data', 'public-keys') + md = {} + + if not os.path.isdir(seed_d): + raise MAASSeedDirNone("%s: not a directory") + + for fname in files: + try: + with open(os.path.join(seed_d, fname)) as fp: + md[fname] = fp.read() + fp.close() + except IOError as e: + if e.errno != errno.ENOENT: + raise + + return(check_seed_contents(md, seed_d)) + + +def read_maas_seed_url(seed_url, header_cb=None, timeout=None, + version=MD_VERSION): + """ + Read the maas datasource at seed_url. + header_cb is a method that should return a headers dictionary that will + be given to urllib2.Request() + + Expected format of seed_url is are the following files: + * //meta-data/instance-id + * //meta-data/local-hostname + * //user-data + """ + files = ('meta-data/local-hostname', + 'meta-data/instance-id', + 'meta-data/public-keys', + 'user-data') + + base_url = "%s/%s" % (seed_url, version) + md = {} + for fname in files: + url = "%s/%s" % (base_url, fname) + if header_cb: + headers = header_cb(url) + else: + headers = {} + + try: + req = urllib2.Request(url, data=None, headers=headers) + resp = urllib2.urlopen(req, timeout=timeout) + md[os.path.basename(fname)] = resp.read() + except urllib2.HTTPError as e: + if e.code != 404: + raise + + return(check_seed_contents(md, seed_url)) + + +def check_seed_contents(content, seed): + """Validate if content is Is the content a dict that is valid as a + return for a datasource. + Either return a (userdata, metadata) tuple or + Raise MAASSeedDirMalformed or MAASSeedDirNone + """ + md_required = ('instance-id', 'local-hostname') + found = content.keys() + + if len(content) == 0: + raise MAASSeedDirNone("%s: no data files found" % seed) + + missing = [k for k in md_required if k not in found] + if len(missing): + raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing)) + + userdata = content.get('user-data', "") + md = {} + for (key, val) in content.iteritems(): + if key == 'user-data': + continue + md[key] = val + + return(userdata, md) + + +def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): + consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) + token = oauth.OAuthToken(token_key, token_secret) + params = { + 'oauth_version': "1.0", + 'oauth_nonce': oauth.generate_nonce(), + 'oauth_timestamp': int(time.time()), + 'oauth_token': token.key, + 'oauth_consumer_key': consumer.key, + } + req = oauth.OAuthRequest(http_url=url, parameters=params) + req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(), + consumer, token) + return(req.to_header()) + + +class MAASSeedDirNone(Exception): + pass + + +class MAASSeedDirMalformed(Exception): + pass + + +datasources = [ + (DataSourceMAAS, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +] + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) + + +if __name__ == "__main__": + def main(): + """ + Call with single argument of directory or http or https url. + If url is given additional arguments are allowed, which will be + interpreted as consumer_key, token_key, token_secret, consumer_secret + """ + import argparse + import pprint + + parser = argparse.ArgumentParser(description='Interact with MAAS DS') + parser.add_argument("--config", metavar="file", + help="specify DS config file", default=None) + parser.add_argument("--ckey", metavar="key", + help="the consumer key to auth with", default=None) + parser.add_argument("--tkey", metavar="key", + help="the token key to auth with", default=None) + parser.add_argument("--csec", metavar="secret", + help="the consumer secret (likely '')", default="") + parser.add_argument("--tsec", metavar="secret", + help="the token secret to auth with", default=None) + parser.add_argument("--apiver", metavar="version", + help="the apiver to use ("" can be used)", default=MD_VERSION) + + subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") + subcmds.add_parser('crawl', help="crawl the datasource") + subcmds.add_parser('get', help="do a single GET of provided url") + subcmds.add_parser('check-seed', help="read andn verify seed at url") + + parser.add_argument("url", help="the data source to query") + + args = parser.parse_args() + + creds = {'consumer_key': args.ckey, 'token_key': args.tkey, + 'token_secret': args.tsec, 'consumer_secret': args.csec} + + if args.config: + import yaml + with open(args.config) as fp: + cfg = yaml.load(fp) + if 'datasource' in cfg: + cfg = cfg['datasource']['MAAS'] + for key in creds.keys(): + if key in cfg and creds[key] == None: + creds[key] = cfg[key] + + def geturl(url, headers_cb): + req = urllib2.Request(url, data=None, headers=headers_cb(url)) + return(urllib2.urlopen(req).read()) + + def printurl(url, headers_cb): + print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) + + def crawl(url, headers_cb=None): + if url.endswith("/"): + for line in geturl(url, headers_cb).splitlines(): + if line.endswith("/"): + crawl("%s%s" % (url, line), headers_cb) + else: + printurl("%s%s" % (url, line), headers_cb) + else: + printurl(url, headers_cb) + + def my_headers(url): + headers = {} + if creds.get('consumer_key', None) != None: + headers = oauth_headers(url, **creds) + return headers + + if args.subcmd == "check-seed": + if args.url.startswith("http"): + (userdata, metadata) = read_maas_seed_url(args.url, + header_cb=my_headers, version=args.apiver) + else: + (userdata, metadata) = read_maas_seed_url(args.url) + print "=== userdata ===" + print userdata + print "=== metadata ===" + pprint.pprint(metadata) + + elif args.subcmd == "get": + printurl(args.url, my_headers) + + elif args.subcmd == "crawl": + if not args.url.endswith("/"): + args.url = "%s/" % args.url + crawl(args.url, my_headers) + + main() diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py new file mode 100644 index 00000000..e8c56b8f --- /dev/null +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -0,0 +1,232 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import errno +import subprocess + + +class DataSourceNoCloud(DataSource.DataSource): + metadata = None + userdata = None + userdata_raw = None + supported_seed_starts = ("/", "file://") + dsmode = "local" + seed = None + cmdline_id = "ds=nocloud" + seeddir = base_seeddir + '/nocloud' + + def __str__(self): + mstr = "DataSourceNoCloud" + mstr = mstr + " [seed=%s]" % self.seed + return(mstr) + + def get_data(self): + defaults = { + "instance-id": "nocloud", "dsmode": self.dsmode + } + + found = [] + md = {} + ud = "" + + try: + # parse the kernel command line, getting data passed in + if parse_cmdline_data(self.cmdline_id, md): + found.append("cmdline") + except: + util.logexc(log) + return False + + # check to see if the seeddir has data. + seedret = {} + if util.read_optional_seed(seedret, base=self.seeddir + "/"): + md = util.mergedict(md, seedret['meta-data']) + ud = seedret['user-data'] + found.append(self.seeddir) + log.debug("using seeded cache data in %s" % self.seeddir) + + # if the datasource config had a 'seedfrom' entry, then that takes + # precedence over a 'seedfrom' that was found in a filesystem + # but not over external medi + if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']: + found.append("ds_config") + md["seedfrom"] = self.ds_cfg['seedfrom'] + + fslist = util.find_devs_with("TYPE=vfat") + fslist.extend(util.find_devs_with("TYPE=iso9660")) + + label_list = util.find_devs_with("LABEL=cidata") + devlist = list(set(fslist) & set(label_list)) + devlist.sort(reverse=True) + + for dev in devlist: + try: + (newmd, newud) = util.mount_callback_umount(dev, + util.read_seeded) + md = util.mergedict(newmd, md) + ud = newud + + # for seed from a device, the default mode is 'net'. + # that is more likely to be what is desired. + # If they want dsmode of local, then they must + # specify that. + if 'dsmode' not in md: + md['dsmode'] = "net" + + log.debug("using data from %s" % dev) + found.append(dev) + break + except OSError, e: + if e.errno != errno.ENOENT: + raise + except util.mountFailedError: + log.warn("Failed to mount %s when looking for seed" % dev) + + # there was no indication on kernel cmdline or data + # in the seeddir suggesting this handler should be used. + if len(found) == 0: + return False + + seeded_interfaces = None + + # the special argument "seedfrom" indicates we should + # attempt to seed the userdata / metadata from its value + # its primarily value is in allowing the user to type less + # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg + if "seedfrom" in md: + seedfrom = md["seedfrom"] + seedfound = False + for proto in self.supported_seed_starts: + if seedfrom.startswith(proto): + seedfound = proto + break + if not seedfound: + log.debug("seed from %s not supported by %s" % + (seedfrom, self.__class__)) + return False + + if 'network-interfaces' in md: + seeded_interfaces = self.dsmode + + # this could throw errors, but the user told us to do it + # so if errors are raised, let them raise + (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) + log.debug("using seeded cache data from %s" % seedfrom) + + # values in the command line override those from the seed + md = util.mergedict(md, md_seed) + found.append(seedfrom) + + md = util.mergedict(md, defaults) + + # update the network-interfaces if metadata had 'network-interfaces' + # entry and this is the local datasource, or 'seedfrom' was used + # and the source of the seed was self.dsmode + # ('local' for NoCloud, 'net' for NoCloudNet') + if ('network-interfaces' in md and + (self.dsmode in ("local", seeded_interfaces))): + log.info("updating network interfaces from nocloud") + + util.write_file("/etc/network/interfaces", + md['network-interfaces']) + try: + (out, err) = util.subp(['ifup', '--all']) + if len(out) or len(err): + log.warn("ifup --all had stderr: %s" % err) + + except subprocess.CalledProcessError as exc: + log.warn("ifup --all failed: %s" % (exc.output[1])) + + self.seed = ",".join(found) + self.metadata = md + self.userdata_raw = ud + + if md['dsmode'] == self.dsmode: + return True + + log.debug("%s: not claiming datasource, dsmode=%s" % + (self, md['dsmode'])) + return False + + +# returns true or false indicating if cmdline indicated +# that this module should be used +# example cmdline: +# root=LABEL=uec-rootfs ro ds=nocloud +def parse_cmdline_data(ds_id, fill, cmdline=None): + if cmdline is None: + cmdline = util.get_cmdline() + cmdline = " %s " % cmdline + + if not (" %s " % ds_id in cmdline or " %s;" % ds_id in cmdline): + return False + + argline = "" + # cmdline can contain: + # ds=nocloud[;key=val;key=val] + for tok in cmdline.split(): + if tok.startswith(ds_id): + argline = tok.split("=", 1) + + # argline array is now 'nocloud' followed optionally by + # a ';' and then key=value pairs also terminated with ';' + tmp = argline[1].split(";") + if len(tmp) > 1: + kvpairs = tmp[1:] + else: + kvpairs = () + + # short2long mapping to save cmdline typing + s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"} + for item in kvpairs: + try: + (k, v) = item.split("=", 1) + except: + k = item + v = None + if k in s2l: + k = s2l[k] + fill[k] = v + + return(True) + + +class DataSourceNoCloudNet(DataSourceNoCloud): + cmdline_id = "ds=nocloud-net" + supported_seed_starts = ("http://", "https://", "ftp://") + seeddir = base_seeddir + '/nocloud-net' + dsmode = "net" + + +datasources = ( + (DataSourceNoCloud, (DataSource.DEP_FILESYSTEM, )), + (DataSourceNoCloudNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +) + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py new file mode 100644 index 00000000..a0b1b518 --- /dev/null +++ b/cloudinit/sources/DataSourceOVF.py @@ -0,0 +1,332 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import os.path +import os +from xml.dom import minidom +import base64 +import re +import tempfile +import subprocess + + +class DataSourceOVF(DataSource.DataSource): + seed = None + seeddir = base_seeddir + '/ovf' + environment = None + cfg = {} + userdata_raw = None + metadata = None + supported_seed_starts = ("/", "file://") + + def __str__(self): + mstr = "DataSourceOVF" + mstr = mstr + " [seed=%s]" % self.seed + return(mstr) + + def get_data(self): + found = [] + md = {} + ud = "" + + defaults = { + "instance-id": "iid-dsovf" + } + + (seedfile, contents) = get_ovf_env(base_seeddir) + if seedfile: + # found a seed dir + seed = "%s/%s" % (base_seeddir, seedfile) + (md, ud, cfg) = read_ovf_environment(contents) + self.environment = contents + + found.append(seed) + else: + np = {'iso': transport_iso9660, + 'vmware-guestd': transport_vmware_guestd, } + name = None + for name, transfunc in np.iteritems(): + (contents, _dev, _fname) = transfunc() + if contents: + break + + if contents: + (md, ud, cfg) = read_ovf_environment(contents) + self.environment = contents + found.append(name) + + # There was no OVF transports found + if len(found) == 0: + return False + + if 'seedfrom' in md and md['seedfrom']: + seedfrom = md['seedfrom'] + seedfound = False + for proto in self.supported_seed_starts: + if seedfrom.startswith(proto): + seedfound = proto + break + if not seedfound: + log.debug("seed from %s not supported by %s" % + (seedfrom, self.__class__)) + return False + + (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) + log.debug("using seeded cache data from %s" % seedfrom) + + md = util.mergedict(md, md_seed) + found.append(seedfrom) + + md = util.mergedict(md, defaults) + self.seed = ",".join(found) + self.metadata = md + self.userdata_raw = ud + self.cfg = cfg + return True + + def get_public_ssh_keys(self): + if not 'public-keys' in self.metadata: + return([]) + return([self.metadata['public-keys'], ]) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return(self.cfg) + + +class DataSourceOVFNet(DataSourceOVF): + seeddir = base_seeddir + '/ovf-net' + supported_seed_starts = ("http://", "https://", "ftp://") + + +# this will return a dict with some content +# meta-data, user-data +def read_ovf_environment(contents): + props = getProperties(contents) + md = {} + cfg = {} + ud = "" + cfg_props = ['password', ] + md_props = ['seedfrom', 'local-hostname', 'public-keys', 'instance-id'] + for prop, val in props.iteritems(): + if prop == 'hostname': + prop = "local-hostname" + if prop in md_props: + md[prop] = val + elif prop in cfg_props: + cfg[prop] = val + elif prop == "user-data": + try: + ud = base64.decodestring(val) + except: + ud = val + return(md, ud, cfg) + + +# returns tuple of filename (in 'dirname', and the contents of the file) +# on "not found", returns 'None' for filename and False for contents +def get_ovf_env(dirname): + env_names = ("ovf-env.xml", "ovf_env.xml", "OVF_ENV.XML", "OVF-ENV.XML") + for fname in env_names: + if os.path.isfile("%s/%s" % (dirname, fname)): + fp = open("%s/%s" % (dirname, fname)) + contents = fp.read() + fp.close() + return(fname, contents) + return(None, False) + + +# transport functions take no input and return +# a 3 tuple of content, path, filename +def transport_iso9660(require_iso=True): + + # default_regex matches values in + # /lib/udev/rules.d/60-cdrom_id.rules + # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end" + envname = "CLOUD_INIT_CDROM_DEV_REGEX" + default_regex = "^(sr[0-9]+|hd[a-z]|xvd.*)" + + devname_regex = os.environ.get(envname, default_regex) + cdmatch = re.compile(devname_regex) + + # go through mounts to see if it was already mounted + fp = open("/proc/mounts") + mounts = fp.readlines() + fp.close() + + mounted = {} + for mpline in mounts: + (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() + mounted[dev] = (dev, fstype, mp, False) + mp = mp.replace("\\040", " ") + if fstype != "iso9660" and require_iso: + continue + + if cdmatch.match(dev[5:]) == None: # take off '/dev/' + continue + + (fname, contents) = get_ovf_env(mp) + if contents is not False: + return(contents, dev, fname) + + tmpd = None + dvnull = None + + devs = os.listdir("/dev/") + devs.sort() + + for dev in devs: + fullp = "/dev/%s" % dev + + if fullp in mounted or not cdmatch.match(dev) or os.path.isdir(fullp): + continue + + fp = None + try: + fp = open(fullp, "rb") + fp.read(512) + fp.close() + except: + if fp: + fp.close() + continue + + if tmpd is None: + tmpd = tempfile.mkdtemp() + if dvnull is None: + try: + dvnull = open("/dev/null") + except: + pass + + cmd = ["mount", "-o", "ro", fullp, tmpd] + if require_iso: + cmd.extend(('-t', 'iso9660')) + + rc = subprocess.call(cmd, stderr=dvnull, stdout=dvnull, stdin=dvnull) + if rc: + continue + + (fname, contents) = get_ovf_env(tmpd) + + subprocess.call(["umount", tmpd]) + + if contents is not False: + os.rmdir(tmpd) + return(contents, fullp, fname) + + if tmpd: + os.rmdir(tmpd) + + if dvnull: + dvnull.close() + + return(False, None, None) + + +def transport_vmware_guestd(): + # http://blogs.vmware.com/vapp/2009/07/ \ + # selfconfiguration-and-the-ovf-environment.html + # try: + # cmd = ['vmware-guestd', '--cmd', 'info-get guestinfo.ovfEnv'] + # (out, err) = subp(cmd) + # return(out, 'guestinfo.ovfEnv', 'vmware-guestd') + # except: + # # would need to error check here and see why this failed + # # to know if log/error should be raised + # return(False, None, None) + return(False, None, None) + + +def findChild(node, filter_func): + ret = [] + if not node.hasChildNodes(): + return ret + for child in node.childNodes: + if filter_func(child): + ret.append(child) + return(ret) + + +def getProperties(environString): + dom = minidom.parseString(environString) + if dom.documentElement.localName != "Environment": + raise Exception("No Environment Node") + + if not dom.documentElement.hasChildNodes(): + raise Exception("No Child Nodes") + + envNsURI = "http://schemas.dmtf.org/ovf/environment/1" + + # could also check here that elem.namespaceURI == + # "http://schemas.dmtf.org/ovf/environment/1" + propSections = findChild(dom.documentElement, + lambda n: n.localName == "PropertySection") + + if len(propSections) == 0: + raise Exception("No 'PropertySection's") + + props = {} + propElems = findChild(propSections[0], lambda n: n.localName == "Property") + + for elem in propElems: + key = elem.attributes.getNamedItemNS(envNsURI, "key").value + val = elem.attributes.getNamedItemNS(envNsURI, "value").value + props[key] = val + + return(props) + + +datasources = ( + (DataSourceOVF, (DataSource.DEP_FILESYSTEM, )), + (DataSourceOVFNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +) + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) + + +if __name__ == "__main__": + def main(): + import sys + envStr = open(sys.argv[1]).read() + props = getProperties(envStr) + import pprint + pprint.pprint(props) + + md, ud, cfg = read_ovf_environment(envStr) + print "=== md ===" + pprint.pprint(md) + print "=== ud ===" + pprint.pprint(ud) + print "=== cfg ===" + pprint.pprint(cfg) + + main() diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py new file mode 100644 index 00000000..e69de29b -- cgit v1.2.3 From 58e73f5ab22c5dcd1c10ca873028e779ef2470a7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 12:48:22 -0700 Subject: Add a file that just deals with handling modules and part data --- cloudinit/handling.py | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 cloudinit/handling.py diff --git a/cloudinit/handling.py b/cloudinit/handling.py new file mode 100644 index 00000000..553abe4f --- /dev/null +++ b/cloudinit/handling.py @@ -0,0 +1,101 @@ +import os + +from cloudinit import importer +from cloudinit import log as logging +from cloudinit.constants import (PER_INSTANCE, PER_ALWAYS) + +LOG = logging.getLogger(__name__) + + +class InternalPartHandler: + freq = PER_INSTANCE + mtypes = [] + handler_version = 1 + handler = None + + def __init__(self, handler, mtypes, frequency, version=2): + self.handler = handler + self.mtypes = mtypes + self.frequency = frequency + self.handler_version = version + + def __repr__(self): + return("InternalPartHandler: [%s]" % self.mtypes) + + def list_types(self): + return(self.mtypes) + + def handle_part(self, data, ctype, filename, payload, frequency): + return(self.handler(data, ctype, filename, payload, frequency)) + + +def handler_register(mod, part_handlers, data, frequency=PER_INSTANCE): + if not hasattr(mod, "handler_version"): + setattr(mod, "handler_version", 1) + + for mtype in mod.list_types(): + part_handlers[mtype] = mod + + handler_call_begin(mod, data, frequency) + return mod + + +def handler_call_begin(mod, data, frequency): + handler_handle_part(mod, data, "__begin__", None, None, frequency) + + +def handler_call_end(mod, data, frequency): + handler_handle_part(mod, data, "__end__", None, None, frequency) + + +def handler_handle_part(mod, data, ctype, filename, payload, frequency): + # only add the handler if the module should run + modfreq = getattr(mod, "frequency", PER_INSTANCE) + if not (modfreq == PER_ALWAYS or + (frequency == PER_INSTANCE and modfreq == PER_INSTANCE)): + return + try: + if mod.handler_version == 1: + mod.handle_part(data, ctype, filename, payload) + else: + mod.handle_part(data, ctype, filename, payload, frequency) + except: + util.logexc(log) + traceback.print_exc(file=sys.stderr) + + +def partwalker_handle_handler(pdata, _ctype, _filename, payload): + curcount = pdata['handlercount'] + modname = 'part-handler-%03d' % curcount + frequency = pdata['frequency'] + + modfname = modname + ".py" + util.write_file(os.path.join(pdata['handlerdir'], modfname), payload, 0600) + + try: + mod = importer.import_module(modname) + handler_register(mod, pdata['handlers'], pdata['data'], frequency) + pdata['handlercount'] = curcount + 1 + except: + LOG.exception("Could not import module %s", modname) + + +def partwalker_callback(pdata, ctype, filename, payload): + # data here is the part_handlers array and then the data to pass through + if ctype == "text/part-handler": + if 'handlercount' not in pdata: + pdata['handlercount'] = 0 + partwalker_handle_handler(pdata, ctype, filename, payload) + return + if ctype not in pdata['handlers']: + if ctype == "text/x-not-multipart": + # Extract the first line or 24 bytes for displaying in the log + start = payload.split("\n", 1)[0][:24] + if start < payload: + details = "starting '%s...'" % start.encode("string-escape") + else: + details = repr(payload) + LOG.warning("Unhandled non-multipart userdata %s", details) + return + handler_handle_part(pdata['handlers'][ctype], pdata['data'], + ctype, filename, payload, pdata['frequency']) -- cgit v1.2.3 From d51e2c431e7b30e0424ba8ab01736b27b1264e17 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 12:48:48 -0700 Subject: Add a file that just deals with url like functions --- cloudinit/url_helper.py | 147 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 cloudinit/url_helper.py diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py new file mode 100644 index 00000000..0f0a9d0c --- /dev/null +++ b/cloudinit/url_helper.py @@ -0,0 +1,147 @@ +import errno +import httplib +import time +import urllib +import urllib2 + +from StringIO import StringIO + +from contextlib import closing + +from cloudinit import log as logging +from cloudinit import shell as sh + +LOG = logging.getLogger(__name__) + + +def ok_http_code(st): + return st in xrange(200, 400) + + +def readurl(url, data=None, timeout=None, retries=0, sec_between=1, read_cb=None, headers=None): + openargs = {} + if timeout is not None: + openargs['timeout'] = int(timeout) + + if data is None: + req = urllib2.Request(url, headers=headers) + else: + req = urllib2.Request(url, data=urllib.urlencode(data), headers=headers) + + if retries <= 0: + retries = 1 + + last_excp = None + LOG.debug("Attempting to read from %s with %s attempts to be performed", url, retries) + for i in range(0, retries): + try: + with closing(urllib2.urlopen(req, **openargs)) as rh: + ofh = StringIO() + sh.pipe_in_out(rh, ofh, chunk_cb=read_cb) + return (ofh.getvalue(), rh.getcode()) + except urllib2.HTTPError as e: + last_excp = e + LOG.exception("Failed at reading from %s.", url) + except urllib2.URLError as e: + # This can be a message string or + # another exception instance (socket.error for remote URLs, OSError for local URLs). + if (isinstance(e.reason, OSError) and + e.reason.errno == errno.ENOENT): + last_excp = e.reason + else: + last_excp = e + LOG.exception("Failed at reading from %s.", url) + LOG.debug("Please wait %s seconds while we wait to try again.", sec_between) + time.sleep(sec_between) + + # Didn't work out + LOG.warn("Failed downloading from %s after %s attempts", url, i + 1) + if last_excp is not None: + raise last_excp + + +def wait_for_url(urls, max_wait=None, timeout=None, + status_cb=None, headers_cb=None, sleep_time=1): + """ + urls: a list of urls to try + max_wait: roughly the maximum time to wait before giving up + The max time is *actually* len(urls)*timeout as each url will + be tried once and given the timeout provided. + timeout: the timeout provided to urllib2.urlopen + status_cb: call method with string message when a url is not available + headers_cb: call method with single argument of url to get headers + for request. + + the idea of this routine is to wait for the EC2 metdata service to + come up. On both Eucalyptus and EC2 we have seen the case where + the instance hit the MD before the MD service was up. EC2 seems + to have permenantely fixed this, though. + + In openstack, the metadata service might be painfully slow, and + unable to avoid hitting a timeout of even up to 10 seconds or more + (LP: #894279) for a simple GET. + + Offset those needs with the need to not hang forever (and block boot) + on a system where cloud-init is configured to look for EC2 Metadata + service but is not going to find one. It is possible that the instance + data host (169.254.169.254) may be firewalled off Entirely for a sytem, + meaning that the connection will block forever unless a timeout is set. + """ + starttime = time.time() + + def nullstatus_cb(msg): + return + + if status_cb is None: + status_cb = nullstatus_cb + + def timeup(max_wait, starttime): + return ((max_wait <= 0 or max_wait is None) or + (time.time() - starttime > max_wait)) + + loop_n = 0 + while True: + sleeptime = int(loop_n / 5) + 1 + for url in urls: + now = time.time() + if loop_n != 0: + if timeup(max_wait, starttime): + break + if timeout and (now + timeout > (starttime + max_wait)): + # shorten timeout to not run way over max_time + timeout = int((starttime + max_wait) - now) + + reason = "" + try: + if headers_cb is not None: + headers = headers_cb(url) + else: + headers = {} + + (resp, status_code) = readurl(url, headers=headers, timeout=timeout) + if not resp: + reason = "empty response [%s]" % status_code + elif not ok_http_code(status_code): + reason = "bad status code [%s]" % status_code + else: + return url + except urllib2.HTTPError as e: + reason = "http error [%s]" % e.code + except urllib2.URLError as e: + reason = "url error [%s]" % e.reason + except socket.timeout as e: + reason = "socket timeout [%s]" % e + except Exception as e: + reason = "unexpected error [%s]" % e + + status_cb("'%s' failed [%s/%ss]: %s" % + (url, int(time.time() - starttime), max_wait, + reason)) + + if timeup(max_wait, starttime): + break + + loop_n = loop_n + 1 + time.sleep(sleeptime) + + return False -- cgit v1.2.3 From 567a69f10fb65655f7f8bde886345600e4094e98 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 12:49:07 -0700 Subject: Add in a helper that is useful for running pep8/pylint --- checks.sh | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ pylintrc | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100755 checks.sh create mode 100644 pylintrc diff --git a/checks.sh b/checks.sh new file mode 100755 index 00000000..f54cad36 --- /dev/null +++ b/checks.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +set -u + +function find_src { + files=`find bin cloudinit -type f | grep "py\$"` + echo $files +} + +function run_pep8 { + echo "Running pep8 ..." + files=$(find_src) + ignores="E202,E501" + output_filename="pep8.log" + opts="--ignore=$ignores --repeat" + pep8 ${opts} ${files} 2>&1 > $output_filename + if [ "$?" -ne "0" ]; then + echo "Some badness was found!" + fi + echo "Check '$output_filename' for a full report." +} + +function run_pylint { + echo "Running pylint ..." + opts="--rcfile=pylintrc --output-format=parseable" + files=$(find_src) + output_filename="pylint.log" + pylint ${opts} ${files} 2>&1 > $output_filename + if [ "$?" -eq "1" ]; then + # pylint --long-help + # * 0 if everything went fine + # * 1 if a fatal message was issued + # * 2 if an error message was issued + # * 4 if a warning message was issued + # * 8 if a refactor message was issued + # * 16 if a convention message was issued + # * 32 on usage error + echo "A fatal pylint error occurred!" + else + if [ "$?" -eq "0" ]; then + echo "Your code is perfect you code master!" + else + echo "You are not yet a code master." + grep -i "Your code" $output_filename + fi + fi + echo "Check '$output_filename' for a full report." +} + + +run_pep8 +run_pylint + diff --git a/pylintrc b/pylintrc new file mode 100644 index 00000000..a7447a19 --- /dev/null +++ b/pylintrc @@ -0,0 +1,42 @@ +# The format of this file isn't really documented; just use --generate-rcfile + +[Master] + +[Messages Control] + +# http://pylint-messages.wikidot.com/all-codes +# NOTE(justinsb): We might want to have a 2nd strict pylintrc in future +# C0111: Don't require docstrings on every method +# R0912: Too many branches (huh) +# R0914: Too many local variables is odd. +# W0142: *args and **kwargs are fine. +# W0511: TODOs in code comments are fine. +# W0613: Unused argument '??' should be ok (they are useful sometimes to know intention of variable) +# W0622: Redefining id is fine. +disable=C0111,W0142,W0622,C0301,R0902,R0201,R0914,W0613,R0912,R0801 + +[Basic] + +# Variable names can be 1 to 31 characters long, with lowercase and underscores +variable-rgx=[a-z_][a-z0-9_]{0,30}$ + +# Argument names can be 2 to 31 characters long, with lowercase and underscores +argument-rgx=[a-z_][a-z0-9_]{1,30}$ + +# Method names should be at least 3 characters long +# and be lowercased with underscores +method-rgx=([a-z_][a-z0-9_]{2,50}|setUp|tearDown)$ + +# Don't require docstrings on tests. +no-docstring-rgx=((__.*__)|([tT]est.*)|setUp|tearDown)$ + +[Design] + +max-public-methods=100 +min-public-methods=0 +max-args=6 + +[Variables] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. -- cgit v1.2.3 From 32362c0ac63f10d7a33e3b95dd91a544a1cbdf54 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 13:46:54 -0700 Subject: 1. Move cloud init and cloud config objects to a cloud file. 2. Cleanup main __init__ file with shell additions, constants usage, os.path usage. --- cloudinit/__init__.py | 608 ++++-------------------------------------- cloudinit/sources/__init__.py | 214 +++++++++++++++ 2 files changed, 260 insertions(+), 562 deletions(-) diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py index 85c6fd1b..f223fbe8 100644 --- a/cloudinit/__init__.py +++ b/cloudinit/__init__.py @@ -20,627 +20,111 @@ # along with this program. If not, see . # -varlibdir = '/var/lib/cloud' -cur_instance_link = varlibdir + "/instance" -boot_finished = cur_instance_link + "/boot-finished" -system_config = '/etc/cloud/cloud.cfg' -seeddir = varlibdir + "/seed" -cfg_env_name = "CLOUD_CFG" - -cfg_builtin = """ -log_cfgs: [] -datasource_list: ["NoCloud", "ConfigDrive", "OVF", "MAAS", "Ec2", "CloudStack"] -def_log_file: /var/log/cloud-init.log -syslog_fix_perms: syslog:adm -""" -logger_name = "cloudinit" - -pathmap = { - "handlers": "/handlers", - "scripts": "/scripts", - "sem": "/sem", - "boothooks": "/boothooks", - "userdata_raw": "/user-data.txt", - "userdata": "/user-data.txt.i", - "obj_pkl": "/obj.pkl", - "cloud_config": "/cloud-config.txt", - "data": "/data", - None: "", -} - -per_instance = "once-per-instance" -per_always = "always" -per_once = "once" - -parsed_cfgs = {} - import os -import cPickle import sys import os.path import errno import subprocess import yaml -import logging -import logging.config -import StringIO import glob import traceback +import cloudinit.log as logging +import cloudinit.shell as sh import cloudinit.util as util +from cloudinit.constants import (VAR_LIB_DIR, CFG_BUILTIN, CLOUD_CONFIG, + BOOT_FINISHED, CUR_INSTANCE_LINK, PATH_MAP) -class NullHandler(logging.Handler): - def emit(self, record): - pass - - -log = logging.getLogger(logger_name) -log.addHandler(NullHandler()) - - -def logging_set_from_cfg_file(cfg_file=system_config): - logging_set_from_cfg(util.get_base_cfg(cfg_file, cfg_builtin, parsed_cfgs)) - - -def logging_set_from_cfg(cfg): - log_cfgs = [] - logcfg = util.get_cfg_option_str(cfg, "log_cfg", False) - if logcfg: - # if there is a 'logcfg' entry in the config, respect - # it, it is the old keyname - log_cfgs = [logcfg] - elif "log_cfgs" in cfg: - for cfg in cfg['log_cfgs']: - if isinstance(cfg, list): - log_cfgs.append('\n'.join(cfg)) - else: - log_cfgs.append() - - if not len(log_cfgs): - sys.stderr.write("Warning, no logging configured\n") - return - - for logcfg in log_cfgs: - try: - logging.config.fileConfig(StringIO.StringIO(logcfg)) - return - except: - pass - - raise Exception("no valid logging found\n") - - -import cloudinit.DataSource as DataSource -import cloudinit.UserDataHandler as UserDataHandler - - -class CloudInit: - cfg = None - part_handlers = {} - old_conffile = '/etc/ec2-init/ec2-config.cfg' - ds_deps = [DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK] - datasource = None - cloud_config_str = '' - datasource_name = '' - - builtin_handlers = [] - - def __init__(self, ds_deps=None, sysconfig=system_config): - self.builtin_handlers = [ - ['text/x-shellscript', self.handle_user_script, per_always], - ['text/cloud-config', self.handle_cloud_config, per_always], - ['text/upstart-job', self.handle_upstart_job, per_instance], - ['text/cloud-boothook', self.handle_cloud_boothook, per_always], - ] - - if ds_deps != None: - self.ds_deps = ds_deps - - self.sysconfig = sysconfig - - self.cfg = self.read_cfg() - - def read_cfg(self): - if self.cfg: - return(self.cfg) - - try: - conf = util.get_base_cfg(self.sysconfig, cfg_builtin, parsed_cfgs) - except Exception: - conf = get_builtin_cfg() - - # support reading the old ConfigObj format file and merging - # it into the yaml dictionary - try: - from configobj import ConfigObj - oldcfg = ConfigObj(self.old_conffile) - if oldcfg is None: - oldcfg = {} - conf = util.mergedict(conf, oldcfg) - except: - pass - - return(conf) - - def restore_from_cache(self): - try: - # we try to restore from a current link and static path - # by using the instance link, if purge_cache was called - # the file wont exist - cache = get_ipath_cur('obj_pkl') - f = open(cache, "rb") - data = cPickle.load(f) - f.close() - self.datasource = data - return True - except: - return False - - def write_to_cache(self): - cache = self.get_ipath("obj_pkl") - try: - os.makedirs(os.path.dirname(cache)) - except OSError as e: - if e.errno != errno.EEXIST: - return False - - try: - f = open(cache, "wb") - cPickle.dump(self.datasource, f) - f.close() - os.chmod(cache, 0400) - except: - raise - - def get_data_source(self): - if self.datasource is not None: - return True - - if self.restore_from_cache(): - log.debug("restored from cache type %s" % self.datasource) - return True - - cfglist = self.cfg['datasource_list'] - dslist = list_sources(cfglist, self.ds_deps) - dsnames = [f.__name__ for f in dslist] - - log.debug("searching for data source in %s" % dsnames) - for cls in dslist: - ds = cls.__name__ - try: - s = cls(sys_cfg=self.cfg) - if s.get_data(): - self.datasource = s - self.datasource_name = ds - log.debug("found data source %s" % ds) - return True - except Exception as e: - log.warn("get_data of %s raised %s" % (ds, e)) - util.logexc(log) - msg = "Did not find data source. searched classes: %s" % dsnames - log.debug(msg) - raise DataSourceNotFoundException(msg) - - def set_cur_instance(self): - try: - os.unlink(cur_instance_link) - except OSError as e: - if e.errno != errno.ENOENT: - raise - - iid = self.get_instance_id() - os.symlink("./instances/%s" % iid, cur_instance_link) - idir = self.get_ipath() - dlist = [] - for d in ["handlers", "scripts", "sem"]: - dlist.append("%s/%s" % (idir, d)) +LOG = logging.getLogger(__name__) - util.ensure_dirs(dlist) +INIT_SUBDIRS = [ + 'scripts', + os.path.join('scripts', 'per-instance'), + os.path.join('scripts', 'per-once'), + os.path.join('scripts', 'per-boot'), + 'seed', + 'instances', + 'handlers', + 'sem', + 'data' +] - ds = "%s: %s\n" % (self.datasource.__class__, str(self.datasource)) - dp = self.get_cpath('data') - util.write_file("%s/%s" % (idir, 'datasource'), ds) - util.write_file("%s/%s" % (dp, 'previous-datasource'), ds) - util.write_file("%s/%s" % (dp, 'previous-instance-id'), "%s\n" % iid) - - def get_userdata(self): - return(self.datasource.get_userdata()) - - def get_userdata_raw(self): - return(self.datasource.get_userdata_raw()) - - def get_instance_id(self): - return(self.datasource.get_instance_id()) - - def update_cache(self): - self.write_to_cache() - self.store_userdata() - - def store_userdata(self): - util.write_file(self.get_ipath('userdata_raw'), - self.datasource.get_userdata_raw(), 0600) - util.write_file(self.get_ipath('userdata'), - self.datasource.get_userdata(), 0600) - - def sem_getpath(self, name, freq): - if freq == 'once-per-instance': - return("%s/%s" % (self.get_ipath("sem"), name)) - return("%s/%s.%s" % (get_cpath("sem"), name, freq)) - - def sem_has_run(self, name, freq): - if freq == per_always: - return False - semfile = self.sem_getpath(name, freq) - if os.path.exists(semfile): - return True - return False - - def sem_acquire(self, name, freq): - from time import time - semfile = self.sem_getpath(name, freq) - - try: - os.makedirs(os.path.dirname(semfile)) - except OSError as e: - if e.errno != errno.EEXIST: - raise e - - if os.path.exists(semfile) and freq != per_always: - return False - - # race condition - try: - f = open(semfile, "w") - f.write("%s\n" % str(time())) - f.close() - except: - return(False) - return(True) - def sem_clear(self, name, freq): - semfile = self.sem_getpath(name, freq) - try: - os.unlink(semfile) - except OSError as e: - if e.errno != errno.ENOENT: - return False - - return True - - # acquire lock on 'name' for given 'freq' - # if that does not exist, then call 'func' with given 'args' - # if 'clear_on_fail' is True and func throws an exception - # then remove the lock (so it would run again) - def sem_and_run(self, semname, freq, func, args=None, clear_on_fail=False): - if args is None: - args = [] - if self.sem_has_run(semname, freq): - log.debug("%s already ran %s", semname, freq) - return False - try: - if not self.sem_acquire(semname, freq): - raise Exception("Failed to acquire lock on %s" % semname) - - func(*args) - except: - if clear_on_fail: - self.sem_clear(semname, freq) - raise - - return True - - # get_ipath : get the instance path for a name in pathmap - # (/var/lib/cloud/instances//name)) - def get_ipath(self, name=None): - return("%s/instances/%s%s" - % (varlibdir, self.get_instance_id(), pathmap[name])) - - def consume_userdata(self, frequency=per_instance): - self.get_userdata() - data = self - - cdir = get_cpath("handlers") - idir = self.get_ipath("handlers") - - # add the path to the plugins dir to the top of our list for import - # instance dir should be read before cloud-dir - sys.path.insert(0, cdir) - sys.path.insert(0, idir) - - part_handlers = {} - # add handlers in cdir - for fname in glob.glob("%s/*.py" % cdir): - if not os.path.isfile(fname): - continue - modname = os.path.basename(fname)[0:-3] - try: - mod = __import__(modname) - handler_register(mod, part_handlers, data, frequency) - log.debug("added handler for [%s] from %s" % (mod.list_types(), - fname)) - except: - log.warn("failed to initialize handler in %s" % fname) - util.logexc(log) - - # add the internal handers if their type hasn't been already claimed - for (btype, bhand, bfreq) in self.builtin_handlers: - if btype in part_handlers: - continue - handler_register(InternalPartHandler(bhand, [btype], bfreq), - part_handlers, data, frequency) - - # walk the data - pdata = {'handlers': part_handlers, 'handlerdir': idir, - 'data': data, 'frequency': frequency} - UserDataHandler.walk_userdata(self.get_userdata(), - partwalker_callback, data=pdata) - - # give callbacks opportunity to finalize - called = [] - for (_mtype, mod) in part_handlers.iteritems(): - if mod in called: - continue - handler_call_end(mod, data, frequency) - - def handle_user_script(self, _data, ctype, filename, payload, _frequency): - if ctype == "__end__": - return - if ctype == "__begin__": - # maybe delete existing things here - return - - filename = filename.replace(os.sep, '_') - scriptsdir = get_ipath_cur('scripts') - util.write_file("%s/%s" % - (scriptsdir, filename), util.dos2unix(payload), 0700) - - def handle_upstart_job(self, _data, ctype, filename, payload, frequency): - # upstart jobs are only written on the first boot - if frequency != per_instance: - return - - if ctype == "__end__" or ctype == "__begin__": - return - if not filename.endswith(".conf"): - filename = filename + ".conf" - - util.write_file("%s/%s" % ("/etc/init", filename), - util.dos2unix(payload), 0644) - - def handle_cloud_config(self, _data, ctype, filename, payload, _frequency): - if ctype == "__begin__": - self.cloud_config_str = "" - return - if ctype == "__end__": - cloud_config = self.get_ipath("cloud_config") - util.write_file(cloud_config, self.cloud_config_str, 0600) - - ## this could merge the cloud config with the system config - ## for now, not doing this as it seems somewhat circular - ## as CloudConfig does that also, merging it with this cfg - ## - # ccfg = yaml.load(self.cloud_config_str) - # if ccfg is None: ccfg = {} - # self.cfg = util.mergedict(ccfg, self.cfg) - - return - - self.cloud_config_str += "\n#%s\n%s" % (filename, payload) - - def handle_cloud_boothook(self, _data, ctype, filename, payload, - _frequency): - if ctype == "__end__": - return - if ctype == "__begin__": - return - - filename = filename.replace(os.sep, '_') - payload = util.dos2unix(payload) - prefix = "#cloud-boothook" - start = 0 - if payload.startswith(prefix): - start = len(prefix) + 1 - - boothooks_dir = self.get_ipath("boothooks") - filepath = "%s/%s" % (boothooks_dir, filename) - util.write_file(filepath, payload[start:], 0700) - try: - env = os.environ.copy() - env['INSTANCE_ID'] = self.datasource.get_instance_id() - subprocess.check_call([filepath], env=env) - except subprocess.CalledProcessError as e: - log.error("boothooks script %s returned %i" % - (filepath, e.returncode)) - except Exception as e: - log.error("boothooks unknown exception %s when running %s" % - (e, filepath)) - - def get_public_ssh_keys(self): - return(self.datasource.get_public_ssh_keys()) - - def get_locale(self): - return(self.datasource.get_locale()) - - def get_mirror(self): - return(self.datasource.get_local_mirror()) - - def get_hostname(self, fqdn=False): - return(self.datasource.get_hostname(fqdn=fqdn)) - - def device_name_to_device(self, name): - return(self.datasource.device_name_to_device(name)) - - # I really don't know if this should be here or not, but - # I needed it in cc_update_hostname, where that code had a valid 'cloud' - # reference, but did not have a cloudinit handle - # (ie, no cloudinit.get_cpath()) - def get_cpath(self, name=None): - return(get_cpath(name)) +# TODO: get rid of this global +parsed_cfgs = {} def initfs(): - subds = ['scripts/per-instance', 'scripts/per-once', 'scripts/per-boot', - 'seed', 'instances', 'handlers', 'sem', 'data'] + + # TODO don't do this every time this function is called? dlist = [] - for subd in subds: - dlist.append("%s/%s" % (varlibdir, subd)) - util.ensure_dirs(dlist) + for subd in INIT_SUBDIRS: + dlist.append(os.path.join(VAR_LIB_DIR, subd)) + sh.ensure_dirs(dlist) - cfg = util.get_base_cfg(system_config, cfg_builtin, parsed_cfgs) + cfg = util.get_base_cfg(CLOUD_CONFIG, get_builtin_cfg(), parsed_cfgs) log_file = util.get_cfg_option_str(cfg, 'def_log_file', None) perms = util.get_cfg_option_str(cfg, 'syslog_fix_perms', None) if log_file: - fp = open(log_file, "ab") - fp.close() + sh.ensure_file(log_file) if log_file and perms: (u, g) = perms.split(':', 1) if u == "-1" or u == "None": u = None if g == "-1" or g == "None": g = None - util.chownbyname(log_file, u, g) + sh.chownbyname(log_file, u, g) def purge_cache(rmcur=True): - rmlist = [boot_finished] + rmlist = [BOOT_FINISHED] if rmcur: - rmlist.append(cur_instance_link) + rmlist.append(CUR_INSTANCE_LINK) for f in rmlist: try: - os.unlink(f) + sh.unlink(f) except OSError as e: if e.errno == errno.ENOENT: continue - return(False) + return False except: - return(False) - return(True) + return False + return True # get_ipath_cur: get the current instance path for an item def get_ipath_cur(name=None): - return("%s/%s%s" % (varlibdir, "instance", pathmap[name])) + add_on = PATH_MAP.get(name) + ipath = os.path.join(VAR_LIB_DIR, 'instance') + if add_on: + ipath = os.path.join(ipath, add_on) + return ipath # get_cpath : get the "clouddir" (/var/lib/cloud/) # for a name in dirmap def get_cpath(name=None): - return("%s%s" % (varlibdir, pathmap[name])) + cpath = VAR_LIB_DIR + add_on = PATH_MAP.get(name) + if add_on: + cpath = os.path.join(cpath, add_on) + return cpath def get_base_cfg(cfg_path=None): if cfg_path is None: - cfg_path = system_config - return(util.get_base_cfg(cfg_path, cfg_builtin, parsed_cfgs)) + cfg_path = CLOUD_CONFIG + return util.get_base_cfg(cfg_path, get_builtin_cfg(), parsed_cfgs) def get_builtin_cfg(): - return(yaml.load(cfg_builtin)) - - -class DataSourceNotFoundException(Exception): - pass + return dict(CFG_BUILTIN) def list_sources(cfg_list, depends): - return(DataSource.list_sources(cfg_list, depends, ["cloudinit", ""])) - - -def handler_register(mod, part_handlers, data, frequency=per_instance): - if not hasattr(mod, "handler_version"): - setattr(mod, "handler_version", 1) - - for mtype in mod.list_types(): - part_handlers[mtype] = mod - - handler_call_begin(mod, data, frequency) - return(mod) - - -def handler_call_begin(mod, data, frequency): - handler_handle_part(mod, data, "__begin__", None, None, frequency) - - -def handler_call_end(mod, data, frequency): - handler_handle_part(mod, data, "__end__", None, None, frequency) - - -def handler_handle_part(mod, data, ctype, filename, payload, frequency): - # only add the handler if the module should run - modfreq = getattr(mod, "frequency", per_instance) - if not (modfreq == per_always or - (frequency == per_instance and modfreq == per_instance)): - return - try: - if mod.handler_version == 1: - mod.handle_part(data, ctype, filename, payload) - else: - mod.handle_part(data, ctype, filename, payload, frequency) - except: - util.logexc(log) - traceback.print_exc(file=sys.stderr) - - -def partwalker_handle_handler(pdata, _ctype, _filename, payload): - curcount = pdata['handlercount'] - modname = 'part-handler-%03d' % curcount - frequency = pdata['frequency'] - - modfname = modname + ".py" - util.write_file("%s/%s" % (pdata['handlerdir'], modfname), payload, 0600) - - try: - mod = __import__(modname) - handler_register(mod, pdata['handlers'], pdata['data'], frequency) - pdata['handlercount'] = curcount + 1 - except: - util.logexc(log) - traceback.print_exc(file=sys.stderr) - - -def partwalker_callback(pdata, ctype, filename, payload): - # data here is the part_handlers array and then the data to pass through - if ctype == "text/part-handler": - if 'handlercount' not in pdata: - pdata['handlercount'] = 0 - partwalker_handle_handler(pdata, ctype, filename, payload) - return - if ctype not in pdata['handlers']: - if ctype == "text/x-not-multipart": - # Extract the first line or 24 bytes for displaying in the log - start = payload.split("\n", 1)[0][:24] - if start < payload: - details = "starting '%s...'" % start.encode("string-escape") - else: - details = repr(payload) - log.warning("Unhandled non-multipart userdata %s", details) - return - handler_handle_part(pdata['handlers'][ctype], pdata['data'], - ctype, filename, payload, pdata['frequency']) - - -class InternalPartHandler: - freq = per_instance - mtypes = [] - handler_version = 1 - handler = None - - def __init__(self, handler, mtypes, frequency, version=2): - self.handler = handler - self.mtypes = mtypes - self.frequency = frequency - self.handler_version = version - - def __repr__(self): - return("InternalPartHandler: [%s]" % self.mtypes) - - def list_types(self): - return(self.mtypes) - - def handle_part(self, data, ctype, filename, payload, frequency): - return(self.handler(data, ctype, filename, payload, frequency)) + return (DataSource.list_sources(cfg_list, depends, ["cloudinit", ""])) def get_cmdline_url(names=('cloud-config-url', 'url'), diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index e69de29b..e2a9150d 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -0,0 +1,214 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +DEP_FILESYSTEM = "FILESYSTEM" +DEP_NETWORK = "NETWORK" + +import cloudinit.UserDataHandler as ud +import cloudinit.util as util +import socket + + +class DataSource: + userdata = None + metadata = None + userdata_raw = None + cfgname = "" + # system config (passed in from cloudinit, + # cloud-config before input from the DataSource) + sys_cfg = {} + # datasource config, the cloud-config['datasource']['__name__'] + ds_cfg = {} # datasource config + + def __init__(self, sys_cfg=None): + if not self.cfgname: + name = str(self.__class__).split(".")[-1] + if name.startswith("DataSource"): + name = name[len("DataSource"):] + self.cfgname = name + if sys_cfg: + self.sys_cfg = sys_cfg + + self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, + ("datasource", self.cfgname), self.ds_cfg) + + def get_userdata(self): + if self.userdata == None: + self.userdata = ud.preprocess_userdata(self.userdata_raw) + return self.userdata + + def get_userdata_raw(self): + return(self.userdata_raw) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return({}) + + def get_public_ssh_keys(self): + keys = [] + if 'public-keys' not in self.metadata: + return([]) + + if isinstance(self.metadata['public-keys'], str): + return(str(self.metadata['public-keys']).splitlines()) + + if isinstance(self.metadata['public-keys'], list): + return(self.metadata['public-keys']) + + for _keyname, klist in self.metadata['public-keys'].items(): + # lp:506332 uec metadata service responds with + # data that makes boto populate a string for 'klist' rather + # than a list. + if isinstance(klist, str): + klist = [klist] + for pkey in klist: + # there is an empty string at the end of the keylist, trim it + if pkey: + keys.append(pkey) + + return(keys) + + def device_name_to_device(self, _name): + # translate a 'name' to a device + # the primary function at this point is on ec2 + # to consult metadata service, that has + # ephemeral0: sdb + # and return 'sdb' for input 'ephemeral0' + return(None) + + def get_locale(self): + return('en_US.UTF-8') + + def get_local_mirror(self): + return None + + def get_instance_id(self): + if 'instance-id' not in self.metadata: + return "iid-datasource" + return(self.metadata['instance-id']) + + def get_hostname(self, fqdn=False): + defdomain = "localdomain" + defhost = "localhost" + + domain = defdomain + if not 'local-hostname' in self.metadata: + + # this is somewhat questionable really. + # the cloud datasource was asked for a hostname + # and didn't have one. raising error might be more appropriate + # but instead, basically look up the existing hostname + toks = [] + + hostname = socket.gethostname() + + fqdn = util.get_fqdn_from_hosts(hostname) + + if fqdn and fqdn.find(".") > 0: + toks = str(fqdn).split(".") + elif hostname: + toks = [hostname, defdomain] + else: + toks = [defhost, defdomain] + + else: + # if there is an ipv4 address in 'local-hostname', then + # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx + lhost = self.metadata['local-hostname'] + if is_ipv4(lhost): + toks = "ip-%s" % lhost.replace(".", "-") + else: + toks = lhost.split(".") + + if len(toks) > 1: + hostname = toks[0] + domain = '.'.join(toks[1:]) + else: + hostname = toks[0] + + if fqdn: + return "%s.%s" % (hostname, domain) + else: + return hostname + + +# return a list of classes that have the same depends as 'depends' +# iterate through cfg_list, loading "DataSourceCollections" modules +# and calling their "get_datasource_list". +# return an ordered list of classes that match +# +# - modules must be named "DataSource", where 'item' is an entry +# in cfg_list +# - if pkglist is given, it will iterate try loading from that package +# ie, pkglist=[ "foo", "" ] +# will first try to load foo.DataSource +# then DataSource +def list_sources(cfg_list, depends, pkglist=None): + if pkglist is None: + pkglist = [] + retlist = [] + for ds_coll in cfg_list: + for pkg in pkglist: + if pkg: + pkg = "%s." % pkg + try: + mod = __import__("%sDataSource%s" % (pkg, ds_coll)) + if pkg: + mod = getattr(mod, "DataSource%s" % ds_coll) + lister = getattr(mod, "get_datasource_list") + retlist.extend(lister(depends)) + break + except: + raise + return(retlist) + + +# depends is a list of dependencies (DEP_FILESYSTEM) +# dslist is a list of 2 item lists +# dslist = [ +# ( class, ( depends-that-this-class-needs ) ) +# } +# it returns a list of 'class' that matched these deps exactly +# it is a helper function for DataSourceCollections +def list_from_depends(depends, dslist): + retlist = [] + depset = set(depends) + for elem in dslist: + (cls, deps) = elem + if depset == set(deps): + retlist.append(cls) + return(retlist) + + +def is_ipv4(instr): + """ determine if input string is a ipv4 address. return boolean""" + toks = instr.split('.') + if len(toks) != 4: + return False + + try: + toks = [x for x in toks if (int(x) < 256 and int(x) > 0)] + except: + return False + + return (len(toks) == 4) -- cgit v1.2.3 From 0383be9602699e163896195dd6b4c4ac515db460 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 13:47:47 -0700 Subject: Move the contents of this file to __init__ in sources. --- cloudinit/sources/DataSource.py | 214 ---------------------------------------- 1 file changed, 214 deletions(-) delete mode 100644 cloudinit/sources/DataSource.py diff --git a/cloudinit/sources/DataSource.py b/cloudinit/sources/DataSource.py deleted file mode 100644 index e2a9150d..00000000 --- a/cloudinit/sources/DataSource.py +++ /dev/null @@ -1,214 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -DEP_FILESYSTEM = "FILESYSTEM" -DEP_NETWORK = "NETWORK" - -import cloudinit.UserDataHandler as ud -import cloudinit.util as util -import socket - - -class DataSource: - userdata = None - metadata = None - userdata_raw = None - cfgname = "" - # system config (passed in from cloudinit, - # cloud-config before input from the DataSource) - sys_cfg = {} - # datasource config, the cloud-config['datasource']['__name__'] - ds_cfg = {} # datasource config - - def __init__(self, sys_cfg=None): - if not self.cfgname: - name = str(self.__class__).split(".")[-1] - if name.startswith("DataSource"): - name = name[len("DataSource"):] - self.cfgname = name - if sys_cfg: - self.sys_cfg = sys_cfg - - self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, - ("datasource", self.cfgname), self.ds_cfg) - - def get_userdata(self): - if self.userdata == None: - self.userdata = ud.preprocess_userdata(self.userdata_raw) - return self.userdata - - def get_userdata_raw(self): - return(self.userdata_raw) - - # the data sources' config_obj is a cloud-config formated - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return({}) - - def get_public_ssh_keys(self): - keys = [] - if 'public-keys' not in self.metadata: - return([]) - - if isinstance(self.metadata['public-keys'], str): - return(str(self.metadata['public-keys']).splitlines()) - - if isinstance(self.metadata['public-keys'], list): - return(self.metadata['public-keys']) - - for _keyname, klist in self.metadata['public-keys'].items(): - # lp:506332 uec metadata service responds with - # data that makes boto populate a string for 'klist' rather - # than a list. - if isinstance(klist, str): - klist = [klist] - for pkey in klist: - # there is an empty string at the end of the keylist, trim it - if pkey: - keys.append(pkey) - - return(keys) - - def device_name_to_device(self, _name): - # translate a 'name' to a device - # the primary function at this point is on ec2 - # to consult metadata service, that has - # ephemeral0: sdb - # and return 'sdb' for input 'ephemeral0' - return(None) - - def get_locale(self): - return('en_US.UTF-8') - - def get_local_mirror(self): - return None - - def get_instance_id(self): - if 'instance-id' not in self.metadata: - return "iid-datasource" - return(self.metadata['instance-id']) - - def get_hostname(self, fqdn=False): - defdomain = "localdomain" - defhost = "localhost" - - domain = defdomain - if not 'local-hostname' in self.metadata: - - # this is somewhat questionable really. - # the cloud datasource was asked for a hostname - # and didn't have one. raising error might be more appropriate - # but instead, basically look up the existing hostname - toks = [] - - hostname = socket.gethostname() - - fqdn = util.get_fqdn_from_hosts(hostname) - - if fqdn and fqdn.find(".") > 0: - toks = str(fqdn).split(".") - elif hostname: - toks = [hostname, defdomain] - else: - toks = [defhost, defdomain] - - else: - # if there is an ipv4 address in 'local-hostname', then - # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx - lhost = self.metadata['local-hostname'] - if is_ipv4(lhost): - toks = "ip-%s" % lhost.replace(".", "-") - else: - toks = lhost.split(".") - - if len(toks) > 1: - hostname = toks[0] - domain = '.'.join(toks[1:]) - else: - hostname = toks[0] - - if fqdn: - return "%s.%s" % (hostname, domain) - else: - return hostname - - -# return a list of classes that have the same depends as 'depends' -# iterate through cfg_list, loading "DataSourceCollections" modules -# and calling their "get_datasource_list". -# return an ordered list of classes that match -# -# - modules must be named "DataSource", where 'item' is an entry -# in cfg_list -# - if pkglist is given, it will iterate try loading from that package -# ie, pkglist=[ "foo", "" ] -# will first try to load foo.DataSource -# then DataSource -def list_sources(cfg_list, depends, pkglist=None): - if pkglist is None: - pkglist = [] - retlist = [] - for ds_coll in cfg_list: - for pkg in pkglist: - if pkg: - pkg = "%s." % pkg - try: - mod = __import__("%sDataSource%s" % (pkg, ds_coll)) - if pkg: - mod = getattr(mod, "DataSource%s" % ds_coll) - lister = getattr(mod, "get_datasource_list") - retlist.extend(lister(depends)) - break - except: - raise - return(retlist) - - -# depends is a list of dependencies (DEP_FILESYSTEM) -# dslist is a list of 2 item lists -# dslist = [ -# ( class, ( depends-that-this-class-needs ) ) -# } -# it returns a list of 'class' that matched these deps exactly -# it is a helper function for DataSourceCollections -def list_from_depends(depends, dslist): - retlist = [] - depset = set(depends) - for elem in dslist: - (cls, deps) = elem - if depset == set(deps): - retlist.append(cls) - return(retlist) - - -def is_ipv4(instr): - """ determine if input string is a ipv4 address. return boolean""" - toks = instr.split('.') - if len(toks) != 4: - return False - - try: - toks = [x for x in toks if (int(x) < 256 and int(x) > 0)] - except: - return False - - return (len(toks) == 4) -- cgit v1.2.3 From 7adde5f221ab3e8d79d3cf053b374c541a75fbf1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 13:48:30 -0700 Subject: Start moving code from there to here. --- cloudinit/handling.py | 63 ++++++++++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/cloudinit/handling.py b/cloudinit/handling.py index 553abe4f..8f6424e3 100644 --- a/cloudinit/handling.py +++ b/cloudinit/handling.py @@ -7,29 +7,7 @@ from cloudinit.constants import (PER_INSTANCE, PER_ALWAYS) LOG = logging.getLogger(__name__) -class InternalPartHandler: - freq = PER_INSTANCE - mtypes = [] - handler_version = 1 - handler = None - - def __init__(self, handler, mtypes, frequency, version=2): - self.handler = handler - self.mtypes = mtypes - self.frequency = frequency - self.handler_version = version - - def __repr__(self): - return("InternalPartHandler: [%s]" % self.mtypes) - - def list_types(self): - return(self.mtypes) - - def handle_part(self, data, ctype, filename, payload, frequency): - return(self.handler(data, ctype, filename, payload, frequency)) - - -def handler_register(mod, part_handlers, data, frequency=PER_INSTANCE): +def handler_register(mod, part_handlers, data, frequency=per_instance): if not hasattr(mod, "handler_version"): setattr(mod, "handler_version", 1) @@ -37,7 +15,7 @@ def handler_register(mod, part_handlers, data, frequency=PER_INSTANCE): part_handlers[mtype] = mod handler_call_begin(mod, data, frequency) - return mod + return(mod) def handler_call_begin(mod, data, frequency): @@ -50,9 +28,9 @@ def handler_call_end(mod, data, frequency): def handler_handle_part(mod, data, ctype, filename, payload, frequency): # only add the handler if the module should run - modfreq = getattr(mod, "frequency", PER_INSTANCE) - if not (modfreq == PER_ALWAYS or - (frequency == PER_INSTANCE and modfreq == PER_INSTANCE)): + modfreq = getattr(mod, "frequency", per_instance) + if not (modfreq == per_always or + (frequency == per_instance and modfreq == per_instance)): return try: if mod.handler_version == 1: @@ -70,14 +48,15 @@ def partwalker_handle_handler(pdata, _ctype, _filename, payload): frequency = pdata['frequency'] modfname = modname + ".py" - util.write_file(os.path.join(pdata['handlerdir'], modfname), payload, 0600) + util.write_file("%s/%s" % (pdata['handlerdir'], modfname), payload, 0600) try: - mod = importer.import_module(modname) + mod = __import__(modname) handler_register(mod, pdata['handlers'], pdata['data'], frequency) pdata['handlercount'] = curcount + 1 except: - LOG.exception("Could not import module %s", modname) + util.logexc(log) + traceback.print_exc(file=sys.stderr) def partwalker_callback(pdata, ctype, filename, payload): @@ -95,7 +74,29 @@ def partwalker_callback(pdata, ctype, filename, payload): details = "starting '%s...'" % start.encode("string-escape") else: details = repr(payload) - LOG.warning("Unhandled non-multipart userdata %s", details) + log.warning("Unhandled non-multipart userdata %s", details) return handler_handle_part(pdata['handlers'][ctype], pdata['data'], ctype, filename, payload, pdata['frequency']) + + +class InternalPartHandler: + freq = per_instance + mtypes = [] + handler_version = 1 + handler = None + + def __init__(self, handler, mtypes, frequency, version=2): + self.handler = handler + self.mtypes = mtypes + self.frequency = frequency + self.handler_version = version + + def __repr__(self): + return("InternalPartHandler: [%s]" % self.mtypes) + + def list_types(self): + return(self.mtypes) + + def handle_part(self, data, ctype, filename, payload, frequency): + return(self.handler(data, ctype, filename, payload, frequency)) -- cgit v1.2.3 From 9aa8060d463c8cdefa6f8b381584838794d4c509 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 13:48:49 -0700 Subject: Add this file which will contain the cloud init and cloud config classes. --- cloudinit/cloud.py | 368 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 368 insertions(+) create mode 100644 cloudinit/cloud.py diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py new file mode 100644 index 00000000..eb71439b --- /dev/null +++ b/cloudinit/cloud.py @@ -0,0 +1,368 @@ +import os + +import cPickle as pickle + + +class CloudInit(object): + cfg = None + part_handlers = {} + old_conffile = '/etc/ec2-init/ec2-config.cfg' + ds_deps = [DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK] + datasource = None + cloud_config_str = '' + datasource_name = '' + + builtin_handlers = [] + + def __init__(self, ds_deps=None, sysconfig=system_config): + self.builtin_handlers = [ + ['text/x-shellscript', self.handle_user_script, per_always], + ['text/cloud-config', self.handle_cloud_config, per_always], + ['text/upstart-job', self.handle_upstart_job, per_instance], + ['text/cloud-boothook', self.handle_cloud_boothook, per_always], + ] + + if ds_deps != None: + self.ds_deps = ds_deps + + self.sysconfig = sysconfig + + self.cfg = self.read_cfg() + + def read_cfg(self): + if self.cfg: + return(self.cfg) + + try: + conf = util.get_base_cfg(self.sysconfig, cfg_builtin, parsed_cfgs) + except Exception: + conf = get_builtin_cfg() + + # support reading the old ConfigObj format file and merging + # it into the yaml dictionary + try: + from configobj import ConfigObj + oldcfg = ConfigObj(self.old_conffile) + if oldcfg is None: + oldcfg = {} + conf = util.mergedict(conf, oldcfg) + except: + pass + + return(conf) + + def restore_from_cache(self): + try: + # we try to restore from a current link and static path + # by using the instance link, if purge_cache was called + # the file wont exist + cache = get_ipath_cur('obj_pkl') + f = open(cache, "rb") + data = cPickle.load(f) + f.close() + self.datasource = data + return True + except: + return False + + def write_to_cache(self): + cache = self.get_ipath("obj_pkl") + try: + os.makedirs(os.path.dirname(cache)) + except OSError as e: + if e.errno != errno.EEXIST: + return False + + try: + f = open(cache, "wb") + cPickle.dump(self.datasource, f) + f.close() + os.chmod(cache, 0400) + except: + raise + + def get_data_source(self): + if self.datasource is not None: + return True + + if self.restore_from_cache(): + log.debug("restored from cache type %s" % self.datasource) + return True + + cfglist = self.cfg['datasource_list'] + dslist = list_sources(cfglist, self.ds_deps) + dsnames = [f.__name__ for f in dslist] + + log.debug("searching for data source in %s" % dsnames) + for cls in dslist: + ds = cls.__name__ + try: + s = cls(sys_cfg=self.cfg) + if s.get_data(): + self.datasource = s + self.datasource_name = ds + log.debug("found data source %s" % ds) + return True + except Exception as e: + log.warn("get_data of %s raised %s" % (ds, e)) + util.logexc(log) + msg = "Did not find data source. searched classes: %s" % dsnames + log.debug(msg) + raise DataSourceNotFoundException(msg) + + def set_cur_instance(self): + try: + os.unlink(cur_instance_link) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + iid = self.get_instance_id() + os.symlink("./instances/%s" % iid, cur_instance_link) + idir = self.get_ipath() + dlist = [] + for d in ["handlers", "scripts", "sem"]: + dlist.append("%s/%s" % (idir, d)) + + util.ensure_dirs(dlist) + + ds = "%s: %s\n" % (self.datasource.__class__, str(self.datasource)) + dp = self.get_cpath('data') + util.write_file("%s/%s" % (idir, 'datasource'), ds) + util.write_file("%s/%s" % (dp, 'previous-datasource'), ds) + util.write_file("%s/%s" % (dp, 'previous-instance-id'), "%s\n" % iid) + + def get_userdata(self): + return(self.datasource.get_userdata()) + + def get_userdata_raw(self): + return(self.datasource.get_userdata_raw()) + + def get_instance_id(self): + return(self.datasource.get_instance_id()) + + def update_cache(self): + self.write_to_cache() + self.store_userdata() + + def store_userdata(self): + util.write_file(self.get_ipath('userdata_raw'), + self.datasource.get_userdata_raw(), 0600) + util.write_file(self.get_ipath('userdata'), + self.datasource.get_userdata(), 0600) + + def sem_getpath(self, name, freq): + if freq == 'once-per-instance': + return("%s/%s" % (self.get_ipath("sem"), name)) + return("%s/%s.%s" % (get_cpath("sem"), name, freq)) + + def sem_has_run(self, name, freq): + if freq == per_always: + return False + semfile = self.sem_getpath(name, freq) + if os.path.exists(semfile): + return True + return False + + def sem_acquire(self, name, freq): + from time import time + semfile = self.sem_getpath(name, freq) + + try: + os.makedirs(os.path.dirname(semfile)) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + + if os.path.exists(semfile) and freq != per_always: + return False + + # race condition + try: + f = open(semfile, "w") + f.write("%s\n" % str(time())) + f.close() + except: + return(False) + return(True) + + def sem_clear(self, name, freq): + semfile = self.sem_getpath(name, freq) + try: + os.unlink(semfile) + except OSError as e: + if e.errno != errno.ENOENT: + return False + + return True + + # acquire lock on 'name' for given 'freq' + # if that does not exist, then call 'func' with given 'args' + # if 'clear_on_fail' is True and func throws an exception + # then remove the lock (so it would run again) + def sem_and_run(self, semname, freq, func, args=None, clear_on_fail=False): + if args is None: + args = [] + if self.sem_has_run(semname, freq): + log.debug("%s already ran %s", semname, freq) + return False + try: + if not self.sem_acquire(semname, freq): + raise Exception("Failed to acquire lock on %s" % semname) + + func(*args) + except: + if clear_on_fail: + self.sem_clear(semname, freq) + raise + + return True + + # get_ipath : get the instance path for a name in pathmap + # (/var/lib/cloud/instances//name)) + def get_ipath(self, name=None): + return("%s/instances/%s%s" + % (varlibdir, self.get_instance_id(), pathmap[name])) + + def consume_userdata(self, frequency=per_instance): + self.get_userdata() + data = self + + cdir = get_cpath("handlers") + idir = self.get_ipath("handlers") + + # add the path to the plugins dir to the top of our list for import + # instance dir should be read before cloud-dir + sys.path.insert(0, cdir) + sys.path.insert(0, idir) + + part_handlers = {} + # add handlers in cdir + for fname in glob.glob("%s/*.py" % cdir): + if not os.path.isfile(fname): + continue + modname = os.path.basename(fname)[0:-3] + try: + mod = __import__(modname) + handler_register(mod, part_handlers, data, frequency) + log.debug("added handler for [%s] from %s" % (mod.list_types(), + fname)) + except: + log.warn("failed to initialize handler in %s" % fname) + util.logexc(log) + + # add the internal handers if their type hasn't been already claimed + for (btype, bhand, bfreq) in self.builtin_handlers: + if btype in part_handlers: + continue + handler_register(InternalPartHandler(bhand, [btype], bfreq), + part_handlers, data, frequency) + + # walk the data + pdata = {'handlers': part_handlers, 'handlerdir': idir, + 'data': data, 'frequency': frequency} + UserDataHandler.walk_userdata(self.get_userdata(), + partwalker_callback, data=pdata) + + # give callbacks opportunity to finalize + called = [] + for (_mtype, mod) in part_handlers.iteritems(): + if mod in called: + continue + handler_call_end(mod, data, frequency) + + def handle_user_script(self, _data, ctype, filename, payload, _frequency): + if ctype == "__end__": + return + if ctype == "__begin__": + # maybe delete existing things here + return + + filename = filename.replace(os.sep, '_') + scriptsdir = get_ipath_cur('scripts') + util.write_file("%s/%s" % + (scriptsdir, filename), util.dos2unix(payload), 0700) + + def handle_upstart_job(self, _data, ctype, filename, payload, frequency): + # upstart jobs are only written on the first boot + if frequency != per_instance: + return + + if ctype == "__end__" or ctype == "__begin__": + return + if not filename.endswith(".conf"): + filename = filename + ".conf" + + util.write_file("%s/%s" % ("/etc/init", filename), + util.dos2unix(payload), 0644) + + def handle_cloud_config(self, _data, ctype, filename, payload, _frequency): + if ctype == "__begin__": + self.cloud_config_str = "" + return + if ctype == "__end__": + cloud_config = self.get_ipath("cloud_config") + util.write_file(cloud_config, self.cloud_config_str, 0600) + + ## this could merge the cloud config with the system config + ## for now, not doing this as it seems somewhat circular + ## as CloudConfig does that also, merging it with this cfg + ## + # ccfg = yaml.load(self.cloud_config_str) + # if ccfg is None: ccfg = {} + # self.cfg = util.mergedict(ccfg, self.cfg) + + return + + self.cloud_config_str += "\n#%s\n%s" % (filename, payload) + + def handle_cloud_boothook(self, _data, ctype, filename, payload, + _frequency): + if ctype == "__end__": + return + if ctype == "__begin__": + return + + filename = filename.replace(os.sep, '_') + payload = util.dos2unix(payload) + prefix = "#cloud-boothook" + start = 0 + if payload.startswith(prefix): + start = len(prefix) + 1 + + boothooks_dir = self.get_ipath("boothooks") + filepath = "%s/%s" % (boothooks_dir, filename) + util.write_file(filepath, payload[start:], 0700) + try: + env = os.environ.copy() + env['INSTANCE_ID'] = self.datasource.get_instance_id() + subprocess.check_call([filepath], env=env) + except subprocess.CalledProcessError as e: + log.error("boothooks script %s returned %i" % + (filepath, e.returncode)) + except Exception as e: + log.error("boothooks unknown exception %s when running %s" % + (e, filepath)) + + def get_public_ssh_keys(self): + return(self.datasource.get_public_ssh_keys()) + + def get_locale(self): + return(self.datasource.get_locale()) + + def get_mirror(self): + return(self.datasource.get_local_mirror()) + + def get_hostname(self, fqdn=False): + return(self.datasource.get_hostname(fqdn=fqdn)) + + def device_name_to_device(self, name): + return(self.datasource.device_name_to_device(name)) + + # I really don't know if this should be here or not, but + # I needed it in cc_update_hostname, where that code had a valid 'cloud' + # reference, but did not have a cloudinit handle + # (ie, no cloudinit.get_cpath()) + def get_cpath(self, name=None): + return(get_cpath(name)) + -- cgit v1.2.3 From 8b71365df508dbcad52f7fb85ecf777e5cec324d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 13:49:07 -0700 Subject: Add in a utility class that will handle all shell-like actions. --- cloudinit/shell.py | 183 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 cloudinit/shell.py diff --git a/cloudinit/shell.py b/cloudinit/shell.py new file mode 100644 index 00000000..17dae817 --- /dev/null +++ b/cloudinit/shell.py @@ -0,0 +1,183 @@ +import os +import shutil +import subprocess + +from StringIO import StringIO + +from cloudinit import exceptions as excp +from cloudinit import log as logging + +try: + import selinux + HAVE_LIBSELINUX = True +except ImportError: + HAVE_LIBSELINUX = False + + +LOG = logging.getLogger(__name__) + + +class SeLinuxGuard(object): + def __init__(self, path, recursive=False): + self.path = path + self.recursive = recursive + self.engaged = False + if HAVE_LIBSELINUX and selinux.is_selinux_enabled(): + self.engaged = True + + def __enter__(self): + return self.engaged + + def __exit__(self, type, value, traceback): + if self.engaged: + LOG.debug("Disengaging selinux mode for %s: %s", self.path, self.recursive) + selinux.restorecon(self.path, recursive=self.recursive) + + +def read_file(fname, read_cb=None): + LOG.debug("Reading from %s", fname) + with open(fname, 'rb') as fh: + ofh = StringIO() + pipe_in_out(fh, ofh, chunk_cb=read_cb) + return ofh.getvalue() + + +def pipe_in_out(in_fh, out_fh, chunk_size=1024, chunk_cb=None): + bytes_piped = 0 + LOG.debug("Transferring the contents of %s to %s in chunks of size %s.", in_fh, out_fh, chunk_size) + while True: + data = in_fh.read(chunk_size) + if data == '': + break + else: + out_fh.write(data) + bytes_piped += len(data) + if chunk_cb: + chunk_cb(bytes_piped) + out_fh.flush() + return bytes_piped + + +def chownbyname(fname, user=None, group=None): + uid = -1 + gid = -1 + if user == None and group == None: + return + if user: + # TODO: why is this late imported + import pwd + uid = pwd.getpwnam(user).pw_uid + if group: + # TODO: why is this late imported + import grp + gid = grp.getgrnam(group).gr_gid + + os.chown(fname, uid, gid) + + +def ensure_dirs(dirlist, mode=0755): + for d in dirlist: + ensure_dir(d, mode) + + +def ensure_dir(path, mode=0755): + if not os.path.isdir(path): + fixmodes = [] + LOG.debug("Ensuring directory exists at path %s", dir_name) + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + if mode is not None: + os.chmod(path, mode) + +def del_file(path): + LOG.debug("Attempting to remove %s", path) + os.unlink(path) + + +def ensure_file(path): + if not os.path.isfile(path): + write_file(path, content='') + + +def write_file(filename, content, mode=0644, omode="wb"): + """ + Writes a file with the given content and sets the file mode as specified. + Resotres the SELinux context if possible. + + @param filename: The full path of the file to write. + @param content: The content to write to the file. + @param mode: The filesystem mode to set on the file. + @param omode: The open mode used when opening the file (r, rb, a, etc.) + """ + try: + os.makedirs(os.path.dirname(filename)) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + + LOG.debug("Writing to %s (%o) %s bytes", filename, mode, len(content)) + with open(filename, omode) as fh: + with SeLinuxGuard(filename): + fh.write(content) + if mode is not None: + os.chmod(filename, mode) + +def delete_dir_contents(dirname): + """ + Deletes all contents of a directory without deleting the directory itself. + + @param dirname: The directory whose contents should be deleted. + """ + for node in os.listdir(dirname): + node_fullpath = os.path.join(dirname, node) + if os.path.isdir(node_fullpath): + shutil.rmtree(node_fullpath) + else: + os.unlink(node_fullpath) + + +def subp(args, input_data=None, allowed_rc=None): + if allowed_rc is None: + allowed_rc = [0] + try: + sp = subprocess.Popen(args, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (out, err) = sp.communicate(input_data) + except OSError as e: + raise excp.ProcessExecutionError(cmd=args, reason=e) + rc = sp.returncode + if rc not in allowed_rc: + raise excp.ProcessExecutionError(stdout=out, stderr=err, + exit_code=rc, + cmd=args) + # Just ensure blank instead of none?? + if not out: + out = '' + if not err: + err = '' + return (out, err) + + +# shellify, takes a list of commands +# for each entry in the list +# if it is an array, shell protect it (with single ticks) +# if it is a string, do nothing +def shellify(cmdlist, add_header=True): + content = '' + if add_header: + content += "#!/bin/sh\n" + escaped = "%s%s%s%s" % ("'", '\\', "'", "'") + for args in cmdlist: + # if the item is a list, wrap all items in single tick + # if its not, then just write it directly + if isinstance(args, list): + fixed = [] + for f in args: + fixed.append("'%s'" % str(f).replace("'", escaped)) + content = "%s%s\n" % (content, ' '.join(fixed)) + else: + content = "%s%s\n" % (content, str(args)) + return content -- cgit v1.2.3 From c29abc03def42008c22df2f43fca3e43f62e3adb Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 13:50:38 -0700 Subject: Add a standard exception holding file. --- cloudinit/exceptions.py | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 cloudinit/exceptions.py diff --git a/cloudinit/exceptions.py b/cloudinit/exceptions.py new file mode 100644 index 00000000..235ded7a --- /dev/null +++ b/cloudinit/exceptions.py @@ -0,0 +1,62 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + + +class ProcessExecutionError(IOError): + + message_tmpl = ('%(description)s\nCommand: %(cmd)s\n' + 'Exit code: %(exit_code)s\nStdout: %(stdout)r\n' + 'Stderr: %(stderr)r') + + def __init__(self, stdout=None, stderr=None, + exit_code=None, cmd=None, + description=None, reason=None): + if not cmd: + self.cmd = '-' + else: + self.cmd = cmd + + if not description: + self.description = 'Unexpected error while running command.' + else: + self.description = description + + if not isinstance(exit_code, (long, int)): + self.exit_code = '-' + else: + self.exit_code = exit_code + + if not stderr: + self.stderr = '' + else: + self.stderr = stderr + + if not stdout: + self.stdout = '' + else: + self.stdout = stdout + + message = self.message_tmpl % { + 'description': self.description, + 'cmd': self.cmd, + 'exit_code': self.exit_code, + 'stdout': self.stdout, + 'stderr': self.stderr, + } + IOError.__init__(self, message) + self.reason = reason + + +class MountFailedError(Exception): + pass + + +class StackExceeded(Exception): + pass + + +class RecursiveInclude(Exception): + pass + + +class DataSourceNotFoundException(Exception): + pass -- cgit v1.2.3 From 5767a1d20ee49137f51b033e09cf2a754b57ef71 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 17:53:42 -0700 Subject: Moving exceptions to modules where they are used. Seems to make sense. --- cloudinit/exceptions.py | 62 ------------------------------------------------- 1 file changed, 62 deletions(-) delete mode 100644 cloudinit/exceptions.py diff --git a/cloudinit/exceptions.py b/cloudinit/exceptions.py deleted file mode 100644 index 235ded7a..00000000 --- a/cloudinit/exceptions.py +++ /dev/null @@ -1,62 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - - -class ProcessExecutionError(IOError): - - message_tmpl = ('%(description)s\nCommand: %(cmd)s\n' - 'Exit code: %(exit_code)s\nStdout: %(stdout)r\n' - 'Stderr: %(stderr)r') - - def __init__(self, stdout=None, stderr=None, - exit_code=None, cmd=None, - description=None, reason=None): - if not cmd: - self.cmd = '-' - else: - self.cmd = cmd - - if not description: - self.description = 'Unexpected error while running command.' - else: - self.description = description - - if not isinstance(exit_code, (long, int)): - self.exit_code = '-' - else: - self.exit_code = exit_code - - if not stderr: - self.stderr = '' - else: - self.stderr = stderr - - if not stdout: - self.stdout = '' - else: - self.stdout = stdout - - message = self.message_tmpl % { - 'description': self.description, - 'cmd': self.cmd, - 'exit_code': self.exit_code, - 'stdout': self.stdout, - 'stderr': self.stderr, - } - IOError.__init__(self, message) - self.reason = reason - - -class MountFailedError(Exception): - pass - - -class StackExceeded(Exception): - pass - - -class RecursiveInclude(Exception): - pass - - -class DataSourceNotFoundException(Exception): - pass -- cgit v1.2.3 From d0a477cd6eb66b201363898f35cb8eecc3559759 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 17:54:21 -0700 Subject: Moved this to a file called parts which handles this type of functionality. --- cloudinit/handling.py | 102 -------------------------------------------------- 1 file changed, 102 deletions(-) delete mode 100644 cloudinit/handling.py diff --git a/cloudinit/handling.py b/cloudinit/handling.py deleted file mode 100644 index 8f6424e3..00000000 --- a/cloudinit/handling.py +++ /dev/null @@ -1,102 +0,0 @@ -import os - -from cloudinit import importer -from cloudinit import log as logging -from cloudinit.constants import (PER_INSTANCE, PER_ALWAYS) - -LOG = logging.getLogger(__name__) - - -def handler_register(mod, part_handlers, data, frequency=per_instance): - if not hasattr(mod, "handler_version"): - setattr(mod, "handler_version", 1) - - for mtype in mod.list_types(): - part_handlers[mtype] = mod - - handler_call_begin(mod, data, frequency) - return(mod) - - -def handler_call_begin(mod, data, frequency): - handler_handle_part(mod, data, "__begin__", None, None, frequency) - - -def handler_call_end(mod, data, frequency): - handler_handle_part(mod, data, "__end__", None, None, frequency) - - -def handler_handle_part(mod, data, ctype, filename, payload, frequency): - # only add the handler if the module should run - modfreq = getattr(mod, "frequency", per_instance) - if not (modfreq == per_always or - (frequency == per_instance and modfreq == per_instance)): - return - try: - if mod.handler_version == 1: - mod.handle_part(data, ctype, filename, payload) - else: - mod.handle_part(data, ctype, filename, payload, frequency) - except: - util.logexc(log) - traceback.print_exc(file=sys.stderr) - - -def partwalker_handle_handler(pdata, _ctype, _filename, payload): - curcount = pdata['handlercount'] - modname = 'part-handler-%03d' % curcount - frequency = pdata['frequency'] - - modfname = modname + ".py" - util.write_file("%s/%s" % (pdata['handlerdir'], modfname), payload, 0600) - - try: - mod = __import__(modname) - handler_register(mod, pdata['handlers'], pdata['data'], frequency) - pdata['handlercount'] = curcount + 1 - except: - util.logexc(log) - traceback.print_exc(file=sys.stderr) - - -def partwalker_callback(pdata, ctype, filename, payload): - # data here is the part_handlers array and then the data to pass through - if ctype == "text/part-handler": - if 'handlercount' not in pdata: - pdata['handlercount'] = 0 - partwalker_handle_handler(pdata, ctype, filename, payload) - return - if ctype not in pdata['handlers']: - if ctype == "text/x-not-multipart": - # Extract the first line or 24 bytes for displaying in the log - start = payload.split("\n", 1)[0][:24] - if start < payload: - details = "starting '%s...'" % start.encode("string-escape") - else: - details = repr(payload) - log.warning("Unhandled non-multipart userdata %s", details) - return - handler_handle_part(pdata['handlers'][ctype], pdata['data'], - ctype, filename, payload, pdata['frequency']) - - -class InternalPartHandler: - freq = per_instance - mtypes = [] - handler_version = 1 - handler = None - - def __init__(self, handler, mtypes, frequency, version=2): - self.handler = handler - self.mtypes = mtypes - self.frequency = frequency - self.handler_version = version - - def __repr__(self): - return("InternalPartHandler: [%s]" % self.mtypes) - - def list_types(self): - return(self.mtypes) - - def handle_part(self, data, ctype, filename, payload, frequency): - return(self.handler(data, ctype, filename, payload, frequency)) -- cgit v1.2.3 From d3e6be27c8a2cd89baa545c1116442d3d5d5b53a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 17:54:45 -0700 Subject: Seems like the util function extraction was not actually needed. --- cloudinit/includer.py | 65 --------------------------------------------------- 1 file changed, 65 deletions(-) delete mode 100644 cloudinit/includer.py diff --git a/cloudinit/includer.py b/cloudinit/includer.py deleted file mode 100644 index d1022c5a..00000000 --- a/cloudinit/includer.py +++ /dev/null @@ -1,65 +0,0 @@ -import os -import re - -from cloudinit import downloader as down -from cloudinit import exceptions as excp -from cloudinit import log as logging -from cloudinit import shell as sh - -INCLUDE_PATT = re.compile("^#(opt_include|include)[ \t](.*)$", re.MULTILINE) -OPT_PATS = ['opt_include'] - -LOG = logging.getLogger(__name__) - - -class Includer(object): - - def __init__(self, root_fn, stack_limit=10): - self.root_fn = root_fn - self.stack_limit = stack_limit - - def _read_file(self, fname): - return sh.read_file(fname) - - def _read(self, fname, stack, rel): - if len(stack) >= self.stack_limit: - raise excp.StackExceeded("Stack limit of %s reached while including %s" % (self.stack_limit, fname)) - - canon_fname = self._canon_name(fname, rel) - if canon_fname in stack: - raise excp.RecursiveInclude("File %s recursively included" % (canon_fname)) - - stack.add(canon_fname) - new_rel = os.path.dirname(canon_fname) - contents = self._read_file(canon_fname) - - def include_cb(match): - is_optional = (match.group(1).lower() in OPT_PATS) - fn = match.group(2).strip() - if not fn: - # Should we die?? - return match.group(0) - else: - try: - LOG.debug("Including file %s", fn) - return self._read(fn, stack, new_rel) - except IOError: - if is_optional: - return '' - else: - raise - - adjusted_contents = INCLUDE_PATT.sub(include_cb, contents) - stack.remove(fname) - return adjusted_contents - - def _canon_name(self, fname, rel): - fname = fname.strip() - if not fname.startswith("/"): - fname = os.path.sep.join([rel, fname]) - return os.path.realpath(fname) - - def read(self, relative_to="."): - stack = set() - return self._read(self.root_fn, stack, rel=relative_to) - -- cgit v1.2.3 From 14676dd74dbd14ea501bd7ff6875dc5ca54edc4b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 17:55:02 -0700 Subject: Putting back into utils for now (until that gets to big). --- cloudinit/shell.py | 183 ----------------------------------------------------- 1 file changed, 183 deletions(-) delete mode 100644 cloudinit/shell.py diff --git a/cloudinit/shell.py b/cloudinit/shell.py deleted file mode 100644 index 17dae817..00000000 --- a/cloudinit/shell.py +++ /dev/null @@ -1,183 +0,0 @@ -import os -import shutil -import subprocess - -from StringIO import StringIO - -from cloudinit import exceptions as excp -from cloudinit import log as logging - -try: - import selinux - HAVE_LIBSELINUX = True -except ImportError: - HAVE_LIBSELINUX = False - - -LOG = logging.getLogger(__name__) - - -class SeLinuxGuard(object): - def __init__(self, path, recursive=False): - self.path = path - self.recursive = recursive - self.engaged = False - if HAVE_LIBSELINUX and selinux.is_selinux_enabled(): - self.engaged = True - - def __enter__(self): - return self.engaged - - def __exit__(self, type, value, traceback): - if self.engaged: - LOG.debug("Disengaging selinux mode for %s: %s", self.path, self.recursive) - selinux.restorecon(self.path, recursive=self.recursive) - - -def read_file(fname, read_cb=None): - LOG.debug("Reading from %s", fname) - with open(fname, 'rb') as fh: - ofh = StringIO() - pipe_in_out(fh, ofh, chunk_cb=read_cb) - return ofh.getvalue() - - -def pipe_in_out(in_fh, out_fh, chunk_size=1024, chunk_cb=None): - bytes_piped = 0 - LOG.debug("Transferring the contents of %s to %s in chunks of size %s.", in_fh, out_fh, chunk_size) - while True: - data = in_fh.read(chunk_size) - if data == '': - break - else: - out_fh.write(data) - bytes_piped += len(data) - if chunk_cb: - chunk_cb(bytes_piped) - out_fh.flush() - return bytes_piped - - -def chownbyname(fname, user=None, group=None): - uid = -1 - gid = -1 - if user == None and group == None: - return - if user: - # TODO: why is this late imported - import pwd - uid = pwd.getpwnam(user).pw_uid - if group: - # TODO: why is this late imported - import grp - gid = grp.getgrnam(group).gr_gid - - os.chown(fname, uid, gid) - - -def ensure_dirs(dirlist, mode=0755): - for d in dirlist: - ensure_dir(d, mode) - - -def ensure_dir(path, mode=0755): - if not os.path.isdir(path): - fixmodes = [] - LOG.debug("Ensuring directory exists at path %s", dir_name) - try: - os.makedirs(path) - except OSError as e: - if e.errno != errno.EEXIST: - raise e - if mode is not None: - os.chmod(path, mode) - -def del_file(path): - LOG.debug("Attempting to remove %s", path) - os.unlink(path) - - -def ensure_file(path): - if not os.path.isfile(path): - write_file(path, content='') - - -def write_file(filename, content, mode=0644, omode="wb"): - """ - Writes a file with the given content and sets the file mode as specified. - Resotres the SELinux context if possible. - - @param filename: The full path of the file to write. - @param content: The content to write to the file. - @param mode: The filesystem mode to set on the file. - @param omode: The open mode used when opening the file (r, rb, a, etc.) - """ - try: - os.makedirs(os.path.dirname(filename)) - except OSError as e: - if e.errno != errno.EEXIST: - raise e - - LOG.debug("Writing to %s (%o) %s bytes", filename, mode, len(content)) - with open(filename, omode) as fh: - with SeLinuxGuard(filename): - fh.write(content) - if mode is not None: - os.chmod(filename, mode) - -def delete_dir_contents(dirname): - """ - Deletes all contents of a directory without deleting the directory itself. - - @param dirname: The directory whose contents should be deleted. - """ - for node in os.listdir(dirname): - node_fullpath = os.path.join(dirname, node) - if os.path.isdir(node_fullpath): - shutil.rmtree(node_fullpath) - else: - os.unlink(node_fullpath) - - -def subp(args, input_data=None, allowed_rc=None): - if allowed_rc is None: - allowed_rc = [0] - try: - sp = subprocess.Popen(args, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, stdin=subprocess.PIPE) - (out, err) = sp.communicate(input_data) - except OSError as e: - raise excp.ProcessExecutionError(cmd=args, reason=e) - rc = sp.returncode - if rc not in allowed_rc: - raise excp.ProcessExecutionError(stdout=out, stderr=err, - exit_code=rc, - cmd=args) - # Just ensure blank instead of none?? - if not out: - out = '' - if not err: - err = '' - return (out, err) - - -# shellify, takes a list of commands -# for each entry in the list -# if it is an array, shell protect it (with single ticks) -# if it is a string, do nothing -def shellify(cmdlist, add_header=True): - content = '' - if add_header: - content += "#!/bin/sh\n" - escaped = "%s%s%s%s" % ("'", '\\', "'", "'") - for args in cmdlist: - # if the item is a list, wrap all items in single tick - # if its not, then just write it directly - if isinstance(args, list): - fixed = [] - for f in args: - fixed.append("'%s'" % str(f).replace("'", escaped)) - content = "%s%s\n" % (content, ' '.join(fixed)) - else: - content = "%s%s\n" % (content, str(args)) - return content -- cgit v1.2.3 From 52f6d8473d514a2a1eef86f3ec892d9f873c6da5 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 17:55:23 -0700 Subject: Renamed this, since its more of settings, not all constants. --- cloudinit/constants.py | 37 ------------------------------------- cloudinit/settings.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 37 deletions(-) delete mode 100644 cloudinit/constants.py create mode 100644 cloudinit/settings.py diff --git a/cloudinit/constants.py b/cloudinit/constants.py deleted file mode 100644 index 7bc90f27..00000000 --- a/cloudinit/constants.py +++ /dev/null @@ -1,37 +0,0 @@ -import os - -VAR_LIB_DIR = '/var/lib/cloud' -CUR_INSTANCE_LINK = os.path.join(VAR_LIB_DIR, "instance") -BOOT_FINISHED = os.path.join(CUR_INSTANCE_LINK, "boot-finished") -SEED_DIR = os.path.join(VAR_LIB_DIR, "seed") - -CFG_ENV_NAME = "CLOUD_CFG" -CLOUD_CONFIG = '/etc/cloud/cloud.cfg' - -CFG_BUILTIN = { - 'datasource_list': ['NoCloud', - 'ConfigDrive', - 'OVF', - 'MAAS', - 'Ec2', - 'CloudStack'], - 'def_log_file': '/var/log/cloud-init.log', - 'log_cfgs': [], - 'syslog_fix_perms': 'syslog:adm' -} - -PATH_MAP = { - "handlers": "handlers", - "scripts": "scripts", - "sem": "sem", - "boothooks": "boothooks", - "userdata_raw": "user-data.txt", - "userdata": "user-data.txt.i", - "obj_pkl": "obj.pkl", - "cloud_config": "cloud-config.txt", - "data": "data", -} - -PER_INSTANCE = "once-per-instance" -PER_ALWAYS = "always" -PER_ONCE = "once" diff --git a/cloudinit/settings.py b/cloudinit/settings.py new file mode 100644 index 00000000..830d970d --- /dev/null +++ b/cloudinit/settings.py @@ -0,0 +1,40 @@ +import os + +VAR_LIB_DIR = '/var/lib/cloud' +CUR_INSTANCE_LINK = os.path.join(VAR_LIB_DIR, "instance") +BOOT_FINISHED = os.path.join(CUR_INSTANCE_LINK, "boot-finished") +SEED_DIR = os.path.join(VAR_LIB_DIR, "seed") + +CFG_ENV_NAME = "CLOUD_CFG" +CLOUD_CONFIG = '/etc/cloud/cloud.cfg' +OLD_CLOUD_CONFIG = '/etc/ec2-init/ec2-config.cfg' + +CFG_BUILTIN = { + 'datasource_list': ['NoCloud', + 'ConfigDrive', + 'OVF', + 'MAAS', + 'Ec2', + 'CloudStack'], + 'def_log_file': '/var/log/cloud-init.log', + 'log_cfgs': [], + 'syslog_fix_perms': 'syslog:adm' +} + +PATH_MAP = { + "handlers": "handlers", + "scripts": "scripts", + "sem": "sem", + "boothooks": "boothooks", + "userdata_raw": "user-data.txt", + "userdata": "user-data.txt.i", + "obj_pkl": "obj.pkl", + "cloud_config": "cloud-config.txt", + "data": "data", +} + +PER_INSTANCE = "once-per-instance" +PER_ALWAYS = "always" +PER_ONCE = "once" + +TEMPLATE_DIR = '/etc/cloud/templates/' -- cgit v1.2.3 From 3e6217b64ba6436af991e52163fea850f21bb770 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 17:55:52 -0700 Subject: This now only holds some simple top level functions. TODO: maybe just move them to utils? --- cloudinit/__init__.py | 53 +++++++++++++-------------------------------------- 1 file changed, 13 insertions(+), 40 deletions(-) diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py index f223fbe8..2c291ff5 100644 --- a/cloudinit/__init__.py +++ b/cloudinit/__init__.py @@ -23,18 +23,17 @@ import os import sys -import os.path import errno import subprocess import yaml import glob -import traceback -import cloudinit.log as logging -import cloudinit.shell as sh -import cloudinit.util as util +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import url_helper as uhelp +from cloudinit import util -from cloudinit.constants import (VAR_LIB_DIR, CFG_BUILTIN, CLOUD_CONFIG, +from cloudinit.settings import (VAR_LIB_DIR, CFG_BUILTIN, CLOUD_CONFIG, BOOT_FINISHED, CUR_INSTANCE_LINK, PATH_MAP) LOG = logging.getLogger(__name__) @@ -62,20 +61,20 @@ def initfs(): dlist = [] for subd in INIT_SUBDIRS: dlist.append(os.path.join(VAR_LIB_DIR, subd)) - sh.ensure_dirs(dlist) + util.ensure_dirs(dlist) cfg = util.get_base_cfg(CLOUD_CONFIG, get_builtin_cfg(), parsed_cfgs) log_file = util.get_cfg_option_str(cfg, 'def_log_file', None) perms = util.get_cfg_option_str(cfg, 'syslog_fix_perms', None) if log_file: - sh.ensure_file(log_file) + util.ensure_file(log_file) if log_file and perms: (u, g) = perms.split(':', 1) if u == "-1" or u == "None": u = None if g == "-1" or g == "None": g = None - sh.chownbyname(log_file, u, g) + util.chownbyname(log_file, u, g) def purge_cache(rmcur=True): @@ -83,36 +82,10 @@ def purge_cache(rmcur=True): if rmcur: rmlist.append(CUR_INSTANCE_LINK) for f in rmlist: - try: - sh.unlink(f) - except OSError as e: - if e.errno == errno.ENOENT: - continue - return False - except: - return False + util.unlink(f) return True -# get_ipath_cur: get the current instance path for an item -def get_ipath_cur(name=None): - add_on = PATH_MAP.get(name) - ipath = os.path.join(VAR_LIB_DIR, 'instance') - if add_on: - ipath = os.path.join(ipath, add_on) - return ipath - - -# get_cpath : get the "clouddir" (/var/lib/cloud/) -# for a name in dirmap -def get_cpath(name=None): - cpath = VAR_LIB_DIR - add_on = PATH_MAP.get(name) - if add_on: - cpath = os.path.join(cpath, add_on) - return cpath - - def get_base_cfg(cfg_path=None): if cfg_path is None: cfg_path = CLOUD_CONFIG @@ -124,7 +97,7 @@ def get_builtin_cfg(): def list_sources(cfg_list, depends): - return (DataSource.list_sources(cfg_list, depends, ["cloudinit", ""])) + return (sources.list_sources(cfg_list, depends, ["cloudinit", ""])) def get_cmdline_url(names=('cloud-config-url', 'url'), @@ -140,11 +113,11 @@ def get_cmdline_url(names=('cloud-config-url', 'url'), if key in data: url = data[key] break - if url == None: - return (None, None, None) - contents = util.readurl(url) + if url is None: + return (None, None, None) + contents = uhelp.readurl(url) if contents.startswith(starts): return (key, url, contents) -- cgit v1.2.3 From e389ecb7af6387de477b1a50e48044b51e65a98c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 17:56:15 -0700 Subject: This now holds the following classes: CloudInit - cut up to only provide some basic init processes CloudPartData - provided to handlers so that they can fetch needed data without providing the whole enchilda of cloud init. CloudPaths - holds the paths that should be used, for instances, for non-instances and such. CloudSemaphores - holds the concept of cloud inits sempaphores, but cleaned up, using context manager to help here. CloudHandlers - holds the user data handlers to be activated CloudConfig - the cloud config object (to be cleaned up) --- cloudinit/cloud.py | 675 +++++++++++++++++++++++++++++------------------------ 1 file changed, 372 insertions(+), 303 deletions(-) diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index eb71439b..cfb1c011 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -1,368 +1,437 @@ -import os +from time import time import cPickle as pickle +import contextlib +import os +import sys +import weakref -class CloudInit(object): - cfg = None - part_handlers = {} - old_conffile = '/etc/ec2-init/ec2-config.cfg' - ds_deps = [DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK] - datasource = None - cloud_config_str = '' - datasource_name = '' - - builtin_handlers = [] - - def __init__(self, ds_deps=None, sysconfig=system_config): - self.builtin_handlers = [ - ['text/x-shellscript', self.handle_user_script, per_always], - ['text/cloud-config', self.handle_cloud_config, per_always], - ['text/upstart-job', self.handle_upstart_job, per_instance], - ['text/cloud-boothook', self.handle_cloud_boothook, per_always], - ] - - if ds_deps != None: - self.ds_deps = ds_deps +from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, + OLD_CLOUD_CONFIG, CLOUD_CONFIG, + CFG_BUILTIN, CUR_INSTANCE_LINK) +from cloudinit import (get_builtin_cfg, get_base_cfg) +from cloudinit import log as logging +from cloudinit import parts +from cloudinit import sources +from cloudinit import util +from cloudinit import user_data - self.sysconfig = sysconfig +LOG = logging.getLogger(__name__) - self.cfg = self.read_cfg() - def read_cfg(self): - if self.cfg: - return(self.cfg) +class CloudSemaphores(object): + def __init__(self, paths): + self.paths = paths - try: - conf = util.get_base_cfg(self.sysconfig, cfg_builtin, parsed_cfgs) - except Exception: - conf = get_builtin_cfg() + # acquire lock on 'name' for given 'freq' and run function 'func' + # if 'clear_on_fail' is True and 'func' throws an exception + # then remove the lock (so it would run again) + def run_functor(self, name, freq, functor, args=None, clear_on_fail=False): + if not args: + args = [] + if self.has_run(name, freq): + LOG.debug("%s already ran %s", name, freq) + return False + with self.lock(name, freq, clear_on_fail) as lock: + if not lock: + raise RuntimeError("Failed to acquire lock on %s" % name) + else: + LOG.debug("Running %s with args %s using lock %s", func, args, lock) + func(*args) + return True - # support reading the old ConfigObj format file and merging - # it into the yaml dictionary + @contextlib.contextmanager + def lock(self, name, freq, clear_on_fail=False): try: - from configobj import ConfigObj - oldcfg = ConfigObj(self.old_conffile) - if oldcfg is None: - oldcfg = {} - conf = util.mergedict(conf, oldcfg) + yield self._acquire(name, freq) except: - pass - - return(conf) + if clear_on_fail: + self.clear(name, freq) + raise - def restore_from_cache(self): + def clear(self, name, freq): + sem_file = self._getpath(name, freq) try: - # we try to restore from a current link and static path - # by using the instance link, if purge_cache was called - # the file wont exist - cache = get_ipath_cur('obj_pkl') - f = open(cache, "rb") - data = cPickle.load(f) - f.close() - self.datasource = data - return True - except: + util.del_file(sem_file) + except IOError: return False + return True - def write_to_cache(self): - cache = self.get_ipath("obj_pkl") - try: - os.makedirs(os.path.dirname(cache)) - except OSError as e: - if e.errno != errno.EEXIST: - return False - + def _acquire(self, name, freq): + if self.has_run(name, freq): + return None + # This is a race condition since nothing atomic is happening + # here, but this should be ok due to the nature of when + # and where cloud-init runs... (file writing is not a lock..) + sem_file = self._getpath(name, freq) + contents = "%s\n" % str(time()) try: - f = open(cache, "wb") - cPickle.dump(self.datasource, f) - f.close() - os.chmod(cache, 0400) - except: - raise + util.write_file(sem_file, contents) + except (IOError, OSError): + return None + return sem_file - def get_data_source(self): - if self.datasource is not None: + def has_run(self, name, freq): + if freq == PER_ALWAYS: + return False + sem_file = self._get_path(name, freq) + if os.path.exists(sem_file): return True + return False - if self.restore_from_cache(): - log.debug("restored from cache type %s" % self.datasource) - return True + def _get_path(self, name, freq): + sem_path = self.init.get_ipath("sem") + if freq == PER_INSTANCE: + return os.path.join(sem_path, name) + return os.path.join(sem_path, "%s.%s" % (name, freq)) + + +class CloudPaths(object): + def __init__(self, init): + self.config = CLOUD_CONFIG + self.old_config = OLD_CLOUD_CONFIG + self.var_dir = VAR_LIB_DIR + self.instance_link = CUR_INSTANCE_LINK + self.init = weakref.proxy(init) + self.upstart_conf_d = "/etc/init" + + def _get_path_key(self, name): + return PATH_MAP.get(name) + + # get_ipath_cur: get the current instance path for an item + def get_ipath_cur(self, name=None): + add_on = self._get_path_key(name) + ipath = os.path.join(self.var_dir, 'instance') + if add_on: + ipath = os.path.join(ipath, add_on) + return ipath + + # get_cpath : get the "clouddir" (/var/lib/cloud/) + # for a name in dirmap + def get_cpath(self, name=None): + cpath = self.var_dir + add_on = self._get_path_key(name) + if add_on: + cpath = os.path.join(cpath, add_on) + return cpath - cfglist = self.cfg['datasource_list'] - dslist = list_sources(cfglist, self.ds_deps) - dsnames = [f.__name__ for f in dslist] + # get_ipath : get the instance path for a name in pathmap + # (/var/lib/cloud/instances//) + def get_ipath(self, name=None): + iid = self.init.datasource.get_instance_id() + ipath = os.path.join(self.var_dir, 'instances', iid) + add_on = self._get_path_key(name) + if add_on: + ipath = os.path.join(ipath, add_on) + return ipath - log.debug("searching for data source in %s" % dsnames) - for cls in dslist: - ds = cls.__name__ - try: - s = cls(sys_cfg=self.cfg) - if s.get_data(): - self.datasource = s - self.datasource_name = ds - log.debug("found data source %s" % ds) - return True - except Exception as e: - log.warn("get_data of %s raised %s" % (ds, e)) - util.logexc(log) - msg = "Did not find data source. searched classes: %s" % dsnames - log.debug(msg) - raise DataSourceNotFoundException(msg) - def set_cur_instance(self): - try: - os.unlink(cur_instance_link) - except OSError as e: - if e.errno != errno.ENOENT: - raise - - iid = self.get_instance_id() - os.symlink("./instances/%s" % iid, cur_instance_link) - idir = self.get_ipath() - dlist = [] - for d in ["handlers", "scripts", "sem"]: - dlist.append("%s/%s" % (idir, d)) +class CloudPartData(object): + def __init__(self, datasource, paths): + self.datasource = datasource + self.paths = paths - util.ensure_dirs(dlist) + def get_userdata(self): + return self.datasource.get_userdata() - ds = "%s: %s\n" % (self.datasource.__class__, str(self.datasource)) - dp = self.get_cpath('data') - util.write_file("%s/%s" % (idir, 'datasource'), ds) - util.write_file("%s/%s" % (dp, 'previous-datasource'), ds) - util.write_file("%s/%s" % (dp, 'previous-instance-id'), "%s\n" % iid) + def get_public_ssh_keys(self): + return self.datasource.get_public_ssh_keys() - def get_userdata(self): - return(self.datasource.get_userdata()) + def get_locale(self): + return self.datasource.get_locale() - def get_userdata_raw(self): - return(self.datasource.get_userdata_raw()) + def get_mirror(self): + return self.datasource.get_local_mirror() - def get_instance_id(self): - return(self.datasource.get_instance_id()) + def get_hostname(self, fqdn=False): + return self.datasource.get_hostname(fqdn=fqdn) - def update_cache(self): - self.write_to_cache() - self.store_userdata() + def device_name_to_device(self, name): + return self.datasource.device_name_to_device(name) - def store_userdata(self): - util.write_file(self.get_ipath('userdata_raw'), - self.datasource.get_userdata_raw(), 0600) - util.write_file(self.get_ipath('userdata'), - self.datasource.get_userdata(), 0600) - - def sem_getpath(self, name, freq): - if freq == 'once-per-instance': - return("%s/%s" % (self.get_ipath("sem"), name)) - return("%s/%s.%s" % (get_cpath("sem"), name, freq)) - - def sem_has_run(self, name, freq): - if freq == per_always: - return False - semfile = self.sem_getpath(name, freq) - if os.path.exists(semfile): - return True - return False + def get_ipath_cur(self, name=None): + return self.paths.get_ipath_cur(name) - def sem_acquire(self, name, freq): - from time import time - semfile = self.sem_getpath(name, freq) + def get_cpath(self, name=None): + return self.paths.get_cpath(name) - try: - os.makedirs(os.path.dirname(semfile)) - except OSError as e: - if e.errno != errno.EEXIST: - raise e + def get_ipath(self, name=None): + return self.paths.get_ipath(name) - if os.path.exists(semfile) and freq != per_always: - return False - # race condition - try: - f = open(semfile, "w") - f.write("%s\n" % str(time())) - f.close() - except: - return(False) - return(True) +class CloudInit(object): + def __init__(self, ds_deps=None): + self.datasource = None + if ds_deps: + self.ds_deps = ds_deps + else: + self.ds_deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] + self.paths = CloudPaths(self) + self.sems = CloudSemaphores(self.paths) + self.cfg = self._read_cfg() - def sem_clear(self, name, freq): - semfile = self.sem_getpath(name, freq) + def _read_cfg_old(self): + # support reading the old ConfigObj format file and merging + # it into the yaml dictionary try: - os.unlink(semfile) - except OSError as e: - if e.errno != errno.ENOENT: - return False + from configobj import ConfigObj + except ImportError: + ConfigObj = None + if not ConfigObj: + return {} + old_cfg = ConfigObj(self.paths.old_config_fn) + return dict(old_cfg) - return True + def read_cfg(self): + if not self.cfg: + self.cfg = self._read_cfg() + return self.cfg - # acquire lock on 'name' for given 'freq' - # if that does not exist, then call 'func' with given 'args' - # if 'clear_on_fail' is True and func throws an exception - # then remove the lock (so it would run again) - def sem_and_run(self, semname, freq, func, args=None, clear_on_fail=False): - if args is None: - args = [] - if self.sem_has_run(semname, freq): - log.debug("%s already ran %s", semname, freq) + def _read_cfg(self): + starting_config = get_builtin_cfg() + try: + conf = get_base_cfg(self.paths.config, starting_config) + except Exception: + conf = starting_config + old_conf = self._read_cfg_old() + conf = util.mergedict(conf, old_conf) + return conf + + def restore_from_cache(self): + pickled_fn = self.paths.get_ipath_cur('obj_pkl') + try: + # we try to restore from a current link and static path + # by using the instance link, if purge_cache was called + # the file wont exist + self.datasource = pickle.loads(util.load_file(pickled_fn)) + return True + except Exception as e: + LOG.debug("Failed loading pickled datasource from %s due to %s", pickled_fn, e) return False + + def write_to_cache(self): + pickled_fn = self.paths.get_ipath_cur("obj_pkl") try: - if not self.sem_acquire(semname, freq): - raise Exception("Failed to acquire lock on %s" % semname) - - func(*args) - except: - if clear_on_fail: - self.sem_clear(semname, freq) - raise - + contents = pickle.dumps(self.datasource) + util.write_file(pickled_fn, contents, mode=0400) + except Exception as e: + LOG.debug("Failed pickling datasource to %s due to %s", pickled_fn, e) + return False + + def get_data_source(self): + if self.datasource: + return True + if self.restore_from_cache(): + LOG.debug("Restored from cache datasource: %s" % self.datasource) + return True + (ds, dsname) = sources.find_source(self.cfg, self.ds_deps) + LOG.debug("Loaded datasource %s:%s", dsname, ds) + self.datasource = ds return True + + def set_cur_instance(self): + # Ensure we are hooked into the right symlink for the current instance + idir = self.paths.get_ipath() + util.del_file(self.paths.instance_link) + util.sym_link(idir, self.paths.instance_link) - # get_ipath : get the instance path for a name in pathmap - # (/var/lib/cloud/instances//name)) - def get_ipath(self, name=None): - return("%s/instances/%s%s" - % (varlibdir, self.get_instance_id(), pathmap[name])) + dlist = [] + for d in ["handlers", "scripts", "sem"]: + dlist.append(os.path.join(idir, d)) + util.ensure_dirs(dlist) - def consume_userdata(self, frequency=per_instance): - self.get_userdata() - data = self + # Write out information on what is being used for the current instance + # and what may have been used for a previous instance... + dp = self.paths.get_cpath('data') + ds = "%s: %s\n" % (self.datasource.__class__, self.datasource) + previous_ds = '' + ds_fn = os.path.join(idir, 'datasource') + try: + previous_ds = util.load_file(ds_fn).strip() + except IOError as e: + pass + if not previous_ds: + # TODO: ?? is this right + previous_ds = ds + util.write_file(ds_fn, ds) + util.write_file(os.path.join(dp, 'previous-datasource'), previous_ds) + iid = self.datasource.get_instance_id() + previous_iid = '' + p_iid_fn = os.path.join(dp, 'previous-instance-id') + try: + previous_iid = util.load_file(p_iid_fn).strip() + except IOError as e: + pass + if not previous_iid: + # TODO: ?? is this right + previous_iid = iid + util.write_file(p_iid_fn, "%s\n" % previous_iid) - cdir = get_cpath("handlers") - idir = self.get_ipath("handlers") + def update_cache(self): + self.write_to_cache() + self.store_userdata() - # add the path to the plugins dir to the top of our list for import + def store_userdata(self): + raw_ud = "%s" % (self.datasource.get_userdata_raw()) + util.write_file(self.paths.get_ipath('userdata_raw'), raw_ud, 0600) + ud = "%s" % (self.datasource.get_userdata()) + util.write_file(self.paths.get_ipath('userdata'), ud, 0600) + + def consume_userdata(self, frequency=PER_INSTANCE): + cdir = self.paths.get_cpath("handlers") + idir = self.paths.get_ipath("handlers") + + # Add the path to the plugins dir to the top of our list for import # instance dir should be read before cloud-dir sys.path.insert(0, cdir) sys.path.insert(0, idir) - part_handlers = {} - # add handlers in cdir - for fname in glob.glob("%s/*.py" % cdir): + # Data will be a little proxy that modules can use + data = CloudPartData(self.datasource, self.paths) + + # This keeps track of all the active handlers + handlers = CloudHandlers(self) + + # Add handlers in cdir + for fname in glob.glob(os.path.join(cdir, "*.py")): if not os.path.isfile(fname): continue modname = os.path.basename(fname)[0:-3] try: - mod = __import__(modname) - handler_register(mod, part_handlers, data, frequency) - log.debug("added handler for [%s] from %s" % (mod.list_types(), - fname)) + mod = parts.fixup_module(importer.import_module(modname)) + types = handlers.register(mod) + LOG.debug("Added handler for [%s] from %s", types, fname) except: - log.warn("failed to initialize handler in %s" % fname) - util.logexc(log) - - # add the internal handers if their type hasn't been already claimed - for (btype, bhand, bfreq) in self.builtin_handlers: - if btype in part_handlers: - continue - handler_register(InternalPartHandler(bhand, [btype], bfreq), - part_handlers, data, frequency) + LOG.exception("Failed to register handler in %s", fname) - # walk the data - pdata = {'handlers': part_handlers, 'handlerdir': idir, - 'data': data, 'frequency': frequency} - UserDataHandler.walk_userdata(self.get_userdata(), - partwalker_callback, data=pdata) + def_handlers = handlers.register_defaults() + if def_handlers: + LOG.debug("Registered default handlers for [%s]", def_handlers) - # give callbacks opportunity to finalize + # Init the handlers first + # Ensure userdata fetched before activation called = [] - for (_mtype, mod) in part_handlers.iteritems(): + for (_mtype, mod) in handlers.iteritems(): if mod in called: continue - handler_call_end(mod, data, frequency) - - def handle_user_script(self, _data, ctype, filename, payload, _frequency): - if ctype == "__end__": - return - if ctype == "__begin__": - # maybe delete existing things here - return - - filename = filename.replace(os.sep, '_') - scriptsdir = get_ipath_cur('scripts') - util.write_file("%s/%s" % - (scriptsdir, filename), util.dos2unix(payload), 0700) - - def handle_upstart_job(self, _data, ctype, filename, payload, frequency): - # upstart jobs are only written on the first boot - if frequency != per_instance: - return - - if ctype == "__end__" or ctype == "__begin__": - return - if not filename.endswith(".conf"): - filename = filename + ".conf" - - util.write_file("%s/%s" % ("/etc/init", filename), - util.dos2unix(payload), 0644) - - def handle_cloud_config(self, _data, ctype, filename, payload, _frequency): - if ctype == "__begin__": - self.cloud_config_str = "" - return - if ctype == "__end__": - cloud_config = self.get_ipath("cloud_config") - util.write_file(cloud_config, self.cloud_config_str, 0600) - - ## this could merge the cloud config with the system config - ## for now, not doing this as it seems somewhat circular - ## as CloudConfig does that also, merging it with this cfg - ## - # ccfg = yaml.load(self.cloud_config_str) - # if ccfg is None: ccfg = {} - # self.cfg = util.mergedict(ccfg, self.cfg) - - return - - self.cloud_config_str += "\n#%s\n%s" % (filename, payload) - - def handle_cloud_boothook(self, _data, ctype, filename, payload, - _frequency): - if ctype == "__end__": - return - if ctype == "__begin__": - return - - filename = filename.replace(os.sep, '_') - payload = util.dos2unix(payload) - prefix = "#cloud-boothook" - start = 0 - if payload.startswith(prefix): - start = len(prefix) + 1 - - boothooks_dir = self.get_ipath("boothooks") - filepath = "%s/%s" % (boothooks_dir, filename) - util.write_file(filepath, payload[start:], 0700) - try: - env = os.environ.copy() - env['INSTANCE_ID'] = self.datasource.get_instance_id() - subprocess.check_call([filepath], env=env) - except subprocess.CalledProcessError as e: - log.error("boothooks script %s returned %i" % - (filepath, e.returncode)) - except Exception as e: - log.error("boothooks unknown exception %s when running %s" % - (e, filepath)) + parts.call_begin(mod, data, frequency) + called.append(mod) + + # Walk the user data + part_data = { + 'handlers': handlers, + 'handlerdir': idir, + 'data': data, + 'frequency': frequency, + 'handlercount': 0, + } + user_data.walk(data.get_userdata(), parts.walker_callback, data=part_data) + + # Give callbacks opportunity to finalize + called = [] + for (_mtype, mod) in handlers.iteritems(): + if mod in called: + continue + parts.call_end(mod, data, frequency) + called.append(mod) - def get_public_ssh_keys(self): - return(self.datasource.get_public_ssh_keys()) - def get_locale(self): - return(self.datasource.get_locale()) +class CloudHandlers(object): - def get_mirror(self): - return(self.datasource.get_local_mirror()) + def __init__(self, paths): + self.paths = paths + self.registered = {} - def get_hostname(self, fqdn=False): - return(self.datasource.get_hostname(fqdn=fqdn)) + def __contains__(self, item): + return self.is_registered(item) - def device_name_to_device(self, name): - return(self.datasource.device_name_to_device(name)) + def __getitem__(self, key): + return self._get_handler(key) - # I really don't know if this should be here or not, but - # I needed it in cc_update_hostname, where that code had a valid 'cloud' - # reference, but did not have a cloudinit handle - # (ie, no cloudinit.get_cpath()) - def get_cpath(self, name=None): - return(get_cpath(name)) + def is_registered(self, content_type): + return content_type in self.registered + + def register(self, mod): + types = set() + for t in mod.list_types(): + self.registered[t] = handler + types.add(t) + return types + + def _get_handler(self, content_type): + return self.registered[content_type] + + def items(self): + return self.registered.items() + + def iteritems(self): + return self.registered.iteritems() + def _get_default_handlers(self): + def_handlers = [] + if self.paths.get_ipath("cloud_config"): + def_handlers.append(parts.CloudConfigPartHandler(self.paths.get_ipath("cloud_config"))) + if self.paths.get_ipath_cur('scripts'): + def_handlers.append(parts.ShellScriptPartHandler(self.paths.get_ipath_cur('scripts'))) + if self.paths.get_ipath("boothooks"): + def_handlers.append(parts.BootHookPartHandler(self.paths.get_ipath("boothooks"))) + if self.paths.upstart_conf_d: + def_handlers.append(parts.UpstartJobPartHandler(self.paths.upstart_conf_d)) + return def_handlers + + def register_defaults(self): + registered = set() + for h in self._get_default_handlers(): + for t in h.list_types(): + if not self.is_registered(t) + self.register_handler(t, h) + registered.add(t) + return registered + + +class CloudConfig(object): + cfgfile = None + cfg = None + + def __init__(self, cfgfile, cloud=None, ds_deps=None): + if cloud == None: + self.cloud = cloudinit.CloudInit(ds_deps) + self.cloud.get_data_source() + else: + self.cloud = cloud + self.cfg = self.get_config_obj(cfgfile) + + def get_config_obj(self, cfgfile): + try: + cfg = util.read_conf(cfgfile) + except: + # TODO: this 'log' could/should be passed in + cloudinit.log.critical("Failed loading of cloud config '%s'. " + "Continuing with empty config\n" % cfgfile) + cloudinit.log.debug(traceback.format_exc() + "\n") + cfg = None + if cfg is None: + cfg = {} + + try: + ds_cfg = self.cloud.datasource.get_config_obj() + except: + ds_cfg = {} + + cfg = util.mergedict(cfg, ds_cfg) + return(util.mergedict(cfg, self.cloud.cfg)) + + def handle(self, name, args, freq=None): + try: + mod = __import__("cc_" + name.replace("-", "_"), globals()) + def_freq = getattr(mod, "frequency", per_instance) + handler = getattr(mod, "handle") + + if not freq: + freq = def_freq + + self.cloud.sem_and_run("config-" + name, freq, handler, + [name, self.cfg, self.cloud, cloudinit.log, args]) + except: + raise -- cgit v1.2.3 From 675341b3b1431eda2513d87d741c5b72d74e4405 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 17:58:37 -0700 Subject: Working on making this have the old setup (where strings are taken in for config) as well as file names (for those that have files). --- cloudinit/log.py | 44 +++++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/cloudinit/log.py b/cloudinit/log.py index 4d270045..2dda405d 100644 --- a/cloudinit/log.py +++ b/cloudinit/log.py @@ -2,8 +2,13 @@ import logging import logging.handlers +import logging.config + +import os import sys +from StringIO import StringIO + # Logging levels for easy access CRITICAL = logging.CRITICAL FATAL = logging.FATAL @@ -14,10 +19,6 @@ INFO = logging.INFO DEBUG = logging.DEBUG NOTSET = logging.NOTSET -# File log rotation settings -ROTATE_AMOUNT = 10 # Only keep the past 9 + 1 active -ROTATE_SIZE = 10 * 1024 * 1024 # 10 MB - class ConsoleFormatter(logging.Formatter): @@ -31,50 +32,39 @@ class ConsoleFormatter(logging.Formatter): record.message = record.getMessage() rdict = dict(record.__dict__) rdict['minilevelname'] = self._get_mini_level(record) - # Skipping exception info for the console... return self._fmt % (rdict) -def setupLogging(level, filename=None, filelevel=logging.DEBUG): - root = getLogger() - consolelg = logging.StreamHandler(sys.stdout) - consolelg.setFormatter(ConsoleFormatter('%(minilevelname)s%(message)s')) - consolelg.setLevel(level) - root.addHandler(consolelg) - if filename: - filelg = logging.handlers.RotatingFileHandler(filename, maxBytes=ROTATE_SIZE, backupCount=ROTATE_AMOUNT) - filelg.setFormatter(logging.Formatter('%(levelname)s: @%(name)s : %(message)s')) - filelg.setLevel(filelevel) - root.addHandler(filelg) - root.setLevel(level) - - -def logging_set_from_cfg(cfg): +def setupLogging(cfg): log_cfgs = [] - logcfg = util.get_cfg_option_str(cfg, "log_cfg", False) - if logcfg: + log_cfg = cfg.get('logcfg') + if log_cfg: # if there is a 'logcfg' entry in the config, respect # it, it is the old keyname - log_cfgs = [logcfg] + log_cfgs = [log_cfg] elif "log_cfgs" in cfg: for cfg in cfg['log_cfgs']: if isinstance(cfg, list): log_cfgs.append('\n'.join(cfg)) else: - log_cfgs.append() + log_cfgs.append(cfg) if not len(log_cfgs): sys.stderr.write("Warning, no logging configured\n") return + am_worked = 0 for logcfg in log_cfgs: try: - logging.config.fileConfig(StringIO.StringIO(logcfg)) - return + if not os.path.isfile(logcfg): + logcfg = StringIO(logcfg) + logging.config.fileConfig(logcfg) + am_worked += 1 except: pass - raise Exception("no valid logging found\n") + if not am_worked: + sys.stderr.write("Warning, no logging configured\n") def getLogger(name='cloudinit'): -- cgit v1.2.3 From 1a415b187119fbdc4e08112a6afb23f1caf2b9ad Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 17:59:31 -0700 Subject: Ensure files end with ".tmpl" if they don't initially. --- cloudinit/templater.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cloudinit/templater.py b/cloudinit/templater.py index b6b320ab..5a3563a2 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -2,14 +2,17 @@ import os from Cheetah.Template import Template +from cloudinit import settings from cloudinit import util -TEMPLATE_DIR = '/etc/cloud/templates/' - def render_to_file(template, outfile, searchList): - contents = Template(file=os.path.join(TEMPLATE_DIR, template), - searchList=[searchList]).respond() + fn = template + (base, ext) = os.path.splitext(fn) + if ext != ".tmpl": + fn = "%s.tmpl" % (fn) + fn = os.path.join(settings.TEMPLATE_DIR, fn) + contents = Template(file=fn, searchList=[searchList]).respond() util.write_file(outfile, contents) -- cgit v1.2.3 From edff7583b501a0eab5376b88f8378a6b774257c7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 18:00:52 -0700 Subject: Cleaning this up, logic should be more sound around attempts and retries now. --- cloudinit/url_helper.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 0f0a9d0c..ed78c92e 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -1,15 +1,11 @@ import errno -import httplib import time import urllib import urllib2 -from StringIO import StringIO - from contextlib import closing from cloudinit import log as logging -from cloudinit import shell as sh LOG = logging.getLogger(__name__) @@ -18,7 +14,7 @@ def ok_http_code(st): return st in xrange(200, 400) -def readurl(url, data=None, timeout=None, retries=0, sec_between=1, read_cb=None, headers=None): +def readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=None): openargs = {} if timeout is not None: openargs['timeout'] = int(timeout) @@ -31,14 +27,13 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, read_cb=None if retries <= 0: retries = 1 + attempts = retries + 1 last_excp = None - LOG.debug("Attempting to read from %s with %s attempts to be performed", url, retries) - for i in range(0, retries): + LOG.debug("Attempting to read from %s with %s attempts to be performed", url, attempts) + for i in range(0, attempts): try: with closing(urllib2.urlopen(req, **openargs)) as rh: - ofh = StringIO() - sh.pipe_in_out(rh, ofh, chunk_cb=read_cb) - return (ofh.getvalue(), rh.getcode()) + return (rh.read(), rh.getcode()) except urllib2.HTTPError as e: last_excp = e LOG.exception("Failed at reading from %s.", url) @@ -51,11 +46,12 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, read_cb=None else: last_excp = e LOG.exception("Failed at reading from %s.", url) - LOG.debug("Please wait %s seconds while we wait to try again.", sec_between) - time.sleep(sec_between) + if i + 1 < attempts: + LOG.debug("Please wait %s seconds while we wait to try again.", sec_between) + time.sleep(sec_between) # Didn't work out - LOG.warn("Failed downloading from %s after %s attempts", url, i + 1) + LOG.warn("Failed downloading from %s after %s attempts", url, attempts) if last_excp is not None: raise last_excp @@ -118,11 +114,11 @@ def wait_for_url(urls, max_wait=None, timeout=None, else: headers = {} - (resp, status_code) = readurl(url, headers=headers, timeout=timeout) + (resp, sc) = readurl(url, headers=headers, timeout=timeout) if not resp: - reason = "empty response [%s]" % status_code - elif not ok_http_code(status_code): - reason = "bad status code [%s]" % status_code + reason = "empty response [%s]" % sc + elif not ok_http_code(sc): + reason = "bad status code [%s]" % sc else: return url except urllib2.HTTPError as e: -- cgit v1.2.3 From 0a078d626ae2b84f1e33a3e3eb5348e919e039c4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 18:01:24 -0700 Subject: Large amounts of refactoring. Now there exists a class which processes the user data down to a mime message and just some small utility methods to walk and determine types. Large amount of content type cleanups & constant creation. --- cloudinit/user_data.py | 387 +++++++++++++++++++++++++------------------------ 1 file changed, 198 insertions(+), 189 deletions(-) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index ec914480..f35e5d38 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -18,19 +18,23 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import email +import hashlib +import os +import urllib +import email from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.mime.base import MIMEBase + import yaml -import cloudinit -import cloudinit.util as util -import hashlib -import urllib +from cloudinit import url_helper +from cloudinit import util -starts_with_mappings = { + +# Different file beginnings to there content type +INCLUSION_TYPES_MAP = { '#include': 'text/x-include-url', '#include-once': 'text/x-include-once-url', '#!': 'text/x-shellscript', @@ -41,170 +45,210 @@ starts_with_mappings = { '#cloud-config-archive': 'text/cloud-config-archive', } +# Various special content types +TYPE_NEEDED = ["text/plain", "text/x-not-multipart"] +INCLUDE_TYPES = ['text/x-include-url', 'text/x-include-once-url'] +ARCHIVE_TYPES = ["text/cloud-config-archive"] +UNDEF_TYPE = "text/plain" +ARCHIVE_UNDEF_TYPE = "text/cloud-config" +NOT_MULTIPART_TYPE = "text/x-not-multipart" +OCTET_TYPE = 'application/octet-stream' -# if 'string' is compressed return decompressed otherwise return it -def decomp_str(string): - import StringIO - import gzip - try: - uncomp = gzip.GzipFile(None, "rb", 1, StringIO.StringIO(string)).read() - return(uncomp) - except: - return(string) - - -def do_include(content, appendmsg): - import os - # is just a list of urls, one per line - # also support '#include ' - includeonce = False - for line in content.splitlines(): - if line == "#include": - continue - if line == "#include-once": - includeonce = True - continue - if line.startswith("#include-once"): - line = line[len("#include-once"):].lstrip() - includeonce = True - elif line.startswith("#include"): - line = line[len("#include"):].lstrip() - if line.startswith("#"): - continue - if line.strip() == "": - continue - - # urls cannot not have leading or trailing white space - msum = hashlib.md5() # pylint: disable=E1101 - msum.update(line.strip()) - includeonce_filename = "%s/urlcache/%s" % ( - cloudinit.get_ipath_cur("data"), msum.hexdigest()) - try: - if includeonce and os.path.isfile(includeonce_filename): - with open(includeonce_filename, "r") as fp: - content = fp.read() - else: - content = urllib.urlopen(line).read() - if includeonce: - util.write_file(includeonce_filename, content, mode=0600) - except Exception: - raise - - process_includes(message_from_string(decomp_str(content)), appendmsg) - - -def explode_cc_archive(archive, appendmsg): - for ent in yaml.load(archive): - # ent can be one of: - # dict { 'filename' : 'value', 'content' : 'value', 'type' : 'value' } - # filename and type not be present - # or - # scalar(payload) - - def_type = "text/cloud-config" - if isinstance(ent, str): - ent = {'content': ent} - - content = ent.get('content', '') - mtype = ent.get('type', None) - if mtype == None: - mtype = type_from_startswith(content, def_type) - - maintype, subtype = mtype.split('/', 1) - if maintype == "text": - msg = MIMEText(content, _subtype=subtype) - else: - msg = MIMEBase(maintype, subtype) - msg.set_payload(content) - - if 'filename' in ent: - msg.add_header('Content-Disposition', 'attachment', - filename=ent['filename']) - - for header in ent.keys(): - if header in ('content', 'filename', 'type'): - continue - msg.add_header(header, ent['header']) +# Sorted longest first +INCLUSION_SRCH = sorted(INCLUSION_TYPES_MAP.keys(), key=(lambda e: 0 - len(e))) - _attach_part(appendmsg, msg) +# Msg header used to track attachments +ATTACHMENT_FIELD = 'Number-Attachments' +# This will be used to create a filename from a url (or like) entry +# When we want to make sure a entry isn't included more than once across sessions. +INCLUDE_ONCE_HASHER = 'md5' -def multi_part_count(outermsg, newcount=None): - """ - Return the number of attachments to this MIMEMultipart by looking - at its 'Number-Attachments' header. - """ - nfield = 'Number-Attachments' - if nfield not in outermsg: - outermsg[nfield] = "0" +# For those pieces without filenames +PART_FN_TPL = 'part-%03d' - if newcount != None: - outermsg.replace_header(nfield, str(newcount)) - return(int(outermsg.get('Number-Attachments', 0))) +class UserDataProcessor(object): + def __init__(self, paths): + self.paths = paths + def process(self, blob): + base_msg = convert_string(blob) + process_msg = MIMEMultipart() + self._process_msg(base_msg, process_msg) + return process_msg -def _attach_part(outermsg, part): - """ - Attach an part to an outer message. outermsg must be a MIMEMultipart. - Modifies a header in outermsg to keep track of number of attachments. - """ - cur = multi_part_count(outermsg) - if not part.get_filename(None): - part.add_header('Content-Disposition', 'attachment', - filename='part-%03d' % (cur + 1)) - outermsg.attach(part) - multi_part_count(outermsg, cur + 1) + def _process_msg(self, base_msg, append_msg): + for part in base_msg.walk(): + # multipart/* are just containers + if part.get_content_maintype() == 'multipart': + continue + + ctype = None + ctype_orig = part.get_content_type() + payload = part.get_payload(decode=True) + + if not ctype_orig: + ctype_orig = UNDEF_TYPE + + if ctype_orig in TYPE_NEEDED: + ctype = type_from_starts_with(payload) + + if ctype is None: + ctype = ctype_orig + + if ctype in INCLUDE_TYPES: + self._do_include(payload, append_msg) + continue + + if ctype in ARCHIVE_TYPES: + self._explode_archive(payload, append_msg) + continue + + if 'Content-Type' in base_msg: + base_msg.replace_header('Content-Type', ctype) + else: + base_msg['Content-Type'] = ctype + + self._attach_part(append_msg, part) + + def _get_include_once_filename(self, entry): + msum = hashlib.new(INCLUDE_ONCE_HASHER) + msum.update(entry) + entry_fn = msum.hexdigest()[0:64] # Don't get to long now + return os.path.join(self.paths.get_ipath_cur('data'), 'urlcache', entry_fn) + + def _do_include(self, content, append_msg): + # is just a list of urls, one per line + # also support '#include ' + for line in content.splitlines(): + includeonce = False + if line in ("#include", "#include-once"): + continue + if line.startswith("#include-once"): + line = line[len("#include-once"):].lstrip() + includeonce = True + elif line.startswith("#include"): + line = line[len("#include"):].lstrip() + if line.startswith("#"): + continue + include_url = line.strip() + if not include_url: + continue + includeonce_filename = self._get_include_once_filename(include_url) + if includeonce and os.path.isfile(includeonce_filename): + content = util.load_file(includeonce_filename) + else: + (content, st) = url_helper.readurl(include_url) + if includeonce and url_helper.ok_http_code(st): + util.write_file(includeonce_filename, content, mode=0600) + if not url_helper.ok_http_code(st): + content = '' -def type_from_startswith(payload, default=None): - # slist is sorted longest first - slist = sorted(starts_with_mappings.keys(), key=lambda e: 0 - len(e)) - for sstr in slist: - if payload.startswith(sstr): - return(starts_with_mappings[sstr]) - return default + new_msg = convert_string(content) + self._process_msg(new_msg, append_msg) + def _explode_archive(self, archive, append_msg): + try: + entries = yaml.load(archive) + except: + entries = [] + if not isinstance(entries, (list, set)): + # TODO raise? + entries = [] + + for ent in entries: + # ent can be one of: + # dict { 'filename' : 'value', 'content' : 'value', 'type' : 'value' } + # filename and type not be present + # or + # scalar(payload) + if isinstance(ent, str): + ent = {'content': ent} + if not isinstance(ent, (dict)): + # TODO raise? + continue -def process_includes(msg, appendmsg=None): - if appendmsg == None: - appendmsg = MIMEMultipart() + content = ent.get('content', '') + mtype = ent.get('type') + if not mtype: + mtype = type_from_starts_with(content, ARCHIVE_UNDEF_TYPE) - for part in msg.walk(): + maintype, subtype = mtype.split('/', 1) + if maintype == "text": + msg = MIMEText(content, _subtype=subtype) + else: + msg = MIMEBase(maintype, subtype) + msg.set_payload(content) + + if 'filename' in ent: + msg.add_header('Content-Disposition', 'attachment', filename=ent['filename']) + + for header in ent.keys(): + if header in ('content', 'filename', 'type'): + continue + msg.add_header(header, ent['header']) + + self._attach_part(append_msg, msg) + + def _multi_part_count(self, outer_msg, new_count=None): + """ + Return the number of attachments to this MIMEMultipart by looking + at its 'Number-Attachments' header. + """ + if ATTACHMENT_FIELD not in outer_msg: + outer_msg[ATTACHMENT_FIELD] = str(0) + + if new_count is not None: + outer_msg.replace_header(ATTACHMENT_FIELD, str(new_count)) + + fetched_count = 0 + try: + fetched_count = int(outer_msg.get(ATTACHMENT_FIELD)) + except (ValueError, TypeError): + outer_msg.replace_header(ATTACHMENT_FIELD, str(fetched_count)) + return fetched_count + + def _attach_part(self, outer_msg, part): + """ + Attach an part to an outer message. outermsg must be a MIMEMultipart. + Modifies a header in the message to keep track of number of attachments. + """ + cur = self._multi_part_count(outer_msg) + if not part.get_filename(): + part.add_header('Content-Disposition', 'attachment', filename=PART_FN_TPL % (cur + 1)) + outer_msg.attach(part) + self._multi_part_count(outer_msg, cur + 1) + + +# Callback is a function that will be called with +# (data, content_type, filename, payload) +def walk(ud_msg, callback, data): + partnum = 0 + for part in ud_msg.walk(): # multipart/* are just containers if part.get_content_maintype() == 'multipart': continue - ctype = None - ctype_orig = part.get_content_type() - - payload = part.get_payload(decode=True) - - if ctype_orig in ("text/plain", "text/x-not-multipart"): - ctype = type_from_startswith(payload) - + ctype = part.get_content_type() if ctype is None: - ctype = ctype_orig + ctype = OCTET_TYPE - if ctype in ('text/x-include-url', 'text/x-include-once-url'): - do_include(payload, appendmsg) - continue - - if ctype == "text/cloud-config-archive": - explode_cc_archive(payload, appendmsg) - continue - - if 'Content-Type' in msg: - msg.replace_header('Content-Type', ctype) - else: - msg['Content-Type'] = ctype + filename = part.get_filename() + if not filename: + filename = PART_FN_TPL % partnum - _attach_part(appendmsg, part) + callback(data, ctype, filename, part.get_payload(decode=True)) + partnum = partnum + 1 -def message_from_string(data, headers=None): - if headers is None: +def convert_string(self, raw_data, headers=None): + if not data: + data = '' + if not headers: headers = {} + data = util.decomp_str(raw_data) if "mime-version:" in data[0:4096].lower(): msg = email.message_from_string(data) for (key, val) in headers.items(): @@ -213,50 +257,15 @@ def message_from_string(data, headers=None): else: msg[key] = val else: - mtype = headers.get("Content-Type", "text/x-not-multipart") + mtype = headers.get("Content-Type", NOT_MULTIPART_TYPE) maintype, subtype = mtype.split("/", 1) msg = MIMEBase(maintype, subtype, *headers) msg.set_payload(data) - - return(msg) + return msg -# this is heavily wasteful, reads through userdata string input -def preprocess_userdata(data): - newmsg = MIMEMultipart() - process_includes(message_from_string(decomp_str(data)), newmsg) - return(newmsg.as_string()) - - -# callback is a function that will be called with (data, content_type, -# filename, payload) -def walk_userdata(istr, callback, data=None): - partnum = 0 - for part in message_from_string(istr).walk(): - # multipart/* are just containers - if part.get_content_maintype() == 'multipart': - continue - - ctype = part.get_content_type() - if ctype is None: - ctype = 'application/octet-stream' - - filename = part.get_filename() - if not filename: - filename = 'part-%03d' % partnum - - callback(data, ctype, filename, part.get_payload(decode=True)) - - partnum = partnum + 1 - - -if __name__ == "__main__": - def main(): - import sys - data = decomp_str(file(sys.argv[1]).read()) - newmsg = MIMEMultipart() - process_includes(message_from_string(data), newmsg) - print newmsg - print "#found %s parts" % multi_part_count(newmsg) - - main() +def type_from_starts_with(payload, default=None): + for text in INCLUSION_SRCH: + if payload.startswith(text): + return INCLUSION_TYPES_MAP[text] + return default -- cgit v1.2.3 From 1173fdf9e5cc6ae931a2001e1bb2b56ee2981157 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 18:02:29 -0700 Subject: Moved the shell functionality back in here. --- cloudinit/util.py | 921 +++++++++++++++++++++++++----------------------------- 1 file changed, 422 insertions(+), 499 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 47397418..265a3a97 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -18,24 +18,24 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import yaml +from StringIO import StringIO + +import contextlib +import grp +import gzip import os -import os.path +import platform +import pwd import shutil -import errno import subprocess -from Cheetah.Template import Template -import urllib2 -import urllib -import logging -import re -import socket -import sys -import time -import tempfile -import traceback import urlparse +import yaml + +from cloudinit import log as logging +from cloudinit import url_helper as uhelp + + try: import selinux HAVE_LIBSELINUX = True @@ -43,53 +43,153 @@ except ImportError: HAVE_LIBSELINUX = False +LOG = logging.getLogger(__name__) + +# Helps cleanup filenames to ensure they aren't FS incompatible +FN_REPLACEMENTS = { + os.sep: '_', +} + + +class ProcessExecutionError(IOError): + + MESSAGE_TMPL = ('%(description)s\nCommand: %(cmd)s\n' + 'Exit code: %(exit_code)s\nStdout: %(stdout)r\n' + 'Stderr: %(stderr)r') + + def __init__(self, stdout=None, stderr=None, + exit_code=None, cmd=None, + description=None, reason=None): + if not cmd: + self.cmd = '-' + else: + self.cmd = cmd + + if not description: + self.description = 'Unexpected error while running command.' + else: + self.description = description + + if not isinstance(exit_code, (long, int)): + self.exit_code = '-' + else: + self.exit_code = exit_code + + if not stderr: + self.stderr = '' + else: + self.stderr = stderr + + if not stdout: + self.stdout = '' + else: + self.stdout = stdout + + message = self.MESSAGE_TMPL % { + 'description': self.description, + 'cmd': self.cmd, + 'exit_code': self.exit_code, + 'stdout': self.stdout, + 'stderr': self.stderr, + } + IOError.__init__(self, message) + self.reason = reason + + +class _SeLinuxGuard(object): + def __init__(self, path, recursive=False): + self.path = path + self.recursive = recursive + self.engaged = False + if HAVE_LIBSELINUX and selinux.is_selinux_enabled(): + self.engaged = True + + def __enter__(self): + return self.engaged + + def __exit__(self, type, value, traceback): + if self.engaged: + LOG.debug("Disengaging selinux mode for %s: %s", self.path, self.recursive) + selinux.restorecon(self.path, recursive=self.recursive) + + +def translate_bool(val): + if not val: + return False + if val is isinstance(val, bool): + return val + if str(val).lower().strip() in ['true', '1', 'on', 'yes']: + return True + return False + + def read_conf(fname): try: - stream = open(fname, "r") - conf = yaml.load(stream) - stream.close() - return conf + mp = yaml.load(load_file(fname)) + if not isinstance(mp, (dict)): + return {} + return mp except IOError as e: if e.errno == errno.ENOENT: return {} raise -def get_base_cfg(cfgfile, cfg_builtin="", parsed_cfgs=None): - kerncfg = {} - syscfg = {} +def clean_filename(fn): + for (k, v) in FN_REPLACEMENTS.items(): + fn = fn.replace(k, v) + return fn.strip() + + +def decomp_str(data): + try: + uncomp = gzip.GzipFile(None, "rb", 1, StringIO(data)).read() + return uncomp + except: + return data + + +def is_ipv4(instr): + """ determine if input string is a ipv4 address. return boolean""" + toks = instr.split('.') + if len(toks) != 4: + return False + + try: + toks = [x for x in toks if (int(x) < 256 and int(x) > 0)] + except: + return False + + return (len(toks) == 4) + + +def get_base_cfg(cfgfile, cfg_builtin=None, parsed_cfgs=None): if parsed_cfgs and cfgfile in parsed_cfgs: - return(parsed_cfgs[cfgfile]) + return parsed_cfgs[cfgfile] syscfg = read_conf_with_confd(cfgfile) - kern_contents = read_cc_from_cmdline() + kerncfg = {} if kern_contents: kerncfg = yaml.load(kern_contents) # kernel parameters override system config combined = mergedict(kerncfg, syscfg) - if cfg_builtin: - builtin = yaml.load(cfg_builtin) - fin = mergedict(combined, builtin) + fin = mergedict(combined, cfg_builtin) else: fin = combined - if parsed_cfgs != None: + # Cache it? + if parsed_cfgs: parsed_cfgs[cfgfile] = fin - return(fin) + return fin def get_cfg_option_bool(yobj, key, default=False): if key not in yobj: return default - val = yobj[key] - if val is True: - return True - if str(val).lower() in ['true', '1', 'on', 'yes']: - return True - return False + return translate_bool(yobj[key]) def get_cfg_option_str(yobj, key, default=None): @@ -98,6 +198,15 @@ def get_cfg_option_str(yobj, key, default=None): return yobj[key] +def system_info(): + return { + 'platform': platform.platform(), + 'release': platform.release(), + 'python': platform.python_version(), + 'uname': platform.uname(), + } + + def get_cfg_option_list_or_str(yobj, key, default=None): """ Gets the C{key} config option from C{yobj} as a list of strings. If the @@ -127,7 +236,7 @@ def get_cfg_by_path(yobj, keyp, default=None): if tok not in cur: return(default) cur = cur[tok] - return(cur) + return cur def mergedict(src, cand): @@ -141,50 +250,29 @@ def mergedict(src, cand): src[k] = v else: src[k] = mergedict(src[k], v) + else: + if not isinstance(src, dict): + raise TypeError("Attempting to merge a non dictionary source type: %s" % (type(src))) + if not isinstance(cand, dict): + raise TypeError("Attempting to merge a non dictionary candiate type: %s" % (type(cand))) return src -def delete_dir_contents(dirname): - """ - Deletes all contents of a directory without deleting the directory itself. - - @param dirname: The directory whose contents should be deleted. - """ - for node in os.listdir(dirname): - node_fullpath = os.path.join(dirname, node) - if os.path.isdir(node_fullpath): - shutil.rmtree(node_fullpath) - else: - os.unlink(node_fullpath) - - -def write_file(filename, content, mode=0644, omode="wb"): - """ - Writes a file with the given content and sets the file mode as specified. - Resotres the SELinux context if possible. - - @param filename: The full path of the file to write. - @param content: The content to write to the file. - @param mode: The filesystem mode to set on the file. - @param omode: The open mode used when opening the file (r, rb, a, etc.) - """ +@contextlib.contextmanager +def tempdir(**kwargs): + # This seems like it was only added in python 3.2 + # Make it since its useful... + # See: http://bugs.python.org/file12970/tempdir.patch + tdir = tempfile.mkdtemp(**kwargs) try: - os.makedirs(os.path.dirname(filename)) - except OSError as e: - if e.errno != errno.EEXIST: - raise e - - f = open(filename, omode) - if mode is not None: - os.chmod(filename, mode) - f.write(content) - f.close() - restorecon_if_possible(filename) + yield tdir + finally: + del_dir(tdir) -def restorecon_if_possible(path, recursive=False): - if HAVE_LIBSELINUX and selinux.is_selinux_enabled(): - selinux.restorecon(path, recursive=recursive) +def del_dir(path): + LOG.debug("Recursively deleting %s", path) + shutil.rmtree(path) # get keyid from keyserver @@ -202,7 +290,8 @@ def getkeybyid(keyid, keyserver): [ -n "${armour}" ] && echo "${armour}" """ args = ['sh', '-c', shcmd, "export-gpg-keyid", keyid, keyserver] - return(subp(args)[0]) + (stdout, stderr) = subp(args) + return stdout def runparts(dirp, skip_no_exist=True): @@ -210,38 +299,19 @@ def runparts(dirp, skip_no_exist=True): return failed = 0 + attempted = 0 for exe_name in sorted(os.listdir(dirp)): exe_path = os.path.join(dirp, exe_name) if os.path.isfile(exe_path) and os.access(exe_path, os.X_OK): - popen = subprocess.Popen([exe_path]) - popen.communicate() - if popen.returncode is not 0: + attempted += 1 + try: + subp([exe_path]) + except ProcessExecutionError as e: + LOG.exception("Failed running %s [%i]", exe_path, e.exit_code) failed += 1 - sys.stderr.write("failed: %s [%i]\n" % - (exe_path, popen.returncode)) - if failed: - raise RuntimeError('runparts: %i failures' % failed) - - -def subp(args, input_=None): - sp = subprocess.Popen(args, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, stdin=subprocess.PIPE) - out, err = sp.communicate(input_) - if sp.returncode is not 0: - raise subprocess.CalledProcessError(sp.returncode, args, (out, err)) - return(out, err) - - -def render_to_file(template, outfile, searchList): - t = Template(file='/etc/cloud/templates/%s.tmpl' % template, - searchList=[searchList]) - f = open(outfile, 'w') - f.write(t.respond()) - f.close() - -def render_string(template, searchList): - return(Template(template, searchList=[searchList]).respond()) + if failed and attempted: + raise RuntimeError('runparts: %i failures in %i attempted commands' % (failed, attempted)) # read_optional_seed @@ -254,13 +324,12 @@ def read_optional_seed(fill, base="", ext="", timeout=5): fill['user-data'] = ud fill['meta-data'] = md return True - except OSError, e: + except OSError as e: if e.errno == errno.ENOENT: return False raise -# raise OSError with enoent if not found def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): if base.startswith("/"): base = "file://%s" % base @@ -276,89 +345,14 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): ud_url = "%s%s%s" % (base, "user-data", ext) md_url = "%s%s%s" % (base, "meta-data", ext) - no_exc = object() - raise_err = no_exc - for attempt in range(0, retries + 1): - try: - md_str = readurl(md_url, timeout=timeout) - ud = readurl(ud_url, timeout=timeout) - md = yaml.load(md_str) - - return(md, ud) - except urllib2.HTTPError as e: - raise_err = e - except urllib2.URLError as e: - raise_err = e - if (isinstance(e.reason, OSError) and - e.reason.errno == errno.ENOENT): - raise_err = e.reason - - if attempt == retries: - break - - #print "%s failed, sleeping" % attempt - time.sleep(1) - - raise(raise_err) - - -def logexc(log, lvl=logging.DEBUG): - log.log(lvl, traceback.format_exc()) - - -class RecursiveInclude(Exception): - pass - - -def read_file_with_includes(fname, rel=".", stack=None, patt=None): - if stack is None: - stack = [] - if not fname.startswith("/"): - fname = os.sep.join((rel, fname)) - - fname = os.path.realpath(fname) - - if fname in stack: - raise(RecursiveInclude("%s recursively included" % fname)) - if len(stack) > 10: - raise(RecursiveInclude("%s included, stack size = %i" % - (fname, len(stack)))) - - if patt == None: - patt = re.compile("^#(opt_include|include)[ \t].*$", re.MULTILINE) - - try: - fp = open(fname) - contents = fp.read() - fp.close() - except: - raise - - rel = os.path.dirname(fname) - stack.append(fname) - - cur = 0 - while True: - match = patt.search(contents[cur:]) - if not match: - break - loc = match.start() + cur - endl = match.end() + cur - - (key, cur_fname) = contents[loc:endl].split(None, 2) - cur_fname = cur_fname.strip() - - try: - inc_contents = read_file_with_includes(cur_fname, rel, stack, patt) - except IOError, e: - if e.errno == errno.ENOENT and key == "#opt_include": - inc_contents = "" - else: - raise - contents = contents[0:loc] + inc_contents + contents[endl + 1:] - cur = loc + len(inc_contents) - stack.pop() - return(contents) + (md_str, msc) = uhelp.readurl(md_url, timeout=timeout, retries=retries) + (ud, usc) = uhelp.readurl(ud_url, timeout=timeout, retries=retries) + md = None + if md_str and uhelp.ok_http_code(msc): + md = yaml.load(md_str) + if not uhelp.ok_http_code(usc): + ud = None + return (md, ud) def read_conf_d(confd): @@ -369,46 +363,32 @@ def read_conf_d(confd): confs = [f for f in confs if f.endswith(".cfg")] # remove anything not a file - confs = [f for f in confs if os.path.isfile("%s/%s" % (confd, f))] + confs = [f for f in confs if os.path.isfile(os.path.join(confd, f))] cfg = {} for conf in confs: - cfg = mergedict(cfg, read_conf("%s/%s" % (confd, conf))) + cfg = mergedict(cfg, read_conf(os.path.join(confd, conf))) - return(cfg) + return cfg def read_conf_with_confd(cfgfile): cfg = read_conf(cfgfile) + confd = False if "conf_d" in cfg: if cfg['conf_d'] is not None: confd = cfg['conf_d'] if not isinstance(confd, str): - raise Exception("cfgfile %s contains 'conf_d' " + raise RuntimeError("cfgfile %s contains 'conf_d' " "with non-string" % cfgfile) elif os.path.isdir("%s.d" % cfgfile): confd = "%s.d" % cfgfile if not confd: - return(cfg) - - confd_cfg = read_conf_d(confd) + return cfg - return(mergedict(confd_cfg, cfg)) - - -def get_cmdline(): - if 'DEBUG_PROC_CMDLINE' in os.environ: - cmdline = os.environ["DEBUG_PROC_CMDLINE"] - else: - try: - cmdfp = open("/proc/cmdline") - cmdline = cmdfp.read().strip() - cmdfp.close() - except: - cmdline = "" - return(cmdline) + return mergedict(read_conf_d(confd), cfg) def read_cc_from_cmdline(cmdline=None): @@ -439,147 +419,15 @@ def read_cc_from_cmdline(cmdline=None): begin = cmdline.find(tag_begin, end + end_l) - return('\n'.join(tokens)) - - -def ensure_dirs(dirlist, mode=0755): - fixmodes = [] - for d in dirlist: - try: - if mode != None: - os.makedirs(d) - else: - os.makedirs(d, mode) - except OSError as e: - if e.errno != errno.EEXIST: - raise - if mode != None: - fixmodes.append(d) - - for d in fixmodes: - os.chmod(d, mode) + return '\n'.join(tokens) -def chownbyname(fname, user=None, group=None): - uid = -1 - gid = -1 - if user == None and group == None: - return - if user: - import pwd - uid = pwd.getpwnam(user).pw_uid - if group: - import grp - gid = grp.getgrnam(group).gr_gid - - os.chown(fname, uid, gid) - - -def readurl(url, data=None, timeout=None): - openargs = {} - if timeout != None: - openargs['timeout'] = timeout - - if data is None: - req = urllib2.Request(url) - else: - encoded = urllib.urlencode(data) - req = urllib2.Request(url, encoded) - - response = urllib2.urlopen(req, **openargs) - return(response.read()) - - -# shellify, takes a list of commands -# for each entry in the list -# if it is an array, shell protect it (with single ticks) -# if it is a string, do nothing -def shellify(cmdlist): - content = "#!/bin/sh\n" - escaped = "%s%s%s%s" % ("'", '\\', "'", "'") - for args in cmdlist: - # if the item is a list, wrap all items in single tick - # if its not, then just write it directly - if isinstance(args, list): - fixed = [] - for f in args: - fixed.append("'%s'" % str(f).replace("'", escaped)) - content = "%s%s\n" % (content, ' '.join(fixed)) - else: - content = "%s%s\n" % (content, str(args)) - return content - - -def dos2unix(string): +def dos2unix(contents): # find first end of line - pos = string.find('\n') - if pos <= 0 or string[pos - 1] != '\r': - return(string) - return(string.replace('\r\n', '\n')) - - -def is_container(): - # is this code running in a container of some sort - - for helper in ('running-in-container', 'lxc-is-container'): - try: - # try to run a helper program. if it returns true - # then we're inside a container. otherwise, no - sp = subprocess.Popen(helper, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - sp.communicate(None) - return(sp.returncode == 0) - except OSError as e: - if e.errno != errno.ENOENT: - raise - - # this code is largely from the logic in - # ubuntu's /etc/init/container-detect.conf - try: - # Detect old-style libvirt - # Detect OpenVZ containers - pid1env = get_proc_env(1) - if "container" in pid1env: - return True - - if "LIBVIRT_LXC_UUID" in pid1env: - return True - - except IOError as e: - if e.errno != errno.ENOENT: - pass - - # Detect OpenVZ containers - if os.path.isdir("/proc/vz") and not os.path.isdir("/proc/bc"): - return True - - try: - # Detect Vserver containers - with open("/proc/self/status") as fp: - lines = fp.read().splitlines() - for line in lines: - if line.startswith("VxID:"): - (_key, val) = line.strip().split(":", 1) - if val != "0": - return True - except IOError as e: - if e.errno != errno.ENOENT: - pass - - return False - - -def get_proc_env(pid): - # return the environment in a dict that a given process id was started with - env = {} - with open("/proc/%s/environ" % pid) as fp: - toks = fp.read().split("\0") - for tok in toks: - if tok == "": - continue - (name, val) = tok.split("=", 1) - env[name] = val - return env + pos = contents.find('\n') + if pos <= 0 or contents[pos - 1] != '\r': + return contents + return contents.replace('\r\n', '\n') def get_hostname_fqdn(cfg, cloud): @@ -603,7 +451,7 @@ def get_hostname_fqdn(cfg, cloud): hostname = cfg['hostname'] else: hostname = cloud.get_hostname() - return(hostname, fqdn) + return (hostname, fqdn) def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): @@ -612,26 +460,22 @@ def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): # did not have did not have 'bind' in the order attribute fqdn = None try: - with open(filename, "r") as hfp: - for line in hfp.readlines(): - hashpos = line.find("#") - if hashpos >= 0: - line = line[0:hashpos] - toks = line.split() - - # if there there is less than 3 entries (ip, canonical, alias) - # then ignore this line - if len(toks) < 3: - continue - - if hostname in toks[2:]: - fqdn = toks[1] - break - hfp.close() + for line in load_file(filename).splitlines(): + hashpos = line.find("#") + if hashpos >= 0: + line = line[0:hashpos] + toks = line.split() + + # if there there is less than 3 entries (ip, canonical, alias) + # then ignore this line + if len(toks) < 3: + continue + + if hostname in toks[2:]: + fqdn = toks[1] + break except IOError as e: - if e.errno == errno.ENOENT: - pass - + pass return fqdn @@ -646,7 +490,7 @@ def is_resolvable(name): def is_resolvable_url(url): """ determine if this url is resolvable (existing or ip) """ - return(is_resolvable(urlparse.urlparse(url).hostname)) + return (is_resolvable(urlparse.urlparse(url).hostname)) def search_for_mirror(candidates): @@ -656,8 +500,7 @@ def search_for_mirror(candidates): if is_resolvable_url(cand): return cand except Exception: - raise - + pass return None @@ -669,7 +512,7 @@ def close_stdin(): if _CLOUD_INIT_SAVE_STDIN is set in environment to a non empty or '0' value then input will not be closed (only useful potentially for debugging). """ - if os.environ.get("_CLOUD_INIT_SAVE_STDIN") in ("", "0", False): + if os.environ.get("_CLOUD_INIT_SAVE_STDIN") in ("", "0", 'False'): return with open(os.devnull) as fp: os.dup2(fp.fileno(), sys.stdin.fileno()) @@ -685,163 +528,244 @@ def find_devs_with(criteria): """ try: (out, _err) = subp(['blkid', '-t%s' % criteria, '-odevice']) - except subprocess.CalledProcessError: - return([]) - return(str(out).splitlines()) + except ProcessExecutionError: + return [] + return (out.splitlines()) -class mountFailedError(Exception): - pass +def load_file(fname, read_cb=None): + LOG.debug("Reading from %s", fname) + with open(fname, 'rb') as fh: + ofh = StringIO() + pipe_in_out(fh, ofh, chunk_cb=read_cb) + return ofh.getvalue() -def mount_callback_umount(device, callback, data=None): - """ - mount the device, call method 'callback' passing the directory - in which it was mounted, then unmount. Return whatever 'callback' - returned. If data != None, also pass data to callback. - """ - - def _cleanup(umount, tmpd): - if umount: - try: - subp(["umount", '-l', umount]) - except subprocess.CalledProcessError: - raise - if tmpd: - os.rmdir(tmpd) - - # go through mounts to see if it was already mounted - fp = open("/proc/mounts") - mounts = fp.readlines() - fp.close() - - tmpd = None - - mounted = {} - for mpline in mounts: - (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() - mp = mp.replace("\\040", " ") - mounted[dev] = (dev, fstype, mp, False) - - umount = False - if device in mounted: - mountpoint = "%s/" % mounted[device][2] +def get_cmdline(): + if 'DEBUG_PROC_CMDLINE' in os.environ: + cmdline = os.environ["DEBUG_PROC_CMDLINE"] else: - tmpd = tempfile.mkdtemp() + try: + cmdline = load_file("/proc/cmdline").strip() + except: + cmdline = "" + return cmdline + + +def pipe_in_out(in_fh, out_fh, chunk_size=1024, chunk_cb=None): + bytes_piped = 0 + LOG.debug("Transferring the contents of %s to %s in chunks of size %s.", in_fh, out_fh, chunk_size) + while True: + data = in_fh.read(chunk_size) + if data == '': + break + else: + out_fh.write(data) + bytes_piped += len(data) + if chunk_cb: + chunk_cb(bytes_piped) + out_fh.flush() + return bytes_piped + - mountcmd = ["mount", "-o", "ro", device, tmpd] +def chownbyid(fname, uid=None, gid=None): + if uid == None and gid == None: + return + LOG.debug("Changing the ownership of %s to %s:%s", fname, uid, gid) + os.chown(fname, uid, gid) + + +def chownbyname(fname, user=None, group=None): + uid = -1 + gid = -1 + if user: + uid = pwd.getpwnam(user).pw_uid + if group: + gid = grp.getgrnam(group).gr_gid + chownbyid(fname, uid, gid) + +def ensure_dirs(dirlist, mode=0755): + for d in dirlist: + ensure_dir(d, mode) + + +def ensure_dir(path, mode=0755): + if not os.path.isdir(path): + fixmodes = [] + LOG.debug("Ensuring directory exists at path %s (perms=%s)", dir_name, mode) try: - (_out, _err) = subp(mountcmd) - umount = tmpd - except subprocess.CalledProcessError as exc: - _cleanup(umount, tmpd) - raise mountFailedError(exc.output[1]) + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + if mode is not None: + os.chmod(path, mode) + - mountpoint = "%s/" % tmpd +def sym_link(source, link): + LOG.debug("Creating symbolic link from %r => %r" % (link, source)) + os.symlink(source, link) + +def del_file(path): + LOG.debug("Attempting to remove %s", path) try: - if data == None: - ret = callback(mountpoint) - else: - ret = callback(mountpoint, data) + os.unlink(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise e - except Exception as exc: - _cleanup(umount, tmpd) - raise exc - _cleanup(umount, tmpd) +def ensure_file(path): + write_file(path, content='', omode="ab") - return(ret) +def write_file(filename, content, mode=0644, omode="wb"): + """ + Writes a file with the given content and sets the file mode as specified. + Resotres the SELinux context if possible. -def wait_for_url(urls, max_wait=None, timeout=None, - status_cb=None, headers_cb=None): + @param filename: The full path of the file to write. + @param content: The content to write to the file. + @param mode: The filesystem mode to set on the file. + @param omode: The open mode used when opening the file (r, rb, a, etc.) """ - urls: a list of urls to try - max_wait: roughly the maximum time to wait before giving up - The max time is *actually* len(urls)*timeout as each url will - be tried once and given the timeout provided. - timeout: the timeout provided to urllib2.urlopen - status_cb: call method with string message when a url is not available - headers_cb: call method with single argument of url to get headers - for request. - - the idea of this routine is to wait for the EC2 metdata service to - come up. On both Eucalyptus and EC2 we have seen the case where - the instance hit the MD before the MD service was up. EC2 seems - to have permenantely fixed this, though. - - In openstack, the metadata service might be painfully slow, and - unable to avoid hitting a timeout of even up to 10 seconds or more - (LP: #894279) for a simple GET. - - Offset those needs with the need to not hang forever (and block boot) - on a system where cloud-init is configured to look for EC2 Metadata - service but is not going to find one. It is possible that the instance - data host (169.254.169.254) may be firewalled off Entirely for a sytem, - meaning that the connection will block forever unless a timeout is set. + ensure_dir(os.path.dirname(filename)) + LOG.debug("Writing to %s - %s (perms=%s) %s bytes", filename, omode, mode, len(content)) + with open(filename, omode) as fh: + with _SeLinuxGuard(filename): + fh.write(content) + fh.flush() + if mode is not None: + os.chmod(filename, mode) + + +def delete_dir_contents(dirname): """ - starttime = time.time() + Deletes all contents of a directory without deleting the directory itself. - sleeptime = 1 + @param dirname: The directory whose contents should be deleted. + """ + for node in os.listdir(dirname): + node_fullpath = os.path.join(dirname, node) + if os.path.isdir(node_fullpath): + del_dir(node_fullpath) + else: + del_file(node_fullpath) - def nullstatus_cb(msg): - return - if status_cb == None: - status_cb = nullstatus_cb +def subp(args, input_data=None, allowed_rc=None, env=None): + if allowed_rc is None: + allowed_rc = [0] + try: + LOG.debug("Running command %s with allowed return codes %s", args, allowed_rc) + sp = subprocess.Popen(args, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, stdin=subprocess.PIPE, + env=env) + (out, err) = sp.communicate(input_data) + except OSError as e: + raise ProcessExecutionError(cmd=args, reason=e) + rc = sp.returncode + if rc not in allowed_rc: + raise ProcessExecutionError(stdout=out, stderr=err, + exit_code=rc, + cmd=args) + # Just ensure blank instead of none?? + if not out: + out = '' + if not err: + err = '' + return (out, err) - def timeup(max_wait, starttime): - return((max_wait <= 0 or max_wait == None) or - (time.time() - starttime > max_wait)) - loop_n = 0 - while True: - sleeptime = int(loop_n / 5) + 1 - for url in urls: - now = time.time() - if loop_n != 0: - if timeup(max_wait, starttime): - break - if timeout and (now + timeout > (starttime + max_wait)): - # shorten timeout to not run way over max_time - timeout = int((starttime + max_wait) - now) - - reason = "" - try: - if headers_cb != None: - headers = headers_cb(url) - else: - headers = {} - - req = urllib2.Request(url, data=None, headers=headers) - resp = urllib2.urlopen(req, timeout=timeout) - if resp.read() != "": - return url - reason = "empty data [%s]" % resp.getcode() - except urllib2.HTTPError as e: - reason = "http error [%s]" % e.code - except urllib2.URLError as e: - reason = "url error [%s]" % e.reason - except socket.timeout as e: - reason = "socket timeout [%s]" % e - except Exception as e: - reason = "unexpected error [%s]" % e - - status_cb("'%s' failed [%s/%ss]: %s" % - (url, int(time.time() - starttime), max_wait, - reason)) - - if timeup(max_wait, starttime): - break +# shellify, takes a list of commands +# for each entry in the list +# if it is an array, shell protect it (with single ticks) +# if it is a string, do nothing +def shellify(cmdlist, add_header=True): + content = '' + if add_header: + content += "#!/bin/sh\n" + escaped = "%s%s%s%s" % ("'", '\\', "'", "'") + for args in cmdlist: + # if the item is a list, wrap all items in single tick + # if its not, then just write it directly + if isinstance(args, list): + fixed = [] + for f in args: + fixed.append("'%s'" % str(f).replace("'", escaped)) + content = "%s%s\n" % (content, ' '.join(fixed)) + else: + content = "%s%s\n" % (content, str(args)) + return content + + +def is_container(): + # is this code running in a container of some sort + + for helper in ('running-in-container', 'lxc-is-container'): + try: + # try to run a helper program. if it returns true/zero + # then we're inside a container. otherwise, no + cmd = [helper] + (stdout, stderr) = subp(cmd, allowed_rc=[0]) + return True + except IOError as e: + pass + # Is this really needed? + # if e.errno != errno.ENOENT: + # raise + + # this code is largely from the logic in + # ubuntu's /etc/init/container-detect.conf + try: + # Detect old-style libvirt + # Detect OpenVZ containers + pid1env = get_proc_env(1) + if "container" in pid1env: + return True + if "LIBVIRT_LXC_UUID" in pid1env: + return True + except IOError as e: + pass - loop_n = loop_n + 1 - time.sleep(sleeptime) + # Detect OpenVZ containers + if os.path.isdir("/proc/vz") and not os.path.isdir("/proc/bc"): + return True + + try: + # Detect Vserver containers + lines = load_file("/proc/self/status").splitlines() + for line in lines: + if line.startswith("VxID:"): + (_key, val) = line.strip().split(":", 1) + if val != "0": + return True + except IOError as e: + pass return False +def get_proc_env(pid): + # return the environment in a dict that a given process id was started with + env = {} + fn = os.path.join("/proc/", str(pid), "environ") + try: + contents = load_file(fn) + toks = contents.split("\0") + for tok in toks: + if tok == "": + continue + (name, val) = tok.split("=", 1) + if not name: + env[name] = val + except IOError: + pass + return env + + def keyval_str_to_dict(kvstring): ret = {} for tok in kvstring.split(): @@ -851,5 +775,4 @@ def keyval_str_to_dict(kvstring): key = tok val = True ret[key] = val - - return(ret) + return ret -- cgit v1.2.3 From 5b99f3b79dfab917a1672b8c5a882fd51351964c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 18:02:45 -0700 Subject: Moved some of the mounting logic here. Also introduced a context manager which will help unmount. --- cloudinit/mounting.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 cloudinit/mounting.py diff --git a/cloudinit/mounting.py b/cloudinit/mounting.py new file mode 100644 index 00000000..b72f729a --- /dev/null +++ b/cloudinit/mounting.py @@ -0,0 +1,51 @@ +import contextlib + +from cloudinit import util + + +class MountFailedError(Exception): + pass + + +@contextlib.contextmanager +def unmounter(umount): + try: + yield umount + finally: + if umount: + sh.subp(["umount", '-l', umount]) + + +def mount_callback_umount(device, callback, data=None): + """ + mount the device, call method 'callback' passing the directory + in which it was mounted, then unmount. Return whatever 'callback' + returned. If data != None, also pass data to callback. + """ + + # go through mounts to see if it was already mounted + mounts = sh.load_file("/proc/mounts").splitlines() + mounted = {} + for mpline in mounts: + (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() + mp = mp.replace("\\040", " ") + mounted[dev] = (dev, fstype, mp, False) + + with util.tempdir() as tmpd: + umount = False + if device in mounted: + mountpoint = "%s/" % mounted[device][2] + else: + try: + mountcmd = ["mount", "-o", "ro", device, tmpd] + util.subp(mountcmd) + umount = tmpd + except IOError as exc: + raise MountFailedError("%s" % (exc)) + mountpoint = "%s/" % tmpd + with unmounter(umount): + if data is None: + ret = callback(mountpoint) + else: + ret = callback(mountpoint, data) + return ret -- cgit v1.2.3 From 88a0e724f8397219ae84c557d05c03c0d1ed9029 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 18:03:22 -0700 Subject: This file now holds the default part handlers (shell script, cloud-config, upstart-jobs, boot-hook) as classes as well as some utility methods to fixup imported modules (ensuring they have the right members/properties) and moving the walking handler callbacks and other part running functionality here. --- cloudinit/parts.py | 200 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 cloudinit/parts.py diff --git a/cloudinit/parts.py b/cloudinit/parts.py new file mode 100644 index 00000000..9cd24c5a --- /dev/null +++ b/cloudinit/parts.py @@ -0,0 +1,200 @@ +import os + +from cloudinit import util +from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) +from cloudinit import log as logging + +LOG = logging.getLogger(__name__) + +CONTENT_END = "__end__" +CONTENT_START = "__begin__" +PART_CONTENT_TYPES = ["text/part-handler"] +PART_HANDLER_FN_TMPL = 'part-handler-%03d' + + +class PartHandler(object): + def __init__(self, frequency, version=2): + self.handler_version = version + self.frequency = frequency + + def __repr__(self): + return "%s: [%s]" % (self.__class__.__name__, self.list_types()) + + def list_types(self): + raise NotImplementedError() + + def handle_part(self, data, ctype, filename, payload, frequency): + return self._handle_part(data, ctype, filename, payload, frequency) + + def _handle_part(self, data, ctype, filename, payload, frequency): + raise NotImplementedError() + + +class BootHookPartHandler(PartHandler): + def __init__(self, boothook_dir, instance_id): + PartHandler.__init__(self, PER_ALWAYS) + self.boothook_dir = boothook_dir + self.instance_id = instance_id + + def list_types(self): + return ['text/cloud-boothook'] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype in [CONTENT_START, CONTENT_END]: + return + + filename = util.clean_filename(filename) + payload = util.dos2unix(payload) + prefix = "#cloud-boothook" + start = 0 + if payload.startswith(prefix): + start = len(prefix) + 1 + + filepath = os.path.join(self.boothook_dir, filename) + util.write_file(filepath, payload[start:], 0700) + try: + env = os.environ.copy() + env['INSTANCE_ID'] = str(self.instance_id) + util.subp([filepath], env=env) + except util.ProcessExecutionError as e: + LOG.error("Boothooks script %s returned %s", filepath, e.exit_code) + except Exception as e: + LOG.error("Boothooks unknown exception %s when running %s", e, filepath) + + +class UpstartJobPartHandler(PartHandler): + def __init__(self, upstart_dir): + PartHandler.__init__(self, PER_INSTANCE) + self.upstart_dir = upstart_dir + + def list_types(self): + return ['text/upstart-job'] + + def _handle_part(self, _data, ctype, filename, payload, frequency): + if ctype in [CONTENT_START, CONTENT_END]: + return + + filename = utils.clean_filename(filename) + (name, ext) = os.path.splitext(filename) + ext = ext.lower() + if ext != ".conf": + filename = filename + ".conf" + + payload = util.dos2unix(payload) + util.write_file(os.path.join(self.upstart_dir, filename), payload, 0644) + + +class ShellScriptPartHandler(PartHandler): + + def __init__(self, script_dir): + PartHandler.__init__(self, PER_ALWAYS) + self.script_dir = script_dir + + def list_types(self): + return ['text/x-shellscript'] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype in [CONTENT_START, CONTENT_END]: + # maybe delete existing things here + return + + filename = util.clean_filename(filename) + payload = util.dos2unix(payload) + util.write_file(os.path.join(self.script_dir, filename), payload, 0700) + + +class CloudConfigPartHandler(PartHandler): + def __init__(self, cloud_fn): + PartHandler.__init__(self, PER_ALWAYS) + self.cloud_buf = [] + self.cloud_fn = cloud_fn + + def list_types(self): + return ['text/cloud-config'] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype == CONTENT_START: + self.cloud_buf = [] + return + + if ctype == CONTENT_END: + payload = "\n".join(self.cloud_buf) + util.write_file(self.cloud_fn, payload, 0600) + self.cloud_buf = [] + return + + filename = util.clean_filename(filename) + entry = "\n".join(["#%s" % (filename), str(payload)]) + self.config_buf.append(entry) + + +def fixup_module(mod): + if not hasattr(mod, "handler_version"): + setattr(mod, "handler_version", 1) + if not hasattr(mod, 'list_types'): + def empty_types(): + return [] + setattr(mod, 'list_types', empty_types) + if not hasattr(mod, frequency): + setattr(mod, 'frequency', PER_INSTANCE) + return mod + + +def run_part(mod, data, ctype, filename, payload, frequency): + # only add the handler if the module should run + mod_freq = getattr(mod, "frequency") + if not (mod_freq == PER_ALWAYS or + (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)): + return + try: + mod_ver = getattr(mod, 'handler_version') + if mod_ver == 1: + mod.handle_part(data, ctype, filename, payload) + else: + mod.handle_part(data, ctype, filename, payload, frequency) + except: + LOG.exception("Failed calling mod %s (%s, %s, %s) with frequency %s", mod, ctype, filename, mod_ver, frequency) + + +def call_begin(mod, data, frequency): + run_part(mod, data, CONTENT_START, None, None, frequency) + + +def call_end(mod, data, frequency): + run_part(mod, data, CONTENT_END, None, None, frequency) + + +def walker_handle_handler(pdata, _ctype, _filename, payload): + curcount = pdata['handlercount'] + modname = PART_HANDLER_FN_TMPL % (curcount) + frequency = pdata['frequency'] + modfname = os.path.join(pdata['handlerdir'], "%s.py" % (modname)) + # TODO: Check if path exists?? + util.write_file(modfname, payload, 0600) + handlers = pdata['handlers'] + try: + mod = fixup_module(importer.import_module(modname)) + handlers.register(mod) + call_begin(mod, pdata['data'], frequency) + pdata['handlercount'] = curcount + 1 + except: + LOG.exception("Failed at registered python file %s", modfname) + + +def walker_callback(pdata, ctype, filename, payload): + # data here is the part_handlers array and then the data to pass through + if ctype in PART_CONTENT_TYPES: + walker_handle_handler(pdata, ctype, filename, payload) + return + handlers = pdata['handlers'] + if ctype not in handlers: + if ctype == "text/x-not-multipart": + # Extract the first line or 24 bytes for displaying in the log + start = payload.split("\n", 1)[0][:24] + if start < payload: + details = "starting '%s...'" % start.encode("string-escape") + else: + details = repr(payload) + LOG.warning("Unhandled non-multipart userdata: %s", details) + return + run_part(handlers[ctype], pdata['data'], ctype, filename, payload, pdata['frequency']) -- cgit v1.2.3 From 8cd2871e7d7885076cc4a760d44323e480eb1b9a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 18:04:56 -0700 Subject: Remove cloud config from here and move to cloud.py --- cloudinit/handlers/__init__.py | 60 +++++++----------------------------------- 1 file changed, 9 insertions(+), 51 deletions(-) diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index a16bdde6..3b0cdd4e 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -19,65 +19,23 @@ # along with this program. If not, see . # -import yaml -import cloudinit -import cloudinit.util as util -import sys -import traceback import os import subprocess +import sys import time +import traceback -per_instance = cloudinit.per_instance -per_always = cloudinit.per_always -per_once = cloudinit.per_once - - -class CloudConfig(): - cfgfile = None - cfg = None - - def __init__(self, cfgfile, cloud=None, ds_deps=None): - if cloud == None: - self.cloud = cloudinit.CloudInit(ds_deps) - self.cloud.get_data_source() - else: - self.cloud = cloud - self.cfg = self.get_config_obj(cfgfile) - - def get_config_obj(self, cfgfile): - try: - cfg = util.read_conf(cfgfile) - except: - # TODO: this 'log' could/should be passed in - cloudinit.log.critical("Failed loading of cloud config '%s'. " - "Continuing with empty config\n" % cfgfile) - cloudinit.log.debug(traceback.format_exc() + "\n") - cfg = None - if cfg is None: - cfg = {} - - try: - ds_cfg = self.cloud.datasource.get_config_obj() - except: - ds_cfg = {} +import yaml - cfg = util.mergedict(cfg, ds_cfg) - return(util.mergedict(cfg, self.cloud.cfg)) +from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE) - def handle(self, name, args, freq=None): - try: - mod = __import__("cc_" + name.replace("-", "_"), globals()) - def_freq = getattr(mod, "frequency", per_instance) - handler = getattr(mod, "handle") +from cloudinit import log as logging +from cloudinit import util - if not freq: - freq = def_freq +LOG = logging.getLogger(__name__) - self.cloud.sem_and_run("config-" + name, freq, handler, - [name, self.cfg, self.cloud, cloudinit.log, args]) - except: - raise +DEF_HANDLER_VERSION = 1 +DEF_FREQ = PER_INSTANCE # reads a cloudconfig module list, returns -- cgit v1.2.3 From d78bc08df59051bc0f118e990bf1a1660584c38e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 18:05:27 -0700 Subject: Remove is ipv4 function from here and move to utils + move exceptions to here as well as other find datasource function. --- cloudinit/sources/__init__.py | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index e2a9150d..9a9c1316 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -18,13 +18,16 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from cloudinit import user_data as ud +from cloudinit import util + +import socket DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" -import cloudinit.UserDataHandler as ud -import cloudinit.util as util -import socket +class DataSourceNotFoundException(Exception): + pass class DataSource: @@ -152,6 +155,25 @@ class DataSource: return hostname +def find_source(cfg, ds_deps): + cfglist = cfg.get('datasource_list') or [] + dslist = list_sources(cfglist, ds_deps) + dsnames = [f.__name__ for f in dslist] + + LOG.debug("Searching for data source in %s", dsnames) + for cls in dslist: + ds = cls.__name__ + try: + s = cls(sys_cfg=cfg) + if s.get_data(): + return (s, ds) + except Exception as e: + LOG.exception("Getting data from %s raised %s", ds, e) + + msg = "Did not find any data source, searched classes: %s" % dsnames + raise DataSourceNotFoundException(msg) + + # return a list of classes that have the same depends as 'depends' # iterate through cfg_list, loading "DataSourceCollections" modules # and calling their "get_datasource_list". @@ -198,17 +220,3 @@ def list_from_depends(depends, dslist): if depset == set(deps): retlist.append(cls) return(retlist) - - -def is_ipv4(instr): - """ determine if input string is a ipv4 address. return boolean""" - toks = instr.split('.') - if len(toks) != 4: - return False - - try: - toks = [x for x in toks if (int(x) < 256 and int(x) > 0)] - except: - return False - - return (len(toks) == 4) -- cgit v1.2.3 From 707e310c2aade8bd1cd024b008f5ecfeb4155063 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 18:09:24 -0700 Subject: Remove the main function from this, seems like that should be in a test if needed. --- cloudinit/ssh_util.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index a081fbe8..1483f718 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -188,40 +188,3 @@ def parse_ssh_config(fname="/etc/ssh/sshd_config"): ret[key] = val fp.close() return(ret) - -if __name__ == "__main__": - def main(): - import sys - # usage: orig_file, new_keys, [key_prefix] - # prints out merged, where 'new_keys' will trump old - ## example - ## ### begin auth_keys ### - # ssh-rsa AAAAB3NzaC1xxxxxxxxxV3csgm8cJn7UveKHkYjJp8= smoser-work - # ssh-rsa AAAAB3NzaC1xxxxxxxxxCmXp5Kt5/82cD/VN3NtHw== smoser@brickies - # ### end authorized_keys ### - # - # ### begin new_keys ### - # ssh-rsa nonmatch smoser@newhost - # ssh-rsa AAAAB3NzaC1xxxxxxxxxV3csgm8cJn7UveKHkYjJp8= new_comment - # ### end new_keys ### - # - # Then run as: - # program auth_keys new_keys \ - # 'no-port-forwarding,command=\"echo hi world;\"' - def_prefix = None - orig_key_file = sys.argv[1] - new_key_file = sys.argv[2] - if len(sys.argv) > 3: - def_prefix = sys.argv[3] - fp = open(new_key_file) - - newkeys = [] - for line in fp.readlines(): - newkeys.append(AuthKeyEntry(line, def_prefix)) - - fp.close() - print update_authorized_keys(orig_key_file, newkeys) - - main() - -# vi: ts=4 expandtab -- cgit v1.2.3 From ddc2dc57653cfe1378c0b40946261a2872fbfe3c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 18:09:44 -0700 Subject: Remove the main function from this, seems like it should also be in a test if needed. --- cloudinit/netinfo.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index aac4af04..1f0fa571 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -110,7 +110,3 @@ def debug_info(pre="ci-info: "): (pre, n, r[0], r[1], r[2], r[7], r[3])) n = n + 1 return('\n'.join(lines)) - - -if __name__ == '__main__': - print debug_info() -- cgit v1.2.3 From 3fa70e06646d0d1f40562786bbfe9552bdd33936 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 18:42:06 -0700 Subject: Fixup CloudConfig by using new module functions and calling objects. --- cloudinit/cloud.py | 69 ++++++++++++++++++++++++++---------------------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index cfb1c011..a98bfbbd 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -16,6 +16,7 @@ from cloudinit import parts from cloudinit import sources from cloudinit import util from cloudinit import user_data +from cloudinit import handlers LOG = logging.getLogger(__name__) @@ -168,7 +169,6 @@ class CloudInit(object): else: self.ds_deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] self.paths = CloudPaths(self) - self.sems = CloudSemaphores(self.paths) self.cfg = self._read_cfg() def _read_cfg_old(self): @@ -294,16 +294,14 @@ class CloudInit(object): handlers = CloudHandlers(self) # Add handlers in cdir - for fname in glob.glob(os.path.join(cdir, "*.py")): - if not os.path.isfile(fname): - continue - modname = os.path.basename(fname)[0:-3] + potential_handlers = parts.find_module_files(cdir) + for (fname, modname) in potential_handlers.iteritems(): try: mod = parts.fixup_module(importer.import_module(modname)) types = handlers.register(mod) LOG.debug("Added handler for [%s] from %s", types, fname) except: - LOG.exception("Failed to register handler in %s", fname) + LOG.exception("Failed to register handler from %s", fname) def_handlers = handlers.register_defaults() if def_handlers: @@ -391,47 +389,46 @@ class CloudHandlers(object): class CloudConfig(object): - cfgfile = None - cfg = None - def __init__(self, cfgfile, cloud=None, ds_deps=None): - if cloud == None: - self.cloud = cloudinit.CloudInit(ds_deps) - self.cloud.get_data_source() - else: - self.cloud = cloud - self.cfg = self.get_config_obj(cfgfile) + def __init__(self, cfgfile, cloud): + self.cloud = cloud + self.cfg = self._get_config(cfgfile) + self.paths = cloud.paths + self.sems = CloudSemaphores(self.paths) + + def _get_config(self, cfgfile): - def get_config_obj(self, cfgfile): + cfg = None try: cfg = util.read_conf(cfgfile) except: - # TODO: this 'log' could/should be passed in - cloudinit.log.critical("Failed loading of cloud config '%s'. " - "Continuing with empty config\n" % cfgfile) - cloudinit.log.debug(traceback.format_exc() + "\n") - cfg = None - if cfg is None: + LOG.exception(("Failed loading of cloud config '%s'. " + "Continuing with empty config."), cfgfile) + if not cfg: cfg = {} + ds_cfg = None try: ds_cfg = self.cloud.datasource.get_config_obj() except: + LOG.exception("Failed loading of datasource config.") + if not ds_cfg: ds_cfg = {} cfg = util.mergedict(cfg, ds_cfg) - return(util.mergedict(cfg, self.cloud.cfg)) + cloud_cfg = self.cloud.cfg or {} + return util.mergedict(cfg, cloud_cfg) - def handle(self, name, args, freq=None): - try: - mod = __import__("cc_" + name.replace("-", "_"), globals()) - def_freq = getattr(mod, "frequency", per_instance) - handler = getattr(mod, "handle") - - if not freq: - freq = def_freq - - self.cloud.sem_and_run("config-" + name, freq, handler, - [name, self.cfg, self.cloud, cloudinit.log, args]) - except: - raise + def extract(self, name): + modname = handlers.form_module_name(name) + if not modname: + return None + return handlers.fixup_module(importer.import_module(modname)) + + def handle(self, name, mod, args, freq=None): + def_freq = mod.frequency + if not freq: + freq = def_freq + c_name = "config-%s" % (name) + real_args = [name, self.cfg, self.cloud, LOG, args] + return self.sems.run_functor(c_name, freq, mod.handle, real_args) -- cgit v1.2.3 From c6ed90775e79fb31c467bf1222fb06ca17f1aa21 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 18:42:31 -0700 Subject: Move how user part data modules are found to here. --- cloudinit/parts.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cloudinit/parts.py b/cloudinit/parts.py index 9cd24c5a..6af1ab7c 100644 --- a/cloudinit/parts.py +++ b/cloudinit/parts.py @@ -137,9 +137,23 @@ def fixup_module(mod): setattr(mod, 'list_types', empty_types) if not hasattr(mod, frequency): setattr(mod, 'frequency', PER_INSTANCE) + if not hasattr(mod, 'handle_part'): + def empty_handler(data, ctype, filename, payload): + pass + setattr(mod, 'handle_part', empty_handler) return mod +def find_module_files(root_dir): + entries = dict() + for fname in glob.glob(os.path.join(root_dir, "*.py")): + if not os.path.isfile(fname): + continue + modname = os.path.basename(fname)[0:-3] + entries[fname] = modname + return entries + + def run_part(mod, data, ctype, filename, payload, frequency): # only add the handler if the module should run mod_freq = getattr(mod, "frequency") -- cgit v1.2.3 From 65a2249c9ba9503afe8c2da06afd54a63cf68fbd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 18:42:54 -0700 Subject: Move how handler module names are found to here as well as a fixup module function for bad modules. --- cloudinit/handlers/__init__.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 3b0cdd4e..ae74b683 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -36,6 +36,7 @@ LOG = logging.getLogger(__name__) DEF_HANDLER_VERSION = 1 DEF_FREQ = PER_INSTANCE +HANDLER_TPL = "cc_%s" # reads a cloudconfig module list, returns @@ -230,3 +231,25 @@ def update_package_sources(): def install_packages(pkglist): update_package_sources() apt_get("install", pkglist) + + +def form_module_name(name): + canon_name = name.replace("-", "_") + if canon_name.endswith(".py"): + canon_name = canon_name[0:(len(canon_name) - 3)] + canon_name = canon_name.strip() + if not canon_name: + return None + return HANDLER_TPL % (canon_name) + + +def fixup_module(mod): + freq = getattr(mod, "frequency", None) + if not freq: + setattr(mod, 'frequency', PER_INSTANCE) + handler = getattr(mod, "handle", None) + if not handler: + def empty_handle(_name, _cfg, _cloud, _log, _args): + pass + setattr(mod, 'handle', empty_handle) + return mod -- cgit v1.2.3 From 3025072fa803124b4067181c712a758f732f771b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 19:10:58 -0700 Subject: Remove parsed config cache, and cleanup cmdline function. --- cloudinit/__init__.py | 62 ++++++++++++++++++++++----------------------------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py index 2c291ff5..c38b3b09 100644 --- a/cloudinit/__init__.py +++ b/cloudinit/__init__.py @@ -34,47 +34,37 @@ from cloudinit import url_helper as uhelp from cloudinit import util from cloudinit.settings import (VAR_LIB_DIR, CFG_BUILTIN, CLOUD_CONFIG, - BOOT_FINISHED, CUR_INSTANCE_LINK, PATH_MAP) + BOOT_FINISHED, CUR_INSTANCE_LINK) LOG = logging.getLogger(__name__) INIT_SUBDIRS = [ - 'scripts', - os.path.join('scripts', 'per-instance'), - os.path.join('scripts', 'per-once'), - os.path.join('scripts', 'per-boot'), - 'seed', - 'instances', - 'handlers', - 'sem', - 'data' + os.path.join(VAR_LIB_DIR, 'scripts'), + os.path.join(VAR_LIB_DIR, 'scripts', 'per-instance'), + os.path.join(VAR_LIB_DIR, 'scripts', 'per-once'), + os.path.join(VAR_LIB_DIR, 'scripts', 'per-boot'), + os.path.join(VAR_LIB_DIR, 'seed'), + os.path.join(VAR_LIB_DIR, 'instances'), + os.path.join(VAR_LIB_DIR, 'handlers'), + os.path.join(VAR_LIB_DIR, 'sem'), + os.path.join(VAR_LIB_DIR, 'data'), ] -# TODO: get rid of this global -parsed_cfgs = {} - - def initfs(): - - # TODO don't do this every time this function is called? - dlist = [] - for subd in INIT_SUBDIRS: - dlist.append(os.path.join(VAR_LIB_DIR, subd)) - util.ensure_dirs(dlist) - - cfg = util.get_base_cfg(CLOUD_CONFIG, get_builtin_cfg(), parsed_cfgs) + util.ensure_dirs(INIT_SUBDIRS) + cfg = util.get_base_cfg(CLOUD_CONFIG, get_builtin_cfg()) log_file = util.get_cfg_option_str(cfg, 'def_log_file', None) perms = util.get_cfg_option_str(cfg, 'syslog_fix_perms', None) if log_file: util.ensure_file(log_file) - if log_file and perms: - (u, g) = perms.split(':', 1) - if u == "-1" or u == "None": - u = None - if g == "-1" or g == "None": - g = None - util.chownbyname(log_file, u, g) + if perms: + (u, g) = perms.split(':', 1) + if u == "-1" or u == "None": + u = None + if g == "-1" or g == "None": + g = None + util.chownbyname(log_file, u, g) def purge_cache(rmcur=True): @@ -89,7 +79,7 @@ def purge_cache(rmcur=True): def get_base_cfg(cfg_path=None): if cfg_path is None: cfg_path = CLOUD_CONFIG - return util.get_base_cfg(cfg_path, get_builtin_cfg(), parsed_cfgs) + return util.get_base_cfg(cfg_path, get_builtin_cfg()) def get_builtin_cfg(): @@ -97,14 +87,16 @@ def get_builtin_cfg(): def list_sources(cfg_list, depends): - return (sources.list_sources(cfg_list, depends, ["cloudinit", ""])) - + return sources.list_sources(cfg_list, depends, ["cloudinit", ""]) -def get_cmdline_url(names=('cloud-config-url', 'url'), - starts="#cloud-config", cmdline=None): - if cmdline == None: +def get_cmdline_url(names=None, starts=None, cmdline=None): + if cmdline is None: cmdline = util.get_cmdline() + if not names: + names = ('cloud-config-url', 'url') + if not starts: + starts = "#cloud-config" data = util.keyval_str_to_dict(cmdline) url = None -- cgit v1.2.3 From 262e54bd5eb4d35e021b2401995eac8ea8d06bb2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 19:11:14 -0700 Subject: Move to having a parts directory/module + seperate modules. --- cloudinit/cloud.py | 37 +++---- cloudinit/parts.py | 214 ---------------------------------------- cloudinit/parts/__init__.py | 117 ++++++++++++++++++++++ cloudinit/parts/boot_hook.py | 42 ++++++++ cloudinit/parts/cloud_config.py | 36 +++++++ cloudinit/parts/shell_script.py | 27 +++++ cloudinit/parts/upstart_job.py | 30 ++++++ 7 files changed, 272 insertions(+), 231 deletions(-) delete mode 100644 cloudinit/parts.py create mode 100644 cloudinit/parts/__init__.py create mode 100644 cloudinit/parts/boot_hook.py create mode 100644 cloudinit/parts/cloud_config.py create mode 100644 cloudinit/parts/shell_script.py create mode 100644 cloudinit/parts/upstart_job.py diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index a98bfbbd..f9c0d531 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -2,22 +2,27 @@ from time import time import cPickle as pickle import contextlib +import copy import os import sys import weakref - from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, OLD_CLOUD_CONFIG, CLOUD_CONFIG, - CFG_BUILTIN, CUR_INSTANCE_LINK) + CUR_INSTANCE_LINK) from cloudinit import (get_builtin_cfg, get_base_cfg) from cloudinit import log as logging -from cloudinit import parts from cloudinit import sources from cloudinit import util from cloudinit import user_data from cloudinit import handlers +from cloudinit import parts +from cloudinit.parts import boot_hook as bh_part +from cloudinit.parts import cloud_config as cc_part +from cloudinit.parts import upstart_job as up_part +from cloudinit.parts import shell_script as ss_part + LOG = logging.getLogger(__name__) @@ -128,10 +133,10 @@ class CloudPaths(object): return ipath -class CloudPartData(object): - def __init__(self, datasource, paths): - self.datasource = datasource - self.paths = paths +class CloudSimple(object): + def __init__(self, init): + self.datasource = init.datasource + self.paths = init.paths def get_userdata(self): return self.datasource.get_userdata() @@ -288,7 +293,7 @@ class CloudInit(object): sys.path.insert(0, idir) # Data will be a little proxy that modules can use - data = CloudPartData(self.datasource, self.paths) + data = CloudSimple(self) # This keeps track of all the active handlers handlers = CloudHandlers(self) @@ -369,13 +374,13 @@ class CloudHandlers(object): def _get_default_handlers(self): def_handlers = [] if self.paths.get_ipath("cloud_config"): - def_handlers.append(parts.CloudConfigPartHandler(self.paths.get_ipath("cloud_config"))) + def_handlers.append(cc_part.CloudConfigPartHandler(self.paths.get_ipath("cloud_config"))) if self.paths.get_ipath_cur('scripts'): - def_handlers.append(parts.ShellScriptPartHandler(self.paths.get_ipath_cur('scripts'))) + def_handlers.append(ss_part.ShellScriptPartHandler(self.paths.get_ipath_cur('scripts'))) if self.paths.get_ipath("boothooks"): - def_handlers.append(parts.BootHookPartHandler(self.paths.get_ipath("boothooks"))) + def_handlers.append(bh_part.BootHookPartHandler(self.paths.get_ipath("boothooks"))) if self.paths.upstart_conf_d: - def_handlers.append(parts.UpstartJobPartHandler(self.paths.upstart_conf_d)) + def_handlers.append(up_part.UpstartJobPartHandler(self.paths.upstart_conf_d)) return def_handlers def register_defaults(self): @@ -391,13 +396,11 @@ class CloudHandlers(object): class CloudConfig(object): def __init__(self, cfgfile, cloud): - self.cloud = cloud + self.cloud = CloudSimple(cloud) self.cfg = self._get_config(cfgfile) - self.paths = cloud.paths - self.sems = CloudSemaphores(self.paths) + self.sems = CloudSemaphores(self.cloud.paths) def _get_config(self, cfgfile): - cfg = None try: cfg = util.read_conf(cfgfile) @@ -430,5 +433,5 @@ class CloudConfig(object): if not freq: freq = def_freq c_name = "config-%s" % (name) - real_args = [name, self.cfg, self.cloud, LOG, args] + real_args = [name, copy.deepcopy(self.cfg), self.cloud, LOG, copy.deepcopy(args)] return self.sems.run_functor(c_name, freq, mod.handle, real_args) diff --git a/cloudinit/parts.py b/cloudinit/parts.py deleted file mode 100644 index 6af1ab7c..00000000 --- a/cloudinit/parts.py +++ /dev/null @@ -1,214 +0,0 @@ -import os - -from cloudinit import util -from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) - -CONTENT_END = "__end__" -CONTENT_START = "__begin__" -PART_CONTENT_TYPES = ["text/part-handler"] -PART_HANDLER_FN_TMPL = 'part-handler-%03d' - - -class PartHandler(object): - def __init__(self, frequency, version=2): - self.handler_version = version - self.frequency = frequency - - def __repr__(self): - return "%s: [%s]" % (self.__class__.__name__, self.list_types()) - - def list_types(self): - raise NotImplementedError() - - def handle_part(self, data, ctype, filename, payload, frequency): - return self._handle_part(data, ctype, filename, payload, frequency) - - def _handle_part(self, data, ctype, filename, payload, frequency): - raise NotImplementedError() - - -class BootHookPartHandler(PartHandler): - def __init__(self, boothook_dir, instance_id): - PartHandler.__init__(self, PER_ALWAYS) - self.boothook_dir = boothook_dir - self.instance_id = instance_id - - def list_types(self): - return ['text/cloud-boothook'] - - def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype in [CONTENT_START, CONTENT_END]: - return - - filename = util.clean_filename(filename) - payload = util.dos2unix(payload) - prefix = "#cloud-boothook" - start = 0 - if payload.startswith(prefix): - start = len(prefix) + 1 - - filepath = os.path.join(self.boothook_dir, filename) - util.write_file(filepath, payload[start:], 0700) - try: - env = os.environ.copy() - env['INSTANCE_ID'] = str(self.instance_id) - util.subp([filepath], env=env) - except util.ProcessExecutionError as e: - LOG.error("Boothooks script %s returned %s", filepath, e.exit_code) - except Exception as e: - LOG.error("Boothooks unknown exception %s when running %s", e, filepath) - - -class UpstartJobPartHandler(PartHandler): - def __init__(self, upstart_dir): - PartHandler.__init__(self, PER_INSTANCE) - self.upstart_dir = upstart_dir - - def list_types(self): - return ['text/upstart-job'] - - def _handle_part(self, _data, ctype, filename, payload, frequency): - if ctype in [CONTENT_START, CONTENT_END]: - return - - filename = utils.clean_filename(filename) - (name, ext) = os.path.splitext(filename) - ext = ext.lower() - if ext != ".conf": - filename = filename + ".conf" - - payload = util.dos2unix(payload) - util.write_file(os.path.join(self.upstart_dir, filename), payload, 0644) - - -class ShellScriptPartHandler(PartHandler): - - def __init__(self, script_dir): - PartHandler.__init__(self, PER_ALWAYS) - self.script_dir = script_dir - - def list_types(self): - return ['text/x-shellscript'] - - def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype in [CONTENT_START, CONTENT_END]: - # maybe delete existing things here - return - - filename = util.clean_filename(filename) - payload = util.dos2unix(payload) - util.write_file(os.path.join(self.script_dir, filename), payload, 0700) - - -class CloudConfigPartHandler(PartHandler): - def __init__(self, cloud_fn): - PartHandler.__init__(self, PER_ALWAYS) - self.cloud_buf = [] - self.cloud_fn = cloud_fn - - def list_types(self): - return ['text/cloud-config'] - - def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype == CONTENT_START: - self.cloud_buf = [] - return - - if ctype == CONTENT_END: - payload = "\n".join(self.cloud_buf) - util.write_file(self.cloud_fn, payload, 0600) - self.cloud_buf = [] - return - - filename = util.clean_filename(filename) - entry = "\n".join(["#%s" % (filename), str(payload)]) - self.config_buf.append(entry) - - -def fixup_module(mod): - if not hasattr(mod, "handler_version"): - setattr(mod, "handler_version", 1) - if not hasattr(mod, 'list_types'): - def empty_types(): - return [] - setattr(mod, 'list_types', empty_types) - if not hasattr(mod, frequency): - setattr(mod, 'frequency', PER_INSTANCE) - if not hasattr(mod, 'handle_part'): - def empty_handler(data, ctype, filename, payload): - pass - setattr(mod, 'handle_part', empty_handler) - return mod - - -def find_module_files(root_dir): - entries = dict() - for fname in glob.glob(os.path.join(root_dir, "*.py")): - if not os.path.isfile(fname): - continue - modname = os.path.basename(fname)[0:-3] - entries[fname] = modname - return entries - - -def run_part(mod, data, ctype, filename, payload, frequency): - # only add the handler if the module should run - mod_freq = getattr(mod, "frequency") - if not (mod_freq == PER_ALWAYS or - (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)): - return - try: - mod_ver = getattr(mod, 'handler_version') - if mod_ver == 1: - mod.handle_part(data, ctype, filename, payload) - else: - mod.handle_part(data, ctype, filename, payload, frequency) - except: - LOG.exception("Failed calling mod %s (%s, %s, %s) with frequency %s", mod, ctype, filename, mod_ver, frequency) - - -def call_begin(mod, data, frequency): - run_part(mod, data, CONTENT_START, None, None, frequency) - - -def call_end(mod, data, frequency): - run_part(mod, data, CONTENT_END, None, None, frequency) - - -def walker_handle_handler(pdata, _ctype, _filename, payload): - curcount = pdata['handlercount'] - modname = PART_HANDLER_FN_TMPL % (curcount) - frequency = pdata['frequency'] - modfname = os.path.join(pdata['handlerdir'], "%s.py" % (modname)) - # TODO: Check if path exists?? - util.write_file(modfname, payload, 0600) - handlers = pdata['handlers'] - try: - mod = fixup_module(importer.import_module(modname)) - handlers.register(mod) - call_begin(mod, pdata['data'], frequency) - pdata['handlercount'] = curcount + 1 - except: - LOG.exception("Failed at registered python file %s", modfname) - - -def walker_callback(pdata, ctype, filename, payload): - # data here is the part_handlers array and then the data to pass through - if ctype in PART_CONTENT_TYPES: - walker_handle_handler(pdata, ctype, filename, payload) - return - handlers = pdata['handlers'] - if ctype not in handlers: - if ctype == "text/x-not-multipart": - # Extract the first line or 24 bytes for displaying in the log - start = payload.split("\n", 1)[0][:24] - if start < payload: - details = "starting '%s...'" % start.encode("string-escape") - else: - details = repr(payload) - LOG.warning("Unhandled non-multipart userdata: %s", details) - return - run_part(handlers[ctype], pdata['data'], ctype, filename, payload, pdata['frequency']) diff --git a/cloudinit/parts/__init__.py b/cloudinit/parts/__init__.py new file mode 100644 index 00000000..20d4bd3b --- /dev/null +++ b/cloudinit/parts/__init__.py @@ -0,0 +1,117 @@ +import os + +from cloudinit import util +from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) +from cloudinit import log as logging + +LOG = logging.getLogger(__name__) + +CONTENT_END = "__end__" +CONTENT_START = "__begin__" +PART_CONTENT_TYPES = ["text/part-handler"] +PART_HANDLER_FN_TMPL = 'part-handler-%03d' +UNDEF_HANDLER_VERSION = 1 + + +class PartHandler(object): + def __init__(self, frequency, version=2): + self.handler_version = version + self.frequency = frequency + + def __repr__(self): + return "%s: [%s]" % (self.__class__.__name__, self.list_types()) + + def list_types(self): + raise NotImplementedError() + + def handle_part(self, data, ctype, filename, payload, frequency): + return self._handle_part(data, ctype, filename, payload, frequency) + + def _handle_part(self, data, ctype, filename, payload, frequency): + raise NotImplementedError() + + +def fixup_module(mod): + if not hasattr(mod, "handler_version"): + setattr(mod, "handler_version", UNDEF_HANDLER_VERSION) + if not hasattr(mod, 'list_types'): + def empty_types(): + return [] + setattr(mod, 'list_types', empty_types) + if not hasattr(mod, frequency): + setattr(mod, 'frequency', PER_INSTANCE) + if not hasattr(mod, 'handle_part'): + def empty_handler(data, ctype, filename, payload): + pass + setattr(mod, 'handle_part', empty_handler) + return mod + + +def find_module_files(root_dir): + entries = dict() + for fname in glob.glob(os.path.join(root_dir, "*.py")): + if not os.path.isfile(fname): + continue + modname = os.path.basename(fname)[0:-3] + entries[fname] = modname + return entries + + +def run_part(mod, data, ctype, filename, payload, frequency): + # only add the handler if the module should run + mod_freq = getattr(mod, "frequency") + if not (mod_freq == PER_ALWAYS or + (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)): + return + try: + mod_ver = getattr(mod, 'handler_version') + if mod_ver == 1: + mod.handle_part(data, ctype, filename, payload) + else: + mod.handle_part(data, ctype, filename, payload, frequency) + except: + LOG.exception("Failed calling mod %s (%s, %s, %s) with frequency %s", mod, ctype, filename, mod_ver, frequency) + + +def call_begin(mod, data, frequency): + run_part(mod, data, CONTENT_START, None, None, frequency) + + +def call_end(mod, data, frequency): + run_part(mod, data, CONTENT_END, None, None, frequency) + + +def walker_handle_handler(pdata, _ctype, _filename, payload): + curcount = pdata['handlercount'] + modname = PART_HANDLER_FN_TMPL % (curcount) + frequency = pdata['frequency'] + modfname = os.path.join(pdata['handlerdir'], "%s.py" % (modname)) + # TODO: Check if path exists?? + util.write_file(modfname, payload, 0600) + handlers = pdata['handlers'] + try: + mod = fixup_module(importer.import_module(modname)) + handlers.register(mod) + call_begin(mod, pdata['data'], frequency) + pdata['handlercount'] = curcount + 1 + except: + LOG.exception("Failed at registered python file %s", modfname) + + +def walker_callback(pdata, ctype, filename, payload): + # data here is the part_handlers array and then the data to pass through + if ctype in PART_CONTENT_TYPES: + walker_handle_handler(pdata, ctype, filename, payload) + return + handlers = pdata['handlers'] + if ctype not in handlers: + if ctype == "text/x-not-multipart": + # Extract the first line or 24 bytes for displaying in the log + start = payload.split("\n", 1)[0][:24] + if start < payload: + details = "starting '%s...'" % start.encode("string-escape") + else: + details = repr(payload) + LOG.warning("Unhandled non-multipart userdata: %s", details) + return + run_part(handlers[ctype], pdata['data'], ctype, filename, payload, pdata['frequency']) \ No newline at end of file diff --git a/cloudinit/parts/boot_hook.py b/cloudinit/parts/boot_hook.py new file mode 100644 index 00000000..881ffc58 --- /dev/null +++ b/cloudinit/parts/boot_hook.py @@ -0,0 +1,42 @@ +import os + +from cloudinit import util +from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) +from cloudinit import log as logging +from cloudinit import parts + +LOG = logging.getLogger(__name__) + + + +class BootHookPartHandler(parts.PartHandler): + def __init__(self, boothook_dir, instance_id): + parts.PartHandler.__init__(self, PER_ALWAYS) + self.boothook_dir = boothook_dir + self.instance_id = instance_id + + def list_types(self): + return ['text/cloud-boothook'] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype in [CONTENT_START, CONTENT_END]: + return + + filename = util.clean_filename(filename) + payload = util.dos2unix(payload) + prefix = "#cloud-boothook" + start = 0 + if payload.startswith(prefix): + start = len(prefix) + 1 + + filepath = os.path.join(self.boothook_dir, filename) + util.write_file(filepath, payload[start:], 0700) + try: + env = os.environ.copy() + env['INSTANCE_ID'] = str(self.instance_id) + util.subp([filepath], env=env) + except util.ProcessExecutionError as e: + LOG.error("Boothooks script %s returned %s", filepath, e.exit_code) + except Exception as e: + LOG.error("Boothooks unknown exception %s when running %s", e, filepath) + diff --git a/cloudinit/parts/cloud_config.py b/cloudinit/parts/cloud_config.py new file mode 100644 index 00000000..dab0e5f5 --- /dev/null +++ b/cloudinit/parts/cloud_config.py @@ -0,0 +1,36 @@ +import os + +from cloudinit import util +from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) +from cloudinit import log as logging +from cloudinit import parts + +LOG = logging.getLogger(__name__) + + + +class CloudConfigPartHandler(parts.PartHandler): + def __init__(self, cloud_fn): + parts.PartHandler.__init__(self, PER_ALWAYS) + self.cloud_buf = [] + self.cloud_fn = cloud_fn + + def list_types(self): + return ['text/cloud-config'] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype == CONTENT_START: + self.cloud_buf = [] + return + + if ctype == CONTENT_END: + payload = "\n".join(self.cloud_buf) + util.write_file(self.cloud_fn, payload, 0600) + self.cloud_buf = [] + return + + filename = util.clean_filename(filename) + entry = "\n".join(["#%s" % (filename), str(payload)]) + self.config_buf.append(entry) + + diff --git a/cloudinit/parts/shell_script.py b/cloudinit/parts/shell_script.py new file mode 100644 index 00000000..a248f198 --- /dev/null +++ b/cloudinit/parts/shell_script.py @@ -0,0 +1,27 @@ +import os + +from cloudinit import util +from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) +from cloudinit import log as logging +from cloudinit import parts + +LOG = logging.getLogger(__name__) + + +class ShellScriptPartHandler(parts.PartHandler): + + def __init__(self, script_dir): + parts.PartHandler.__init__(self, PER_ALWAYS) + self.script_dir = script_dir + + def list_types(self): + return ['text/x-shellscript'] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype in [CONTENT_START, CONTENT_END]: + # maybe delete existing things here + return + + filename = util.clean_filename(filename) + payload = util.dos2unix(payload) + util.write_file(os.path.join(self.script_dir, filename), payload, 0700) diff --git a/cloudinit/parts/upstart_job.py b/cloudinit/parts/upstart_job.py new file mode 100644 index 00000000..7b290d26 --- /dev/null +++ b/cloudinit/parts/upstart_job.py @@ -0,0 +1,30 @@ +import os + +from cloudinit import util +from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) +from cloudinit import log as logging +from cloudinit import parts + +LOG = logging.getLogger(__name__) + + +class UpstartJobPartHandler(parts.PartHandler): + def __init__(self, upstart_dir): + parts.PartHandler.__init__(self, PER_INSTANCE) + self.upstart_dir = upstart_dir + + def list_types(self): + return ['text/upstart-job'] + + def _handle_part(self, _data, ctype, filename, payload, frequency): + if ctype in [CONTENT_START, CONTENT_END]: + return + + filename = utils.clean_filename(filename) + (name, ext) = os.path.splitext(filename) + ext = ext.lower() + if ext != ".conf": + filename = filename + ".conf" + + payload = util.dos2unix(payload) + util.write_file(os.path.join(self.upstart_dir, filename), payload, 0644) -- cgit v1.2.3 From 3f9d5977891611aa89dbe0db811adbb943851692 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 9 Jun 2012 12:30:43 -0700 Subject: Not needed. --- checks.sh | 53 ----------------------------------------------------- 1 file changed, 53 deletions(-) delete mode 100755 checks.sh diff --git a/checks.sh b/checks.sh deleted file mode 100755 index f54cad36..00000000 --- a/checks.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -set -u - -function find_src { - files=`find bin cloudinit -type f | grep "py\$"` - echo $files -} - -function run_pep8 { - echo "Running pep8 ..." - files=$(find_src) - ignores="E202,E501" - output_filename="pep8.log" - opts="--ignore=$ignores --repeat" - pep8 ${opts} ${files} 2>&1 > $output_filename - if [ "$?" -ne "0" ]; then - echo "Some badness was found!" - fi - echo "Check '$output_filename' for a full report." -} - -function run_pylint { - echo "Running pylint ..." - opts="--rcfile=pylintrc --output-format=parseable" - files=$(find_src) - output_filename="pylint.log" - pylint ${opts} ${files} 2>&1 > $output_filename - if [ "$?" -eq "1" ]; then - # pylint --long-help - # * 0 if everything went fine - # * 1 if a fatal message was issued - # * 2 if an error message was issued - # * 4 if a warning message was issued - # * 8 if a refactor message was issued - # * 16 if a convention message was issued - # * 32 on usage error - echo "A fatal pylint error occurred!" - else - if [ "$?" -eq "0" ]; then - echo "Your code is perfect you code master!" - else - echo "You are not yet a code master." - grep -i "Your code" $output_filename - fi - fi - echo "Check '$output_filename' for a full report." -} - - -run_pep8 -run_pylint - -- cgit v1.2.3 From 84197ecd42fecd51919c578e3f10b3c85b50bf84 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 9 Jun 2012 12:30:57 -0700 Subject: Start adding place where distro specifics can go. --- cloudinit/distros/__init__.py | 0 cloudinit/distros/ubuntu.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 cloudinit/distros/__init__.py create mode 100644 cloudinit/distros/ubuntu.py diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py new file mode 100644 index 00000000..e69de29b -- cgit v1.2.3 From 0a598f5a66ee1ba661c729b09f6b4e83271e7f7f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 9 Jun 2012 12:31:19 -0700 Subject: Rename these to user_data module + submodules. --- cloudinit/parts/__init__.py | 117 ---------------- cloudinit/parts/boot_hook.py | 42 ------ cloudinit/parts/cloud_config.py | 36 ----- cloudinit/parts/shell_script.py | 27 ---- cloudinit/parts/upstart_job.py | 30 ---- cloudinit/user_data.py | 271 ------------------------------------ cloudinit/user_data/__init__.py | 210 ++++++++++++++++++++++++++++ cloudinit/user_data/boot_hook.py | 65 +++++++++ cloudinit/user_data/cloud_config.py | 59 ++++++++ cloudinit/user_data/processor.py | 205 +++++++++++++++++++++++++++ cloudinit/user_data/shell_script.py | 53 +++++++ cloudinit/user_data/upstart_job.py | 56 ++++++++ 12 files changed, 648 insertions(+), 523 deletions(-) delete mode 100644 cloudinit/parts/__init__.py delete mode 100644 cloudinit/parts/boot_hook.py delete mode 100644 cloudinit/parts/cloud_config.py delete mode 100644 cloudinit/parts/shell_script.py delete mode 100644 cloudinit/parts/upstart_job.py delete mode 100644 cloudinit/user_data.py create mode 100644 cloudinit/user_data/__init__.py create mode 100644 cloudinit/user_data/boot_hook.py create mode 100644 cloudinit/user_data/cloud_config.py create mode 100644 cloudinit/user_data/processor.py create mode 100644 cloudinit/user_data/shell_script.py create mode 100644 cloudinit/user_data/upstart_job.py diff --git a/cloudinit/parts/__init__.py b/cloudinit/parts/__init__.py deleted file mode 100644 index 20d4bd3b..00000000 --- a/cloudinit/parts/__init__.py +++ /dev/null @@ -1,117 +0,0 @@ -import os - -from cloudinit import util -from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) - -CONTENT_END = "__end__" -CONTENT_START = "__begin__" -PART_CONTENT_TYPES = ["text/part-handler"] -PART_HANDLER_FN_TMPL = 'part-handler-%03d' -UNDEF_HANDLER_VERSION = 1 - - -class PartHandler(object): - def __init__(self, frequency, version=2): - self.handler_version = version - self.frequency = frequency - - def __repr__(self): - return "%s: [%s]" % (self.__class__.__name__, self.list_types()) - - def list_types(self): - raise NotImplementedError() - - def handle_part(self, data, ctype, filename, payload, frequency): - return self._handle_part(data, ctype, filename, payload, frequency) - - def _handle_part(self, data, ctype, filename, payload, frequency): - raise NotImplementedError() - - -def fixup_module(mod): - if not hasattr(mod, "handler_version"): - setattr(mod, "handler_version", UNDEF_HANDLER_VERSION) - if not hasattr(mod, 'list_types'): - def empty_types(): - return [] - setattr(mod, 'list_types', empty_types) - if not hasattr(mod, frequency): - setattr(mod, 'frequency', PER_INSTANCE) - if not hasattr(mod, 'handle_part'): - def empty_handler(data, ctype, filename, payload): - pass - setattr(mod, 'handle_part', empty_handler) - return mod - - -def find_module_files(root_dir): - entries = dict() - for fname in glob.glob(os.path.join(root_dir, "*.py")): - if not os.path.isfile(fname): - continue - modname = os.path.basename(fname)[0:-3] - entries[fname] = modname - return entries - - -def run_part(mod, data, ctype, filename, payload, frequency): - # only add the handler if the module should run - mod_freq = getattr(mod, "frequency") - if not (mod_freq == PER_ALWAYS or - (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)): - return - try: - mod_ver = getattr(mod, 'handler_version') - if mod_ver == 1: - mod.handle_part(data, ctype, filename, payload) - else: - mod.handle_part(data, ctype, filename, payload, frequency) - except: - LOG.exception("Failed calling mod %s (%s, %s, %s) with frequency %s", mod, ctype, filename, mod_ver, frequency) - - -def call_begin(mod, data, frequency): - run_part(mod, data, CONTENT_START, None, None, frequency) - - -def call_end(mod, data, frequency): - run_part(mod, data, CONTENT_END, None, None, frequency) - - -def walker_handle_handler(pdata, _ctype, _filename, payload): - curcount = pdata['handlercount'] - modname = PART_HANDLER_FN_TMPL % (curcount) - frequency = pdata['frequency'] - modfname = os.path.join(pdata['handlerdir'], "%s.py" % (modname)) - # TODO: Check if path exists?? - util.write_file(modfname, payload, 0600) - handlers = pdata['handlers'] - try: - mod = fixup_module(importer.import_module(modname)) - handlers.register(mod) - call_begin(mod, pdata['data'], frequency) - pdata['handlercount'] = curcount + 1 - except: - LOG.exception("Failed at registered python file %s", modfname) - - -def walker_callback(pdata, ctype, filename, payload): - # data here is the part_handlers array and then the data to pass through - if ctype in PART_CONTENT_TYPES: - walker_handle_handler(pdata, ctype, filename, payload) - return - handlers = pdata['handlers'] - if ctype not in handlers: - if ctype == "text/x-not-multipart": - # Extract the first line or 24 bytes for displaying in the log - start = payload.split("\n", 1)[0][:24] - if start < payload: - details = "starting '%s...'" % start.encode("string-escape") - else: - details = repr(payload) - LOG.warning("Unhandled non-multipart userdata: %s", details) - return - run_part(handlers[ctype], pdata['data'], ctype, filename, payload, pdata['frequency']) \ No newline at end of file diff --git a/cloudinit/parts/boot_hook.py b/cloudinit/parts/boot_hook.py deleted file mode 100644 index 881ffc58..00000000 --- a/cloudinit/parts/boot_hook.py +++ /dev/null @@ -1,42 +0,0 @@ -import os - -from cloudinit import util -from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) -from cloudinit import log as logging -from cloudinit import parts - -LOG = logging.getLogger(__name__) - - - -class BootHookPartHandler(parts.PartHandler): - def __init__(self, boothook_dir, instance_id): - parts.PartHandler.__init__(self, PER_ALWAYS) - self.boothook_dir = boothook_dir - self.instance_id = instance_id - - def list_types(self): - return ['text/cloud-boothook'] - - def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype in [CONTENT_START, CONTENT_END]: - return - - filename = util.clean_filename(filename) - payload = util.dos2unix(payload) - prefix = "#cloud-boothook" - start = 0 - if payload.startswith(prefix): - start = len(prefix) + 1 - - filepath = os.path.join(self.boothook_dir, filename) - util.write_file(filepath, payload[start:], 0700) - try: - env = os.environ.copy() - env['INSTANCE_ID'] = str(self.instance_id) - util.subp([filepath], env=env) - except util.ProcessExecutionError as e: - LOG.error("Boothooks script %s returned %s", filepath, e.exit_code) - except Exception as e: - LOG.error("Boothooks unknown exception %s when running %s", e, filepath) - diff --git a/cloudinit/parts/cloud_config.py b/cloudinit/parts/cloud_config.py deleted file mode 100644 index dab0e5f5..00000000 --- a/cloudinit/parts/cloud_config.py +++ /dev/null @@ -1,36 +0,0 @@ -import os - -from cloudinit import util -from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) -from cloudinit import log as logging -from cloudinit import parts - -LOG = logging.getLogger(__name__) - - - -class CloudConfigPartHandler(parts.PartHandler): - def __init__(self, cloud_fn): - parts.PartHandler.__init__(self, PER_ALWAYS) - self.cloud_buf = [] - self.cloud_fn = cloud_fn - - def list_types(self): - return ['text/cloud-config'] - - def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype == CONTENT_START: - self.cloud_buf = [] - return - - if ctype == CONTENT_END: - payload = "\n".join(self.cloud_buf) - util.write_file(self.cloud_fn, payload, 0600) - self.cloud_buf = [] - return - - filename = util.clean_filename(filename) - entry = "\n".join(["#%s" % (filename), str(payload)]) - self.config_buf.append(entry) - - diff --git a/cloudinit/parts/shell_script.py b/cloudinit/parts/shell_script.py deleted file mode 100644 index a248f198..00000000 --- a/cloudinit/parts/shell_script.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -from cloudinit import util -from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) -from cloudinit import log as logging -from cloudinit import parts - -LOG = logging.getLogger(__name__) - - -class ShellScriptPartHandler(parts.PartHandler): - - def __init__(self, script_dir): - parts.PartHandler.__init__(self, PER_ALWAYS) - self.script_dir = script_dir - - def list_types(self): - return ['text/x-shellscript'] - - def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype in [CONTENT_START, CONTENT_END]: - # maybe delete existing things here - return - - filename = util.clean_filename(filename) - payload = util.dos2unix(payload) - util.write_file(os.path.join(self.script_dir, filename), payload, 0700) diff --git a/cloudinit/parts/upstart_job.py b/cloudinit/parts/upstart_job.py deleted file mode 100644 index 7b290d26..00000000 --- a/cloudinit/parts/upstart_job.py +++ /dev/null @@ -1,30 +0,0 @@ -import os - -from cloudinit import util -from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) -from cloudinit import log as logging -from cloudinit import parts - -LOG = logging.getLogger(__name__) - - -class UpstartJobPartHandler(parts.PartHandler): - def __init__(self, upstart_dir): - parts.PartHandler.__init__(self, PER_INSTANCE) - self.upstart_dir = upstart_dir - - def list_types(self): - return ['text/upstart-job'] - - def _handle_part(self, _data, ctype, filename, payload, frequency): - if ctype in [CONTENT_START, CONTENT_END]: - return - - filename = utils.clean_filename(filename) - (name, ext) = os.path.splitext(filename) - ext = ext.lower() - if ext != ".conf": - filename = filename + ".conf" - - payload = util.dos2unix(payload) - util.write_file(os.path.join(self.upstart_dir, filename), payload, 0644) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py deleted file mode 100644 index f35e5d38..00000000 --- a/cloudinit/user_data.py +++ /dev/null @@ -1,271 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import hashlib -import os -import urllib - -import email -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText -from email.mime.base import MIMEBase - -import yaml - -from cloudinit import url_helper -from cloudinit import util - - -# Different file beginnings to there content type -INCLUSION_TYPES_MAP = { - '#include': 'text/x-include-url', - '#include-once': 'text/x-include-once-url', - '#!': 'text/x-shellscript', - '#cloud-config': 'text/cloud-config', - '#upstart-job': 'text/upstart-job', - '#part-handler': 'text/part-handler', - '#cloud-boothook': 'text/cloud-boothook', - '#cloud-config-archive': 'text/cloud-config-archive', -} - -# Various special content types -TYPE_NEEDED = ["text/plain", "text/x-not-multipart"] -INCLUDE_TYPES = ['text/x-include-url', 'text/x-include-once-url'] -ARCHIVE_TYPES = ["text/cloud-config-archive"] -UNDEF_TYPE = "text/plain" -ARCHIVE_UNDEF_TYPE = "text/cloud-config" -NOT_MULTIPART_TYPE = "text/x-not-multipart" -OCTET_TYPE = 'application/octet-stream' - -# Sorted longest first -INCLUSION_SRCH = sorted(INCLUSION_TYPES_MAP.keys(), key=(lambda e: 0 - len(e))) - -# Msg header used to track attachments -ATTACHMENT_FIELD = 'Number-Attachments' - -# This will be used to create a filename from a url (or like) entry -# When we want to make sure a entry isn't included more than once across sessions. -INCLUDE_ONCE_HASHER = 'md5' - -# For those pieces without filenames -PART_FN_TPL = 'part-%03d' - - -class UserDataProcessor(object): - def __init__(self, paths): - self.paths = paths - - def process(self, blob): - base_msg = convert_string(blob) - process_msg = MIMEMultipart() - self._process_msg(base_msg, process_msg) - return process_msg - - def _process_msg(self, base_msg, append_msg): - for part in base_msg.walk(): - # multipart/* are just containers - if part.get_content_maintype() == 'multipart': - continue - - ctype = None - ctype_orig = part.get_content_type() - payload = part.get_payload(decode=True) - - if not ctype_orig: - ctype_orig = UNDEF_TYPE - - if ctype_orig in TYPE_NEEDED: - ctype = type_from_starts_with(payload) - - if ctype is None: - ctype = ctype_orig - - if ctype in INCLUDE_TYPES: - self._do_include(payload, append_msg) - continue - - if ctype in ARCHIVE_TYPES: - self._explode_archive(payload, append_msg) - continue - - if 'Content-Type' in base_msg: - base_msg.replace_header('Content-Type', ctype) - else: - base_msg['Content-Type'] = ctype - - self._attach_part(append_msg, part) - - def _get_include_once_filename(self, entry): - msum = hashlib.new(INCLUDE_ONCE_HASHER) - msum.update(entry) - entry_fn = msum.hexdigest()[0:64] # Don't get to long now - return os.path.join(self.paths.get_ipath_cur('data'), 'urlcache', entry_fn) - - def _do_include(self, content, append_msg): - # is just a list of urls, one per line - # also support '#include ' - for line in content.splitlines(): - includeonce = False - if line in ("#include", "#include-once"): - continue - if line.startswith("#include-once"): - line = line[len("#include-once"):].lstrip() - includeonce = True - elif line.startswith("#include"): - line = line[len("#include"):].lstrip() - if line.startswith("#"): - continue - include_url = line.strip() - if not include_url: - continue - - includeonce_filename = self._get_include_once_filename(include_url) - if includeonce and os.path.isfile(includeonce_filename): - content = util.load_file(includeonce_filename) - else: - (content, st) = url_helper.readurl(include_url) - if includeonce and url_helper.ok_http_code(st): - util.write_file(includeonce_filename, content, mode=0600) - if not url_helper.ok_http_code(st): - content = '' - - new_msg = convert_string(content) - self._process_msg(new_msg, append_msg) - - def _explode_archive(self, archive, append_msg): - try: - entries = yaml.load(archive) - except: - entries = [] - if not isinstance(entries, (list, set)): - # TODO raise? - entries = [] - - for ent in entries: - # ent can be one of: - # dict { 'filename' : 'value', 'content' : 'value', 'type' : 'value' } - # filename and type not be present - # or - # scalar(payload) - if isinstance(ent, str): - ent = {'content': ent} - if not isinstance(ent, (dict)): - # TODO raise? - continue - - content = ent.get('content', '') - mtype = ent.get('type') - if not mtype: - mtype = type_from_starts_with(content, ARCHIVE_UNDEF_TYPE) - - maintype, subtype = mtype.split('/', 1) - if maintype == "text": - msg = MIMEText(content, _subtype=subtype) - else: - msg = MIMEBase(maintype, subtype) - msg.set_payload(content) - - if 'filename' in ent: - msg.add_header('Content-Disposition', 'attachment', filename=ent['filename']) - - for header in ent.keys(): - if header in ('content', 'filename', 'type'): - continue - msg.add_header(header, ent['header']) - - self._attach_part(append_msg, msg) - - def _multi_part_count(self, outer_msg, new_count=None): - """ - Return the number of attachments to this MIMEMultipart by looking - at its 'Number-Attachments' header. - """ - if ATTACHMENT_FIELD not in outer_msg: - outer_msg[ATTACHMENT_FIELD] = str(0) - - if new_count is not None: - outer_msg.replace_header(ATTACHMENT_FIELD, str(new_count)) - - fetched_count = 0 - try: - fetched_count = int(outer_msg.get(ATTACHMENT_FIELD)) - except (ValueError, TypeError): - outer_msg.replace_header(ATTACHMENT_FIELD, str(fetched_count)) - return fetched_count - - def _attach_part(self, outer_msg, part): - """ - Attach an part to an outer message. outermsg must be a MIMEMultipart. - Modifies a header in the message to keep track of number of attachments. - """ - cur = self._multi_part_count(outer_msg) - if not part.get_filename(): - part.add_header('Content-Disposition', 'attachment', filename=PART_FN_TPL % (cur + 1)) - outer_msg.attach(part) - self._multi_part_count(outer_msg, cur + 1) - - -# Callback is a function that will be called with -# (data, content_type, filename, payload) -def walk(ud_msg, callback, data): - partnum = 0 - for part in ud_msg.walk(): - # multipart/* are just containers - if part.get_content_maintype() == 'multipart': - continue - - ctype = part.get_content_type() - if ctype is None: - ctype = OCTET_TYPE - - filename = part.get_filename() - if not filename: - filename = PART_FN_TPL % partnum - - callback(data, ctype, filename, part.get_payload(decode=True)) - partnum = partnum + 1 - - -def convert_string(self, raw_data, headers=None): - if not data: - data = '' - if not headers: - headers = {} - data = util.decomp_str(raw_data) - if "mime-version:" in data[0:4096].lower(): - msg = email.message_from_string(data) - for (key, val) in headers.items(): - if key in msg: - msg.replace_header(key, val) - else: - msg[key] = val - else: - mtype = headers.get("Content-Type", NOT_MULTIPART_TYPE) - maintype, subtype = mtype.split("/", 1) - msg = MIMEBase(maintype, subtype, *headers) - msg.set_payload(data) - return msg - - -def type_from_starts_with(payload, default=None): - for text in INCLUSION_SRCH: - if payload.startswith(text): - return INCLUSION_TYPES_MAP[text] - return default diff --git a/cloudinit/user_data/__init__.py b/cloudinit/user_data/__init__.py new file mode 100644 index 00000000..6264a6cc --- /dev/null +++ b/cloudinit/user_data/__init__.py @@ -0,0 +1,210 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import os +import glob + +import email + +from email.mime.base import MIMEBase + +from cloudinit import log as logging +from cloudinit import util + +from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) + +LOG = logging.getLogger(__name__) + +# Special content types that signal the start and end of processing +CONTENT_END = "__end__" +CONTENT_START = "__begin__" +CONTENT_SIGNALS = [CONTENT_START, CONTENT_END] + +# Used when a part-handler type is encountered +# to allow for registration of new types. +PART_CONTENT_TYPES = ["text/part-handler"] +PART_HANDLER_FN_TMPL = 'part-handler-%03d' + +# For parts without filenames +PART_FN_TPL = 'part-%03d' + +# Used as the content type when a message is not multipart +# and it doesn't contain its own content-type +NOT_MULTIPART_TYPE = "text/x-not-multipart" + +# Different file beginnings to there content type +INCLUSION_TYPES_MAP = { + '#include': 'text/x-include-url', + '#include-once': 'text/x-include-once-url', + '#!': 'text/x-shellscript', + '#cloud-config': 'text/cloud-config', + '#upstart-job': 'text/upstart-job', + '#part-handler': 'text/part-handler', + '#cloud-boothook': 'text/cloud-boothook', + '#cloud-config-archive': 'text/cloud-config-archive', +} + +# Sorted longest first +INCLUSION_SRCH = sorted(INCLUSION_TYPES_MAP.keys(), key=(lambda e: 0 - len(e))) + + +class PartHandler(object): + def __init__(self, frequency, version=2): + self.handler_version = version + self.frequency = frequency + + def __repr__(self): + return "%s: [%s]" % (self.__class__.__name__, self.list_types()) + + def list_types(self): + raise NotImplementedError() + + def handle_part(self, data, ctype, filename, payload, frequency): + return self._handle_part(data, ctype, filename, payload, frequency) + + def _handle_part(self, data, ctype, filename, payload, frequency): + raise NotImplementedError() + + +def fixup_module(mod): + if not hasattr(mod, "handler_version"): + setattr(mod, "handler_version", 1) + if not hasattr(mod, 'list_types'): + def empty_types(): + return [] + setattr(mod, 'list_types', empty_types) + if not hasattr(mod, frequency): + setattr(mod, 'frequency', PER_INSTANCE) + if not hasattr(mod, 'handle_part'): + def empty_handler(data, ctype, filename, payload): + pass + setattr(mod, 'handle_part', empty_handler) + return mod + + +def run_part(mod, data, ctype, filename, payload, frequency): + mod_freq = mod.frequency + if not (mod_freq == PER_ALWAYS or + (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)): + return + mod_ver = mod.handler_version + try: + if mod_ver == 1: + mod.handle_part(data, ctype, filename, payload) + else: + mod.handle_part(data, ctype, filename, payload, frequency) + except: + LOG.exception("Failed calling mod %s (%s, %s, %s) with frequency %s", mod, ctype, filename, mod_ver, frequency) + + +def call_begin(mod, data, frequency): + run_part(mod, data, CONTENT_START, None, None, frequency) + + +def call_end(mod, data, frequency): + run_part(mod, data, CONTENT_END, None, None, frequency) + + +def walker_handle_handler(pdata, _ctype, _filename, payload): + curcount = pdata['handlercount'] + modname = PART_HANDLER_FN_TMPL % (curcount) + frequency = pdata['frequency'] + modfname = os.path.join(pdata['handlerdir'], "%s.py" % (modname)) + # TODO: Check if path exists?? + util.write_file(modfname, payload, 0600) + handlers = pdata['handlers'] + try: + mod = fixup_module(importer.import_module(modname)) + handlers.register(mod) + call_begin(mod, pdata['data'], frequency) + pdata['handlercount'] = curcount + 1 + except: + LOG.exception("Failed at registered python file: %s", modfname) + + +def walker_callback(pdata, ctype, filename, payload): + if ctype in PART_CONTENT_TYPES: + walker_handle_handler(pdata, ctype, filename, payload) + return + handlers = pdata['handlers'] + if ctype not in handlers: + if ctype == NOT_MULTIPART_TYPE: + # Extract the first line or 24 bytes for displaying in the log + start = payload.split("\n", 1)[0][:24] + if start < payload: + details = "starting '%s...'" % start.encode("string-escape") + else: + details = repr(payload) + LOG.warning("Unhandled non-multipart userdata: %s", details) + return + run_part(handlers[ctype], pdata['data'], ctype, filename, payload, pdata['frequency']) + + +# Callback is a function that will be called with +# (data, content_type, filename, payload) +def walk(msg, callback, data): + partnum = 0 + for part in msg.walk(): + # multipart/* are just containers + if part.get_content_maintype() == 'multipart': + continue + + ctype = part.get_content_type() + if ctype is None: + ctype = OCTET_TYPE + + filename = part.get_filename() + if not filename: + filename = PART_FN_TPL % (partnum) + + callback(data, ctype, filename, part.get_payload(decode=True)) + partnum = partnum + 1 + + +# Coverts a raw string into a mime message +def convert_string(self, raw_data, headers=None): + if not raw_data: + raw_data = '' + if not headers: + headers = {} + data = util.decomp_str(raw_data) + if "mime-version:" in data[0:4096].lower(): + msg = email.message_from_string(data) + for (key, val) in headers.items(): + if key in msg: + msg.replace_header(key, val) + else: + msg[key] = val + else: + mtype = headers.get("Content-Type", NOT_MULTIPART_TYPE) + maintype, subtype = mtype.split("/", 1) + msg = MIMEBase(maintype, subtype, *headers) + msg.set_payload(data) + return msg + + +def type_from_starts_with(payload, default=None): + for text in INCLUSION_SRCH: + if payload.startswith(text): + return INCLUSION_TYPES_MAP[text] + return default \ No newline at end of file diff --git a/cloudinit/user_data/boot_hook.py b/cloudinit/user_data/boot_hook.py new file mode 100644 index 00000000..4ce398ac --- /dev/null +++ b/cloudinit/user_data/boot_hook.py @@ -0,0 +1,65 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os + +from cloudinit import log as logging +from cloudinit import user_data as ud +from cloudinit import util + +from cloudinit.settings import (PER_ALWAYS) + +LOG = logging.getLogger(__name__) + + +class BootHookPartHandler(ud.PartHandler): + def __init__(self, boothook_dir, instance_id): + ud.PartHandler.__init__(self, PER_ALWAYS) + self.boothook_dir = boothook_dir + self.instance_id = instance_id + + def list_types(self): + return [ + ud.type_from_starts_with("#cloud-boothook"), + ] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype in ud.CONTENT_SIGNALS: + return + + filename = util.clean_filename(filename) + payload = util.dos2unix(payload) + prefix = "#cloud-boothook" + start = 0 + if payload.startswith(prefix): + start = len(prefix) + 1 + + filepath = os.path.join(self.boothook_dir, filename) + util.write_file(filepath, payload[start:], 0700) + try: + env = os.environ.copy() + env['INSTANCE_ID'] = str(self.instance_id) + util.subp([filepath], env=env) + except util.ProcessExecutionError as e: + LOG.error("Boothooks script %s returned %s", filepath, e.exit_code) + except Exception as e: + LOG.error("Boothooks unknown exception %s when running %s", e, filepath) diff --git a/cloudinit/user_data/cloud_config.py b/cloudinit/user_data/cloud_config.py new file mode 100644 index 00000000..1c43f3a1 --- /dev/null +++ b/cloudinit/user_data/cloud_config.py @@ -0,0 +1,59 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import os + +from cloudinit import log as logging +from cloudinit import user_data as ud +from cloudinit import util + +from cloudinit.settings import (PER_ALWAYS) + +LOG = logging.getLogger(__name__) + + +class CloudConfigPartHandler(ud.PartHandler): + def __init__(self, cloud_fn): + ud.PartHandler.__init__(self, PER_ALWAYS) + self.cloud_buf = [] + self.cloud_fn = cloud_fn + + def list_types(self): + return [ + ud.type_from_starts_with("#cloud-config"), + ] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype == ud.CONTENT_START: + self.cloud_buf = [] + return + + if ctype == ud.CONTENT_END: + payload = "\n".join(self.cloud_buf) + util.write_file(self.cloud_fn, payload, 0600) + self.cloud_buf = [] + return + + filename = util.clean_filename(filename) + entry = "\n".join(["#%s" % (filename), str(payload)]) + self.config_buf.append(entry) diff --git a/cloudinit/user_data/processor.py b/cloudinit/user_data/processor.py new file mode 100644 index 00000000..d4de9470 --- /dev/null +++ b/cloudinit/user_data/processor.py @@ -0,0 +1,205 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import hashlib +import os +import urllib + +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.mime.base import MIMEBase + +import yaml + +from cloudinit import url_helper +from cloudinit import user_data as ud +from cloudinit import util + +# Various special content types +TYPE_NEEDED = ["text/plain", "text/x-not-multipart"] +INCLUDE_TYPES = ['text/x-include-url', 'text/x-include-once-url'] +ARCHIVE_TYPES = ["text/cloud-config-archive"] +UNDEF_TYPE = "text/plain" +ARCHIVE_UNDEF_TYPE = "text/cloud-config" +OCTET_TYPE = 'application/octet-stream' + +# Msg header used to track attachments +ATTACHMENT_FIELD = 'Number-Attachments' + +# This will be used to create a filename from a url (or like) entry +# When we want to make sure a entry isn't included more than once across sessions. +INCLUDE_ONCE_HASHER = 'md5' + + +class UserDataProcessor(object): + def __init__(self, paths): + self.paths = paths + + def process(self, blob): + base_msg = ud.convert_string(blob) + process_msg = MIMEMultipart() + self._process_msg(base_msg, process_msg) + return process_msg + + def _process_msg(self, base_msg, append_msg): + for part in base_msg.walk(): + # multipart/* are just containers + if part.get_content_maintype() == 'multipart': + continue + + ctype = None + ctype_orig = part.get_content_type() + payload = part.get_payload(decode=True) + + if not ctype_orig: + ctype_orig = UNDEF_TYPE + + if ctype_orig in TYPE_NEEDED: + ctype = ud.type_from_starts_with(payload) + + if ctype is None: + ctype = ctype_orig + + if ctype in INCLUDE_TYPES: + self._do_include(payload, append_msg) + continue + + if ctype in ARCHIVE_TYPES: + self._explode_archive(payload, append_msg) + continue + + if 'Content-Type' in base_msg: + base_msg.replace_header('Content-Type', ctype) + else: + base_msg['Content-Type'] = ctype + + self._attach_part(append_msg, part) + + def _get_include_once_filename(self, entry): + msum = hashlib.new(INCLUDE_ONCE_HASHER) + msum.update(entry) + entry_fn = msum.hexdigest()[0:64] # Don't get to long now + return os.path.join(self.paths.get_ipath_cur('data'), 'urlcache', entry_fn) + + def _do_include(self, content, append_msg): + # is just a list of urls, one per line + # also support '#include ' + for line in content.splitlines(): + includeonce = False + if line in ("#include", "#include-once"): + continue + if line.startswith("#include-once"): + line = line[len("#include-once"):].lstrip() + includeonce = True + elif line.startswith("#include"): + line = line[len("#include"):].lstrip() + if line.startswith("#"): + continue + include_url = line.strip() + if not include_url: + continue + + includeonce_filename = self._get_include_once_filename(include_url) + if includeonce and os.path.isfile(includeonce_filename): + content = util.load_file(includeonce_filename) + else: + (content, st) = url_helper.readurl(include_url) + if includeonce and url_helper.ok_http_code(st): + util.write_file(includeonce_filename, content, mode=0600) + if not url_helper.ok_http_code(st): + content = '' + + new_msg = ud.convert_string(content) + self._process_msg(new_msg, append_msg) + + def _explode_archive(self, archive, append_msg): + try: + entries = yaml.load(archive) + except: + entries = [] + if not isinstance(entries, (list, set)): + # TODO raise? + entries = [] + + for ent in entries: + # ent can be one of: + # dict { 'filename' : 'value', 'content' : 'value', 'type' : 'value' } + # filename and type not be present + # or + # scalar(payload) + if isinstance(ent, str): + ent = {'content': ent} + if not isinstance(ent, (dict)): + # TODO raise? + continue + + content = ent.get('content', '') + mtype = ent.get('type') + if not mtype: + mtype = ud.type_from_starts_with(content, ARCHIVE_UNDEF_TYPE) + + maintype, subtype = mtype.split('/', 1) + if maintype == "text": + msg = MIMEText(content, _subtype=subtype) + else: + msg = MIMEBase(maintype, subtype) + msg.set_payload(content) + + if 'filename' in ent: + msg.add_header('Content-Disposition', 'attachment', filename=ent['filename']) + + for header in ent.keys(): + if header in ('content', 'filename', 'type'): + continue + msg.add_header(header, ent['header']) + + self._attach_part(append_msg, msg) + + def _multi_part_count(self, outer_msg, new_count=None): + """ + Return the number of attachments to this MIMEMultipart by looking + at its 'Number-Attachments' header. + """ + if ATTACHMENT_FIELD not in outer_msg: + outer_msg[ATTACHMENT_FIELD] = str(0) + + if new_count is not None: + outer_msg.replace_header(ATTACHMENT_FIELD, str(new_count)) + + fetched_count = 0 + try: + fetched_count = int(outer_msg.get(ATTACHMENT_FIELD)) + except (ValueError, TypeError): + outer_msg.replace_header(ATTACHMENT_FIELD, str(fetched_count)) + return fetched_count + + def _attach_part(self, outer_msg, part): + """ + Attach an part to an outer message. outermsg must be a MIMEMultipart. + Modifies a header in the message to keep track of number of attachments. + """ + cur = self._multi_part_count(outer_msg) + if not part.get_filename(): + fn = ud.PART_FN_TPL % (cur + 1) + part.add_header('Content-Disposition', 'attachment', filename=fn) + outer_msg.attach(part) + self._multi_part_count(outer_msg, cur + 1) diff --git a/cloudinit/user_data/shell_script.py b/cloudinit/user_data/shell_script.py new file mode 100644 index 00000000..d666b9c4 --- /dev/null +++ b/cloudinit/user_data/shell_script.py @@ -0,0 +1,53 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import os + +from cloudinit import log as logging +from cloudinit import user_data as ud +from cloudinit import util + +from cloudinit.settings import (PER_INSTANCE) + +LOG = logging.getLogger(__name__) + + +class ShellScriptPartHandler(ud.PartHandler): + + def __init__(self, script_dir): + ud.PartHandler.__init__(self, PER_ALWAYS) + self.script_dir = script_dir + + def list_types(self): + return [ + ud.type_from_starts_with("#!"), + ] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype in ud.CONTENT_SIGNALS: + # maybe delete existing things here + return + + filename = util.clean_filename(filename) + payload = util.dos2unix(payload) + util.write_file(os.path.join(self.script_dir, filename), payload, 0700) diff --git a/cloudinit/user_data/upstart_job.py b/cloudinit/user_data/upstart_job.py new file mode 100644 index 00000000..d15e66c4 --- /dev/null +++ b/cloudinit/user_data/upstart_job.py @@ -0,0 +1,56 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import os + +from cloudinit import log as logging +from cloudinit import user_data as ud +from cloudinit import util + +from cloudinit.settings import (PER_INSTANCE) + +LOG = logging.getLogger(__name__) + + +class UpstartJobPartHandler(ud.PartHandler): + def __init__(self, upstart_dir): + ud.PartHandler.__init__(self, PER_INSTANCE) + self.upstart_dir = upstart_dir + + def list_types(self): + return [ + ud.type_from_starts_with("#upstart-job"), + ] + + def _handle_part(self, _data, ctype, filename, payload, frequency): + if ctype in ud.CONTENT_SIGNALS: + return + + filename = utils.clean_filename(filename) + (name, ext) = os.path.splitext(filename) + ext = ext.lower() + if ext != ".conf": + filename = filename + ".conf" + + payload = util.dos2unix(payload) + util.write_file(os.path.join(self.upstart_dir, filename), payload, 0644) -- cgit v1.2.3 From e78c20b9aba7e031d7e29e9ab4763a6ab430a0dd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 9 Jun 2012 12:31:56 -0700 Subject: Move most of this functionality to either cloud-init object or utils. --- cloudinit/__init__.py | 93 ++++----------------------------------------------- 1 file changed, 6 insertions(+), 87 deletions(-) diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py index c38b3b09..ab0603e0 100644 --- a/cloudinit/__init__.py +++ b/cloudinit/__init__.py @@ -1,11 +1,12 @@ # vi: ts=4 expandtab # -# Common code for the EC2 initialisation scripts in Ubuntu -# Copyright (C) 2008-2009 Canonical Ltd +# Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. # -# Author: Soren Hansen +# Author: Scott Moser # Author: Juerg Haefliger +# Author: Joshua Harlow # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -18,99 +19,17 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# - -import os - -import sys -import errno -import subprocess -import yaml -import glob -from cloudinit import log as logging -from cloudinit import sources -from cloudinit import url_helper as uhelp from cloudinit import util -from cloudinit.settings import (VAR_LIB_DIR, CFG_BUILTIN, CLOUD_CONFIG, - BOOT_FINISHED, CUR_INSTANCE_LINK) - -LOG = logging.getLogger(__name__) - -INIT_SUBDIRS = [ - os.path.join(VAR_LIB_DIR, 'scripts'), - os.path.join(VAR_LIB_DIR, 'scripts', 'per-instance'), - os.path.join(VAR_LIB_DIR, 'scripts', 'per-once'), - os.path.join(VAR_LIB_DIR, 'scripts', 'per-boot'), - os.path.join(VAR_LIB_DIR, 'seed'), - os.path.join(VAR_LIB_DIR, 'instances'), - os.path.join(VAR_LIB_DIR, 'handlers'), - os.path.join(VAR_LIB_DIR, 'sem'), - os.path.join(VAR_LIB_DIR, 'data'), -] - - -def initfs(): - util.ensure_dirs(INIT_SUBDIRS) - cfg = util.get_base_cfg(CLOUD_CONFIG, get_builtin_cfg()) - log_file = util.get_cfg_option_str(cfg, 'def_log_file', None) - perms = util.get_cfg_option_str(cfg, 'syslog_fix_perms', None) - if log_file: - util.ensure_file(log_file) - if perms: - (u, g) = perms.split(':', 1) - if u == "-1" or u == "None": - u = None - if g == "-1" or g == "None": - g = None - util.chownbyname(log_file, u, g) - - -def purge_cache(rmcur=True): - rmlist = [BOOT_FINISHED] - if rmcur: - rmlist.append(CUR_INSTANCE_LINK) - for f in rmlist: - util.unlink(f) - return True +from cloudinit.settings import (CFG_BUILTIN, CLOUD_CONFIG) def get_base_cfg(cfg_path=None): - if cfg_path is None: + if not cfg_path: cfg_path = CLOUD_CONFIG return util.get_base_cfg(cfg_path, get_builtin_cfg()) def get_builtin_cfg(): return dict(CFG_BUILTIN) - - -def list_sources(cfg_list, depends): - return sources.list_sources(cfg_list, depends, ["cloudinit", ""]) - - -def get_cmdline_url(names=None, starts=None, cmdline=None): - if cmdline is None: - cmdline = util.get_cmdline() - if not names: - names = ('cloud-config-url', 'url') - if not starts: - starts = "#cloud-config" - - data = util.keyval_str_to_dict(cmdline) - url = None - key = None - for key in names: - if key in data: - url = data[key] - break - - if url is None: - return (None, None, None) - - contents = uhelp.readurl(url) - if contents.startswith(starts): - return (key, url, contents) - - return (key, url, None) -- cgit v1.2.3 From b587b3184673b43af57e7f9119794c0ef7f166a5 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 9 Jun 2012 12:32:50 -0700 Subject: More init fs here, cleanup after user_data file moves + other cleanups. --- cloudinit/cloud.py | 199 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 144 insertions(+), 55 deletions(-) diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index f9c0d531..80d4f1ce 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -1,34 +1,57 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + from time import time import cPickle as pickle + import contextlib import copy import os import sys import weakref -from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, - OLD_CLOUD_CONFIG, CLOUD_CONFIG, - CUR_INSTANCE_LINK) +from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS) +from cloudinit.settings import (OLD_CLOUD_CONFIG, CLOUD_CONFIG) + from cloudinit import (get_builtin_cfg, get_base_cfg) from cloudinit import log as logging from cloudinit import sources from cloudinit import util -from cloudinit import user_data from cloudinit import handlers -from cloudinit import parts -from cloudinit.parts import boot_hook as bh_part -from cloudinit.parts import cloud_config as cc_part -from cloudinit.parts import upstart_job as up_part -from cloudinit.parts import shell_script as ss_part +from cloudinit import user_data as ud +from cloudinit.user_data import boot_hook as bh_part +from cloudinit.user_data import cloud_config as cc_part +from cloudinit.user_data import processor as ud_proc +from cloudinit.user_data import shell_script as ss_part +from cloudinit.user_data import upstart_job as up_part LOG = logging.getLogger(__name__) class CloudSemaphores(object): - def __init__(self, paths): - self.paths = paths + def __init__(self, sem_path): + self.sem_path = sem_path # acquire lock on 'name' for given 'freq' and run function 'func' # if 'clear_on_fail' is True and 'func' throws an exception @@ -71,7 +94,7 @@ class CloudSemaphores(object): # here, but this should be ok due to the nature of when # and where cloud-init runs... (file writing is not a lock..) sem_file = self._getpath(name, freq) - contents = "%s\n" % str(time()) + contents = "%s: %s\n" % (os.getpid(), time()) try: util.write_file(sem_file, contents) except (IOError, OSError): @@ -87,28 +110,37 @@ class CloudSemaphores(object): return False def _get_path(self, name, freq): - sem_path = self.init.get_ipath("sem") + sem_path = self.sem_path if freq == PER_INSTANCE: return os.path.join(sem_path, name) return os.path.join(sem_path, "%s.%s" % (name, freq)) class CloudPaths(object): - def __init__(self, init): - self.config = CLOUD_CONFIG - self.old_config = OLD_CLOUD_CONFIG - self.var_dir = VAR_LIB_DIR - self.instance_link = CUR_INSTANCE_LINK - self.init = weakref.proxy(init) - self.upstart_conf_d = "/etc/init" - - def _get_path_key(self, name): - return PATH_MAP.get(name) + def __init__(self, sys_info): + self.cloud_dir = sys_info['cloud_dir'] + self.instance_link = os.path.join(self.cloud_dir, 'instance') + self.boot_finished = os.path.join(self.instance_link, "boot-finished") + self.upstart_conf_d = sys_info.get('upstart_dir') + self.template_dir = sys_info['templates_dir'] + self.seed_dir = os.path.join(self.cloud_dir, 'seed') + self.datasource = None + self.lookups = { + "handlers": "handlers", + "scripts": "scripts", + "sem": "sem", + "boothooks": "boothooks", + "userdata_raw": "user-data.txt", + "userdata": "user-data.txt.i", + "obj_pkl": "obj.pkl", + "cloud_config": "cloud-config.txt", + "data": "data", + } # get_ipath_cur: get the current instance path for an item def get_ipath_cur(self, name=None): - add_on = self._get_path_key(name) - ipath = os.path.join(self.var_dir, 'instance') + ipath = os.path.join(self.cloud_dir, 'instance') + add_on = self.lookups.get(name) if add_on: ipath = os.path.join(ipath, add_on) return ipath @@ -117,7 +149,7 @@ class CloudPaths(object): # for a name in dirmap def get_cpath(self, name=None): cpath = self.var_dir - add_on = self._get_path_key(name) + add_on = self.lookups.get(name) if add_on: cpath = os.path.join(cpath, add_on) return cpath @@ -125,18 +157,21 @@ class CloudPaths(object): # get_ipath : get the instance path for a name in pathmap # (/var/lib/cloud/instances//) def get_ipath(self, name=None): - iid = self.init.datasource.get_instance_id() - ipath = os.path.join(self.var_dir, 'instances', iid) - add_on = self._get_path_key(name) + if not self.datasource: + raise RuntimeError("Unable to get instance path, datasource not available/set.") + iid = self.datasource.get_instance_id() + ipath = os.path.join(self.cloud_dir, 'instances', iid) + add_on = self.lookups.get(name) if add_on: ipath = os.path.join(ipath, add_on) return ipath class CloudSimple(object): - def __init__(self, init): + def __init__(self, ci): self.datasource = init.datasource self.paths = init.paths + self.cfg = copy.deepcopy(ci.cfg) def get_userdata(self): return self.datasource.get_userdata() @@ -173,8 +208,8 @@ class CloudInit(object): self.ds_deps = ds_deps else: self.ds_deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] - self.paths = CloudPaths(self) self.cfg = self._read_cfg() + self.paths = CloudPaths(self.cfg['system_info']) def _read_cfg_old(self): # support reading the old ConfigObj format file and merging @@ -185,56 +220,110 @@ class CloudInit(object): ConfigObj = None if not ConfigObj: return {} - old_cfg = ConfigObj(self.paths.old_config_fn) + old_cfg = ConfigObj(OLD_CLOUD_CONFIG) return dict(old_cfg) - def read_cfg(self): - if not self.cfg: - self.cfg = self._read_cfg() - return self.cfg + def _initial_subdirs(self): + c_dir = self.paths.cloud_dir + initial_dirs = [ + os.path.join(c_dir, 'scripts'), + os.path.join(c_dir, 'scripts', 'per-instance'), + os.path.join(c_dir, 'scripts', 'per-once'), + os.path.join(c_dir, 'scripts', 'per-boot'), + os.path.join(c_dir, 'seed'), + os.path.join(c_dir, 'instances'), + os.path.join(c_dir, 'handlers'), + os.path.join(c_dir, 'sem'), + os.path.join(c_dir, 'data'), + ] + return initial_dirs + + def purge_cache(self, rmcur=True): + rmlist = [] + rmlist.append(self.paths.boot_finished) + if rmcur: + rmlist.append(self.paths.instance_link) + for f in rmlist: + util.unlink(f) + return len(rmlist) + + def init_fs(self): + util.ensure_dirs(self._initial_subdirs()) + log_file = util.get_cfg_option_str(self.cfg, 'def_log_file', None) + perms = util.get_cfg_option_str(self.cfg, 'syslog_fix_perms', None) + if log_file: + util.ensure_file(log_file) + if perms: + (u, g) = perms.split(':', 1) + if u == "-1" or u == "None": + u = None + if g == "-1" or g == "None": + g = None + util.chownbyname(log_file, u, g) def _read_cfg(self): starting_config = get_builtin_cfg() try: - conf = get_base_cfg(self.paths.config, starting_config) + conf = get_base_cfg(CLOUD_CONFIG, starting_config) except Exception: conf = starting_config old_conf = self._read_cfg_old() conf = util.mergedict(conf, old_conf) return conf - def restore_from_cache(self): + def _restore_from_cache(self): pickled_fn = self.paths.get_ipath_cur('obj_pkl') try: # we try to restore from a current link and static path # by using the instance link, if purge_cache was called # the file wont exist - self.datasource = pickle.loads(util.load_file(pickled_fn)) - return True + return pickle.loads(util.load_file(pickled_fn)) except Exception as e: LOG.debug("Failed loading pickled datasource from %s due to %s", pickled_fn, e) return False - + def write_to_cache(self): pickled_fn = self.paths.get_ipath_cur("obj_pkl") try: contents = pickle.dumps(self.datasource) util.write_file(pickled_fn, contents, mode=0400) except Exception as e: - LOG.debug("Failed pickling datasource to %s due to %s", pickled_fn, e) + LOG.debug("Failed pickling datasource to %s due to: %s", pickled_fn, e) return False - + + def _get_processor(self): + return ud_proc.UserDataProcessor(self.paths) + + def _get_datasources(self): + # Any config provided??? + pkg_list = self.cfg.get('datasource_pkg_list') or [] + # Add the defaults at the end + for n in [util.obj_name(sources), '']: + if n not in pkg_list: + pkg_list.append(n) + cfg_list = self.cfg.get('datasource_list') or [] + return (cfg_list, pkg_list) + def get_data_source(self): if self.datasource: return True - if self.restore_from_cache(): - LOG.debug("Restored from cache datasource: %s" % self.datasource) - return True - (ds, dsname) = sources.find_source(self.cfg, self.ds_deps) - LOG.debug("Loaded datasource %s:%s", dsname, ds) + ds = self._restore_from_cache() + if ds: + LOG.debug("Restored from cache datasource: %s" % ds) + else: + (cfg_list, pkg_list) = self._get_datasources() + ud_proc = self._get_processor() + (ds, dsname) = sources.find_source(self.cfg, + self.ds_deps, + cfg_list=cfg_list, + pkg_list=pkg_list, + ud_proc=ud_proc) + LOG.debug("Loaded datasource %s - %s", dsname, ds) self.datasource = ds + # This allows the paths obj to have an ipath function that works + self.paths.datasource = ds return True - + def set_cur_instance(self): # Ensure we are hooked into the right symlink for the current instance idir = self.paths.get_ipath() @@ -299,7 +388,7 @@ class CloudInit(object): handlers = CloudHandlers(self) # Add handlers in cdir - potential_handlers = parts.find_module_files(cdir) + potential_handlers = utils.find_modules(cdir) for (fname, modname) in potential_handlers.iteritems(): try: mod = parts.fixup_module(importer.import_module(modname)) @@ -329,7 +418,7 @@ class CloudInit(object): 'frequency': frequency, 'handlercount': 0, } - user_data.walk(data.get_userdata(), parts.walker_callback, data=part_data) + ud.walk(data.get_userdata(), parts.walker_callback, data=part_data) # Give callbacks opportunity to finalize called = [] @@ -394,11 +483,11 @@ class CloudHandlers(object): class CloudConfig(object): - def __init__(self, cfgfile, cloud): - self.cloud = CloudSimple(cloud) + self.cloud = cloud self.cfg = self._get_config(cfgfile) - self.sems = CloudSemaphores(self.cloud.paths) + self.paths = cloud.paths + self.sems = CloudSemaphores(self.paths.get_ipath("sem")) def _get_config(self, cfgfile): cfg = None @@ -433,5 +522,5 @@ class CloudConfig(object): if not freq: freq = def_freq c_name = "config-%s" % (name) - real_args = [name, copy.deepcopy(self.cfg), self.cloud, LOG, copy.deepcopy(args)] + real_args = [name, copy.deepcopy(self.cfg), CloudSimple(self.cloud), LOG, copy.deepcopy(args)] return self.sems.run_functor(c_name, freq, mod.handle, real_args) -- cgit v1.2.3 From 1b4f105bcc4f37644e896a8e090382ce049dac91 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 9 Jun 2012 12:33:20 -0700 Subject: Add copyright + index hash directly instead of get. --- cloudinit/importer.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/cloudinit/importer.py b/cloudinit/importer.py index deea0956..0344d0de 100644 --- a/cloudinit/importer.py +++ b/cloudinit/importer.py @@ -1,4 +1,24 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . import sys @@ -6,6 +26,6 @@ import sys def import_module(module_name): try: __import__(module_name) - return sys.modules.get(module_name, None) + return sys.modules[module_name] except ImportError as err: raise RuntimeError('Could not load module %s: %s' % (module_name, err)) -- cgit v1.2.3 From b5abbfe706fd862731433a77d5239eaca840304d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 9 Jun 2012 12:33:44 -0700 Subject: Add copyright. --- cloudinit/log.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/cloudinit/log.py b/cloudinit/log.py index 2dda405d..341b329b 100644 --- a/cloudinit/log.py +++ b/cloudinit/log.py @@ -1,4 +1,25 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + import logging import logging.handlers -- cgit v1.2.3 From 25dcab0f30bf0b0f15a4baf70348ce78d6d5da5d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 9 Jun 2012 12:34:06 -0700 Subject: Add copyright. --- cloudinit/mounting.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cloudinit/mounting.py b/cloudinit/mounting.py index b72f729a..b78cb5bd 100644 --- a/cloudinit/mounting.py +++ b/cloudinit/mounting.py @@ -1,3 +1,25 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + import contextlib from cloudinit import util -- cgit v1.2.3 From 5ee615c0542a664a7dd19bbfadcc7eeeaff0d6e1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 9 Jun 2012 12:34:52 -0700 Subject: Remove pathmap and make this a part of the CloudPath object, add copyright and format built in config --- cloudinit/settings.py | 63 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 830d970d..f58c4b52 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -1,40 +1,51 @@ -import os - -VAR_LIB_DIR = '/var/lib/cloud' -CUR_INSTANCE_LINK = os.path.join(VAR_LIB_DIR, "instance") -BOOT_FINISHED = os.path.join(CUR_INSTANCE_LINK, "boot-finished") -SEED_DIR = os.path.join(VAR_LIB_DIR, "seed") +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . CFG_ENV_NAME = "CLOUD_CFG" CLOUD_CONFIG = '/etc/cloud/cloud.cfg' OLD_CLOUD_CONFIG = '/etc/ec2-init/ec2-config.cfg' CFG_BUILTIN = { - 'datasource_list': ['NoCloud', - 'ConfigDrive', - 'OVF', - 'MAAS', - 'Ec2', - 'CloudStack'], + 'datasource_list': [ + 'NoCloud', + 'ConfigDrive', + 'OVF', + 'MAAS', + 'Ec2', + 'CloudStack' + ], 'def_log_file': '/var/log/cloud-init.log', 'log_cfgs': [], - 'syslog_fix_perms': 'syslog:adm' -} - -PATH_MAP = { - "handlers": "handlers", - "scripts": "scripts", - "sem": "sem", - "boothooks": "boothooks", - "userdata_raw": "user-data.txt", - "userdata": "user-data.txt.i", - "obj_pkl": "obj.pkl", - "cloud_config": "cloud-config.txt", - "data": "data", + 'syslog_fix_perms': 'syslog:adm', + 'system_info': { + 'paths': { + 'cloud_dir': '/var/lib/cloud', + 'templates_dir': '/etc/cloud/templates/', + }, + 'distro': 'ubuntu', + }, } PER_INSTANCE = "once-per-instance" PER_ALWAYS = "always" PER_ONCE = "once" -TEMPLATE_DIR = '/etc/cloud/templates/' -- cgit v1.2.3 From 2d831d8a0e0c57bc85de1e1e2def2788fa6ac525 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 9 Jun 2012 12:35:39 -0700 Subject: Cleanup this and add refactoring around large constructors (add a parse method). Handle error cases better... --- cloudinit/ssh_util.py | 277 ++++++++++++++++++++++++++++---------------------- 1 file changed, 155 insertions(+), 122 deletions(-) diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index 1483f718..93fd55dd 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -20,42 +20,70 @@ # along with this program. If not, see . import os -import os.path -import cloudinit.util as util - - -class AuthKeyEntry(): - # lines are options, keytype, base64-encoded key, comment - # man page says the following which I did not understand: - # The options field is optional; its presence is determined by whether - # the line starts with a number or not (the options field never starts - # with a number) - options = None - keytype = None - base64 = None - comment = None - is_comment = False - line_in = "" +import pwd + +from cloudinit import log as logging +from cloudinit import util + +LOG = logging.getLogger(__name__) + + +class AuthKeyEntry(object): + """ + AUTHORIZED_KEYS FILE FORMAT + AuthorizedKeysFile specifies the file containing public keys for public + key authentication; if none is specified, the default is + ~/.ssh/authorized_keys. Each line of the file contains one key (empty + (because of the size of the public key encoding) up to a limit of 8 kilo- + bytes, which permits DSA keys up to 8 kilobits and RSA keys up to 16 + kilobits. You don't want to type them in; instead, copy the + identity.pub, id_dsa.pub, or the id_rsa.pub file and edit it. + + sshd enforces a minimum RSA key modulus size for protocol 1 and protocol + 2 keys of 768 bits. + + The options (if present) consist of comma-separated option specifica- + tions. No spaces are permitted, except within double quotes. The fol- + lowing option specifications are supported (note that option keywords are + case-insensitive): + """ def __init__(self, line, def_opt=None): - line = line.rstrip("\n\r") - self.line_in = line - if line.startswith("#") or line.strip() == "": - self.is_comment = True + self.line = str(line) + (self.value, self.components) = self._parse(self.line, def_opt) + + def _form_components(self, toks): + components = {} + if len(toks) == 1: + components['base64'] = toks[0] + elif len(toks) == 2: + components['base64'] = toks[0] + components['comment'] = toks[1] + elif len(toks) == 3: + components['keytype'] = toks[0] + components['base64'] = toks[1] + components['comment'] = toks[2] + return components + + def get(self, piece): + return self.components.get(piece) + + def _parse(self, in_line, def_opt): + line = in_line.rstrip("\r\n") + if line.startswith("#") or line.strip() == '': + return (False, {}) else: ent = line.strip() toks = ent.split(None, 3) - if len(toks) == 1: - self.base64 = toks[0] - elif len(toks) == 2: - (self.base64, self.comment) = toks - elif len(toks) == 3: - (self.keytype, self.base64, self.comment) = toks - elif len(toks) == 4: + tmp_components = {} + if def_opt: + tmp_components['options'] = def_opt + if len(toks) < 4: + tmp_components.update(self._form_components(toks)) + else: + # taken from auth_rsa_key_allowed in auth-rsa.c i = 0 - ent = line.strip() quoted = False - # taken from auth_rsa_key_allowed in auth-rsa.c try: while (i < len(ent) and ((quoted) or (ent[i] not in (" ", "\t")))): @@ -67,124 +95,129 @@ class AuthKeyEntry(): quoted = not quoted i = i + 1 except IndexError: - self.is_comment = True - return - + return (False, {}) try: - self.options = ent[0:i] - (self.keytype, self.base64, self.comment) = \ - ent[i + 1:].split(None, 3) - except ValueError: - # we did not understand this line - self.is_comment = True - - if self.options == None and def_opt: - self.options = def_opt - - return - - def debug(self): - print("line_in=%s\ncomment: %s\noptions=%s\nkeytype=%s\nbase64=%s\n" - "comment=%s\n" % (self.line_in, self.is_comment, self.options, - self.keytype, self.base64, self.comment)), - - def __repr__(self): - if self.is_comment: - return(self.line_in) + options = ent[0:i] + toks = ent[i + 1:].split(None, 3) + if options: + tmp_components['options'] = options + tmp_components.update(self._form_components(toks)) + except (IndexError, ValueError): + return (False, {}) + # We got some useful value! + return (True, tmp_components) + + def __str__(self): + if not self.value: + return self.line else: toks = [] - for e in (self.options, self.keytype, self.base64, self.comment): - if e: - toks.append(e) - - return(' '.join(toks)) + if 'options' in self.components: + toks.append(self.components['options']) + if 'keytype' in self.components: + toks.append(self.components['keytype']) + if 'base64' in self.components: + toks.append(self.components['base64']) + if 'comment' in self.components: + toks.append(self.components['comment']) + if not toks: + return '' + return ' '.join(toks) def update_authorized_keys(fname, keys): - # keys is a list of AuthKeyEntries - # key_prefix is the prefix (options) to prepend + lines = [] try: - fp = open(fname, "r") - lines = fp.readlines() # lines have carriage return - fp.close() - except IOError: + if os.path.isfile(fname): + lines = util.load_file(fname).splitlines() + except (IOError, OSError): + LOG.exception("Error reading lines from %s", fname) lines = [] - ka_stats = {} # keys_added status - for k in keys: - ka_stats[k] = False - - to_add = [] - for key in keys: - to_add.append(key) - + to_add = list(keys) for i in range(0, len(lines)): ent = AuthKeyEntry(lines[i]) + if not ent.value: + continue + # Replace those with the same base64 for k in keys: - if k.base64 == ent.base64 and not k.is_comment: + if not k.value: + continue + if k.get('base64') == ent.get('base64'): + # Replace it with our better one ent = k - try: - to_add.remove(k) - except ValueError: - pass + # Don't add it later + to_add.remove(k) lines[i] = str(ent) - # now append any entries we did not match above + # Now append any entries we did not match above for key in to_add: lines.append(str(key)) - if len(lines) == 0: - return("") - else: - return('\n'.join(lines) + "\n") + # Ensure it ends with a newline + lines.append('') + return '\n'.join(lines) -def setup_user_keys(keys, user, key_prefix, log=None): - import pwd - saved_umask = os.umask(077) - +def setup_user_keys(keys, user, key_prefix, sshd_config_fn="/etc/ssh/sshd_config"): pwent = pwd.getpwnam(user) - ssh_dir = '%s/.ssh' % pwent.pw_dir + ssh_dir = os.path.join(pwent.pw_dir, '.ssh') if not os.path.exists(ssh_dir): - os.mkdir(ssh_dir) - os.chown(ssh_dir, pwent.pw_uid, pwent.pw_gid) - - try: - ssh_cfg = parse_ssh_config() - akeys = ssh_cfg.get("AuthorizedKeysFile", "%h/.ssh/authorized_keys") - akeys = akeys.replace("%h", pwent.pw_dir) - akeys = akeys.replace("%u", user) - if not akeys.startswith('/'): - akeys = os.path.join(pwent.pw_dir, akeys) - authorized_keys = akeys - except Exception: - authorized_keys = '%s/.ssh/authorized_keys' % pwent.pw_dir - if log: - util.logexc(log) + util.ensure_dir(ssh_dir, mode=0700) + util.chownbyid(ssh_dir, pwent.pw_uid, pwent.pw_gid) key_entries = [] for k in keys: - ke = AuthKeyEntry(k, def_opt=key_prefix) - key_entries.append(ke) - - content = update_authorized_keys(authorized_keys, key_entries) - util.write_file(authorized_keys, content, 0600) - - os.chown(authorized_keys, pwent.pw_uid, pwent.pw_gid) - util.restorecon_if_possible(ssh_dir, recursive=True) - - os.umask(saved_umask) - - -def parse_ssh_config(fname="/etc/ssh/sshd_config"): + key_entries.append(AuthKeyEntry(k, def_opt=key_prefix)) + + with util.SeLinuxGuard(ssh_dir, recursive=True): + try: + """ + AuthorizedKeysFile may contain tokens + of the form %T which are substituted during connection set-up. + The following tokens are defined: %% is replaced by a literal + '%', %h is replaced by the home directory of the user being + authenticated and %u is replaced by the username of that user. + """ + ssh_cfg = parse_ssh_config(sshd_config_fn) + akeys = ssh_cfg.get("authorizedkeysfile", '') + akeys = akeys.strip() + if not akeys: + akeys = "%h/.ssh/authorized_keys" + akeys = akeys.replace("%h", pwent.pw_dir) + akeys = akeys.replace("%u", user) + akeys = akeys.replace("%%", '%') + if not akeys.startswith('/'): + akeys = os.path.join(pwent.pw_dir, akeys) + authorized_keys = akeys + except (IOError, OSError): + authorized_keys = os.path.join(ssh_dir, 'authorized_keys') + LOG.exception(("Failed extracting 'AuthorizedKeysFile' in ssh config" + " from %s, using 'AuthorizedKeysFile' file %s instead."), + sshd_config_fn, authorized_keys) + + content = update_authorized_keys(authorized_keys, key_entries) + util.ensure_dir(os.path.dirname(authorized_keys), mode=0700) + util.write_file(authorized_keys, content, mode=0600) + util.chownbyid(authorized_keys, pwent.pw_uid, pwent.pw_gid) + + +def parse_ssh_config(fname): + """ + The file contains keyword-argu-ment pairs, one per line. + Lines starting with '#' and empty lines are interpreted as comments. + Note: key-words are case-insensitive and arguments are case-sensitive + """ ret = {} - fp = open(fname) - for l in fp.readlines(): - l = l.strip() - if not l or l.startswith("#"): + if not os.path.isfile(fname): + return ret + for line in util.load_file(fname).splitlines(): + line = line.strip() + if not line or line.startswith("#"): continue - key, val = l.split(None, 1) - ret[key] = val - fp.close() - return(ret) + (key, val) = line.split(None, 1) + key = key.strip().lower() + if key: + ret[key] = val + return ret -- cgit v1.2.3 From 46e451abd8e5945a9b944dfd794a2556a3830811 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 9 Jun 2012 12:36:18 -0700 Subject: Remove unused imports and reduce lines needed + add copy right. --- cloudinit/templater.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/cloudinit/templater.py b/cloudinit/templater.py index 5a3563a2..04cc5a6f 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -1,18 +1,32 @@ -import os +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . from Cheetah.Template import Template -from cloudinit import settings from cloudinit import util -def render_to_file(template, outfile, searchList): - fn = template - (base, ext) = os.path.splitext(fn) - if ext != ".tmpl": - fn = "%s.tmpl" % (fn) - fn = os.path.join(settings.TEMPLATE_DIR, fn) - contents = Template(file=fn, searchList=[searchList]).respond() +def render_to_file(template_fn, outfile, searchList): + contents = Template(file=template_fn, searchList=[searchList]).respond() util.write_file(outfile, contents) -- cgit v1.2.3 From 8377022620aaaaebcd32fd8c0db1dbc769ce282d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 9 Jun 2012 12:37:10 -0700 Subject: Move cmdline functions here. Cleanup exception trapping. Fixup copyright. --- cloudinit/util.py | 113 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 88 insertions(+), 25 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 265a3a97..5930ff3f 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1,10 +1,12 @@ # vi: ts=4 expandtab # -# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser -# Author: Juerg Hafliger +# Author: Juerg Haefliger +# Author: Joshua Harlow # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -21,13 +23,16 @@ from StringIO import StringIO import contextlib +import glob import grp import gzip import os import platform import pwd import shutil +import socket import subprocess +import types import urlparse import yaml @@ -96,7 +101,7 @@ class ProcessExecutionError(IOError): self.reason = reason -class _SeLinuxGuard(object): +class SeLinuxGuard(object): def __init__(self, path, recursive=False): self.path = path self.recursive = recursive @@ -149,6 +154,18 @@ def decomp_str(data): return data +def find_modules(root_dir): + entries = dict() + for fname in glob.glob(os.path.join(root_dir, "*.py")): + if not os.path.isfile(fname): + continue + modname = os.path.basename(fname)[0:-3] + modname = modname.strip() + if modname and modname.find(".") == -1: + entries[fname] = modname + return entries + + def is_ipv4(instr): """ determine if input string is a ipv4 address. return boolean""" toks = instr.split('.') @@ -163,15 +180,16 @@ def is_ipv4(instr): return (len(toks) == 4) -def get_base_cfg(cfgfile, cfg_builtin=None, parsed_cfgs=None): - if parsed_cfgs and cfgfile in parsed_cfgs: - return parsed_cfgs[cfgfile] - +def get_base_cfg(cfgfile, cfg_builtin=None): syscfg = read_conf_with_confd(cfgfile) + kern_contents = read_cc_from_cmdline() kerncfg = {} if kern_contents: - kerncfg = yaml.load(kern_contents) + try: + kerncfg = yaml.load(kern_contents) + except: + pass # kernel parameters override system config combined = mergedict(kerncfg, syscfg) @@ -180,9 +198,6 @@ def get_base_cfg(cfgfile, cfg_builtin=None, parsed_cfgs=None): else: fin = combined - # Cache it? - if parsed_cfgs: - parsed_cfgs[cfgfile] = fin return fin @@ -223,7 +238,7 @@ def get_cfg_option_list_or_str(yobj, key, default=None): return default if yobj[key] is None: return [] - if isinstance(yobj[key], list): + if isinstance(yobj[key], (list)): return yobj[key] return [yobj[key]] @@ -239,6 +254,15 @@ def get_cfg_by_path(yobj, keyp, default=None): return cur +def obj_name(obj): + if isinstance(obj, (types.TypeType, + types.ModuleType, + types.FunctionType, + types.LambdaType)): + return str(obj.__name__) + return obj_name(obj.__class__) + + def mergedict(src, cand): """ Merge values from C{cand} into C{src}. If C{src} has a key C{cand} will @@ -258,6 +282,15 @@ def mergedict(src, cand): return src +@contextlib.contextmanager +def umask(n_msk): + old = os.umask(n_msk) + try: + yield old + finally: + os.umask(old) + + @contextlib.contextmanager def tempdir(**kwargs): # This seems like it was only added in python 3.2 @@ -379,13 +412,15 @@ def read_conf_with_confd(cfgfile): if "conf_d" in cfg: if cfg['conf_d'] is not None: confd = cfg['conf_d'] - if not isinstance(confd, str): - raise RuntimeError("cfgfile %s contains 'conf_d' " - "with non-string" % cfgfile) + if not isinstance(confd, (str)): + raise RuntimeError(("Config file %s contains 'conf_d' " + "with non-string") % (cfgfile)) + else: + confd = confd.strip() elif os.path.isdir("%s.d" % cfgfile): confd = "%s.d" % cfgfile - if not confd: + if not confd or not os.path.isdir(confd): return cfg return mergedict(read_conf_d(confd), cfg) @@ -479,6 +514,32 @@ def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): return fqdn +def get_cmdline_url(names=None, starts=None, cmdline=None): + if cmdline is None: + cmdline = get_cmdline() + if not names: + names = ('cloud-config-url', 'url') + if not starts: + starts = "#cloud-config" + + data = keyval_str_to_dict(cmdline) + url = None + key = None + for key in names: + if key in data: + url = data[key] + break + + if not url: + return (None, None, None) + + (contents, sc) = uhelp.readurl(url) + if contents.startswith(starts) and uhelp.ok_http_code(sc): + return (key, url, contents) + + return (key, url, None) + + def is_resolvable(name): """ determine if a url is resolvable, return a boolean """ try: @@ -488,6 +549,11 @@ def is_resolvable(name): return False +def get_hostname(): + hostname = socket.gethostname() + return hostname + + def is_resolvable_url(url): """ determine if this url is resolvable (existing or ip) """ return (is_resolvable(urlparse.urlparse(url).hostname)) @@ -634,7 +700,7 @@ def write_file(filename, content, mode=0644, omode="wb"): ensure_dir(os.path.dirname(filename)) LOG.debug("Writing to %s - %s (perms=%s) %s bytes", filename, omode, mode, len(content)) with open(filename, omode) as fh: - with _SeLinuxGuard(filename): + with SeLinuxGuard(filename): fh.write(content) fh.flush() if mode is not None: @@ -711,11 +777,8 @@ def is_container(): cmd = [helper] (stdout, stderr) = subp(cmd, allowed_rc=[0]) return True - except IOError as e: + except (IOError, OSError): pass - # Is this really needed? - # if e.errno != errno.ENOENT: - # raise # this code is largely from the logic in # ubuntu's /etc/init/container-detect.conf @@ -727,7 +790,7 @@ def is_container(): return True if "LIBVIRT_LXC_UUID" in pid1env: return True - except IOError as e: + except (IOError, OSError): pass # Detect OpenVZ containers @@ -742,7 +805,7 @@ def is_container(): (_key, val) = line.strip().split(":", 1) if val != "0": return True - except IOError as e: + except (IOError, OSError): pass return False @@ -759,9 +822,9 @@ def get_proc_env(pid): if tok == "": continue (name, val) = tok.split("=", 1) - if not name: + if name: env[name] = val - except IOError: + except (IOError, OSError): pass return env -- cgit v1.2.3 From c2b0379b6ab35f2d283ee0882f5ce61f0ca19c6f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 9 Jun 2012 12:37:40 -0700 Subject: Add copyright --- cloudinit/url_helper.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index ed78c92e..6fa3e44b 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -1,3 +1,26 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + import errno import time import urllib -- cgit v1.2.3 From ad9122034ff59fb6113adfac2713c44af09bc91b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 9 Jun 2012 12:38:19 -0700 Subject: Initial cleanups --- cloudinit/sources/__init__.py | 188 +++++++++++++++++++++--------------------- 1 file changed, 93 insertions(+), 95 deletions(-) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 9a9c1316..05c8bfad 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -1,10 +1,12 @@ # vi: ts=4 expandtab # -# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser -# Author: Juerg Hafliger +# Author: Juerg Haefliger +# Author: Joshua Harlow # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -18,78 +20,78 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from cloudinit import user_data as ud +from cloudinit import importer +from cloudinit import log as logging from cloudinit import util -import socket - DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" +DS_PREFIX = 'DataSource' +LOG = logging.getLogger(__name__) + class DataSourceNotFoundException(Exception): pass -class DataSource: - userdata = None - metadata = None - userdata_raw = None - cfgname = "" - # system config (passed in from cloudinit, - # cloud-config before input from the DataSource) - sys_cfg = {} - # datasource config, the cloud-config['datasource']['__name__'] - ds_cfg = {} # datasource config - - def __init__(self, sys_cfg=None): - if not self.cfgname: - name = str(self.__class__).split(".")[-1] - if name.startswith("DataSource"): - name = name[len("DataSource"):] - self.cfgname = name +class DataSource(object): + def __init__(self, ud_proc, cfg): + name = util.obj_name(self) + if name.startswith(DS_PREFIX): + name = name[DS_PREFIX:] + self.cfgname = name if sys_cfg: self.sys_cfg = sys_cfg - + else: + self.sys_cfg = {} + self.ud_proc = ud_proc + self.userdata = None + self.metadata = None + self.userdata_raw = None self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, ("datasource", self.cfgname), self.ds_cfg) def get_userdata(self): - if self.userdata == None: - self.userdata = ud.preprocess_userdata(self.userdata_raw) + if self.userdata is None: + raw_data = self.get_userdata_raw() + self.userdata = self.ud_proc.process(raw_data) return self.userdata def get_userdata_raw(self): - return(self.userdata_raw) + return self.userdata_raw # the data sources' config_obj is a cloud-config formated # object that came to it from ways other than cloud-config # because cloud-config content would be handled elsewhere def get_config_obj(self): - return({}) + return {} def get_public_ssh_keys(self): keys = [] - if 'public-keys' not in self.metadata: - return([]) - if isinstance(self.metadata['public-keys'], str): - return(str(self.metadata['public-keys']).splitlines()) + if not self.metadata or 'public-keys' not in self.metadata: + return keys - if isinstance(self.metadata['public-keys'], list): - return(self.metadata['public-keys']) + if isinstance(self.metadata['public-keys'], (str)): + return str(self.metadata['public-keys']).splitlines() - for _keyname, klist in self.metadata['public-keys'].items(): - # lp:506332 uec metadata service responds with - # data that makes boto populate a string for 'klist' rather - # than a list. - if isinstance(klist, str): - klist = [klist] - for pkey in klist: - # there is an empty string at the end of the keylist, trim it - if pkey: - keys.append(pkey) + if isinstance(self.metadata['public-keys'], (list, set)): + return list(self.metadata['public-keys']) - return(keys) + if isinstance(self.metadata['public-keys'], (dict)): + for _keyname, klist in self.metadata['public-keys'].items(): + # lp:506332 uec metadata service responds with + # data that makes boto populate a string for 'klist' rather + # than a list. + if isinstance(klist, (str)): + klist = [klist] + if isinstance(klist, (list)): + for pkey in klist: + # there is an empty string at the end of the keylist, trim it + if pkey: + keys.append(pkey) + + return keys def device_name_to_device(self, _name): # translate a 'name' to a device @@ -97,48 +99,43 @@ class DataSource: # to consult metadata service, that has # ephemeral0: sdb # and return 'sdb' for input 'ephemeral0' - return(None) + return None def get_locale(self): - return('en_US.UTF-8') + return 'en_US.UTF-8' def get_local_mirror(self): return None def get_instance_id(self): - if 'instance-id' not in self.metadata: + if not self.metadata or 'instance-id' not in self.metadata: return "iid-datasource" - return(self.metadata['instance-id']) + return str(self.metadata['instance-id']) def get_hostname(self, fqdn=False): defdomain = "localdomain" defhost = "localhost" - domain = defdomain - if not 'local-hostname' in self.metadata: + if not self.metadata or not 'local-hostname' in self.metadata: # this is somewhat questionable really. # the cloud datasource was asked for a hostname # and didn't have one. raising error might be more appropriate # but instead, basically look up the existing hostname toks = [] - - hostname = socket.gethostname() - + hostname = util.get_hostname() fqdn = util.get_fqdn_from_hosts(hostname) - if fqdn and fqdn.find(".") > 0: toks = str(fqdn).split(".") elif hostname: toks = [hostname, defdomain] else: toks = [defhost, defdomain] - else: # if there is an ipv4 address in 'local-hostname', then # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx lhost = self.metadata['local-hostname'] - if is_ipv4(lhost): + if util.is_ipv4(lhost): toks = "ip-%s" % lhost.replace(".", "-") else: toks = lhost.split(".") @@ -155,22 +152,22 @@ class DataSource: return hostname -def find_source(cfg, ds_deps): - cfglist = cfg.get('datasource_list') or [] - dslist = list_sources(cfglist, ds_deps) - dsnames = [f.__name__ for f in dslist] - - LOG.debug("Searching for data source in %s", dsnames) - for cls in dslist: - ds = cls.__name__ +def find_source(cfg, ds_deps, cfg_list, pkg_list, **kwargs): + ds_list = list_sources(cfg_list, ds_deps, pkg_list) + ds_names = [util.obj_name(f) for f in ds_list] + ds_args = dict(kwargs) + ds_args['cfg'] = cfg + LOG.info("Searching for data source in: %s", ds_names) + for cls in ds_list: + ds = util.obj_name(cls) try: - s = cls(sys_cfg=cfg) + s = cls(**ds_args) if s.get_data(): return (s, ds) except Exception as e: - LOG.exception("Getting data from %s raised %s", ds, e) + LOG.exception("Getting data from %s failed", ds) - msg = "Did not find any data source, searched classes: %s" % dsnames + msg = "Did not find any data source, searched classes: %s" % (ds_names) raise DataSourceNotFoundException(msg) @@ -178,31 +175,33 @@ def find_source(cfg, ds_deps): # iterate through cfg_list, loading "DataSourceCollections" modules # and calling their "get_datasource_list". # return an ordered list of classes that match -# -# - modules must be named "DataSource", where 'item' is an entry -# in cfg_list -# - if pkglist is given, it will iterate try loading from that package -# ie, pkglist=[ "foo", "" ] -# will first try to load foo.DataSource -# then DataSource -def list_sources(cfg_list, depends, pkglist=None): - if pkglist is None: - pkglist = [] - retlist = [] +def list_sources(cfg_list, depends, pkg_list): + src_list = [] + LOG.info("Looking for for data source in: %s, %s that match %s", cfg_list, pkg_list, depends) for ds_coll in cfg_list: - for pkg in pkglist: + ds_name = str(ds_coll) + if not ds_name.startswith(DS_PREFIX): + ds_name = '%s%s' % (DS_PREFIX, ds_name) + for pkg in pkg_list: + pkg_name = [] + if pkg: + pkg_name.append(str(pkg)) + pkg_name.append(ds_name) + mod_name = ".".join(pkg_name) + mod = importer.import_module(mod_name) if pkg: - pkg = "%s." % pkg - try: - mod = __import__("%sDataSource%s" % (pkg, ds_coll)) - if pkg: - mod = getattr(mod, "DataSource%s" % ds_coll) - lister = getattr(mod, "get_datasource_list") - retlist.extend(lister(depends)) - break - except: - raise - return(retlist) + mod = getattr(mod, ds_name, None) + if not mod: + continue + lister = getattr(mod, "get_datasource_list", None) + if not lister: + continue + cls_matches = lister(depends) + if not cls_matches: + continue + src_list.extend(cls_matches) + break + return src_list # depends is a list of dependencies (DEP_FILESYSTEM) @@ -213,10 +212,9 @@ def list_sources(cfg_list, depends, pkglist=None): # it returns a list of 'class' that matched these deps exactly # it is a helper function for DataSourceCollections def list_from_depends(depends, dslist): - retlist = [] + ret_list = [] depset = set(depends) - for elem in dslist: - (cls, deps) = elem + for (cls, deps) in dslist: if depset == set(deps): - retlist.append(cls) - return(retlist) + ret_list.append(cls) + return ret_list -- cgit v1.2.3 From ac1ccf5183fa7412247d3161e99da93c32f56ff2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 9 Jun 2012 12:38:32 -0700 Subject: Start adding a system section. --- config/cloud.cfg | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config/cloud.cfg b/config/cloud.cfg index 3688f88d..33feec48 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -1,7 +1,6 @@ user: ubuntu disable_root: 1 preserve_hostname: False -# datasource_list: ["NoCloud", "ConfigDrive", "OVF", "MAAS", "Ec2", "CloudStack"] cloud_init_modules: - bootcmd @@ -40,3 +39,10 @@ cloud_final_modules: - keys-to-console - phone-home - final-message + +system_info: + paths: + cloud_dir: /var/lib/cloud/ + templates_dir: /etc/cloud/templates/ + upstart_dir: /etc/init/ + distro: ubuntu -- cgit v1.2.3 From 747d59838fbaed299c5555984895affc9fc7f012 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:09:10 -0700 Subject: Not needed, this functionality can go in utils. --- cloudinit/mounting.py | 73 --------------------------------------------------- 1 file changed, 73 deletions(-) delete mode 100644 cloudinit/mounting.py diff --git a/cloudinit/mounting.py b/cloudinit/mounting.py deleted file mode 100644 index b78cb5bd..00000000 --- a/cloudinit/mounting.py +++ /dev/null @@ -1,73 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import contextlib - -from cloudinit import util - - -class MountFailedError(Exception): - pass - - -@contextlib.contextmanager -def unmounter(umount): - try: - yield umount - finally: - if umount: - sh.subp(["umount", '-l', umount]) - - -def mount_callback_umount(device, callback, data=None): - """ - mount the device, call method 'callback' passing the directory - in which it was mounted, then unmount. Return whatever 'callback' - returned. If data != None, also pass data to callback. - """ - - # go through mounts to see if it was already mounted - mounts = sh.load_file("/proc/mounts").splitlines() - mounted = {} - for mpline in mounts: - (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() - mp = mp.replace("\\040", " ") - mounted[dev] = (dev, fstype, mp, False) - - with util.tempdir() as tmpd: - umount = False - if device in mounted: - mountpoint = "%s/" % mounted[device][2] - else: - try: - mountcmd = ["mount", "-o", "ro", device, tmpd] - util.subp(mountcmd) - umount = tmpd - except IOError as exc: - raise MountFailedError("%s" % (exc)) - mountpoint = "%s/" % tmpd - with unmounter(umount): - if data is None: - ret = callback(mountpoint) - else: - ret = callback(mountpoint, data) - return ret -- cgit v1.2.3 From c4fad2ad01a66065ebe904a1f7842613a355b891 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:09:24 -0700 Subject: Removed, not needed. --- pylintrc | 42 ------------------------------------------ 1 file changed, 42 deletions(-) delete mode 100644 pylintrc diff --git a/pylintrc b/pylintrc deleted file mode 100644 index a7447a19..00000000 --- a/pylintrc +++ /dev/null @@ -1,42 +0,0 @@ -# The format of this file isn't really documented; just use --generate-rcfile - -[Master] - -[Messages Control] - -# http://pylint-messages.wikidot.com/all-codes -# NOTE(justinsb): We might want to have a 2nd strict pylintrc in future -# C0111: Don't require docstrings on every method -# R0912: Too many branches (huh) -# R0914: Too many local variables is odd. -# W0142: *args and **kwargs are fine. -# W0511: TODOs in code comments are fine. -# W0613: Unused argument '??' should be ok (they are useful sometimes to know intention of variable) -# W0622: Redefining id is fine. -disable=C0111,W0142,W0622,C0301,R0902,R0201,R0914,W0613,R0912,R0801 - -[Basic] - -# Variable names can be 1 to 31 characters long, with lowercase and underscores -variable-rgx=[a-z_][a-z0-9_]{0,30}$ - -# Argument names can be 2 to 31 characters long, with lowercase and underscores -argument-rgx=[a-z_][a-z0-9_]{1,30}$ - -# Method names should be at least 3 characters long -# and be lowercased with underscores -method-rgx=([a-z_][a-z0-9_]{2,50}|setUp|tearDown)$ - -# Don't require docstrings on tests. -no-docstring-rgx=((__.*__)|([tT]est.*)|setUp|tearDown)$ - -[Design] - -max-public-methods=100 -min-public-methods=0 -max-args=6 - -[Variables] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -- cgit v1.2.3 From 6b37d05a4848e00d34c34b4b23a74d9187f29bba Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:09:38 -0700 Subject: All main stage helpers go here, ie semaphores, handler registrys, runners... --- cloudinit/helpers.py | 204 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 cloudinit/helpers.py diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py new file mode 100644 index 00000000..cdb8a07e --- /dev/null +++ b/cloudinit/helpers.py @@ -0,0 +1,204 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from time import time + +import contextlib +import os + +from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE) + +from cloudinit import log as logging +from cloudinit import util + +from cloudinit.user_data import boot_hook as bh_part +from cloudinit.user_data import cloud_config as cc_part +from cloudinit.user_data import shell_script as ss_part +from cloudinit.user_data import upstart_job as up_part + +LOG = logging.getLogger(__name__) + + +class DummySemaphores(object): + def __init__(self): + pass + + @contextlib.contextmanager + def lock(self, _name, _freq, _clear_on_fail): + yield True + + def has_run(self, _name, _freq): + return False + + +class Semaphores(object): + def __init__(self, sem_path): + self.sem_path = sem_path + + @contextlib.contextmanager + def lock(self, name, freq, clear_on_fail): + try: + yield self._acquire(name, freq) + except: + if clear_on_fail: + self.clear(name, freq) + raise + + def clear(self, name, freq): + sem_file = self._get_path(name, freq) + try: + util.del_file(sem_file) + except (IOError, OSError): + return False + return True + + def _acquire(self, name, freq): + if self.has_run(name, freq): + return None + # This is a race condition since nothing atomic is happening + # here, but this should be ok due to the nature of when + # and where cloud-init runs... (file writing is not a lock..) + sem_file = self._get_path(name, freq) + contents = "%s: %s\n" % (os.getpid(), time()) + try: + util.write_file(sem_file, contents) + except (IOError, OSError): + return None + return sem_file + + def has_run(self, name, freq): + if freq == PER_ALWAYS: + return False + sem_file = self._get_path(name, freq) + if os.path.exists(sem_file): + return True + return False + + def _get_path(self, name, freq): + sem_path = self.sem_path + if not freq or freq == PER_INSTANCE: + return os.path.join(sem_path, name) + else: + return os.path.join(sem_path, "%s.%s" % (name, freq)) + + +class Runners(object): + def __init__(self, paths): + self.paths = paths + self.sems = {} + + def _get_sem(self, freq): + if freq == PER_ALWAYS or not freq: + return None + sem_path = None + if freq == PER_INSTANCE: + sem_path = self.paths.get_ipath("sem") + elif freq == PER_ONCE: + sem_path = self.paths.get_cpath("sem") + if not sem_path: + return None + if sem_path not in self.sems: + self.sems[sem_path] = Semaphores(sem_path) + return self.sems[sem_path] + + def run(self, name, functor, args, freq=None, clear_on_fail=False): + sem = self._get_sem(freq) + if not sem: + sem = DummySemaphores() + if not args: + args = [] + if sem.has_run(name, freq): + LOG.info("%s already ran (freq=%s)", name, freq) + return None + with sem.lock(name, freq, clear_on_fail) as lk: + if not lk: + raise RuntimeError("Failed to acquire lock on %s" % name) + else: + LOG.debug("Running %s with args %s using lock %s", + functor, args, lk) + return functor(*args) + + +class ContentHandlers(object): + + def __init__(self, paths): + self.paths = paths + self.registered = {} + + def __contains__(self, item): + return self.is_registered(item) + + def __getitem__(self, key): + return self._get_handler(key) + + def is_registered(self, content_type): + return content_type in self.registered + + def register(self, mod): + types = set() + for t in mod.list_types(): + self.registered[t] = mod + types.add(t) + return types + + def _get_handler(self, content_type): + return self.registered[content_type] + + def items(self): + return self.registered.items() + + def iteritems(self): + return self.registered.iteritems() + + def _get_default_handlers(self): + def_handlers = [] + + cc_path = self.paths.get_ipath("cloud_config") + if cc_path: + cc_h = cc_part.CloudConfigPartHandler(cc_path) + def_handlers.append(cc_h) + + sc_path = self.paths.get_ipath_cur('scripts') + if sc_path: + ss_h = ss_part.ShellScriptPartHandler(sc_path) + def_handlers.append(ss_h) + + bh_path = self.paths.get_ipath("boothooks") + if bh_path: + bh_h = bh_part.BootHookPartHandler(bh_path) + def_handlers.append(bh_h) + + upstart_pth = self.paths.upstart_conf_d + if upstart_pth: + up_h = up_part.UpstartJobPartHandler(upstart_pth) + def_handlers.append(up_h) + + return def_handlers + + def register_defaults(self): + registered = set() + for mod in self._get_default_handlers(): + for t in mod.list_types(): + if not self.is_registered(t): + self.registered[t] = mod + registered.add(t) + return registered -- cgit v1.2.3 From 830c54fec367f852b2f4b0caa71849aa621b5377 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:11:46 -0700 Subject: Move paths to here, since it also qualifies. --- cloudinit/helpers.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index cdb8a07e..0bd13c78 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -202,3 +202,66 @@ class ContentHandlers(object): self.registered[t] = mod registered.add(t) return registered + + +class Paths(object): + def __init__(self, sys_info): + self.cloud_dir = sys_info.get('cloud_dir', '/var/lib/cloud') + self.instance_link = os.path.join(self.cloud_dir, 'instance') + self.boot_finished = os.path.join(self.instance_link, "boot-finished") + self.upstart_conf_d = sys_info.get('upstart_dir') + template_dir = sys_info.get('templates_dir', '/etc/cloud/templates/') + self.template_tpl = os.path.join(template_dir, '%s.tmpl') + self.seed_dir = os.path.join(self.cloud_dir, 'seed') + self.lookups = { + "handlers": "handlers", + "scripts": "scripts", + "sem": "sem", + "boothooks": "boothooks", + "userdata_raw": "user-data.txt", + "userdata": "user-data.txt.i", + "obj_pkl": "obj.pkl", + "cloud_config": "cloud-config.txt", + "data": "data", + } + # Set when a datasource becomes active + self.datasource = None + + # get_ipath_cur: get the current instance path for an item + def get_ipath_cur(self, name=None): + ipath = self.instance_link + add_on = self.lookups.get(name) + if add_on: + ipath = os.path.join(ipath, add_on) + return ipath + + # get_cpath : get the "clouddir" (/var/lib/cloud/) + # for a name in dirmap + def get_cpath(self, name=None): + cpath = self.cloud_dir + add_on = self.lookups.get(name) + if add_on: + cpath = os.path.join(cpath, add_on) + return cpath + + def _get_ipath(self, name=None): + if not self.datasource: + return None + iid = self.datasource.get_instance_id() + if iid is None: + return None + ipath = os.path.join(self.cloud_dir, 'instances', iid) + add_on = self.lookups.get(name) + if add_on: + ipath = os.path.join(ipath, add_on) + return ipath + + # (/var/lib/cloud/instances//) + def get_ipath(self, name=None): + ipath = self._get_ipath(name) + if not ipath: + LOG.warn(("No per instance semaphores available, " + "is there an datasource/iid set?")) + return None + else: + return ipath -- cgit v1.2.3 From cad4b0c217427f5497cc32fe9e19bd3e5071a131 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:12:00 -0700 Subject: Add this file, which contains the main init stage, and the handlers stage (as seperate objects). --- cloudinit/stages.py | 411 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 411 insertions(+) create mode 100644 cloudinit/stages.py diff --git a/cloudinit/stages.py b/cloudinit/stages.py new file mode 100644 index 00000000..55ad143d --- /dev/null +++ b/cloudinit/stages.py @@ -0,0 +1,411 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cPickle as pickle + +import copy +import os +import sys + +try: + from configobj import ConfigObj +except ImportError: + ConfigObj = None + +from cloudinit.settings import (PER_INSTANCE) +from cloudinit.settings import (OLD_CLOUD_CONFIG) + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import handlers +from cloudinit import helpers +from cloudinit import importer +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util + +from cloudinit import user_data as ud + +LOG = logging.getLogger(__name__) + + +class Init(object): + def __init__(self, ds_deps=None): + self.datasource = None + if ds_deps: + self.ds_deps = ds_deps + else: + self.ds_deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] + # Created on first use + self.cached_cfg = None + self.cached_distro = None + self.cached_paths = None + + def _read_cfg_old(self): + # Support reading the old ConfigObj format file and merging + # it into the yaml dictionary + if not ConfigObj: + return {} + old_cfg = ConfigObj(OLD_CLOUD_CONFIG) + return dict(old_cfg) + + @property + def cfg(self): + if self.cached_cfg is None: + self.cached_cfg = self._read_cfg() + return self.cached_cfg + + @property + def paths(self): + if not self.cached_paths: + sys_info = self.cfg.get('system_info', {}) + self.cached_paths = helpers.Paths(copy.deepcopy(sys_info), + self.datasource) + return self.cached_paths + + def _initial_subdirs(self): + c_dir = self.paths.cloud_dir + initial_dirs = [ + os.path.join(c_dir, 'scripts'), + os.path.join(c_dir, 'scripts', 'per-instance'), + os.path.join(c_dir, 'scripts', 'per-once'), + os.path.join(c_dir, 'scripts', 'per-boot'), + os.path.join(c_dir, 'seed'), + os.path.join(c_dir, 'instances'), + os.path.join(c_dir, 'handlers'), + os.path.join(c_dir, 'sem'), + os.path.join(c_dir, 'data'), + ] + return initial_dirs + + def purge_cache(self, rmcur=True): + rmlist = [] + rmlist.append(self.paths.boot_finished) + if rmcur: + rmlist.append(self.paths.instance_link) + for f in rmlist: + util.del_file(f) + return len(rmlist) + + def initialize(self): + self._initialize_filesystem() + + def _initialize_filesystem(self): + util.ensure_dirs(self._initial_subdirs()) + log_file = util.get_cfg_option_str(self.cfg, 'def_log_file') + perms = util.get_cfg_option_str(self.cfg, 'syslog_fix_perms') + if log_file: + util.ensure_file(log_file) + if perms: + (u, g) = perms.split(':', 1) + if u == "-1" or u == "None": + u = None + if g == "-1" or g == "None": + g = None + util.chownbyname(log_file, u, g) + + def _read_cfg(self): + # Deep copy so that + b_config = util.get_builtin_cfg() + try: + conf = util.get_base_cfg() + except Exception: + conf = b_config + return util.mergedict(conf, self._read_cfg_old()) + + def _restore_from_cache(self): + pickled_fn = self.paths.get_ipath_cur('obj_pkl') + try: + # we try to restore from a current link and static path + # by using the instance link, if purge_cache was called + # the file wont exist + return pickle.loads(util.load_file(pickled_fn)) + except Exception as e: + LOG.exception(("Failed loading pickled datasource from" + " %s due to: %s"), pickled_fn, e) + return None + + def _write_to_cache(self): + pickled_fn = self.paths.get_ipath_cur("obj_pkl") + try: + contents = pickle.dumps(self.datasource) + util.write_file(pickled_fn, contents, mode=0400) + except Exception as e: + LOG.exception(("Failed pickling datasource to" + " %s due to: %s"), pickled_fn, e) + return False + + def _get_datasources(self): + # Any config provided??? + pkg_list = self.cfg.get('datasource_pkg_list') or [] + # Add the defaults at the end + for n in [util.obj_name(sources), '']: + if n not in pkg_list: + pkg_list.append(n) + cfg_list = self.cfg.get('datasource_list') or [] + return (cfg_list, pkg_list) + + def _get_data_source(self): + if self.datasource: + return self.datasource + ds = self._restore_from_cache() + if ds: + LOG.debug("Restored from cache datasource: %s" % ds) + else: + (cfg_list, pkg_list) = self._get_datasources() + # Deep copy so that handlers can not modify (which will + # affect handlers down the line...) + sys_cfg = copy.deepcopy(self.cfg) + ds_deps = copy.deepcopy(self.ds_deps) + distro = distros.fetch(sys_cfg, + cloud.Cloud(self.datasource, + self.paths, sys_cfg)) + (ds, dsname) = sources.find_source(sys_cfg, distro, self.paths, + ds_deps, cfg_list, pkg_list) + LOG.debug("Loaded datasource %s - %s", dsname, ds) + self.datasource = ds + if self.cached_paths: + self.cached_paths.datasource = ds + return ds + + def _reflect_cur_instance(self): + # Ensure we are hooked into the right symlink for the current instance + idir = self.paths.get_ipath() + util.del_file(self.paths.instance_link) + util.sym_link(idir, self.paths.instance_link) + + # Ensures these dirs exist + dir_list = [] + for d in ["handlers", "scripts", "sem"]: + dir_list.append(os.path.join(idir, d)) + util.ensure_dirs(dir_list) + + # Write out information on what is being used for the current instance + # and what may have been used for a previous instance... + dp = self.paths.get_cpath('data') + + # Write what the datasource was and is.. + ds = "%s: %s" % (util.obj_name(self.datasource), self.datasource) + previous_ds = '' + ds_fn = os.path.join(idir, 'datasource') + try: + previous_ds = util.load_file(ds_fn).strip() + except Exception: + pass + if not previous_ds: + # TODO: ?? is this right + previous_ds = ds + util.write_file(ds_fn, "%s\n" % ds) + util.write_file(os.path.join(dp, 'previous-datasource'), + "%s\n" % (previous_ds)) + + # What the instance id was and is... + iid = self.datasource.get_instance_id() + previous_iid = '' + p_iid_fn = os.path.join(dp, 'previous-instance-id') + c_iid_fn = os.path.join(dp, 'instance-id') + try: + previous_iid = util.load_file(p_iid_fn).strip() + except Exception: + pass + if not previous_iid: + # TODO: ?? is this right + previous_iid = iid + util.write_file(c_iid_fn, "%s\n" % iid) + util.write_file(p_iid_fn, "%s\n" % previous_iid) + + def fetch(self): + return self._get_data_source() + + def instancify(self): + self._reflect_cur_instance() + + def update(self): + self._write_to_cache() + self._store_userdata() + + def _store_userdata(self): + raw_ud = "%s" % (self.datasource.get_userdata_raw()) + util.write_file(self.paths.get_ipath('userdata_raw'), raw_ud, 0600) + processed_ud = "%s" % (self.datasource.get_userdata()) + util.write_file(self.paths.get_ipath('userdata'), processed_ud, 0600) + + def consume(self, frequency=PER_INSTANCE): + cdir = self.paths.get_cpath("handlers") + idir = self.paths.get_ipath("handlers") + + # Add the path to the plugins dir to the top of our list for import + # instance dir should be read before cloud-dir + sys.path.insert(0, cdir) + sys.path.insert(0, idir) + + # This keeps track of all the active handlers + c_handlers = helpers.ContentHandlers(self.paths) + + # Add handlers in cdir + potential_handlers = util.find_modules(cdir) + for (fname, modname) in potential_handlers.iteritems(): + try: + mod = ud.fixup_module(importer.import_module(modname)) + types = c_handlers.register(mod) + LOG.debug("Added handler for [%s] from %s", types, fname) + except: + LOG.exception("Failed to register handler from %s", fname) + + def_handlers = c_handlers.register_defaults() + if def_handlers: + LOG.debug("Registered default handlers for [%s]", def_handlers) + + # Form our cloud proxy + data = cloud.Cloud(self.datasource, + self.paths, copy.deepcopy(self.cfg)) + + # Init the handlers first + # Ensure userdata fetched before activation + called = [] + for (_mtype, mod) in c_handlers.iteritems(): + if mod in called: + continue + ud.call_begin(mod, data, frequency) + called.append(mod) + + # Walk the user data + part_data = { + 'handlers': c_handlers, + 'handlerdir': idir, + 'data': data, + 'frequency': frequency, + 'handlercount': 0, + } + ud.walk(data.get_userdata(), ud.walker_callback, data=part_data) + + # Give callbacks opportunity to finalize + called = [] + for (_mtype, mod) in c_handlers.iteritems(): + if mod in called: + continue + ud.call_end(mod, data, frequency) + called.append(mod) + + +class Handlers(object): + def __init__(self, datasource, h_cloud, cfgfile=None, basecfg=None): + self.datasource = datasource + self.cfgfile = cfgfile + self.basecfg = basecfg + self.h_cloud = h_cloud + self.cachedcfg = None + + @property + def cfg(self): + if self.cachedcfg is None: + self.cachedcfg = self._get_config(self.cfgfile) + return self.cachedcfg + + def _get_config(self, cfgfile): + mcfg = None + + if self.cfgfile: + try: + mcfg = util.read_conf(cfgfile) + except: + LOG.exception(("Failed loading of cloud config '%s'. " + "Continuing with an empty config."), cfgfile) + if not mcfg: + mcfg = {} + + ds_cfg = None + try: + ds_cfg = self.datasource.get_config_obj() + except: + LOG.exception("Failed loading of datasource config.") + if not ds_cfg: + ds_cfg = {} + + mcfg = util.mergedict(mcfg, ds_cfg) + if self.basecfg: + return util.mergedict(mcfg, self.basecfg) + else: + return mcfg + + + def _read_modules(self, name): + module_list = [] + if name not in self.cfg: + return module_list + cfg_mods = self.cfg[name] + # Create 'module_list', an array of arrays + # Where array[0] = module name + # array[1] = frequency + # array[2:] = arguments + for item in cfg_mods: + if not item: + continue + if isinstance(item, str): + module_list.append([item]) + elif isinstance(item, list): + module_list.append(item) + else: + raise TypeError("Failed to read '%s' item in config") + return module_list + + def _form_modules(self, raw_mods): + mostly_mods = [] + for raw_mod in raw_mods: + raw_name = raw_mod[0] + freq = None + run_args = None + if len(raw_mod) > 1: + freq = raw_mod[1] + if len(raw_mod) > 2: + run_args = raw_mod[2:] + if not run_args: + run_args = [] + mod_name = handlers.form_module_name(raw_name) + if not mod_name: + continue + mod = handlers.fixup_module(importer.import_module(mod_name)) + mostly_mods.append([mod, raw_name, freq, run_args]) + return mostly_mods + + def _run_modules(self, mostly_mods): + failures = [] + for (mod, name, freq, args) in mostly_mods: + try: + if not freq: + freq = mod.frequency + if not freq: + freq = PER_INSTANCE + func_args = [name, copy.deepcopy(self.cfg), + self.h_cloud, LOG, + args] + run_name = "config-" + name + self.h_cloud.run(run_name, mod.handle, func_args, freq=freq) + except: + LOG.exception("Running %s failed", mod) + failures.append(name) + return failures + + def run(self, name): + raw_mods = self._read_modules(name) + mostly_mods = self._form_modules(raw_mods) + return self._run_modules(mostly_mods) -- cgit v1.2.3 From a987baa675c19c8c2064c7eee786e93468a9c8ec Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:12:29 -0700 Subject: Moved these functions to utils. --- cloudinit/__init__.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/cloudinit/__init__.py b/cloudinit/__init__.py index ab0603e0..da124641 100644 --- a/cloudinit/__init__.py +++ b/cloudinit/__init__.py @@ -19,17 +19,3 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - -from cloudinit import util - -from cloudinit.settings import (CFG_BUILTIN, CLOUD_CONFIG) - - -def get_base_cfg(cfg_path=None): - if not cfg_path: - cfg_path = CLOUD_CONFIG - return util.get_base_cfg(cfg_path, get_builtin_cfg()) - - -def get_builtin_cfg(): - return dict(CFG_BUILTIN) -- cgit v1.2.3 From 6a58c8ee7345c53dda47ae0845b0e8cae0043dd4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:12:49 -0700 Subject: This will now just contain the "public" view of the cloud objects, backed by whatever other set of objects. --- cloudinit/cloud.py | 482 ++--------------------------------------------------- 1 file changed, 11 insertions(+), 471 deletions(-) diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index 80d4f1ce..765e7d3a 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -20,158 +20,23 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from time import time - -import cPickle as pickle - -import contextlib -import copy -import os -import sys -import weakref - -from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS) -from cloudinit.settings import (OLD_CLOUD_CONFIG, CLOUD_CONFIG) - -from cloudinit import (get_builtin_cfg, get_base_cfg) +from cloudinit import distros +from cloudinit import helpers from cloudinit import log as logging -from cloudinit import sources -from cloudinit import util -from cloudinit import handlers - -from cloudinit import user_data as ud -from cloudinit.user_data import boot_hook as bh_part -from cloudinit.user_data import cloud_config as cc_part -from cloudinit.user_data import processor as ud_proc -from cloudinit.user_data import shell_script as ss_part -from cloudinit.user_data import upstart_job as up_part LOG = logging.getLogger(__name__) -class CloudSemaphores(object): - def __init__(self, sem_path): - self.sem_path = sem_path - - # acquire lock on 'name' for given 'freq' and run function 'func' - # if 'clear_on_fail' is True and 'func' throws an exception - # then remove the lock (so it would run again) - def run_functor(self, name, freq, functor, args=None, clear_on_fail=False): - if not args: - args = [] - if self.has_run(name, freq): - LOG.debug("%s already ran %s", name, freq) - return False - with self.lock(name, freq, clear_on_fail) as lock: - if not lock: - raise RuntimeError("Failed to acquire lock on %s" % name) - else: - LOG.debug("Running %s with args %s using lock %s", func, args, lock) - func(*args) - return True - - @contextlib.contextmanager - def lock(self, name, freq, clear_on_fail=False): - try: - yield self._acquire(name, freq) - except: - if clear_on_fail: - self.clear(name, freq) - raise - - def clear(self, name, freq): - sem_file = self._getpath(name, freq) - try: - util.del_file(sem_file) - except IOError: - return False - return True - - def _acquire(self, name, freq): - if self.has_run(name, freq): - return None - # This is a race condition since nothing atomic is happening - # here, but this should be ok due to the nature of when - # and where cloud-init runs... (file writing is not a lock..) - sem_file = self._getpath(name, freq) - contents = "%s: %s\n" % (os.getpid(), time()) - try: - util.write_file(sem_file, contents) - except (IOError, OSError): - return None - return sem_file - - def has_run(self, name, freq): - if freq == PER_ALWAYS: - return False - sem_file = self._get_path(name, freq) - if os.path.exists(sem_file): - return True - return False - - def _get_path(self, name, freq): - sem_path = self.sem_path - if freq == PER_INSTANCE: - return os.path.join(sem_path, name) - return os.path.join(sem_path, "%s.%s" % (name, freq)) - - -class CloudPaths(object): - def __init__(self, sys_info): - self.cloud_dir = sys_info['cloud_dir'] - self.instance_link = os.path.join(self.cloud_dir, 'instance') - self.boot_finished = os.path.join(self.instance_link, "boot-finished") - self.upstart_conf_d = sys_info.get('upstart_dir') - self.template_dir = sys_info['templates_dir'] - self.seed_dir = os.path.join(self.cloud_dir, 'seed') - self.datasource = None - self.lookups = { - "handlers": "handlers", - "scripts": "scripts", - "sem": "sem", - "boothooks": "boothooks", - "userdata_raw": "user-data.txt", - "userdata": "user-data.txt.i", - "obj_pkl": "obj.pkl", - "cloud_config": "cloud-config.txt", - "data": "data", - } - - # get_ipath_cur: get the current instance path for an item - def get_ipath_cur(self, name=None): - ipath = os.path.join(self.cloud_dir, 'instance') - add_on = self.lookups.get(name) - if add_on: - ipath = os.path.join(ipath, add_on) - return ipath - - # get_cpath : get the "clouddir" (/var/lib/cloud/) - # for a name in dirmap - def get_cpath(self, name=None): - cpath = self.var_dir - add_on = self.lookups.get(name) - if add_on: - cpath = os.path.join(cpath, add_on) - return cpath - - # get_ipath : get the instance path for a name in pathmap - # (/var/lib/cloud/instances//) - def get_ipath(self, name=None): - if not self.datasource: - raise RuntimeError("Unable to get instance path, datasource not available/set.") - iid = self.datasource.get_instance_id() - ipath = os.path.join(self.cloud_dir, 'instances', iid) - add_on = self.lookups.get(name) - if add_on: - ipath = os.path.join(ipath, add_on) - return ipath - +class Cloud(object): + def __init__(self, datasource, paths, cfg): + self.datasource = datasource + self.paths = paths + self.cfg = cfg + self.distro = distros.fetch(cfg, self) + self.runners = helpers.Runners(paths) -class CloudSimple(object): - def __init__(self, ci): - self.datasource = init.datasource - self.paths = init.paths - self.cfg = copy.deepcopy(ci.cfg) + def run(self, name, functor, args, freq=None, clear_on_fail=False): + return self.runners.run(name, functor, args, freq, clear_on_fail) def get_userdata(self): return self.datasource.get_userdata() @@ -199,328 +64,3 @@ class CloudSimple(object): def get_ipath(self, name=None): return self.paths.get_ipath(name) - - -class CloudInit(object): - def __init__(self, ds_deps=None): - self.datasource = None - if ds_deps: - self.ds_deps = ds_deps - else: - self.ds_deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] - self.cfg = self._read_cfg() - self.paths = CloudPaths(self.cfg['system_info']) - - def _read_cfg_old(self): - # support reading the old ConfigObj format file and merging - # it into the yaml dictionary - try: - from configobj import ConfigObj - except ImportError: - ConfigObj = None - if not ConfigObj: - return {} - old_cfg = ConfigObj(OLD_CLOUD_CONFIG) - return dict(old_cfg) - - def _initial_subdirs(self): - c_dir = self.paths.cloud_dir - initial_dirs = [ - os.path.join(c_dir, 'scripts'), - os.path.join(c_dir, 'scripts', 'per-instance'), - os.path.join(c_dir, 'scripts', 'per-once'), - os.path.join(c_dir, 'scripts', 'per-boot'), - os.path.join(c_dir, 'seed'), - os.path.join(c_dir, 'instances'), - os.path.join(c_dir, 'handlers'), - os.path.join(c_dir, 'sem'), - os.path.join(c_dir, 'data'), - ] - return initial_dirs - - def purge_cache(self, rmcur=True): - rmlist = [] - rmlist.append(self.paths.boot_finished) - if rmcur: - rmlist.append(self.paths.instance_link) - for f in rmlist: - util.unlink(f) - return len(rmlist) - - def init_fs(self): - util.ensure_dirs(self._initial_subdirs()) - log_file = util.get_cfg_option_str(self.cfg, 'def_log_file', None) - perms = util.get_cfg_option_str(self.cfg, 'syslog_fix_perms', None) - if log_file: - util.ensure_file(log_file) - if perms: - (u, g) = perms.split(':', 1) - if u == "-1" or u == "None": - u = None - if g == "-1" or g == "None": - g = None - util.chownbyname(log_file, u, g) - - def _read_cfg(self): - starting_config = get_builtin_cfg() - try: - conf = get_base_cfg(CLOUD_CONFIG, starting_config) - except Exception: - conf = starting_config - old_conf = self._read_cfg_old() - conf = util.mergedict(conf, old_conf) - return conf - - def _restore_from_cache(self): - pickled_fn = self.paths.get_ipath_cur('obj_pkl') - try: - # we try to restore from a current link and static path - # by using the instance link, if purge_cache was called - # the file wont exist - return pickle.loads(util.load_file(pickled_fn)) - except Exception as e: - LOG.debug("Failed loading pickled datasource from %s due to %s", pickled_fn, e) - return False - - def write_to_cache(self): - pickled_fn = self.paths.get_ipath_cur("obj_pkl") - try: - contents = pickle.dumps(self.datasource) - util.write_file(pickled_fn, contents, mode=0400) - except Exception as e: - LOG.debug("Failed pickling datasource to %s due to: %s", pickled_fn, e) - return False - - def _get_processor(self): - return ud_proc.UserDataProcessor(self.paths) - - def _get_datasources(self): - # Any config provided??? - pkg_list = self.cfg.get('datasource_pkg_list') or [] - # Add the defaults at the end - for n in [util.obj_name(sources), '']: - if n not in pkg_list: - pkg_list.append(n) - cfg_list = self.cfg.get('datasource_list') or [] - return (cfg_list, pkg_list) - - def get_data_source(self): - if self.datasource: - return True - ds = self._restore_from_cache() - if ds: - LOG.debug("Restored from cache datasource: %s" % ds) - else: - (cfg_list, pkg_list) = self._get_datasources() - ud_proc = self._get_processor() - (ds, dsname) = sources.find_source(self.cfg, - self.ds_deps, - cfg_list=cfg_list, - pkg_list=pkg_list, - ud_proc=ud_proc) - LOG.debug("Loaded datasource %s - %s", dsname, ds) - self.datasource = ds - # This allows the paths obj to have an ipath function that works - self.paths.datasource = ds - return True - - def set_cur_instance(self): - # Ensure we are hooked into the right symlink for the current instance - idir = self.paths.get_ipath() - util.del_file(self.paths.instance_link) - util.sym_link(idir, self.paths.instance_link) - - dlist = [] - for d in ["handlers", "scripts", "sem"]: - dlist.append(os.path.join(idir, d)) - util.ensure_dirs(dlist) - - # Write out information on what is being used for the current instance - # and what may have been used for a previous instance... - dp = self.paths.get_cpath('data') - ds = "%s: %s\n" % (self.datasource.__class__, self.datasource) - previous_ds = '' - ds_fn = os.path.join(idir, 'datasource') - try: - previous_ds = util.load_file(ds_fn).strip() - except IOError as e: - pass - if not previous_ds: - # TODO: ?? is this right - previous_ds = ds - util.write_file(ds_fn, ds) - util.write_file(os.path.join(dp, 'previous-datasource'), previous_ds) - iid = self.datasource.get_instance_id() - previous_iid = '' - p_iid_fn = os.path.join(dp, 'previous-instance-id') - try: - previous_iid = util.load_file(p_iid_fn).strip() - except IOError as e: - pass - if not previous_iid: - # TODO: ?? is this right - previous_iid = iid - util.write_file(p_iid_fn, "%s\n" % previous_iid) - - def update_cache(self): - self.write_to_cache() - self.store_userdata() - - def store_userdata(self): - raw_ud = "%s" % (self.datasource.get_userdata_raw()) - util.write_file(self.paths.get_ipath('userdata_raw'), raw_ud, 0600) - ud = "%s" % (self.datasource.get_userdata()) - util.write_file(self.paths.get_ipath('userdata'), ud, 0600) - - def consume_userdata(self, frequency=PER_INSTANCE): - cdir = self.paths.get_cpath("handlers") - idir = self.paths.get_ipath("handlers") - - # Add the path to the plugins dir to the top of our list for import - # instance dir should be read before cloud-dir - sys.path.insert(0, cdir) - sys.path.insert(0, idir) - - # Data will be a little proxy that modules can use - data = CloudSimple(self) - - # This keeps track of all the active handlers - handlers = CloudHandlers(self) - - # Add handlers in cdir - potential_handlers = utils.find_modules(cdir) - for (fname, modname) in potential_handlers.iteritems(): - try: - mod = parts.fixup_module(importer.import_module(modname)) - types = handlers.register(mod) - LOG.debug("Added handler for [%s] from %s", types, fname) - except: - LOG.exception("Failed to register handler from %s", fname) - - def_handlers = handlers.register_defaults() - if def_handlers: - LOG.debug("Registered default handlers for [%s]", def_handlers) - - # Init the handlers first - # Ensure userdata fetched before activation - called = [] - for (_mtype, mod) in handlers.iteritems(): - if mod in called: - continue - parts.call_begin(mod, data, frequency) - called.append(mod) - - # Walk the user data - part_data = { - 'handlers': handlers, - 'handlerdir': idir, - 'data': data, - 'frequency': frequency, - 'handlercount': 0, - } - ud.walk(data.get_userdata(), parts.walker_callback, data=part_data) - - # Give callbacks opportunity to finalize - called = [] - for (_mtype, mod) in handlers.iteritems(): - if mod in called: - continue - parts.call_end(mod, data, frequency) - called.append(mod) - - -class CloudHandlers(object): - - def __init__(self, paths): - self.paths = paths - self.registered = {} - - def __contains__(self, item): - return self.is_registered(item) - - def __getitem__(self, key): - return self._get_handler(key) - - def is_registered(self, content_type): - return content_type in self.registered - - def register(self, mod): - types = set() - for t in mod.list_types(): - self.registered[t] = handler - types.add(t) - return types - - def _get_handler(self, content_type): - return self.registered[content_type] - - def items(self): - return self.registered.items() - - def iteritems(self): - return self.registered.iteritems() - - def _get_default_handlers(self): - def_handlers = [] - if self.paths.get_ipath("cloud_config"): - def_handlers.append(cc_part.CloudConfigPartHandler(self.paths.get_ipath("cloud_config"))) - if self.paths.get_ipath_cur('scripts'): - def_handlers.append(ss_part.ShellScriptPartHandler(self.paths.get_ipath_cur('scripts'))) - if self.paths.get_ipath("boothooks"): - def_handlers.append(bh_part.BootHookPartHandler(self.paths.get_ipath("boothooks"))) - if self.paths.upstart_conf_d: - def_handlers.append(up_part.UpstartJobPartHandler(self.paths.upstart_conf_d)) - return def_handlers - - def register_defaults(self): - registered = set() - for h in self._get_default_handlers(): - for t in h.list_types(): - if not self.is_registered(t) - self.register_handler(t, h) - registered.add(t) - return registered - - -class CloudConfig(object): - def __init__(self, cfgfile, cloud): - self.cloud = cloud - self.cfg = self._get_config(cfgfile) - self.paths = cloud.paths - self.sems = CloudSemaphores(self.paths.get_ipath("sem")) - - def _get_config(self, cfgfile): - cfg = None - try: - cfg = util.read_conf(cfgfile) - except: - LOG.exception(("Failed loading of cloud config '%s'. " - "Continuing with empty config."), cfgfile) - if not cfg: - cfg = {} - - ds_cfg = None - try: - ds_cfg = self.cloud.datasource.get_config_obj() - except: - LOG.exception("Failed loading of datasource config.") - if not ds_cfg: - ds_cfg = {} - - cfg = util.mergedict(cfg, ds_cfg) - cloud_cfg = self.cloud.cfg or {} - return util.mergedict(cfg, cloud_cfg) - - def extract(self, name): - modname = handlers.form_module_name(name) - if not modname: - return None - return handlers.fixup_module(importer.import_module(modname)) - - def handle(self, name, mod, args, freq=None): - def_freq = mod.frequency - if not freq: - freq = def_freq - c_name = "config-%s" % (name) - real_args = [name, copy.deepcopy(self.cfg), CloudSimple(self.cloud), LOG, copy.deepcopy(args)] - return self.sems.run_functor(c_name, freq, mod.handle, real_args) -- cgit v1.2.3 From 70640d62e1dffeabceacf16b52a3122fe914c297 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:13:19 -0700 Subject: Adding in the root distro class + a util function to fetch various distros. --- cloudinit/distros/__init__.py | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index e69de29b..f7f48d1f 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -0,0 +1,45 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import contextlib + +from cloudinit import importer + + +class Distro(object): + def __init__(self, cloud): + self.cloud = cloud + + def install_packages(self, pkglist): + raise NotImplementedError() + + def apply_network(self, settings): + raise NotImplementedError() + + +def fetch(cfg, cloud): + sys_info = cfg.get('system_info', {}) + distro = sys_info.get('distro', 'ubuntu') + mod_name = "%s.%s" % (__name__, distro) + mod = importer.import_module(mod_name) + distro_cls = getattr(mod, 'Distro') + return distro_cls(cloud) \ No newline at end of file -- cgit v1.2.3 From 3b964024040bed3c217b6d4f043af55d0dbc5469 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:14:46 -0700 Subject: Adding in the base of a ubuntu distro subclass that can install pkgs (right now). --- cloudinit/distros/ubuntu.py | 55 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index e69de29b..d19ef63e 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -0,0 +1,55 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os + +from cloudinit import distros +from cloudinit import log as logging +from cloudinit import util + +from cloudinit.settings import (PER_INSTANCE) + + +LOG = logging.getLogger(__name__) + + +class Distro(distros.Distro): + + def install_packages(self, pkglist): + self.update_package_sources() + self.apt_get('install', pkglist) + + def apply_network(self, settings): + pass + + # apt_get top level command (install, update...), and args to pass it + def apt_get(self, tlc, args=None): + e = os.environ.copy() + e['DEBIAN_FRONTEND'] = 'noninteractive' + cmd = ['apt-get', '--option', 'Dpkg::Options::=--force-confold', + '--assume-yes', tlc] + if args: + cmd.extend(args) + util.subp(cmd, env=e) + + def update_package_sources(self): + self.cloud.run("update-sources", self.apt_get, ["update"], freq=PER_INSTANCE) \ No newline at end of file -- cgit v1.2.3 From 82243cdf875cd7d8b2fbd962fa92ab139c653043 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:15:16 -0700 Subject: Removing distro specifics from here, as well as running functionality (which is now elsewhere). --- cloudinit/handlers/__init__.py | 40 +++------------------------------------- 1 file changed, 3 insertions(+), 37 deletions(-) diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index ae74b683..5d70ac43 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -36,7 +36,6 @@ LOG = logging.getLogger(__name__) DEF_HANDLER_VERSION = 1 DEF_FREQ = PER_INSTANCE -HANDLER_TPL = "cc_%s" # reads a cloudconfig module list, returns @@ -198,41 +197,6 @@ def redirect_output(outfmt, errfmt, o_out=sys.stdout, o_err=sys.stderr): return -def run_per_instance(name, func, args, clear_on_fail=False): - semfile = "%s/%s" % (cloudinit.get_ipath_cur("data"), name) - if os.path.exists(semfile): - return - - util.write_file(semfile, str(time.time())) - try: - func(*args) - except: - if clear_on_fail: - os.unlink(semfile) - raise - - -# apt_get top level command (install, update...), and args to pass it -def apt_get(tlc, args=None): - if args is None: - args = [] - e = os.environ.copy() - e['DEBIAN_FRONTEND'] = 'noninteractive' - cmd = ['apt-get', '--option', 'Dpkg::Options::=--force-confold', - '--assume-yes', tlc] - cmd.extend(args) - subprocess.check_call(cmd, env=e) - - -def update_package_sources(): - run_per_instance("update-sources", apt_get, ("update",)) - - -def install_packages(pkglist): - update_package_sources() - apt_get("install", pkglist) - - def form_module_name(name): canon_name = name.replace("-", "_") if canon_name.endswith(".py"): @@ -240,7 +204,9 @@ def form_module_name(name): canon_name = canon_name.strip() if not canon_name: return None - return HANDLER_TPL % (canon_name) + if not canon_name.startswith("cc_"): + canon_name = 'cc_%s' % (canon_name) + return canon_name def fixup_module(mod): -- cgit v1.2.3 From ebe5943575771be771657bdbc82a7e36669ff9a1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:15:39 -0700 Subject: Added log setup and a basic log (for when one is not enabled) --- cloudinit/log.py | 66 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/cloudinit/log.py b/cloudinit/log.py index 341b329b..6e7424e1 100644 --- a/cloudinit/log.py +++ b/cloudinit/log.py @@ -40,52 +40,56 @@ INFO = logging.INFO DEBUG = logging.DEBUG NOTSET = logging.NOTSET +# Default basic format +DEF_FORMAT = '%(levelname)s: @%(name)s : %(message)s' -class ConsoleFormatter(logging.Formatter): - def _get_mini_level(self, record): - if record.levelno in [INFO, NOTSET] or not record.levelname: - return '' - lvlname = record.levelname - return lvlname[0].upper() + ": " +def setupBasicLogging(level=INFO, fmt=DEF_FORMAT): + root = getLogger() + console = logging.StreamHandler(sys.stdout) + console.setFormatter(logging.Formatter(fmt)) + console.setLevel(level) + root.addHandler(console) + root.setLevel(level) - def format(self, record): - record.message = record.getMessage() - rdict = dict(record.__dict__) - rdict['minilevelname'] = self._get_mini_level(record) - return self._fmt % (rdict) +def setupLogging(cfg=None): + # See if the config provides any logging conf... + if not cfg: + cfg = {} -def setupLogging(cfg): log_cfgs = [] log_cfg = cfg.get('logcfg') - if log_cfg: - # if there is a 'logcfg' entry in the config, respect - # it, it is the old keyname - log_cfgs = [log_cfg] - elif "log_cfgs" in cfg: - for cfg in cfg['log_cfgs']: - if isinstance(cfg, list): - log_cfgs.append('\n'.join(cfg)) + if log_cfg and isinstance(log_cfg, (str, basestring)): + # Ff there is a 'logcfg' entry in the config, + # respect it, it is the old keyname + log_cfgs.append(str(log_cfg)) + elif "log_cfgs" in cfg and isinstance(cfg['log_cfgs'], (set, list)): + for a_cfg in cfg['log_cfgs']: + if isinstance(a_cfg, (list, set, dict)): + cfg_str = [str(c) for c in a_cfg] + log_cfgs.append('\n'.join(cfg_str)) else: - log_cfgs.append(cfg) - - if not len(log_cfgs): - sys.stderr.write("Warning, no logging configured\n") - return + log_cfgs.append(str(a_cfg)) + # See if any of them actually load... am_worked = 0 - for logcfg in log_cfgs: + for log_cfg in log_cfgs: try: - if not os.path.isfile(logcfg): - logcfg = StringIO(logcfg) - logging.config.fileConfig(logcfg) + if not os.path.isfile(log_cfg): + log_cfg = StringIO(log_cfg) + logging.config.fileConfig(log_cfg) am_worked += 1 - except: + except Exception: pass + # If it didn't work, at least setup a basic logger + basic_enabled = cfg.get('log_basic', True) if not am_worked: - sys.stderr.write("Warning, no logging configured\n") + sys.stderr.write("Warning, no logging configured!\n") + if basic_enabled: + sys.stderr.write("Setting up basic logging...\n") + setupBasicLogging() def getLogger(name='cloudinit'): -- cgit v1.2.3 From 6a80e79a485aec3dcf1072be411441b415302038 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:16:07 -0700 Subject: Start of cleaning this up. --- cloudinit/netinfo.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index 1f0fa571..541d2911 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -22,6 +22,11 @@ import cloudinit.util as util +class NetInfo(object): + def __init__(self): + pass + + def netdev_info(empty=""): fields = ("hwaddr", "addr", "bcast", "mask") (ifcfg_out, _err) = util.subp(["ifconfig", "-a"]) -- cgit v1.2.3 From 2803567ebaf428aa0711ce00be9aa532a3f35410 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:16:19 -0700 Subject: Fix this up to use the new datasource class hierachy, as well as other new objects/logging added... --- cloudinit/sources/DataSourceCloudStack.py | 76 +++++++++++++++++-------------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 5afdf7b6..33fb3491 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -18,62 +18,68 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util from socket import inet_ntoa +from struct import pack + +import os import time + import boto.utils as boto_utils -from struct import pack +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util + +LOG = logging.getLogger(__name__) -class DataSourceCloudStack(DataSource.DataSource): - api_ver = 'latest' - seeddir = base_seeddir + '/cs' - metadata_address = None - def __init__(self, sys_cfg=None): - DataSource.DataSource.__init__(self, sys_cfg) +class DataSourceCloudStack(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.seed_dir = os.path.join(paths.seed_dir, 'cs') # Cloudstack has its metadata/userdata URLs located at # http:///latest/ - self.metadata_address = "http://%s/" % self.get_default_gateway() + self.api_ver = 'latest' + gw_addr = self.get_default_gateway() + if not gw_addr: + raise RuntimeError("No default gateway found!") + self.metadata_address = "http://%s/" % (gw_addr) def get_default_gateway(self): """ Returns the default gateway ip address in the dotted format """ - with open("/proc/net/route", "r") as f: - for line in f.readlines(): - items = line.split("\t") - if items[1] == "00000000": - # found the default route, get the gateway - gw = inet_ntoa(pack(" Date: Mon, 11 Jun 2012 17:17:05 -0700 Subject: Add more logging in onto what is being searched for. --- cloudinit/sources/__init__.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 05c8bfad..dfd1fff3 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -24,9 +24,12 @@ from cloudinit import importer from cloudinit import log as logging from cloudinit import util +from cloudinit.user_data import processor as ud_proc + DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" DS_PREFIX = 'DataSource' + LOG = logging.getLogger(__name__) @@ -35,26 +38,25 @@ class DataSourceNotFoundException(Exception): class DataSource(object): - def __init__(self, ud_proc, cfg): + def __init__(self, sys_cfg, distro, paths): name = util.obj_name(self) if name.startswith(DS_PREFIX): name = name[DS_PREFIX:] self.cfgname = name - if sys_cfg: - self.sys_cfg = sys_cfg - else: - self.sys_cfg = {} - self.ud_proc = ud_proc + self.sys_cfg = sys_cfg + self.distro = distro + self.paths = paths + self.userdata_proc = ud_proc.UserDataProcessor(paths) self.userdata = None self.metadata = None self.userdata_raw = None self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, - ("datasource", self.cfgname), self.ds_cfg) + ("datasource", self.cfgname), {}) def get_userdata(self): if self.userdata is None: raw_data = self.get_userdata_raw() - self.userdata = self.ud_proc.process(raw_data) + self.userdata = self.userdata_proc.process(raw_data) return self.userdata def get_userdata_raw(self): @@ -85,7 +87,7 @@ class DataSource(object): # than a list. if isinstance(klist, (str)): klist = [klist] - if isinstance(klist, (list)): + if isinstance(klist, (list, set)): for pkey in klist: # there is an empty string at the end of the keylist, trim it if pkey: @@ -105,6 +107,7 @@ class DataSource(object): return 'en_US.UTF-8' def get_local_mirror(self): + # ?? return None def get_instance_id(self): @@ -152,20 +155,18 @@ class DataSource(object): return hostname -def find_source(cfg, ds_deps, cfg_list, pkg_list, **kwargs): +def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): ds_list = list_sources(cfg_list, ds_deps, pkg_list) ds_names = [util.obj_name(f) for f in ds_list] - ds_args = dict(kwargs) - ds_args['cfg'] = cfg LOG.info("Searching for data source in: %s", ds_names) for cls in ds_list: ds = util.obj_name(cls) try: - s = cls(**ds_args) + s = cls(distro, sys_cfg, paths) if s.get_data(): return (s, ds) except Exception as e: - LOG.exception("Getting data from %s failed", ds) + LOG.exception("Getting data from %s failed due to %s", ds, e) msg = "Did not find any data source, searched classes: %s" % (ds_names) raise DataSourceNotFoundException(msg) @@ -187,8 +188,7 @@ def list_sources(cfg_list, depends, pkg_list): if pkg: pkg_name.append(str(pkg)) pkg_name.append(ds_name) - mod_name = ".".join(pkg_name) - mod = importer.import_module(mod_name) + mod = importer.import_module(".".join(pkg_name)) if pkg: mod = getattr(mod, ds_name, None) if not mod: @@ -196,10 +196,13 @@ def list_sources(cfg_list, depends, pkg_list): lister = getattr(mod, "get_datasource_list", None) if not lister: continue + LOG.debug("Seeing if %s matches using function %s", mod, lister) cls_matches = lister(depends) if not cls_matches: continue src_list.extend(cls_matches) + LOG.debug("Found a match for data source %s in %s with matches %s", + ds_name, mod, cls_matches) break return src_list -- cgit v1.2.3 From 7a719072faac3b0947d163968bd6e311859ceb3b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:17:51 -0700 Subject: Add comments on formats, add in function that handles option extraction + other pylint cleanups. --- cloudinit/ssh_util.py | 105 +++++++++++++++++++++++++++++--------------------- 1 file changed, 62 insertions(+), 43 deletions(-) diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index 93fd55dd..c97b3819 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -19,6 +19,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from StringIO import StringIO + +import csv import os import pwd @@ -26,6 +29,7 @@ from cloudinit import log as logging from cloudinit import util LOG = logging.getLogger(__name__) +DEF_SSHD_CFG = "/etc/ssh/sshd_config" class AuthKeyEntry(object): @@ -52,6 +56,40 @@ class AuthKeyEntry(object): self.line = str(line) (self.value, self.components) = self._parse(self.line, def_opt) + def _extract_options(self, ent): + """ + The options (if present) consist of comma-separated option specifica- + tions. No spaces are permitted, except within double quotes. + Note that option keywords are case-insensitive. + """ + quoted = False + i = 0 + while (i < len(ent) and + ((quoted) or (ent[i] not in (" ", "\t")))): + curc = ent[i] + if i + 1 >= len(ent): + i = i + 1 + break + nextc = ent[i + 1] + if curc == "\\" and nextc == '"': + i = i + 1 + elif curc == '"': + quoted = not quoted + i = i + 1 + + options = ent[0:i] + options_lst = [] + reader = csv.reader(StringIO(options), quoting=csv.QUOTE_NONE) + for row in reader: + for e in row: + e = e.strip() + if e: + options_lst.append(e) + toks = [] + if i + 1 < len(ent): + toks = ent[i + 1:].split(None, 3) + return (options_lst, toks) + def _form_components(self, toks): components = {} if len(toks) == 1: @@ -81,29 +119,10 @@ class AuthKeyEntry(object): if len(toks) < 4: tmp_components.update(self._form_components(toks)) else: - # taken from auth_rsa_key_allowed in auth-rsa.c - i = 0 - quoted = False - try: - while (i < len(ent) and - ((quoted) or (ent[i] not in (" ", "\t")))): - curc = ent[i] - nextc = ent[i + 1] - if curc == "\\" and nextc == '"': - i = i + 1 - elif curc == '"': - quoted = not quoted - i = i + 1 - except IndexError: - return (False, {}) - try: - options = ent[0:i] - toks = ent[i + 1:].split(None, 3) - if options: - tmp_components['options'] = options - tmp_components.update(self._form_components(toks)) - except (IndexError, ValueError): - return (False, {}) + (options, toks) = self._extract_options(ent) + if options: + tmp_components['options'] = ",".join(options) + tmp_components.update(self._form_components(toks)) # We got some useful value! return (True, tmp_components) @@ -125,7 +144,7 @@ class AuthKeyEntry(object): return ' '.join(toks) -def update_authorized_keys(fname, keys): +def _update_authorized_keys(fname, keys): lines = [] try: if os.path.isfile(fname): @@ -159,9 +178,11 @@ def update_authorized_keys(fname, keys): return '\n'.join(lines) -def setup_user_keys(keys, user, key_prefix, sshd_config_fn="/etc/ssh/sshd_config"): - pwent = pwd.getpwnam(user) +def setup_user_keys(keys, user, key_prefix, sshd_config_fn=None): + if not sshd_config_fn: + sshd_config_fn = DEF_SSHD_CFG + pwent = pwd.getpwnam(user) ssh_dir = os.path.join(pwent.pw_dir, '.ssh') if not os.path.exists(ssh_dir): util.ensure_dir(ssh_dir, mode=0700) @@ -173,14 +194,12 @@ def setup_user_keys(keys, user, key_prefix, sshd_config_fn="/etc/ssh/sshd_config with util.SeLinuxGuard(ssh_dir, recursive=True): try: - """ - AuthorizedKeysFile may contain tokens - of the form %T which are substituted during connection set-up. - The following tokens are defined: %% is replaced by a literal - '%', %h is replaced by the home directory of the user being - authenticated and %u is replaced by the username of that user. - """ - ssh_cfg = parse_ssh_config(sshd_config_fn) + # AuthorizedKeysFile may contain tokens + # of the form %T which are substituted during connection set-up. + # The following tokens are defined: %% is replaced by a literal + # '%', %h is replaced by the home directory of the user being + # authenticated and %u is replaced by the username of that user. + ssh_cfg = _parse_ssh_config(sshd_config_fn) akeys = ssh_cfg.get("authorizedkeysfile", '') akeys = akeys.strip() if not akeys: @@ -193,22 +212,22 @@ def setup_user_keys(keys, user, key_prefix, sshd_config_fn="/etc/ssh/sshd_config authorized_keys = akeys except (IOError, OSError): authorized_keys = os.path.join(ssh_dir, 'authorized_keys') - LOG.exception(("Failed extracting 'AuthorizedKeysFile' in ssh config" - " from %s, using 'AuthorizedKeysFile' file %s instead."), + LOG.exception(("Failed extracting 'AuthorizedKeysFile'" + " in ssh config" + " from %s, using 'AuthorizedKeysFile' file" + " %s instead"), sshd_config_fn, authorized_keys) - content = update_authorized_keys(authorized_keys, key_entries) + content = _update_authorized_keys(authorized_keys, key_entries) util.ensure_dir(os.path.dirname(authorized_keys), mode=0700) util.write_file(authorized_keys, content, mode=0600) util.chownbyid(authorized_keys, pwent.pw_uid, pwent.pw_gid) -def parse_ssh_config(fname): - """ - The file contains keyword-argu-ment pairs, one per line. - Lines starting with '#' and empty lines are interpreted as comments. - Note: key-words are case-insensitive and arguments are case-sensitive - """ +def _parse_ssh_config(fname): + # The file contains keyword-argument pairs, one per line. + # Lines starting with '#' and empty lines are interpreted as comments. + # Note: key-words are case-insensitive and arguments are case-sensitive ret = {} if not os.path.isfile(fname): return ret -- cgit v1.2.3 From 1c11a941ca832c06ba125e1da226030504c58033 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:18:37 -0700 Subject: Start using tempita instead of the more complicated cheetah. --- cloudinit/templater.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/cloudinit/templater.py b/cloudinit/templater.py index 04cc5a6f..5839911c 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -20,15 +20,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from Cheetah.Template import Template +from tempita import Template from cloudinit import util -def render_to_file(template_fn, outfile, searchList): - contents = Template(file=template_fn, searchList=[searchList]).respond() +def render_from_file(fn, params): + return render_string(util.load_file(fn), params, name=fn) + + +def render_to_file(name, outfile, params): + contents = render_from_file(name, params) util.write_file(outfile, contents) -def render_string(template, searchList): - return Template(template, searchList=[searchList]).respond() +def render_string(content, params, name=None): + tpl = Template(content, name=name) + if not params: + params = dict() + return tpl.substitute(params) -- cgit v1.2.3 From bcf1477b78473e9f9cfdb4fb66c50972ef156cae Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:19:19 -0700 Subject: Logging fixups + pylint fixups. --- cloudinit/url_helper.py | 92 +++++++++++++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 38 deletions(-) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 6fa3e44b..1bf24c4f 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -20,14 +20,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from contextlib import closing import errno +import socket import time import urllib import urllib2 -from contextlib import closing - from cloudinit import log as logging LOG = logging.getLogger(__name__) @@ -37,46 +37,58 @@ def ok_http_code(st): return st in xrange(200, 400) -def readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=None): - openargs = {} - if timeout is not None: - openargs['timeout'] = int(timeout) - - if data is None: - req = urllib2.Request(url, headers=headers) - else: - req = urllib2.Request(url, data=urllib.urlencode(data), headers=headers) +def readurl(url, data=None, timeout=None, + retries=0, sec_between=1, headers=None): - if retries <= 0: - retries = 1 + req_args = {} + req_args['url'] = url + if data is not None: + req_args['data'] = urllib.urlencode(data) + if headers is not None: + req_args['headers'] = dict(headers) + req = urllib2.Request(**req_args) + retries = max(retries, 0) attempts = retries + 1 - last_excp = None - LOG.debug("Attempting to read from %s with %s attempts to be performed", url, attempts) + + last_excp = Exception("??") + LOG.info(("Attempting to read from %s with %s attempts" + " (%s retries) to be performed"), url, attempts, retries) + open_args = {} + if timeout is not None: + open_args['timeout'] = int(timeout) for i in range(0, attempts): try: - with closing(urllib2.urlopen(req, **openargs)) as rh: - return (rh.read(), rh.getcode()) + with closing(urllib2.urlopen(req, **open_args)) as rh: + content = rh.read() + status = rh.getcode() + if status is None: + # This seems to happen when files are read... + status = 200 + LOG.info("Read from %s (%s, %sb) after %s attempts", + url, status, len(content), (i + 1)) + return (content, status) except urllib2.HTTPError as e: last_excp = e LOG.exception("Failed at reading from %s.", url) except urllib2.URLError as e: # This can be a message string or - # another exception instance (socket.error for remote URLs, OSError for local URLs). + # another exception instance + # (socket.error for remote URLs, OSError for local URLs). if (isinstance(e.reason, OSError) and e.reason.errno == errno.ENOENT): last_excp = e.reason else: last_excp = e - LOG.exception("Failed at reading from %s.", url) + LOG.exception("Failed at reading from %s", url) if i + 1 < attempts: - LOG.debug("Please wait %s seconds while we wait to try again.", sec_between) + LOG.info("Please wait %s seconds while we wait to try again", + sec_between) time.sleep(sec_between) # Didn't work out - LOG.warn("Failed downloading from %s after %s attempts", url, attempts) - if last_excp is not None: - raise last_excp + LOG.warn("Failed reading from %s after %s attempts", url, attempts) + raise last_excp def wait_for_url(urls, max_wait=None, timeout=None, @@ -106,29 +118,29 @@ def wait_for_url(urls, max_wait=None, timeout=None, data host (169.254.169.254) may be firewalled off Entirely for a sytem, meaning that the connection will block forever unless a timeout is set. """ - starttime = time.time() + start_time = time.time() - def nullstatus_cb(msg): - return + def log_status_cb(msg): + LOG.info(msg) if status_cb is None: - status_cb = nullstatus_cb + status_cb = log_status_cb - def timeup(max_wait, starttime): + def timeup(max_wait, start_time): return ((max_wait <= 0 or max_wait is None) or - (time.time() - starttime > max_wait)) + (time.time() - start_time > max_wait)) loop_n = 0 while True: - sleeptime = int(loop_n / 5) + 1 + sleep_time = int(loop_n / 5) + 1 for url in urls: now = time.time() if loop_n != 0: - if timeup(max_wait, starttime): + if timeup(max_wait, start_time): break - if timeout and (now + timeout > (starttime + max_wait)): + if timeout and (now + timeout > (start_time + max_wait)): # shorten timeout to not run way over max_time - timeout = int((starttime + max_wait) - now) + timeout = int((start_time + max_wait) - now) reason = "" try: @@ -153,14 +165,18 @@ def wait_for_url(urls, max_wait=None, timeout=None, except Exception as e: reason = "unexpected error [%s]" % e - status_cb("'%s' failed [%s/%ss]: %s" % - (url, int(time.time() - starttime), max_wait, - reason)) + time_taken = int(time.time() - start_time) + status_msg = "Calling '%s' failed [%s/%ss]: %s" % (url, + time_taken, + max_wait, reason) + status_cb(status_msg) - if timeup(max_wait, starttime): + if timeup(max_wait, start_time): break loop_n = loop_n + 1 - time.sleep(sleeptime) + LOG.info("Please wait %s seconds while we wait to try again", + sleep_time) + time.sleep(sleep_time) return False -- cgit v1.2.3 From bccf7ec19694f2f29717e18e62dcd33688cb424d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:19:50 -0700 Subject: Pylint fixups. --- cloudinit/user_data/__init__.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/cloudinit/user_data/__init__.py b/cloudinit/user_data/__init__.py index 6264a6cc..1910ba28 100644 --- a/cloudinit/user_data/__init__.py +++ b/cloudinit/user_data/__init__.py @@ -28,6 +28,7 @@ import email from email.mime.base import MIMEBase +from cloudinit import importer from cloudinit import log as logging from cloudinit import util @@ -51,6 +52,7 @@ PART_FN_TPL = 'part-%03d' # Used as the content type when a message is not multipart # and it doesn't contain its own content-type NOT_MULTIPART_TYPE = "text/x-not-multipart" +OCTET_TYPE = 'application/octet-stream' # Different file beginnings to there content type INCLUSION_TYPES_MAP = { @@ -86,17 +88,17 @@ class PartHandler(object): raise NotImplementedError() -def fixup_module(mod): +def fixup_module(mod, def_freq=PER_INSTANCE): if not hasattr(mod, "handler_version"): setattr(mod, "handler_version", 1) if not hasattr(mod, 'list_types'): def empty_types(): return [] setattr(mod, 'list_types', empty_types) - if not hasattr(mod, frequency): - setattr(mod, 'frequency', PER_INSTANCE) + if not hasattr(mod, 'frequency'): + setattr(mod, 'frequency', def_freq) if not hasattr(mod, 'handle_part'): - def empty_handler(data, ctype, filename, payload): + def empty_handler(_data, _ctype, _filename, _payload): pass setattr(mod, 'handle_part', empty_handler) return mod @@ -114,7 +116,9 @@ def run_part(mod, data, ctype, filename, payload, frequency): else: mod.handle_part(data, ctype, filename, payload, frequency) except: - LOG.exception("Failed calling mod %s (%s, %s, %s) with frequency %s", mod, ctype, filename, mod_ver, frequency) + LOG.exception(("Failed calling mod %s (%s, %s, %s)" + " with frequency %s"), mod, ctype, filename, + mod_ver, frequency) def call_begin(mod, data, frequency): @@ -157,7 +161,8 @@ def walker_callback(pdata, ctype, filename, payload): details = repr(payload) LOG.warning("Unhandled non-multipart userdata: %s", details) return - run_part(handlers[ctype], pdata['data'], ctype, filename, payload, pdata['frequency']) + run_part(handlers[ctype], pdata['data'], ctype, filename, + payload, pdata['frequency']) # Callback is a function that will be called with @@ -182,7 +187,7 @@ def walk(msg, callback, data): # Coverts a raw string into a mime message -def convert_string(self, raw_data, headers=None): +def convert_string(raw_data, headers=None): if not raw_data: raw_data = '' if not headers: -- cgit v1.2.3 From 27133cc16d219e5594d7f17f25a48e3fb56db3a6 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:20:04 -0700 Subject: Pylint fixups. --- cloudinit/user_data/boot_hook.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudinit/user_data/boot_hook.py b/cloudinit/user_data/boot_hook.py index 4ce398ac..87e7a3ec 100644 --- a/cloudinit/user_data/boot_hook.py +++ b/cloudinit/user_data/boot_hook.py @@ -60,6 +60,7 @@ class BootHookPartHandler(ud.PartHandler): env['INSTANCE_ID'] = str(self.instance_id) util.subp([filepath], env=env) except util.ProcessExecutionError as e: - LOG.error("Boothooks script %s returned %s", filepath, e.exit_code) + LOG.error("Boothooks script %s execution error %s", filepath, e) except Exception as e: - LOG.error("Boothooks unknown exception %s when running %s", e, filepath) + LOG.exception(("Boothooks unknown " + "error %s when running %s"), e, filepath) -- cgit v1.2.3 From 993e32d9e00718bca829bb9e5c3ae7b0be6ca78b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:20:17 -0700 Subject: Pylint fixups. --- cloudinit/user_data/cloud_config.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cloudinit/user_data/cloud_config.py b/cloudinit/user_data/cloud_config.py index 1c43f3a1..f0e88eeb 100644 --- a/cloudinit/user_data/cloud_config.py +++ b/cloudinit/user_data/cloud_config.py @@ -20,9 +20,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . - -import os - from cloudinit import log as logging from cloudinit import user_data as ud from cloudinit import util @@ -56,4 +53,4 @@ class CloudConfigPartHandler(ud.PartHandler): filename = util.clean_filename(filename) entry = "\n".join(["#%s" % (filename), str(payload)]) - self.config_buf.append(entry) + self.cloud_buf.append(entry) -- cgit v1.2.3 From cfecfaf2a57ef258a7ad12bda7ee1e2808c736d4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:20:35 -0700 Subject: Use common yaml loading function + pylint cleanups. --- cloudinit/user_data/processor.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/cloudinit/user_data/processor.py b/cloudinit/user_data/processor.py index d4de9470..6e796f1c 100644 --- a/cloudinit/user_data/processor.py +++ b/cloudinit/user_data/processor.py @@ -22,14 +22,12 @@ import hashlib import os -import urllib from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.mime.base import MIMEBase -import yaml - +from cloudinit import log as logging from cloudinit import url_helper from cloudinit import user_data as ud from cloudinit import util @@ -46,8 +44,12 @@ OCTET_TYPE = 'application/octet-stream' ATTACHMENT_FIELD = 'Number-Attachments' # This will be used to create a filename from a url (or like) entry -# When we want to make sure a entry isn't included more than once across sessions. +# When we want to make sure a entry isn't included +# more than once across sessions. INCLUDE_ONCE_HASHER = 'md5' +MAX_INCLUDE_FN_LEN = 64 + +LOG = logging.getLogger(__name__) class UserDataProcessor(object): @@ -97,8 +99,10 @@ class UserDataProcessor(object): def _get_include_once_filename(self, entry): msum = hashlib.new(INCLUDE_ONCE_HASHER) msum.update(entry) - entry_fn = msum.hexdigest()[0:64] # Don't get to long now - return os.path.join(self.paths.get_ipath_cur('data'), 'urlcache', entry_fn) + # Don't get to long now + entry_fn = msum.hexdigest()[0:MAX_INCLUDE_FN_LEN] + return os.path.join(self.paths.get_ipath_cur('data'), + 'urlcache', entry_fn) def _do_include(self, content, append_msg): # is just a list of urls, one per line @@ -132,17 +136,11 @@ class UserDataProcessor(object): self._process_msg(new_msg, append_msg) def _explode_archive(self, archive, append_msg): - try: - entries = yaml.load(archive) - except: - entries = [] - if not isinstance(entries, (list, set)): - # TODO raise? - entries = [] - + entries = util.load_yaml(archive, default=[], allowed=[list, set]) for ent in entries: # ent can be one of: - # dict { 'filename' : 'value', 'content' : 'value', 'type' : 'value' } + # dict { 'filename' : 'value', 'content' : + # 'value', 'type' : 'value' } # filename and type not be present # or # scalar(payload) @@ -165,7 +163,8 @@ class UserDataProcessor(object): msg.set_payload(content) if 'filename' in ent: - msg.add_header('Content-Disposition', 'attachment', filename=ent['filename']) + msg.add_header('Content-Disposition', 'attachment', + filename=ent['filename']) for header in ent.keys(): if header in ('content', 'filename', 'type'): -- cgit v1.2.3 From 31f92c74b194430f6db3e69ac6960daa1646a7b8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:20:57 -0700 Subject: Pylint cleanups. --- cloudinit/user_data/shell_script.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/user_data/shell_script.py b/cloudinit/user_data/shell_script.py index d666b9c4..564e4623 100644 --- a/cloudinit/user_data/shell_script.py +++ b/cloudinit/user_data/shell_script.py @@ -20,14 +20,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . - import os from cloudinit import log as logging from cloudinit import user_data as ud from cloudinit import util -from cloudinit.settings import (PER_INSTANCE) +from cloudinit.settings import (PER_ALWAYS) LOG = logging.getLogger(__name__) @@ -45,9 +44,10 @@ class ShellScriptPartHandler(ud.PartHandler): def _handle_part(self, _data, ctype, filename, payload, _frequency): if ctype in ud.CONTENT_SIGNALS: - # maybe delete existing things here + # TODO: maybe delete existing things here return filename = util.clean_filename(filename) payload = util.dos2unix(payload) - util.write_file(os.path.join(self.script_dir, filename), payload, 0700) + path = os.path.join(self.script_dir, filename) + util.write_file(path, payload, 0700) -- cgit v1.2.3 From feeb0620b31d2d6aff59449c53a17b193a2d8d8e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:21:09 -0700 Subject: Pylint cleanups. --- cloudinit/user_data/upstart_job.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cloudinit/user_data/upstart_job.py b/cloudinit/user_data/upstart_job.py index d15e66c4..568a644a 100644 --- a/cloudinit/user_data/upstart_job.py +++ b/cloudinit/user_data/upstart_job.py @@ -46,11 +46,14 @@ class UpstartJobPartHandler(ud.PartHandler): if ctype in ud.CONTENT_SIGNALS: return - filename = utils.clean_filename(filename) - (name, ext) = os.path.splitext(filename) + filename = util.clean_filename(filename) + (_name, ext) = os.path.splitext(filename) + if not ext: + ext = '' ext = ext.lower() if ext != ".conf": filename = filename + ".conf" payload = util.dos2unix(payload) - util.write_file(os.path.join(self.upstart_dir, filename), payload, 0644) + path = os.path.join(self.upstart_dir, filename) + util.write_file(path, payload, 0644) -- cgit v1.2.3 From eeff4524aceaa35577ab74a1066b4c169f9e865d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:21:43 -0700 Subject: Pylint cleanups, add a load yaml function that handles errors nicely + other cleanups. --- cloudinit/util.py | 295 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 232 insertions(+), 63 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 5930ff3f..6cf75916 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -23,6 +23,8 @@ from StringIO import StringIO import contextlib +import copy +import errno import glob import grp import gzip @@ -32,6 +34,8 @@ import pwd import shutil import socket import subprocess +import sys +import tempfile import types import urlparse @@ -40,6 +44,8 @@ import yaml from cloudinit import log as logging from cloudinit import url_helper as uhelp +from cloudinit.settings import (CFG_BUILTIN, CLOUD_CONFIG) + try: import selinux @@ -55,6 +61,9 @@ FN_REPLACEMENTS = { os.sep: '_', } +# Helper utils to see if running in a container +CONTAINER_TESTS = ['running-in-container', 'lxc-is-container'] + class ProcessExecutionError(IOError): @@ -112,12 +121,17 @@ class SeLinuxGuard(object): def __enter__(self): return self.engaged - def __exit__(self, type, value, traceback): + def __exit__(self, excp_type, excp_value, excp_traceback): if self.engaged: - LOG.debug("Disengaging selinux mode for %s: %s", self.path, self.recursive) + LOG.debug("Disengaging selinux mode for %s: %s", + self.path, self.recursive) selinux.restorecon(self.path, recursive=self.recursive) +class MountFailedError(Exception): + pass + + def translate_bool(val): if not val: return False @@ -130,14 +144,12 @@ def translate_bool(val): def read_conf(fname): try: - mp = yaml.load(load_file(fname)) - if not isinstance(mp, (dict)): - return {} - return mp + return load_yaml(load_file(fname), default={}) except IOError as e: if e.errno == errno.ENOENT: return {} - raise + else: + raise def clean_filename(fn): @@ -148,8 +160,9 @@ def clean_filename(fn): def decomp_str(data): try: - uncomp = gzip.GzipFile(None, "rb", 1, StringIO(data)).read() - return uncomp + buf = StringIO(str(data)) + with contextlib.closing(gzip.GzipFile(None, "rb", 1, buf)) as gh: + return gh.read() except: return data @@ -180,16 +193,13 @@ def is_ipv4(instr): return (len(toks) == 4) -def get_base_cfg(cfgfile, cfg_builtin=None): +def merge_base_cfg(cfgfile, cfg_builtin=None): syscfg = read_conf_with_confd(cfgfile) kern_contents = read_cc_from_cmdline() kerncfg = {} if kern_contents: - try: - kerncfg = yaml.load(kern_contents) - except: - pass + kerncfg = load_yaml(kern_contents, default={}) # kernel parameters override system config combined = mergedict(kerncfg, syscfg) @@ -265,8 +275,9 @@ def obj_name(obj): def mergedict(src, cand): """ - Merge values from C{cand} into C{src}. If C{src} has a key C{cand} will - not override. Nested dictionaries are merged recursively. + Merge values from C{cand} into C{src}. + If C{src} has a key C{cand} will not override. + Nested dictionaries are merged recursively. """ if isinstance(src, dict) and isinstance(cand, dict): for k, v in cand.iteritems(): @@ -276,9 +287,11 @@ def mergedict(src, cand): src[k] = mergedict(src[k], v) else: if not isinstance(src, dict): - raise TypeError("Attempting to merge a non dictionary source type: %s" % (type(src))) + raise TypeError(("Attempting to merge a non dictionary " + "source type: %s") % (obj_name(src))) if not isinstance(cand, dict): - raise TypeError("Attempting to merge a non dictionary candiate type: %s" % (type(cand))) + raise TypeError(("Attempting to merge a non dictionary " + "candidate type: %s") % (obj_name(cand))) return src @@ -308,8 +321,9 @@ def del_dir(path): shutil.rmtree(path) -# get keyid from keyserver +# get gpg keyid from keyserver def getkeybyid(keyid, keyserver): + # TODO fix this... shcmd = """ k=${1} ks=${2}; exec 2>/dev/null @@ -323,7 +337,7 @@ def getkeybyid(keyid, keyserver): [ -n "${armour}" ] && echo "${armour}" """ args = ['sh', '-c', shcmd, "export-gpg-keyid", keyid, keyserver] - (stdout, stderr) = subp(args) + (stdout, _stderr) = subp(args) return stdout @@ -340,11 +354,12 @@ def runparts(dirp, skip_no_exist=True): try: subp([exe_path]) except ProcessExecutionError as e: - LOG.exception("Failed running %s [%i]", exe_path, e.exit_code) + LOG.exception("Failed running %s [%s]", exe_path, e.exit_code) failed += 1 if failed and attempted: - raise RuntimeError('runparts: %i failures in %i attempted commands' % (failed, attempted)) + raise RuntimeError('Runparts: %s failures in %s attempted commands' + % (failed, attempted)) # read_optional_seed @@ -363,6 +378,32 @@ def read_optional_seed(fill, base="", ext="", timeout=5): raise +def read_file_or_url(url, timeout, retries, file_retries): + if url.startswith("/"): + url = "file://%s" % url + if url.startswith("file://"): + retries = file_retries + return uhelp.readurl(url, timeout=timeout, retries=retries) + + +def load_yaml(blob, default=None, allowed=(dict,)): + loaded = default + try: + blob = str(blob) + LOG.debug(("Attempting to load yaml from string " + "of length %s with allowed root types %s"), + len(blob), allowed) + converted = yaml.load(blob) + if not isinstance(converted, allowed): + # Yes this will just be caught, but thats ok for now... + raise TypeError("Yaml load allows %s types, but got %s instead" % + (allowed, obj_name(converted))) + loaded = converted + except (yaml.YAMLError, TypeError, ValueError) as exc: + LOG.exception("Failed loading yaml due to: %s", exc) + return loaded + + def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): if base.startswith("/"): base = "file://%s" % base @@ -378,13 +419,16 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): ud_url = "%s%s%s" % (base, "user-data", ext) md_url = "%s%s%s" % (base, "meta-data", ext) - (md_str, msc) = uhelp.readurl(md_url, timeout=timeout, retries=retries) - (ud, usc) = uhelp.readurl(ud_url, timeout=timeout, retries=retries) + (md_str, msc) = read_file_or_url(md_url, timeout, retries, file_retries) md = None if md_str and uhelp.ok_http_code(msc): - md = yaml.load(md_str) - if not uhelp.ok_http_code(usc): - ud = None + md = load_yaml(md_str, default={}) + + (ud_str, usc) = read_file_or_url(ud_url, timeout, retries, file_retries) + ud = None + if ud_str and uhelp.ok_http_code(usc): + ud = ud_str + return (md, ud) @@ -410,13 +454,14 @@ def read_conf_with_confd(cfgfile): confd = False if "conf_d" in cfg: - if cfg['conf_d'] is not None: - confd = cfg['conf_d'] - if not isinstance(confd, (str)): - raise RuntimeError(("Config file %s contains 'conf_d' " - "with non-string") % (cfgfile)) + confd = cfg['conf_d'] + if confd: + if not isinstance(confd, (str, basestring)): + raise TypeError(("Config file %s contains 'conf_d' " + "with non-string type %s") % + (cfgfile, obj_name(confd))) else: - confd = confd.strip() + confd = str(confd).strip() elif os.path.isdir("%s.d" % cfgfile): confd = "%s.d" % cfgfile @@ -490,26 +535,41 @@ def get_hostname_fqdn(cfg, cloud): def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): - # this parses /etc/hosts to get a fqdn. It should return the same - # result as 'hostname -f ' if /etc/hosts.conf - # did not have did not have 'bind' in the order attribute + """ + For each host a single line should be present with + the following information: + + IP_address canonical_hostname [aliases...] + + Fields of the entry are separated by any number of blanks and/or tab + characters. Text from a "#" character until the end of the line is a + comment, and is ignored. Host names may contain only alphanumeric + characters, minus signs ("-"), and periods ("."). They must begin with + an alphabetic character and end with an alphanumeric character. + Optional aliases provide for name changes, alternate spellings, shorter + hostnames, or generic hostnames (for example, localhost). + """ fqdn = None try: for line in load_file(filename).splitlines(): hashpos = line.find("#") if hashpos >= 0: line = line[0:hashpos] - toks = line.split() - - # if there there is less than 3 entries (ip, canonical, alias) + line = line.strip() + if not line: + continue + + # If there there is less than 3 entries + # (IP_address, canonical_hostname, alias) # then ignore this line + toks = line.split() if len(toks) < 3: continue - + if hostname in toks[2:]: fqdn = toks[1] break - except IOError as e: + except IOError: pass return fqdn @@ -584,7 +644,7 @@ def close_stdin(): os.dup2(fp.fileno(), sys.stdin.fileno()) -def find_devs_with(criteria): +def find_devs_with(criteria=None): """ find devices matching given criteria (via blkid) criteria can be *one* of: @@ -593,10 +653,26 @@ def find_devs_with(criteria): UUID= """ try: - (out, _err) = subp(['blkid', '-t%s' % criteria, '-odevice']) + blk_id_cmd = ['blkid'] + if criteria: + # Search for block devices with tokens named NAME that + # have the value 'value' and display any devices which are found. + # Common values for NAME include TYPE, LABEL, and UUID. + # If there are no devices specified on the command line, + # all block devices will be searched; otherwise, + # only search the devices specified by the user. + blk_id_cmd.append("-t%s" % (criteria)) + # Only print the device name + blk_id_cmd.append('-odevice') + (out, _err) = subp(blk_id_cmd) + entries = [] + for line in out.splitlines(): + line = line.strip() + if line: + entries.append(line) + return entries except ProcessExecutionError: return [] - return (out.splitlines()) def load_file(fname, read_cb=None): @@ -604,7 +680,10 @@ def load_file(fname, read_cb=None): with open(fname, 'rb') as fh: ofh = StringIO() pipe_in_out(fh, ofh, chunk_cb=read_cb) - return ofh.getvalue() + ofh.flush() + contents = ofh.getvalue() + LOG.debug("Read %s bytes from %s", len(contents), fname) + return contents def get_cmdline(): @@ -620,7 +699,8 @@ def get_cmdline(): def pipe_in_out(in_fh, out_fh, chunk_size=1024, chunk_cb=None): bytes_piped = 0 - LOG.debug("Transferring the contents of %s to %s in chunks of size %s.", in_fh, out_fh, chunk_size) + LOG.debug(("Transferring the contents of %s " + "to %s in chunks of size %sb"), in_fh, out_fh, chunk_size) while True: data = in_fh.read(chunk_size) if data == '': @@ -658,15 +738,87 @@ def ensure_dirs(dirlist, mode=0755): def ensure_dir(path, mode=0755): if not os.path.isdir(path): - fixmodes = [] - LOG.debug("Ensuring directory exists at path %s (perms=%s)", dir_name, mode) - try: - os.makedirs(path) - except OSError as e: - if e.errno != errno.EEXIST: - raise e - if mode is not None: - os.chmod(path, mode) + # Make the dir and adjust the mode + LOG.debug("Ensuring directory exists at path %s", path) + os.makedirs(path) + chmod(path, mode) + else: + # Just adjust the mode + chmod(path, mode) + + +def get_base_cfg(cfg_path=None): + if not cfg_path: + cfg_path = CLOUD_CONFIG + return merge_base_cfg(cfg_path, get_builtin_cfg()) + + +@contextlib.contextmanager +def unmounter(umount): + try: + yield umount + finally: + if umount: + umount_cmd = ["umount", '-l', umount] + subp(umount_cmd) + + +def mounts(): + mounted = {} + try: + # Go through mounts to see if it was already mounted + mount_locs = load_file("/proc/mounts").splitlines() + for mpline in mount_locs: + # Format at: http://linux.die.net/man/5/fstab + try: + (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() + except: + continue + # If the name of the mount point contains spaces these + # can be escaped as '\040', so undo that.. + mp = mp.replace("\\040", " ") + mounted[dev] = (dev, fstype, mp, False) + except (IOError, OSError): + pass + return mounted + + +def mount_cb(device, callback, data=None, rw=False): + """ + Mount the device, call method 'callback' passing the directory + in which it was mounted, then unmount. Return whatever 'callback' + returned. If data != None, also pass data to callback. + """ + mounted = mounts() + with tempdir() as tmpd: + umount = False + if device in mounted: + mountpoint = "%s/" % mounted[device][2] + else: + try: + mountcmd = ['mount', "-o"] + if rw: + mountcmd.append('rw') + else: + mountcmd.append('ro') + mountcmd.append(device) + mountcmd.append(tmpd) + subp(mountcmd) + umount = tmpd + except IOError as exc: + raise MountFailedError("%s" % (exc)) + mountpoint = "%s/" % tmpd + with unmounter(umount): + if data is None: + ret = callback(mountpoint) + else: + ret = callback(mountpoint, data) + return ret + + +def get_builtin_cfg(): + # Deep copy so that others can't modify + return copy.deepcopy(CFG_BUILTIN) def sym_link(source, link): @@ -687,6 +839,18 @@ def ensure_file(path): write_file(path, content='', omode="ab") +def chmod(path, mode): + real_mode = None + try: + real_mode = int(mode) + except (ValueError, TypeError): + pass + if path and real_mode: + LOG.debug("Adjusting the permissions of %s (perms=%o)", + path, real_mode) + os.chmod(path, real_mode) + + def write_file(filename, content, mode=0644, omode="wb"): """ Writes a file with the given content and sets the file mode as specified. @@ -698,13 +862,12 @@ def write_file(filename, content, mode=0644, omode="wb"): @param omode: The open mode used when opening the file (r, rb, a, etc.) """ ensure_dir(os.path.dirname(filename)) - LOG.debug("Writing to %s - %s (perms=%s) %s bytes", filename, omode, mode, len(content)) + LOG.debug("Writing to %s - %s, %s bytes", filename, omode, len(content)) with open(filename, omode) as fh: with SeLinuxGuard(filename): fh.write(content) fh.flush() - if mode is not None: - os.chmod(filename, mode) + chmod(filename, mode) def delete_dir_contents(dirname): @@ -725,7 +888,8 @@ def subp(args, input_data=None, allowed_rc=None, env=None): if allowed_rc is None: allowed_rc = [0] try: - LOG.debug("Running command %s with allowed return codes %s", args, allowed_rc) + LOG.debug("Running command %s with allowed return codes %s", + args, allowed_rc) sp = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, env=env) @@ -768,14 +932,16 @@ def shellify(cmdlist, add_header=True): def is_container(): - # is this code running in a container of some sort + """ + Checks to see if this code running in a container of some sort + """ - for helper in ('running-in-container', 'lxc-is-container'): + for helper in CONTAINER_TESTS: try: # try to run a helper program. if it returns true/zero # then we're inside a container. otherwise, no cmd = [helper] - (stdout, stderr) = subp(cmd, allowed_rc=[0]) + subp(cmd, allowed_rc=[0]) return True except (IOError, OSError): pass @@ -812,7 +978,10 @@ def is_container(): def get_proc_env(pid): - # return the environment in a dict that a given process id was started with + """ + Return the environment in a dict that a given process id was started with. + """ + env = {} fn = os.path.join("/proc/", str(pid), "environ") try: -- cgit v1.2.3 From 2a17d8fa939a3b85b5592b81d9b0bd146197578a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 18:01:45 -0700 Subject: Add initial network writing here. --- cloudinit/distros/ubuntu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index d19ef63e..ccf2cec4 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -39,7 +39,7 @@ class Distro(distros.Distro): self.apt_get('install', pkglist) def apply_network(self, settings): - pass + util.write_file("/etc/network/interfaces", settings) # apt_get top level command (install, update...), and args to pass it def apt_get(self, tlc, args=None): -- cgit v1.2.3 From 00399cd672e14b7e6953954eaf126f0ccfe35ecd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 18:01:59 -0700 Subject: Fix log statement. --- cloudinit/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 0bd13c78..16548001 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -260,8 +260,8 @@ class Paths(object): def get_ipath(self, name=None): ipath = self._get_ipath(name) if not ipath: - LOG.warn(("No per instance semaphores available, " - "is there an datasource/iid set?")) + LOG.warn(("No per instance data available, " + "is there an datasource/iid set?")) return None else: return ipath -- cgit v1.2.3 From dab0b7c7ebcc92c772bfadce334ba118955f5a59 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 18:02:32 -0700 Subject: Fix logging + fetch user data before user data content handlers are activated. --- cloudinit/stages.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 55ad143d..2cb9d0ec 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -124,7 +124,6 @@ class Init(object): util.chownbyname(log_file, u, g) def _read_cfg(self): - # Deep copy so that b_config = util.get_builtin_cfg() try: conf = util.get_base_cfg() @@ -279,8 +278,10 @@ class Init(object): data = cloud.Cloud(self.datasource, self.paths, copy.deepcopy(self.cfg)) - # Init the handlers first # Ensure userdata fetched before activation + ud_obj = data.get_userdata() + + # Init the handlers first called = [] for (_mtype, mod) in c_handlers.iteritems(): if mod in called: @@ -294,9 +295,12 @@ class Init(object): 'handlerdir': idir, 'data': data, 'frequency': frequency, + # This will be used when new handlers are found + # to help write there contents to files with numbered + # names... 'handlercount': 0, } - ud.walk(data.get_userdata(), ud.walker_callback, data=part_data) + ud.walk(ud_obj, ud.walker_callback, data=part_data) # Give callbacks opportunity to finalize called = [] -- cgit v1.2.3 From b46190b8bb0aaa5a45245df5490f543e6c6497cb Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 18:02:56 -0700 Subject: Fix this up to work with new utils/logging/datasource... --- cloudinit/sources/DataSourceConfigDrive.py | 218 ++++++++++++++--------------- 1 file changed, 109 insertions(+), 109 deletions(-) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 2db4a76a..ca4bb7cf 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -14,54 +14,61 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import os.path -import os import json -import subprocess - -DEFAULT_IID = "iid-dsconfigdrive" +import os +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util -class DataSourceConfigDrive(DataSource.DataSource): - seed = None - seeddir = base_seeddir + '/config_drive' - cfg = {} - userdata_raw = None - metadata = None - dsmode = "local" +LOG = logging.getLogger(__name__) +DEFAULT_IID = "iid-dsconfigdrive" +DEFAULT_MODE = 'pass' +CFG_DRIVE_FILES = [ + "etc/network/interfaces", + "root/.ssh/authorized_keys", + "meta.js", +] +DEFAULT_METADATA = { + "instance-id": DEFAULT_IID, + "dsmode": DEFAULT_MODE, +} +IF_UP_CMD = ['ifup', '--all'] +CFG_DRIVE_DEV_ENV = 'CLOUD_INIT_CONFIG_DRIVE_DEVICE' + + +class DataSourceConfigDrive(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.seed = None + self.cfg = {} + self.dsmode = 'local' + self.seed_dir = os.path.join(self.paths.seed_dir, 'config_drive') def __str__(self): - mstr = "DataSourceConfigDrive[%s]" % self.dsmode - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) + mstr = "%s[%s]" % (util.obj_name(self), self.dsmode) + mstr = mstr + " [seed=%s]" % (self.seed) + return mstr def get_data(self): found = None md = {} ud = "" - defaults = {"instance-id": DEFAULT_IID, "dsmode": "pass"} - - if os.path.isdir(self.seeddir): + if os.path.isdir(self.seed_dir): try: - (md, ud) = read_config_drive_dir(self.seeddir) - found = self.seeddir - except nonConfigDriveDir: - pass - + (md, ud) = read_config_drive_dir(self.seed_dir) + found = self.seed_dir + except NonConfigDriveDir: + LOG.exception("Failed reading config drive from %s", + self.seed_dir) if not found: - dev = cfg_drive_device() + dev = find_cfg_drive_device() if dev: try: - (md, ud) = util.mount_callback_umount(dev, - read_config_drive_dir) + (md, ud) = util.mount_cb(dev, read_config_drive_dir) found = dev - except (nonConfigDriveDir, util.mountFailedError): + except (NonConfigDriveDir, util.MountFailedError): pass if not found: @@ -70,25 +77,24 @@ class DataSourceConfigDrive(DataSource.DataSource): if 'dsconfig' in md: self.cfg = md['dscfg'] - md = util.mergedict(md, defaults) + md = util.mergedict(md, DEFAULT_METADATA) - # update interfaces and ifup only on the local datasource + # Update interfaces and ifup only on the local datasource # this way the DataSourceConfigDriveNet doesn't do it also. if 'network-interfaces' in md and self.dsmode == "local": if md['dsmode'] == "pass": - log.info("updating network interfaces from configdrive") + LOG.info("Updating network interfaces from configdrive") else: - log.debug("updating network interfaces from configdrive") + LOG.debug("Updating network interfaces from configdrive") - util.write_file("/etc/network/interfaces", - md['network-interfaces']) + self.distro.apply_network(md['network-interfaces']) try: - (out, err) = util.subp(['ifup', '--all']) - if len(out) or len(err): - log.warn("ifup --all had stderr: %s" % err) - - except subprocess.CalledProcessError as exc: - log.warn("ifup --all failed: %s" % (exc.output[1])) + (_out, err) = util.subp(IF_UP_CMD) + if len(err): + LOG.warn("Running %s resulted in stderr output: %s", + IF_UP_CMD, err) + except util.ProcessExecutionError: + LOG.exception("Running %s failed", IF_UP_CMD) self.seed = found self.metadata = md @@ -97,99 +103,107 @@ class DataSourceConfigDrive(DataSource.DataSource): if md['dsmode'] == self.dsmode: return True - log.debug("%s: not claiming datasource, dsmode=%s" % - (self, md['dsmode'])) + LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode']) return False def get_public_ssh_keys(self): if not 'public-keys' in self.metadata: - return([]) - return(self.metadata['public-keys']) + return [] + return list(self.metadata['public-keys']) - # the data sources' config_obj is a cloud-config formated + # The data sources' config_obj is a cloud-config formated # object that came to it from ways other than cloud-config # because cloud-config content would be handled elsewhere def get_config_obj(self): - return(self.cfg) + return self.cfg class DataSourceConfigDriveNet(DataSourceConfigDrive): - dsmode = "net" + def __init__(self, sys_cfg, paths): + DataSourceConfigDrive.__init__(self, sys_cfg, paths) + self.dsmode = 'net' -class nonConfigDriveDir(Exception): +class NonConfigDriveDir(Exception): pass -def cfg_drive_device(): - """ get the config drive device. return a string like '/dev/vdb' +def find_cfg_drive_device(): + """ Get the config drive device. Return a string like '/dev/vdb' or None (if there is no non-root device attached). This does not check the contents, only reports that if there *were* a config_drive attached, it would be this device. - per config_drive documentation, this is - "associated as the last available disk on the instance" + Note: per config_drive documentation, this is + "associated as the last available disk on the instance" """ - if 'CLOUD_INIT_CONFIG_DRIVE_DEVICE' in os.environ: - return(os.environ['CLOUD_INIT_CONFIG_DRIVE_DEVICE']) - - # we are looking for a raw block device (sda, not sda1) with a vfat - # filesystem on it. + # This seems to be for debugging?? + if CFG_DRIVE_DEV_ENV in os.environ: + return os.environ[CFG_DRIVE_DEV_ENV] + # We are looking for a raw block device (sda, not sda1) with a vfat + # filesystem on it.... letters = "abcdefghijklmnopqrstuvwxyz" devs = util.find_devs_with("TYPE=vfat") - # filter out anything not ending in a letter (ignore partitions) + # Filter out anything not ending in a letter (ignore partitions) devs = [f for f in devs if f[-1] in letters] - # sort them in reverse so "last" device is first + # Sort them in reverse so "last" device is first devs.sort(reverse=True) - if len(devs): - return(devs[0]) + if devs: + return devs[0] - return(None) + return None def read_config_drive_dir(source_dir): """ read_config_drive_dir(source_dir): read source_dir, and return a tuple with metadata dict and user-data - string populated. If not a valid dir, raise a nonConfigDriveDir + string populated. If not a valid dir, raise a NonConfigDriveDir """ - md = {} - ud = "" - flist = ("etc/network/interfaces", "root/.ssh/authorized_keys", "meta.js") - found = [f for f in flist if os.path.isfile("%s/%s" % (source_dir, f))] - keydata = "" + # TODO: fix this for other operating systems... + # Ie: this is where https://fedorahosted.org/netcf/ or similar should + # be hooked in... (or could be) + found = {} + for af in CFG_DRIVE_FILES: + fn = os.path.join(source_dir, af) + if os.path.isfile(fn): + found[af] = fn if len(found) == 0: - raise nonConfigDriveDir("%s: %s" % (source_dir, "no files found")) + raise NonConfigDriveDir("%s: %s" % (source_dir, "no files found")) + md = {} + ud = "" + keydata = "" if "etc/network/interfaces" in found: - with open("%s/%s" % (source_dir, "/etc/network/interfaces")) as fp: - md['network-interfaces'] = fp.read() + fn = found["etc/network/interfaces"] + md['network-interfaces'] = util.load_file(fn) if "root/.ssh/authorized_keys" in found: - with open("%s/%s" % (source_dir, "root/.ssh/authorized_keys")) as fp: - keydata = fp.read() + fn = found["root/.ssh/authorized_keys"] + keydata = util.load_file(fn) meta_js = {} - if "meta.js" in found: - content = '' - with open("%s/%s" % (source_dir, "meta.js")) as fp: - content = fp.read() - md['meta_js'] = content + fn = found['meta.js'] + content = util.load_file(fn) try: + # Just check if its really json... meta_js = json.loads(content) - except ValueError: - raise nonConfigDriveDir("%s: %s" % - (source_dir, "invalid json in meta.js")) + if not isinstance(meta_js, (dict)): + raise TypeError("Dict expected for meta.js root node") + except (ValueError, TypeError) as e: + raise NonConfigDriveDir("%s: %s, %s" % + (source_dir, "invalid json in meta.js", e)) + md['meta_js'] = content + # Key data override?? keydata = meta_js.get('public-keys', keydata) - if keydata: lines = keydata.splitlines() md['public-keys'] = [l for l in lines @@ -202,30 +216,16 @@ def read_config_drive_dir(source_dir): if 'user-data' in meta_js: ud = meta_js['user-data'] - return(md, ud) + return (md, ud) + -datasources = ( - (DataSourceConfigDrive, (DataSource.DEP_FILESYSTEM, )), - (DataSourceConfigDriveNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -) +# Used to match classes to dependencies +datasources = [ + (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), + (DataSourceConfigDriveNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] -# return a list of data sources that match this set of dependencies +# Used to match classes to dependencies def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - -if __name__ == "__main__": - def main(): - import sys - import pprint - print cfg_drive_device() - (md, ud) = read_config_drive_dir(sys.argv[1]) - print "=== md ===" - pprint.pprint(md) - print "=== ud ===" - print(ud) - - main() - -# vi: ts=4 expandtab + return sources.list_from_depends(depends, datasources) -- cgit v1.2.3 From 3a74809e375f32e491d20fc303ad9220aae555d8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 20:11:03 -0700 Subject: Add centering method + more descriptive mount failure exception message. --- cloudinit/util.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 6cf75916..f02fcfe9 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -316,6 +316,11 @@ def tempdir(**kwargs): del_dir(tdir) +def center(text, fill, max_len): + return '{0:{fill}{align}{size}}'.format(text, fill=fill, + align="^", size=max_len) + + def del_dir(path): LOG.debug("Recursively deleting %s", path) shutil.rmtree(path) @@ -805,8 +810,10 @@ def mount_cb(device, callback, data=None, rw=False): mountcmd.append(tmpd) subp(mountcmd) umount = tmpd - except IOError as exc: - raise MountFailedError("%s" % (exc)) + except (IOError, OSError) as exc: + raise MountFailedError(("Failed mounting %s " + "to %s due to: %s") % + (device, tmpd, exc)) mountpoint = "%s/" % tmpd with unmounter(umount): if data is None: -- cgit v1.2.3 From 08fa7fc2fb32764eac25fba9d83f61ce1243aa7b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 20:11:24 -0700 Subject: Use pretty table to get a parseable format out. --- cloudinit/netinfo.py | 93 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 67 insertions(+), 26 deletions(-) diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index 541d2911..a211788c 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -19,12 +19,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util +import os +import cloudinit.util as util -class NetInfo(object): - def __init__(self): - pass +from prettytable import PrettyTable def netdev_info(empty=""): @@ -71,47 +70,89 @@ def netdev_info(empty=""): if dev[field] == "": dev[field] = empty - return(devs) + return devs def route_info(): (route_out, _err) = util.subp(["route", "-n"]) routes = [] - for line in str(route_out).splitlines()[1:]: + entries = route_out.splitlines()[1:] + for line in entries: if not line: continue toks = line.split() if toks[0] == "Kernel" or toks[0] == "Destination": continue - routes.append(toks) - return(routes) + entry = { + 'destination': toks[0], + 'gateway': toks[1], + 'genmask': toks[2], + 'flags': toks[3], + 'metric': toks[4], + 'ref': toks[5], + 'use': toks[6], + 'iface': toks[7], + } + routes.append(entry) + return routes def getgateway(): - for r in route_info(): - if r[3].find("G") >= 0: - return("%s[%s]" % (r[1], r[7])) - return(None) + routes = [] + try: + routes = route_info() + except: + pass + for r in routes: + if r['flags'].find("G") >= 0: + return "%s[%s]" % (r['gateway'], r['iface']) + return None -def debug_info(pre="ci-info: "): +def netdev_pformat(): lines = [] try: netdev = netdev_info(empty=".") except Exception: - lines.append("netdev_info failed!") - netdev = {} - for (dev, d) in netdev.iteritems(): - lines.append("%s%-6s: %i %-15s %-15s %s" % - (pre, dev, d["up"], d["addr"], d["mask"], d["hwaddr"])) + lines.append(util.center("Net device info failed", '!', 80)) + netdev = None + if netdev is not None: + fields = ['Device', 'Up', 'Address', 'Mask', 'Hw-Address'] + tbl = PrettyTable(fields) + for (dev, d) in netdev.iteritems(): + tbl.add_row([dev, d["up"], d["addr"], d["mask"], d["hwaddr"]]) + netdev_s = tbl.get_string() + max_len = len(max(netdev_s.splitlines(), key=len)) + header = util.center("Net device info", "+", max_len) + lines.extend([header, netdev_s]) + return os.linesep.join(lines) + + +def route_pformat(): + lines = [] try: routes = route_info() except Exception: - lines.append("route_info failed") - routes = [] - n = 0 - for r in routes: - lines.append("%sroute-%d: %-15s %-15s %-15s %-6s %s" % - (pre, n, r[0], r[1], r[2], r[7], r[3])) - n = n + 1 - return('\n'.join(lines)) + lines.append(util.center('Route info failed', '!', 80)) + routes = None + if routes is not None: + fields = ['Route', 'Destination', 'Gateway', + 'Genmask', 'Interface', 'Flags'] + tbl = PrettyTable(fields) + for n, r in enumerate(routes): + route_id = str(n) + tbl.add_row([str(n), r['destination'], + r['gateway'], r['genmask'], + r['iface'], r['flags']]) + route_s = tbl.get_string() + max_len = len(max(route_s.splitlines(), key=len)) + header = util.center("Route info", "+", max_len) + lines.extend([header, route_s]) + return os.linesep.join(lines) + + +def debug_info(pre=""): + lines = [] + lines.append(netdev_pformat()) + lines.append(route_pformat()) + return os.linesep.join(lines) -- cgit v1.2.3 From 1f120980a3217289f4bad02b813053664732e4bf Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 20:11:44 -0700 Subject: Rename sem class to file sem class, add clear_all methods, add empty methods to dummy sem class. --- cloudinit/helpers.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 16548001..e5f33a26 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -49,8 +49,14 @@ class DummySemaphores(object): def has_run(self, _name, _freq): return False + def clear(self, _name, _freq): + return True + + def clear_all(self): + pass + -class Semaphores(object): +class FileSemaphores(object): def __init__(self, sem_path): self.sem_path = sem_path @@ -71,6 +77,12 @@ class Semaphores(object): return False return True + def clear_all(self): + try: + util.del_dir(self.sem_path) + except (IOError, OSError): + pass + def _acquire(self, name, freq): if self.has_run(name, freq): return None @@ -117,7 +129,7 @@ class Runners(object): if not sem_path: return None if sem_path not in self.sems: - self.sems[sem_path] = Semaphores(sem_path) + self.sems[sem_path] = FileSemaphores(sem_path) return self.sems[sem_path] def run(self, name, functor, args, freq=None, clear_on_fail=False): -- cgit v1.2.3 From f061dd55bf755efc6caff4feeb59c62d0c8ae94b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 20:12:36 -0700 Subject: Fix copy right. --- cloudinit/netinfo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index a211788c..874c2674 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -1,11 +1,12 @@ -#!/usr/bin/python # vi: ts=4 expandtab # # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser # Author: Juerg Haefliger +# Author: Joshua Harlow # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as -- cgit v1.2.3 From ae765a7182d60e0984b5ebc6684aac895b8f3c8e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 22:15:11 -0700 Subject: Update with parsing of a requirments file, changelog for this new refactoring stuff and setup.py for both of those. --- ChangeLog | 2 ++ Requires | 2 ++ setup.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 56 insertions(+), 14 deletions(-) create mode 100644 Requires diff --git a/ChangeLog b/ChangeLog index 0964877e..f701927b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ +0.7.0: + - Huge refactor [Joshua Harlow] 0.6.4: - support relative path in AuthorizedKeysFile (LP: #970071). 0.6.3: diff --git a/Requires b/Requires new file mode 100644 index 00000000..6a7064af --- /dev/null +++ b/Requires @@ -0,0 +1,2 @@ +Tempita>=0.4 +PrettyTable>=0.4 diff --git a/setup.py b/setup.py index f32662b8..96f889d8 100755 --- a/setup.py +++ b/setup.py @@ -1,10 +1,12 @@ -#!/usr/bin/python # vi: ts=4 expandtab # # Distutils magic for ec2-init +# # Copyright (C) 2009 Canonical Ltd. +# Copyright (C) 2012 Yahoo! Inc. # # Author: Soren Hansen +# Author: Joshua Harlow # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -17,24 +19,59 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# -from distutils.core import setup + from glob import glob -import os.path -import subprocess + +import os +import re + +from distutils.core import setup +from setuptools import find_packages + def is_f(p): - return(os.path.isfile(p)) + return os.path.isfile(p) + + +def versions(fn="ChangeLog"): + with open(fn, 'r') as fh: + lines = fh.read().splitlines() + versions = [] + for line in lines: + line = line.strip() + if line.startswith("-") or not line: + continue + if not re.match(r"[\d]", line): + continue + line = line.strip(":") + if (re.match(r"^[\d+]\.[\d+]\.[\d+]$", line) or + re.match(r"^[\d+]\.[\d+]$", line)): + versions.append(line) + return versions + + +def requires(fn='Requires'): + requires = [] + with open(fn, 'r') as fh: + lines = fh.read().splitlines() + for line in lines: + line = line.strip() + if not line or line[0] == '#': + continue + else: + requires.append(line) + return requires + setup(name='cloud-init', - version='0.6.3', + version=versions()[0], description='EC2 initialisation magic', author='Scott Moser', author_email='scott.moser@canonical.com', url='http://launchpad.net/cloud-init/', - packages=['cloudinit', 'cloudinit.CloudConfig' ], - scripts=['cloud-init.py', - 'cloud-init-cfg.py', + packages=find_packages(), + scripts=['bin/cloud-init.py', + 'bin/cloud-init-cfg.py', 'tools/cloud-init-per', ], data_files=[('/etc/cloud', glob('config/*.cfg')), @@ -42,11 +79,12 @@ setup(name='cloud-init', ('/etc/cloud/templates', glob('templates/*')), ('/etc/init', glob('upstart/*.conf')), ('/usr/share/cloud-init', []), - ('/usr/lib/cloud-init', + ('/usr/lib/cloud-init', ['tools/uncloud-init', 'tools/write-ssh-key-fingerprints']), - ('/usr/share/doc/cloud-init', filter(is_f,glob('doc/*'))), - ('/usr/share/doc/cloud-init/examples', filter(is_f,glob('doc/examples/*'))), - ('/usr/share/doc/cloud-init/examples/seed', filter(is_f,glob('doc/examples/seed/*'))), + ('/usr/share/doc/cloud-init', filter(is_f, glob('doc/*'))), + ('/usr/share/doc/cloud-init/examples', filter(is_f, glob('doc/examples/*'))), + ('/usr/share/doc/cloud-init/examples/seed', filter(is_f, glob('doc/examples/seed/*'))), ('/etc/profile.d', ['tools/Z99-cloud-locale-test.sh']), ], + install_requires=requires(), ) -- cgit v1.2.3 From 63d4586534b14be7a8524eea33fa406d0bc26d6c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 13 Jun 2012 12:48:38 -0700 Subject: Add in a make url function (might be useful). Adjust usage of logexc to use that in the utils file for exceptions that occur. Add in more options to sub function to capture outputs, shell mode and such. --- cloudinit/util.py | 222 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 195 insertions(+), 27 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index f02fcfe9..7d5932c1 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -36,6 +36,7 @@ import socket import subprocess import sys import tempfile +import traceback import types import urlparse @@ -259,11 +260,79 @@ def get_cfg_by_path(yobj, keyp, default=None): cur = yobj for tok in keyp: if tok not in cur: - return(default) + return default cur = cur[tok] return cur +# redirect_output(outfmt, errfmt, orig_out, orig_err) +# replace orig_out and orig_err with filehandles specified in outfmt or errfmt +# fmt can be: +# > FILEPATH +# >> FILEPATH +# | program [ arg1 [ arg2 [ ... ] ] ] +# +# with a '|', arguments are passed to shell, so one level of +# shell escape is required. +def redirect_output(outfmt, errfmt, o_out=sys.stdout, o_err=sys.stderr): + if outfmt: + (mode, arg) = outfmt.split(" ", 1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": + owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) + new_fp = proc.stdin + else: + raise TypeError("Invalid type for outfmt: %s" % outfmt) + + if o_out: + os.dup2(new_fp.fileno(), o_out.fileno()) + if errfmt == outfmt: + os.dup2(new_fp.fileno(), o_err.fileno()) + return + + if errfmt: + (mode, arg) = errfmt.split(" ", 1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": + owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) + new_fp = proc.stdin + else: + raise TypeError("Invalid type for errfmt: %s" % errfmt) + + if o_err: + os.dup2(new_fp.fileno(), o_err.fileno()) + + +def make_url(scheme, host, port=None, + path='', params='', query='', fragment=''): + + pieces = [] + pieces.append(scheme or '') + + netloc = '' + if host: + netloc = str(host) + + if port is not None: + netloc += ":" + "%s" % (port) + + pieces.append(netloc or '') + pieces.append(path or '') + pieces.append(params or '') + pieces.append(query or '') + pieces.append(fragment or '') + + return urlparse.urlunparse(pieces) + + def obj_name(obj): if isinstance(obj, (types.TypeType, types.ModuleType, @@ -359,7 +428,7 @@ def runparts(dirp, skip_no_exist=True): try: subp([exe_path]) except ProcessExecutionError as e: - LOG.exception("Failed running %s [%s]", exe_path, e.exit_code) + logexc(LOG, "Failed running %s [%s]", exe_path, e.exit_code) failed += 1 if failed and attempted: @@ -405,7 +474,7 @@ def load_yaml(blob, default=None, allowed=(dict,)): (allowed, obj_name(converted))) loaded = converted except (yaml.YAMLError, TypeError, ValueError) as exc: - LOG.exception("Failed loading yaml due to: %s", exc) + logexc(LOG, "Failed loading yaml blob") return loaded @@ -682,9 +751,9 @@ def find_devs_with(criteria=None): def load_file(fname, read_cb=None): LOG.debug("Reading from %s", fname) - with open(fname, 'rb') as fh: + with open(fname, 'rb') as ifh: ofh = StringIO() - pipe_in_out(fh, ofh, chunk_cb=read_cb) + pipe_in_out(ifh, ofh, chunk_cb=read_cb) ofh.flush() contents = ofh.getvalue() LOG.debug("Read %s bytes from %s", len(contents), fname) @@ -736,12 +805,90 @@ def chownbyname(fname, user=None, group=None): chownbyid(fname, uid, gid) +# Always returns well formated values +# cfg is expected to have an entry 'output' in it, which is a dictionary +# that includes entries for 'init', 'config', 'final' or 'all' +# init: /var/log/cloud.out +# config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ] +# final: +# output: "| logger -p" +# error: "> /dev/null" +# this returns the specific 'mode' entry, cleanly formatted, with value +def get_output_cfg(cfg, mode="init"): + ret = [None, None] + if not cfg or not 'output' in cfg: + return ret + + outcfg = cfg['output'] + if mode in outcfg: + modecfg = outcfg[mode] + else: + if 'all' not in outcfg: + return ret + # if there is a 'all' item in the output list + # then it applies to all users of this (init, config, final) + modecfg = outcfg['all'] + + # if value is a string, it specifies stdout and stderr + if isinstance(modecfg, str): + ret = [modecfg, modecfg] + + # if its a list, then we expect (stdout, stderr) + if isinstance(modecfg, list): + if len(modecfg) > 0: + ret[0] = modecfg[0] + if len(modecfg) > 1: + ret[1] = modecfg[1] + + # if it is a dictionary, expect 'out' and 'error' + # items, which indicate out and error + if isinstance(modecfg, dict): + if 'output' in modecfg: + ret[0] = modecfg['output'] + if 'error' in modecfg: + ret[1] = modecfg['error'] + + # if err's entry == "&1", then make it same as stdout + # as in shell syntax of "echo foo >/dev/null 2>&1" + if ret[1] == "&1": + ret[1] = ret[0] + + swlist = [">>", ">", "|"] + for i in range(len(ret)): + if not ret[i]: + continue + val = ret[i].lstrip() + found = False + for s in swlist: + if val.startswith(s): + val = "%s %s" % (s, val[len(s):].strip()) + found = True + break + if not found: + # default behavior is append + val = "%s %s" % (">>", val.strip()) + ret[i] = val + + return ret + + +def logexc(log, msg='', *args): + # Setting this here allows this to change + # levels easily (not always error level) + # or even desirable to have that much junk + # coming out to a non-debug stream + if msg: + log.warn(msg, *args) + # Debug gets the full trace + log.debug(msg, exc_info=1, *args) + + def ensure_dirs(dirlist, mode=0755): for d in dirlist: ensure_dir(d, mode) -def ensure_dir(path, mode=0755): +def ensure_dir(path, mode=None): if not os.path.isdir(path): # Make the dir and adjust the mode LOG.debug("Ensuring directory exists at path %s", path) @@ -771,24 +918,29 @@ def unmounter(umount): def mounts(): mounted = {} try: - # Go through mounts to see if it was already mounted + # Go through mounts to see what is already mounted mount_locs = load_file("/proc/mounts").splitlines() for mpline in mount_locs: # Format at: http://linux.die.net/man/5/fstab try: - (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() + (dev, mp, fstype, opts, _freq, _passno) = mpline.split() except: continue # If the name of the mount point contains spaces these # can be escaped as '\040', so undo that.. mp = mp.replace("\\040", " ") - mounted[dev] = (dev, fstype, mp, False) + mounted[dev] = { + 'fstype': fstype, + 'mountpoint': mp, + 'opts': opts, + } + LOG.debug("Fetched %s mounts from %s", mounted, "/proc/mounts") except (IOError, OSError): - pass + logexc(LOG, "Failed fetching mount points from /proc/mounts") return mounted -def mount_cb(device, callback, data=None, rw=False): +def mount_cb(device, callback, data=None, rw=False, mtype=None): """ Mount the device, call method 'callback' passing the directory in which it was mounted, then unmount. Return whatever 'callback' @@ -798,7 +950,7 @@ def mount_cb(device, callback, data=None, rw=False): with tempdir() as tmpd: umount = False if device in mounted: - mountpoint = "%s/" % mounted[device][2] + mountpoint = "%s/" % mounted[device]['mountpoint'] else: try: mountcmd = ['mount', "-o"] @@ -806,6 +958,8 @@ def mount_cb(device, callback, data=None, rw=False): mountcmd.append('rw') else: mountcmd.append('ro') + if mtype: + mountcmd.extend(['-t', mtype]) mountcmd.append(device) mountcmd.append(tmpd) subp(mountcmd) @@ -891,28 +1045,42 @@ def delete_dir_contents(dirname): del_file(node_fullpath) -def subp(args, input_data=None, allowed_rc=None, env=None): - if allowed_rc is None: - allowed_rc = [0] +def subp(args, data=None, rcs=None, env=None, capture=True, shell=False): + if rcs is None: + rcs = [0] try: - LOG.debug("Running command %s with allowed return codes %s", - args, allowed_rc) - sp = subprocess.Popen(args, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, stdin=subprocess.PIPE, - env=env) - (out, err) = sp.communicate(input_data) + LOG.debug(("Running command %s with allowed return codes %s" + " (shell=%s, capture=%s)"), args, rcs, shell, capture) + if not capture: + stdout = None + stderr = None + else: + stdout = subprocess.PIPE + stderr = subprocess.PIPE + # Always pipe stdin (for now) + # harlowja: I don't see why anyone would want to pipe stdin + # since cloud-init shuts it down (via the method close stdin) + stdin = subprocess.PIPE + sp = subprocess.Popen(args, stdout=stdout, + stderr=stderr, stdin=stdin, + env=env, shell=shell) + (out, err) = sp.communicate(data) except OSError as e: raise ProcessExecutionError(cmd=args, reason=e) rc = sp.returncode - if rc not in allowed_rc: + if rc not in rcs: raise ProcessExecutionError(stdout=out, stderr=err, - exit_code=rc, - cmd=args) - # Just ensure blank instead of none?? - if not out: + exit_code=rc, + cmd=args) + # Just ensure blank instead of none?? (iff capturing) + if not out and capture: out = '' - if not err: + if not err and capture: err = '' + # Useful to note what happened... + if capture: + LOG.debug("Stdout: %s", out) + LOG.debug("Stderr: %s", err) return (out, err) -- cgit v1.2.3 From 3861f78c9f183c9e5f046d81336320209d5edff8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:33:55 -0700 Subject: Moved the contents of this file back to user_data.py It seems to make sense to have that file, then have a handler directory for user data handlers. --- cloudinit/user_data/__init__.py | 215 ---------------------------------------- 1 file changed, 215 deletions(-) delete mode 100644 cloudinit/user_data/__init__.py diff --git a/cloudinit/user_data/__init__.py b/cloudinit/user_data/__init__.py deleted file mode 100644 index 1910ba28..00000000 --- a/cloudinit/user_data/__init__.py +++ /dev/null @@ -1,215 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -import os -import glob - -import email - -from email.mime.base import MIMEBase - -from cloudinit import importer -from cloudinit import log as logging -from cloudinit import util - -from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) - -LOG = logging.getLogger(__name__) - -# Special content types that signal the start and end of processing -CONTENT_END = "__end__" -CONTENT_START = "__begin__" -CONTENT_SIGNALS = [CONTENT_START, CONTENT_END] - -# Used when a part-handler type is encountered -# to allow for registration of new types. -PART_CONTENT_TYPES = ["text/part-handler"] -PART_HANDLER_FN_TMPL = 'part-handler-%03d' - -# For parts without filenames -PART_FN_TPL = 'part-%03d' - -# Used as the content type when a message is not multipart -# and it doesn't contain its own content-type -NOT_MULTIPART_TYPE = "text/x-not-multipart" -OCTET_TYPE = 'application/octet-stream' - -# Different file beginnings to there content type -INCLUSION_TYPES_MAP = { - '#include': 'text/x-include-url', - '#include-once': 'text/x-include-once-url', - '#!': 'text/x-shellscript', - '#cloud-config': 'text/cloud-config', - '#upstart-job': 'text/upstart-job', - '#part-handler': 'text/part-handler', - '#cloud-boothook': 'text/cloud-boothook', - '#cloud-config-archive': 'text/cloud-config-archive', -} - -# Sorted longest first -INCLUSION_SRCH = sorted(INCLUSION_TYPES_MAP.keys(), key=(lambda e: 0 - len(e))) - - -class PartHandler(object): - def __init__(self, frequency, version=2): - self.handler_version = version - self.frequency = frequency - - def __repr__(self): - return "%s: [%s]" % (self.__class__.__name__, self.list_types()) - - def list_types(self): - raise NotImplementedError() - - def handle_part(self, data, ctype, filename, payload, frequency): - return self._handle_part(data, ctype, filename, payload, frequency) - - def _handle_part(self, data, ctype, filename, payload, frequency): - raise NotImplementedError() - - -def fixup_module(mod, def_freq=PER_INSTANCE): - if not hasattr(mod, "handler_version"): - setattr(mod, "handler_version", 1) - if not hasattr(mod, 'list_types'): - def empty_types(): - return [] - setattr(mod, 'list_types', empty_types) - if not hasattr(mod, 'frequency'): - setattr(mod, 'frequency', def_freq) - if not hasattr(mod, 'handle_part'): - def empty_handler(_data, _ctype, _filename, _payload): - pass - setattr(mod, 'handle_part', empty_handler) - return mod - - -def run_part(mod, data, ctype, filename, payload, frequency): - mod_freq = mod.frequency - if not (mod_freq == PER_ALWAYS or - (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)): - return - mod_ver = mod.handler_version - try: - if mod_ver == 1: - mod.handle_part(data, ctype, filename, payload) - else: - mod.handle_part(data, ctype, filename, payload, frequency) - except: - LOG.exception(("Failed calling mod %s (%s, %s, %s)" - " with frequency %s"), mod, ctype, filename, - mod_ver, frequency) - - -def call_begin(mod, data, frequency): - run_part(mod, data, CONTENT_START, None, None, frequency) - - -def call_end(mod, data, frequency): - run_part(mod, data, CONTENT_END, None, None, frequency) - - -def walker_handle_handler(pdata, _ctype, _filename, payload): - curcount = pdata['handlercount'] - modname = PART_HANDLER_FN_TMPL % (curcount) - frequency = pdata['frequency'] - modfname = os.path.join(pdata['handlerdir'], "%s.py" % (modname)) - # TODO: Check if path exists?? - util.write_file(modfname, payload, 0600) - handlers = pdata['handlers'] - try: - mod = fixup_module(importer.import_module(modname)) - handlers.register(mod) - call_begin(mod, pdata['data'], frequency) - pdata['handlercount'] = curcount + 1 - except: - LOG.exception("Failed at registered python file: %s", modfname) - - -def walker_callback(pdata, ctype, filename, payload): - if ctype in PART_CONTENT_TYPES: - walker_handle_handler(pdata, ctype, filename, payload) - return - handlers = pdata['handlers'] - if ctype not in handlers: - if ctype == NOT_MULTIPART_TYPE: - # Extract the first line or 24 bytes for displaying in the log - start = payload.split("\n", 1)[0][:24] - if start < payload: - details = "starting '%s...'" % start.encode("string-escape") - else: - details = repr(payload) - LOG.warning("Unhandled non-multipart userdata: %s", details) - return - run_part(handlers[ctype], pdata['data'], ctype, filename, - payload, pdata['frequency']) - - -# Callback is a function that will be called with -# (data, content_type, filename, payload) -def walk(msg, callback, data): - partnum = 0 - for part in msg.walk(): - # multipart/* are just containers - if part.get_content_maintype() == 'multipart': - continue - - ctype = part.get_content_type() - if ctype is None: - ctype = OCTET_TYPE - - filename = part.get_filename() - if not filename: - filename = PART_FN_TPL % (partnum) - - callback(data, ctype, filename, part.get_payload(decode=True)) - partnum = partnum + 1 - - -# Coverts a raw string into a mime message -def convert_string(raw_data, headers=None): - if not raw_data: - raw_data = '' - if not headers: - headers = {} - data = util.decomp_str(raw_data) - if "mime-version:" in data[0:4096].lower(): - msg = email.message_from_string(data) - for (key, val) in headers.items(): - if key in msg: - msg.replace_header(key, val) - else: - msg[key] = val - else: - mtype = headers.get("Content-Type", NOT_MULTIPART_TYPE) - maintype, subtype = mtype.split("/", 1) - msg = MIMEBase(maintype, subtype, *headers) - msg.set_payload(data) - return msg - - -def type_from_starts_with(payload, default=None): - for text in INCLUSION_SRCH: - if payload.startswith(text): - return INCLUSION_TYPES_MAP[text] - return default \ No newline at end of file -- cgit v1.2.3 From 2631a5cea83ae32ea992d8d4dc9a01ff3da7725d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:34:28 -0700 Subject: Moved this class which does the user data process back to the main user_data.py file. It seems to make a little more sense for now to move it to there. --- cloudinit/user_data/processor.py | 204 --------------------------------------- 1 file changed, 204 deletions(-) delete mode 100644 cloudinit/user_data/processor.py diff --git a/cloudinit/user_data/processor.py b/cloudinit/user_data/processor.py deleted file mode 100644 index 6e796f1c..00000000 --- a/cloudinit/user_data/processor.py +++ /dev/null @@ -1,204 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import hashlib -import os - -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText -from email.mime.base import MIMEBase - -from cloudinit import log as logging -from cloudinit import url_helper -from cloudinit import user_data as ud -from cloudinit import util - -# Various special content types -TYPE_NEEDED = ["text/plain", "text/x-not-multipart"] -INCLUDE_TYPES = ['text/x-include-url', 'text/x-include-once-url'] -ARCHIVE_TYPES = ["text/cloud-config-archive"] -UNDEF_TYPE = "text/plain" -ARCHIVE_UNDEF_TYPE = "text/cloud-config" -OCTET_TYPE = 'application/octet-stream' - -# Msg header used to track attachments -ATTACHMENT_FIELD = 'Number-Attachments' - -# This will be used to create a filename from a url (or like) entry -# When we want to make sure a entry isn't included -# more than once across sessions. -INCLUDE_ONCE_HASHER = 'md5' -MAX_INCLUDE_FN_LEN = 64 - -LOG = logging.getLogger(__name__) - - -class UserDataProcessor(object): - def __init__(self, paths): - self.paths = paths - - def process(self, blob): - base_msg = ud.convert_string(blob) - process_msg = MIMEMultipart() - self._process_msg(base_msg, process_msg) - return process_msg - - def _process_msg(self, base_msg, append_msg): - for part in base_msg.walk(): - # multipart/* are just containers - if part.get_content_maintype() == 'multipart': - continue - - ctype = None - ctype_orig = part.get_content_type() - payload = part.get_payload(decode=True) - - if not ctype_orig: - ctype_orig = UNDEF_TYPE - - if ctype_orig in TYPE_NEEDED: - ctype = ud.type_from_starts_with(payload) - - if ctype is None: - ctype = ctype_orig - - if ctype in INCLUDE_TYPES: - self._do_include(payload, append_msg) - continue - - if ctype in ARCHIVE_TYPES: - self._explode_archive(payload, append_msg) - continue - - if 'Content-Type' in base_msg: - base_msg.replace_header('Content-Type', ctype) - else: - base_msg['Content-Type'] = ctype - - self._attach_part(append_msg, part) - - def _get_include_once_filename(self, entry): - msum = hashlib.new(INCLUDE_ONCE_HASHER) - msum.update(entry) - # Don't get to long now - entry_fn = msum.hexdigest()[0:MAX_INCLUDE_FN_LEN] - return os.path.join(self.paths.get_ipath_cur('data'), - 'urlcache', entry_fn) - - def _do_include(self, content, append_msg): - # is just a list of urls, one per line - # also support '#include ' - for line in content.splitlines(): - includeonce = False - if line in ("#include", "#include-once"): - continue - if line.startswith("#include-once"): - line = line[len("#include-once"):].lstrip() - includeonce = True - elif line.startswith("#include"): - line = line[len("#include"):].lstrip() - if line.startswith("#"): - continue - include_url = line.strip() - if not include_url: - continue - - includeonce_filename = self._get_include_once_filename(include_url) - if includeonce and os.path.isfile(includeonce_filename): - content = util.load_file(includeonce_filename) - else: - (content, st) = url_helper.readurl(include_url) - if includeonce and url_helper.ok_http_code(st): - util.write_file(includeonce_filename, content, mode=0600) - if not url_helper.ok_http_code(st): - content = '' - - new_msg = ud.convert_string(content) - self._process_msg(new_msg, append_msg) - - def _explode_archive(self, archive, append_msg): - entries = util.load_yaml(archive, default=[], allowed=[list, set]) - for ent in entries: - # ent can be one of: - # dict { 'filename' : 'value', 'content' : - # 'value', 'type' : 'value' } - # filename and type not be present - # or - # scalar(payload) - if isinstance(ent, str): - ent = {'content': ent} - if not isinstance(ent, (dict)): - # TODO raise? - continue - - content = ent.get('content', '') - mtype = ent.get('type') - if not mtype: - mtype = ud.type_from_starts_with(content, ARCHIVE_UNDEF_TYPE) - - maintype, subtype = mtype.split('/', 1) - if maintype == "text": - msg = MIMEText(content, _subtype=subtype) - else: - msg = MIMEBase(maintype, subtype) - msg.set_payload(content) - - if 'filename' in ent: - msg.add_header('Content-Disposition', 'attachment', - filename=ent['filename']) - - for header in ent.keys(): - if header in ('content', 'filename', 'type'): - continue - msg.add_header(header, ent['header']) - - self._attach_part(append_msg, msg) - - def _multi_part_count(self, outer_msg, new_count=None): - """ - Return the number of attachments to this MIMEMultipart by looking - at its 'Number-Attachments' header. - """ - if ATTACHMENT_FIELD not in outer_msg: - outer_msg[ATTACHMENT_FIELD] = str(0) - - if new_count is not None: - outer_msg.replace_header(ATTACHMENT_FIELD, str(new_count)) - - fetched_count = 0 - try: - fetched_count = int(outer_msg.get(ATTACHMENT_FIELD)) - except (ValueError, TypeError): - outer_msg.replace_header(ATTACHMENT_FIELD, str(fetched_count)) - return fetched_count - - def _attach_part(self, outer_msg, part): - """ - Attach an part to an outer message. outermsg must be a MIMEMultipart. - Modifies a header in the message to keep track of number of attachments. - """ - cur = self._multi_part_count(outer_msg) - if not part.get_filename(): - fn = ud.PART_FN_TPL % (cur + 1) - part.add_header('Content-Disposition', 'attachment', filename=fn) - outer_msg.attach(part) - self._multi_part_count(outer_msg, cur + 1) -- cgit v1.2.3 From 4d74f499a0257a4f27f7def4b76402bcbde63567 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:35:07 -0700 Subject: Moved the handlers from a user_data directory to a handler directory. --- cloudinit/handlers/__init__.py | 221 ---------------------- cloudinit/handlers/boot_hook.py | 66 +++++++ cloudinit/handlers/cc_apt_pipelining.py | 53 ------ cloudinit/handlers/cc_apt_update_upgrade.py | 241 ------------------------ cloudinit/handlers/cc_bootcmd.py | 48 ----- cloudinit/handlers/cc_byobu.py | 77 -------- cloudinit/handlers/cc_ca_certs.py | 90 --------- cloudinit/handlers/cc_chef.py | 119 ------------ cloudinit/handlers/cc_disable_ec2_metadata.py | 30 --- cloudinit/handlers/cc_final_message.py | 58 ------ cloudinit/handlers/cc_foo.py | 29 --- cloudinit/handlers/cc_grub_dpkg.py | 64 ------- cloudinit/handlers/cc_keys_to_console.py | 42 ----- cloudinit/handlers/cc_landscape.py | 75 -------- cloudinit/handlers/cc_locale.py | 54 ------ cloudinit/handlers/cc_mcollective.py | 99 ---------- cloudinit/handlers/cc_mounts.py | 179 ------------------ cloudinit/handlers/cc_phone_home.py | 106 ----------- cloudinit/handlers/cc_puppet.py | 108 ----------- cloudinit/handlers/cc_resizefs.py | 108 ----------- cloudinit/handlers/cc_rightscale_userdata.py | 78 -------- cloudinit/handlers/cc_rsyslog.py | 101 ---------- cloudinit/handlers/cc_runcmd.py | 32 ---- cloudinit/handlers/cc_salt_minion.py | 56 ------ cloudinit/handlers/cc_scripts_per_boot.py | 34 ---- cloudinit/handlers/cc_scripts_per_instance.py | 34 ---- cloudinit/handlers/cc_scripts_per_once.py | 34 ---- cloudinit/handlers/cc_scripts_user.py | 34 ---- cloudinit/handlers/cc_set_hostname.py | 42 ----- cloudinit/handlers/cc_set_passwords.py | 129 ------------- cloudinit/handlers/cc_ssh.py | 106 ----------- cloudinit/handlers/cc_ssh_import_id.py | 50 ----- cloudinit/handlers/cc_timezone.py | 67 ------- cloudinit/handlers/cc_update_etc_hosts.py | 87 --------- cloudinit/handlers/cc_update_hostname.py | 101 ---------- cloudinit/transforms/__init__.py | 221 ++++++++++++++++++++++ cloudinit/transforms/cc_apt_pipelining.py | 53 ++++++ cloudinit/transforms/cc_apt_update_upgrade.py | 241 ++++++++++++++++++++++++ cloudinit/transforms/cc_bootcmd.py | 48 +++++ cloudinit/transforms/cc_byobu.py | 77 ++++++++ cloudinit/transforms/cc_ca_certs.py | 90 +++++++++ cloudinit/transforms/cc_chef.py | 119 ++++++++++++ cloudinit/transforms/cc_disable_ec2_metadata.py | 30 +++ cloudinit/transforms/cc_final_message.py | 58 ++++++ cloudinit/transforms/cc_foo.py | 29 +++ cloudinit/transforms/cc_grub_dpkg.py | 64 +++++++ cloudinit/transforms/cc_keys_to_console.py | 42 +++++ cloudinit/transforms/cc_landscape.py | 75 ++++++++ cloudinit/transforms/cc_locale.py | 54 ++++++ cloudinit/transforms/cc_mcollective.py | 99 ++++++++++ cloudinit/transforms/cc_mounts.py | 179 ++++++++++++++++++ cloudinit/transforms/cc_phone_home.py | 106 +++++++++++ cloudinit/transforms/cc_puppet.py | 108 +++++++++++ cloudinit/transforms/cc_resizefs.py | 108 +++++++++++ cloudinit/transforms/cc_rightscale_userdata.py | 78 ++++++++ cloudinit/transforms/cc_rsyslog.py | 101 ++++++++++ cloudinit/transforms/cc_runcmd.py | 32 ++++ cloudinit/transforms/cc_salt_minion.py | 56 ++++++ cloudinit/transforms/cc_scripts_per_boot.py | 34 ++++ cloudinit/transforms/cc_scripts_per_instance.py | 34 ++++ cloudinit/transforms/cc_scripts_per_once.py | 34 ++++ cloudinit/transforms/cc_scripts_user.py | 34 ++++ cloudinit/transforms/cc_set_hostname.py | 42 +++++ cloudinit/transforms/cc_set_passwords.py | 129 +++++++++++++ cloudinit/transforms/cc_ssh.py | 106 +++++++++++ cloudinit/transforms/cc_ssh_import_id.py | 50 +++++ cloudinit/transforms/cc_timezone.py | 67 +++++++ cloudinit/transforms/cc_update_etc_hosts.py | 87 +++++++++ cloudinit/transforms/cc_update_hostname.py | 101 ++++++++++ cloudinit/user_data/boot_hook.py | 66 ------- 70 files changed, 2852 insertions(+), 2852 deletions(-) delete mode 100644 cloudinit/handlers/__init__.py create mode 100644 cloudinit/handlers/boot_hook.py delete mode 100644 cloudinit/handlers/cc_apt_pipelining.py delete mode 100644 cloudinit/handlers/cc_apt_update_upgrade.py delete mode 100644 cloudinit/handlers/cc_bootcmd.py delete mode 100644 cloudinit/handlers/cc_byobu.py delete mode 100644 cloudinit/handlers/cc_ca_certs.py delete mode 100644 cloudinit/handlers/cc_chef.py delete mode 100644 cloudinit/handlers/cc_disable_ec2_metadata.py delete mode 100644 cloudinit/handlers/cc_final_message.py delete mode 100644 cloudinit/handlers/cc_foo.py delete mode 100644 cloudinit/handlers/cc_grub_dpkg.py delete mode 100644 cloudinit/handlers/cc_keys_to_console.py delete mode 100644 cloudinit/handlers/cc_landscape.py delete mode 100644 cloudinit/handlers/cc_locale.py delete mode 100644 cloudinit/handlers/cc_mcollective.py delete mode 100644 cloudinit/handlers/cc_mounts.py delete mode 100644 cloudinit/handlers/cc_phone_home.py delete mode 100644 cloudinit/handlers/cc_puppet.py delete mode 100644 cloudinit/handlers/cc_resizefs.py delete mode 100644 cloudinit/handlers/cc_rightscale_userdata.py delete mode 100644 cloudinit/handlers/cc_rsyslog.py delete mode 100644 cloudinit/handlers/cc_runcmd.py delete mode 100644 cloudinit/handlers/cc_salt_minion.py delete mode 100644 cloudinit/handlers/cc_scripts_per_boot.py delete mode 100644 cloudinit/handlers/cc_scripts_per_instance.py delete mode 100644 cloudinit/handlers/cc_scripts_per_once.py delete mode 100644 cloudinit/handlers/cc_scripts_user.py delete mode 100644 cloudinit/handlers/cc_set_hostname.py delete mode 100644 cloudinit/handlers/cc_set_passwords.py delete mode 100644 cloudinit/handlers/cc_ssh.py delete mode 100644 cloudinit/handlers/cc_ssh_import_id.py delete mode 100644 cloudinit/handlers/cc_timezone.py delete mode 100644 cloudinit/handlers/cc_update_etc_hosts.py delete mode 100644 cloudinit/handlers/cc_update_hostname.py create mode 100644 cloudinit/transforms/__init__.py create mode 100644 cloudinit/transforms/cc_apt_pipelining.py create mode 100644 cloudinit/transforms/cc_apt_update_upgrade.py create mode 100644 cloudinit/transforms/cc_bootcmd.py create mode 100644 cloudinit/transforms/cc_byobu.py create mode 100644 cloudinit/transforms/cc_ca_certs.py create mode 100644 cloudinit/transforms/cc_chef.py create mode 100644 cloudinit/transforms/cc_disable_ec2_metadata.py create mode 100644 cloudinit/transforms/cc_final_message.py create mode 100644 cloudinit/transforms/cc_foo.py create mode 100644 cloudinit/transforms/cc_grub_dpkg.py create mode 100644 cloudinit/transforms/cc_keys_to_console.py create mode 100644 cloudinit/transforms/cc_landscape.py create mode 100644 cloudinit/transforms/cc_locale.py create mode 100644 cloudinit/transforms/cc_mcollective.py create mode 100644 cloudinit/transforms/cc_mounts.py create mode 100644 cloudinit/transforms/cc_phone_home.py create mode 100644 cloudinit/transforms/cc_puppet.py create mode 100644 cloudinit/transforms/cc_resizefs.py create mode 100644 cloudinit/transforms/cc_rightscale_userdata.py create mode 100644 cloudinit/transforms/cc_rsyslog.py create mode 100644 cloudinit/transforms/cc_runcmd.py create mode 100644 cloudinit/transforms/cc_salt_minion.py create mode 100644 cloudinit/transforms/cc_scripts_per_boot.py create mode 100644 cloudinit/transforms/cc_scripts_per_instance.py create mode 100644 cloudinit/transforms/cc_scripts_per_once.py create mode 100644 cloudinit/transforms/cc_scripts_user.py create mode 100644 cloudinit/transforms/cc_set_hostname.py create mode 100644 cloudinit/transforms/cc_set_passwords.py create mode 100644 cloudinit/transforms/cc_ssh.py create mode 100644 cloudinit/transforms/cc_ssh_import_id.py create mode 100644 cloudinit/transforms/cc_timezone.py create mode 100644 cloudinit/transforms/cc_update_etc_hosts.py create mode 100644 cloudinit/transforms/cc_update_hostname.py delete mode 100644 cloudinit/user_data/boot_hook.py diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py deleted file mode 100644 index 5d70ac43..00000000 --- a/cloudinit/handlers/__init__.py +++ /dev/null @@ -1,221 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2008-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Chuck Short -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -import os -import subprocess -import sys -import time -import traceback - -import yaml - -from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE) - -from cloudinit import log as logging -from cloudinit import util - -LOG = logging.getLogger(__name__) - -DEF_HANDLER_VERSION = 1 -DEF_FREQ = PER_INSTANCE - - -# reads a cloudconfig module list, returns -# a 2 dimensional array suitable to pass to run_cc_modules -def read_cc_modules(cfg, name): - if name not in cfg: - return([]) - module_list = [] - # create 'module_list', an array of arrays - # where array[0] = config - # array[1] = freq - # array[2:] = arguemnts - for item in cfg[name]: - if isinstance(item, str): - module_list.append((item,)) - elif isinstance(item, list): - module_list.append(item) - else: - raise TypeError("failed to read '%s' item in config") - return(module_list) - - -def run_cc_modules(cc, module_list, log): - failures = [] - for cfg_mod in module_list: - name = cfg_mod[0] - freq = None - run_args = [] - if len(cfg_mod) > 1: - freq = cfg_mod[1] - if len(cfg_mod) > 2: - run_args = cfg_mod[2:] - - try: - log.debug("handling %s with freq=%s and args=%s" % - (name, freq, run_args)) - cc.handle(name, run_args, freq=freq) - except: - log.warn(traceback.format_exc()) - log.error("config handling of %s, %s, %s failed\n" % - (name, freq, run_args)) - failures.append(name) - - return(failures) - - -# always returns well formated values -# cfg is expected to have an entry 'output' in it, which is a dictionary -# that includes entries for 'init', 'config', 'final' or 'all' -# init: /var/log/cloud.out -# config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ] -# final: -# output: "| logger -p" -# error: "> /dev/null" -# this returns the specific 'mode' entry, cleanly formatted, with value -# None if if none is given -def get_output_cfg(cfg, mode="init"): - ret = [None, None] - if not 'output' in cfg: - return ret - - outcfg = cfg['output'] - if mode in outcfg: - modecfg = outcfg[mode] - else: - if 'all' not in outcfg: - return ret - # if there is a 'all' item in the output list - # then it applies to all users of this (init, config, final) - modecfg = outcfg['all'] - - # if value is a string, it specifies stdout and stderr - if isinstance(modecfg, str): - ret = [modecfg, modecfg] - - # if its a list, then we expect (stdout, stderr) - if isinstance(modecfg, list): - if len(modecfg) > 0: - ret[0] = modecfg[0] - if len(modecfg) > 1: - ret[1] = modecfg[1] - - # if it is a dictionary, expect 'out' and 'error' - # items, which indicate out and error - if isinstance(modecfg, dict): - if 'output' in modecfg: - ret[0] = modecfg['output'] - if 'error' in modecfg: - ret[1] = modecfg['error'] - - # if err's entry == "&1", then make it same as stdout - # as in shell syntax of "echo foo >/dev/null 2>&1" - if ret[1] == "&1": - ret[1] = ret[0] - - swlist = [">>", ">", "|"] - for i in range(len(ret)): - if not ret[i]: - continue - val = ret[i].lstrip() - found = False - for s in swlist: - if val.startswith(s): - val = "%s %s" % (s, val[len(s):].strip()) - found = True - break - if not found: - # default behavior is append - val = "%s %s" % (">>", val.strip()) - ret[i] = val - - return(ret) - - -# redirect_output(outfmt, errfmt, orig_out, orig_err) -# replace orig_out and orig_err with filehandles specified in outfmt or errfmt -# fmt can be: -# > FILEPATH -# >> FILEPATH -# | program [ arg1 [ arg2 [ ... ] ] ] -# -# with a '|', arguments are passed to shell, so one level of -# shell escape is required. -def redirect_output(outfmt, errfmt, o_out=sys.stdout, o_err=sys.stderr): - if outfmt: - (mode, arg) = outfmt.split(" ", 1) - if mode == ">" or mode == ">>": - owith = "ab" - if mode == ">": - owith = "wb" - new_fp = open(arg, owith) - elif mode == "|": - proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin - else: - raise TypeError("invalid type for outfmt: %s" % outfmt) - - if o_out: - os.dup2(new_fp.fileno(), o_out.fileno()) - if errfmt == outfmt: - os.dup2(new_fp.fileno(), o_err.fileno()) - return - - if errfmt: - (mode, arg) = errfmt.split(" ", 1) - if mode == ">" or mode == ">>": - owith = "ab" - if mode == ">": - owith = "wb" - new_fp = open(arg, owith) - elif mode == "|": - proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin - else: - raise TypeError("invalid type for outfmt: %s" % outfmt) - - if o_err: - os.dup2(new_fp.fileno(), o_err.fileno()) - return - - -def form_module_name(name): - canon_name = name.replace("-", "_") - if canon_name.endswith(".py"): - canon_name = canon_name[0:(len(canon_name) - 3)] - canon_name = canon_name.strip() - if not canon_name: - return None - if not canon_name.startswith("cc_"): - canon_name = 'cc_%s' % (canon_name) - return canon_name - - -def fixup_module(mod): - freq = getattr(mod, "frequency", None) - if not freq: - setattr(mod, 'frequency', PER_INSTANCE) - handler = getattr(mod, "handle", None) - if not handler: - def empty_handle(_name, _cfg, _cloud, _log, _args): - pass - setattr(mod, 'handle', empty_handle) - return mod diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py new file mode 100644 index 00000000..c75aeb72 --- /dev/null +++ b/cloudinit/handlers/boot_hook.py @@ -0,0 +1,66 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os + +from cloudinit import log as logging +from cloudinit import user_data as ud +from cloudinit import util + +from cloudinit.settings import (PER_ALWAYS) + +LOG = logging.getLogger(__name__) + + +class BootHookPartHandler(ud.PartHandler): + def __init__(self, boothook_dir, instance_id): + ud.PartHandler.__init__(self, PER_ALWAYS) + self.boothook_dir = boothook_dir + self.instance_id = instance_id + + def list_types(self): + return [ + ud.type_from_starts_with("#cloud-boothook"), + ] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype in ud.CONTENT_SIGNALS: + return + + filename = util.clean_filename(filename) + payload = util.dos2unix(payload) + prefix = "#cloud-boothook" + start = 0 + if payload.startswith(prefix): + start = len(prefix) + 1 + + filepath = os.path.join(self.boothook_dir, filename) + util.write_file(filepath, payload[start:], 0700) + try: + env = os.environ.copy() + env['INSTANCE_ID'] = str(self.instance_id) + util.subp([filepath], env=env) + except util.ProcessExecutionError as e: + util.logexc(LOG, "Boothooks script %s execution error", filepath) + except Exception as e: + util.logexc(LOG, ("Boothooks unknown " + "error when running %s"), filepath) diff --git a/cloudinit/handlers/cc_apt_pipelining.py b/cloudinit/handlers/cc_apt_pipelining.py deleted file mode 100644 index 0286a9ae..00000000 --- a/cloudinit/handlers/cc_apt_pipelining.py +++ /dev/null @@ -1,53 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# -# Author: Ben Howard -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance - -frequency = per_instance -default_file = "/etc/apt/apt.conf.d/90cloud-init-pipelining" - - -def handle(_name, cfg, _cloud, log, _args): - - apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) - apt_pipe_value = str(apt_pipe_value).lower() - - if apt_pipe_value == "false": - write_apt_snippet("0", log) - - elif apt_pipe_value in ("none", "unchanged", "os"): - return - - elif apt_pipe_value in str(range(0, 6)): - write_apt_snippet(apt_pipe_value, log) - - else: - log.warn("Invalid option for apt_pipeling: %s" % apt_pipe_value) - - -def write_apt_snippet(setting, log, f_name=default_file): - """ Writes f_name with apt pipeline depth 'setting' """ - - acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' - file_contents = ("//Written by cloud-init per 'apt_pipelining'\n" - + (acquire_pipeline_depth % setting)) - - util.write_file(f_name, file_contents) - - log.debug("Wrote %s with APT pipeline setting" % f_name) diff --git a/cloudinit/handlers/cc_apt_update_upgrade.py b/cloudinit/handlers/cc_apt_update_upgrade.py deleted file mode 100644 index a7049bce..00000000 --- a/cloudinit/handlers/cc_apt_update_upgrade.py +++ /dev/null @@ -1,241 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import traceback -import os -import glob -import cloudinit.CloudConfig as cc - - -def handle(_name, cfg, cloud, log, _args): - update = util.get_cfg_option_bool(cfg, 'apt_update', False) - upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) - - release = get_release() - - mirror = find_apt_mirror(cloud, cfg) - - log.debug("selected mirror at: %s" % mirror) - - if not util.get_cfg_option_bool(cfg, \ - 'apt_preserve_sources_list', False): - generate_sources_list(release, mirror) - old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', \ - "archive.ubuntu.com/ubuntu") - rename_apt_lists(old_mir, mirror) - - # set up proxy - proxy = cfg.get("apt_proxy", None) - proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy" - if proxy: - try: - contents = "Acquire::HTTP::Proxy \"%s\";\n" - with open(proxy_filename, "w") as fp: - fp.write(contents % proxy) - except Exception as e: - log.warn("Failed to write proxy to %s" % proxy_filename) - elif os.path.isfile(proxy_filename): - os.unlink(proxy_filename) - - # process 'apt_sources' - if 'apt_sources' in cfg: - errors = add_sources(cfg['apt_sources'], - {'MIRROR': mirror, 'RELEASE': release}) - for e in errors: - log.warn("Source Error: %s\n" % ':'.join(e)) - - dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False) - if dconf_sel: - log.debug("setting debconf selections per cloud config") - try: - util.subp(('debconf-set-selections', '-'), dconf_sel) - except: - log.error("Failed to run debconf-set-selections") - log.debug(traceback.format_exc()) - - pkglist = util.get_cfg_option_list_or_str(cfg, 'packages', []) - - errors = [] - if update or len(pkglist) or upgrade: - try: - cc.update_package_sources() - except subprocess.CalledProcessError as e: - log.warn("apt-get update failed") - log.debug(traceback.format_exc()) - errors.append(e) - - if upgrade: - try: - cc.apt_get("upgrade") - except subprocess.CalledProcessError as e: - log.warn("apt upgrade failed") - log.debug(traceback.format_exc()) - errors.append(e) - - if len(pkglist): - try: - cc.install_packages(pkglist) - except subprocess.CalledProcessError as e: - log.warn("Failed to install packages: %s " % pkglist) - log.debug(traceback.format_exc()) - errors.append(e) - - if len(errors): - raise errors[0] - - return(True) - - -def mirror2lists_fileprefix(mirror): - string = mirror - # take of http:// or ftp:// - if string.endswith("/"): - string = string[0:-1] - pos = string.find("://") - if pos >= 0: - string = string[pos + 3:] - string = string.replace("/", "_") - return string - - -def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"): - oprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(omirror)) - nprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(new_mirror)) - if(oprefix == nprefix): - return - olen = len(oprefix) - for filename in glob.glob("%s_*" % oprefix): - os.rename(filename, "%s%s" % (nprefix, filename[olen:])) - - -def get_release(): - stdout, _stderr = subprocess.Popen(['lsb_release', '-cs'], - stdout=subprocess.PIPE).communicate() - return(str(stdout).strip()) - - -def generate_sources_list(codename, mirror): - util.render_to_file('sources.list', '/etc/apt/sources.list', \ - {'mirror': mirror, 'codename': codename}) - - -def add_sources(srclist, searchList=None): - """ - add entries in /etc/apt/sources.list.d for each abbreviated - sources.list entry in 'srclist'. When rendering template, also - include the values in dictionary searchList - """ - if searchList is None: - searchList = {} - elst = [] - - for ent in srclist: - if 'source' not in ent: - elst.append(["", "missing source"]) - continue - - source = ent['source'] - if source.startswith("ppa:"): - try: - util.subp(["add-apt-repository", source]) - except: - elst.append([source, "add-apt-repository failed"]) - continue - - source = util.render_string(source, searchList) - - if 'filename' not in ent: - ent['filename'] = 'cloud_config_sources.list' - - if not ent['filename'].startswith("/"): - ent['filename'] = "%s/%s" % \ - ("/etc/apt/sources.list.d/", ent['filename']) - - if ('keyid' in ent and 'key' not in ent): - ks = "keyserver.ubuntu.com" - if 'keyserver' in ent: - ks = ent['keyserver'] - try: - ent['key'] = util.getkeybyid(ent['keyid'], ks) - except: - elst.append([source, "failed to get key from %s" % ks]) - continue - - if 'key' in ent: - try: - util.subp(('apt-key', 'add', '-'), ent['key']) - except: - elst.append([source, "failed add key"]) - - try: - util.write_file(ent['filename'], source + "\n", omode="ab") - except: - elst.append([source, "failed write to file %s" % ent['filename']]) - - return(elst) - - -def find_apt_mirror(cloud, cfg): - """ find an apt_mirror given the cloud and cfg provided """ - - # TODO: distro and defaults should be configurable - distro = "ubuntu" - defaults = { - 'ubuntu': "http://archive.ubuntu.com/ubuntu", - 'debian': "http://archive.debian.org/debian", - } - mirror = None - - cfg_mirror = cfg.get("apt_mirror", None) - if cfg_mirror: - mirror = cfg["apt_mirror"] - elif "apt_mirror_search" in cfg: - mirror = util.search_for_mirror(cfg['apt_mirror_search']) - else: - if cloud: - mirror = cloud.get_mirror() - - mydom = "" - - doms = [] - - if not mirror and cloud: - # if we have a fqdn, then search its domain portion first - (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - mydom = ".".join(fqdn.split(".")[1:]) - if mydom: - doms.append(".%s" % mydom) - - if not mirror: - doms.extend((".localdomain", "",)) - - mirror_list = [] - mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) - for post in doms: - mirror_list.append(mirrorfmt % post) - - mirror = util.search_for_mirror(mirror_list) - - if not mirror: - mirror = defaults[distro] - - return mirror diff --git a/cloudinit/handlers/cc_bootcmd.py b/cloudinit/handlers/cc_bootcmd.py deleted file mode 100644 index f584da02..00000000 --- a/cloudinit/handlers/cc_bootcmd.py +++ /dev/null @@ -1,48 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import cloudinit.util as util -import subprocess -import tempfile -import os -from cloudinit.CloudConfig import per_always -frequency = per_always - - -def handle(_name, cfg, cloud, log, _args): - if "bootcmd" not in cfg: - return - - try: - content = util.shellify(cfg["bootcmd"]) - tmpf = tempfile.TemporaryFile() - tmpf.write(content) - tmpf.seek(0) - except: - log.warn("failed to shellify bootcmd") - raise - - try: - env = os.environ.copy() - env['INSTANCE_ID'] = cloud.get_instance_id() - subprocess.check_call(['/bin/sh'], env=env, stdin=tmpf) - tmpf.close() - except: - log.warn("failed to run commands from bootcmd") - raise diff --git a/cloudinit/handlers/cc_byobu.py b/cloudinit/handlers/cc_byobu.py deleted file mode 100644 index e821b261..00000000 --- a/cloudinit/handlers/cc_byobu.py +++ /dev/null @@ -1,77 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import traceback - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - value = args[0] - else: - value = util.get_cfg_option_str(cfg, "byobu_by_default", "") - - if not value: - return - - if value == "user" or value == "system": - value = "enable-%s" % value - - valid = ("enable-user", "enable-system", "enable", - "disable-user", "disable-system", "disable") - if not value in valid: - log.warn("Unknown value %s for byobu_by_default" % value) - - mod_user = value.endswith("-user") - mod_sys = value.endswith("-system") - if value.startswith("enable"): - bl_inst = "install" - dc_val = "byobu byobu/launch-by-default boolean true" - mod_sys = True - else: - if value == "disable": - mod_user = True - mod_sys = True - bl_inst = "uninstall" - dc_val = "byobu byobu/launch-by-default boolean false" - - shcmd = "" - if mod_user: - user = util.get_cfg_option_str(cfg, "user", "ubuntu") - shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst) - shcmd += " || X=$(($X+1)); " - if mod_sys: - shcmd += "echo \"%s\" | debconf-set-selections" % dc_val - shcmd += " && dpkg-reconfigure byobu --frontend=noninteractive" - shcmd += " || X=$(($X+1)); " - - cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] - - log.debug("setting byobu to %s" % value) - - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) - except OSError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/handlers/cc_ca_certs.py b/cloudinit/handlers/cc_ca_certs.py deleted file mode 100644 index 3af6238a..00000000 --- a/cloudinit/handlers/cc_ca_certs.py +++ /dev/null @@ -1,90 +0,0 @@ -# vi: ts=4 expandtab -# -# Author: Mike Milner -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import os -from subprocess import check_call -from cloudinit.util import (write_file, get_cfg_option_list_or_str, - delete_dir_contents, subp) - -CA_CERT_PATH = "/usr/share/ca-certificates/" -CA_CERT_FILENAME = "cloud-init-ca-certs.crt" -CA_CERT_CONFIG = "/etc/ca-certificates.conf" -CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" - - -def update_ca_certs(): - """ - Updates the CA certificate cache on the current machine. - """ - check_call(["update-ca-certificates"]) - - -def add_ca_certs(certs): - """ - Adds certificates to the system. To actually apply the new certificates - you must also call L{update_ca_certs}. - - @param certs: A list of certificate strings. - """ - if certs: - cert_file_contents = "\n".join(certs) - cert_file_fullpath = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) - write_file(cert_file_fullpath, cert_file_contents, mode=0644) - # Append cert filename to CA_CERT_CONFIG file. - write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="a") - - -def remove_default_ca_certs(): - """ - Removes all default trusted CA certificates from the system. To actually - apply the change you must also call L{update_ca_certs}. - """ - delete_dir_contents(CA_CERT_PATH) - delete_dir_contents(CA_CERT_SYSTEM_PATH) - write_file(CA_CERT_CONFIG, "", mode=0644) - debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" - subp(('debconf-set-selections', '-'), debconf_sel) - - -def handle(_name, cfg, _cloud, log, _args): - """ - Call to handle ca-cert sections in cloud-config file. - - @param name: The module name "ca-cert" from cloud.cfg - @param cfg: A nested dict containing the entire cloud config contents. - @param cloud: The L{CloudInit} object in use. - @param log: Pre-initialized Python logger object to use for logging. - @param args: Any module arguments from cloud.cfg - """ - # If there isn't a ca-certs section in the configuration don't do anything - if "ca-certs" not in cfg: - return - ca_cert_cfg = cfg['ca-certs'] - - # If there is a remove-defaults option set to true, remove the system - # default trusted CA certs first. - if ca_cert_cfg.get("remove-defaults", False): - log.debug("removing default certificates") - remove_default_ca_certs() - - # If we are given any new trusted CA certs to add, add them. - if "trusted" in ca_cert_cfg: - trusted_certs = get_cfg_option_list_or_str(ca_cert_cfg, "trusted") - if trusted_certs: - log.debug("adding %d certificates" % len(trusted_certs)) - add_ca_certs(trusted_certs) - - # Update the system with the new cert configuration. - update_ca_certs() diff --git a/cloudinit/handlers/cc_chef.py b/cloudinit/handlers/cc_chef.py deleted file mode 100644 index 941e04fe..00000000 --- a/cloudinit/handlers/cc_chef.py +++ /dev/null @@ -1,119 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Avishai Ish-Shalom -# Author: Mike Moulton -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import subprocess -import json -import cloudinit.CloudConfig as cc -import cloudinit.util as util - -ruby_version_default = "1.8" - - -def handle(_name, cfg, cloud, log, _args): - # If there isn't a chef key in the configuration don't do anything - if 'chef' not in cfg: - return - chef_cfg = cfg['chef'] - - # ensure the chef directories we use exist - mkdirs(['/etc/chef', '/var/log/chef', '/var/lib/chef', - '/var/cache/chef', '/var/backups/chef', '/var/run/chef']) - - # set the validation key based on the presence of either 'validation_key' - # or 'validation_cert'. In the case where both exist, 'validation_key' - # takes precedence - for key in ('validation_key', 'validation_cert'): - if key in chef_cfg and chef_cfg[key]: - with open('/etc/chef/validation.pem', 'w') as validation_key_fh: - validation_key_fh.write(chef_cfg[key]) - break - - # create the chef config from template - util.render_to_file('chef_client.rb', '/etc/chef/client.rb', - {'server_url': chef_cfg['server_url'], - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', - cloud.datasource.get_instance_id()), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - '_default'), - 'validation_name': chef_cfg['validation_name']}) - - # set the firstboot json - with open('/etc/chef/firstboot.json', 'w') as firstboot_json_fh: - initial_json = {} - if 'run_list' in chef_cfg: - initial_json['run_list'] = chef_cfg['run_list'] - if 'initial_attributes' in chef_cfg: - initial_attributes = chef_cfg['initial_attributes'] - for k in initial_attributes.keys(): - initial_json[k] = initial_attributes[k] - firstboot_json_fh.write(json.dumps(initial_json)) - - # If chef is not installed, we install chef based on 'install_type' - if not os.path.isfile('/usr/bin/chef-client'): - install_type = util.get_cfg_option_str(chef_cfg, 'install_type', - 'packages') - if install_type == "gems": - # this will install and run the chef-client from gems - chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) - ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', - ruby_version_default) - install_chef_from_gems(ruby_version, chef_version) - # and finally, run chef-client - log.debug('running chef-client') - subprocess.check_call(['/usr/bin/chef-client', '-d', '-i', '1800', - '-s', '20']) - else: - # this will install and run the chef-client from packages - cc.install_packages(('chef',)) - - -def get_ruby_packages(version): - # return a list of packages needed to install ruby at version - pkgs = ['ruby%s' % version, 'ruby%s-dev' % version] - if version == "1.8": - pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8')) - return(pkgs) - - -def install_chef_from_gems(ruby_version, chef_version=None): - cc.install_packages(get_ruby_packages(ruby_version)) - if not os.path.exists('/usr/bin/gem'): - os.symlink('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') - if not os.path.exists('/usr/bin/ruby'): - os.symlink('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') - if chef_version: - subprocess.check_call(['/usr/bin/gem', 'install', 'chef', - '-v %s' % chef_version, '--no-ri', - '--no-rdoc', '--bindir', '/usr/bin', '-q']) - else: - subprocess.check_call(['/usr/bin/gem', 'install', 'chef', - '--no-ri', '--no-rdoc', '--bindir', - '/usr/bin', '-q']) - - -def ensure_dir(d): - if not os.path.exists(d): - os.makedirs(d) - - -def mkdirs(dirs): - for d in dirs: - ensure_dir(d) diff --git a/cloudinit/handlers/cc_disable_ec2_metadata.py b/cloudinit/handlers/cc_disable_ec2_metadata.py deleted file mode 100644 index 6b31ea8e..00000000 --- a/cloudinit/handlers/cc_disable_ec2_metadata.py +++ /dev/null @@ -1,30 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import cloudinit.util as util -import subprocess -from cloudinit.CloudConfig import per_always - -frequency = per_always - - -def handle(_name, cfg, _cloud, _log, _args): - if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False): - fwall = "route add -host 169.254.169.254 reject" - subprocess.call(fwall.split(' ')) diff --git a/cloudinit/handlers/cc_final_message.py b/cloudinit/handlers/cc_final_message.py deleted file mode 100644 index abb4ca32..00000000 --- a/cloudinit/handlers/cc_final_message.py +++ /dev/null @@ -1,58 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.CloudConfig import per_always -import sys -from cloudinit import util, boot_finished -import time - -frequency = per_always - -final_message = "cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds" - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - msg_in = args[0] - else: - msg_in = util.get_cfg_option_str(cfg, "final_message", final_message) - - try: - uptimef = open("/proc/uptime") - uptime = uptimef.read().split(" ")[0] - uptimef.close() - except IOError as e: - log.warn("unable to open /proc/uptime\n") - uptime = "na" - - try: - ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) - except: - ts = "na" - - try: - subs = {'UPTIME': uptime, 'TIMESTAMP': ts} - sys.stdout.write("%s\n" % util.render_string(msg_in, subs)) - except Exception as e: - log.warn("failed to render string to stdout: %s" % e) - - fp = open(boot_finished, "wb") - fp.write(uptime + "\n") - fp.close() diff --git a/cloudinit/handlers/cc_foo.py b/cloudinit/handlers/cc_foo.py deleted file mode 100644 index 35ec3fa7..00000000 --- a/cloudinit/handlers/cc_foo.py +++ /dev/null @@ -1,29 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -#import cloudinit -#import cloudinit.util as util -from cloudinit.CloudConfig import per_instance - -frequency = per_instance - - -def handle(_name, _cfg, _cloud, _log, _args): - print "hi" diff --git a/cloudinit/handlers/cc_grub_dpkg.py b/cloudinit/handlers/cc_grub_dpkg.py deleted file mode 100644 index 9f3a7eaf..00000000 --- a/cloudinit/handlers/cc_grub_dpkg.py +++ /dev/null @@ -1,64 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import traceback -import os - - -def handle(_name, cfg, _cloud, log, _args): - idevs = None - idevs_empty = None - - if "grub-dpkg" in cfg: - idevs = util.get_cfg_option_str(cfg["grub-dpkg"], - "grub-pc/install_devices", None) - idevs_empty = util.get_cfg_option_str(cfg["grub-dpkg"], - "grub-pc/install_devices_empty", None) - - if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): - if idevs == None: - idevs = "" - if idevs_empty == None: - idevs_empty = "true" - else: - if idevs_empty == None: - idevs_empty = "false" - if idevs == None: - idevs = "/dev/sda" - for dev in ("/dev/sda", "/dev/vda", "/dev/sda1", "/dev/vda1"): - if os.path.exists(dev): - idevs = dev - break - - # now idevs and idevs_empty are set to determined values - # or, those set by user - - dconf_sel = "grub-pc grub-pc/install_devices string %s\n" % idevs + \ - "grub-pc grub-pc/install_devices_empty boolean %s\n" % idevs_empty - log.debug("setting grub debconf-set-selections with '%s','%s'" % - (idevs, idevs_empty)) - - try: - util.subp(('debconf-set-selections'), dconf_sel) - except: - log.error("Failed to run debconf-set-selections for grub-dpkg") - log.debug(traceback.format_exc()) diff --git a/cloudinit/handlers/cc_keys_to_console.py b/cloudinit/handlers/cc_keys_to_console.py deleted file mode 100644 index 73a477c0..00000000 --- a/cloudinit/handlers/cc_keys_to_console.py +++ /dev/null @@ -1,42 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.CloudConfig import per_instance -import cloudinit.util as util -import subprocess - -frequency = per_instance - - -def handle(_name, cfg, _cloud, log, _args): - cmd = ['/usr/lib/cloud-init/write-ssh-key-fingerprints'] - fp_blacklist = util.get_cfg_option_list_or_str(cfg, - "ssh_fp_console_blacklist", []) - key_blacklist = util.get_cfg_option_list_or_str(cfg, - "ssh_key_console_blacklist", ["ssh-dss"]) - try: - confp = open('/dev/console', "wb") - cmd.append(','.join(fp_blacklist)) - cmd.append(','.join(key_blacklist)) - subprocess.call(cmd, stdout=confp) - confp.close() - except: - log.warn("writing keys to console value") - raise diff --git a/cloudinit/handlers/cc_landscape.py b/cloudinit/handlers/cc_landscape.py deleted file mode 100644 index a4113cbe..00000000 --- a/cloudinit/handlers/cc_landscape.py +++ /dev/null @@ -1,75 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import os.path -from cloudinit.CloudConfig import per_instance -from configobj import ConfigObj - -frequency = per_instance - -lsc_client_cfg_file = "/etc/landscape/client.conf" - -# defaults taken from stock client.conf in landscape-client 11.07.1.1-0ubuntu2 -lsc_builtincfg = { - 'client': { - 'log_level': "info", - 'url': "https://landscape.canonical.com/message-system", - 'ping_url': "http://landscape.canonical.com/ping", - 'data_path': "/var/lib/landscape/client", - } -} - - -def handle(_name, cfg, _cloud, log, _args): - """ - Basically turn a top level 'landscape' entry with a 'client' dict - and render it to ConfigObj format under '[client]' section in - /etc/landscape/client.conf - """ - - ls_cloudcfg = cfg.get("landscape", {}) - - if not isinstance(ls_cloudcfg, dict): - raise(Exception("'landscape' existed in config, but not a dict")) - - merged = mergeTogether([lsc_builtincfg, lsc_client_cfg_file, ls_cloudcfg]) - - if not os.path.isdir(os.path.dirname(lsc_client_cfg_file)): - os.makedirs(os.path.dirname(lsc_client_cfg_file)) - - with open(lsc_client_cfg_file, "w") as fp: - merged.write(fp) - - log.debug("updated %s" % lsc_client_cfg_file) - - -def mergeTogether(objs): - """ - merge together ConfigObj objects or things that ConfigObj() will take in - later entries override earlier - """ - cfg = ConfigObj({}) - for obj in objs: - if isinstance(obj, ConfigObj): - cfg.merge(obj) - else: - cfg.merge(ConfigObj(obj)) - return cfg diff --git a/cloudinit/handlers/cc_locale.py b/cloudinit/handlers/cc_locale.py deleted file mode 100644 index 2bb22fdb..00000000 --- a/cloudinit/handlers/cc_locale.py +++ /dev/null @@ -1,54 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import os.path -import subprocess -import traceback - - -def apply_locale(locale, cfgfile): - if os.path.exists('/usr/sbin/locale-gen'): - subprocess.Popen(['locale-gen', locale]).communicate() - if os.path.exists('/usr/sbin/update-locale'): - subprocess.Popen(['update-locale', locale]).communicate() - - util.render_to_file('default-locale', cfgfile, {'locale': locale}) - - -def handle(_name, cfg, cloud, log, args): - if len(args) != 0: - locale = args[0] - else: - locale = util.get_cfg_option_str(cfg, "locale", cloud.get_locale()) - - locale_cfgfile = util.get_cfg_option_str(cfg, "locale_configfile", - "/etc/default/locale") - - if not locale: - return - - log.debug("setting locale to %s" % locale) - - try: - apply_locale(locale, locale_cfgfile) - except Exception as e: - log.debug(traceback.format_exc(e)) - raise Exception("failed to apply locale %s" % locale) diff --git a/cloudinit/handlers/cc_mcollective.py b/cloudinit/handlers/cc_mcollective.py deleted file mode 100644 index a2a6230c..00000000 --- a/cloudinit/handlers/cc_mcollective.py +++ /dev/null @@ -1,99 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Marc Cluet -# Based on code by Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import subprocess -import StringIO -import ConfigParser -import cloudinit.CloudConfig as cc -import cloudinit.util as util - -pubcert_file = "/etc/mcollective/ssl/server-public.pem" -pricert_file = "/etc/mcollective/ssl/server-private.pem" - - -# Our fake header section -class FakeSecHead(object): - def __init__(self, fp): - self.fp = fp - self.sechead = '[nullsection]\n' - - def readline(self): - if self.sechead: - try: - return self.sechead - finally: - self.sechead = None - else: - return self.fp.readline() - - -def handle(_name, cfg, _cloud, _log, _args): - # If there isn't a mcollective key in the configuration don't do anything - if 'mcollective' not in cfg: - return - mcollective_cfg = cfg['mcollective'] - # Start by installing the mcollective package ... - cc.install_packages(("mcollective",)) - - # ... and then update the mcollective configuration - if 'conf' in mcollective_cfg: - # Create object for reading server.cfg values - mcollective_config = ConfigParser.ConfigParser() - # Read server.cfg values from original file in order to be able to mix - # the rest up - mcollective_config.readfp(FakeSecHead(open('/etc/mcollective/' - 'server.cfg'))) - for cfg_name, cfg in mcollective_cfg['conf'].iteritems(): - if cfg_name == 'public-cert': - util.write_file(pubcert_file, cfg, mode=0644) - mcollective_config.set(cfg_name, - 'plugin.ssl_server_public', pubcert_file) - mcollective_config.set(cfg_name, 'securityprovider', 'ssl') - elif cfg_name == 'private-cert': - util.write_file(pricert_file, cfg, mode=0600) - mcollective_config.set(cfg_name, - 'plugin.ssl_server_private', pricert_file) - mcollective_config.set(cfg_name, 'securityprovider', 'ssl') - else: - # Iterate throug the config items, we'll use ConfigParser.set - # to overwrite or create new items as needed - for o, v in cfg.iteritems(): - mcollective_config.set(cfg_name, o, v) - # We got all our config as wanted we'll rename - # the previous server.cfg and create our new one - os.rename('/etc/mcollective/server.cfg', - '/etc/mcollective/server.cfg.old') - outputfile = StringIO.StringIO() - mcollective_config.write(outputfile) - # Now we got the whole file, write to disk except first line - # Note below, that we've just used ConfigParser because it generally - # works. Below, we remove the initial 'nullsection' header - # and then change 'key = value' to 'key: value'. The global - # search and replace of '=' with ':' could be problematic though. - # this most likely needs fixing. - util.write_file('/etc/mcollective/server.cfg', - outputfile.getvalue().replace('[nullsection]\n', '').replace(' =', - ':'), - mode=0644) - - # Start mcollective - subprocess.check_call(['service', 'mcollective', 'start']) diff --git a/cloudinit/handlers/cc_mounts.py b/cloudinit/handlers/cc_mounts.py deleted file mode 100644 index 6cdd74e8..00000000 --- a/cloudinit/handlers/cc_mounts.py +++ /dev/null @@ -1,179 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import os -import re -from string import whitespace # pylint: disable=W0402 - - -def is_mdname(name): - # return true if this is a metadata service name - if name in ["ami", "root", "swap"]: - return True - # names 'ephemeral0' or 'ephemeral1' - # 'ebs[0-9]' appears when '--block-device-mapping sdf=snap-d4d90bbc' - for enumname in ("ephemeral", "ebs"): - if name.startswith(enumname) and name.find(":") == -1: - return True - return False - - -def handle(_name, cfg, cloud, log, _args): - # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno - defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] - defvals = cfg.get("mount_default_fields", defvals) - - # these are our default set of mounts - defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"], - ["swap", "none", "swap", "sw", "0", "0"]] - - cfgmnt = [] - if "mounts" in cfg: - cfgmnt = cfg["mounts"] - - # shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 - shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" - shortname = re.compile(shortname_filter) - - for i in range(len(cfgmnt)): - # skip something that wasn't a list - if not isinstance(cfgmnt[i], list): - continue - - # workaround, allow user to specify 'ephemeral' - # rather than more ec2 correct 'ephemeral0' - if cfgmnt[i][0] == "ephemeral": - cfgmnt[i][0] = "ephemeral0" - - if is_mdname(cfgmnt[i][0]): - newname = cloud.device_name_to_device(cfgmnt[i][0]) - if not newname: - log.debug("ignoring nonexistant named mount %s" % cfgmnt[i][0]) - cfgmnt[i][1] = None - else: - if newname.startswith("/"): - cfgmnt[i][0] = newname - else: - cfgmnt[i][0] = "/dev/%s" % newname - else: - if shortname.match(cfgmnt[i][0]): - cfgmnt[i][0] = "/dev/%s" % cfgmnt[i][0] - - # in case the user did not quote a field (likely fs-freq, fs_passno) - # but do not convert None to 'None' (LP: #898365) - for j in range(len(cfgmnt[i])): - if isinstance(cfgmnt[i][j], int): - cfgmnt[i][j] = str(cfgmnt[i][j]) - - for i in range(len(cfgmnt)): - # fill in values with defaults from defvals above - for j in range(len(defvals)): - if len(cfgmnt[i]) <= j: - cfgmnt[i].append(defvals[j]) - elif cfgmnt[i][j] is None: - cfgmnt[i][j] = defvals[j] - - # if the second entry in the list is 'None' this - # clears all previous entries of that same 'fs_spec' - # (fs_spec is the first field in /etc/fstab, ie, that device) - if cfgmnt[i][1] is None: - for j in range(i): - if cfgmnt[j][0] == cfgmnt[i][0]: - cfgmnt[j][1] = None - - # for each of the "default" mounts, add them only if no other - # entry has the same device name - for defmnt in defmnts: - devname = cloud.device_name_to_device(defmnt[0]) - if devname is None: - continue - if devname.startswith("/"): - defmnt[0] = devname - else: - defmnt[0] = "/dev/%s" % devname - - cfgmnt_has = False - for cfgm in cfgmnt: - if cfgm[0] == defmnt[0]: - cfgmnt_has = True - break - - if cfgmnt_has: - continue - cfgmnt.append(defmnt) - - # now, each entry in the cfgmnt list has all fstab values - # if the second field is None (not the string, the value) we skip it - actlist = [x for x in cfgmnt if x[1] is not None] - - if len(actlist) == 0: - return - - comment = "comment=cloudconfig" - cc_lines = [] - needswap = False - dirs = [] - for line in actlist: - # write 'comment' in the fs_mntops, entry, claiming this - line[3] = "%s,comment=cloudconfig" % line[3] - if line[2] == "swap": - needswap = True - if line[1].startswith("/"): - dirs.append(line[1]) - cc_lines.append('\t'.join(line)) - - fstab_lines = [] - fstab = open("/etc/fstab", "r+") - ws = re.compile("[%s]+" % whitespace) - for line in fstab.read().splitlines(): - try: - toks = ws.split(line) - if toks[3].find(comment) != -1: - continue - except: - pass - fstab_lines.append(line) - - fstab_lines.extend(cc_lines) - - fstab.seek(0) - fstab.write("%s\n" % '\n'.join(fstab_lines)) - fstab.truncate() - fstab.close() - - if needswap: - try: - util.subp(("swapon", "-a")) - except: - log.warn("Failed to enable swap") - - for d in dirs: - if os.path.exists(d): - continue - try: - os.makedirs(d) - except: - log.warn("Failed to make '%s' config-mount\n", d) - - try: - util.subp(("mount", "-a")) - except: - log.warn("'mount -a' failed") diff --git a/cloudinit/handlers/cc_phone_home.py b/cloudinit/handlers/cc_phone_home.py deleted file mode 100644 index a7ff74e1..00000000 --- a/cloudinit/handlers/cc_phone_home.py +++ /dev/null @@ -1,106 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -from cloudinit.CloudConfig import per_instance -import cloudinit.util as util -from time import sleep - -frequency = per_instance -post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', - 'hostname'] - - -# phone_home: -# url: http://my.foo.bar/$INSTANCE/ -# post: all -# tries: 10 -# -# phone_home: -# url: http://my.foo.bar/$INSTANCE_ID/ -# post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id -# -def handle(_name, cfg, cloud, log, args): - if len(args) != 0: - ph_cfg = util.read_conf(args[0]) - else: - if not 'phone_home' in cfg: - return - ph_cfg = cfg['phone_home'] - - if 'url' not in ph_cfg: - log.warn("no 'url' token in phone_home") - return - - url = ph_cfg['url'] - post_list = ph_cfg.get('post', 'all') - tries = ph_cfg.get('tries', 10) - try: - tries = int(tries) - except: - log.warn("tries is not an integer. using 10") - tries = 10 - - if post_list == "all": - post_list = post_list_all - - all_keys = {} - all_keys['instance_id'] = cloud.get_instance_id() - all_keys['hostname'] = cloud.get_hostname() - - pubkeys = { - 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', - 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', - 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', - } - - for n, path in pubkeys.iteritems(): - try: - fp = open(path, "rb") - all_keys[n] = fp.read() - fp.close() - except: - log.warn("%s: failed to open in phone_home" % path) - - submit_keys = {} - for k in post_list: - if k in all_keys: - submit_keys[k] = all_keys[k] - else: - submit_keys[k] = "N/A" - log.warn("requested key %s from 'post' list not available") - - url = util.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) - - null_exc = object() - last_e = null_exc - for i in range(0, tries): - try: - util.readurl(url, submit_keys) - log.debug("succeeded submit to %s on try %i" % (url, i + 1)) - return - except Exception as e: - log.debug("failed to post to %s on try %i" % (url, i + 1)) - last_e = e - sleep(3) - - log.warn("failed to post to %s in %i tries" % (url, tries)) - if last_e is not null_exc: - raise(last_e) - - return diff --git a/cloudinit/handlers/cc_puppet.py b/cloudinit/handlers/cc_puppet.py deleted file mode 100644 index 6fc475f6..00000000 --- a/cloudinit/handlers/cc_puppet.py +++ /dev/null @@ -1,108 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import os.path -import pwd -import socket -import subprocess -import StringIO -import ConfigParser -import cloudinit.CloudConfig as cc -import cloudinit.util as util - - -def handle(_name, cfg, cloud, log, _args): - # If there isn't a puppet key in the configuration don't do anything - if 'puppet' not in cfg: - return - puppet_cfg = cfg['puppet'] - # Start by installing the puppet package ... - cc.install_packages(("puppet",)) - - # ... and then update the puppet configuration - if 'conf' in puppet_cfg: - # Add all sections from the conf object to puppet.conf - puppet_conf_fh = open('/etc/puppet/puppet.conf', 'r') - # Create object for reading puppet.conf values - puppet_config = ConfigParser.ConfigParser() - # Read puppet.conf values from original file in order to be able to - # mix the rest up - puppet_config.readfp(StringIO.StringIO(''.join(i.lstrip() for i in - puppet_conf_fh.readlines()))) - # Close original file, no longer needed - puppet_conf_fh.close() - for cfg_name, cfg in puppet_cfg['conf'].iteritems(): - # ca_cert configuration is a special case - # Dump the puppetmaster ca certificate in the correct place - if cfg_name == 'ca_cert': - # Puppet ssl sub-directory isn't created yet - # Create it with the proper permissions and ownership - os.makedirs('/var/lib/puppet/ssl') - os.chmod('/var/lib/puppet/ssl', 0771) - os.chown('/var/lib/puppet/ssl', - pwd.getpwnam('puppet').pw_uid, 0) - os.makedirs('/var/lib/puppet/ssl/certs/') - os.chown('/var/lib/puppet/ssl/certs/', - pwd.getpwnam('puppet').pw_uid, 0) - ca_fh = open('/var/lib/puppet/ssl/certs/ca.pem', 'w') - ca_fh.write(cfg) - ca_fh.close() - os.chown('/var/lib/puppet/ssl/certs/ca.pem', - pwd.getpwnam('puppet').pw_uid, 0) - util.restorecon_if_possible('/var/lib/puppet', recursive=True) - else: - #puppet_conf_fh.write("\n[%s]\n" % (cfg_name)) - # If puppet.conf already has this section we don't want to - # write it again - if puppet_config.has_section(cfg_name) == False: - puppet_config.add_section(cfg_name) - # Iterate throug the config items, we'll use ConfigParser.set - # to overwrite or create new items as needed - for o, v in cfg.iteritems(): - if o == 'certname': - # Expand %f as the fqdn - v = v.replace("%f", socket.getfqdn()) - # Expand %i as the instance id - v = v.replace("%i", - cloud.datasource.get_instance_id()) - # certname needs to be downcase - v = v.lower() - puppet_config.set(cfg_name, o, v) - #puppet_conf_fh.write("%s=%s\n" % (o, v)) - # We got all our config as wanted we'll rename - # the previous puppet.conf and create our new one - os.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') - with open('/etc/puppet/puppet.conf', 'wb') as configfile: - puppet_config.write(configfile) - util.restorecon_if_possible('/etc/puppet/puppet.conf') - # Set puppet to automatically start - if os.path.exists('/etc/default/puppet'): - subprocess.check_call(['sed', '-i', - '-e', 's/^START=.*/START=yes/', - '/etc/default/puppet']) - elif os.path.exists('/bin/systemctl'): - subprocess.check_call(['/bin/systemctl', 'enable', 'puppet.service']) - elif os.path.exists('/sbin/chkconfig'): - subprocess.check_call(['/sbin/chkconfig', 'puppet', 'on']) - else: - log.warn("Do not know how to enable puppet service on this system") - # Start puppetd - subprocess.check_call(['service', 'puppet', 'start']) diff --git a/cloudinit/handlers/cc_resizefs.py b/cloudinit/handlers/cc_resizefs.py deleted file mode 100644 index 2dc66def..00000000 --- a/cloudinit/handlers/cc_resizefs.py +++ /dev/null @@ -1,108 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import os -import stat -import sys -import time -import tempfile -from cloudinit.CloudConfig import per_always - -frequency = per_always - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - resize_root = False - if str(args[0]).lower() in ['true', '1', 'on', 'yes']: - resize_root = True - else: - resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) - - if str(resize_root).lower() in ['false', '0']: - return - - # we use mktemp rather than mkstemp because early in boot nothing - # else should be able to race us for this, and we need to mknod. - devpth = tempfile.mktemp(prefix="cloudinit.resizefs.", dir="/run") - - try: - st_dev = os.stat("/").st_dev - dev = os.makedev(os.major(st_dev), os.minor(st_dev)) - os.mknod(devpth, 0400 | stat.S_IFBLK, dev) - except: - if util.is_container(): - log.debug("inside container, ignoring mknod failure in resizefs") - return - log.warn("Failed to make device node to resize /") - raise - - cmd = ['blkid', '-c', '/dev/null', '-sTYPE', '-ovalue', devpth] - try: - (fstype, _err) = util.subp(cmd) - except subprocess.CalledProcessError as e: - log.warn("Failed to get filesystem type of maj=%s, min=%s via: %s" % - (os.major(st_dev), os.minor(st_dev), cmd)) - log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) - os.unlink(devpth) - raise - - if str(fstype).startswith("ext"): - resize_cmd = ['resize2fs', devpth] - elif fstype == "xfs": - resize_cmd = ['xfs_growfs', devpth] - else: - os.unlink(devpth) - log.debug("not resizing unknown filesystem %s" % fstype) - return - - if resize_root == "noblock": - fid = os.fork() - if fid == 0: - try: - do_resize(resize_cmd, devpth, log) - os._exit(0) # pylint: disable=W0212 - except Exception as exc: - sys.stderr.write("Failed: %s" % exc) - os._exit(1) # pylint: disable=W0212 - else: - do_resize(resize_cmd, devpth, log) - - log.debug("resizing root filesystem (type=%s, maj=%i, min=%i, val=%s)" % - (str(fstype).rstrip("\n"), os.major(st_dev), os.minor(st_dev), - resize_root)) - - return - - -def do_resize(resize_cmd, devpth, log): - try: - start = time.time() - util.subp(resize_cmd) - except subprocess.CalledProcessError as e: - log.warn("Failed to resize filesystem (%s)" % resize_cmd) - log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) - os.unlink(devpth) - raise - - os.unlink(devpth) - log.debug("resize took %s seconds" % (time.time() - start)) diff --git a/cloudinit/handlers/cc_rightscale_userdata.py b/cloudinit/handlers/cc_rightscale_userdata.py deleted file mode 100644 index 5ed0848f..00000000 --- a/cloudinit/handlers/cc_rightscale_userdata.py +++ /dev/null @@ -1,78 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -## -## The purpose of this script is to allow cloud-init to consume -## rightscale style userdata. rightscale user data is key-value pairs -## in a url-query-string like format. -## -## for cloud-init support, there will be a key named -## 'CLOUD_INIT_REMOTE_HOOK'. -## -## This cloud-config module will -## - read the blob of data from raw user data, and parse it as key/value -## - for each key that is found, download the content to -## the local instance/scripts directory and set them executable. -## - the files in that directory will be run by the user-scripts module -## Therefore, this must run before that. -## -## - -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_ipath_cur -from urlparse import parse_qs - -frequency = per_instance -my_name = "cc_rightscale_userdata" -my_hookname = 'CLOUD_INIT_REMOTE_HOOK' - - -def handle(_name, _cfg, cloud, log, _args): - try: - ud = cloud.get_userdata_raw() - except: - log.warn("failed to get raw userdata in %s" % my_name) - return - - try: - mdict = parse_qs(ud) - if not my_hookname in mdict: - return - except: - log.warn("failed to urlparse.parse_qa(userdata_raw())") - raise - - scripts_d = get_ipath_cur('scripts') - i = 0 - first_e = None - for url in mdict[my_hookname]: - fname = "%s/rightscale-%02i" % (scripts_d, i) - i = i + 1 - try: - content = util.readurl(url) - util.write_file(fname, content, mode=0700) - except Exception as e: - if not first_e: - first_e = None - log.warn("%s failed to read %s: %s" % (my_name, url, e)) - - if first_e: - raise(e) diff --git a/cloudinit/handlers/cc_rsyslog.py b/cloudinit/handlers/cc_rsyslog.py deleted file mode 100644 index ac7f2c74..00000000 --- a/cloudinit/handlers/cc_rsyslog.py +++ /dev/null @@ -1,101 +0,0 @@ -# vi: ts=4 expandtab syntax=python -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit -import logging -import cloudinit.util as util -import traceback - -DEF_FILENAME = "20-cloud-config.conf" -DEF_DIR = "/etc/rsyslog.d" - - -def handle(_name, cfg, _cloud, log, _args): - # rsyslog: - # - "*.* @@192.158.1.1" - # - content: "*.* @@192.0.2.1:10514" - # - filename: 01-examplecom.conf - # content: | - # *.* @@syslogd.example.com - - # process 'rsyslog' - if not 'rsyslog' in cfg: - return - - def_dir = cfg.get('rsyslog_dir', DEF_DIR) - def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) - - files = [] - elst = [] - for ent in cfg['rsyslog']: - if isinstance(ent, dict): - if not "content" in ent: - elst.append((ent, "no 'content' entry")) - continue - content = ent['content'] - filename = ent.get("filename", def_fname) - else: - content = ent - filename = def_fname - - if not filename.startswith("/"): - filename = "%s/%s" % (def_dir, filename) - - omode = "ab" - # truncate filename first time you see it - if filename not in files: - omode = "wb" - files.append(filename) - - try: - util.write_file(filename, content + "\n", omode=omode) - except Exception as e: - log.debug(traceback.format_exc(e)) - elst.append((content, "failed to write to %s" % filename)) - - # need to restart syslogd - restarted = False - try: - # if this config module is running at cloud-init time - # (before rsyslog is running) we don't actually have to - # restart syslog. - # - # upstart actually does what we want here, in that it doesn't - # start a service that wasn't running already on 'restart' - # it will also return failure on the attempt, so 'restarted' - # won't get set - log.debug("restarting rsyslog") - util.subp(['service', 'rsyslog', 'restart']) - restarted = True - - except Exception as e: - elst.append(("restart", str(e))) - - if restarted: - # this only needs to run if we *actually* restarted - # syslog above. - cloudinit.logging_set_from_cfg_file() - log = logging.getLogger() - log.debug("rsyslog configured %s" % files) - - for e in elst: - log.warn("rsyslog error: %s\n" % ':'.join(e)) - - return diff --git a/cloudinit/handlers/cc_runcmd.py b/cloudinit/handlers/cc_runcmd.py deleted file mode 100644 index f7e8c671..00000000 --- a/cloudinit/handlers/cc_runcmd.py +++ /dev/null @@ -1,32 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util - - -def handle(_name, cfg, cloud, log, _args): - if "runcmd" not in cfg: - return - outfile = "%s/runcmd" % cloud.get_ipath('scripts') - try: - content = util.shellify(cfg["runcmd"]) - util.write_file(outfile, content, 0700) - except: - log.warn("failed to open %s for runcmd" % outfile) diff --git a/cloudinit/handlers/cc_salt_minion.py b/cloudinit/handlers/cc_salt_minion.py deleted file mode 100644 index 1a3b5039..00000000 --- a/cloudinit/handlers/cc_salt_minion.py +++ /dev/null @@ -1,56 +0,0 @@ -# vi: ts=4 expandtab -# -# Author: Jeff Bauer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import os.path -import subprocess -import cloudinit.CloudConfig as cc -import yaml - - -def handle(_name, cfg, _cloud, _log, _args): - # If there isn't a salt key in the configuration don't do anything - if 'salt_minion' not in cfg: - return - salt_cfg = cfg['salt_minion'] - # Start by installing the salt package ... - cc.install_packages(("salt",)) - config_dir = '/etc/salt' - if not os.path.isdir(config_dir): - os.makedirs(config_dir) - # ... and then update the salt configuration - if 'conf' in salt_cfg: - # Add all sections from the conf object to /etc/salt/minion - minion_config = os.path.join(config_dir, 'minion') - yaml.dump(salt_cfg['conf'], - file(minion_config, 'w'), - default_flow_style=False) - # ... copy the key pair if specified - if 'public_key' in salt_cfg and 'private_key' in salt_cfg: - pki_dir = '/etc/salt/pki' - cumask = os.umask(077) - if not os.path.isdir(pki_dir): - os.makedirs(pki_dir) - pub_name = os.path.join(pki_dir, 'minion.pub') - pem_name = os.path.join(pki_dir, 'minion.pem') - with open(pub_name, 'w') as f: - f.write(salt_cfg['public_key']) - with open(pem_name, 'w') as f: - f.write(salt_cfg['private_key']) - os.umask(cumask) - - # Start salt-minion - subprocess.check_call(['service', 'salt-minion', 'start']) diff --git a/cloudinit/handlers/cc_scripts_per_boot.py b/cloudinit/handlers/cc_scripts_per_boot.py deleted file mode 100644 index 41a74754..00000000 --- a/cloudinit/handlers/cc_scripts_per_boot.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_always -from cloudinit import get_cpath - -frequency = per_always -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-boot") - - -def handle(_name, _cfg, _cloud, log, _args): - try: - util.runparts(runparts_path) - except: - log.warn("failed to run-parts in %s" % runparts_path) - raise diff --git a/cloudinit/handlers/cc_scripts_per_instance.py b/cloudinit/handlers/cc_scripts_per_instance.py deleted file mode 100644 index a2981eab..00000000 --- a/cloudinit/handlers/cc_scripts_per_instance.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_cpath - -frequency = per_instance -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-instance") - - -def handle(_name, _cfg, _cloud, log, _args): - try: - util.runparts(runparts_path) - except: - log.warn("failed to run-parts in %s" % runparts_path) - raise diff --git a/cloudinit/handlers/cc_scripts_per_once.py b/cloudinit/handlers/cc_scripts_per_once.py deleted file mode 100644 index a69151da..00000000 --- a/cloudinit/handlers/cc_scripts_per_once.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_once -from cloudinit import get_cpath - -frequency = per_once -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-once") - - -def handle(_name, _cfg, _cloud, log, _args): - try: - util.runparts(runparts_path) - except: - log.warn("failed to run-parts in %s" % runparts_path) - raise diff --git a/cloudinit/handlers/cc_scripts_user.py b/cloudinit/handlers/cc_scripts_user.py deleted file mode 100644 index 933aa4e0..00000000 --- a/cloudinit/handlers/cc_scripts_user.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_ipath_cur - -frequency = per_instance -runparts_path = "%s/%s" % (get_ipath_cur(), "scripts") - - -def handle(_name, _cfg, _cloud, log, _args): - try: - util.runparts(runparts_path) - except: - log.warn("failed to run-parts in %s" % runparts_path) - raise diff --git a/cloudinit/handlers/cc_set_hostname.py b/cloudinit/handlers/cc_set_hostname.py deleted file mode 100644 index acea74d9..00000000 --- a/cloudinit/handlers/cc_set_hostname.py +++ /dev/null @@ -1,42 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util - - -def handle(_name, cfg, cloud, log, _args): - if util.get_cfg_option_bool(cfg, "preserve_hostname", False): - log.debug("preserve_hostname is set. not setting hostname") - return(True) - - (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) - try: - set_hostname(hostname, log) - except Exception: - util.logexc(log) - log.warn("failed to set hostname to %s\n", hostname) - - return(True) - - -def set_hostname(hostname, log): - util.subp(['hostname', hostname]) - util.write_file("/etc/hostname", "%s\n" % hostname, 0644) - log.debug("populated /etc/hostname with %s on first boot", hostname) diff --git a/cloudinit/handlers/cc_set_passwords.py b/cloudinit/handlers/cc_set_passwords.py deleted file mode 100644 index 9d0bbdb8..00000000 --- a/cloudinit/handlers/cc_set_passwords.py +++ /dev/null @@ -1,129 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import sys -import random -from string import letters, digits # pylint: disable=W0402 - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - # if run from command line, and give args, wipe the chpasswd['list'] - password = args[0] - if 'chpasswd' in cfg and 'list' in cfg['chpasswd']: - del cfg['chpasswd']['list'] - else: - password = util.get_cfg_option_str(cfg, "password", None) - - expire = True - pw_auth = "no" - change_pwauth = False - plist = None - - if 'chpasswd' in cfg: - chfg = cfg['chpasswd'] - plist = util.get_cfg_option_str(chfg, 'list', plist) - expire = util.get_cfg_option_bool(chfg, 'expire', expire) - - if not plist and password: - user = util.get_cfg_option_str(cfg, "user", "ubuntu") - plist = "%s:%s" % (user, password) - - errors = [] - if plist: - plist_in = [] - randlist = [] - users = [] - for line in plist.splitlines(): - u, p = line.split(':', 1) - if p == "R" or p == "RANDOM": - p = rand_user_password() - randlist.append("%s:%s" % (u, p)) - plist_in.append("%s:%s" % (u, p)) - users.append(u) - - ch_in = '\n'.join(plist_in) - try: - util.subp(['chpasswd'], ch_in) - log.debug("changed password for %s:" % users) - except Exception as e: - errors.append(e) - log.warn("failed to set passwords with chpasswd: %s" % e) - - if len(randlist): - sys.stdout.write("%s\n%s\n" % ("Set the following passwords\n", - '\n'.join(randlist))) - - if expire: - enum = len(errors) - for u in users: - try: - util.subp(['passwd', '--expire', u]) - except Exception as e: - errors.append(e) - log.warn("failed to expire account for %s" % u) - if enum == len(errors): - log.debug("expired passwords for: %s" % u) - - if 'ssh_pwauth' in cfg: - val = str(cfg['ssh_pwauth']).lower() - if val in ("true", "1", "yes"): - pw_auth = "yes" - change_pwauth = True - elif val in ("false", "0", "no"): - pw_auth = "no" - change_pwauth = True - else: - change_pwauth = False - - if change_pwauth: - pa_s = "\(#*\)\(PasswordAuthentication[[:space:]]\+\)\(yes\|no\)" - msg = "set PasswordAuthentication to '%s'" % pw_auth - try: - cmd = ['sed', '-i', 's,%s,\\2%s,' % (pa_s, pw_auth), - '/etc/ssh/sshd_config'] - util.subp(cmd) - log.debug(msg) - except Exception as e: - log.warn("failed %s" % msg) - errors.append(e) - - try: - p = util.subp(['service', cfg.get('ssh_svcname', 'ssh'), - 'restart']) - log.debug("restarted sshd") - except: - log.warn("restart of ssh failed") - - if len(errors): - raise(errors[0]) - - return - - -def rand_str(strlen=32, select_from=letters + digits): - return("".join([random.choice(select_from) for _x in range(0, strlen)])) - - -def rand_user_password(pwlen=9): - selfrom = (letters.translate(None, 'loLOI') + - digits.translate(None, '01')) - return(rand_str(pwlen, select_from=selfrom)) diff --git a/cloudinit/handlers/cc_ssh.py b/cloudinit/handlers/cc_ssh.py deleted file mode 100644 index 48eb58bc..00000000 --- a/cloudinit/handlers/cc_ssh.py +++ /dev/null @@ -1,106 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import cloudinit.SshUtil as sshutil -import os -import glob -import subprocess - -DISABLE_ROOT_OPTS = "no-port-forwarding,no-agent-forwarding," \ -"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " \ -"rather than the user \\\"root\\\".\';echo;sleep 10\"" - - -def handle(_name, cfg, cloud, log, _args): - - # remove the static keys from the pristine image - if cfg.get("ssh_deletekeys", True): - for f in glob.glob("/etc/ssh/ssh_host_*key*"): - try: - os.unlink(f) - except: - pass - - if "ssh_keys" in cfg: - # if there are keys in cloud-config, use them - key2file = { - "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), - "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), - "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), - "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), - "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), - "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), - } - - for key, val in cfg["ssh_keys"].items(): - if key in key2file: - util.write_file(key2file[key][0], val, key2file[key][1]) - - priv2pub = {'rsa_private': 'rsa_public', 'dsa_private': 'dsa_public', - 'ecdsa_private': 'ecdsa_public', } - - cmd = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' - for priv, pub in priv2pub.iteritems(): - if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: - continue - pair = (key2file[priv][0], key2file[pub][0]) - subprocess.call(('sh', '-xc', cmd % pair)) - log.debug("generated %s from %s" % pair) - else: - # if not, generate them - for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', - ['rsa', 'dsa', 'ecdsa']): - keyfile = '/etc/ssh/ssh_host_%s_key' % keytype - if not os.path.exists(keyfile): - subprocess.call(['ssh-keygen', '-t', keytype, '-N', '', - '-f', keyfile]) - - util.restorecon_if_possible('/etc/ssh', recursive=True) - - try: - user = util.get_cfg_option_str(cfg, 'user') - disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) - disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", - DISABLE_ROOT_OPTS) - keys = cloud.get_public_ssh_keys() - - if "ssh_authorized_keys" in cfg: - cfgkeys = cfg["ssh_authorized_keys"] - keys.extend(cfgkeys) - - apply_credentials(keys, user, disable_root, disable_root_opts, log) - except: - util.logexc(log) - log.warn("applying credentials failed!\n") - - -def apply_credentials(keys, user, disable_root, - disable_root_opts=DISABLE_ROOT_OPTS, log=None): - keys = set(keys) - if user: - sshutil.setup_user_keys(keys, user, '', log) - - if disable_root: - key_prefix = disable_root_opts.replace('$USER', user) - else: - key_prefix = '' - - sshutil.setup_user_keys(keys, 'root', key_prefix, log) diff --git a/cloudinit/handlers/cc_ssh_import_id.py b/cloudinit/handlers/cc_ssh_import_id.py deleted file mode 100644 index bbf5bd83..00000000 --- a/cloudinit/handlers/cc_ssh_import_id.py +++ /dev/null @@ -1,50 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import traceback - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - user = args[0] - ids = [] - if len(args) > 1: - ids = args[1:] - else: - user = util.get_cfg_option_str(cfg, "user", "ubuntu") - ids = util.get_cfg_option_list_or_str(cfg, "ssh_import_id", []) - - if len(ids) == 0: - return - - cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids - - log.debug("importing ssh ids. cmd = %s" % cmd) - - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) - except OSError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/handlers/cc_timezone.py b/cloudinit/handlers/cc_timezone.py deleted file mode 100644 index e5c9901b..00000000 --- a/cloudinit/handlers/cc_timezone.py +++ /dev/null @@ -1,67 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.CloudConfig import per_instance -from cloudinit import util -import os.path -import shutil - -frequency = per_instance -tz_base = "/usr/share/zoneinfo" - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - timezone = args[0] - else: - timezone = util.get_cfg_option_str(cfg, "timezone", False) - - if not timezone: - return - - tz_file = "%s/%s" % (tz_base, timezone) - - if not os.path.isfile(tz_file): - log.debug("Invalid timezone %s" % tz_file) - raise Exception("Invalid timezone %s" % tz_file) - - try: - fp = open("/etc/timezone", "wb") - fp.write("%s\n" % timezone) - fp.close() - except: - log.debug("failed to write to /etc/timezone") - raise - if os.path.exists("/etc/sysconfig/clock"): - try: - with open("/etc/sysconfig/clock", "w") as fp: - fp.write('ZONE="%s"\n' % timezone) - except: - log.debug("failed to write to /etc/sysconfig/clock") - raise - - try: - shutil.copy(tz_file, "/etc/localtime") - except: - log.debug("failed to copy %s to /etc/localtime" % tz_file) - raise - - log.debug("set timezone to %s" % timezone) - return diff --git a/cloudinit/handlers/cc_update_etc_hosts.py b/cloudinit/handlers/cc_update_etc_hosts.py deleted file mode 100644 index 6ad2fca8..00000000 --- a/cloudinit/handlers/cc_update_etc_hosts.py +++ /dev/null @@ -1,87 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_always -import StringIO - -frequency = per_always - - -def handle(_name, cfg, cloud, log, _args): - (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - - manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) - if manage_hosts in ("True", "true", True, "template"): - # render from template file - try: - if not hostname: - log.info("manage_etc_hosts was set, but no hostname found") - return - - util.render_to_file('hosts', '/etc/hosts', - {'hostname': hostname, 'fqdn': fqdn}) - except Exception: - log.warn("failed to update /etc/hosts") - raise - elif manage_hosts == "localhost": - log.debug("managing 127.0.1.1 in /etc/hosts") - update_etc_hosts(hostname, fqdn, log) - return - else: - if manage_hosts not in ("False", False): - log.warn("Unknown value for manage_etc_hosts. Assuming False") - else: - log.debug("not managing /etc/hosts") - - -def update_etc_hosts(hostname, fqdn, _log): - with open('/etc/hosts', 'r') as etchosts: - header = "# Added by cloud-init\n" - hosts_line = "127.0.1.1\t%s %s\n" % (fqdn, hostname) - need_write = False - need_change = True - new_etchosts = StringIO.StringIO() - for line in etchosts: - split_line = [s.strip() for s in line.split()] - if len(split_line) < 2: - new_etchosts.write(line) - continue - if line == header: - continue - ip, hosts = split_line[0], split_line[1:] - if ip == "127.0.1.1": - if sorted([hostname, fqdn]) == sorted(hosts): - need_change = False - if need_change == True: - line = "%s%s" % (header, hosts_line) - need_change = False - need_write = True - new_etchosts.write(line) - etchosts.close() - if need_change == True: - new_etchosts.write("%s%s" % (header, hosts_line)) - need_write = True - if need_write == True: - new_etcfile = open('/etc/hosts', 'wb') - new_etcfile.write(new_etchosts.getvalue()) - new_etcfile.close() - new_etchosts.close() - return diff --git a/cloudinit/handlers/cc_update_hostname.py b/cloudinit/handlers/cc_update_hostname.py deleted file mode 100644 index b9d1919a..00000000 --- a/cloudinit/handlers/cc_update_hostname.py +++ /dev/null @@ -1,101 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import errno -from cloudinit.CloudConfig import per_always - -frequency = per_always - - -def handle(_name, cfg, cloud, log, _args): - if util.get_cfg_option_bool(cfg, "preserve_hostname", False): - log.debug("preserve_hostname is set. not updating hostname") - return - - (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) - try: - prev = "%s/%s" % (cloud.get_cpath('data'), "previous-hostname") - update_hostname(hostname, prev, log) - except Exception: - log.warn("failed to set hostname\n") - raise - - -# read hostname from a 'hostname' file -# allow for comments and stripping line endings. -# if file doesn't exist, or no contents, return default -def read_hostname(filename, default=None): - try: - fp = open(filename, "r") - lines = fp.readlines() - fp.close() - for line in lines: - hpos = line.find("#") - if hpos != -1: - line = line[0:hpos] - line = line.rstrip() - if line: - return line - except IOError as e: - if e.errno != errno.ENOENT: - raise - return default - - -def update_hostname(hostname, prev_file, log): - etc_file = "/etc/hostname" - - hostname_prev = None - hostname_in_etc = None - - try: - hostname_prev = read_hostname(prev_file) - except Exception as e: - log.warn("Failed to open %s: %s" % (prev_file, e)) - - try: - hostname_in_etc = read_hostname(etc_file) - except: - log.warn("Failed to open %s" % etc_file) - - update_files = [] - if not hostname_prev or hostname_prev != hostname: - update_files.append(prev_file) - - if (not hostname_in_etc or - (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): - update_files.append(etc_file) - - try: - for fname in update_files: - util.write_file(fname, "%s\n" % hostname, 0644) - log.debug("wrote %s to %s" % (hostname, fname)) - except: - log.warn("failed to write hostname to %s" % fname) - - if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: - log.debug("%s differs from %s. assuming user maintained" % - (prev_file, etc_file)) - - if etc_file in update_files: - log.debug("setting hostname to %s" % hostname) - subprocess.Popen(['hostname', hostname]).communicate() diff --git a/cloudinit/transforms/__init__.py b/cloudinit/transforms/__init__.py new file mode 100644 index 00000000..5d70ac43 --- /dev/null +++ b/cloudinit/transforms/__init__.py @@ -0,0 +1,221 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2008-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Chuck Short +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import os +import subprocess +import sys +import time +import traceback + +import yaml + +from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE) + +from cloudinit import log as logging +from cloudinit import util + +LOG = logging.getLogger(__name__) + +DEF_HANDLER_VERSION = 1 +DEF_FREQ = PER_INSTANCE + + +# reads a cloudconfig module list, returns +# a 2 dimensional array suitable to pass to run_cc_modules +def read_cc_modules(cfg, name): + if name not in cfg: + return([]) + module_list = [] + # create 'module_list', an array of arrays + # where array[0] = config + # array[1] = freq + # array[2:] = arguemnts + for item in cfg[name]: + if isinstance(item, str): + module_list.append((item,)) + elif isinstance(item, list): + module_list.append(item) + else: + raise TypeError("failed to read '%s' item in config") + return(module_list) + + +def run_cc_modules(cc, module_list, log): + failures = [] + for cfg_mod in module_list: + name = cfg_mod[0] + freq = None + run_args = [] + if len(cfg_mod) > 1: + freq = cfg_mod[1] + if len(cfg_mod) > 2: + run_args = cfg_mod[2:] + + try: + log.debug("handling %s with freq=%s and args=%s" % + (name, freq, run_args)) + cc.handle(name, run_args, freq=freq) + except: + log.warn(traceback.format_exc()) + log.error("config handling of %s, %s, %s failed\n" % + (name, freq, run_args)) + failures.append(name) + + return(failures) + + +# always returns well formated values +# cfg is expected to have an entry 'output' in it, which is a dictionary +# that includes entries for 'init', 'config', 'final' or 'all' +# init: /var/log/cloud.out +# config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ] +# final: +# output: "| logger -p" +# error: "> /dev/null" +# this returns the specific 'mode' entry, cleanly formatted, with value +# None if if none is given +def get_output_cfg(cfg, mode="init"): + ret = [None, None] + if not 'output' in cfg: + return ret + + outcfg = cfg['output'] + if mode in outcfg: + modecfg = outcfg[mode] + else: + if 'all' not in outcfg: + return ret + # if there is a 'all' item in the output list + # then it applies to all users of this (init, config, final) + modecfg = outcfg['all'] + + # if value is a string, it specifies stdout and stderr + if isinstance(modecfg, str): + ret = [modecfg, modecfg] + + # if its a list, then we expect (stdout, stderr) + if isinstance(modecfg, list): + if len(modecfg) > 0: + ret[0] = modecfg[0] + if len(modecfg) > 1: + ret[1] = modecfg[1] + + # if it is a dictionary, expect 'out' and 'error' + # items, which indicate out and error + if isinstance(modecfg, dict): + if 'output' in modecfg: + ret[0] = modecfg['output'] + if 'error' in modecfg: + ret[1] = modecfg['error'] + + # if err's entry == "&1", then make it same as stdout + # as in shell syntax of "echo foo >/dev/null 2>&1" + if ret[1] == "&1": + ret[1] = ret[0] + + swlist = [">>", ">", "|"] + for i in range(len(ret)): + if not ret[i]: + continue + val = ret[i].lstrip() + found = False + for s in swlist: + if val.startswith(s): + val = "%s %s" % (s, val[len(s):].strip()) + found = True + break + if not found: + # default behavior is append + val = "%s %s" % (">>", val.strip()) + ret[i] = val + + return(ret) + + +# redirect_output(outfmt, errfmt, orig_out, orig_err) +# replace orig_out and orig_err with filehandles specified in outfmt or errfmt +# fmt can be: +# > FILEPATH +# >> FILEPATH +# | program [ arg1 [ arg2 [ ... ] ] ] +# +# with a '|', arguments are passed to shell, so one level of +# shell escape is required. +def redirect_output(outfmt, errfmt, o_out=sys.stdout, o_err=sys.stderr): + if outfmt: + (mode, arg) = outfmt.split(" ", 1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": + owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) + new_fp = proc.stdin + else: + raise TypeError("invalid type for outfmt: %s" % outfmt) + + if o_out: + os.dup2(new_fp.fileno(), o_out.fileno()) + if errfmt == outfmt: + os.dup2(new_fp.fileno(), o_err.fileno()) + return + + if errfmt: + (mode, arg) = errfmt.split(" ", 1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": + owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) + new_fp = proc.stdin + else: + raise TypeError("invalid type for outfmt: %s" % outfmt) + + if o_err: + os.dup2(new_fp.fileno(), o_err.fileno()) + return + + +def form_module_name(name): + canon_name = name.replace("-", "_") + if canon_name.endswith(".py"): + canon_name = canon_name[0:(len(canon_name) - 3)] + canon_name = canon_name.strip() + if not canon_name: + return None + if not canon_name.startswith("cc_"): + canon_name = 'cc_%s' % (canon_name) + return canon_name + + +def fixup_module(mod): + freq = getattr(mod, "frequency", None) + if not freq: + setattr(mod, 'frequency', PER_INSTANCE) + handler = getattr(mod, "handle", None) + if not handler: + def empty_handle(_name, _cfg, _cloud, _log, _args): + pass + setattr(mod, 'handle', empty_handle) + return mod diff --git a/cloudinit/transforms/cc_apt_pipelining.py b/cloudinit/transforms/cc_apt_pipelining.py new file mode 100644 index 00000000..0286a9ae --- /dev/null +++ b/cloudinit/transforms/cc_apt_pipelining.py @@ -0,0 +1,53 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Ben Howard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance + +frequency = per_instance +default_file = "/etc/apt/apt.conf.d/90cloud-init-pipelining" + + +def handle(_name, cfg, _cloud, log, _args): + + apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) + apt_pipe_value = str(apt_pipe_value).lower() + + if apt_pipe_value == "false": + write_apt_snippet("0", log) + + elif apt_pipe_value in ("none", "unchanged", "os"): + return + + elif apt_pipe_value in str(range(0, 6)): + write_apt_snippet(apt_pipe_value, log) + + else: + log.warn("Invalid option for apt_pipeling: %s" % apt_pipe_value) + + +def write_apt_snippet(setting, log, f_name=default_file): + """ Writes f_name with apt pipeline depth 'setting' """ + + acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' + file_contents = ("//Written by cloud-init per 'apt_pipelining'\n" + + (acquire_pipeline_depth % setting)) + + util.write_file(f_name, file_contents) + + log.debug("Wrote %s with APT pipeline setting" % f_name) diff --git a/cloudinit/transforms/cc_apt_update_upgrade.py b/cloudinit/transforms/cc_apt_update_upgrade.py new file mode 100644 index 00000000..a7049bce --- /dev/null +++ b/cloudinit/transforms/cc_apt_update_upgrade.py @@ -0,0 +1,241 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import traceback +import os +import glob +import cloudinit.CloudConfig as cc + + +def handle(_name, cfg, cloud, log, _args): + update = util.get_cfg_option_bool(cfg, 'apt_update', False) + upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) + + release = get_release() + + mirror = find_apt_mirror(cloud, cfg) + + log.debug("selected mirror at: %s" % mirror) + + if not util.get_cfg_option_bool(cfg, \ + 'apt_preserve_sources_list', False): + generate_sources_list(release, mirror) + old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', \ + "archive.ubuntu.com/ubuntu") + rename_apt_lists(old_mir, mirror) + + # set up proxy + proxy = cfg.get("apt_proxy", None) + proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy" + if proxy: + try: + contents = "Acquire::HTTP::Proxy \"%s\";\n" + with open(proxy_filename, "w") as fp: + fp.write(contents % proxy) + except Exception as e: + log.warn("Failed to write proxy to %s" % proxy_filename) + elif os.path.isfile(proxy_filename): + os.unlink(proxy_filename) + + # process 'apt_sources' + if 'apt_sources' in cfg: + errors = add_sources(cfg['apt_sources'], + {'MIRROR': mirror, 'RELEASE': release}) + for e in errors: + log.warn("Source Error: %s\n" % ':'.join(e)) + + dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False) + if dconf_sel: + log.debug("setting debconf selections per cloud config") + try: + util.subp(('debconf-set-selections', '-'), dconf_sel) + except: + log.error("Failed to run debconf-set-selections") + log.debug(traceback.format_exc()) + + pkglist = util.get_cfg_option_list_or_str(cfg, 'packages', []) + + errors = [] + if update or len(pkglist) or upgrade: + try: + cc.update_package_sources() + except subprocess.CalledProcessError as e: + log.warn("apt-get update failed") + log.debug(traceback.format_exc()) + errors.append(e) + + if upgrade: + try: + cc.apt_get("upgrade") + except subprocess.CalledProcessError as e: + log.warn("apt upgrade failed") + log.debug(traceback.format_exc()) + errors.append(e) + + if len(pkglist): + try: + cc.install_packages(pkglist) + except subprocess.CalledProcessError as e: + log.warn("Failed to install packages: %s " % pkglist) + log.debug(traceback.format_exc()) + errors.append(e) + + if len(errors): + raise errors[0] + + return(True) + + +def mirror2lists_fileprefix(mirror): + string = mirror + # take of http:// or ftp:// + if string.endswith("/"): + string = string[0:-1] + pos = string.find("://") + if pos >= 0: + string = string[pos + 3:] + string = string.replace("/", "_") + return string + + +def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"): + oprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(omirror)) + nprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(new_mirror)) + if(oprefix == nprefix): + return + olen = len(oprefix) + for filename in glob.glob("%s_*" % oprefix): + os.rename(filename, "%s%s" % (nprefix, filename[olen:])) + + +def get_release(): + stdout, _stderr = subprocess.Popen(['lsb_release', '-cs'], + stdout=subprocess.PIPE).communicate() + return(str(stdout).strip()) + + +def generate_sources_list(codename, mirror): + util.render_to_file('sources.list', '/etc/apt/sources.list', \ + {'mirror': mirror, 'codename': codename}) + + +def add_sources(srclist, searchList=None): + """ + add entries in /etc/apt/sources.list.d for each abbreviated + sources.list entry in 'srclist'. When rendering template, also + include the values in dictionary searchList + """ + if searchList is None: + searchList = {} + elst = [] + + for ent in srclist: + if 'source' not in ent: + elst.append(["", "missing source"]) + continue + + source = ent['source'] + if source.startswith("ppa:"): + try: + util.subp(["add-apt-repository", source]) + except: + elst.append([source, "add-apt-repository failed"]) + continue + + source = util.render_string(source, searchList) + + if 'filename' not in ent: + ent['filename'] = 'cloud_config_sources.list' + + if not ent['filename'].startswith("/"): + ent['filename'] = "%s/%s" % \ + ("/etc/apt/sources.list.d/", ent['filename']) + + if ('keyid' in ent and 'key' not in ent): + ks = "keyserver.ubuntu.com" + if 'keyserver' in ent: + ks = ent['keyserver'] + try: + ent['key'] = util.getkeybyid(ent['keyid'], ks) + except: + elst.append([source, "failed to get key from %s" % ks]) + continue + + if 'key' in ent: + try: + util.subp(('apt-key', 'add', '-'), ent['key']) + except: + elst.append([source, "failed add key"]) + + try: + util.write_file(ent['filename'], source + "\n", omode="ab") + except: + elst.append([source, "failed write to file %s" % ent['filename']]) + + return(elst) + + +def find_apt_mirror(cloud, cfg): + """ find an apt_mirror given the cloud and cfg provided """ + + # TODO: distro and defaults should be configurable + distro = "ubuntu" + defaults = { + 'ubuntu': "http://archive.ubuntu.com/ubuntu", + 'debian': "http://archive.debian.org/debian", + } + mirror = None + + cfg_mirror = cfg.get("apt_mirror", None) + if cfg_mirror: + mirror = cfg["apt_mirror"] + elif "apt_mirror_search" in cfg: + mirror = util.search_for_mirror(cfg['apt_mirror_search']) + else: + if cloud: + mirror = cloud.get_mirror() + + mydom = "" + + doms = [] + + if not mirror and cloud: + # if we have a fqdn, then search its domain portion first + (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + mydom = ".".join(fqdn.split(".")[1:]) + if mydom: + doms.append(".%s" % mydom) + + if not mirror: + doms.extend((".localdomain", "",)) + + mirror_list = [] + mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) + for post in doms: + mirror_list.append(mirrorfmt % post) + + mirror = util.search_for_mirror(mirror_list) + + if not mirror: + mirror = defaults[distro] + + return mirror diff --git a/cloudinit/transforms/cc_bootcmd.py b/cloudinit/transforms/cc_bootcmd.py new file mode 100644 index 00000000..f584da02 --- /dev/null +++ b/cloudinit/transforms/cc_bootcmd.py @@ -0,0 +1,48 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import cloudinit.util as util +import subprocess +import tempfile +import os +from cloudinit.CloudConfig import per_always +frequency = per_always + + +def handle(_name, cfg, cloud, log, _args): + if "bootcmd" not in cfg: + return + + try: + content = util.shellify(cfg["bootcmd"]) + tmpf = tempfile.TemporaryFile() + tmpf.write(content) + tmpf.seek(0) + except: + log.warn("failed to shellify bootcmd") + raise + + try: + env = os.environ.copy() + env['INSTANCE_ID'] = cloud.get_instance_id() + subprocess.check_call(['/bin/sh'], env=env, stdin=tmpf) + tmpf.close() + except: + log.warn("failed to run commands from bootcmd") + raise diff --git a/cloudinit/transforms/cc_byobu.py b/cloudinit/transforms/cc_byobu.py new file mode 100644 index 00000000..e821b261 --- /dev/null +++ b/cloudinit/transforms/cc_byobu.py @@ -0,0 +1,77 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import traceback + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + value = args[0] + else: + value = util.get_cfg_option_str(cfg, "byobu_by_default", "") + + if not value: + return + + if value == "user" or value == "system": + value = "enable-%s" % value + + valid = ("enable-user", "enable-system", "enable", + "disable-user", "disable-system", "disable") + if not value in valid: + log.warn("Unknown value %s for byobu_by_default" % value) + + mod_user = value.endswith("-user") + mod_sys = value.endswith("-system") + if value.startswith("enable"): + bl_inst = "install" + dc_val = "byobu byobu/launch-by-default boolean true" + mod_sys = True + else: + if value == "disable": + mod_user = True + mod_sys = True + bl_inst = "uninstall" + dc_val = "byobu byobu/launch-by-default boolean false" + + shcmd = "" + if mod_user: + user = util.get_cfg_option_str(cfg, "user", "ubuntu") + shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst) + shcmd += " || X=$(($X+1)); " + if mod_sys: + shcmd += "echo \"%s\" | debconf-set-selections" % dc_val + shcmd += " && dpkg-reconfigure byobu --frontend=noninteractive" + shcmd += " || X=$(($X+1)); " + + cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] + + log.debug("setting byobu to %s" % value) + + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) + except OSError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/transforms/cc_ca_certs.py b/cloudinit/transforms/cc_ca_certs.py new file mode 100644 index 00000000..3af6238a --- /dev/null +++ b/cloudinit/transforms/cc_ca_certs.py @@ -0,0 +1,90 @@ +# vi: ts=4 expandtab +# +# Author: Mike Milner +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import os +from subprocess import check_call +from cloudinit.util import (write_file, get_cfg_option_list_or_str, + delete_dir_contents, subp) + +CA_CERT_PATH = "/usr/share/ca-certificates/" +CA_CERT_FILENAME = "cloud-init-ca-certs.crt" +CA_CERT_CONFIG = "/etc/ca-certificates.conf" +CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" + + +def update_ca_certs(): + """ + Updates the CA certificate cache on the current machine. + """ + check_call(["update-ca-certificates"]) + + +def add_ca_certs(certs): + """ + Adds certificates to the system. To actually apply the new certificates + you must also call L{update_ca_certs}. + + @param certs: A list of certificate strings. + """ + if certs: + cert_file_contents = "\n".join(certs) + cert_file_fullpath = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) + write_file(cert_file_fullpath, cert_file_contents, mode=0644) + # Append cert filename to CA_CERT_CONFIG file. + write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="a") + + +def remove_default_ca_certs(): + """ + Removes all default trusted CA certificates from the system. To actually + apply the change you must also call L{update_ca_certs}. + """ + delete_dir_contents(CA_CERT_PATH) + delete_dir_contents(CA_CERT_SYSTEM_PATH) + write_file(CA_CERT_CONFIG, "", mode=0644) + debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" + subp(('debconf-set-selections', '-'), debconf_sel) + + +def handle(_name, cfg, _cloud, log, _args): + """ + Call to handle ca-cert sections in cloud-config file. + + @param name: The module name "ca-cert" from cloud.cfg + @param cfg: A nested dict containing the entire cloud config contents. + @param cloud: The L{CloudInit} object in use. + @param log: Pre-initialized Python logger object to use for logging. + @param args: Any module arguments from cloud.cfg + """ + # If there isn't a ca-certs section in the configuration don't do anything + if "ca-certs" not in cfg: + return + ca_cert_cfg = cfg['ca-certs'] + + # If there is a remove-defaults option set to true, remove the system + # default trusted CA certs first. + if ca_cert_cfg.get("remove-defaults", False): + log.debug("removing default certificates") + remove_default_ca_certs() + + # If we are given any new trusted CA certs to add, add them. + if "trusted" in ca_cert_cfg: + trusted_certs = get_cfg_option_list_or_str(ca_cert_cfg, "trusted") + if trusted_certs: + log.debug("adding %d certificates" % len(trusted_certs)) + add_ca_certs(trusted_certs) + + # Update the system with the new cert configuration. + update_ca_certs() diff --git a/cloudinit/transforms/cc_chef.py b/cloudinit/transforms/cc_chef.py new file mode 100644 index 00000000..941e04fe --- /dev/null +++ b/cloudinit/transforms/cc_chef.py @@ -0,0 +1,119 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Avishai Ish-Shalom +# Author: Mike Moulton +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import subprocess +import json +import cloudinit.CloudConfig as cc +import cloudinit.util as util + +ruby_version_default = "1.8" + + +def handle(_name, cfg, cloud, log, _args): + # If there isn't a chef key in the configuration don't do anything + if 'chef' not in cfg: + return + chef_cfg = cfg['chef'] + + # ensure the chef directories we use exist + mkdirs(['/etc/chef', '/var/log/chef', '/var/lib/chef', + '/var/cache/chef', '/var/backups/chef', '/var/run/chef']) + + # set the validation key based on the presence of either 'validation_key' + # or 'validation_cert'. In the case where both exist, 'validation_key' + # takes precedence + for key in ('validation_key', 'validation_cert'): + if key in chef_cfg and chef_cfg[key]: + with open('/etc/chef/validation.pem', 'w') as validation_key_fh: + validation_key_fh.write(chef_cfg[key]) + break + + # create the chef config from template + util.render_to_file('chef_client.rb', '/etc/chef/client.rb', + {'server_url': chef_cfg['server_url'], + 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', + cloud.datasource.get_instance_id()), + 'environment': util.get_cfg_option_str(chef_cfg, 'environment', + '_default'), + 'validation_name': chef_cfg['validation_name']}) + + # set the firstboot json + with open('/etc/chef/firstboot.json', 'w') as firstboot_json_fh: + initial_json = {} + if 'run_list' in chef_cfg: + initial_json['run_list'] = chef_cfg['run_list'] + if 'initial_attributes' in chef_cfg: + initial_attributes = chef_cfg['initial_attributes'] + for k in initial_attributes.keys(): + initial_json[k] = initial_attributes[k] + firstboot_json_fh.write(json.dumps(initial_json)) + + # If chef is not installed, we install chef based on 'install_type' + if not os.path.isfile('/usr/bin/chef-client'): + install_type = util.get_cfg_option_str(chef_cfg, 'install_type', + 'packages') + if install_type == "gems": + # this will install and run the chef-client from gems + chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) + ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', + ruby_version_default) + install_chef_from_gems(ruby_version, chef_version) + # and finally, run chef-client + log.debug('running chef-client') + subprocess.check_call(['/usr/bin/chef-client', '-d', '-i', '1800', + '-s', '20']) + else: + # this will install and run the chef-client from packages + cc.install_packages(('chef',)) + + +def get_ruby_packages(version): + # return a list of packages needed to install ruby at version + pkgs = ['ruby%s' % version, 'ruby%s-dev' % version] + if version == "1.8": + pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8')) + return(pkgs) + + +def install_chef_from_gems(ruby_version, chef_version=None): + cc.install_packages(get_ruby_packages(ruby_version)) + if not os.path.exists('/usr/bin/gem'): + os.symlink('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') + if not os.path.exists('/usr/bin/ruby'): + os.symlink('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') + if chef_version: + subprocess.check_call(['/usr/bin/gem', 'install', 'chef', + '-v %s' % chef_version, '--no-ri', + '--no-rdoc', '--bindir', '/usr/bin', '-q']) + else: + subprocess.check_call(['/usr/bin/gem', 'install', 'chef', + '--no-ri', '--no-rdoc', '--bindir', + '/usr/bin', '-q']) + + +def ensure_dir(d): + if not os.path.exists(d): + os.makedirs(d) + + +def mkdirs(dirs): + for d in dirs: + ensure_dir(d) diff --git a/cloudinit/transforms/cc_disable_ec2_metadata.py b/cloudinit/transforms/cc_disable_ec2_metadata.py new file mode 100644 index 00000000..6b31ea8e --- /dev/null +++ b/cloudinit/transforms/cc_disable_ec2_metadata.py @@ -0,0 +1,30 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import cloudinit.util as util +import subprocess +from cloudinit.CloudConfig import per_always + +frequency = per_always + + +def handle(_name, cfg, _cloud, _log, _args): + if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False): + fwall = "route add -host 169.254.169.254 reject" + subprocess.call(fwall.split(' ')) diff --git a/cloudinit/transforms/cc_final_message.py b/cloudinit/transforms/cc_final_message.py new file mode 100644 index 00000000..abb4ca32 --- /dev/null +++ b/cloudinit/transforms/cc_final_message.py @@ -0,0 +1,58 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.CloudConfig import per_always +import sys +from cloudinit import util, boot_finished +import time + +frequency = per_always + +final_message = "cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds" + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + msg_in = args[0] + else: + msg_in = util.get_cfg_option_str(cfg, "final_message", final_message) + + try: + uptimef = open("/proc/uptime") + uptime = uptimef.read().split(" ")[0] + uptimef.close() + except IOError as e: + log.warn("unable to open /proc/uptime\n") + uptime = "na" + + try: + ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) + except: + ts = "na" + + try: + subs = {'UPTIME': uptime, 'TIMESTAMP': ts} + sys.stdout.write("%s\n" % util.render_string(msg_in, subs)) + except Exception as e: + log.warn("failed to render string to stdout: %s" % e) + + fp = open(boot_finished, "wb") + fp.write(uptime + "\n") + fp.close() diff --git a/cloudinit/transforms/cc_foo.py b/cloudinit/transforms/cc_foo.py new file mode 100644 index 00000000..35ec3fa7 --- /dev/null +++ b/cloudinit/transforms/cc_foo.py @@ -0,0 +1,29 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#import cloudinit +#import cloudinit.util as util +from cloudinit.CloudConfig import per_instance + +frequency = per_instance + + +def handle(_name, _cfg, _cloud, _log, _args): + print "hi" diff --git a/cloudinit/transforms/cc_grub_dpkg.py b/cloudinit/transforms/cc_grub_dpkg.py new file mode 100644 index 00000000..9f3a7eaf --- /dev/null +++ b/cloudinit/transforms/cc_grub_dpkg.py @@ -0,0 +1,64 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import traceback +import os + + +def handle(_name, cfg, _cloud, log, _args): + idevs = None + idevs_empty = None + + if "grub-dpkg" in cfg: + idevs = util.get_cfg_option_str(cfg["grub-dpkg"], + "grub-pc/install_devices", None) + idevs_empty = util.get_cfg_option_str(cfg["grub-dpkg"], + "grub-pc/install_devices_empty", None) + + if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or + (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): + if idevs == None: + idevs = "" + if idevs_empty == None: + idevs_empty = "true" + else: + if idevs_empty == None: + idevs_empty = "false" + if idevs == None: + idevs = "/dev/sda" + for dev in ("/dev/sda", "/dev/vda", "/dev/sda1", "/dev/vda1"): + if os.path.exists(dev): + idevs = dev + break + + # now idevs and idevs_empty are set to determined values + # or, those set by user + + dconf_sel = "grub-pc grub-pc/install_devices string %s\n" % idevs + \ + "grub-pc grub-pc/install_devices_empty boolean %s\n" % idevs_empty + log.debug("setting grub debconf-set-selections with '%s','%s'" % + (idevs, idevs_empty)) + + try: + util.subp(('debconf-set-selections'), dconf_sel) + except: + log.error("Failed to run debconf-set-selections for grub-dpkg") + log.debug(traceback.format_exc()) diff --git a/cloudinit/transforms/cc_keys_to_console.py b/cloudinit/transforms/cc_keys_to_console.py new file mode 100644 index 00000000..73a477c0 --- /dev/null +++ b/cloudinit/transforms/cc_keys_to_console.py @@ -0,0 +1,42 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.CloudConfig import per_instance +import cloudinit.util as util +import subprocess + +frequency = per_instance + + +def handle(_name, cfg, _cloud, log, _args): + cmd = ['/usr/lib/cloud-init/write-ssh-key-fingerprints'] + fp_blacklist = util.get_cfg_option_list_or_str(cfg, + "ssh_fp_console_blacklist", []) + key_blacklist = util.get_cfg_option_list_or_str(cfg, + "ssh_key_console_blacklist", ["ssh-dss"]) + try: + confp = open('/dev/console', "wb") + cmd.append(','.join(fp_blacklist)) + cmd.append(','.join(key_blacklist)) + subprocess.call(cmd, stdout=confp) + confp.close() + except: + log.warn("writing keys to console value") + raise diff --git a/cloudinit/transforms/cc_landscape.py b/cloudinit/transforms/cc_landscape.py new file mode 100644 index 00000000..a4113cbe --- /dev/null +++ b/cloudinit/transforms/cc_landscape.py @@ -0,0 +1,75 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +from cloudinit.CloudConfig import per_instance +from configobj import ConfigObj + +frequency = per_instance + +lsc_client_cfg_file = "/etc/landscape/client.conf" + +# defaults taken from stock client.conf in landscape-client 11.07.1.1-0ubuntu2 +lsc_builtincfg = { + 'client': { + 'log_level': "info", + 'url': "https://landscape.canonical.com/message-system", + 'ping_url': "http://landscape.canonical.com/ping", + 'data_path': "/var/lib/landscape/client", + } +} + + +def handle(_name, cfg, _cloud, log, _args): + """ + Basically turn a top level 'landscape' entry with a 'client' dict + and render it to ConfigObj format under '[client]' section in + /etc/landscape/client.conf + """ + + ls_cloudcfg = cfg.get("landscape", {}) + + if not isinstance(ls_cloudcfg, dict): + raise(Exception("'landscape' existed in config, but not a dict")) + + merged = mergeTogether([lsc_builtincfg, lsc_client_cfg_file, ls_cloudcfg]) + + if not os.path.isdir(os.path.dirname(lsc_client_cfg_file)): + os.makedirs(os.path.dirname(lsc_client_cfg_file)) + + with open(lsc_client_cfg_file, "w") as fp: + merged.write(fp) + + log.debug("updated %s" % lsc_client_cfg_file) + + +def mergeTogether(objs): + """ + merge together ConfigObj objects or things that ConfigObj() will take in + later entries override earlier + """ + cfg = ConfigObj({}) + for obj in objs: + if isinstance(obj, ConfigObj): + cfg.merge(obj) + else: + cfg.merge(ConfigObj(obj)) + return cfg diff --git a/cloudinit/transforms/cc_locale.py b/cloudinit/transforms/cc_locale.py new file mode 100644 index 00000000..2bb22fdb --- /dev/null +++ b/cloudinit/transforms/cc_locale.py @@ -0,0 +1,54 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import os.path +import subprocess +import traceback + + +def apply_locale(locale, cfgfile): + if os.path.exists('/usr/sbin/locale-gen'): + subprocess.Popen(['locale-gen', locale]).communicate() + if os.path.exists('/usr/sbin/update-locale'): + subprocess.Popen(['update-locale', locale]).communicate() + + util.render_to_file('default-locale', cfgfile, {'locale': locale}) + + +def handle(_name, cfg, cloud, log, args): + if len(args) != 0: + locale = args[0] + else: + locale = util.get_cfg_option_str(cfg, "locale", cloud.get_locale()) + + locale_cfgfile = util.get_cfg_option_str(cfg, "locale_configfile", + "/etc/default/locale") + + if not locale: + return + + log.debug("setting locale to %s" % locale) + + try: + apply_locale(locale, locale_cfgfile) + except Exception as e: + log.debug(traceback.format_exc(e)) + raise Exception("failed to apply locale %s" % locale) diff --git a/cloudinit/transforms/cc_mcollective.py b/cloudinit/transforms/cc_mcollective.py new file mode 100644 index 00000000..a2a6230c --- /dev/null +++ b/cloudinit/transforms/cc_mcollective.py @@ -0,0 +1,99 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Marc Cluet +# Based on code by Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import subprocess +import StringIO +import ConfigParser +import cloudinit.CloudConfig as cc +import cloudinit.util as util + +pubcert_file = "/etc/mcollective/ssl/server-public.pem" +pricert_file = "/etc/mcollective/ssl/server-private.pem" + + +# Our fake header section +class FakeSecHead(object): + def __init__(self, fp): + self.fp = fp + self.sechead = '[nullsection]\n' + + def readline(self): + if self.sechead: + try: + return self.sechead + finally: + self.sechead = None + else: + return self.fp.readline() + + +def handle(_name, cfg, _cloud, _log, _args): + # If there isn't a mcollective key in the configuration don't do anything + if 'mcollective' not in cfg: + return + mcollective_cfg = cfg['mcollective'] + # Start by installing the mcollective package ... + cc.install_packages(("mcollective",)) + + # ... and then update the mcollective configuration + if 'conf' in mcollective_cfg: + # Create object for reading server.cfg values + mcollective_config = ConfigParser.ConfigParser() + # Read server.cfg values from original file in order to be able to mix + # the rest up + mcollective_config.readfp(FakeSecHead(open('/etc/mcollective/' + 'server.cfg'))) + for cfg_name, cfg in mcollective_cfg['conf'].iteritems(): + if cfg_name == 'public-cert': + util.write_file(pubcert_file, cfg, mode=0644) + mcollective_config.set(cfg_name, + 'plugin.ssl_server_public', pubcert_file) + mcollective_config.set(cfg_name, 'securityprovider', 'ssl') + elif cfg_name == 'private-cert': + util.write_file(pricert_file, cfg, mode=0600) + mcollective_config.set(cfg_name, + 'plugin.ssl_server_private', pricert_file) + mcollective_config.set(cfg_name, 'securityprovider', 'ssl') + else: + # Iterate throug the config items, we'll use ConfigParser.set + # to overwrite or create new items as needed + for o, v in cfg.iteritems(): + mcollective_config.set(cfg_name, o, v) + # We got all our config as wanted we'll rename + # the previous server.cfg and create our new one + os.rename('/etc/mcollective/server.cfg', + '/etc/mcollective/server.cfg.old') + outputfile = StringIO.StringIO() + mcollective_config.write(outputfile) + # Now we got the whole file, write to disk except first line + # Note below, that we've just used ConfigParser because it generally + # works. Below, we remove the initial 'nullsection' header + # and then change 'key = value' to 'key: value'. The global + # search and replace of '=' with ':' could be problematic though. + # this most likely needs fixing. + util.write_file('/etc/mcollective/server.cfg', + outputfile.getvalue().replace('[nullsection]\n', '').replace(' =', + ':'), + mode=0644) + + # Start mcollective + subprocess.check_call(['service', 'mcollective', 'start']) diff --git a/cloudinit/transforms/cc_mounts.py b/cloudinit/transforms/cc_mounts.py new file mode 100644 index 00000000..6cdd74e8 --- /dev/null +++ b/cloudinit/transforms/cc_mounts.py @@ -0,0 +1,179 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import os +import re +from string import whitespace # pylint: disable=W0402 + + +def is_mdname(name): + # return true if this is a metadata service name + if name in ["ami", "root", "swap"]: + return True + # names 'ephemeral0' or 'ephemeral1' + # 'ebs[0-9]' appears when '--block-device-mapping sdf=snap-d4d90bbc' + for enumname in ("ephemeral", "ebs"): + if name.startswith(enumname) and name.find(":") == -1: + return True + return False + + +def handle(_name, cfg, cloud, log, _args): + # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno + defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] + defvals = cfg.get("mount_default_fields", defvals) + + # these are our default set of mounts + defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"], + ["swap", "none", "swap", "sw", "0", "0"]] + + cfgmnt = [] + if "mounts" in cfg: + cfgmnt = cfg["mounts"] + + # shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 + shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" + shortname = re.compile(shortname_filter) + + for i in range(len(cfgmnt)): + # skip something that wasn't a list + if not isinstance(cfgmnt[i], list): + continue + + # workaround, allow user to specify 'ephemeral' + # rather than more ec2 correct 'ephemeral0' + if cfgmnt[i][0] == "ephemeral": + cfgmnt[i][0] = "ephemeral0" + + if is_mdname(cfgmnt[i][0]): + newname = cloud.device_name_to_device(cfgmnt[i][0]) + if not newname: + log.debug("ignoring nonexistant named mount %s" % cfgmnt[i][0]) + cfgmnt[i][1] = None + else: + if newname.startswith("/"): + cfgmnt[i][0] = newname + else: + cfgmnt[i][0] = "/dev/%s" % newname + else: + if shortname.match(cfgmnt[i][0]): + cfgmnt[i][0] = "/dev/%s" % cfgmnt[i][0] + + # in case the user did not quote a field (likely fs-freq, fs_passno) + # but do not convert None to 'None' (LP: #898365) + for j in range(len(cfgmnt[i])): + if isinstance(cfgmnt[i][j], int): + cfgmnt[i][j] = str(cfgmnt[i][j]) + + for i in range(len(cfgmnt)): + # fill in values with defaults from defvals above + for j in range(len(defvals)): + if len(cfgmnt[i]) <= j: + cfgmnt[i].append(defvals[j]) + elif cfgmnt[i][j] is None: + cfgmnt[i][j] = defvals[j] + + # if the second entry in the list is 'None' this + # clears all previous entries of that same 'fs_spec' + # (fs_spec is the first field in /etc/fstab, ie, that device) + if cfgmnt[i][1] is None: + for j in range(i): + if cfgmnt[j][0] == cfgmnt[i][0]: + cfgmnt[j][1] = None + + # for each of the "default" mounts, add them only if no other + # entry has the same device name + for defmnt in defmnts: + devname = cloud.device_name_to_device(defmnt[0]) + if devname is None: + continue + if devname.startswith("/"): + defmnt[0] = devname + else: + defmnt[0] = "/dev/%s" % devname + + cfgmnt_has = False + for cfgm in cfgmnt: + if cfgm[0] == defmnt[0]: + cfgmnt_has = True + break + + if cfgmnt_has: + continue + cfgmnt.append(defmnt) + + # now, each entry in the cfgmnt list has all fstab values + # if the second field is None (not the string, the value) we skip it + actlist = [x for x in cfgmnt if x[1] is not None] + + if len(actlist) == 0: + return + + comment = "comment=cloudconfig" + cc_lines = [] + needswap = False + dirs = [] + for line in actlist: + # write 'comment' in the fs_mntops, entry, claiming this + line[3] = "%s,comment=cloudconfig" % line[3] + if line[2] == "swap": + needswap = True + if line[1].startswith("/"): + dirs.append(line[1]) + cc_lines.append('\t'.join(line)) + + fstab_lines = [] + fstab = open("/etc/fstab", "r+") + ws = re.compile("[%s]+" % whitespace) + for line in fstab.read().splitlines(): + try: + toks = ws.split(line) + if toks[3].find(comment) != -1: + continue + except: + pass + fstab_lines.append(line) + + fstab_lines.extend(cc_lines) + + fstab.seek(0) + fstab.write("%s\n" % '\n'.join(fstab_lines)) + fstab.truncate() + fstab.close() + + if needswap: + try: + util.subp(("swapon", "-a")) + except: + log.warn("Failed to enable swap") + + for d in dirs: + if os.path.exists(d): + continue + try: + os.makedirs(d) + except: + log.warn("Failed to make '%s' config-mount\n", d) + + try: + util.subp(("mount", "-a")) + except: + log.warn("'mount -a' failed") diff --git a/cloudinit/transforms/cc_phone_home.py b/cloudinit/transforms/cc_phone_home.py new file mode 100644 index 00000000..a7ff74e1 --- /dev/null +++ b/cloudinit/transforms/cc_phone_home.py @@ -0,0 +1,106 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +from cloudinit.CloudConfig import per_instance +import cloudinit.util as util +from time import sleep + +frequency = per_instance +post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', + 'hostname'] + + +# phone_home: +# url: http://my.foo.bar/$INSTANCE/ +# post: all +# tries: 10 +# +# phone_home: +# url: http://my.foo.bar/$INSTANCE_ID/ +# post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id +# +def handle(_name, cfg, cloud, log, args): + if len(args) != 0: + ph_cfg = util.read_conf(args[0]) + else: + if not 'phone_home' in cfg: + return + ph_cfg = cfg['phone_home'] + + if 'url' not in ph_cfg: + log.warn("no 'url' token in phone_home") + return + + url = ph_cfg['url'] + post_list = ph_cfg.get('post', 'all') + tries = ph_cfg.get('tries', 10) + try: + tries = int(tries) + except: + log.warn("tries is not an integer. using 10") + tries = 10 + + if post_list == "all": + post_list = post_list_all + + all_keys = {} + all_keys['instance_id'] = cloud.get_instance_id() + all_keys['hostname'] = cloud.get_hostname() + + pubkeys = { + 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', + 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', + 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', + } + + for n, path in pubkeys.iteritems(): + try: + fp = open(path, "rb") + all_keys[n] = fp.read() + fp.close() + except: + log.warn("%s: failed to open in phone_home" % path) + + submit_keys = {} + for k in post_list: + if k in all_keys: + submit_keys[k] = all_keys[k] + else: + submit_keys[k] = "N/A" + log.warn("requested key %s from 'post' list not available") + + url = util.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) + + null_exc = object() + last_e = null_exc + for i in range(0, tries): + try: + util.readurl(url, submit_keys) + log.debug("succeeded submit to %s on try %i" % (url, i + 1)) + return + except Exception as e: + log.debug("failed to post to %s on try %i" % (url, i + 1)) + last_e = e + sleep(3) + + log.warn("failed to post to %s in %i tries" % (url, tries)) + if last_e is not null_exc: + raise(last_e) + + return diff --git a/cloudinit/transforms/cc_puppet.py b/cloudinit/transforms/cc_puppet.py new file mode 100644 index 00000000..6fc475f6 --- /dev/null +++ b/cloudinit/transforms/cc_puppet.py @@ -0,0 +1,108 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +import pwd +import socket +import subprocess +import StringIO +import ConfigParser +import cloudinit.CloudConfig as cc +import cloudinit.util as util + + +def handle(_name, cfg, cloud, log, _args): + # If there isn't a puppet key in the configuration don't do anything + if 'puppet' not in cfg: + return + puppet_cfg = cfg['puppet'] + # Start by installing the puppet package ... + cc.install_packages(("puppet",)) + + # ... and then update the puppet configuration + if 'conf' in puppet_cfg: + # Add all sections from the conf object to puppet.conf + puppet_conf_fh = open('/etc/puppet/puppet.conf', 'r') + # Create object for reading puppet.conf values + puppet_config = ConfigParser.ConfigParser() + # Read puppet.conf values from original file in order to be able to + # mix the rest up + puppet_config.readfp(StringIO.StringIO(''.join(i.lstrip() for i in + puppet_conf_fh.readlines()))) + # Close original file, no longer needed + puppet_conf_fh.close() + for cfg_name, cfg in puppet_cfg['conf'].iteritems(): + # ca_cert configuration is a special case + # Dump the puppetmaster ca certificate in the correct place + if cfg_name == 'ca_cert': + # Puppet ssl sub-directory isn't created yet + # Create it with the proper permissions and ownership + os.makedirs('/var/lib/puppet/ssl') + os.chmod('/var/lib/puppet/ssl', 0771) + os.chown('/var/lib/puppet/ssl', + pwd.getpwnam('puppet').pw_uid, 0) + os.makedirs('/var/lib/puppet/ssl/certs/') + os.chown('/var/lib/puppet/ssl/certs/', + pwd.getpwnam('puppet').pw_uid, 0) + ca_fh = open('/var/lib/puppet/ssl/certs/ca.pem', 'w') + ca_fh.write(cfg) + ca_fh.close() + os.chown('/var/lib/puppet/ssl/certs/ca.pem', + pwd.getpwnam('puppet').pw_uid, 0) + util.restorecon_if_possible('/var/lib/puppet', recursive=True) + else: + #puppet_conf_fh.write("\n[%s]\n" % (cfg_name)) + # If puppet.conf already has this section we don't want to + # write it again + if puppet_config.has_section(cfg_name) == False: + puppet_config.add_section(cfg_name) + # Iterate throug the config items, we'll use ConfigParser.set + # to overwrite or create new items as needed + for o, v in cfg.iteritems(): + if o == 'certname': + # Expand %f as the fqdn + v = v.replace("%f", socket.getfqdn()) + # Expand %i as the instance id + v = v.replace("%i", + cloud.datasource.get_instance_id()) + # certname needs to be downcase + v = v.lower() + puppet_config.set(cfg_name, o, v) + #puppet_conf_fh.write("%s=%s\n" % (o, v)) + # We got all our config as wanted we'll rename + # the previous puppet.conf and create our new one + os.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') + with open('/etc/puppet/puppet.conf', 'wb') as configfile: + puppet_config.write(configfile) + util.restorecon_if_possible('/etc/puppet/puppet.conf') + # Set puppet to automatically start + if os.path.exists('/etc/default/puppet'): + subprocess.check_call(['sed', '-i', + '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet']) + elif os.path.exists('/bin/systemctl'): + subprocess.check_call(['/bin/systemctl', 'enable', 'puppet.service']) + elif os.path.exists('/sbin/chkconfig'): + subprocess.check_call(['/sbin/chkconfig', 'puppet', 'on']) + else: + log.warn("Do not know how to enable puppet service on this system") + # Start puppetd + subprocess.check_call(['service', 'puppet', 'start']) diff --git a/cloudinit/transforms/cc_resizefs.py b/cloudinit/transforms/cc_resizefs.py new file mode 100644 index 00000000..2dc66def --- /dev/null +++ b/cloudinit/transforms/cc_resizefs.py @@ -0,0 +1,108 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import os +import stat +import sys +import time +import tempfile +from cloudinit.CloudConfig import per_always + +frequency = per_always + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + resize_root = False + if str(args[0]).lower() in ['true', '1', 'on', 'yes']: + resize_root = True + else: + resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) + + if str(resize_root).lower() in ['false', '0']: + return + + # we use mktemp rather than mkstemp because early in boot nothing + # else should be able to race us for this, and we need to mknod. + devpth = tempfile.mktemp(prefix="cloudinit.resizefs.", dir="/run") + + try: + st_dev = os.stat("/").st_dev + dev = os.makedev(os.major(st_dev), os.minor(st_dev)) + os.mknod(devpth, 0400 | stat.S_IFBLK, dev) + except: + if util.is_container(): + log.debug("inside container, ignoring mknod failure in resizefs") + return + log.warn("Failed to make device node to resize /") + raise + + cmd = ['blkid', '-c', '/dev/null', '-sTYPE', '-ovalue', devpth] + try: + (fstype, _err) = util.subp(cmd) + except subprocess.CalledProcessError as e: + log.warn("Failed to get filesystem type of maj=%s, min=%s via: %s" % + (os.major(st_dev), os.minor(st_dev), cmd)) + log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) + os.unlink(devpth) + raise + + if str(fstype).startswith("ext"): + resize_cmd = ['resize2fs', devpth] + elif fstype == "xfs": + resize_cmd = ['xfs_growfs', devpth] + else: + os.unlink(devpth) + log.debug("not resizing unknown filesystem %s" % fstype) + return + + if resize_root == "noblock": + fid = os.fork() + if fid == 0: + try: + do_resize(resize_cmd, devpth, log) + os._exit(0) # pylint: disable=W0212 + except Exception as exc: + sys.stderr.write("Failed: %s" % exc) + os._exit(1) # pylint: disable=W0212 + else: + do_resize(resize_cmd, devpth, log) + + log.debug("resizing root filesystem (type=%s, maj=%i, min=%i, val=%s)" % + (str(fstype).rstrip("\n"), os.major(st_dev), os.minor(st_dev), + resize_root)) + + return + + +def do_resize(resize_cmd, devpth, log): + try: + start = time.time() + util.subp(resize_cmd) + except subprocess.CalledProcessError as e: + log.warn("Failed to resize filesystem (%s)" % resize_cmd) + log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) + os.unlink(devpth) + raise + + os.unlink(devpth) + log.debug("resize took %s seconds" % (time.time() - start)) diff --git a/cloudinit/transforms/cc_rightscale_userdata.py b/cloudinit/transforms/cc_rightscale_userdata.py new file mode 100644 index 00000000..5ed0848f --- /dev/null +++ b/cloudinit/transforms/cc_rightscale_userdata.py @@ -0,0 +1,78 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +## +## The purpose of this script is to allow cloud-init to consume +## rightscale style userdata. rightscale user data is key-value pairs +## in a url-query-string like format. +## +## for cloud-init support, there will be a key named +## 'CLOUD_INIT_REMOTE_HOOK'. +## +## This cloud-config module will +## - read the blob of data from raw user data, and parse it as key/value +## - for each key that is found, download the content to +## the local instance/scripts directory and set them executable. +## - the files in that directory will be run by the user-scripts module +## Therefore, this must run before that. +## +## + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance +from cloudinit import get_ipath_cur +from urlparse import parse_qs + +frequency = per_instance +my_name = "cc_rightscale_userdata" +my_hookname = 'CLOUD_INIT_REMOTE_HOOK' + + +def handle(_name, _cfg, cloud, log, _args): + try: + ud = cloud.get_userdata_raw() + except: + log.warn("failed to get raw userdata in %s" % my_name) + return + + try: + mdict = parse_qs(ud) + if not my_hookname in mdict: + return + except: + log.warn("failed to urlparse.parse_qa(userdata_raw())") + raise + + scripts_d = get_ipath_cur('scripts') + i = 0 + first_e = None + for url in mdict[my_hookname]: + fname = "%s/rightscale-%02i" % (scripts_d, i) + i = i + 1 + try: + content = util.readurl(url) + util.write_file(fname, content, mode=0700) + except Exception as e: + if not first_e: + first_e = None + log.warn("%s failed to read %s: %s" % (my_name, url, e)) + + if first_e: + raise(e) diff --git a/cloudinit/transforms/cc_rsyslog.py b/cloudinit/transforms/cc_rsyslog.py new file mode 100644 index 00000000..ac7f2c74 --- /dev/null +++ b/cloudinit/transforms/cc_rsyslog.py @@ -0,0 +1,101 @@ +# vi: ts=4 expandtab syntax=python +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit +import logging +import cloudinit.util as util +import traceback + +DEF_FILENAME = "20-cloud-config.conf" +DEF_DIR = "/etc/rsyslog.d" + + +def handle(_name, cfg, _cloud, log, _args): + # rsyslog: + # - "*.* @@192.158.1.1" + # - content: "*.* @@192.0.2.1:10514" + # - filename: 01-examplecom.conf + # content: | + # *.* @@syslogd.example.com + + # process 'rsyslog' + if not 'rsyslog' in cfg: + return + + def_dir = cfg.get('rsyslog_dir', DEF_DIR) + def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) + + files = [] + elst = [] + for ent in cfg['rsyslog']: + if isinstance(ent, dict): + if not "content" in ent: + elst.append((ent, "no 'content' entry")) + continue + content = ent['content'] + filename = ent.get("filename", def_fname) + else: + content = ent + filename = def_fname + + if not filename.startswith("/"): + filename = "%s/%s" % (def_dir, filename) + + omode = "ab" + # truncate filename first time you see it + if filename not in files: + omode = "wb" + files.append(filename) + + try: + util.write_file(filename, content + "\n", omode=omode) + except Exception as e: + log.debug(traceback.format_exc(e)) + elst.append((content, "failed to write to %s" % filename)) + + # need to restart syslogd + restarted = False + try: + # if this config module is running at cloud-init time + # (before rsyslog is running) we don't actually have to + # restart syslog. + # + # upstart actually does what we want here, in that it doesn't + # start a service that wasn't running already on 'restart' + # it will also return failure on the attempt, so 'restarted' + # won't get set + log.debug("restarting rsyslog") + util.subp(['service', 'rsyslog', 'restart']) + restarted = True + + except Exception as e: + elst.append(("restart", str(e))) + + if restarted: + # this only needs to run if we *actually* restarted + # syslog above. + cloudinit.logging_set_from_cfg_file() + log = logging.getLogger() + log.debug("rsyslog configured %s" % files) + + for e in elst: + log.warn("rsyslog error: %s\n" % ':'.join(e)) + + return diff --git a/cloudinit/transforms/cc_runcmd.py b/cloudinit/transforms/cc_runcmd.py new file mode 100644 index 00000000..f7e8c671 --- /dev/null +++ b/cloudinit/transforms/cc_runcmd.py @@ -0,0 +1,32 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util + + +def handle(_name, cfg, cloud, log, _args): + if "runcmd" not in cfg: + return + outfile = "%s/runcmd" % cloud.get_ipath('scripts') + try: + content = util.shellify(cfg["runcmd"]) + util.write_file(outfile, content, 0700) + except: + log.warn("failed to open %s for runcmd" % outfile) diff --git a/cloudinit/transforms/cc_salt_minion.py b/cloudinit/transforms/cc_salt_minion.py new file mode 100644 index 00000000..1a3b5039 --- /dev/null +++ b/cloudinit/transforms/cc_salt_minion.py @@ -0,0 +1,56 @@ +# vi: ts=4 expandtab +# +# Author: Jeff Bauer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +import subprocess +import cloudinit.CloudConfig as cc +import yaml + + +def handle(_name, cfg, _cloud, _log, _args): + # If there isn't a salt key in the configuration don't do anything + if 'salt_minion' not in cfg: + return + salt_cfg = cfg['salt_minion'] + # Start by installing the salt package ... + cc.install_packages(("salt",)) + config_dir = '/etc/salt' + if not os.path.isdir(config_dir): + os.makedirs(config_dir) + # ... and then update the salt configuration + if 'conf' in salt_cfg: + # Add all sections from the conf object to /etc/salt/minion + minion_config = os.path.join(config_dir, 'minion') + yaml.dump(salt_cfg['conf'], + file(minion_config, 'w'), + default_flow_style=False) + # ... copy the key pair if specified + if 'public_key' in salt_cfg and 'private_key' in salt_cfg: + pki_dir = '/etc/salt/pki' + cumask = os.umask(077) + if not os.path.isdir(pki_dir): + os.makedirs(pki_dir) + pub_name = os.path.join(pki_dir, 'minion.pub') + pem_name = os.path.join(pki_dir, 'minion.pem') + with open(pub_name, 'w') as f: + f.write(salt_cfg['public_key']) + with open(pem_name, 'w') as f: + f.write(salt_cfg['private_key']) + os.umask(cumask) + + # Start salt-minion + subprocess.check_call(['service', 'salt-minion', 'start']) diff --git a/cloudinit/transforms/cc_scripts_per_boot.py b/cloudinit/transforms/cc_scripts_per_boot.py new file mode 100644 index 00000000..41a74754 --- /dev/null +++ b/cloudinit/transforms/cc_scripts_per_boot.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_always +from cloudinit import get_cpath + +frequency = per_always +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-boot") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/transforms/cc_scripts_per_instance.py b/cloudinit/transforms/cc_scripts_per_instance.py new file mode 100644 index 00000000..a2981eab --- /dev/null +++ b/cloudinit/transforms/cc_scripts_per_instance.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance +from cloudinit import get_cpath + +frequency = per_instance +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-instance") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/transforms/cc_scripts_per_once.py b/cloudinit/transforms/cc_scripts_per_once.py new file mode 100644 index 00000000..a69151da --- /dev/null +++ b/cloudinit/transforms/cc_scripts_per_once.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_once +from cloudinit import get_cpath + +frequency = per_once +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-once") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/transforms/cc_scripts_user.py b/cloudinit/transforms/cc_scripts_user.py new file mode 100644 index 00000000..933aa4e0 --- /dev/null +++ b/cloudinit/transforms/cc_scripts_user.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance +from cloudinit import get_ipath_cur + +frequency = per_instance +runparts_path = "%s/%s" % (get_ipath_cur(), "scripts") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/transforms/cc_set_hostname.py b/cloudinit/transforms/cc_set_hostname.py new file mode 100644 index 00000000..acea74d9 --- /dev/null +++ b/cloudinit/transforms/cc_set_hostname.py @@ -0,0 +1,42 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util + + +def handle(_name, cfg, cloud, log, _args): + if util.get_cfg_option_bool(cfg, "preserve_hostname", False): + log.debug("preserve_hostname is set. not setting hostname") + return(True) + + (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) + try: + set_hostname(hostname, log) + except Exception: + util.logexc(log) + log.warn("failed to set hostname to %s\n", hostname) + + return(True) + + +def set_hostname(hostname, log): + util.subp(['hostname', hostname]) + util.write_file("/etc/hostname", "%s\n" % hostname, 0644) + log.debug("populated /etc/hostname with %s on first boot", hostname) diff --git a/cloudinit/transforms/cc_set_passwords.py b/cloudinit/transforms/cc_set_passwords.py new file mode 100644 index 00000000..9d0bbdb8 --- /dev/null +++ b/cloudinit/transforms/cc_set_passwords.py @@ -0,0 +1,129 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import sys +import random +from string import letters, digits # pylint: disable=W0402 + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + # if run from command line, and give args, wipe the chpasswd['list'] + password = args[0] + if 'chpasswd' in cfg and 'list' in cfg['chpasswd']: + del cfg['chpasswd']['list'] + else: + password = util.get_cfg_option_str(cfg, "password", None) + + expire = True + pw_auth = "no" + change_pwauth = False + plist = None + + if 'chpasswd' in cfg: + chfg = cfg['chpasswd'] + plist = util.get_cfg_option_str(chfg, 'list', plist) + expire = util.get_cfg_option_bool(chfg, 'expire', expire) + + if not plist and password: + user = util.get_cfg_option_str(cfg, "user", "ubuntu") + plist = "%s:%s" % (user, password) + + errors = [] + if plist: + plist_in = [] + randlist = [] + users = [] + for line in plist.splitlines(): + u, p = line.split(':', 1) + if p == "R" or p == "RANDOM": + p = rand_user_password() + randlist.append("%s:%s" % (u, p)) + plist_in.append("%s:%s" % (u, p)) + users.append(u) + + ch_in = '\n'.join(plist_in) + try: + util.subp(['chpasswd'], ch_in) + log.debug("changed password for %s:" % users) + except Exception as e: + errors.append(e) + log.warn("failed to set passwords with chpasswd: %s" % e) + + if len(randlist): + sys.stdout.write("%s\n%s\n" % ("Set the following passwords\n", + '\n'.join(randlist))) + + if expire: + enum = len(errors) + for u in users: + try: + util.subp(['passwd', '--expire', u]) + except Exception as e: + errors.append(e) + log.warn("failed to expire account for %s" % u) + if enum == len(errors): + log.debug("expired passwords for: %s" % u) + + if 'ssh_pwauth' in cfg: + val = str(cfg['ssh_pwauth']).lower() + if val in ("true", "1", "yes"): + pw_auth = "yes" + change_pwauth = True + elif val in ("false", "0", "no"): + pw_auth = "no" + change_pwauth = True + else: + change_pwauth = False + + if change_pwauth: + pa_s = "\(#*\)\(PasswordAuthentication[[:space:]]\+\)\(yes\|no\)" + msg = "set PasswordAuthentication to '%s'" % pw_auth + try: + cmd = ['sed', '-i', 's,%s,\\2%s,' % (pa_s, pw_auth), + '/etc/ssh/sshd_config'] + util.subp(cmd) + log.debug(msg) + except Exception as e: + log.warn("failed %s" % msg) + errors.append(e) + + try: + p = util.subp(['service', cfg.get('ssh_svcname', 'ssh'), + 'restart']) + log.debug("restarted sshd") + except: + log.warn("restart of ssh failed") + + if len(errors): + raise(errors[0]) + + return + + +def rand_str(strlen=32, select_from=letters + digits): + return("".join([random.choice(select_from) for _x in range(0, strlen)])) + + +def rand_user_password(pwlen=9): + selfrom = (letters.translate(None, 'loLOI') + + digits.translate(None, '01')) + return(rand_str(pwlen, select_from=selfrom)) diff --git a/cloudinit/transforms/cc_ssh.py b/cloudinit/transforms/cc_ssh.py new file mode 100644 index 00000000..48eb58bc --- /dev/null +++ b/cloudinit/transforms/cc_ssh.py @@ -0,0 +1,106 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import cloudinit.SshUtil as sshutil +import os +import glob +import subprocess + +DISABLE_ROOT_OPTS = "no-port-forwarding,no-agent-forwarding," \ +"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " \ +"rather than the user \\\"root\\\".\';echo;sleep 10\"" + + +def handle(_name, cfg, cloud, log, _args): + + # remove the static keys from the pristine image + if cfg.get("ssh_deletekeys", True): + for f in glob.glob("/etc/ssh/ssh_host_*key*"): + try: + os.unlink(f) + except: + pass + + if "ssh_keys" in cfg: + # if there are keys in cloud-config, use them + key2file = { + "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), + "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), + "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), + "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), + "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), + "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), + } + + for key, val in cfg["ssh_keys"].items(): + if key in key2file: + util.write_file(key2file[key][0], val, key2file[key][1]) + + priv2pub = {'rsa_private': 'rsa_public', 'dsa_private': 'dsa_public', + 'ecdsa_private': 'ecdsa_public', } + + cmd = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' + for priv, pub in priv2pub.iteritems(): + if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: + continue + pair = (key2file[priv][0], key2file[pub][0]) + subprocess.call(('sh', '-xc', cmd % pair)) + log.debug("generated %s from %s" % pair) + else: + # if not, generate them + for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', + ['rsa', 'dsa', 'ecdsa']): + keyfile = '/etc/ssh/ssh_host_%s_key' % keytype + if not os.path.exists(keyfile): + subprocess.call(['ssh-keygen', '-t', keytype, '-N', '', + '-f', keyfile]) + + util.restorecon_if_possible('/etc/ssh', recursive=True) + + try: + user = util.get_cfg_option_str(cfg, 'user') + disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) + disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", + DISABLE_ROOT_OPTS) + keys = cloud.get_public_ssh_keys() + + if "ssh_authorized_keys" in cfg: + cfgkeys = cfg["ssh_authorized_keys"] + keys.extend(cfgkeys) + + apply_credentials(keys, user, disable_root, disable_root_opts, log) + except: + util.logexc(log) + log.warn("applying credentials failed!\n") + + +def apply_credentials(keys, user, disable_root, + disable_root_opts=DISABLE_ROOT_OPTS, log=None): + keys = set(keys) + if user: + sshutil.setup_user_keys(keys, user, '', log) + + if disable_root: + key_prefix = disable_root_opts.replace('$USER', user) + else: + key_prefix = '' + + sshutil.setup_user_keys(keys, 'root', key_prefix, log) diff --git a/cloudinit/transforms/cc_ssh_import_id.py b/cloudinit/transforms/cc_ssh_import_id.py new file mode 100644 index 00000000..bbf5bd83 --- /dev/null +++ b/cloudinit/transforms/cc_ssh_import_id.py @@ -0,0 +1,50 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import traceback + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + user = args[0] + ids = [] + if len(args) > 1: + ids = args[1:] + else: + user = util.get_cfg_option_str(cfg, "user", "ubuntu") + ids = util.get_cfg_option_list_or_str(cfg, "ssh_import_id", []) + + if len(ids) == 0: + return + + cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids + + log.debug("importing ssh ids. cmd = %s" % cmd) + + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) + except OSError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/transforms/cc_timezone.py b/cloudinit/transforms/cc_timezone.py new file mode 100644 index 00000000..e5c9901b --- /dev/null +++ b/cloudinit/transforms/cc_timezone.py @@ -0,0 +1,67 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.CloudConfig import per_instance +from cloudinit import util +import os.path +import shutil + +frequency = per_instance +tz_base = "/usr/share/zoneinfo" + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + timezone = args[0] + else: + timezone = util.get_cfg_option_str(cfg, "timezone", False) + + if not timezone: + return + + tz_file = "%s/%s" % (tz_base, timezone) + + if not os.path.isfile(tz_file): + log.debug("Invalid timezone %s" % tz_file) + raise Exception("Invalid timezone %s" % tz_file) + + try: + fp = open("/etc/timezone", "wb") + fp.write("%s\n" % timezone) + fp.close() + except: + log.debug("failed to write to /etc/timezone") + raise + if os.path.exists("/etc/sysconfig/clock"): + try: + with open("/etc/sysconfig/clock", "w") as fp: + fp.write('ZONE="%s"\n' % timezone) + except: + log.debug("failed to write to /etc/sysconfig/clock") + raise + + try: + shutil.copy(tz_file, "/etc/localtime") + except: + log.debug("failed to copy %s to /etc/localtime" % tz_file) + raise + + log.debug("set timezone to %s" % timezone) + return diff --git a/cloudinit/transforms/cc_update_etc_hosts.py b/cloudinit/transforms/cc_update_etc_hosts.py new file mode 100644 index 00000000..6ad2fca8 --- /dev/null +++ b/cloudinit/transforms/cc_update_etc_hosts.py @@ -0,0 +1,87 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_always +import StringIO + +frequency = per_always + + +def handle(_name, cfg, cloud, log, _args): + (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + + manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) + if manage_hosts in ("True", "true", True, "template"): + # render from template file + try: + if not hostname: + log.info("manage_etc_hosts was set, but no hostname found") + return + + util.render_to_file('hosts', '/etc/hosts', + {'hostname': hostname, 'fqdn': fqdn}) + except Exception: + log.warn("failed to update /etc/hosts") + raise + elif manage_hosts == "localhost": + log.debug("managing 127.0.1.1 in /etc/hosts") + update_etc_hosts(hostname, fqdn, log) + return + else: + if manage_hosts not in ("False", False): + log.warn("Unknown value for manage_etc_hosts. Assuming False") + else: + log.debug("not managing /etc/hosts") + + +def update_etc_hosts(hostname, fqdn, _log): + with open('/etc/hosts', 'r') as etchosts: + header = "# Added by cloud-init\n" + hosts_line = "127.0.1.1\t%s %s\n" % (fqdn, hostname) + need_write = False + need_change = True + new_etchosts = StringIO.StringIO() + for line in etchosts: + split_line = [s.strip() for s in line.split()] + if len(split_line) < 2: + new_etchosts.write(line) + continue + if line == header: + continue + ip, hosts = split_line[0], split_line[1:] + if ip == "127.0.1.1": + if sorted([hostname, fqdn]) == sorted(hosts): + need_change = False + if need_change == True: + line = "%s%s" % (header, hosts_line) + need_change = False + need_write = True + new_etchosts.write(line) + etchosts.close() + if need_change == True: + new_etchosts.write("%s%s" % (header, hosts_line)) + need_write = True + if need_write == True: + new_etcfile = open('/etc/hosts', 'wb') + new_etcfile.write(new_etchosts.getvalue()) + new_etcfile.close() + new_etchosts.close() + return diff --git a/cloudinit/transforms/cc_update_hostname.py b/cloudinit/transforms/cc_update_hostname.py new file mode 100644 index 00000000..b9d1919a --- /dev/null +++ b/cloudinit/transforms/cc_update_hostname.py @@ -0,0 +1,101 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import errno +from cloudinit.CloudConfig import per_always + +frequency = per_always + + +def handle(_name, cfg, cloud, log, _args): + if util.get_cfg_option_bool(cfg, "preserve_hostname", False): + log.debug("preserve_hostname is set. not updating hostname") + return + + (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) + try: + prev = "%s/%s" % (cloud.get_cpath('data'), "previous-hostname") + update_hostname(hostname, prev, log) + except Exception: + log.warn("failed to set hostname\n") + raise + + +# read hostname from a 'hostname' file +# allow for comments and stripping line endings. +# if file doesn't exist, or no contents, return default +def read_hostname(filename, default=None): + try: + fp = open(filename, "r") + lines = fp.readlines() + fp.close() + for line in lines: + hpos = line.find("#") + if hpos != -1: + line = line[0:hpos] + line = line.rstrip() + if line: + return line + except IOError as e: + if e.errno != errno.ENOENT: + raise + return default + + +def update_hostname(hostname, prev_file, log): + etc_file = "/etc/hostname" + + hostname_prev = None + hostname_in_etc = None + + try: + hostname_prev = read_hostname(prev_file) + except Exception as e: + log.warn("Failed to open %s: %s" % (prev_file, e)) + + try: + hostname_in_etc = read_hostname(etc_file) + except: + log.warn("Failed to open %s" % etc_file) + + update_files = [] + if not hostname_prev or hostname_prev != hostname: + update_files.append(prev_file) + + if (not hostname_in_etc or + (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): + update_files.append(etc_file) + + try: + for fname in update_files: + util.write_file(fname, "%s\n" % hostname, 0644) + log.debug("wrote %s to %s" % (hostname, fname)) + except: + log.warn("failed to write hostname to %s" % fname) + + if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: + log.debug("%s differs from %s. assuming user maintained" % + (prev_file, etc_file)) + + if etc_file in update_files: + log.debug("setting hostname to %s" % hostname) + subprocess.Popen(['hostname', hostname]).communicate() diff --git a/cloudinit/user_data/boot_hook.py b/cloudinit/user_data/boot_hook.py deleted file mode 100644 index 87e7a3ec..00000000 --- a/cloudinit/user_data/boot_hook.py +++ /dev/null @@ -1,66 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import log as logging -from cloudinit import user_data as ud -from cloudinit import util - -from cloudinit.settings import (PER_ALWAYS) - -LOG = logging.getLogger(__name__) - - -class BootHookPartHandler(ud.PartHandler): - def __init__(self, boothook_dir, instance_id): - ud.PartHandler.__init__(self, PER_ALWAYS) - self.boothook_dir = boothook_dir - self.instance_id = instance_id - - def list_types(self): - return [ - ud.type_from_starts_with("#cloud-boothook"), - ] - - def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype in ud.CONTENT_SIGNALS: - return - - filename = util.clean_filename(filename) - payload = util.dos2unix(payload) - prefix = "#cloud-boothook" - start = 0 - if payload.startswith(prefix): - start = len(prefix) + 1 - - filepath = os.path.join(self.boothook_dir, filename) - util.write_file(filepath, payload[start:], 0700) - try: - env = os.environ.copy() - env['INSTANCE_ID'] = str(self.instance_id) - util.subp([filepath], env=env) - except util.ProcessExecutionError as e: - LOG.error("Boothooks script %s execution error %s", filepath, e) - except Exception as e: - LOG.exception(("Boothooks unknown " - "error %s when running %s"), e, filepath) -- cgit v1.2.3 From 76a7160d4a4eb70e946cd29317ace1fe93ffe5d5 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:35:38 -0700 Subject: Moved the user_data directory back to a user_data.py and made a handler subdir for just the internal handlers. --- cloudinit/handlers/cloud_config.py | 56 +++++++++++++++++++++++++++++++++++ cloudinit/handlers/shell_script.py | 53 +++++++++++++++++++++++++++++++++ cloudinit/handlers/upstart_job.py | 59 +++++++++++++++++++++++++++++++++++++ cloudinit/user_data/cloud_config.py | 56 ----------------------------------- cloudinit/user_data/shell_script.py | 53 --------------------------------- cloudinit/user_data/upstart_job.py | 59 ------------------------------------- 6 files changed, 168 insertions(+), 168 deletions(-) create mode 100644 cloudinit/handlers/cloud_config.py create mode 100644 cloudinit/handlers/shell_script.py create mode 100644 cloudinit/handlers/upstart_job.py delete mode 100644 cloudinit/user_data/cloud_config.py delete mode 100644 cloudinit/user_data/shell_script.py delete mode 100644 cloudinit/user_data/upstart_job.py diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py new file mode 100644 index 00000000..f0e88eeb --- /dev/null +++ b/cloudinit/handlers/cloud_config.py @@ -0,0 +1,56 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit import log as logging +from cloudinit import user_data as ud +from cloudinit import util + +from cloudinit.settings import (PER_ALWAYS) + +LOG = logging.getLogger(__name__) + + +class CloudConfigPartHandler(ud.PartHandler): + def __init__(self, cloud_fn): + ud.PartHandler.__init__(self, PER_ALWAYS) + self.cloud_buf = [] + self.cloud_fn = cloud_fn + + def list_types(self): + return [ + ud.type_from_starts_with("#cloud-config"), + ] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype == ud.CONTENT_START: + self.cloud_buf = [] + return + + if ctype == ud.CONTENT_END: + payload = "\n".join(self.cloud_buf) + util.write_file(self.cloud_fn, payload, 0600) + self.cloud_buf = [] + return + + filename = util.clean_filename(filename) + entry = "\n".join(["#%s" % (filename), str(payload)]) + self.cloud_buf.append(entry) diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py new file mode 100644 index 00000000..564e4623 --- /dev/null +++ b/cloudinit/handlers/shell_script.py @@ -0,0 +1,53 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os + +from cloudinit import log as logging +from cloudinit import user_data as ud +from cloudinit import util + +from cloudinit.settings import (PER_ALWAYS) + +LOG = logging.getLogger(__name__) + + +class ShellScriptPartHandler(ud.PartHandler): + + def __init__(self, script_dir): + ud.PartHandler.__init__(self, PER_ALWAYS) + self.script_dir = script_dir + + def list_types(self): + return [ + ud.type_from_starts_with("#!"), + ] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype in ud.CONTENT_SIGNALS: + # TODO: maybe delete existing things here + return + + filename = util.clean_filename(filename) + payload = util.dos2unix(payload) + path = os.path.join(self.script_dir, filename) + util.write_file(path, payload, 0700) diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py new file mode 100644 index 00000000..568a644a --- /dev/null +++ b/cloudinit/handlers/upstart_job.py @@ -0,0 +1,59 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import os + +from cloudinit import log as logging +from cloudinit import user_data as ud +from cloudinit import util + +from cloudinit.settings import (PER_INSTANCE) + +LOG = logging.getLogger(__name__) + + +class UpstartJobPartHandler(ud.PartHandler): + def __init__(self, upstart_dir): + ud.PartHandler.__init__(self, PER_INSTANCE) + self.upstart_dir = upstart_dir + + def list_types(self): + return [ + ud.type_from_starts_with("#upstart-job"), + ] + + def _handle_part(self, _data, ctype, filename, payload, frequency): + if ctype in ud.CONTENT_SIGNALS: + return + + filename = util.clean_filename(filename) + (_name, ext) = os.path.splitext(filename) + if not ext: + ext = '' + ext = ext.lower() + if ext != ".conf": + filename = filename + ".conf" + + payload = util.dos2unix(payload) + path = os.path.join(self.upstart_dir, filename) + util.write_file(path, payload, 0644) diff --git a/cloudinit/user_data/cloud_config.py b/cloudinit/user_data/cloud_config.py deleted file mode 100644 index f0e88eeb..00000000 --- a/cloudinit/user_data/cloud_config.py +++ /dev/null @@ -1,56 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import log as logging -from cloudinit import user_data as ud -from cloudinit import util - -from cloudinit.settings import (PER_ALWAYS) - -LOG = logging.getLogger(__name__) - - -class CloudConfigPartHandler(ud.PartHandler): - def __init__(self, cloud_fn): - ud.PartHandler.__init__(self, PER_ALWAYS) - self.cloud_buf = [] - self.cloud_fn = cloud_fn - - def list_types(self): - return [ - ud.type_from_starts_with("#cloud-config"), - ] - - def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype == ud.CONTENT_START: - self.cloud_buf = [] - return - - if ctype == ud.CONTENT_END: - payload = "\n".join(self.cloud_buf) - util.write_file(self.cloud_fn, payload, 0600) - self.cloud_buf = [] - return - - filename = util.clean_filename(filename) - entry = "\n".join(["#%s" % (filename), str(payload)]) - self.cloud_buf.append(entry) diff --git a/cloudinit/user_data/shell_script.py b/cloudinit/user_data/shell_script.py deleted file mode 100644 index 564e4623..00000000 --- a/cloudinit/user_data/shell_script.py +++ /dev/null @@ -1,53 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import log as logging -from cloudinit import user_data as ud -from cloudinit import util - -from cloudinit.settings import (PER_ALWAYS) - -LOG = logging.getLogger(__name__) - - -class ShellScriptPartHandler(ud.PartHandler): - - def __init__(self, script_dir): - ud.PartHandler.__init__(self, PER_ALWAYS) - self.script_dir = script_dir - - def list_types(self): - return [ - ud.type_from_starts_with("#!"), - ] - - def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype in ud.CONTENT_SIGNALS: - # TODO: maybe delete existing things here - return - - filename = util.clean_filename(filename) - payload = util.dos2unix(payload) - path = os.path.join(self.script_dir, filename) - util.write_file(path, payload, 0700) diff --git a/cloudinit/user_data/upstart_job.py b/cloudinit/user_data/upstart_job.py deleted file mode 100644 index 568a644a..00000000 --- a/cloudinit/user_data/upstart_job.py +++ /dev/null @@ -1,59 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -import os - -from cloudinit import log as logging -from cloudinit import user_data as ud -from cloudinit import util - -from cloudinit.settings import (PER_INSTANCE) - -LOG = logging.getLogger(__name__) - - -class UpstartJobPartHandler(ud.PartHandler): - def __init__(self, upstart_dir): - ud.PartHandler.__init__(self, PER_INSTANCE) - self.upstart_dir = upstart_dir - - def list_types(self): - return [ - ud.type_from_starts_with("#upstart-job"), - ] - - def _handle_part(self, _data, ctype, filename, payload, frequency): - if ctype in ud.CONTENT_SIGNALS: - return - - filename = util.clean_filename(filename) - (_name, ext) = os.path.splitext(filename) - if not ext: - ext = '' - ext = ext.lower() - if ext != ".conf": - filename = filename + ".conf" - - payload = util.dos2unix(payload) - path = os.path.join(self.upstart_dir, filename) - util.write_file(path, payload, 0644) -- cgit v1.2.3 From aa69b001e0624ca297b199984b28e9d5401a439f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:36:07 -0700 Subject: This provides a nice little utility class that avoids the python config parser throwing when options + values are being added to unknown sections since it handles the creation and checking that those sections exist before the option is added. Also it adds a little helper that can turn that config into a string. --- cloudinit/cfg.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 cloudinit/cfg.py diff --git a/cloudinit/cfg.py b/cloudinit/cfg.py new file mode 100644 index 00000000..dd8f7baf --- /dev/null +++ b/cloudinit/cfg.py @@ -0,0 +1,72 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import io + +from ConfigParser import (NoSectionError, NoOptionError, RawConfigParser) + + +class DefaultingConfigParser(RawConfigParser): + DEF_INT = 0 + DEF_FLOAT = 0.0 + DEF_BOOLEAN = False + DEF_BASE = None + + def get(self, section, option): + value = self.DEF_BASE + try: + value = RawConfigParser.get(self, section, option) + except NoSectionError: + pass + except NoOptionError: + pass + return value + + def set(self, section, option, value): + if not self.has_section(section) and section.lower() != 'default': + self.add_section(section) + RawConfigParser.set(self, section, option, value) + + def remove_option(self, section, option): + if self.has_option(section, option): + RawConfigParser.remove_option(self, section, option) + + def getboolean(self, section, option): + if not self.has_option(section, option): + return self.DEF_BOOLEAN + return RawConfigParser.getboolean(self, section, option) + + def getfloat(self, section, option): + if not self.has_option(section, option): + return self.DEF_FLOAT + return RawConfigParser.getfloat(self, section, option) + + def getint(self, section, option): + if not self.has_option(section, option): + return self.DEF_INT + return RawConfigParser.getint(self, section, option) + + def stringify(self, header=None): + contents = '' + with io.BytesIO() as outputstream: + self.write(outputstream) + outputstream.flush() + contents = outputstream.getvalue() + if header: + contents = "\n".join([header, contents]) + return contents -- cgit v1.2.3 From a2e588b027dd038f029727935ab07c0bacadcdc9 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:37:17 -0700 Subject: Added a new transform that can be used to show and initial welcome message to users. It is similar to the final msg one, but is used at the start of cloud init instead. --- cloudinit/transforms/cc_welcome.py | 64 ++++++++++++++++++++++++++++++++++++++ templates/welcome_msg.tmpl | 15 +++++++++ 2 files changed, 79 insertions(+) create mode 100644 cloudinit/transforms/cc_welcome.py create mode 100644 templates/welcome_msg.tmpl diff --git a/cloudinit/transforms/cc_welcome.py b/cloudinit/transforms/cc_welcome.py new file mode 100644 index 00000000..0db71125 --- /dev/null +++ b/cloudinit/transforms/cc_welcome.py @@ -0,0 +1,64 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.settings import PER_ALWAYS + +from cloudinit import templater +from cloudinit import util +from cloudinit import version + +import sys + +welcome_message_def = ("Cloud-init v. {{version}} starting stage {{stage}} at " + "{{timestamp}}. Up {{uptime}} seconds.") + + +frequency = PER_ALWAYS + + +def handle(name, cfg, cloud, log, args): + + welcome_msg = util.get_cfg_option_str(cfg, "welcome_msg"): + if not welcome_msg: + tpl_fn = cloud.get_template_filename("welcome_msg") + if tpl_fn: + welcome_msg = util.load_file(tpl_fn) + + if not welcome_msg: + welcome_msg = welcome_message_def + + stage = "??" + if args: + stage = args[0] + + tpl_params = { + 'stage': stage, + 'version': version.version_string(), + 'uptime': util.uptime(), + 'timestamp', util.time_rfc2822(), + } + try: + contents = templater.render_string(welcome_msg, tpl_params) + # TODO use log or sys.stderr?? + sys.stderr.write("%s\n" % (contents)) + except: + util.logexc(log, "Failed to render welcome message template") diff --git a/templates/welcome_msg.tmpl b/templates/welcome_msg.tmpl new file mode 100644 index 00000000..999202cc --- /dev/null +++ b/templates/welcome_msg.tmpl @@ -0,0 +1,15 @@ + _ + _( )_ + ( ) + (_______) + _ \ \ \ \ + _( )_ _ + ( ) _( )_ +(_______) ( ) + \ \ \ \ (_______) + \ \ \ \ + +Cloud-init v. {{version}} +Starting stage: {{stage}} on {{timestamp}} +System uptime: {{uptime}} seconds +-- -- cgit v1.2.3 From a4f3d2d118a963b857ebf4bfc6a2687ccd8bd8d7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:38:32 -0700 Subject: Moved the common user data classes and functionality back to this file since for now it seems to make organizational sense to put it here. --- cloudinit/user_data.py | 383 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 383 insertions(+) create mode 100644 cloudinit/user_data.py diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py new file mode 100644 index 00000000..64fc2734 --- /dev/null +++ b/cloudinit/user_data.py @@ -0,0 +1,383 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import os +import glob + +import email + +from email.mime.base import MIMEBase + +from cloudinit import importer +from cloudinit import log as logging +from cloudinit import util + +from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, FREQUENCIES) + +LOG = logging.getLogger(__name__) + +# Special content types that signal the start and end of processing +CONTENT_END = "__end__" +CONTENT_START = "__begin__" +CONTENT_SIGNALS = [CONTENT_START, CONTENT_END] + +# Used when a part-handler type is encountered +# to allow for registration of new types. +PART_CONTENT_TYPES = ["text/part-handler"] +PART_HANDLER_FN_TMPL = 'part-handler-%03d' + +# For parts without filenames +PART_FN_TPL = 'part-%03d' + +# Used as the content type when a message is not multipart +# and it doesn't contain its own content-type +NOT_MULTIPART_TYPE = "text/x-not-multipart" +OCTET_TYPE = 'application/octet-stream' + +# Different file beginnings to there content type +INCLUSION_TYPES_MAP = { + '#include': 'text/x-include-url', + '#include-once': 'text/x-include-once-url', + '#!': 'text/x-shellscript', + '#cloud-config': 'text/cloud-config', + '#upstart-job': 'text/upstart-job', + '#part-handler': 'text/part-handler', + '#cloud-boothook': 'text/cloud-boothook', + '#cloud-config-archive': 'text/cloud-config-archive', +} + +# Sorted longest first +INCLUSION_SRCH = sorted(INCLUSION_TYPES_MAP.keys(), key=(lambda e: 0 - len(e))) + +# Various special content types +TYPE_NEEDED = ["text/plain", "text/x-not-multipart"] +INCLUDE_TYPES = ['text/x-include-url', 'text/x-include-once-url'] +ARCHIVE_TYPES = ["text/cloud-config-archive"] +UNDEF_TYPE = "text/plain" +ARCHIVE_UNDEF_TYPE = "text/cloud-config" +OCTET_TYPE = 'application/octet-stream' + +# Msg header used to track attachments +ATTACHMENT_FIELD = 'Number-Attachments' + + +class UserDataProcessor(object): + def __init__(self, paths): + self.paths = paths + + def process(self, blob): + base_msg = ud.convert_string(blob) + process_msg = MIMEMultipart() + self._process_msg(base_msg, process_msg) + return process_msg + + def _process_msg(self, base_msg, append_msg): + for part in base_msg.walk(): + # multipart/* are just containers + if part.get_content_maintype() == 'multipart': + continue + + ctype = None + ctype_orig = part.get_content_type() + payload = part.get_payload(decode=True) + + if not ctype_orig: + ctype_orig = UNDEF_TYPE + + if ctype_orig in TYPE_NEEDED: + ctype = ud.type_from_starts_with(payload) + + if ctype is None: + ctype = ctype_orig + + if ctype in INCLUDE_TYPES: + self._do_include(payload, append_msg) + continue + + if ctype in ARCHIVE_TYPES: + self._explode_archive(payload, append_msg) + continue + + if 'Content-Type' in base_msg: + base_msg.replace_header('Content-Type', ctype) + else: + base_msg['Content-Type'] = ctype + + self._attach_part(append_msg, part) + + def _get_include_once_filename(self, entry): + entry_fn = util.hash_blob(entry, 'md5', 64) + return os.path.join(self.paths.get_ipath_cur('data'), + 'urlcache', entry_fn) + + def _do_include(self, content, append_msg): + # is just a list of urls, one per line + # also support '#include ' + for line in content.splitlines(): + includeonce = False + if line in ("#include", "#include-once"): + continue + if line.startswith("#include-once"): + line = line[len("#include-once"):].lstrip() + includeonce = True + elif line.startswith("#include"): + line = line[len("#include"):].lstrip() + if line.startswith("#"): + continue + include_url = line.strip() + if not include_url: + continue + + includeonce_filename = self._get_include_once_filename(include_url) + if includeonce and os.path.isfile(includeonce_filename): + content = util.load_file(includeonce_filename) + else: + (content, st) = url_helper.readurl(include_url) + if includeonce and url_helper.ok_http_code(st): + util.write_file(includeonce_filename, content, mode=0600) + if not url_helper.ok_http_code(st): + content = '' + + new_msg = ud.convert_string(content) + self._process_msg(new_msg, append_msg) + + def _explode_archive(self, archive, append_msg): + entries = util.load_yaml(archive, default=[], allowed=[list, set]) + for ent in entries: + # ent can be one of: + # dict { 'filename' : 'value', 'content' : + # 'value', 'type' : 'value' } + # filename and type not be present + # or + # scalar(payload) + if isinstance(ent, (str, basestring)): + ent = {'content': ent} + if not isinstance(ent, (dict)): + # TODO raise? + continue + + content = ent.get('content', '') + mtype = ent.get('type') + if not mtype: + mtype = ud.type_from_starts_with(content, ARCHIVE_UNDEF_TYPE) + + maintype, subtype = mtype.split('/', 1) + if maintype == "text": + msg = MIMEText(content, _subtype=subtype) + else: + msg = MIMEBase(maintype, subtype) + msg.set_payload(content) + + if 'filename' in ent: + msg.add_header('Content-Disposition', 'attachment', + filename=ent['filename']) + + for header in ent.keys(): + if header in ('content', 'filename', 'type'): + continue + msg.add_header(header, ent['header']) + + self._attach_part(append_msg, msg) + + def _multi_part_count(self, outer_msg, new_count=None): + """ + Return the number of attachments to this MIMEMultipart by looking + at its 'Number-Attachments' header. + """ + if ATTACHMENT_FIELD not in outer_msg: + outer_msg[ATTACHMENT_FIELD] = '0' + + if new_count is not None: + outer_msg.replace_header(ATTACHMENT_FIELD, str(new_count)) + + fetched_count = 0 + try: + fetched_count = int(outer_msg.get(ATTACHMENT_FIELD)) + except (ValueError, TypeError): + outer_msg.replace_header(ATTACHMENT_FIELD, str(fetched_count)) + return fetched_count + + def _attach_part(self, outer_msg, part): + """ + Attach an part to an outer message. outermsg must be a MIMEMultipart. + Modifies a header in the message to keep track of number of attachments. + """ + cur = self._multi_part_count(outer_msg) + if not part.get_filename(): + fn = PART_FN_TPL % (cur + 1) + part.add_header('Content-Disposition', 'attachment', filename=fn) + outer_msg.attach(part) + self._multi_part_count(outer_msg, cur + 1) + + +class PartHandler(object): + def __init__(self, frequency, version=2): + self.handler_version = version + self.frequency = frequency + + def __repr__(self): + return "%s: [%s]" % (util.obj_name(self), self.list_types()) + + def list_types(self): + raise NotImplementedError() + + def handle_part(self, data, ctype, filename, payload, frequency): + return self._handle_part(data, ctype, filename, payload, frequency) + + def _handle_part(self, data, ctype, filename, payload, frequency): + raise NotImplementedError() + + +def fixup_module(mod, def_freq=PER_INSTANCE): + if not hasattr(mod, "handler_version"): + setattr(mod, "handler_version", 1) + if not hasattr(mod, 'list_types'): + def empty_types(): + return [] + setattr(mod, 'list_types', empty_types) + if not hasattr(mod, 'frequency'): + setattr(mod, 'frequency', def_freq) + else: + freq = mod.frequency + if freq and freq not in FREQUENCIES: + LOG.warn("Module %s has an unknown frequency %s", mod, freq) + if not hasattr(mod, 'handle_part'): + def empty_handler(_data, _ctype, _filename, _payload): + pass + setattr(mod, 'handle_part', empty_handler) + return mod + + +def run_part(mod, data, ctype, filename, payload, frequency): + mod_freq = mod.frequency + if not (mod_freq == PER_ALWAYS or + (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)): + return + mod_ver = mod.handler_version + try: + if mod_ver == 1: + mod.handle_part(data, ctype, filename, payload) + else: + mod.handle_part(data, ctype, filename, payload, frequency) + except: + util.logexc(LOG, ("Failed calling mod %s (%s, %s, %s)" + " with frequency %s"), + mod, ctype, filename, + mod_ver, frequency) + + +def call_begin(mod, data, frequency): + run_part(mod, data, CONTENT_START, None, None, frequency) + + +def call_end(mod, data, frequency): + run_part(mod, data, CONTENT_END, None, None, frequency) + + +def walker_handle_handler(pdata, _ctype, _filename, payload): + curcount = pdata['handlercount'] + modname = PART_HANDLER_FN_TMPL % (curcount) + frequency = pdata['frequency'] + modfname = os.path.join(pdata['handlerdir'], "%s" % (modname)) + if not modfname.endswith(".py"): + modfname = "%s.py" % (modfname) + # TODO: Check if path exists?? + util.write_file(modfname, payload, 0600) + handlers = pdata['handlers'] + try: + mod = fixup_module(importer.import_module(modname)) + handlers.register(mod) + call_begin(mod, pdata['data'], frequency) + pdata['handlercount'] = curcount + 1 + except: + util.logexc(LOG, "Failed at registered python file: %s", modfname) + + +def walker_callback(pdata, ctype, filename, payload): + if ctype in PART_CONTENT_TYPES: + walker_handle_handler(pdata, ctype, filename, payload) + return + handlers = pdata['handlers'] + if ctype not in handlers: + if ctype == NOT_MULTIPART_TYPE: + # Extract the first line or 24 bytes for displaying in the log + start = payload.split("\n", 1)[0][:24] + if start < payload: + details = "starting '%s...'" % start.encode("string-escape") + else: + details = repr(payload) + LOG.warning("Unhandled non-multipart userdata: %s", details) + return + run_part(handlers[ctype], pdata['data'], ctype, filename, + payload, pdata['frequency']) + + +# Callback is a function that will be called with +# (data, content_type, filename, payload) +def walk(msg, callback, data): + partnum = 0 + for part in msg.walk(): + # multipart/* are just containers + if part.get_content_maintype() == 'multipart': + continue + + ctype = part.get_content_type() + if ctype is None: + ctype = OCTET_TYPE + + filename = part.get_filename() + if not filename: + filename = PART_FN_TPL % (partnum) + + callback(data, ctype, filename, part.get_payload(decode=True)) + partnum = partnum + 1 + + +# Coverts a raw string into a mime message +def convert_string(raw_data, headers=None): + if not raw_data: + raw_data = '' + if not headers: + headers = {} + data = util.decomp_str(raw_data) + if "mime-version:" in data[0:4096].lower(): + msg = email.message_from_string(data) + for (key, val) in headers.items(): + if key in msg: + msg.replace_header(key, val) + else: + msg[key] = val + else: + mtype = headers.get("Content-Type", NOT_MULTIPART_TYPE) + maintype, subtype = mtype.split("/", 1) + msg = MIMEBase(maintype, subtype, *headers) + msg.set_payload(data) + return msg + + +def type_from_starts_with(payload, default=None): + for text in INCLUSION_SRCH: + if payload.startswith(text): + return INCLUSION_TYPES_MAP[text] + return default + -- cgit v1.2.3 From aed43266d2e7b376bc0077f0c763283a159a29e1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:39:42 -0700 Subject: Added a python file which can be used internally to determine the version of cloud init that is running for internal messaging, if desired --- cloudinit/version.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 cloudinit/version.py diff --git a/cloudinit/version.py b/cloudinit/version.py new file mode 100644 index 00000000..4599910c --- /dev/null +++ b/cloudinit/version.py @@ -0,0 +1,27 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from distutils import version as vr + + +def version(): + return vr.StrictVersion("0.7.0") + + +def version_string(): + return str(version()) -- cgit v1.2.3 From 09f6384ac5694be18bef4872c9f19b9601b48f8b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:39:54 -0700 Subject: Added a hosts template file (in tempita format) that provides the base of a "/etc/hosts" file for rhel like systems. --- templates/hosts.redhat.tmpl | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 templates/hosts.redhat.tmpl diff --git a/templates/hosts.redhat.tmpl b/templates/hosts.redhat.tmpl new file mode 100644 index 00000000..cfc40668 --- /dev/null +++ b/templates/hosts.redhat.tmpl @@ -0,0 +1,22 @@ +{{# This file /etc/cloud/templates/hosts.tmpl is only utilized + if enabled in cloud-config. Specifically, in order to enable it + you need to add the following to config: + manage_etc_hosts: True}} +# +# Your system has configured 'manage_etc_hosts' as True. +# As a result, if you wish for changes to this file to persist +# then you will need to either +# a.) make changes to the master file in /etc/cloud/templates/hosts.tmpl +# b.) change or remove the value of 'manage_etc_hosts' in +# /etc/cloud/cloud.cfg or cloud-config from user-data +# +# The following lines are desirable for IPv4 capable hosts +127.0.0.1 {{fqdn}} {{hostname}} +127.0.0.1 localhost.localdomain localhost +127.0.0.1 localhost4.localdomain4 localhost4 + +# The following lines are desirable for IPv6 capable hosts +::1 {{fqdn}} {{hostname}} +::1 localhost.localdomain localhost +::1 localhost6.localdomain6 localhost6 + -- cgit v1.2.3 From 19f5aa0f4bb449a826a0e6a96f318dba4d814443 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:40:55 -0700 Subject: Renamed hosts to match the ubuntu distro. Also converted all the other templates to tempita format instead of the more complicated (and more dependent on other packages) cheetah format. --- templates/chef_client.rb.tmpl | 8 ++-- templates/default-locale.tmpl | 2 +- templates/hosts.tmpl | 25 ----------- templates/hosts.ubuntu.tmpl | 24 ++++++++++ templates/sources.list.tmpl | 101 +++++++++++++++++++++--------------------- 5 files changed, 79 insertions(+), 81 deletions(-) delete mode 100644 templates/hosts.tmpl create mode 100644 templates/hosts.ubuntu.tmpl diff --git a/templates/chef_client.rb.tmpl b/templates/chef_client.rb.tmpl index d3d9a922..35123ced 100644 --- a/templates/chef_client.rb.tmpl +++ b/templates/chef_client.rb.tmpl @@ -1,12 +1,12 @@ log_level :info log_location "/var/log/chef/client.log" ssl_verify_mode :verify_none -validation_client_name "$validation_name" +validation_client_name "{{validation_name}}" validation_key "/etc/chef/validation.pem" client_key "/etc/chef/client.pem" -chef_server_url "$server_url" -environment "$environment" -node_name "$node_name" +chef_server_url "{{server_url}}" +environment "{{environment}}" +node_name "{{node_name}}" json_attribs "/etc/chef/firstboot.json" file_cache_path "/var/cache/chef" file_backup_path "/var/backups/chef" diff --git a/templates/default-locale.tmpl b/templates/default-locale.tmpl index 7940672b..5ee7e454 100644 --- a/templates/default-locale.tmpl +++ b/templates/default-locale.tmpl @@ -1 +1 @@ -LANG="$locale" +LANG="{{locale}}" diff --git a/templates/hosts.tmpl b/templates/hosts.tmpl deleted file mode 100644 index ae120b02..00000000 --- a/templates/hosts.tmpl +++ /dev/null @@ -1,25 +0,0 @@ -## This file (/etc/cloud/templates/hosts.tmpl) is only utilized -## if enabled in cloud-config. Specifically, in order to enable it -## you need to add the following to config: -## manage_etc_hosts: True -## -## Note, double-hash commented lines will not appear in /etc/hosts -# -# Your system has configured 'manage_etc_hosts' as True. -# As a result, if you wish for changes to this file to persist -# then you will need to either -# a.) make changes to the master file in /etc/cloud/templates/hosts.tmpl -# b.) change or remove the value of 'manage_etc_hosts' in -# /etc/cloud/cloud.cfg or cloud-config from user-data -# -## The value '$hostname' will be replaced with the local-hostname -127.0.1.1 $fqdn $hostname -127.0.0.1 localhost - -# The following lines are desirable for IPv6 capable hosts -::1 ip6-localhost ip6-loopback -fe00::0 ip6-localnet -ff00::0 ip6-mcastprefix -ff02::1 ip6-allnodes -ff02::2 ip6-allrouters -ff02::3 ip6-allhosts diff --git a/templates/hosts.ubuntu.tmpl b/templates/hosts.ubuntu.tmpl new file mode 100644 index 00000000..9eebe971 --- /dev/null +++ b/templates/hosts.ubuntu.tmpl @@ -0,0 +1,24 @@ +{{# This file /etc/cloud/templates/hosts.tmpl is only utilized + if enabled in cloud-config. Specifically, in order to enable it + you need to add the following to config: + manage_etc_hosts: True}} +# +# Your system has configured 'manage_etc_hosts' as True. +# As a result, if you wish for changes to this file to persist +# then you will need to either +# a.) make changes to the master file in /etc/cloud/templates/hosts.tmpl +# b.) change or remove the value of 'manage_etc_hosts' in +# /etc/cloud/cloud.cfg or cloud-config from user-data +# +# The following lines are desirable for IPv4 capable hosts +127.0.1.1 {{fqdn}} {{hostname}} +127.0.0.1 localhost + +# The following lines are desirable for IPv6 capable hosts +::1 ip6-localhost ip6-loopback +fe00::0 ip6-localnet +ff00::0 ip6-mcastprefix +ff02::1 ip6-allnodes +ff02::2 ip6-allrouters +ff02::3 ip6-allhosts + diff --git a/templates/sources.list.tmpl b/templates/sources.list.tmpl index f702025f..8acbd7d5 100644 --- a/templates/sources.list.tmpl +++ b/templates/sources.list.tmpl @@ -1,60 +1,59 @@ -\## Note, this file is written by cloud-init on first boot of an instance -\## modifications made here will not survive a re-bundle. -\## if you wish to make changes you can: -\## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg -\## or do the same in user-data -\## b.) add sources in /etc/apt/sources.list.d -\## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl -\### +# Note, this file is written by cloud-init on first boot of an instance +# modifications made here will not survive a re-bundle. +# if you wish to make changes you can: +# a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg +# or do the same in user-data +# b.) add sources in /etc/apt/sources.list.d +# c.) make changes to template file /etc/cloud/templates/sources.list.tmpl # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to # newer versions of the distribution. -deb $mirror $codename main -deb-src $mirror $codename main +deb {{mirror}} {{codename}} main +deb-src {{mirror}} {{codename}} main -\## Major bug fix updates produced after the final release of the -\## distribution. -deb $mirror $codename-updates main -deb-src $mirror $codename-updates main +# Major bug fix updates produced after the final release of the +# distribution. +deb {{mirror}} {{codename}}-updates main +deb-src {{mirror}} {{codename}}-updates main -\## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu -\## team. Also, please note that software in universe WILL NOT receive any -\## review or updates from the Ubuntu security team. -deb $mirror $codename universe -deb-src $mirror $codename universe -deb $mirror $codename-updates universe -deb-src $mirror $codename-updates universe +# N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +# team. Also, please note that software in universe WILL NOT receive any +# review or updates from the Ubuntu security team. +deb {{mirror}} {{codename}} universe +deb-src {{mirror}} {{codename}} universe +deb {{mirror}} {{codename}}-updates universe +deb-src {{mirror}} {{codename}}-updates universe -\## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu -\## team, and may not be under a free licence. Please satisfy yourself as to -\## your rights to use the software. Also, please note that software in -\## multiverse WILL NOT receive any review or updates from the Ubuntu -\## security team. -# deb $mirror $codename multiverse -# deb-src $mirror $codename multiverse -# deb $mirror $codename-updates multiverse -# deb-src $mirror $codename-updates multiverse +# N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +# team, and may not be under a free licence. Please satisfy yourself as to +# your rights to use the software. Also, please note that software in +# multiverse WILL NOT receive any review or updates from the Ubuntu +# security team. +# deb {{mirror}} {{codename}} multiverse +# deb-src {{mirror}} {{codename}} multiverse +# deb {{mirror}} {{codename}}-updates multiverse +# deb-src {{mirror}} {{codename}}-updates multiverse -\## Uncomment the following two lines to add software from the 'backports' -\## repository. -\## N.B. software from this repository may not have been tested as -\## extensively as that contained in the main release, although it includes -\## newer versions of some applications which may provide useful features. -\## Also, please note that software in backports WILL NOT receive any review -\## or updates from the Ubuntu security team. -# deb $mirror $codename-backports main restricted universe multiverse -# deb-src $mirror $codename-backports main restricted universe multiverse +# Uncomment the following two lines to add software from the 'backports' +# repository. +# N.B. software from this repository may not have been tested as +# extensively as that contained in the main release, although it includes +# newer versions of some applications which may provide useful features. +# Also, please note that software in backports WILL NOT receive any review +# or updates from the Ubuntu security team. +# deb {{mirror}} {{codename}}-backports main restricted universe multiverse +# deb-src {{mirror}} {{codename}}-backports main restricted universe multiverse -\## Uncomment the following two lines to add software from Canonical's -\## 'partner' repository. -\## This software is not part of Ubuntu, but is offered by Canonical and the -\## respective vendors as a service to Ubuntu users. -# deb http://archive.canonical.com/ubuntu $codename partner -# deb-src http://archive.canonical.com/ubuntu $codename partner +# Uncomment the following two lines to add software from Canonical's +# 'partner' repository. +# This software is not part of Ubuntu, but is offered by Canonical and the +# respective vendors as a service to Ubuntu users. +# deb http://archive.canonical.com/ubuntu {{codename}} partner +# deb-src http://archive.canonical.com/ubuntu {{codename}} partner -deb http://security.ubuntu.com/ubuntu $codename-security main -deb-src http://security.ubuntu.com/ubuntu $codename-security main -deb http://security.ubuntu.com/ubuntu $codename-security universe -deb-src http://security.ubuntu.com/ubuntu $codename-security universe -# deb http://security.ubuntu.com/ubuntu $codename-security multiverse -# deb-src http://security.ubuntu.com/ubuntu $codename-security multiverse +deb http://security.ubuntu.com/ubuntu {{codename}}-security main +deb-src http://security.ubuntu.com/ubuntu {{codename}}-security main +deb http://security.ubuntu.com/ubuntu {{codename}}-security universe +deb-src http://security.ubuntu.com/ubuntu {{codename}}-security universe +# deb http://security.ubuntu.com/ubuntu {{codename}}-security multiverse +# deb-src http://security.ubuntu.com/ubuntu {{codename}}-security multiverse -- cgit v1.2.3 From a6f387756d74e0542f2247de8cab617043297ced Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:42:05 -0700 Subject: Updated so that pylint and pyflakes will now run over cloudinit/ and bin/ python files --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 0fc6c46b..0a73f987 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,13 @@ +CWD=$(shell pwd) +PY_FILES=$(shell find cloudinit bin -name "*.py") all: test pylint: - pylint cloudinit + $(CWD)/tools/run-pylint $(PY_FILES) pyflakes: - pyflakes . + pyflakes $(PY_FILES) test: nosetests tests/unittests/ -- cgit v1.2.3 From 6e5a65404721e4c05c40102f19cf0d58930887c6 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:43:01 -0700 Subject: Enabled the oauth library, which seems needed for maas datasource. --- Requires | 1 + 1 file changed, 1 insertion(+) diff --git a/Requires b/Requires index 6a7064af..dbf64baf 100644 --- a/Requires +++ b/Requires @@ -1,2 +1,3 @@ Tempita>=0.4 PrettyTable>=0.4 +oauth>=1.0 -- cgit v1.2.3 From d732453a809811e2ea37cd60025ffcfc132329de Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:44:53 -0700 Subject: Continued adding more comments about what each section/option does. Continued adding more system info sections as needed for various components. Enabled the welcome transform. --- config/cloud.cfg | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/config/cloud.cfg b/config/cloud.cfg index 33feec48..f1c43f2f 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -1,8 +1,26 @@ +# The top level settings are used as transforms +# and system configuration. + +# This user will have its passwd adjusted user: ubuntu -disable_root: 1 -preserve_hostname: False +# If this is set, 'root' will not be able to ssh in and they +# will get a message to login instead as the above $user +disable_root: true + +# This will cause the set+update hostname module to not operate (if true) +preserve_hostname: false + +# Example datasource config +# datasource: +# Ec2: +# metadata_urls: [ 'blah.com' ] +# timeout: 5 # (defaults to 50 seconds) +# max_wait: 10 # (defaults to 120 seconds) + +# The transform modules that run in the 'init' stage cloud_init_modules: + - welcome - bootcmd - resizefs - set_hostname @@ -12,6 +30,7 @@ cloud_init_modules: - rsyslog - ssh + # The transform modules that run in the 'config' stage cloud_config_modules: - mounts - ssh-import-id @@ -30,6 +49,7 @@ cloud_config_modules: - runcmd - byobu +# The transform modules that run in the 'final' stage cloud_final_modules: - rightscale_userdata - scripts-per-once @@ -40,9 +60,15 @@ cloud_final_modules: - phone-home - final-message +# System and/or distro specific settings system_info: - paths: - cloud_dir: /var/lib/cloud/ - templates_dir: /etc/cloud/templates/ - upstart_dir: /etc/init/ + # This will affect which distro class gets used distro: ubuntu + # Other config here will be given to the distro class and/or path classes + paths: + cloud_dir: "/var/lib/cloud/" + templates_dir: "/etc/cloud/templates/" + upstart_dir: "/etc/init/" + package_mirror: "http://archive.ubuntu.com/ubuntu" + availability_zone_template: "http://%(zone)s.ec2.archive.ubuntu.com/ubuntu/" + ssh_svcname: ssh -- cgit v1.2.3 From 419b9451684f2c1bc5bef73b26ae8fca1a17383d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:45:52 -0700 Subject: Instead of having the cloud pass large references to its constructor, this has been reduced to actual objects. Added a get template filename helper which can be used to locate template files for various handlers/transforms. Ensured that the config that we give back out is copied, so that it can't be modified by any 'malicous' handlers/transforms. Added helper method cycle_logging that can resetup logging, this is mainly used by the rsyslog transform. --- cloudinit/cloud.py | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index 765e7d3a..b2dfc749 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -20,6 +20,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import copy +import os + from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging @@ -28,18 +31,40 @@ LOG = logging.getLogger(__name__) class Cloud(object): - def __init__(self, datasource, paths, cfg): + def __init__(self, datasource, paths, cfg, distro, runners): self.datasource = datasource self.paths = paths - self.cfg = cfg - self.distro = distros.fetch(cfg, self) - self.runners = helpers.Runners(paths) + self.distro = distro + self._cfg = cfg + self._runners = runners + + # If a transform manipulates logging or logging services + # it is typically useful to cause the logging to be + # setup again. + def cycle_logging(self): + logging.setupLogging(self.cfg) + + @property + def cfg(self): + # Ensure that not indirectly modified + return copy.deepcopy(self._cfg) def run(self, name, functor, args, freq=None, clear_on_fail=False): - return self.runners.run(name, functor, args, freq, clear_on_fail) + return self._runners.run(name, functor, args, freq, clear_on_fail) + + def get_template_filename(self, name): + fn = self.paths.template_tpl % (name) + if not os.path.isfile(fn): + LOG.warn("No template found at %s for template named %s", fn, name) + return None + return fn + # The rest of thes are just useful proxies def get_userdata(self): return self.datasource.get_userdata() + + def get_instance_id(self): + return self.datasource.get_instance_id() def get_public_ssh_keys(self): return self.datasource.get_public_ssh_keys() @@ -47,7 +72,7 @@ class Cloud(object): def get_locale(self): return self.datasource.get_locale() - def get_mirror(self): + def get_local_mirror(self): return self.datasource.get_local_mirror() def get_hostname(self, fqdn=False): -- cgit v1.2.3 From 37c0976f5d94c5abf47a848d92acd76feaf150a7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:48:09 -0700 Subject: Continued adding distro specific functionality to this new parent distro class as needed. --- cloudinit/distros/__init__.py | 133 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 123 insertions(+), 10 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index f7f48d1f..90607668 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -20,26 +20,139 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import contextlib +import abc +import copy from cloudinit import importer +from cloudinit import util + +from StringIO import StringIO + +# TODO: Make this via config?? +IFACE_ACTIONS = { + 'up': ['ifup', '--all'], + 'down': ['ifdown', '--all'], +} class Distro(object): - def __init__(self, cloud): - self.cloud = cloud + __metaclass__ = abc.ABCMeta + + def __init__(self, cfg, runner): + self._runner = runner + self._cfg = util.get_cfg_by_path(cfg, ('system_info', ), {}) + self.name = self._cfg.pop("distro", 'generic') + + @abc.abstractmethod def install_packages(self, pkglist): raise NotImplementedError() - def apply_network(self, settings): + @abc.abstractmethod + def _write_network(self, settings): + # In the future use the python-netcf + # to write this blob out in a distro format + raise NotImplementedError() + + def get_option(self, opt_name, default=None): + return self._cfg.get(opt_name, default) + + @abc.abstractmethod + def set_hostname(self, hostname): raise NotImplementedError() + @abc.abstractmethod + def update_hostname(self, hostname, prev_hostname_fn): + raise NotImplementedError() + + @abc.abstractmethod + def package_command(self, cmd, args=None): + raise NotImplementedError() + + def get_package_mirror(self): + return self.get_option('package_mirror') + + def get_paths(self): + paths = self.get_option("paths") or {} + return copy.deepcopy(paths) + + def apply_network(self, settings, bring_up=True): + # Write it out + self._write_network(settings) + # Now try to bring them up + if bring_up: + self._interface_action('up') + + @abc.abstractmethod + def set_timezone(self, tz): + raise NotImplementedError() + + def _get_localhost_ip(self): + return "127.0.0.1" + + def update_etc_hosts(self, hostname, fqdn): + # Format defined at + # http://unixhelp.ed.ac.uk/CGI/man-cgi?hosts + header = "# Added by cloud-init" + real_header = "%s on %s" % (header, util.time_rfc2822()) + local_ip = self._get_localhost_ip() + hosts_line = "%s\t%s %s" % (local_ip, fqdn, hostname) + new_etchosts = StringIO() + need_write = False + need_change = True + for line in util.load_file("/etc/hosts").splitlines(): + if line.strip().startswith(header): + continue + if not line.strip() or line.strip().startswith("#"): + new_etchosts.write("%s\n" % (line)) + continue + split_line = [s.strip() for s in line.split()] + if len(split_line) < 2: + new_etchosts.write("%s\n" % (line)) + continue + (ip, hosts) = split_line[0], split_line[1:] + if ip == local_ip: + if sorted([hostname, fqdn]) == sorted(hosts): + need_change = False + if need_change: + line = "%s\n%s" % (real_header, hosts_line) + need_change = False + need_write = True + new_etchosts.write("%s\n" % (line)) + if need_change: + new_etchosts.write("%s\n%s\n" % (real_header, hosts_line)) + need_write = True + if need_write: + contents = new_etchosts.getvalue() + util.write_file("/etc/hosts", contents) + + def _interface_action(self, action): + if action not in IFACE_ACTIONS: + raise NotImplementedError("Unknown interface action %s" % (action)) + cmd = IFACE_ACTIONS[action] + try: + LOG.info("Attempting to run %s interface action using command %s", + action, cmd) + (_out, err) = util.subp(cmd) + if len(err): + LOG.warn("Running %s resulted in stderr output: %s", + IF_UP_CMD, err) + return True + except util.ProcessExecutionError as exc: + util.logexc(LOG, "Running %s failed", cmd) + return False + -def fetch(cfg, cloud): - sys_info = cfg.get('system_info', {}) - distro = sys_info.get('distro', 'ubuntu') - mod_name = "%s.%s" % (__name__, distro) - mod = importer.import_module(mod_name) +def fetch(distro_name, mods=(__name__, )): + mod = None + for m in mods: + try: + mod_name = "%s.%s" % (m, distro_name) + mod = importer.import_module(mod_name) + except RuntimeError: + pass + if not mod: + raise RuntimeError("No distribution found for distro %s" % (distro_name)) distro_cls = getattr(mod, 'Distro') - return distro_cls(cloud) \ No newline at end of file + return distro_cls + -- cgit v1.2.3 From 136f3baf4bebc462a7b5f82ffbdd4b2f713b2709 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:49:02 -0700 Subject: Change variable names to reflect usage correctly --- cloudinit/templater.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudinit/templater.py b/cloudinit/templater.py index 5839911c..41315adf 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -29,9 +29,9 @@ def render_from_file(fn, params): return render_string(util.load_file(fn), params, name=fn) -def render_to_file(name, outfile, params): - contents = render_from_file(name, params) - util.write_file(outfile, contents) +def render_to_file(fn, outfn, params): + contents = render_from_file(fn, params) + util.write_file(outfn, contents) def render_string(content, params, name=None): -- cgit v1.2.3 From db4614f3ea4eecd8283b75283effd3e369fb0e48 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:49:28 -0700 Subject: Moved as many distro specific actions to here as should be needed. Mainly this is the following: 1. Hostname updating 2. Package command invocation 3. Network settings writings 4. Timezone setting (needs cleanup) --- cloudinit/distros/ubuntu.py | 81 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 8 deletions(-) diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index ccf2cec4..b8aff03c 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -20,36 +20,101 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from StringIO import StringIO + import os +import socket from cloudinit import distros from cloudinit import log as logging +from cloudinit import templater from cloudinit import util from cloudinit.settings import (PER_INSTANCE) - LOG = logging.getLogger(__name__) class Distro(distros.Distro): def install_packages(self, pkglist): - self.update_package_sources() - self.apt_get('install', pkglist) + self._update_package_sources() + self._apt_get('install', pkglist) - def apply_network(self, settings): + def _write_network(self, settings): util.write_file("/etc/network/interfaces", settings) + def package_command(self, command, args=None): + self._apt_get(command, args) + + def set_hostname(self, hostname): + util.write_file("/etc/hostname", "%s\n" % hostname, 0644) + LOG.debug("Setting hostname to %s", hostname) + util.subp(['hostname', hostname]) + + def update_hostname(self, hostname, prev_file): + hostname_prev = None + prev_name = self._read_hostname(prev_file) + hostname_in_etc = self._read_hostname("/etc/hostname") + update_files = [] + if not hostname_prev or hostname_prev != hostname: + update_files.append(prev_file) + if (not hostname_in_etc or + (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): + update_files.append("/etc/hostname") + for fn in update_files: + try: + util.write_file(fn, "%s\n" % hostname, 0644) + except: + util.logexc(LOG, "Failed to write hostname %s to %s", hostname, fn) + if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: + LOG.debug(("%s differs from /etc/hostname." + " Assuming user maintained hostname."), prev_file) + if "/etc/hostname" in update_files: + LOG.debug("Setting hostname to %s", hostname) + util.subp(['hostname', hostname]) + + def _read_hostname(filename, default=None): + contents = util.load_file(filename, quiet=True) + for line in contents.splitlines(): + hpos = line.find("#") + if hpos != -1: + line = line[0:hpos] + line = line.rstrip() + if line: + return line + return default + + def _get_localhost_ip(self): + # Note: http://www.leonardoborda.com/blog/127-0-1-1-ubuntu-debian/ + return "127.0.1.1" + + def set_timezone(self, tz): + tz_file = os.path.join("/usr/share/zoneinfo", tz) + if not os.path.isfile(tz_file): + raise Exception("Invalid timezone %s, no file found at %s" % (tz, tz_file)) + tz_contents = "%s\n" % tz + util.write_file("/etc/timezone", tz_contents) + # TODO, this should be in a rhel distro subclass?? + if os.path.exists("/etc/sysconfig/clock"): + tz_contents = '"%s"\n' % tz + util.write_file("/etc/sysconfig/clock", tz_contents) + # This ensures that the correct tz will be used for the system + util.copy(tz_file, "/etc/localtime") + + def name(self): + return "ubuntu" + # apt_get top level command (install, update...), and args to pass it - def apt_get(self, tlc, args=None): + def _apt_get(self, tlc, args=None): e = os.environ.copy() e['DEBIAN_FRONTEND'] = 'noninteractive' cmd = ['apt-get', '--option', 'Dpkg::Options::=--force-confold', '--assume-yes', tlc] if args: cmd.extend(args) - util.subp(cmd, env=e) + # Allow the output of this to flow outwards (ie not be captured) + util.subp(cmd, env=e, capture=False) - def update_package_sources(self): - self.cloud.run("update-sources", self.apt_get, ["update"], freq=PER_INSTANCE) \ No newline at end of file + def _update_package_sources(self): + self.runner.run("update-sources", self._apt_get, ["update"], freq=PER_INSTANCE) \ No newline at end of file -- cgit v1.2.3 From 52d8a9617da4cbdb167d8cf4d470967192d17001 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:51:27 -0700 Subject: 1. Handle returning values from handlers better. 2. Throw lock specific exception when semaphores can't be obtained 3. Make the paths constructor config variable named better to its actual usage. --- cloudinit/helpers.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index e5f33a26..2ecda3e9 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -38,12 +38,16 @@ from cloudinit.user_data import upstart_job as up_part LOG = logging.getLogger(__name__) +class LockFailure(Exception): + pass + + class DummySemaphores(object): def __init__(self): pass @contextlib.contextmanager - def lock(self, _name, _freq, _clear_on_fail): + def lock(self, _name, _freq, _clear_on_fail=False): yield True def has_run(self, _name, _freq): @@ -61,7 +65,7 @@ class FileSemaphores(object): self.sem_path = sem_path @contextlib.contextmanager - def lock(self, name, freq, clear_on_fail): + def lock(self, name, freq, clear_on_fail=False): try: yield self._acquire(name, freq) except: @@ -73,15 +77,17 @@ class FileSemaphores(object): sem_file = self._get_path(name, freq) try: util.del_file(sem_file) - except (IOError, OSError): + except (IOError, OSError) as e: + util.logexc(LOG, "Failed deleting semaphore %s", sem_file) return False return True def clear_all(self): try: util.del_dir(self.sem_path) - except (IOError, OSError): - pass + except (IOError, OSError) as e: + LOG.debug("Failed deleting semaphore directory %s due to %s", + self.sem_path, e) def _acquire(self, name, freq): if self.has_run(name, freq): @@ -93,7 +99,8 @@ class FileSemaphores(object): contents = "%s: %s\n" % (os.getpid(), time()) try: util.write_file(sem_file, contents) - except (IOError, OSError): + except (IOError, OSError) as e: + util.logexc(LOG, "Failed writing semaphore file %s", sem_file) return None return sem_file @@ -143,11 +150,14 @@ class Runners(object): return None with sem.lock(name, freq, clear_on_fail) as lk: if not lk: - raise RuntimeError("Failed to acquire lock on %s" % name) + raise LockFailure("Failed to acquire lock for %s" % name) else: LOG.debug("Running %s with args %s using lock %s", functor, args, lk) - return functor(*args) + if isinstance(args, (dict)): + return functor(**args) + else: + return functor(*args) class ContentHandlers(object): @@ -217,12 +227,12 @@ class ContentHandlers(object): class Paths(object): - def __init__(self, sys_info): - self.cloud_dir = sys_info.get('cloud_dir', '/var/lib/cloud') + def __init__(self, path_cfgs, ds=None): + self.cloud_dir = path_cfgs.get('cloud_dir', '/var/lib/cloud') self.instance_link = os.path.join(self.cloud_dir, 'instance') self.boot_finished = os.path.join(self.instance_link, "boot-finished") - self.upstart_conf_d = sys_info.get('upstart_dir') - template_dir = sys_info.get('templates_dir', '/etc/cloud/templates/') + self.upstart_conf_d = path_cfgs.get('upstart_dir') + template_dir = path_cfgs.get('templates_dir', '/etc/cloud/templates/') self.template_tpl = os.path.join(template_dir, '%s.tmpl') self.seed_dir = os.path.join(self.cloud_dir, 'seed') self.lookups = { @@ -237,7 +247,7 @@ class Paths(object): "data": "data", } # Set when a datasource becomes active - self.datasource = None + self.datasource = ds # get_ipath_cur: get the current instance path for an item def get_ipath_cur(self, name=None): @@ -256,6 +266,8 @@ class Paths(object): cpath = os.path.join(cpath, add_on) return cpath + # get_ipath : get the instance path for a name in pathmap + # (/var/lib/cloud/instances//) def _get_ipath(self, name=None): if not self.datasource: return None @@ -268,6 +280,7 @@ class Paths(object): ipath = os.path.join(ipath, add_on) return ipath + # get_ipath : get the instance path for a name in pathmap # (/var/lib/cloud/instances//) def get_ipath(self, name=None): ipath = self._get_ipath(name) -- cgit v1.2.3 From 9f3a59203ae4e51087fb0ad9ad0fa2ad31302c80 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:54:27 -0700 Subject: Ensure that the root logger is manipulated instead of just the cloudinit logger, show how many configs were tried if none succeeded, and for basic logging setup try to mirror more of what is in the default configuration file if all else fails --- cloudinit/log.py | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/cloudinit/log.py b/cloudinit/log.py index 6e7424e1..c247eb9e 100644 --- a/cloudinit/log.py +++ b/cloudinit/log.py @@ -41,16 +41,27 @@ DEBUG = logging.DEBUG NOTSET = logging.NOTSET # Default basic format -DEF_FORMAT = '%(levelname)s: @%(name)s : %(message)s' +DEF_CON_FORMAT = '%(asctime)s - %(filename)s[%(levelname)s]: %(message)s' -def setupBasicLogging(level=INFO, fmt=DEF_FORMAT): - root = getLogger() - console = logging.StreamHandler(sys.stdout) - console.setFormatter(logging.Formatter(fmt)) - console.setLevel(level) +def setupBasicLogging(): + root = logging.getLogger() + # Warnings go to the console + console = logging.StreamHandler(sys.stderr) + console.setFormatter(logging.Formatter(DEF_CON_FORMAT)) + console.setLevel(WARNING) root.addHandler(console) - root.setLevel(level) + # Everything else goes to this file (if we can) + try: + cfile = logging.FileHandler('/var/log/cloud-init.log') + cfile.setFormatter(logging.Formatter(DEF_CON_FORMAT)) + cfile.setLevel(DEBUG) + root.addHandle(cfile) + except (IOError, OSError): + # Likely that u can't write to that file... + # Make console now have DEBUG?? + console.setLevel(DEBUG) + root.setLevel(DEBUG) def setupLogging(cfg=None): @@ -61,7 +72,7 @@ def setupLogging(cfg=None): log_cfgs = [] log_cfg = cfg.get('logcfg') if log_cfg and isinstance(log_cfg, (str, basestring)): - # Ff there is a 'logcfg' entry in the config, + # If there is a 'logcfg' entry in the config, # respect it, it is the old keyname log_cfgs.append(str(log_cfg)) elif "log_cfgs" in cfg and isinstance(cfg['log_cfgs'], (set, list)): @@ -73,20 +84,27 @@ def setupLogging(cfg=None): log_cfgs.append(str(a_cfg)) # See if any of them actually load... + am_tried = 0 am_worked = 0 for log_cfg in log_cfgs: try: - if not os.path.isfile(log_cfg): + am_tried += 1 + # Assume its just a string if not a filename + if log_cfg.startswith("/") and os.path.isfile(log_cfg): + pass + else: log_cfg = StringIO(log_cfg) + # Attempt to load its config logging.config.fileConfig(log_cfg) am_worked += 1 except Exception: pass - # If it didn't work, at least setup a basic logger + # If it didn't work, at least setup a basic logger (if desired) basic_enabled = cfg.get('log_basic', True) if not am_worked: - sys.stderr.write("Warning, no logging configured!\n") + sys.stderr.write(("Warning, no logging configured!" + " (tried %s configs)\n") % (am_tried)) if basic_enabled: sys.stderr.write("Setting up basic logging...\n") setupBasicLogging() @@ -105,5 +123,5 @@ except ImportError: def emit(self, record): pass -logger = getLogger() +logger = logging.getLogger() logger.addHandler(NullHandler()) -- cgit v1.2.3 From 34e3285163a3a4a4455dedf81987cc90e089fad7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:54:52 -0700 Subject: Logic test on split up line length (just incase). Removed un-used pre debug info option, since this is not used in the prettytable case. --- cloudinit/netinfo.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index 874c2674..0d729502 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -82,7 +82,7 @@ def route_info(): if not line: continue toks = line.split() - if toks[0] == "Kernel" or toks[0] == "Destination": + if len(toks) < 8 or toks[0] == "Kernel" or toks[0] == "Destination": continue entry = { 'destination': toks[0], @@ -140,9 +140,9 @@ def route_pformat(): fields = ['Route', 'Destination', 'Gateway', 'Genmask', 'Interface', 'Flags'] tbl = PrettyTable(fields) - for n, r in enumerate(routes): + for (n, r) in enumerate(routes): route_id = str(n) - tbl.add_row([str(n), r['destination'], + tbl.add_row([route_id, r['destination'], r['gateway'], r['genmask'], r['iface'], r['flags']]) route_s = tbl.get_string() @@ -152,7 +152,7 @@ def route_pformat(): return os.linesep.join(lines) -def debug_info(pre=""): +def debug_info(): lines = [] lines.append(netdev_pformat()) lines.append(route_pformat()) -- cgit v1.2.3 From 0743caf836d14e12d980c9e02b77e9358a61fcc4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:56:10 -0700 Subject: Add more comments around what the settings mean, add in a list that can be used to check if user provided frequencies are actually valid or not --- cloudinit/settings.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cloudinit/settings.py b/cloudinit/settings.py index f58c4b52..8a1eaeb3 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -20,10 +20,17 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +# Set and read for determining the cloud config file location CFG_ENV_NAME = "CLOUD_CFG" + +# This is expected to be a yaml formatted file CLOUD_CONFIG = '/etc/cloud/cloud.cfg' + +# This legacy format is expected to be readable by +# configobj, see http://pypi.python.org/pypi/configobj OLD_CLOUD_CONFIG = '/etc/ec2-init/ec2-config.cfg' +# What u get if no config is provided CFG_BUILTIN = { 'datasource_list': [ 'NoCloud', @@ -45,7 +52,10 @@ CFG_BUILTIN = { }, } +# Valid frequencies of handlers/modules PER_INSTANCE = "once-per-instance" PER_ALWAYS = "always" PER_ONCE = "once" +# Used to sanity check incoming handlers/modules frequencies +FREQUENCIES = [PER_INSTANCE, PER_ALWAYS, PER_ONCE] -- cgit v1.2.3 From 7027bd63c39a153a2748d8a7fd326ba5c2e2c40e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:56:56 -0700 Subject: Split up time calculation, use logexc instead of log.exception --- cloudinit/sources/DataSourceCloudStack.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 33fb3491..791df68f 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -2,9 +2,11 @@ # # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Cosmin Luta +# Copyright (C) 2012 Yahoo! Inc. # # Author: Cosmin Luta # Author: Scott Moser +# Author: Joshua Harlow # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -66,7 +68,7 @@ class DataSourceCloudStack(sources.DataSource): if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")): self.userdata_raw = seed_ret['user-data'] self.metadata = seed_ret['meta-data'] - LOG.info("Using seeded cloudstack data from: %s", self.seed_dir) + LOG.debug("Using seeded cloudstack data from: %s", self.seed_dir) return True try: start = time.time() @@ -74,12 +76,12 @@ class DataSourceCloudStack(sources.DataSource): None, self.metadata_address) self.metadata = boto_utils.get_instance_metadata(self.api_ver, self.metadata_address) - LOG.debug("Crawl of metadata service took %ds", - (time.time() - start)) + tot_time = (time.time() - start) + LOG.debug("Crawl of metadata service took %s", int(tot_time)) return True except Exception as e: - LOG.exception(('Failed fetching from metadata ' - 'service %s due to: %s'), self.metadata_address, e) + util.logexc(LOG, ('Failed fetching from metadata ' + 'service %s'), self.metadata_address) return False def get_instance_id(self): -- cgit v1.2.3 From 403430d42701d2eb2ed55077498a213aeb6eaf92 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:58:38 -0700 Subject: Update applying network esttings to use the distro function to handle this, use logexc instead of log.exception --- cloudinit/sources/DataSourceConfigDrive.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index ca4bb7cf..176b62b0 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -1,6 +1,10 @@ +# vi: ts=4 expandtab +# # Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser +# Author: Joshua Harlow # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -22,6 +26,8 @@ from cloudinit import sources from cloudinit import util LOG = logging.getLogger(__name__) + +# Various defaults/constants... DEFAULT_IID = "iid-dsconfigdrive" DEFAULT_MODE = 'pass' CFG_DRIVE_FILES = [ @@ -33,7 +39,6 @@ DEFAULT_METADATA = { "instance-id": DEFAULT_IID, "dsmode": DEFAULT_MODE, } -IF_UP_CMD = ['ifup', '--all'] CFG_DRIVE_DEV_ENV = 'CLOUD_INIT_CONFIG_DRIVE_DEVICE' @@ -43,11 +48,11 @@ class DataSourceConfigDrive(sources.DataSource): self.seed = None self.cfg = {} self.dsmode = 'local' - self.seed_dir = os.path.join(self.paths.seed_dir, 'config_drive') + self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') def __str__(self): mstr = "%s[%s]" % (util.obj_name(self), self.dsmode) - mstr = mstr + " [seed=%s]" % (self.seed) + mstr += " [seed=%s]" % (self.seed) return mstr def get_data(self): @@ -60,8 +65,8 @@ class DataSourceConfigDrive(sources.DataSource): (md, ud) = read_config_drive_dir(self.seed_dir) found = self.seed_dir except NonConfigDriveDir: - LOG.exception("Failed reading config drive from %s", - self.seed_dir) + util.logexc(LOG, "Failed reading config drive from %s", + self.seed_dir) if not found: dev = find_cfg_drive_device() if dev: @@ -86,15 +91,7 @@ class DataSourceConfigDrive(sources.DataSource): LOG.info("Updating network interfaces from configdrive") else: LOG.debug("Updating network interfaces from configdrive") - self.distro.apply_network(md['network-interfaces']) - try: - (_out, err) = util.subp(IF_UP_CMD) - if len(err): - LOG.warn("Running %s resulted in stderr output: %s", - IF_UP_CMD, err) - except util.ProcessExecutionError: - LOG.exception("Running %s failed", IF_UP_CMD) self.seed = found self.metadata = md @@ -109,7 +106,7 @@ class DataSourceConfigDrive(sources.DataSource): def get_public_ssh_keys(self): if not 'public-keys' in self.metadata: return [] - return list(self.metadata['public-keys']) + return self.metadata['public-keys'] # The data sources' config_obj is a cloud-config formated # object that came to it from ways other than cloud-config @@ -226,6 +223,6 @@ datasources = [ ] -# Used to match classes to dependencies +# Return a list of data sources that match this set of dependencies def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) -- cgit v1.2.3 From 36c1da35c2c0cb1b2ee18b7374bc81df8349e3e2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:59:12 -0700 Subject: Complete cleanup for refactoring/rework that makes it comptabile with the new structure, using unified util functions, logging and eliminating code and calls. --- cloudinit/sources/DataSourceEc2.py | 209 ++++++++++++++++++++++--------------- 1 file changed, 123 insertions(+), 86 deletions(-) diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 7051ecda..38be71fa 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -2,9 +2,11 @@ # # Copyright (C) 2009-2010 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser # Author: Juerg Hafliger +# Author: Joshua Harlow # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -18,31 +20,38 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import socket +import os import time + import boto.utils as boto_utils -import os.path +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import url_helper as uhelp +from cloudinit import util + +LOG = logging.getLogger(__name__) +DEF_MD_URL = "http://169.254.169.254" +DEF_MD_VERSION = '2009-04-04' +DEF_MD_URLS = [DEF_MD_URL, "http://instance-data:8773"] -class DataSourceEc2(DataSource.DataSource): - api_ver = '2009-04-04' - seeddir = base_seeddir + '/ec2' - metadata_address = "http://169.254.169.254" + +class DataSourceEc2(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.metadata_address = DEF_MD_URL + self.seed_dir = os.path.join(paths.seed_dir, "ec2") + self.api_ver = DEF_MD_VERSION def __str__(self): - return("DataSourceEc2") + return util.obj_name(self) def get_data(self): - seedret = {} - if util.read_optional_seed(seedret, base=self.seeddir + "/"): - self.userdata_raw = seedret['user-data'] - self.metadata = seedret['meta-data'] - log.debug("using seeded ec2 data in %s" % self.seeddir) + seed_ret = {} + if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")): + self.userdata_raw = seed_ret['user-data'] + self.metadata = seed_ret['meta-data'] + LOG.debug("Using seeded ec2 data from %s", self.seed_dir) return True try: @@ -53,51 +62,61 @@ class DataSourceEc2(DataSource.DataSource): None, self.metadata_address) self.metadata = boto_utils.get_instance_metadata(self.api_ver, self.metadata_address) - log.debug("crawl of metadata service took %ds" % (time.time() - - start)) + tot_time = int(time.time() - start) + LOG.debug("Crawl of metadata service took %s", tot_time) return True - except Exception as e: - print e + except Exception: + util.logexc(LOG, "Failed reading from metadata address %s", + self.metadata_address) return False def get_instance_id(self): - return(self.metadata['instance-id']) + return self.metadata['instance-id'] def get_availability_zone(self): - return(self.metadata['placement']['availability-zone']) + return self.metadata['placement']['availability-zone'] def get_local_mirror(self): - return(self.get_mirror_from_availability_zone()) + return self.get_mirror_from_availability_zone() def get_mirror_from_availability_zone(self, availability_zone=None): - # availability is like 'us-west-1b' or 'eu-west-1a' - if availability_zone == None: + # Availability is like 'us-west-1b' or 'eu-west-1a' + if availability_zone is None: availability_zone = self.get_availability_zone() - fallback = None - if self.is_vpc(): - return fallback + return None - try: - host = "%s.ec2.archive.ubuntu.com" % availability_zone[:-1] - socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM) - return 'http://%s/ubuntu/' % host - except: - return fallback + # Use the distro to get the mirror + if not availability_zone: + return None - def wait_for_metadata_service(self): - mcfg = self.ds_cfg + mirror_tpl = self.distro.get_option('availability_zone_template') + if not mirror_tpl: + return None - if not hasattr(mcfg, "get"): - mcfg = {} + tpl_params = { + 'zone': availability_zone.strip(), + } + mirror_url = mirror_tpl % (tpl_params) + + (max_wait, timeout) = self._get_url_settings() + worked = uhelp.wait_for_url([mirror_url], max_wait=max_wait, + timeout=timeout, status_cb=LOG.warn) + if not worked: + return None + + return mirror_url + def _get_url_settings(self): + mcfg = self.ds_cfg + if not mcfg: + mcfg = {} max_wait = 120 try: max_wait = int(mcfg.get("max_wait", max_wait)) except Exception: - util.logexc(log) - log.warn("Failed to get max wait. using %s" % max_wait) + util.logexc(LOG, "Failed to get max wait. using %s", max_wait) if max_wait == 0: return False @@ -106,91 +125,104 @@ class DataSourceEc2(DataSource.DataSource): try: timeout = int(mcfg.get("timeout", timeout)) except Exception: - util.logexc(log) - log.warn("Failed to get timeout, using %s" % timeout) + util.logexc(LOG, "Failed to get timeout, using %s", timeout) + return (max_wait, timeout) + + def wait_for_metadata_service(self): + mcfg = self.ds_cfg + if not mcfg: + mcfg = {} - def_mdurls = ["http://169.254.169.254", "http://instance-data:8773"] - mdurls = mcfg.get("metadata_urls", def_mdurls) + (max_wait, timeout) = self._get_url_settings() # Remove addresses from the list that wont resolve. + mdurls = mcfg.get("metadata_urls", DEF_MD_URLS) filtered = [x for x in mdurls if util.is_resolvable_url(x)] if set(filtered) != set(mdurls): - log.debug("removed the following from metadata urls: %s" % - list((set(mdurls) - set(filtered)))) + LOG.debug("Removed the following from metadata urls: %s", + list((set(mdurls) - set(filtered)))) if len(filtered): mdurls = filtered else: - log.warn("Empty metadata url list! using default list") - mdurls = def_mdurls + LOG.warn("Empty metadata url list! using default list") + mdurls = DEF_MD_URLS urls = [] - url2base = {False: False} + url2base = {} for url in mdurls: cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver) urls.append(cur) url2base[cur] = url starttime = time.time() - url = util.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=log.warn) + url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, status_cb=LOG.warn) if url: - log.debug("Using metadata source: '%s'" % url2base[url]) + LOG.info("Using metadata source: '%s'", url2base[url]) else: - log.critical("giving up on md after %i seconds\n" % - int(time.time() - starttime)) + LOG.critical("Giving up on md from %s after %i seconds", + urls, int(time.time() - starttime)) - self.metadata_address = url2base[url] - return (bool(url)) + self.metadata_address = url2base.get(url) + return bool(url) + + def _remap_device(self, short_name): + # LP: #611137 + # the metadata service may believe that devices are named 'sda' + # when the kernel named them 'vda' or 'xvda' + # we want to return the correct value for what will actually + # exist in this instance + mappings = {"sd": ("vd", "xvd")} + for (nfrom, tlist) in mappings.iteritems(): + if not short_name.startswith(nfrom): + continue + for nto in tlist: + cand = "/dev/%s%s" % (nto, short_name[len(nfrom):]) + if os.path.exists(cand): + return cand + return None def device_name_to_device(self, name): - # consult metadata service, that has + # Consult metadata service, that has # ephemeral0: sdb # and return 'sdb' for input 'ephemeral0' if 'block-device-mapping' not in self.metadata: - return(None) + return None + # Example: + # 'block-device-mapping': + # {'ami': '/dev/sda1', + # 'ephemeral0': '/dev/sdb', + # 'root': '/dev/sda1'} found = None - for entname, device in self.metadata['block-device-mapping'].items(): + for (entname, device) in self.metadata['block-device-mapping'].items(): if entname == name: found = device break # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0' if entname == "ephemeral" and name == "ephemeral0": found = device - if found == None: - log.debug("unable to convert %s to a device" % name) + if found is None: + LOG.debug("Unable to convert %s to a device", name) return None - # LP: #611137 - # the metadata service may believe that devices are named 'sda' - # when the kernel named them 'vda' or 'xvda' - # we want to return the correct value for what will actually - # exist in this instance - mappings = {"sd": ("vd", "xvd")} ofound = found - short = os.path.basename(found) - if not found.startswith("/"): found = "/dev/%s" % found - if os.path.exists(found): - return(found) + return found - for nfrom, tlist in mappings.items(): - if not short.startswith(nfrom): - continue - for nto in tlist: - cand = "/dev/%s%s" % (nto, short[len(nfrom):]) - if os.path.exists(cand): - log.debug("remapped device name %s => %s" % (found, cand)) - return(cand) + remapped = self._remap_device(os.path.basename(found)) + if remapped: + LOG.debug("Remapped device name %s => %s", (found, remapped)) + return remapped - # on t1.micro, ephemeral0 will appear in block-device-mapping from + # On t1.micro, ephemeral0 will appear in block-device-mapping from # metadata, but it will not exist on disk (and never will) - # at this pint, we've verified that the path did not exist + # at this point, we've verified that the path did not exist # in the special case of 'ephemeral0' return None to avoid bogus # fstab entry (LP: #744019) if name == "ephemeral0": @@ -198,7 +230,11 @@ class DataSourceEc2(DataSource.DataSource): return ofound def is_vpc(self): - # per comment in LP: #615545 + # See: https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/615545 + # Detect that the machine was launched in a VPC. + # But I did notice that when in a VPC, meta-data + # does not have public-ipv4 and public-hostname + # listed as a possibility. ph = "public-hostname" p4 = "public-ipv4" if ((ph not in self.metadata or self.metadata[ph] == "") and @@ -207,11 +243,12 @@ class DataSourceEc2(DataSource.DataSource): return False +# Used to match classes to dependencies datasources = [ - (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), + (DataSourceEc2, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] -# return a list of data sources that match this set of dependencies +# Return a list of data sources that match this set of dependencies def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) + return sources.list_from_depends(depends, datasources) -- cgit v1.2.3 From 508168acb95aee070d493b45656f781a42bdd262 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 18:01:03 -0700 Subject: Complete initial cleanup for refactoring/rework. Some of the cleanups were the following 1. Using standard (logged) utility functions for sub process work, writing, reading files, and other file system/operating system options 2. Having distrobutions impelement there own subclasses to handle system specifics (if applicable) 3. Having a cloud wrapper that provides just the functionality we want to expose (cloud.py) 4. Using a path class instead of globals for all cloud init paths (it is configured via config) 5. Removal of as much shared global state as possible (there should be none, minus a set of constants) 6. Other various cleanups that remove transforms/handlers/modules from reading/writing/chmoding there own files. a. They should be using util functions to take advantage of the logging that is now enabled in those util functions (very useful for debugging) 7. Urls being read and checked from a single module that serves this and only this purpose (+1 for code organization) 8. Updates to log whenever a transform decides not to run 9. Ensure whenever a exception is thrown (and possibly captured) that the util.logexc function is called a. For debugging, tracing this is important to not just drop them on the floor. 10. Code shuffling into utils.py where it makes sense (and where it could serve a benefit for other code now or in the future) --- cloudinit/sources/DataSourceMAAS.py | 226 +++++++---------------- cloudinit/sources/DataSourceNoCloud.py | 143 +++++++-------- cloudinit/sources/DataSourceOVF.py | 227 ++++++++++-------------- cloudinit/sources/__init__.py | 12 +- cloudinit/transforms/__init__.py | 190 ++------------------ cloudinit/transforms/cc_apt_pipelining.py | 9 +- cloudinit/transforms/cc_apt_update_upgrade.py | 116 ++++++------ cloudinit/transforms/cc_bootcmd.py | 50 +++--- cloudinit/transforms/cc_byobu.py | 22 +-- cloudinit/transforms/cc_ca_certs.py | 25 +-- cloudinit/transforms/cc_chef.py | 101 +++++------ cloudinit/transforms/cc_disable_ec2_metadata.py | 14 +- cloudinit/transforms/cc_final_message.py | 63 ++++--- cloudinit/transforms/cc_foo.py | 35 +++- cloudinit/transforms/cc_grub_dpkg.py | 19 +- cloudinit/transforms/cc_keys_to_console.py | 14 +- cloudinit/transforms/cc_landscape.py | 43 +++-- cloudinit/transforms/cc_locale.py | 36 ++-- cloudinit/transforms/cc_mcollective.py | 80 ++++----- cloudinit/transforms/cc_mounts.py | 84 +++++---- cloudinit/transforms/cc_phone_home.py | 53 +++--- cloudinit/transforms/cc_puppet.py | 94 +++++----- cloudinit/transforms/cc_resizefs.py | 142 +++++++++------ cloudinit/transforms/cc_rightscale_userdata.py | 62 ++++--- cloudinit/transforms/cc_rsyslog.py | 52 +++--- cloudinit/transforms/cc_runcmd.py | 15 +- cloudinit/transforms/cc_salt_minion.py | 49 ++--- cloudinit/transforms/cc_scripts_per_boot.py | 20 ++- cloudinit/transforms/cc_scripts_per_instance.py | 20 ++- cloudinit/transforms/cc_scripts_per_once.py | 20 ++- cloudinit/transforms/cc_scripts_user.py | 19 +- cloudinit/transforms/cc_set_hostname.py | 23 +-- cloudinit/transforms/cc_set_passwords.py | 108 ++++++----- cloudinit/transforms/cc_ssh.py | 93 ++++++---- cloudinit/transforms/cc_ssh_import_id.py | 25 ++- cloudinit/transforms/cc_timezone.py | 41 +---- cloudinit/transforms/cc_update_etc_hosts.py | 82 +++------ cloudinit/transforms/cc_update_hostname.py | 80 +-------- 38 files changed, 1113 insertions(+), 1394 deletions(-) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 61a0038f..27196265 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser +# Author: Joshua Harlow # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -16,22 +18,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util +import os import errno import oauth.oauth as oauth -import os.path -import urllib2 import time +import urllib2 +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import url_helper as uhelp +from cloudinit import util +LOG = logging.getLogger(__name__) MD_VERSION = "2012-03-01" -class DataSourceMAAS(DataSource.DataSource): +class DataSourceMAAS(sources.DataSource): """ DataSourceMAAS reads instance information from MAAS. Given a config metadata_url, and oauth tokens, it expects to find @@ -40,61 +42,64 @@ class DataSourceMAAS(DataSource.DataSource): user-data hostname """ - seeddir = base_seeddir + '/maas' - baseurl = None + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.base_url = None + self.seed_dir = os.path.join(paths.seed_dir, 'maas') def __str__(self): - return("DataSourceMAAS[%s]" % self.baseurl) + return "%s[%s]" % (util.obj_name(self), self.base_url) def get_data(self): mcfg = self.ds_cfg try: - (userdata, metadata) = read_maas_seed_dir(self.seeddir) + (userdata, metadata) = read_maas_seed_dir(self.seed_dir) self.userdata_raw = userdata self.metadata = metadata - self.baseurl = self.seeddir + self.base_url = self.seed_dir return True except MAASSeedDirNone: pass except MAASSeedDirMalformed as exc: - log.warn("%s was malformed: %s\n" % (self.seeddir, exc)) + LOG.warn("%s was malformed: %s" % (self.seed_dir, exc)) raise - try: - # if there is no metadata_url, then we're not configured - url = mcfg.get('metadata_url', None) - if url == None: - return False + # If there is no metadata_url, then we're not configured + url = mcfg.get('metadata_url', None) + if not url: + return False + try: if not self.wait_for_metadata_service(url): return False - self.baseurl = url + self.base_url = url - (userdata, metadata) = read_maas_seed_url(self.baseurl, - self.md_headers) + (userdata, metadata) = read_maas_seed_url(self.base_url, + self.md_headers) self.userdata_raw = userdata self.metadata = metadata return True except Exception: - util.logexc(log) + util.logexc(LOG, "Failed fetching metadata from url %s", url) return False def md_headers(self, url): mcfg = self.ds_cfg - # if we are missing token_key, token_secret or consumer_key + # If we are missing token_key, token_secret or consumer_key # then just do non-authed requests for required in ('token_key', 'token_secret', 'consumer_key'): if required not in mcfg: - return({}) + return {} consumer_secret = mcfg.get('consumer_secret', "") - - return(oauth_headers(url=url, consumer_key=mcfg['consumer_key'], - token_key=mcfg['token_key'], token_secret=mcfg['token_secret'], - consumer_secret=consumer_secret)) + return oauth_headers(url=url, + consumer_key=mcfg['consumer_key'], + token_key=mcfg['token_key'], + token_secret=mcfg['token_secret'], + consumer_secret=consumer_secret) def wait_for_metadata_service(self, url): mcfg = self.ds_cfg @@ -103,32 +108,31 @@ class DataSourceMAAS(DataSource.DataSource): try: max_wait = int(mcfg.get("max_wait", max_wait)) except Exception: - util.logexc(log) - log.warn("Failed to get max wait. using %s" % max_wait) + util.logexc(LOG, "Failed to get max wait. using %s", max_wait) if max_wait == 0: return False timeout = 50 try: - timeout = int(mcfg.get("timeout", timeout)) + if timeout in mcfg: + timeout = int(mcfg.get("timeout", timeout)) except Exception: - util.logexc(log) - log.warn("Failed to get timeout, using %s" % timeout) + LOG.warn("Failed to get timeout, using %s" % timeout) starttime = time.time() check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) url = util.wait_for_url(urls=[check_url], max_wait=max_wait, - timeout=timeout, status_cb=log.warn, - headers_cb=self.md_headers) + timeout=timeout, status_cb=LOG.warn, + headers_cb=self.md_headers) if url: - log.debug("Using metadata source: '%s'" % url) + LOG.info("Using metadata source: '%s'", url) else: - log.critical("giving up on md after %i seconds\n" % - int(time.time() - starttime)) + LOG.critical("Giving up on md from %s after %i seconds", + urls, int(time.time() - starttime)) - return (bool(url)) + return bool(url) def read_maas_seed_dir(seed_d): @@ -139,22 +143,19 @@ def read_maas_seed_dir(seed_d): * local-hostname * user-data """ - files = ('local-hostname', 'instance-id', 'user-data', 'public-keys') - md = {} - if not os.path.isdir(seed_d): raise MAASSeedDirNone("%s: not a directory") + files = ('local-hostname', 'instance-id', 'user-data', 'public-keys') + md = {} for fname in files: try: - with open(os.path.join(seed_d, fname)) as fp: - md[fname] = fp.read() - fp.close() + md[fname] = util.load_file(os.path.join(seed_d, fname)) except IOError as e: if e.errno != errno.ENOENT: raise - return(check_seed_contents(md, seed_d)) + return check_seed_contents(md, seed_d) def read_maas_seed_url(seed_url, header_cb=None, timeout=None, @@ -169,29 +170,26 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, * //meta-data/local-hostname * //user-data """ - files = ('meta-data/local-hostname', - 'meta-data/instance-id', - 'meta-data/public-keys', - 'user-data') - base_url = "%s/%s" % (seed_url, version) + files = { + 'local-hostname': "%s/%s" % (base_url, 'meta-data/local-hostname'), + 'instance-id': "%s/%s" % (base_url, 'meta-data/instance-id'), + 'public-keys': "%s/%s" % (base_url, 'meta-data/public-keys'), + 'user-data': "%s/%s" % (base_url, 'user-data'), + } md = {} - for fname in files: - url = "%s/%s" % (base_url, fname) + for (name, url) in files: if header_cb: headers = header_cb(url) else: headers = {} - try: - req = urllib2.Request(url, data=None, headers=headers) - resp = urllib2.urlopen(req, timeout=timeout) - md[os.path.basename(fname)] = resp.read() + (resp, sc) = uhelp.readurl(url, headers=headers, timeout=timeout) + md[name] = resp except urllib2.HTTPError as e: if e.code != 404: raise - - return(check_seed_contents(md, seed_url)) + return check_seed_contents(md, seed_url) def check_seed_contents(content, seed): @@ -201,11 +199,10 @@ def check_seed_contents(content, seed): Raise MAASSeedDirMalformed or MAASSeedDirNone """ md_required = ('instance-id', 'local-hostname') - found = content.keys() - if len(content) == 0: raise MAASSeedDirNone("%s: no data files found" % seed) + found = content.keys() missing = [k for k in md_required if k not in found] if len(missing): raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing)) @@ -217,7 +214,7 @@ def check_seed_contents(content, seed): continue md[key] = val - return(userdata, md) + return (userdata, md) def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): @@ -232,8 +229,8 @@ def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): } req = oauth.OAuthRequest(http_url=url, parameters=params) req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(), - consumer, token) - return(req.to_header()) + consumer, token) + return req.to_header() class MAASSeedDirNone(Exception): @@ -244,102 +241,11 @@ class MAASSeedDirMalformed(Exception): pass +# Used to match classes to dependencies datasources = [ - (DataSourceMAAS, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), + (DataSourceMAAS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] - -# return a list of data sources that match this set of dependencies +# Return a list of data sources that match this set of dependencies def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - - -if __name__ == "__main__": - def main(): - """ - Call with single argument of directory or http or https url. - If url is given additional arguments are allowed, which will be - interpreted as consumer_key, token_key, token_secret, consumer_secret - """ - import argparse - import pprint - - parser = argparse.ArgumentParser(description='Interact with MAAS DS') - parser.add_argument("--config", metavar="file", - help="specify DS config file", default=None) - parser.add_argument("--ckey", metavar="key", - help="the consumer key to auth with", default=None) - parser.add_argument("--tkey", metavar="key", - help="the token key to auth with", default=None) - parser.add_argument("--csec", metavar="secret", - help="the consumer secret (likely '')", default="") - parser.add_argument("--tsec", metavar="secret", - help="the token secret to auth with", default=None) - parser.add_argument("--apiver", metavar="version", - help="the apiver to use ("" can be used)", default=MD_VERSION) - - subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") - subcmds.add_parser('crawl', help="crawl the datasource") - subcmds.add_parser('get', help="do a single GET of provided url") - subcmds.add_parser('check-seed', help="read andn verify seed at url") - - parser.add_argument("url", help="the data source to query") - - args = parser.parse_args() - - creds = {'consumer_key': args.ckey, 'token_key': args.tkey, - 'token_secret': args.tsec, 'consumer_secret': args.csec} - - if args.config: - import yaml - with open(args.config) as fp: - cfg = yaml.load(fp) - if 'datasource' in cfg: - cfg = cfg['datasource']['MAAS'] - for key in creds.keys(): - if key in cfg and creds[key] == None: - creds[key] = cfg[key] - - def geturl(url, headers_cb): - req = urllib2.Request(url, data=None, headers=headers_cb(url)) - return(urllib2.urlopen(req).read()) - - def printurl(url, headers_cb): - print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) - - def crawl(url, headers_cb=None): - if url.endswith("/"): - for line in geturl(url, headers_cb).splitlines(): - if line.endswith("/"): - crawl("%s%s" % (url, line), headers_cb) - else: - printurl("%s%s" % (url, line), headers_cb) - else: - printurl(url, headers_cb) - - def my_headers(url): - headers = {} - if creds.get('consumer_key', None) != None: - headers = oauth_headers(url, **creds) - return headers - - if args.subcmd == "check-seed": - if args.url.startswith("http"): - (userdata, metadata) = read_maas_seed_url(args.url, - header_cb=my_headers, version=args.apiver) - else: - (userdata, metadata) = read_maas_seed_url(args.url) - print "=== userdata ===" - print userdata - print "=== metadata ===" - pprint.pprint(metadata) - - elif args.subcmd == "get": - printurl(args.url, my_headers) - - elif args.subcmd == "crawl": - if not args.url.endswith("/"): - args.url = "%s/" % args.url - crawl(args.url, my_headers) - - main() + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index e8c56b8f..84d0f99d 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -2,9 +2,11 @@ # # Copyright (C) 2009-2010 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser # Author: Juerg Hafliger +# Author: Joshua Harlow # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -18,33 +20,34 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util import errno -import subprocess +import os + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util + +LOG = logging.getLogger(__name__) -class DataSourceNoCloud(DataSource.DataSource): - metadata = None - userdata = None - userdata_raw = None - supported_seed_starts = ("/", "file://") - dsmode = "local" - seed = None - cmdline_id = "ds=nocloud" - seeddir = base_seeddir + '/nocloud' +class DataSourceNoCloud(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.dsmode = 'local' + self.seed = None + self.cmdline_id = "ds=nocloud" + self.seed_dir = os.path.join(paths.seed_dir, 'nocloud') + self.supported_seed_starts = ("/", "file://") def __str__(self): - mstr = "DataSourceNoCloud" - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) + mstr = "%s [seed=%s][dsmode=%s]" % (util.obj_name(self), + self.seed, self.dsmode) + return mstr def get_data(self): defaults = { - "instance-id": "nocloud", "dsmode": self.dsmode + "instance-id": "nocloud", + "dsmode": self.dsmode, } found = [] @@ -52,24 +55,24 @@ class DataSourceNoCloud(DataSource.DataSource): ud = "" try: - # parse the kernel command line, getting data passed in + # Parse the kernel command line, getting data passed in if parse_cmdline_data(self.cmdline_id, md): found.append("cmdline") except: - util.logexc(log) + util.logexc(LOG, "Unable to parse command line data") return False - # check to see if the seeddir has data. + # Check to see if the seed dir has data. seedret = {} - if util.read_optional_seed(seedret, base=self.seeddir + "/"): + if util.read_optional_seed(seedret, base=self.seed_dir + "/"): md = util.mergedict(md, seedret['meta-data']) ud = seedret['user-data'] - found.append(self.seeddir) - log.debug("using seeded cache data in %s" % self.seeddir) + found.append(self.seed_dir) + LOG.debug("Using seeded cache data from %s", self.seed_dir) - # if the datasource config had a 'seedfrom' entry, then that takes + # If the datasource config had a 'seedfrom' entry, then that takes # precedence over a 'seedfrom' that was found in a filesystem - # but not over external medi + # but not over external media if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']: found.append("ds_config") md["seedfrom"] = self.ds_cfg['seedfrom'] @@ -83,35 +86,36 @@ class DataSourceNoCloud(DataSource.DataSource): for dev in devlist: try: - (newmd, newud) = util.mount_callback_umount(dev, - util.read_seeded) + LOG.debug("Attempting to use data from %s", dev) + + (newmd, newud) = util.mount_cb(dev, util.read_seeded) md = util.mergedict(newmd, md) ud = newud - # for seed from a device, the default mode is 'net'. + # For seed from a device, the default mode is 'net'. # that is more likely to be what is desired. # If they want dsmode of local, then they must # specify that. if 'dsmode' not in md: md['dsmode'] = "net" - log.debug("using data from %s" % dev) + LOG.debug("Using data from %s", dev) found.append(dev) break - except OSError, e: + except OSError as e: if e.errno != errno.ENOENT: raise - except util.mountFailedError: - log.warn("Failed to mount %s when looking for seed" % dev) + except util.MountFailedError: + util.logexc(LOG, "Failed to mount %s when looking for seed", dev) - # there was no indication on kernel cmdline or data + # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. if len(found) == 0: return False seeded_interfaces = None - # the special argument "seedfrom" indicates we should + # The special argument "seedfrom" indicates we should # attempt to seed the userdata / metadata from its value # its primarily value is in allowing the user to type less # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg @@ -123,57 +127,46 @@ class DataSourceNoCloud(DataSource.DataSource): seedfound = proto break if not seedfound: - log.debug("seed from %s not supported by %s" % - (seedfrom, self.__class__)) + LOG.debug("Seed from %s not supported by %s", seedfrom, self) return False if 'network-interfaces' in md: seeded_interfaces = self.dsmode - # this could throw errors, but the user told us to do it + # This could throw errors, but the user told us to do it # so if errors are raised, let them raise (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) - log.debug("using seeded cache data from %s" % seedfrom) + LOG.debug("Using seeded cache data from %s", seedfrom) - # values in the command line override those from the seed + # Values in the command line override those from the seed md = util.mergedict(md, md_seed) found.append(seedfrom) + # Now that we have exhausted any other places merge in the defaults md = util.mergedict(md, defaults) - # update the network-interfaces if metadata had 'network-interfaces' + # Update the network-interfaces if metadata had 'network-interfaces' # entry and this is the local datasource, or 'seedfrom' was used # and the source of the seed was self.dsmode # ('local' for NoCloud, 'net' for NoCloudNet') if ('network-interfaces' in md and (self.dsmode in ("local", seeded_interfaces))): - log.info("updating network interfaces from nocloud") - - util.write_file("/etc/network/interfaces", - md['network-interfaces']) - try: - (out, err) = util.subp(['ifup', '--all']) - if len(out) or len(err): - log.warn("ifup --all had stderr: %s" % err) - - except subprocess.CalledProcessError as exc: - log.warn("ifup --all failed: %s" % (exc.output[1])) - - self.seed = ",".join(found) - self.metadata = md - self.userdata_raw = ud - + LOG.info("Updating network interfaces from %s", self) + self.distro.apply_network(md['network-interfaces']) + if md['dsmode'] == self.dsmode: + self.seed = ",".join(found) + self.metadata = md + self.userdata_raw = ud return True - log.debug("%s: not claiming datasource, dsmode=%s" % - (self, md['dsmode'])) + LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode']) return False -# returns true or false indicating if cmdline indicated +# Returns true or false indicating if cmdline indicated # that this module should be used -# example cmdline: +# Example cmdline: # root=LABEL=uec-rootfs ro ds=nocloud def parse_cmdline_data(ds_id, fill, cmdline=None): if cmdline is None: @@ -210,23 +203,25 @@ def parse_cmdline_data(ds_id, fill, cmdline=None): k = s2l[k] fill[k] = v - return(True) + return True class DataSourceNoCloudNet(DataSourceNoCloud): - cmdline_id = "ds=nocloud-net" - supported_seed_starts = ("http://", "https://", "ftp://") - seeddir = base_seeddir + '/nocloud-net' - dsmode = "net" + def __init__(self, sys_cfg, distro, paths): + DataSourceNoCloud.__init__(self, sys_cfg, distro, paths) + self.cmdline_id = "ds=nocloud-net" + self.supported_seed_starts = ("http://", "https://", "ftp://") + self.seed_dir = os.path.join(paths.seed_dir, 'nocloud-net') + self.dsmode = "net" -datasources = ( - (DataSourceNoCloud, (DataSource.DEP_FILESYSTEM, )), - (DataSourceNoCloudNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -) +# Used to match classes to dependencies +datasources = [ + (DataSourceNoCloud, (sources.DEP_FILESYSTEM, )), + (DataSourceNoCloudNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] -# return a list of data sources that match this set of dependencies +# Return a list of data sources that match this set of dependencies def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index a0b1b518..bb0f46c2 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -2,9 +2,11 @@ # # Copyright (C) 2011 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser # Author: Juerg Hafliger +# Author: Joshua Harlow # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -18,33 +20,30 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import os.path -import os from xml.dom import minidom import base64 +import os import re import tempfile -import subprocess +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util -class DataSourceOVF(DataSource.DataSource): - seed = None - seeddir = base_seeddir + '/ovf' - environment = None - cfg = {} - userdata_raw = None - metadata = None - supported_seed_starts = ("/", "file://") +LOG = logging.getLogger(__name__) + + +class DataSourceOVF(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.seed = None + self.seed_dir = os.path.join(paths.seed_dir, 'ovf') + self.environment = None + self.cfg = {} + self.supported_seed_starts = ("/", "file://") def __str__(self): - mstr = "DataSourceOVF" - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) + return "%s [seed=%s]" % (util.obj_name(self), self.seed) def get_data(self): found = [] @@ -55,13 +54,12 @@ class DataSourceOVF(DataSource.DataSource): "instance-id": "iid-dsovf" } - (seedfile, contents) = get_ovf_env(base_seeddir) + (seedfile, contents) = get_ovf_env(self.paths.seed_dir) if seedfile: - # found a seed dir - seed = "%s/%s" % (base_seeddir, seedfile) + # Found a seed dir + seed = os.path.join(self.paths.seed_dir, seedfile) (md, ud, cfg) = read_ovf_environment(contents) self.environment = contents - found.append(seed) else: np = {'iso': transport_iso9660, @@ -71,7 +69,6 @@ class DataSourceOVF(DataSource.DataSource): (contents, _dev, _fname) = transfunc() if contents: break - if contents: (md, ud, cfg) = read_ovf_environment(contents) self.environment = contents @@ -89,17 +86,19 @@ class DataSourceOVF(DataSource.DataSource): seedfound = proto break if not seedfound: - log.debug("seed from %s not supported by %s" % - (seedfrom, self.__class__)) + LOG.debug("Seed from %s not supported by %s", + seedfrom, self) return False (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) - log.debug("using seeded cache data from %s" % seedfrom) + LOG.debug("Using seeded cache data from %s", seedfrom) md = util.mergedict(md, md_seed) found.append(seedfrom) + # Now that we have exhausted any other places merge in the defaults md = util.mergedict(md, defaults) + self.seed = ",".join(found) self.metadata = md self.userdata_raw = ud @@ -108,31 +107,37 @@ class DataSourceOVF(DataSource.DataSource): def get_public_ssh_keys(self): if not 'public-keys' in self.metadata: - return([]) - return([self.metadata['public-keys'], ]) + return [] + pks = self.metadata['public-keys'] + if isinstance(pks, (list)): + return pks + else: + return [pks] - # the data sources' config_obj is a cloud-config formated + # The data sources' config_obj is a cloud-config formatted # object that came to it from ways other than cloud-config # because cloud-config content would be handled elsewhere def get_config_obj(self): - return(self.cfg) + return self.cfg class DataSourceOVFNet(DataSourceOVF): - seeddir = base_seeddir + '/ovf-net' - supported_seed_starts = ("http://", "https://", "ftp://") + def __init__(self, sys_cfg, distro, paths): + DataSourceOVF.__init__(self, sys_cfg, distro, paths) + self.seed_dir = os.path.join(paths.seed_dir, 'ovf-net') + self.supported_seed_starts = ("http://", "https://", "ftp://") -# this will return a dict with some content -# meta-data, user-data +# This will return a dict with some content +# meta-data, user-data, some config def read_ovf_environment(contents): - props = getProperties(contents) + props = get_properties(contents) md = {} cfg = {} ud = "" - cfg_props = ['password', ] + cfg_props = ['password'] md_props = ['seedfrom', 'local-hostname', 'public-keys', 'instance-id'] - for prop, val in props.iteritems(): + for (prop, val) in props.iteritems(): if prop == 'hostname': prop = "local-hostname" if prop in md_props: @@ -144,23 +149,25 @@ def read_ovf_environment(contents): ud = base64.decodestring(val) except: ud = val - return(md, ud, cfg) + return (md, ud, cfg) -# returns tuple of filename (in 'dirname', and the contents of the file) +# Returns tuple of filename (in 'dirname', and the contents of the file) # on "not found", returns 'None' for filename and False for contents def get_ovf_env(dirname): env_names = ("ovf-env.xml", "ovf_env.xml", "OVF_ENV.XML", "OVF-ENV.XML") for fname in env_names: - if os.path.isfile("%s/%s" % (dirname, fname)): - fp = open("%s/%s" % (dirname, fname)) - contents = fp.read() - fp.close() - return(fname, contents) - return(None, False) + full_fn = os.path.join(dirname, fname) + if os.path.isfile(full_fn): + try: + contents = util.load_file(full_fn) + return (fname, contents) + except: + util.logexc(LOG, "Failed loading ovf file %s", full_fn) + return (None, False) -# transport functions take no input and return +# Transport functions take no input and return # a 3 tuple of content, path, filename def transport_iso9660(require_iso=True): @@ -173,79 +180,45 @@ def transport_iso9660(require_iso=True): devname_regex = os.environ.get(envname, default_regex) cdmatch = re.compile(devname_regex) - # go through mounts to see if it was already mounted - fp = open("/proc/mounts") - mounts = fp.readlines() - fp.close() - - mounted = {} - for mpline in mounts: - (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() - mounted[dev] = (dev, fstype, mp, False) - mp = mp.replace("\\040", " ") + # Go through mounts to see if it was already mounted + mounts = util.mounts() + for (dev, info) in mounts.iteritems(): + fstype = info['fstype'] if fstype != "iso9660" and require_iso: continue - if cdmatch.match(dev[5:]) == None: # take off '/dev/' continue - + mp = info['mountpoint'] (fname, contents) = get_ovf_env(mp) if contents is not False: - return(contents, dev, fname) - - tmpd = None - dvnull = None + return (contents, dev, fname) devs = os.listdir("/dev/") devs.sort() - for dev in devs: - fullp = "/dev/%s" % dev + fullp = os.path.join("/dev/", dev) - if fullp in mounted or not cdmatch.match(dev) or os.path.isdir(fullp): + if (fullp in mounted or + not cdmatch.match(dev) or os.path.isdir(fullp)): continue - fp = None try: - fp = open(fullp, "rb") - fp.read(512) - fp.close() + # See if we can read anything at all...?? + with open(fullp, 'rb') as fp: + fp.read(512) except: - if fp: - fp.close() continue - if tmpd is None: - tmpd = tempfile.mkdtemp() - if dvnull is None: - try: - dvnull = open("/dev/null") - except: - pass - - cmd = ["mount", "-o", "ro", fullp, tmpd] - if require_iso: - cmd.extend(('-t', 'iso9660')) - - rc = subprocess.call(cmd, stderr=dvnull, stdout=dvnull, stdin=dvnull) - if rc: + try: + (fname, contents) = utils.mount_cb(fullp, get_ovf_env, mtype="iso9660") + except util.MountFailedError: + util.logexc(LOG, "Failed mounting %s", fullp) continue - (fname, contents) = get_ovf_env(tmpd) - - subprocess.call(["umount", tmpd]) - if contents is not False: - os.rmdir(tmpd) - return(contents, fullp, fname) - - if tmpd: - os.rmdir(tmpd) - - if dvnull: - dvnull.close() + return (contents, fullp, fname) - return(False, None, None) + return (False, None, None) def transport_vmware_guestd(): @@ -259,74 +232,60 @@ def transport_vmware_guestd(): # # would need to error check here and see why this failed # # to know if log/error should be raised # return(False, None, None) - return(False, None, None) + return (False, None, None) -def findChild(node, filter_func): +def find_child(node, filter_func): ret = [] if not node.hasChildNodes(): return ret for child in node.childNodes: if filter_func(child): ret.append(child) - return(ret) + return ret -def getProperties(environString): - dom = minidom.parseString(environString) +def get_properties(contents): + + dom = minidom.parseString(contents) if dom.documentElement.localName != "Environment": - raise Exception("No Environment Node") + raise XmlError("No Environment Node") if not dom.documentElement.hasChildNodes(): - raise Exception("No Child Nodes") + raise XmlError("No Child Nodes") envNsURI = "http://schemas.dmtf.org/ovf/environment/1" # could also check here that elem.namespaceURI == # "http://schemas.dmtf.org/ovf/environment/1" - propSections = findChild(dom.documentElement, + propSections = find_child(dom.documentElement, lambda n: n.localName == "PropertySection") if len(propSections) == 0: - raise Exception("No 'PropertySection's") + raise XmlError("No 'PropertySection's") props = {} - propElems = findChild(propSections[0], lambda n: n.localName == "Property") + propElems = find_child(propSections[0], lambda n: n.localName == "Property") for elem in propElems: key = elem.attributes.getNamedItemNS(envNsURI, "key").value val = elem.attributes.getNamedItemNS(envNsURI, "value").value props[key] = val - return(props) + return props + + +class XmlError(Exception): + pass +# Used to match classes to dependencies datasources = ( - (DataSourceOVF, (DataSource.DEP_FILESYSTEM, )), - (DataSourceOVFNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), + (DataSourceOVF, (sources.DEP_FILESYSTEM, )), + (DataSourceOVFNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ) -# return a list of data sources that match this set of dependencies +# Return a list of data sources that match this set of dependencies def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - - -if __name__ == "__main__": - def main(): - import sys - envStr = open(sys.argv[1]).read() - props = getProperties(envStr) - import pprint - pprint.pprint(props) - - md, ud, cfg = read_ovf_environment(envStr) - print "=== md ===" - pprint.pprint(md) - print "=== ud ===" - pprint.pprint(ud) - print "=== cfg ===" - pprint.pprint(cfg) - - main() + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index dfd1fff3..08669f5d 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -39,10 +39,6 @@ class DataSourceNotFoundException(Exception): class DataSource(object): def __init__(self, sys_cfg, distro, paths): - name = util.obj_name(self) - if name.startswith(DS_PREFIX): - name = name[DS_PREFIX:] - self.cfgname = name self.sys_cfg = sys_cfg self.distro = distro self.paths = paths @@ -50,8 +46,11 @@ class DataSource(object): self.userdata = None self.metadata = None self.userdata_raw = None + name = util.obj_name(self) + if name.startswith(DS_PREFIX): + name = name[DS_PREFIX:] self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, - ("datasource", self.cfgname), {}) + ("datasource", name), {}) def get_userdata(self): if self.userdata is None: @@ -112,6 +111,7 @@ class DataSource(object): def get_instance_id(self): if not self.metadata or 'instance-id' not in self.metadata: + # Return a magic not really instance id string return "iid-datasource" return str(self.metadata['instance-id']) @@ -166,7 +166,7 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): if s.get_data(): return (s, ds) except Exception as e: - LOG.exception("Getting data from %s failed due to %s", ds, e) + util.logexc(LOG, "Getting data from %s failed", ds) msg = "Did not find any data source, searched classes: %s" % (ds_names) raise DataSourceNotFoundException(msg) diff --git a/cloudinit/transforms/__init__.py b/cloudinit/transforms/__init__.py index 5d70ac43..8275b375 100644 --- a/cloudinit/transforms/__init__.py +++ b/cloudinit/transforms/__init__.py @@ -19,183 +19,12 @@ # along with this program. If not, see . # -import os -import subprocess -import sys -import time -import traceback - -import yaml - -from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE) +from cloudinit.settings import (PER_INSTANCE, FREQUENCIES) from cloudinit import log as logging -from cloudinit import util LOG = logging.getLogger(__name__) -DEF_HANDLER_VERSION = 1 -DEF_FREQ = PER_INSTANCE - - -# reads a cloudconfig module list, returns -# a 2 dimensional array suitable to pass to run_cc_modules -def read_cc_modules(cfg, name): - if name not in cfg: - return([]) - module_list = [] - # create 'module_list', an array of arrays - # where array[0] = config - # array[1] = freq - # array[2:] = arguemnts - for item in cfg[name]: - if isinstance(item, str): - module_list.append((item,)) - elif isinstance(item, list): - module_list.append(item) - else: - raise TypeError("failed to read '%s' item in config") - return(module_list) - - -def run_cc_modules(cc, module_list, log): - failures = [] - for cfg_mod in module_list: - name = cfg_mod[0] - freq = None - run_args = [] - if len(cfg_mod) > 1: - freq = cfg_mod[1] - if len(cfg_mod) > 2: - run_args = cfg_mod[2:] - - try: - log.debug("handling %s with freq=%s and args=%s" % - (name, freq, run_args)) - cc.handle(name, run_args, freq=freq) - except: - log.warn(traceback.format_exc()) - log.error("config handling of %s, %s, %s failed\n" % - (name, freq, run_args)) - failures.append(name) - - return(failures) - - -# always returns well formated values -# cfg is expected to have an entry 'output' in it, which is a dictionary -# that includes entries for 'init', 'config', 'final' or 'all' -# init: /var/log/cloud.out -# config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ] -# final: -# output: "| logger -p" -# error: "> /dev/null" -# this returns the specific 'mode' entry, cleanly formatted, with value -# None if if none is given -def get_output_cfg(cfg, mode="init"): - ret = [None, None] - if not 'output' in cfg: - return ret - - outcfg = cfg['output'] - if mode in outcfg: - modecfg = outcfg[mode] - else: - if 'all' not in outcfg: - return ret - # if there is a 'all' item in the output list - # then it applies to all users of this (init, config, final) - modecfg = outcfg['all'] - - # if value is a string, it specifies stdout and stderr - if isinstance(modecfg, str): - ret = [modecfg, modecfg] - - # if its a list, then we expect (stdout, stderr) - if isinstance(modecfg, list): - if len(modecfg) > 0: - ret[0] = modecfg[0] - if len(modecfg) > 1: - ret[1] = modecfg[1] - - # if it is a dictionary, expect 'out' and 'error' - # items, which indicate out and error - if isinstance(modecfg, dict): - if 'output' in modecfg: - ret[0] = modecfg['output'] - if 'error' in modecfg: - ret[1] = modecfg['error'] - - # if err's entry == "&1", then make it same as stdout - # as in shell syntax of "echo foo >/dev/null 2>&1" - if ret[1] == "&1": - ret[1] = ret[0] - - swlist = [">>", ">", "|"] - for i in range(len(ret)): - if not ret[i]: - continue - val = ret[i].lstrip() - found = False - for s in swlist: - if val.startswith(s): - val = "%s %s" % (s, val[len(s):].strip()) - found = True - break - if not found: - # default behavior is append - val = "%s %s" % (">>", val.strip()) - ret[i] = val - - return(ret) - - -# redirect_output(outfmt, errfmt, orig_out, orig_err) -# replace orig_out and orig_err with filehandles specified in outfmt or errfmt -# fmt can be: -# > FILEPATH -# >> FILEPATH -# | program [ arg1 [ arg2 [ ... ] ] ] -# -# with a '|', arguments are passed to shell, so one level of -# shell escape is required. -def redirect_output(outfmt, errfmt, o_out=sys.stdout, o_err=sys.stderr): - if outfmt: - (mode, arg) = outfmt.split(" ", 1) - if mode == ">" or mode == ">>": - owith = "ab" - if mode == ">": - owith = "wb" - new_fp = open(arg, owith) - elif mode == "|": - proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin - else: - raise TypeError("invalid type for outfmt: %s" % outfmt) - - if o_out: - os.dup2(new_fp.fileno(), o_out.fileno()) - if errfmt == outfmt: - os.dup2(new_fp.fileno(), o_err.fileno()) - return - - if errfmt: - (mode, arg) = errfmt.split(" ", 1) - if mode == ">" or mode == ">>": - owith = "ab" - if mode == ">": - owith = "wb" - new_fp = open(arg, owith) - elif mode == "|": - proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin - else: - raise TypeError("invalid type for outfmt: %s" % outfmt) - - if o_err: - os.dup2(new_fp.fileno(), o_err.fileno()) - return - def form_module_name(name): canon_name = name.replace("-", "_") @@ -209,13 +38,18 @@ def form_module_name(name): return canon_name -def fixup_module(mod): - freq = getattr(mod, "frequency", None) - if not freq: - setattr(mod, 'frequency', PER_INSTANCE) - handler = getattr(mod, "handle", None) - if not handler: +def fixup_module(mod, def_freq=PER_INSTANCE): + if not hasattr(mod, 'frequency'): + setattr(mod, 'frequency', def_freq) + else: + freq = mod.frequency + if freq and freq not in FREQUENCIES: + LOG.warn("Module %s has an unknown frequency %s", mod, freq) + if not hasattr(mod, 'handle'): def empty_handle(_name, _cfg, _cloud, _log, _args): pass setattr(mod, 'handle', empty_handle) + # Used only for warning if possibly running on a not checked distro... + if not hasattr(mod, 'distros'): + setattr(mod, 'distros', None) return mod diff --git a/cloudinit/transforms/cc_apt_pipelining.py b/cloudinit/transforms/cc_apt_pipelining.py index 0286a9ae..69027b0c 100644 --- a/cloudinit/transforms/cc_apt_pipelining.py +++ b/cloudinit/transforms/cc_apt_pipelining.py @@ -16,10 +16,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance +from cloudinit import util +from cloudinit.settings import PER_INSTANCE + +frequency = PER_INSTANCE + +distros = ['ubuntu', 'debian'] -frequency = per_instance default_file = "/etc/apt/apt.conf.d/90cloud-init-pipelining" diff --git a/cloudinit/transforms/cc_apt_update_upgrade.py b/cloudinit/transforms/cc_apt_update_upgrade.py index a7049bce..c4a543ed 100644 --- a/cloudinit/transforms/cc_apt_update_upgrade.py +++ b/cloudinit/transforms/cc_apt_update_upgrade.py @@ -18,12 +18,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import subprocess -import traceback -import os import glob -import cloudinit.CloudConfig as cc +import os + +from cloudinit import templater +from cloudinit import util + +distros = ['ubuntu', 'debian'] def handle(_name, cfg, cloud, log, _args): @@ -34,13 +35,13 @@ def handle(_name, cfg, cloud, log, _args): mirror = find_apt_mirror(cloud, cfg) - log.debug("selected mirror at: %s" % mirror) + log.debug("Selected mirror at: %s" % mirror) - if not util.get_cfg_option_bool(cfg, \ - 'apt_preserve_sources_list', False): - generate_sources_list(release, mirror) - old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', \ - "archive.ubuntu.com/ubuntu") + if not util.get_cfg_option_bool(cfg, + 'apt_preserve_sources_list', False): + generate_sources_list(release, mirror, cloud, log) + old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', + "archive.ubuntu.com/ubuntu") rename_apt_lists(old_mir, mirror) # set up proxy @@ -49,19 +50,18 @@ def handle(_name, cfg, cloud, log, _args): if proxy: try: contents = "Acquire::HTTP::Proxy \"%s\";\n" - with open(proxy_filename, "w") as fp: - fp.write(contents % proxy) + util.write_file(proxy_filename, contents % (proxy)) except Exception as e: - log.warn("Failed to write proxy to %s" % proxy_filename) + util.logexc(log, "Failed to write proxy to %s", proxy_filename) elif os.path.isfile(proxy_filename): - os.unlink(proxy_filename) + util.del_file(proxy_filename) # process 'apt_sources' if 'apt_sources' in cfg: errors = add_sources(cfg['apt_sources'], {'MIRROR': mirror, 'RELEASE': release}) for e in errors: - log.warn("Source Error: %s\n" % ':'.join(e)) + log.warn("Source Error: %s", ':'.join(e)) dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False) if dconf_sel: @@ -69,41 +69,35 @@ def handle(_name, cfg, cloud, log, _args): try: util.subp(('debconf-set-selections', '-'), dconf_sel) except: - log.error("Failed to run debconf-set-selections") - log.debug(traceback.format_exc()) + util.logexc(log, "Failed to run debconf-set-selections") pkglist = util.get_cfg_option_list_or_str(cfg, 'packages', []) errors = [] if update or len(pkglist) or upgrade: try: - cc.update_package_sources() - except subprocess.CalledProcessError as e: - log.warn("apt-get update failed") - log.debug(traceback.format_exc()) + cloud.distro.update_package_sources() + except Exception as e: + util.logexc(log, "Package update failed") errors.append(e) if upgrade: try: - cc.apt_get("upgrade") - except subprocess.CalledProcessError as e: - log.warn("apt upgrade failed") - log.debug(traceback.format_exc()) + cloud.distro.package_command("upgrade") + except Exception as e: + util.logexc(log, "Package upgrade failed") errors.append(e) if len(pkglist): try: - cc.install_packages(pkglist) - except subprocess.CalledProcessError as e: - log.warn("Failed to install packages: %s " % pkglist) - log.debug(traceback.format_exc()) + cloud.distro.install_packages(pkglist) + except Exception as e: + util.logexc(log, "Failed to install packages: %s ", pkglist) errors.append(e) if len(errors): raise errors[0] - return(True) - def mirror2lists_fileprefix(mirror): string = mirror @@ -120,37 +114,40 @@ def mirror2lists_fileprefix(mirror): def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"): oprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(omirror)) nprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(new_mirror)) - if(oprefix == nprefix): + if oprefix == nprefix: return olen = len(oprefix) for filename in glob.glob("%s_*" % oprefix): - os.rename(filename, "%s%s" % (nprefix, filename[olen:])) + util.rename(filename, "%s%s" % (nprefix, filename[olen:])) def get_release(): - stdout, _stderr = subprocess.Popen(['lsb_release', '-cs'], - stdout=subprocess.PIPE).communicate() - return(str(stdout).strip()) + (stdout, _stderr) = util.subp(['lsb_release', '-cs']) + return stdout.strip() -def generate_sources_list(codename, mirror): - util.render_to_file('sources.list', '/etc/apt/sources.list', \ - {'mirror': mirror, 'codename': codename}) +def generate_sources_list(codename, mirror, cloud, log): + template_fn = cloud.get_template_filename('sources.list') + if template_fn: + params = {'mirror': mirror, 'codename': codename} + templater.render_to_file(template_fn, '/etc/apt/sources.list', params) + else: + log.warn("No template found, not rendering /etc/apt/sources.list") -def add_sources(srclist, searchList=None): +def add_sources(srclist, template_params=None): """ add entries in /etc/apt/sources.list.d for each abbreviated sources.list entry in 'srclist'. When rendering template, also include the values in dictionary searchList """ - if searchList is None: - searchList = {} - elst = [] + if template_params is None: + template_params = {} + errorlist = [] for ent in srclist: if 'source' not in ent: - elst.append(["", "missing source"]) + errorlist.append(["", "missing source"]) continue source = ent['source'] @@ -158,17 +155,17 @@ def add_sources(srclist, searchList=None): try: util.subp(["add-apt-repository", source]) except: - elst.append([source, "add-apt-repository failed"]) + errorlist.append([source, "add-apt-repository failed"]) continue - source = util.render_string(source, searchList) + source = templater.render_string(source, template_params) if 'filename' not in ent: ent['filename'] = 'cloud_config_sources.list' if not ent['filename'].startswith("/"): - ent['filename'] = "%s/%s" % \ - ("/etc/apt/sources.list.d/", ent['filename']) + ent['filename'] = os.path.join("/etc/apt/sources.list.d/", + ent['filename']) if ('keyid' in ent and 'key' not in ent): ks = "keyserver.ubuntu.com" @@ -177,32 +174,26 @@ def add_sources(srclist, searchList=None): try: ent['key'] = util.getkeybyid(ent['keyid'], ks) except: - elst.append([source, "failed to get key from %s" % ks]) + errorlist.append([source, "failed to get key from %s" % ks]) continue if 'key' in ent: try: util.subp(('apt-key', 'add', '-'), ent['key']) except: - elst.append([source, "failed add key"]) + errorlist.append([source, "failed add key"]) try: util.write_file(ent['filename'], source + "\n", omode="ab") except: - elst.append([source, "failed write to file %s" % ent['filename']]) + errorlist.append([source, "failed write to file %s" % ent['filename']]) - return(elst) + return errorlist def find_apt_mirror(cloud, cfg): """ find an apt_mirror given the cloud and cfg provided """ - # TODO: distro and defaults should be configurable - distro = "ubuntu" - defaults = { - 'ubuntu': "http://archive.ubuntu.com/ubuntu", - 'debian': "http://archive.debian.org/debian", - } mirror = None cfg_mirror = cfg.get("apt_mirror", None) @@ -211,14 +202,13 @@ def find_apt_mirror(cloud, cfg): elif "apt_mirror_search" in cfg: mirror = util.search_for_mirror(cfg['apt_mirror_search']) else: - if cloud: - mirror = cloud.get_mirror() + mirror = cloud.get_local_mirror() mydom = "" doms = [] - if not mirror and cloud: + if not mirror: # if we have a fqdn, then search its domain portion first (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) mydom = ".".join(fqdn.split(".")[1:]) @@ -236,6 +226,6 @@ def find_apt_mirror(cloud, cfg): mirror = util.search_for_mirror(mirror_list) if not mirror: - mirror = defaults[distro] + mirror = cloud.distro.get_package_mirror() return mirror diff --git a/cloudinit/transforms/cc_bootcmd.py b/cloudinit/transforms/cc_bootcmd.py index f584da02..a2efad32 100644 --- a/cloudinit/transforms/cc_bootcmd.py +++ b/cloudinit/transforms/cc_bootcmd.py @@ -17,32 +17,36 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import subprocess -import tempfile + import os -from cloudinit.CloudConfig import per_always -frequency = per_always +import tempfile + +from cloudinit import util +from cloudinit.settings import PER_ALWAYS +frequency = PER_ALWAYS + + +def handle(name, cfg, cloud, log, _args): -def handle(_name, cfg, cloud, log, _args): if "bootcmd" not in cfg: + log.debug("Skipping module named %s, no 'bootcomd' key in configuration", name) return - try: - content = util.shellify(cfg["bootcmd"]) - tmpf = tempfile.TemporaryFile() - tmpf.write(content) - tmpf.seek(0) - except: - log.warn("failed to shellify bootcmd") - raise - - try: - env = os.environ.copy() - env['INSTANCE_ID'] = cloud.get_instance_id() - subprocess.check_call(['/bin/sh'], env=env, stdin=tmpf) - tmpf.close() - except: - log.warn("failed to run commands from bootcmd") - raise + with tempfile.NamedTemporaryFile(suffix=".sh") as tmpf: + try: + content = util.shellify(cfg["bootcmd"]) + tmpf.write(content) + tmpf.flush() + except: + log.warn("Failed to shellify bootcmd") + raise + + try: + env = os.environ.copy() + env['INSTANCE_ID'] = cloud.get_instance_id() + cmd = ['/bin/sh', tmpf.name] + util.subp(cmd, env=env, capture=False) + except: + log.warn("Failed to run commands from bootcmd") + raise diff --git a/cloudinit/transforms/cc_byobu.py b/cloudinit/transforms/cc_byobu.py index e821b261..38586174 100644 --- a/cloudinit/transforms/cc_byobu.py +++ b/cloudinit/transforms/cc_byobu.py @@ -18,18 +18,19 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import subprocess -import traceback +from cloudinit import util +distros = ['ubuntu', 'debian'] -def handle(_name, cfg, _cloud, log, args): + +def handle(name, cfg, _cloud, log, args): if len(args) != 0: value = args[0] else: value = util.get_cfg_option_str(cfg, "byobu_by_default", "") if not value: + log.debug("Skipping module named %s, no 'byobu' values found", name) return if value == "user" or value == "system": @@ -38,7 +39,7 @@ def handle(_name, cfg, _cloud, log, args): valid = ("enable-user", "enable-system", "enable", "disable-user", "disable-system", "disable") if not value in valid: - log.warn("Unknown value %s for byobu_by_default" % value) + log.warn("Unknown value %s for byobu_by_default", value) mod_user = value.endswith("-user") mod_sys = value.endswith("-system") @@ -65,13 +66,6 @@ def handle(_name, cfg, _cloud, log, args): cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] - log.debug("setting byobu to %s" % value) + log.debug("Setting byobu to %s", value) - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) - except OSError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd failed to execute: %s" % (cmd)) + util.subp(cmd) diff --git a/cloudinit/transforms/cc_ca_certs.py b/cloudinit/transforms/cc_ca_certs.py index 3af6238a..8ca9a200 100644 --- a/cloudinit/transforms/cc_ca_certs.py +++ b/cloudinit/transforms/cc_ca_certs.py @@ -13,10 +13,10 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . + import os -from subprocess import check_call -from cloudinit.util import (write_file, get_cfg_option_list_or_str, - delete_dir_contents, subp) + +from cloudinit import util CA_CERT_PATH = "/usr/share/ca-certificates/" CA_CERT_FILENAME = "cloud-init-ca-certs.crt" @@ -28,7 +28,7 @@ def update_ca_certs(): """ Updates the CA certificate cache on the current machine. """ - check_call(["update-ca-certificates"]) + util.subp(["update-ca-certificates"]) def add_ca_certs(certs): @@ -41,9 +41,9 @@ def add_ca_certs(certs): if certs: cert_file_contents = "\n".join(certs) cert_file_fullpath = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) - write_file(cert_file_fullpath, cert_file_contents, mode=0644) + util.write_file(cert_file_fullpath, cert_file_contents, mode=0644) # Append cert filename to CA_CERT_CONFIG file. - write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="a") + util.write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="ab") def remove_default_ca_certs(): @@ -51,14 +51,14 @@ def remove_default_ca_certs(): Removes all default trusted CA certificates from the system. To actually apply the change you must also call L{update_ca_certs}. """ - delete_dir_contents(CA_CERT_PATH) - delete_dir_contents(CA_CERT_SYSTEM_PATH) - write_file(CA_CERT_CONFIG, "", mode=0644) + util.delete_dir_contents(CA_CERT_PATH) + util.delete_dir_contents(CA_CERT_SYSTEM_PATH) + util.write_file(CA_CERT_CONFIG, "", mode=0644) debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" - subp(('debconf-set-selections', '-'), debconf_sel) + util.subp(('debconf-set-selections', '-'), debconf_sel) -def handle(_name, cfg, _cloud, log, _args): +def handle(name, cfg, _cloud, log, _args): """ Call to handle ca-cert sections in cloud-config file. @@ -70,6 +70,7 @@ def handle(_name, cfg, _cloud, log, _args): """ # If there isn't a ca-certs section in the configuration don't do anything if "ca-certs" not in cfg: + log.debug("Skipping module named %s, no 'ca-certs' key in configuration", name) return ca_cert_cfg = cfg['ca-certs'] @@ -81,7 +82,7 @@ def handle(_name, cfg, _cloud, log, _args): # If we are given any new trusted CA certs to add, add them. if "trusted" in ca_cert_cfg: - trusted_certs = get_cfg_option_list_or_str(ca_cert_cfg, "trusted") + trusted_certs = util.get_cfg_option_list_or_str(ca_cert_cfg, "trusted") if trusted_certs: log.debug("adding %d certificates" % len(trusted_certs)) add_ca_certs(trusted_certs) diff --git a/cloudinit/transforms/cc_chef.py b/cloudinit/transforms/cc_chef.py index 941e04fe..12c2f539 100644 --- a/cloudinit/transforms/cc_chef.py +++ b/cloudinit/transforms/cc_chef.py @@ -18,53 +18,59 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import os -import subprocess import json -import cloudinit.CloudConfig as cc -import cloudinit.util as util +import os + +from cloudinit import templater +from cloudinit import util ruby_version_default = "1.8" -def handle(_name, cfg, cloud, log, _args): +def handle(name, cfg, cloud, log, _args): + # If there isn't a chef key in the configuration don't do anything if 'chef' not in cfg: + log.debug("Skipping module named %s, no 'chef' key in configuration", name) return chef_cfg = cfg['chef'] # ensure the chef directories we use exist - mkdirs(['/etc/chef', '/var/log/chef', '/var/lib/chef', - '/var/cache/chef', '/var/backups/chef', '/var/run/chef']) + util.ensure_dirs(['/etc/chef', '/var/log/chef', '/var/lib/chef', + '/var/cache/chef', '/var/backups/chef', '/var/run/chef']) # set the validation key based on the presence of either 'validation_key' # or 'validation_cert'. In the case where both exist, 'validation_key' # takes precedence for key in ('validation_key', 'validation_cert'): if key in chef_cfg and chef_cfg[key]: - with open('/etc/chef/validation.pem', 'w') as validation_key_fh: - validation_key_fh.write(chef_cfg[key]) + util.write_file('/etc/chef/validation.pem', chef_cfg[key]) break # create the chef config from template - util.render_to_file('chef_client.rb', '/etc/chef/client.rb', - {'server_url': chef_cfg['server_url'], - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', - cloud.datasource.get_instance_id()), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - '_default'), - 'validation_name': chef_cfg['validation_name']}) + template_fn = cloud.get_template_filename('chef_client.rb') + if template_fn: + params = { + 'server_url': chef_cfg['server_url'], + 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', + cloud.datasource.get_instance_id()), + 'environment': util.get_cfg_option_str(chef_cfg, 'environment', + '_default'), + 'validation_name': chef_cfg['validation_name'] + } + templater.render_to_file(template_fn, '/etc/chef/client.rb', params) + else: + log.warn("No template found, not rendering to /etc/chef/client.rb") # set the firstboot json - with open('/etc/chef/firstboot.json', 'w') as firstboot_json_fh: - initial_json = {} - if 'run_list' in chef_cfg: - initial_json['run_list'] = chef_cfg['run_list'] - if 'initial_attributes' in chef_cfg: - initial_attributes = chef_cfg['initial_attributes'] - for k in initial_attributes.keys(): - initial_json[k] = initial_attributes[k] - firstboot_json_fh.write(json.dumps(initial_json)) + initial_json = {} + if 'run_list' in chef_cfg: + initial_json['run_list'] = chef_cfg['run_list'] + if 'initial_attributes' in chef_cfg: + initial_attributes = chef_cfg['initial_attributes'] + for k in initial_attributes.keys(): + initial_json[k] = initial_attributes[k] + util.write_file('/etc/chef/firstboot.json', json.dumps(initial_json)) # If chef is not installed, we install chef based on 'install_type' if not os.path.isfile('/usr/bin/chef-client'): @@ -75,14 +81,15 @@ def handle(_name, cfg, cloud, log, _args): chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', ruby_version_default) - install_chef_from_gems(ruby_version, chef_version) + install_chef_from_gems(cloud.distro, ruby_version, chef_version) # and finally, run chef-client - log.debug('running chef-client') - subprocess.check_call(['/usr/bin/chef-client', '-d', '-i', '1800', - '-s', '20']) - else: + log.debug('Running chef-client') + util.subp(['/usr/bin/chef-client', '-d', '-i', '1800', '-s', '20']) + elif install_type == 'packages': # this will install and run the chef-client from packages - cc.install_packages(('chef',)) + cloud.distro.install_packages(('chef',)) + else: + log.warn("Unknown chef install type %s", install_type) def get_ruby_packages(version): @@ -90,30 +97,20 @@ def get_ruby_packages(version): pkgs = ['ruby%s' % version, 'ruby%s-dev' % version] if version == "1.8": pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8')) - return(pkgs) + return pkgs -def install_chef_from_gems(ruby_version, chef_version=None): - cc.install_packages(get_ruby_packages(ruby_version)) +def install_chef_from_gems(ruby_version, chef_version, distro): + distro.install_packages(get_ruby_packages(ruby_version)) if not os.path.exists('/usr/bin/gem'): - os.symlink('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') + util.sym_link('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') if not os.path.exists('/usr/bin/ruby'): - os.symlink('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') + util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') if chef_version: - subprocess.check_call(['/usr/bin/gem', 'install', 'chef', - '-v %s' % chef_version, '--no-ri', - '--no-rdoc', '--bindir', '/usr/bin', '-q']) + util.subp(['/usr/bin/gem', 'install', 'chef', + '-v %s' % chef_version, '--no-ri', + '--no-rdoc', '--bindir', '/usr/bin', '-q']) else: - subprocess.check_call(['/usr/bin/gem', 'install', 'chef', - '--no-ri', '--no-rdoc', '--bindir', - '/usr/bin', '-q']) - - -def ensure_dir(d): - if not os.path.exists(d): - os.makedirs(d) - - -def mkdirs(dirs): - for d in dirs: - ensure_dir(d) + util.subp(['/usr/bin/gem', 'install', 'chef', + '--no-ri', '--no-rdoc', '--bindir', + '/usr/bin', '-q']) diff --git a/cloudinit/transforms/cc_disable_ec2_metadata.py b/cloudinit/transforms/cc_disable_ec2_metadata.py index 6b31ea8e..4d2a7f55 100644 --- a/cloudinit/transforms/cc_disable_ec2_metadata.py +++ b/cloudinit/transforms/cc_disable_ec2_metadata.py @@ -17,14 +17,16 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import subprocess -from cloudinit.CloudConfig import per_always -frequency = per_always +from cloudinit import util + +from cloudinit.settings import PER_ALWAYS + +frequency = PER_ALWAYS + +reject_cmd = ['route', 'add', '-host', '169.254.169.254', 'reject'] def handle(_name, cfg, _cloud, _log, _args): if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False): - fwall = "route add -host 169.254.169.254 reject" - subprocess.call(fwall.split(' ')) + util.subp(reject_cmd) diff --git a/cloudinit/transforms/cc_final_message.py b/cloudinit/transforms/cc_final_message.py index abb4ca32..dc4ae34c 100644 --- a/cloudinit/transforms/cc_final_message.py +++ b/cloudinit/transforms/cc_final_message.py @@ -18,41 +18,54 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from cloudinit.CloudConfig import per_always import sys -from cloudinit import util, boot_finished -import time -frequency = per_always +from cloudinit import templater +from cloudinit import util +from cloudinit import version -final_message = "cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds" +from cloudinit.settings import PER_ALWAYS +frequency = PER_ALWAYS -def handle(_name, cfg, _cloud, log, args): +final_message_def = ("Cloud-init v. {{version}} finished at {{timestamp}}." + " Up {{uptime}} seconds.") + + +def handle(name, cfg, cloud, log, args): + + msg_in = None if len(args) != 0: msg_in = args[0] else: - msg_in = util.get_cfg_option_str(cfg, "final_message", final_message) + msg_in = util.get_cfg_option_str(cfg, "final_message") + + if not msg_in: + template_fn = cloud.get_template_filename('final_message') + if template_fn: + msg_in = util.load_file(template_fn) - try: - uptimef = open("/proc/uptime") - uptime = uptimef.read().split(" ")[0] - uptimef.close() - except IOError as e: - log.warn("unable to open /proc/uptime\n") - uptime = "na" + if not msg_in: + msg_in = final_message_def + uptime = util.uptime() + ts = util.time_rfc2822() + cver = version.version_string() try: - ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) - except: - ts = "na" - - try: - subs = {'UPTIME': uptime, 'TIMESTAMP': ts} - sys.stdout.write("%s\n" % util.render_string(msg_in, subs)) + subs = { + 'uptime': uptime, + 'timestamp': ts, + 'version': cver, + } + # Use stdout, stderr or the logger?? + content = templater.render_string(msg_in, subs) + sys.stderr.write("%s\n" % (content)) except Exception as e: - log.warn("failed to render string to stdout: %s" % e) + util.logexc(log, "Failed to render final message template") - fp = open(boot_finished, "wb") - fp.write(uptime + "\n") - fp.close() + boot_fin_fn = cloud.paths.boot_finished + try: + contents = "%s - %s - v. %s\n" % (uptime, ts, cver) + util.write_file(boot_fin_fn, contents) + except: + util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn) diff --git a/cloudinit/transforms/cc_foo.py b/cloudinit/transforms/cc_foo.py index 35ec3fa7..8007f981 100644 --- a/cloudinit/transforms/cc_foo.py +++ b/cloudinit/transforms/cc_foo.py @@ -18,12 +18,35 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#import cloudinit -#import cloudinit.util as util -from cloudinit.CloudConfig import per_instance +from cloudinit.settings import PER_INSTANCE -frequency = per_instance +# Modules are expected to have the following attributes. +# 1. A required 'handle' method which takes the following params. +# a) The name will not be this files name, but instead +# the name specified in configuration (which is the name +# which will be used to find this module). +# b) A configuration object that is the result of the merging +# of cloud configs configuration with legacy configuration +# as well as any datasource provided configuration +# c) A cloud object that can be used to access various +# datasource and paths for the given distro and data provided +# by the various datasource instance types. +# d) A argument list that may or may not be empty to this module. +# Typically those are from module configuration where the module +# is defined with some extra configuration that will eventually +# be translated from yaml into arguments to this module. +# 2. A optional 'frequency' that defines how often this module should be ran. +# Typically one of PER_INSTANCE, PER_ALWAYS, PER_ONCE. If not +# provided PER_INSTANCE will be assumed. +# See settings.py for these constants. +# 3. A optional 'distros' array/set/tuple that defines the known distros +# this module will work with (if not all of them). This is used to write +# a warning out if a module is being ran on a untested distribution for +# informational purposes. If non existent all distros are assumed and +# no warning occurs. +frequency = settings.PER_INSTANCE -def handle(_name, _cfg, _cloud, _log, _args): - print "hi" + +def handle(name, _cfg, _cloud, _log, _args): + print("Hi from %s" % (name)) diff --git a/cloudinit/transforms/cc_grub_dpkg.py b/cloudinit/transforms/cc_grub_dpkg.py index 9f3a7eaf..c048d5cc 100644 --- a/cloudinit/transforms/cc_grub_dpkg.py +++ b/cloudinit/transforms/cc_grub_dpkg.py @@ -18,10 +18,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import traceback import os +from cloudinit import util + +distros = ['ubuntu', 'debian'] + def handle(_name, cfg, _cloud, log, _args): idevs = None @@ -52,13 +54,14 @@ def handle(_name, cfg, _cloud, log, _args): # now idevs and idevs_empty are set to determined values # or, those set by user - dconf_sel = "grub-pc grub-pc/install_devices string %s\n" % idevs + \ - "grub-pc grub-pc/install_devices_empty boolean %s\n" % idevs_empty - log.debug("setting grub debconf-set-selections with '%s','%s'" % + dconf_sel = ("grub-pc grub-pc/install_devices string %s\n" + "grub-pc grub-pc/install_devices_empty boolean %s\n") % + (idevs, idevs_empty) + + log.debug("Setting grub debconf-set-selections with '%s','%s'" % (idevs, idevs_empty)) try: - util.subp(('debconf-set-selections'), dconf_sel) + util.subp(['debconf-set-selections'], dconf_sel) except: - log.error("Failed to run debconf-set-selections for grub-dpkg") - log.debug(traceback.format_exc()) + util.logexc(log, "Failed to run debconf-set-selections for grub-dpkg") diff --git a/cloudinit/transforms/cc_keys_to_console.py b/cloudinit/transforms/cc_keys_to_console.py index 73a477c0..2f2a5297 100644 --- a/cloudinit/transforms/cc_keys_to_console.py +++ b/cloudinit/transforms/cc_keys_to_console.py @@ -18,11 +18,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from cloudinit.CloudConfig import per_instance -import cloudinit.util as util -import subprocess +from cloudinit.settings import PER_INSTANCE +from cloudinit import util -frequency = per_instance +frequency = PER_INSTANCE def handle(_name, cfg, _cloud, log, _args): @@ -32,11 +31,10 @@ def handle(_name, cfg, _cloud, log, _args): key_blacklist = util.get_cfg_option_list_or_str(cfg, "ssh_key_console_blacklist", ["ssh-dss"]) try: - confp = open('/dev/console', "wb") cmd.append(','.join(fp_blacklist)) cmd.append(','.join(key_blacklist)) - subprocess.call(cmd, stdout=confp) - confp.close() + (stdout, stderr) = util.subp(cmd) + util.write_file('/dev/console', stdout) except: - log.warn("writing keys to console value") + log.warn("Writing keys to console failed!") raise diff --git a/cloudinit/transforms/cc_landscape.py b/cloudinit/transforms/cc_landscape.py index a4113cbe..48491992 100644 --- a/cloudinit/transforms/cc_landscape.py +++ b/cloudinit/transforms/cc_landscape.py @@ -19,14 +19,24 @@ # along with this program. If not, see . import os -import os.path -from cloudinit.CloudConfig import per_instance -from configobj import ConfigObj -frequency = per_instance +from StringIO import StringIO + +try: + from configobj import ConfigObj +except ImportError: + ConfigObj = None + +from cloudinit import util + +from cloudinit.settings import PER_INSTANCE + +frequency = PER_INSTANCE lsc_client_cfg_file = "/etc/landscape/client.conf" +distros = ['ubuntu'] + # defaults taken from stock client.conf in landscape-client 11.07.1.1-0ubuntu2 lsc_builtincfg = { 'client': { @@ -38,36 +48,43 @@ lsc_builtincfg = { } -def handle(_name, cfg, _cloud, log, _args): +def handle(name, cfg, _cloud, log, _args): """ Basically turn a top level 'landscape' entry with a 'client' dict and render it to ConfigObj format under '[client]' section in /etc/landscape/client.conf """ + if not ConfigObj: + log.warn("'ConfigObj' support not enabled, running %s disabled", name) + return ls_cloudcfg = cfg.get("landscape", {}) if not isinstance(ls_cloudcfg, dict): - raise(Exception("'landscape' existed in config, but not a dict")) + raise Exception(("'landscape' key existed in config," + " but not a dictionary type," + " is a %s instead"), util.obj_name(ls_cloudcfg)) - merged = mergeTogether([lsc_builtincfg, lsc_client_cfg_file, ls_cloudcfg]) + merged = merge_together([lsc_builtincfg, lsc_client_cfg_file, ls_cloudcfg]) if not os.path.isdir(os.path.dirname(lsc_client_cfg_file)): - os.makedirs(os.path.dirname(lsc_client_cfg_file)) - - with open(lsc_client_cfg_file, "w") as fp: - merged.write(fp) + util.ensure_dir(os.path.dirname(lsc_client_cfg_file)) - log.debug("updated %s" % lsc_client_cfg_file) + contents = StringIO() + merged.write(contents) + util.write_file(lsc_client_cfg_file, contents.getvalue()) + log.debug("Wrote landscape config file to %s", lsc_client_cfg_file) -def mergeTogether(objs): +def merge_together(objs): """ merge together ConfigObj objects or things that ConfigObj() will take in later entries override earlier """ cfg = ConfigObj({}) for obj in objs: + if not obj: + continue if isinstance(obj, ConfigObj): cfg.merge(obj) else: diff --git a/cloudinit/transforms/cc_locale.py b/cloudinit/transforms/cc_locale.py index 2bb22fdb..3fb4c5d9 100644 --- a/cloudinit/transforms/cc_locale.py +++ b/cloudinit/transforms/cc_locale.py @@ -18,22 +18,28 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import os.path -import subprocess -import traceback +import os +from cloudinit import templater +from cloudinit import util -def apply_locale(locale, cfgfile): + +def apply_locale(locale, cfgfile, cloud, log): + # TODO this command might not work on RH... if os.path.exists('/usr/sbin/locale-gen'): - subprocess.Popen(['locale-gen', locale]).communicate() + util.subp(['locale-gen', locale], capture=False) if os.path.exists('/usr/sbin/update-locale'): - subprocess.Popen(['update-locale', locale]).communicate() - - util.render_to_file('default-locale', cfgfile, {'locale': locale}) + util.subp(['update-locale', locale], capture=False) + if not cfgfile: + return + template_fn = cloud.get_template_filename('default-locale') + if not template_fn: + log.warn("No template filename found to write to %s", cfgfile) + else: + templater.render_to_file(template_fn, cfgfile, {'locale': locale}) -def handle(_name, cfg, cloud, log, args): +def handle(name, cfg, cloud, log, args): if len(args) != 0: locale = args[0] else: @@ -43,12 +49,10 @@ def handle(_name, cfg, cloud, log, args): "/etc/default/locale") if not locale: + log.debug(("Skipping module named %s, " + "no 'locale' configuration found"), name) return - log.debug("setting locale to %s" % locale) + log.debug("Setting locale to %s", locale) - try: - apply_locale(locale, locale_cfgfile) - except Exception as e: - log.debug(traceback.format_exc(e)) - raise Exception("failed to apply locale %s" % locale) + apply_locale(locale, locale_cfgfile, cloud, log) diff --git a/cloudinit/transforms/cc_mcollective.py b/cloudinit/transforms/cc_mcollective.py index a2a6230c..aeeda9d2 100644 --- a/cloudinit/transforms/cc_mcollective.py +++ b/cloudinit/transforms/cc_mcollective.py @@ -19,50 +19,53 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from ConfigParser import ConfigParser +from StringIO import StringIO + import os -import subprocess -import StringIO -import ConfigParser -import cloudinit.CloudConfig as cc -import cloudinit.util as util + +from cloudinit import util +from cloudinit import cfg pubcert_file = "/etc/mcollective/ssl/server-public.pem" pricert_file = "/etc/mcollective/ssl/server-private.pem" -# Our fake header section -class FakeSecHead(object): - def __init__(self, fp): - self.fp = fp - self.sechead = '[nullsection]\n' - - def readline(self): - if self.sechead: - try: - return self.sechead - finally: - self.sechead = None - else: - return self.fp.readline() +def handle(name, cfg, cloud, log, _args): - -def handle(_name, cfg, _cloud, _log, _args): # If there isn't a mcollective key in the configuration don't do anything if 'mcollective' not in cfg: + log.debug(("Skipping module named %s, " + "no 'mcollective' key in configuration"), name) return + mcollective_cfg = cfg['mcollective'] + # Start by installing the mcollective package ... - cc.install_packages(("mcollective",)) + cloud.distro.install_packages(("mcollective",)) # ... and then update the mcollective configuration if 'conf' in mcollective_cfg: # Create object for reading server.cfg values - mcollective_config = ConfigParser.ConfigParser() + mcollective_config = cfg.DefaultingConfigParser() # Read server.cfg values from original file in order to be able to mix # the rest up - mcollective_config.readfp(FakeSecHead(open('/etc/mcollective/' - 'server.cfg'))) - for cfg_name, cfg in mcollective_cfg['conf'].iteritems(): + old_contents = util.load_file('/etc/mcollective/server.cfg') + # It doesn't contain any sections so just add one temporarily + # Use a hash id based off the contents, + # just incase of conflicts... (try to not have any...) + # This is so that an error won't occur when reading (and no + # sections exist in the file) + section_tpl = "[nullsection_%s]" + attempts = 0 + section_head = section_tpl % (attempts) + while old_contents.find(section_head) != -1: + attempts += 1 + section_head = section_tpl % (attempts) + sectioned_contents = "%s\n%s" % (section_head, old_contents) + mcollective_config.readfp(StringIO(sectioned_contents), + filename='/etc/mcollective/server.cfg') + for (cfg_name, cfg) in mcollective_cfg['conf'].iteritems(): if cfg_name == 'public-cert': util.write_file(pubcert_file, cfg, mode=0644) mcollective_config.set(cfg_name, @@ -76,24 +79,19 @@ def handle(_name, cfg, _cloud, _log, _args): else: # Iterate throug the config items, we'll use ConfigParser.set # to overwrite or create new items as needed - for o, v in cfg.iteritems(): + for (o, v) in cfg.iteritems(): mcollective_config.set(cfg_name, o, v) # We got all our config as wanted we'll rename # the previous server.cfg and create our new one - os.rename('/etc/mcollective/server.cfg', - '/etc/mcollective/server.cfg.old') - outputfile = StringIO.StringIO() - mcollective_config.write(outputfile) - # Now we got the whole file, write to disk except first line + util.rename('/etc/mcollective/server.cfg', + '/etc/mcollective/server.cfg.old') + # Now we got the whole file, write to disk except the section + # we added so that config parser won't error out when trying to read. # Note below, that we've just used ConfigParser because it generally - # works. Below, we remove the initial 'nullsection' header - # and then change 'key = value' to 'key: value'. The global - # search and replace of '=' with ':' could be problematic though. - # this most likely needs fixing. - util.write_file('/etc/mcollective/server.cfg', - outputfile.getvalue().replace('[nullsection]\n', '').replace(' =', - ':'), - mode=0644) + # works. Below, we remove the initial 'nullsection' header. + contents = mcollective_config.stringify() + contents = contents.replace("%s\n" % (section_head), "") + util.write_file('/etc/mcollective/server.cfg', contents, mode=0644) # Start mcollective - subprocess.check_call(['service', 'mcollective', 'start']) + util.subp(['service', 'mcollective', 'start'], capture=False) diff --git a/cloudinit/transforms/cc_mounts.py b/cloudinit/transforms/cc_mounts.py index 6cdd74e8..babcbda1 100644 --- a/cloudinit/transforms/cc_mounts.py +++ b/cloudinit/transforms/cc_mounts.py @@ -18,10 +18,17 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util +from string import whitespace # pylint: disable=W0402 + import os import re -from string import whitespace # pylint: disable=W0402 + +from cloudinit import util + +# shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 +shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" +shortname = re.compile(shortname_filter) +ws = re.compile("[%s]+" % whitespace) def is_mdname(name): @@ -49,38 +56,46 @@ def handle(_name, cfg, cloud, log, _args): if "mounts" in cfg: cfgmnt = cfg["mounts"] - # shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 - shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" - shortname = re.compile(shortname_filter) - + for i in range(len(cfgmnt)): # skip something that wasn't a list if not isinstance(cfgmnt[i], list): + log.warn("Mount option %s not a list, got a %s instead", + (i + 1), util.obj_name(cfgmnt[i])) continue + startname = str(cfgmnt[i][0]) + LOG.debug("Attempting to determine the real name of %s", startname) + # workaround, allow user to specify 'ephemeral' # rather than more ec2 correct 'ephemeral0' - if cfgmnt[i][0] == "ephemeral": + if startname == "ephemeral": cfgmnt[i][0] = "ephemeral0" + log.debug("Adjusted mount option %s name from ephemeral to ephemeral0", (i + 1)) - if is_mdname(cfgmnt[i][0]): - newname = cloud.device_name_to_device(cfgmnt[i][0]) + if is_mdname(startname): + newname = cloud.device_name_to_device(startname) if not newname: - log.debug("ignoring nonexistant named mount %s" % cfgmnt[i][0]) + log.debug("Ignoring nonexistant named mount %s", startname) cfgmnt[i][1] = None else: - if newname.startswith("/"): - cfgmnt[i][0] = newname - else: - cfgmnt[i][0] = "/dev/%s" % newname + renamed = newname + if not newname.startswith("/"): + renamed = "/dev/%s" % newname + cfgmnt[i][0] = renamed + log.debug("Mapped metadata name %s to %s", startname, renamed) else: - if shortname.match(cfgmnt[i][0]): - cfgmnt[i][0] = "/dev/%s" % cfgmnt[i][0] + if shortname.match(startname): + renamed = "/dev/%s" % startname + log.debug("Mapped shortname name %s to %s", startname, renamed) + cfgmnt[i][0] = renamed # in case the user did not quote a field (likely fs-freq, fs_passno) # but do not convert None to 'None' (LP: #898365) for j in range(len(cfgmnt[i])): - if isinstance(cfgmnt[i][j], int): + if j is None: + continue + else: cfgmnt[i][j] = str(cfgmnt[i][j]) for i in range(len(cfgmnt)): @@ -102,14 +117,18 @@ def handle(_name, cfg, cloud, log, _args): # for each of the "default" mounts, add them only if no other # entry has the same device name for defmnt in defmnts: - devname = cloud.device_name_to_device(defmnt[0]) + startname = defmnt[0] + devname = cloud.device_name_to_device(startname) if devname is None: + log.debug("Ignoring nonexistant named default mount %s", startname) continue if devname.startswith("/"): defmnt[0] = devname else: defmnt[0] = "/dev/%s" % devname + log.debug("Mapped default device %s to %s", startname, defmnt[0]) + cfgmnt_has = False for cfgm in cfgmnt: if cfgm[0] == defmnt[0]: @@ -117,14 +136,21 @@ def handle(_name, cfg, cloud, log, _args): break if cfgmnt_has: + log.debug("Not including %s, already previously included", startname) continue cfgmnt.append(defmnt) # now, each entry in the cfgmnt list has all fstab values # if the second field is None (not the string, the value) we skip it - actlist = [x for x in cfgmnt if x[1] is not None] + actlist = [] + for x in cfgmnt: + if x[1] is None: + log.debug("Skipping non-existent device named %s", x[0]) + else: + actlist.append(x) if len(actlist) == 0: + log.debug("No modifications to fstab needed.") return comment = "comment=cloudconfig" @@ -141,8 +167,7 @@ def handle(_name, cfg, cloud, log, _args): cc_lines.append('\t'.join(line)) fstab_lines = [] - fstab = open("/etc/fstab", "r+") - ws = re.compile("[%s]+" % whitespace) + fstab = util.load_file("/etc/fstab") for line in fstab.read().splitlines(): try: toks = ws.split(line) @@ -153,27 +178,22 @@ def handle(_name, cfg, cloud, log, _args): fstab_lines.append(line) fstab_lines.extend(cc_lines) - - fstab.seek(0) - fstab.write("%s\n" % '\n'.join(fstab_lines)) - fstab.truncate() - fstab.close() + contents = "%s\n" % ('\n'.join(fstab_lines)) + util.write_file("/etc/fstab", contents) if needswap: try: util.subp(("swapon", "-a")) except: - log.warn("Failed to enable swap") + util.logexc(log, "Activating swap via 'swapon -a' failed") for d in dirs: - if os.path.exists(d): - continue try: - os.makedirs(d) + util.ensure_dir(d) except: - log.warn("Failed to make '%s' config-mount\n", d) + util.logexc(log, "Failed to make '%s' config-mount", d) try: util.subp(("mount", "-a")) except: - log.warn("'mount -a' failed") + util.logexc(log, "Activating mounts via 'mount -a' failed") diff --git a/cloudinit/transforms/cc_phone_home.py b/cloudinit/transforms/cc_phone_home.py index a7ff74e1..36af6dfa 100644 --- a/cloudinit/transforms/cc_phone_home.py +++ b/cloudinit/transforms/cc_phone_home.py @@ -17,13 +17,18 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from cloudinit.CloudConfig import per_instance -import cloudinit.util as util + +from cloudinit import templater +from cloudinit import url_helper as uhelp +from cloudinit import util + +from cloudinit.settings import PER_INSTANCE + from time import sleep -frequency = per_instance -post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', - 'hostname'] +frequency = PER_INSTANCE +post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', + 'instance_id', 'hostname'] # phone_home: @@ -35,7 +40,7 @@ post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', # url: http://my.foo.bar/$INSTANCE_ID/ # post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id # -def handle(_name, cfg, cloud, log, args): +def handle(name, cfg, cloud, log, args): if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: @@ -44,7 +49,8 @@ def handle(_name, cfg, cloud, log, args): ph_cfg = cfg['phone_home'] if 'url' not in ph_cfg: - log.warn("no 'url' token in phone_home") + log.warn(("Skipping module named %s, " + "no 'url' found in 'phone_home' configuration"), name) return url = ph_cfg['url'] @@ -53,8 +59,8 @@ def handle(_name, cfg, cloud, log, args): try: tries = int(tries) except: - log.warn("tries is not an integer. using 10") tries = 10 + util.logexc(log, "Configuration entry 'tries' is not an integer, using %s", tries) if post_list == "all": post_list = post_list_all @@ -71,11 +77,9 @@ def handle(_name, cfg, cloud, log, args): for n, path in pubkeys.iteritems(): try: - fp = open(path, "rb") - all_keys[n] = fp.read() - fp.close() + all_keys[n] = util.load_file(path) except: - log.warn("%s: failed to open in phone_home" % path) + util.logexc(log, "%s: failed to open, can not phone home that data", path) submit_keys = {} for k in post_list: @@ -83,24 +87,11 @@ def handle(_name, cfg, cloud, log, args): submit_keys[k] = all_keys[k] else: submit_keys[k] = "N/A" - log.warn("requested key %s from 'post' list not available") + log.warn("Requested key %s from 'post' configuration list not available", k) - url = util.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) + url = templater.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) - null_exc = object() - last_e = null_exc - for i in range(0, tries): - try: - util.readurl(url, submit_keys) - log.debug("succeeded submit to %s on try %i" % (url, i + 1)) - return - except Exception as e: - log.debug("failed to post to %s on try %i" % (url, i + 1)) - last_e = e - sleep(3) - - log.warn("failed to post to %s in %i tries" % (url, tries)) - if last_e is not null_exc: - raise(last_e) - - return + try: + uhelp.readurl(url, data=submit_keys, retries=tries, sec_between=3) + except: + util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries) diff --git a/cloudinit/transforms/cc_puppet.py b/cloudinit/transforms/cc_puppet.py index 6fc475f6..0a21a929 100644 --- a/cloudinit/transforms/cc_puppet.py +++ b/cloudinit/transforms/cc_puppet.py @@ -18,91 +18,85 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from StringIO import StringIO + import os -import os.path import pwd import socket -import subprocess -import StringIO -import ConfigParser -import cloudinit.CloudConfig as cc -import cloudinit.util as util + +from cloudinit import util +from cloudinit import cfg -def handle(_name, cfg, cloud, log, _args): +def handle(name, cfg, cloud, log, _args): # If there isn't a puppet key in the configuration don't do anything if 'puppet' not in cfg: + log.debug(("Skipping module named %s," + " no 'puppet' configuration found"), name) return + puppet_cfg = cfg['puppet'] + # Start by installing the puppet package ... - cc.install_packages(("puppet",)) + cloud.distro.install_packages(("puppet",)) # ... and then update the puppet configuration if 'conf' in puppet_cfg: # Add all sections from the conf object to puppet.conf - puppet_conf_fh = open('/etc/puppet/puppet.conf', 'r') + contents = util.load_file('/etc/puppet/puppet.conf') # Create object for reading puppet.conf values - puppet_config = ConfigParser.ConfigParser() + puppet_config = cfg.DefaultingConfigParser() # Read puppet.conf values from original file in order to be able to - # mix the rest up - puppet_config.readfp(StringIO.StringIO(''.join(i.lstrip() for i in - puppet_conf_fh.readlines()))) - # Close original file, no longer needed - puppet_conf_fh.close() - for cfg_name, cfg in puppet_cfg['conf'].iteritems(): + # mix the rest up. First clean them up (TODO is this really needed??) + cleaned_contents = '\n'.join([i.lstrip() for i in contents.splitlines()]) + puppet_config.readfp(StringIO(cleaned_contents), + filename='/etc/puppet/puppet.conf') + for (cfg_name, cfg) in puppet_cfg['conf'].iteritems(): # ca_cert configuration is a special case # Dump the puppetmaster ca certificate in the correct place if cfg_name == 'ca_cert': # Puppet ssl sub-directory isn't created yet # Create it with the proper permissions and ownership - os.makedirs('/var/lib/puppet/ssl') - os.chmod('/var/lib/puppet/ssl', 0771) - os.chown('/var/lib/puppet/ssl', - pwd.getpwnam('puppet').pw_uid, 0) - os.makedirs('/var/lib/puppet/ssl/certs/') - os.chown('/var/lib/puppet/ssl/certs/', - pwd.getpwnam('puppet').pw_uid, 0) - ca_fh = open('/var/lib/puppet/ssl/certs/ca.pem', 'w') - ca_fh.write(cfg) - ca_fh.close() - os.chown('/var/lib/puppet/ssl/certs/ca.pem', - pwd.getpwnam('puppet').pw_uid, 0) - util.restorecon_if_possible('/var/lib/puppet', recursive=True) + util.ensure_dir('/var/lib/puppet/ssl', 0771) + util.chownbyid('/var/lib/puppet/ssl', + pwd.getpwnam('puppet').pw_uid, 0) + util.ensure_dir('/var/lib/puppet/ssl/certs/') + util.chownbyid('/var/lib/puppet/ssl/certs/', + pwd.getpwnam('puppet').pw_uid, 0) + util.write_file('/var/lib/puppet/ssl/certs/ca.pem', cfg) + util.chownbyid('/var/lib/puppet/ssl/certs/ca.pem', + pwd.getpwnam('puppet').pw_uid, 0) else: - #puppet_conf_fh.write("\n[%s]\n" % (cfg_name)) - # If puppet.conf already has this section we don't want to - # write it again - if puppet_config.has_section(cfg_name) == False: - puppet_config.add_section(cfg_name) # Iterate throug the config items, we'll use ConfigParser.set # to overwrite or create new items as needed - for o, v in cfg.iteritems(): + for (o, v) in cfg.iteritems(): if o == 'certname': # Expand %f as the fqdn + # TODO should this use the cloud fqdn?? v = v.replace("%f", socket.getfqdn()) # Expand %i as the instance id - v = v.replace("%i", - cloud.datasource.get_instance_id()) - # certname needs to be downcase + v = v.replace("%i", cloud.get_instance_id()) + # certname needs to be downcased v = v.lower() puppet_config.set(cfg_name, o, v) - #puppet_conf_fh.write("%s=%s\n" % (o, v)) # We got all our config as wanted we'll rename # the previous puppet.conf and create our new one - os.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') - with open('/etc/puppet/puppet.conf', 'wb') as configfile: - puppet_config.write(configfile) - util.restorecon_if_possible('/etc/puppet/puppet.conf') + util.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') + contents = puppet_config.stringify() + util.write_file('/etc/puppet/puppet.conf', contents) + # Set puppet to automatically start if os.path.exists('/etc/default/puppet'): - subprocess.check_call(['sed', '-i', - '-e', 's/^START=.*/START=yes/', - '/etc/default/puppet']) + util.subp(['sed', '-i', + '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet'], capture=False) elif os.path.exists('/bin/systemctl'): - subprocess.check_call(['/bin/systemctl', 'enable', 'puppet.service']) + util.subp(['/bin/systemctl', 'enable', 'puppet.service'], capture=False) elif os.path.exists('/sbin/chkconfig'): - subprocess.check_call(['/sbin/chkconfig', 'puppet', 'on']) + util.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) else: - log.warn("Do not know how to enable puppet service on this system") + log.warn(("Sorry we do not know how to enable" + " puppet services on this system")) + # Start puppetd - subprocess.check_call(['service', 'puppet', 'start']) + util.subp(['service', 'puppet', 'start'], capture=False) diff --git a/cloudinit/transforms/cc_resizefs.py b/cloudinit/transforms/cc_resizefs.py index 2dc66def..daaf4da9 100644 --- a/cloudinit/transforms/cc_resizefs.py +++ b/cloudinit/transforms/cc_resizefs.py @@ -18,91 +18,117 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import subprocess +import errno import os import stat import sys -import time import tempfile -from cloudinit.CloudConfig import per_always - -frequency = per_always +import time +from cloudinit import util +from cloudinit.settings import PER_ALWAYS -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - resize_root = False - if str(args[0]).lower() in ['true', '1', 'on', 'yes']: - resize_root = True - else: - resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) +frequency = PER_ALWAYS - if str(resize_root).lower() in ['false', '0']: - return +resize_fs_prefixes_cmds = [ + ('ext', 'resize2fs'), + ('xfs', 'xfs_growfs'), +] - # we use mktemp rather than mkstemp because early in boot nothing - # else should be able to race us for this, and we need to mknod. - devpth = tempfile.mktemp(prefix="cloudinit.resizefs.", dir="/run") +def nodeify_path(devpth, where, log): try: - st_dev = os.stat("/").st_dev + st_dev = os.stat(where).st_dev dev = os.makedev(os.major(st_dev), os.minor(st_dev)) os.mknod(devpth, 0400 | stat.S_IFBLK, dev) + return st_dev except: if util.is_container(): - log.debug("inside container, ignoring mknod failure in resizefs") + log.debug("Inside container, ignoring mknod failure in resizefs") return - log.warn("Failed to make device node to resize /") + log.warn("Failed to make device node to resize %s at %s", where, devpth) raise - cmd = ['blkid', '-c', '/dev/null', '-sTYPE', '-ovalue', devpth] + +def get_fs_type(st_dev, path, log): try: - (fstype, _err) = util.subp(cmd) - except subprocess.CalledProcessError as e: - log.warn("Failed to get filesystem type of maj=%s, min=%s via: %s" % - (os.major(st_dev), os.minor(st_dev), cmd)) - log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) - os.unlink(devpth) + fs_type = util.find_devs_with(tag='TYPE', oformat='value', + no_cache=True, path=path) + return fs_type + except util.ProcessExecutionError: + util.logexc(log, ("Failed to get filesystem type" + " of maj=%s, min=%s for path %s"), + os.major(st_dev), os.minor(st_dev), path) raise - if str(fstype).startswith("ext"): - resize_cmd = ['resize2fs', devpth] - elif fstype == "xfs": - resize_cmd = ['xfs_growfs', devpth] + +def handle(name, cfg, _cloud, log, args): + if len(args) != 0: + resize_root = args[0] else: - os.unlink(devpth) - log.debug("not resizing unknown filesystem %s" % fstype) + resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) + + if not util.translate_bool(resize_root): + log.debug("Skipping module named %s, resizing disabled", name) return - if resize_root == "noblock": - fid = os.fork() - if fid == 0: - try: - do_resize(resize_cmd, devpth, log) - os._exit(0) # pylint: disable=W0212 - except Exception as exc: - sys.stderr.write("Failed: %s" % exc) - os._exit(1) # pylint: disable=W0212 - else: - do_resize(resize_cmd, devpth, log) + # TODO is the directory ok to be used?? + resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run") + util.ensure_dir(resize_root_d) + with util.SilentTemporaryFile(prefix="cloudinit.resizefs.", + dir=resize_root_d, delete=True) as tfh: + devpth = tfh.name + + # Delete the file so that mknod will work + # but don't change the file handle to know that its + # removed so that when a later call that recreates + # occurs this temporary file will still benefit from + # auto deletion + tfh.unlink_now() + + # TODO: allow what is to be resized to + # be configurable?? + st_dev = nodeify_path(devpth, "/", log) + fs_type = get_fs_type(st_dev, devpath, log) + + resizer = None + fstype_lc = fstype.lower() + for (pfix, root_cmd) in resize_fs_prefixes_cmds: + if fstype_lc.startswith(pfix): + resizer = root_cmd + break - log.debug("resizing root filesystem (type=%s, maj=%i, min=%i, val=%s)" % - (str(fstype).rstrip("\n"), os.major(st_dev), os.minor(st_dev), - resize_root)) + if not resizer: + log.warn("Not resizing unknown filesystem type %s", fs_type) + return + + log.debug("Resizing using %s", resizer) + resize_cmd = [resizer, devpth] - return + if resize_root == "noblock": + # Fork to a child that will run + # the resize command + util.fork_cb(do_resize, resize_cmd, log) + # Don't delete the file now in the parent + tfh.delete = False + else: + do_resize(resize_cmd, log) + action = 'Resized' + if resize_root == "noblock": + action = 'Resizing (via forking)' + log.debug("%s root filesystem (type=%s, maj=%i, min=%i, val=%s)", + action, fs_type, os.major(st_dev), os.minor(st_dev), resize_root) -def do_resize(resize_cmd, devpth, log): + +def do_resize(resize_cmd, log): + start = time.time() try: - start = time.time() util.subp(resize_cmd) - except subprocess.CalledProcessError as e: - log.warn("Failed to resize filesystem (%s)" % resize_cmd) - log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) - os.unlink(devpth) + except util.ProcessExecutionError as e: + util.logexc(log, "Failed to resize filesystem (using %s)", resize_cmd) raise - - os.unlink(devpth) - log.debug("resize took %s seconds" % (time.time() - start)) + tot_time = int(time.time() - start) + log.debug("Resizing took %s seconds", tot_time) + # TODO: Should we add a fsck check after this to make + # sure we didn't corrupt anything? diff --git a/cloudinit/transforms/cc_rightscale_userdata.py b/cloudinit/transforms/cc_rightscale_userdata.py index 5ed0848f..cde11b54 100644 --- a/cloudinit/transforms/cc_rightscale_userdata.py +++ b/cloudinit/transforms/cc_rightscale_userdata.py @@ -35,44 +35,64 @@ ## ## -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_ipath_cur +import os + +from cloudinit import url_helper as uhelp +from cloudinit import util +from cloudinit.settings import PER_INSTANCE + from urlparse import parse_qs -frequency = per_instance +frequency = PER_INSTANCE + my_name = "cc_rightscale_userdata" my_hookname = 'CLOUD_INIT_REMOTE_HOOK' -def handle(_name, _cfg, cloud, log, _args): +def handle(name, _cfg, cloud, log, _args): try: ud = cloud.get_userdata_raw() except: - log.warn("failed to get raw userdata in %s" % my_name) + log.warn("Failed to get raw userdata in module %s", name) return try: mdict = parse_qs(ud) - if not my_hookname in mdict: + if not mdict or not my_hookname in mdict: + log.debug("Skipping module %s, did not find %s in parsed raw userdata", name, my_hookname) return except: - log.warn("failed to urlparse.parse_qa(userdata_raw())") + log.warn("Failed to parse query string %s into a dictionary", ud) raise - scripts_d = get_ipath_cur('scripts') - i = 0 - first_e = None - for url in mdict[my_hookname]: - fname = "%s/rightscale-%02i" % (scripts_d, i) - i = i + 1 + wrote_fns = [] + captured_excps = [] + + # These will eventually be then ran by the cc_scripts_user + # TODO: maybe this should just be a new user data handler?? + # Instead of a late transform that acts like a user data handler? + scripts_d = cloud.get_ipath_cur('scripts') + urls = mdict[my_hookname] + for (i, url) in enumerate(urls): + fname = os.path.join(scripts_d, "rightscale-%02i" % (i)) try: - content = util.readurl(url) - util.write_file(fname, content, mode=0700) + (content, st) = uhelp.readurl(url) + # Ensure its a valid http response (and something gotten) + if uhelp.ok_http_code(st) and content: + util.write_file(fname, content, mode=0700) + wrote_fns.append(fname) except Exception as e: - if not first_e: - first_e = None - log.warn("%s failed to read %s: %s" % (my_name, url, e)) + captured_excps.append(e) + util.logexc(log, "%s failed to read %s and write %s", my_name, url, fname) + + if wrote_fns: + log.debug("Wrote out rightscale userdata to %s files", len(wrote_fns)) + + if len(wrote_fns) != len(urls): + skipped = len(urls) - len(wrote_fns) + log.debug("%s urls were skipped or failed", skipped) + + if captured_excps: + log.warn("%s failed with exceptions, re-raising the last one", len(captured_excps)) + raise captured_excps[-1] - if first_e: - raise(e) diff --git a/cloudinit/transforms/cc_rsyslog.py b/cloudinit/transforms/cc_rsyslog.py index ac7f2c74..ccbe68ff 100644 --- a/cloudinit/transforms/cc_rsyslog.py +++ b/cloudinit/transforms/cc_rsyslog.py @@ -18,16 +18,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit -import logging -import cloudinit.util as util -import traceback +import os + +from cloudinit import util DEF_FILENAME = "20-cloud-config.conf" DEF_DIR = "/etc/rsyslog.d" -def handle(_name, cfg, _cloud, log, _args): +def handle(name, cfg, cloud, log, _args): # rsyslog: # - "*.* @@192.158.1.1" # - content: "*.* @@192.0.2.1:10514" @@ -37,17 +36,17 @@ def handle(_name, cfg, _cloud, log, _args): # process 'rsyslog' if not 'rsyslog' in cfg: + log.debug("Skipping module named %s, no 'rsyslog' key in configuration", name) return def_dir = cfg.get('rsyslog_dir', DEF_DIR) def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) files = [] - elst = [] - for ent in cfg['rsyslog']: + for i, ent in enumerate(cfg['rsyslog']): if isinstance(ent, dict): if not "content" in ent: - elst.append((ent, "no 'content' entry")) + log.warn("No 'content' entry in config entry %s", i + 1) continue content = ent['content'] filename = ent.get("filename", def_fname) @@ -55,8 +54,13 @@ def handle(_name, cfg, _cloud, log, _args): content = ent filename = def_fname + filename = filename.strip() + if not filename: + log.warn("Entry %s has an empty filename", i + 1) + continue + if not filename.startswith("/"): - filename = "%s/%s" % (def_dir, filename) + filename = os.path.join(def_dir, filename) omode = "ab" # truncate filename first time you see it @@ -67,35 +71,29 @@ def handle(_name, cfg, _cloud, log, _args): try: util.write_file(filename, content + "\n", omode=omode) except Exception as e: - log.debug(traceback.format_exc(e)) - elst.append((content, "failed to write to %s" % filename)) + util.logexc(log, "Failed to write to %s", filename) - # need to restart syslogd + # Attempt to restart syslogd restarted = False try: - # if this config module is running at cloud-init time + # If this config module is running at cloud-init time # (before rsyslog is running) we don't actually have to # restart syslog. # - # upstart actually does what we want here, in that it doesn't + # Upstart actually does what we want here, in that it doesn't # start a service that wasn't running already on 'restart' # it will also return failure on the attempt, so 'restarted' - # won't get set - log.debug("restarting rsyslog") + # won't get set. + log.debug("Restarting rsyslog") util.subp(['service', 'rsyslog', 'restart']) restarted = True - except Exception as e: - elst.append(("restart", str(e))) + util.logexc("Failed restarting rsyslog") if restarted: - # this only needs to run if we *actually* restarted + # This only needs to run if we *actually* restarted # syslog above. - cloudinit.logging_set_from_cfg_file() - log = logging.getLogger() - log.debug("rsyslog configured %s" % files) - - for e in elst: - log.warn("rsyslog error: %s\n" % ':'.join(e)) - - return + cloud.cycle_logging() + # This should now use rsyslog if + # the logging was setup to use it... + log.debug("%s configured %s files", name, files) diff --git a/cloudinit/transforms/cc_runcmd.py b/cloudinit/transforms/cc_runcmd.py index f7e8c671..19c0e721 100644 --- a/cloudinit/transforms/cc_runcmd.py +++ b/cloudinit/transforms/cc_runcmd.py @@ -18,15 +18,20 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util +import os +from cloudinit import util -def handle(_name, cfg, cloud, log, _args): + +def handle(name, cfg, cloud, log, _args): if "runcmd" not in cfg: + log.debug("Skipping module named %s, no 'runcmd' key in configuration", name) return - outfile = "%s/runcmd" % cloud.get_ipath('scripts') + + outfile = os.path.join(cloud.get_ipath('scripts'), "runcmd") + cmd = cfg["runcmd"] try: - content = util.shellify(cfg["runcmd"]) + content = util.shellify(cmd) util.write_file(outfile, content, 0700) except: - log.warn("failed to open %s for runcmd" % outfile) + util.logexc(log, "Failed to shellify %s into file %s", cmd, outfile) diff --git a/cloudinit/transforms/cc_salt_minion.py b/cloudinit/transforms/cc_salt_minion.py index 1a3b5039..47cbc194 100644 --- a/cloudinit/transforms/cc_salt_minion.py +++ b/cloudinit/transforms/cc_salt_minion.py @@ -15,42 +15,43 @@ # along with this program. If not, see . import os -import os.path -import subprocess -import cloudinit.CloudConfig as cc -import yaml +from cloudinit import util -def handle(_name, cfg, _cloud, _log, _args): +# Note: see http://saltstack.org/topics/installation/ + + +def handle(name, cfg, cloud, _log, _args): # If there isn't a salt key in the configuration don't do anything if 'salt_minion' not in cfg: + log.debug("Skipping module named %s, no 'salt_minion' key in configuration", name) return + salt_cfg = cfg['salt_minion'] + # Start by installing the salt package ... - cc.install_packages(("salt",)) - config_dir = '/etc/salt' - if not os.path.isdir(config_dir): - os.makedirs(config_dir) + cloud.distro.install_packages(("salt",)) + + # Ensure we can configure files at the right dir + config_dir = salt_cfg.get("config_dir", '/etc/salt') + util.ensure_dir(config_dir) + # ... and then update the salt configuration if 'conf' in salt_cfg: # Add all sections from the conf object to /etc/salt/minion minion_config = os.path.join(config_dir, 'minion') - yaml.dump(salt_cfg['conf'], - file(minion_config, 'w'), - default_flow_style=False) + minion_data = util.yaml_dumps(salt_cfg.get('conf')) + util.write_file(minion_config, minion_data) + # ... copy the key pair if specified if 'public_key' in salt_cfg and 'private_key' in salt_cfg: - pki_dir = '/etc/salt/pki' - cumask = os.umask(077) - if not os.path.isdir(pki_dir): - os.makedirs(pki_dir) - pub_name = os.path.join(pki_dir, 'minion.pub') - pem_name = os.path.join(pki_dir, 'minion.pem') - with open(pub_name, 'w') as f: - f.write(salt_cfg['public_key']) - with open(pem_name, 'w') as f: - f.write(salt_cfg['private_key']) - os.umask(cumask) + pki_dir = salt_cfg.get('pki_dir', '/etc/salt/pki') + with util.umask(077): + util.ensure_dir(pki_dir) + pub_name = os.path.join(pki_dir, 'minion.pub') + pem_name = os.path.join(pki_dir, 'minion.pem') + util.write_file(pub_name, salt_cfg['public_key']) + util.write_file(pem_name, salt_cfg['private_key']) # Start salt-minion - subprocess.check_call(['service', 'salt-minion', 'start']) + util.subp(['service', 'salt-minion', 'start'], capture=False) diff --git a/cloudinit/transforms/cc_scripts_per_boot.py b/cloudinit/transforms/cc_scripts_per_boot.py index 41a74754..bcdf4400 100644 --- a/cloudinit/transforms/cc_scripts_per_boot.py +++ b/cloudinit/transforms/cc_scripts_per_boot.py @@ -18,17 +18,23 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -from cloudinit.CloudConfig import per_always -from cloudinit import get_cpath +import os -frequency = per_always -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-boot") +from cloudinit import util +from cloudinit.settings import PER_ALWAYS -def handle(_name, _cfg, _cloud, log, _args): +frequency = PER_ALWAYS + +script_subdir = 'per-boot' + + +def handle(_name, _cfg, cloud, log, _args): + # Comes from the following: + # https://forums.aws.amazon.com/thread.jspa?threadID=96918 + runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) try: util.runparts(runparts_path) except: - log.warn("failed to run-parts in %s" % runparts_path) + log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_scripts_per_instance.py b/cloudinit/transforms/cc_scripts_per_instance.py index a2981eab..8d6609a1 100644 --- a/cloudinit/transforms/cc_scripts_per_instance.py +++ b/cloudinit/transforms/cc_scripts_per_instance.py @@ -18,17 +18,23 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_cpath +import os -frequency = per_instance -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-instance") +from cloudinit import util +from cloudinit.settings import PER_INSTANCE -def handle(_name, _cfg, _cloud, log, _args): +frequency = PER_INSTANCE + +script_subdir = 'per-instance' + + +def handle(_name, _cfg, cloud, log, _args): + # Comes from the following: + # https://forums.aws.amazon.com/thread.jspa?threadID=96918 + runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) try: util.runparts(runparts_path) except: - log.warn("failed to run-parts in %s" % runparts_path) + log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_scripts_per_once.py b/cloudinit/transforms/cc_scripts_per_once.py index a69151da..dbcec05d 100644 --- a/cloudinit/transforms/cc_scripts_per_once.py +++ b/cloudinit/transforms/cc_scripts_per_once.py @@ -18,17 +18,23 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -from cloudinit.CloudConfig import per_once -from cloudinit import get_cpath +import os -frequency = per_once -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-once") +from cloudinit import util +from cloudinit.settings import PER_ONCE -def handle(_name, _cfg, _cloud, log, _args): +frequency = PER_ONCE + +script_subdir = 'per-once' + + +def handle(_name, _cfg, cloud, log, _args): + # Comes from the following: + # https://forums.aws.amazon.com/thread.jspa?threadID=96918 + runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) try: util.runparts(runparts_path) except: - log.warn("failed to run-parts in %s" % runparts_path) + log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_scripts_user.py b/cloudinit/transforms/cc_scripts_user.py index 933aa4e0..1e438ee6 100644 --- a/cloudinit/transforms/cc_scripts_user.py +++ b/cloudinit/transforms/cc_scripts_user.py @@ -18,17 +18,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_ipath_cur +import os -frequency = per_instance -runparts_path = "%s/%s" % (get_ipath_cur(), "scripts") +from cloudinit import util +from cloudinit.settings import PER_INSTANCE -def handle(_name, _cfg, _cloud, log, _args): +frequency = PER_INSTANCE + + +def handle(_name, _cfg, cloud, log, _args): + # This is written to by the user data handlers + # Ie, any custom shell scripts that come down + # go here... + runparts_path = os.path.join(cloud.get_ipath_cur(), "scripts") try: util.runparts(runparts_path) except: - log.warn("failed to run-parts in %s" % runparts_path) + log.warn("Failed to run-parts(%s) in %s", "user-data", runparts_path) raise diff --git a/cloudinit/transforms/cc_set_hostname.py b/cloudinit/transforms/cc_set_hostname.py index acea74d9..fa2b59c2 100644 --- a/cloudinit/transforms/cc_set_hostname.py +++ b/cloudinit/transforms/cc_set_hostname.py @@ -18,25 +18,18 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util +from cloudinit import util -def handle(_name, cfg, cloud, log, _args): +def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): - log.debug("preserve_hostname is set. not setting hostname") - return(True) + log.debug(("Configuration option 'preserve_hostname' is set," + " not setting the hostname in %s"), name) + return (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) try: - set_hostname(hostname, log) + log.debug("Setting hostname to %s", hostname) + cloud.distro.set_hostname(hostname) except Exception: - util.logexc(log) - log.warn("failed to set hostname to %s\n", hostname) - - return(True) - - -def set_hostname(hostname, log): - util.subp(['hostname', hostname]) - util.write_file("/etc/hostname", "%s\n" % hostname, 0644) - log.debug("populated /etc/hostname with %s on first boot", hostname) + util.logexc(log, "Failed to set hostname to %s", hostname) diff --git a/cloudinit/transforms/cc_set_passwords.py b/cloudinit/transforms/cc_set_passwords.py index 9d0bbdb8..4f2cdb97 100644 --- a/cloudinit/transforms/cc_set_passwords.py +++ b/cloudinit/transforms/cc_set_passwords.py @@ -18,13 +18,18 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util import sys -import random -from string import letters, digits # pylint: disable=W0402 +from cloudinit import util -def handle(_name, cfg, _cloud, log, args): +from string import letters, digits + +# We are removing certain 'painful' letters/numbers +pw_set = (letters.translate(None, 'loLOI') + + digits.translate(None, '01')) + + +def handle(_name, cfg, cloud, log, args): if len(args) != 0: # if run from command line, and give args, wipe the chpasswd['list'] password = args[0] @@ -62,68 +67,83 @@ def handle(_name, cfg, _cloud, log, args): ch_in = '\n'.join(plist_in) try: + log.debug("Changing password for %s:", users) util.subp(['chpasswd'], ch_in) - log.debug("changed password for %s:" % users) except Exception as e: errors.append(e) - log.warn("failed to set passwords with chpasswd: %s" % e) + util.logexc(log, "Failed to set passwords with chpasswd for %s", users) if len(randlist): - sys.stdout.write("%s\n%s\n" % ("Set the following passwords\n", + sys.stderr.write("%s\n%s\n" % ("Set the following 'random' passwords\n", '\n'.join(randlist))) if expire: - enum = len(errors) + expired_users = [] for u in users: try: util.subp(['passwd', '--expire', u]) + expired_users.append(u) except Exception as e: errors.append(e) - log.warn("failed to expire account for %s" % u) - if enum == len(errors): - log.debug("expired passwords for: %s" % u) + util.logexc(log, "Failed to set 'expire' for %s", u) + if expired_users: + log.debug("Expired passwords for: %s users", expired_users) + change_pwauth = False + pw_auth = None if 'ssh_pwauth' in cfg: - val = str(cfg['ssh_pwauth']).lower() - if val in ("true", "1", "yes"): - pw_auth = "yes" - change_pwauth = True - elif val in ("false", "0", "no"): - pw_auth = "no" - change_pwauth = True - else: - change_pwauth = False + change_pwauth = True + if util.is_true_str(cfg['ssh_pwauth']): + pw_auth = 'yes' + if util.is_false_str(cfg['ssh_pwauth']): + pw_auth = 'no' if change_pwauth: - pa_s = "\(#*\)\(PasswordAuthentication[[:space:]]\+\)\(yes\|no\)" - msg = "set PasswordAuthentication to '%s'" % pw_auth - try: - cmd = ['sed', '-i', 's,%s,\\2%s,' % (pa_s, pw_auth), - '/etc/ssh/sshd_config'] - util.subp(cmd) - log.debug(msg) - except Exception as e: - log.warn("failed %s" % msg) - errors.append(e) + new_lines = [] + replaced_auth = False + replacement = "PasswordAuthentication %s" % (pw_auth) + + # See http://linux.die.net/man/5/sshd_config + old_lines = util.load_file('/etc/ssh/sshd_config').splitlines() + for i, line in enumerate(old_lines): + if not line.strip() or line.startswith("#"): + new_lines.append(line) + continue + splitup = line.split(None, 1) + if len(splitup) <= 1: + new_lines.append(line) + continue + (cmd, args) = splitup + # Keywords are case-insensitive and arguments are case-sensitive + cmd = cmd.lower().strip() + if cmd == 'passwordauthentication': + log.debug("Replacing auth line %s with %s", i + 1, replacement) + replaced_auth = True + new_lines.append(replacement) + else: + new_lines.append(line) + + if not replaced_auth: + log.debug("Adding new auth line %s", replacement) + replaced_auth = True + new_lines.append(replacement) + + new_contents = "\n".join(new_lines) + util.write_file('/etc/ssh/sshd_config', new_contents) try: - p = util.subp(['service', cfg.get('ssh_svcname', 'ssh'), - 'restart']) - log.debug("restarted sshd") + cmd = ['service'] + cmd.append(cloud.distro.get_option('ssh_svcname', 'ssh')) + cmd.append('restart') + util.subp(cmd) + log.debug("Restarted the ssh daemon") except: - log.warn("restart of ssh failed") + util.logexc(log, "Restarting of the ssh daemon failed") if len(errors): - raise(errors[0]) - - return - - -def rand_str(strlen=32, select_from=letters + digits): - return("".join([random.choice(select_from) for _x in range(0, strlen)])) + log.debug("%s errors occured, re-raising the last one", len(errors)) + raise errors[-1] def rand_user_password(pwlen=9): - selfrom = (letters.translate(None, 'loLOI') + - digits.translate(None, '01')) - return(rand_str(pwlen, select_from=selfrom)) + return util.rand_str(pwlen, select_from=pw_set) diff --git a/cloudinit/transforms/cc_ssh.py b/cloudinit/transforms/cc_ssh.py index 48eb58bc..db6848d9 100644 --- a/cloudinit/transforms/cc_ssh.py +++ b/cloudinit/transforms/cc_ssh.py @@ -18,15 +18,34 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import cloudinit.SshUtil as sshutil import os import glob -import subprocess -DISABLE_ROOT_OPTS = "no-port-forwarding,no-agent-forwarding," \ -"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " \ -"rather than the user \\\"root\\\".\';echo;sleep 10\"" +from cloudinit import util +from cloudinit import ssh_util + +DISABLE_ROOT_OPTS = ( "no-port-forwarding,no-agent-forwarding," +"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " +"rather than the user \\\"root\\\".\';echo;sleep 10\"") + +key2file = { + "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), + "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), + "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), + "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), + "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), + "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), +} + +priv2pub = { + 'rsa_private': 'rsa_public', + 'dsa_private': 'dsa_public', + 'ecdsa_private': 'ecdsa_public', +} + +key_gen_tpl = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' + +generate_keys = ['rsa', 'dsa', 'ecdsa'] def handle(_name, cfg, cloud, log, _args): @@ -35,72 +54,70 @@ def handle(_name, cfg, cloud, log, _args): if cfg.get("ssh_deletekeys", True): for f in glob.glob("/etc/ssh/ssh_host_*key*"): try: - os.unlink(f) + util.del_file(f) except: - pass - + util.logexc(log, "Failed deleting key file %s", f) + if "ssh_keys" in cfg: # if there are keys in cloud-config, use them - key2file = { - "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), - "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), - "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), - "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), - "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), - "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), - } - - for key, val in cfg["ssh_keys"].items(): + for (key, val) in cfg["ssh_keys"].iteritems(): if key in key2file: - util.write_file(key2file[key][0], val, key2file[key][1]) - - priv2pub = {'rsa_private': 'rsa_public', 'dsa_private': 'dsa_public', - 'ecdsa_private': 'ecdsa_public', } - + tgt_fn = key2file[key][0] + tgt_perms = key2file[key][1] + util.write_file(tgt_fn, val, tgt_perms) + cmd = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' for priv, pub in priv2pub.iteritems(): if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: continue pair = (key2file[priv][0], key2file[pub][0]) - subprocess.call(('sh', '-xc', cmd % pair)) - log.debug("generated %s from %s" % pair) + cmd = ['sh', '-xc', key_gen_tpl % pair] + try: + # TODO: Is this guard needed? + with util.SeLinuxGuard("/etc/ssh", recursive=True): + util.subp(cmd, capture=False) + log.debug("Generated a key for %s from %s", pair[0], pair[1]) + except: + util.logexc(log, "Failed generated a key for %s from %s", pair[0], pair[1]) else: # if not, generate them - for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', - ['rsa', 'dsa', 'ecdsa']): + for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', generate_keys): keyfile = '/etc/ssh/ssh_host_%s_key' % keytype if not os.path.exists(keyfile): - subprocess.call(['ssh-keygen', '-t', keytype, '-N', '', - '-f', keyfile]) - - util.restorecon_if_possible('/etc/ssh', recursive=True) + cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] + try: + # TODO: Is this guard needed? + with util.SeLinuxGuard("/etc/ssh", recursive=True): + util.subp(cmd, capture=False) + except: + util.logexc(log, "Failed generating key type %s to file %s", keytype, keyfile) try: user = util.get_cfg_option_str(cfg, 'user') disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", DISABLE_ROOT_OPTS) - keys = cloud.get_public_ssh_keys() + keys = cloud.get_public_ssh_keys() or [] if "ssh_authorized_keys" in cfg: cfgkeys = cfg["ssh_authorized_keys"] keys.extend(cfgkeys) apply_credentials(keys, user, disable_root, disable_root_opts, log) except: - util.logexc(log) - log.warn("applying credentials failed!\n") + util.logexc(log, "Applying ssh credentials failed!") def apply_credentials(keys, user, disable_root, disable_root_opts=DISABLE_ROOT_OPTS, log=None): + keys = set(keys) if user: - sshutil.setup_user_keys(keys, user, '', log) + ssh_util.setup_user_keys(keys, user, '') - if disable_root: + if disable_root and user: key_prefix = disable_root_opts.replace('$USER', user) else: key_prefix = '' - sshutil.setup_user_keys(keys, 'root', key_prefix, log) + ssh_util.setup_user_keys(keys, 'root', key_prefix) diff --git a/cloudinit/transforms/cc_ssh_import_id.py b/cloudinit/transforms/cc_ssh_import_id.py index bbf5bd83..019413d4 100644 --- a/cloudinit/transforms/cc_ssh_import_id.py +++ b/cloudinit/transforms/cc_ssh_import_id.py @@ -18,12 +18,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import subprocess -import traceback +from cloudinit import util +# The ssh-import-id only seems to exist on ubuntu (for now) +# https://launchpad.net/ssh-import-id +distros = ['ubuntu'] -def handle(_name, cfg, _cloud, log, args): + +def handle(name, cfg, _cloud, log, args): if len(args) != 0: user = args[0] ids = [] @@ -34,17 +36,14 @@ def handle(_name, cfg, _cloud, log, args): ids = util.get_cfg_option_list_or_str(cfg, "ssh_import_id", []) if len(ids) == 0: + log.debug("Skipping module named %s, no ids found to import", name) return cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids - - log.debug("importing ssh ids. cmd = %s" % cmd) + log.debug("Importing ssh ids for user %s.", user) try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) - except OSError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd failed to execute: %s" % (cmd)) + util.subp(cmd, capture=False) + except util.ProcessExecutionError as e: + util.logexc(log, "Failed to run command to import %s ssh ids", user) + raise e diff --git a/cloudinit/transforms/cc_timezone.py b/cloudinit/transforms/cc_timezone.py index e5c9901b..6fb5edc0 100644 --- a/cloudinit/transforms/cc_timezone.py +++ b/cloudinit/transforms/cc_timezone.py @@ -18,50 +18,19 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from cloudinit.CloudConfig import per_instance -from cloudinit import util -import os.path -import shutil +from cloudinit.settings import PER_INSTANCE -frequency = per_instance -tz_base = "/usr/share/zoneinfo" +frequency = PER_INSTANCE -def handle(_name, cfg, _cloud, log, args): +def handle(_name, cfg, cloud, log, args): if len(args) != 0: timezone = args[0] else: timezone = util.get_cfg_option_str(cfg, "timezone", False) if not timezone: + log.debug("Skipping module named %s, no 'timezone' specified", name) return - tz_file = "%s/%s" % (tz_base, timezone) - - if not os.path.isfile(tz_file): - log.debug("Invalid timezone %s" % tz_file) - raise Exception("Invalid timezone %s" % tz_file) - - try: - fp = open("/etc/timezone", "wb") - fp.write("%s\n" % timezone) - fp.close() - except: - log.debug("failed to write to /etc/timezone") - raise - if os.path.exists("/etc/sysconfig/clock"): - try: - with open("/etc/sysconfig/clock", "w") as fp: - fp.write('ZONE="%s"\n' % timezone) - except: - log.debug("failed to write to /etc/sysconfig/clock") - raise - - try: - shutil.copy(tz_file, "/etc/localtime") - except: - log.debug("failed to copy %s to /etc/localtime" % tz_file) - raise - - log.debug("set timezone to %s" % timezone) - return + cloud.distro.set_timezone(timezone) diff --git a/cloudinit/transforms/cc_update_etc_hosts.py b/cloudinit/transforms/cc_update_etc_hosts.py index 6ad2fca8..361097a6 100644 --- a/cloudinit/transforms/cc_update_etc_hosts.py +++ b/cloudinit/transforms/cc_update_etc_hosts.py @@ -18,70 +18,34 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -from cloudinit.CloudConfig import per_always -import StringIO +from cloudinit import util +from cloudinit import templater -frequency = per_always +from cloudinit.settings import PER_ALWAYS +frequency = PER_ALWAYS -def handle(_name, cfg, cloud, log, _args): - (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) +def handle(name, cfg, cloud, log, _args): manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) - if manage_hosts in ("True", "true", True, "template"): - # render from template file - try: - if not hostname: - log.info("manage_etc_hosts was set, but no hostname found") - return - - util.render_to_file('hosts', '/etc/hosts', + if util.translate_bool(manage_hosts, addons=['template']): + (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + # Render from template file + if not hostname: + log.warn("Option 'manage_etc_hosts' was set, but no hostname was found") + return + tpl_fn_name = cloud.get_template_filename("hosts.%s" % (cloud.distro.name())) + if not tpl_fn_name: + raise Exception("No hosts template could be found for distro %s" % (cloud.distro.name())) + templater.render_to_file(tpl_fn_name, '/etc/hosts', {'hostname': hostname, 'fqdn': fqdn}) - except Exception: - log.warn("failed to update /etc/hosts") - raise elif manage_hosts == "localhost": - log.debug("managing 127.0.1.1 in /etc/hosts") - update_etc_hosts(hostname, fqdn, log) - return + log.debug("Managing localhost in /etc/hosts") + (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + if not hostname: + log.warn("Option 'manage_etc_hosts' was set, but no hostname was found") + return + cloud.distro.update_etc_hosts(hostname, fqdn) else: - if manage_hosts not in ("False", False): - log.warn("Unknown value for manage_etc_hosts. Assuming False") - else: - log.debug("not managing /etc/hosts") - - -def update_etc_hosts(hostname, fqdn, _log): - with open('/etc/hosts', 'r') as etchosts: - header = "# Added by cloud-init\n" - hosts_line = "127.0.1.1\t%s %s\n" % (fqdn, hostname) - need_write = False - need_change = True - new_etchosts = StringIO.StringIO() - for line in etchosts: - split_line = [s.strip() for s in line.split()] - if len(split_line) < 2: - new_etchosts.write(line) - continue - if line == header: - continue - ip, hosts = split_line[0], split_line[1:] - if ip == "127.0.1.1": - if sorted([hostname, fqdn]) == sorted(hosts): - need_change = False - if need_change == True: - line = "%s%s" % (header, hosts_line) - need_change = False - need_write = True - new_etchosts.write(line) - etchosts.close() - if need_change == True: - new_etchosts.write("%s%s" % (header, hosts_line)) - need_write = True - if need_write == True: - new_etcfile = open('/etc/hosts', 'wb') - new_etcfile.write(new_etchosts.getvalue()) - new_etcfile.close() - new_etchosts.close() - return + log.debug(("Configuration option 'manage_etc_hosts' is not set," + " not managing /etc/hosts in %s"), name) diff --git a/cloudinit/transforms/cc_update_hostname.py b/cloudinit/transforms/cc_update_hostname.py index b9d1919a..439bdcb3 100644 --- a/cloudinit/transforms/cc_update_hostname.py +++ b/cloudinit/transforms/cc_update_hostname.py @@ -18,84 +18,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -import subprocess -import errno -from cloudinit.CloudConfig import per_always +from cloudinit import util +from cloudinit.settings import PER_ALWAYS -frequency = per_always +frequency = PER_ALWAYS -def handle(_name, cfg, cloud, log, _args): +def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): - log.debug("preserve_hostname is set. not updating hostname") + log.debug(("Configuration option 'preserve_hostname' is set," + " not updating the hostname in %s"), name) return (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) try: - prev = "%s/%s" % (cloud.get_cpath('data'), "previous-hostname") - update_hostname(hostname, prev, log) + prev_fn = os.path.join(cloud.get_cpath('data'), "previous-hostname") + cloud.distro.update_hostname(hostname, prev_fn) except Exception: - log.warn("failed to set hostname\n") + util.logexc(log, "Failed to set the hostname to %s", hostname) raise - - -# read hostname from a 'hostname' file -# allow for comments and stripping line endings. -# if file doesn't exist, or no contents, return default -def read_hostname(filename, default=None): - try: - fp = open(filename, "r") - lines = fp.readlines() - fp.close() - for line in lines: - hpos = line.find("#") - if hpos != -1: - line = line[0:hpos] - line = line.rstrip() - if line: - return line - except IOError as e: - if e.errno != errno.ENOENT: - raise - return default - - -def update_hostname(hostname, prev_file, log): - etc_file = "/etc/hostname" - - hostname_prev = None - hostname_in_etc = None - - try: - hostname_prev = read_hostname(prev_file) - except Exception as e: - log.warn("Failed to open %s: %s" % (prev_file, e)) - - try: - hostname_in_etc = read_hostname(etc_file) - except: - log.warn("Failed to open %s" % etc_file) - - update_files = [] - if not hostname_prev or hostname_prev != hostname: - update_files.append(prev_file) - - if (not hostname_in_etc or - (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): - update_files.append(etc_file) - - try: - for fname in update_files: - util.write_file(fname, "%s\n" % hostname, 0644) - log.debug("wrote %s to %s" % (hostname, fname)) - except: - log.warn("failed to write hostname to %s" % fname) - - if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: - log.debug("%s differs from %s. assuming user maintained" % - (prev_file, etc_file)) - - if etc_file in update_files: - log.debug("setting hostname to %s" % hostname) - subprocess.Popen(['hostname', hostname]).communicate() -- cgit v1.2.3 From 7b3bf46487e599b375acfdf99176294810805ef0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 18:24:04 -0700 Subject: Ensure when an exception is captured that we use the util.logexc helper. --- cloudinit/ssh_util.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index c97b3819..f6941a29 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -144,13 +144,13 @@ class AuthKeyEntry(object): return ' '.join(toks) -def _update_authorized_keys(fname, keys): +def update_authorized_keys(fname, keys): lines = [] try: if os.path.isfile(fname): lines = util.load_file(fname).splitlines() except (IOError, OSError): - LOG.exception("Error reading lines from %s", fname) + util.logexc(LOG, "Error reading lines from %s", fname) lines = [] to_add = list(keys) @@ -199,7 +199,7 @@ def setup_user_keys(keys, user, key_prefix, sshd_config_fn=None): # The following tokens are defined: %% is replaced by a literal # '%', %h is replaced by the home directory of the user being # authenticated and %u is replaced by the username of that user. - ssh_cfg = _parse_ssh_config(sshd_config_fn) + ssh_cfg = parse_ssh_config(sshd_config_fn) akeys = ssh_cfg.get("authorizedkeysfile", '') akeys = akeys.strip() if not akeys: @@ -212,19 +212,19 @@ def setup_user_keys(keys, user, key_prefix, sshd_config_fn=None): authorized_keys = akeys except (IOError, OSError): authorized_keys = os.path.join(ssh_dir, 'authorized_keys') - LOG.exception(("Failed extracting 'AuthorizedKeysFile'" - " in ssh config" - " from %s, using 'AuthorizedKeysFile' file" - " %s instead"), - sshd_config_fn, authorized_keys) + util.logexc(LOG, ("Failed extracting 'AuthorizedKeysFile'" + " in ssh config" + " from %s, using 'AuthorizedKeysFile' file" + " %s instead"), + sshd_config_fn, authorized_keys) - content = _update_authorized_keys(authorized_keys, key_entries) + content = update_authorized_keys(authorized_keys, key_entries) util.ensure_dir(os.path.dirname(authorized_keys), mode=0700) util.write_file(authorized_keys, content, mode=0600) util.chownbyid(authorized_keys, pwent.pw_uid, pwent.pw_gid) -def _parse_ssh_config(fname): +def parse_ssh_config(fname): # The file contains keyword-argument pairs, one per line. # Lines starting with '#' and empty lines are interpreted as comments. # Note: key-words are case-insensitive and arguments are case-sensitive -- cgit v1.2.3 From 7f94f5b510c2ed8f0d449a7eaa9b7f15e041bf44 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 18:24:40 -0700 Subject: 1. Modify the check http code function to allow or not allow redirect status from being valid codes. 2. Log how much we will sleep for 3. If not headers are provided, add a set which will have a cloud init user agent + version --- cloudinit/url_helper.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 1bf24c4f..7ae0226a 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -29,12 +29,16 @@ import urllib import urllib2 from cloudinit import log as logging +from cloudinit import version LOG = logging.getLogger(__name__) -def ok_http_code(st): - return st in xrange(200, 400) +def ok_http_code(st, redirects_ok=False): + if redirects_ok: + return st in xrange(200, 400) + else: + return st in xrange(200, 300) def readurl(url, data=None, timeout=None, @@ -44,8 +48,13 @@ def readurl(url, data=None, timeout=None, req_args['url'] = url if data is not None: req_args['data'] = urllib.urlencode(data) - if headers is not None: - req_args['headers'] = dict(headers) + + if not headers: + headers = { + 'User-Agent': 'Cloud-Init/%s' % (version.version_string()), + } + + req_args['headers'] = headers req = urllib2.Request(**req_args) retries = max(retries, 0) @@ -70,19 +79,17 @@ def readurl(url, data=None, timeout=None, return (content, status) except urllib2.HTTPError as e: last_excp = e - LOG.exception("Failed at reading from %s.", url) except urllib2.URLError as e: # This can be a message string or # another exception instance # (socket.error for remote URLs, OSError for local URLs). - if (isinstance(e.reason, OSError) and + if (isinstance(e.reason, (OSError)) and e.reason.errno == errno.ENOENT): last_excp = e.reason else: last_excp = e - LOG.exception("Failed at reading from %s", url) if i + 1 < attempts: - LOG.info("Please wait %s seconds while we wait to try again", + LOG.debug("Please wait %s seconds while we wait to try again", sec_between) time.sleep(sec_between) -- cgit v1.2.3 From a456f4fdc26eba06ee98ffe2eeeafc6e15a39e88 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 18:26:34 -0700 Subject: Inclusion of more utility functions included: 1. Adjustments to using more selinux guards around directory creation, chmod... 2. Adding util functions to check if values are false or true (used internally and externally) 3. Move find_devs_with to util.py and allow it to serve multiple use cases 4. Add fork_cb which will fork a process and then call a certain callback (used right now by the resize nonblocking mode) 5. Move functions that performed time_rfc2822 time fetching and uptime fetching to here. 6. Allow the subp util function to act in shell mode 7. Increase logging usefulness in shellify function --- cloudinit/util.py | 280 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 220 insertions(+), 60 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 7d5932c1..7259d933 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -28,14 +28,18 @@ import errno import glob import grp import gzip +import hashlib import os import platform import pwd +import random import shutil import socket +import string import subprocess import sys import tempfile +import time import traceback import types import urlparse @@ -68,8 +72,11 @@ CONTAINER_TESTS = ['running-in-container', 'lxc-is-container'] class ProcessExecutionError(IOError): - MESSAGE_TMPL = ('%(description)s\nCommand: %(cmd)s\n' - 'Exit code: %(exit_code)s\nStdout: %(stdout)r\n' + MESSAGE_TMPL = ('%(description)s\n' + 'Command: %(cmd)s\n' + 'Exit code: %(exit_code)s\n' + 'Reason: %(reason)s\n' + 'Stdout: %(stdout)r\n' 'Stderr: %(stderr)r') def __init__(self, stdout=None, stderr=None, @@ -100,31 +107,37 @@ class ProcessExecutionError(IOError): else: self.stdout = stdout + if reason: + self.reason = reason + else: + self.reason = '-' + message = self.MESSAGE_TMPL % { 'description': self.description, 'cmd': self.cmd, 'exit_code': self.exit_code, 'stdout': self.stdout, 'stderr': self.stderr, + 'reason': self.reason, } IOError.__init__(self, message) - self.reason = reason class SeLinuxGuard(object): def __init__(self, path, recursive=False): self.path = path self.recursive = recursive - self.engaged = False + self.enabled = False if HAVE_LIBSELINUX and selinux.is_selinux_enabled(): - self.engaged = True + self.enabled = True def __enter__(self): - return self.engaged + # TODO: Should we try to engage selinux here?? + return None def __exit__(self, excp_type, excp_value, excp_traceback): - if self.engaged: - LOG.debug("Disengaging selinux mode for %s: %s", + if self.enabled: + LOG.debug("Restoring selinux mode for %s (recursive=%s)", self.path, self.recursive) selinux.restorecon(self.path, recursive=self.recursive) @@ -133,14 +146,72 @@ class MountFailedError(Exception): pass -def translate_bool(val): +def SilentTemporaryFile(**kwargs): + fh = tempfile.NamedTemporaryFile(**kwargs) + # Replace its unlink with a quiet version + # that does not raise errors when the + # file to unlink has been unlinked elsewhere.. + LOG.debug("Created temporary file %s", fh.name) + fh.unlink = del_file + # Add a new method that will unlink + # right 'now' but still lets the exit + # method attempt to remove it (which will + # not throw due to our del file being quiet + # about files that are not there) + def unlink_now(): + fh.unlink(fh.name) + setattr(fh, 'unlink_now', unlink_now) + return fh + + +def fork_cb(child_cb, *args): + fid = os.fork() + if fid == 0: + try: + child_cb(*args) + os._exit(0) # pylint: disable=W0212 + except: + logexc(LOG, "Failed forking and calling callback %s", obj_name(child_cb)) + os._exit(1) # pylint: disable=W0212 + else: + LOG.debug("Forked child %s who will run callback %s", + fid, obj_name(child_cb)) + + +def is_true_str(val, addons=None): + check_set = ['true', '1', 'on', 'yes'] + if addons: + check_set = check_set + addons + if str(val).lower().strip() in check_set: + return True + return False + + +def is_false_str(val, addons=None): + check_set = ['off', '0', 'no', 'false'] + if addons: + check_set = check_set + addons + if str(val).lower().strip() in check_set: + return True + return False + + +def translate_bool(val, addons=None): if not val: + # This handles empty lists and false and + # other things that python believes are false return False - if val is isinstance(val, bool): + # If its already a boolean skip + if isinstance(val, (bool)): return val - if str(val).lower().strip() in ['true', '1', 'on', 'yes']: - return True - return False + return is_true_str(val, addons) + + +def rand_str(strlen=32, select_from=None): + if not select_from: + select_from = string.letters + string.digits + return "".join([random.choice(select_from) for _x in range(0, strlen)]) + def read_conf(fname): @@ -221,7 +292,10 @@ def get_cfg_option_bool(yobj, key, default=False): def get_cfg_option_str(yobj, key, default=None): if key not in yobj: return default - return yobj[key] + val = yobj[key] + if not isinstance(val, (str, basestring)): + val = str(val) + return val def system_info(): @@ -233,7 +307,7 @@ def system_info(): } -def get_cfg_option_list_or_str(yobj, key, default=None): +def get_cfg_option_list(yobj, key, default=None): """ Gets the C{key} config option from C{yobj} as a list of strings. If the key is present as a single string it will be returned as a list with one @@ -249,9 +323,14 @@ def get_cfg_option_list_or_str(yobj, key, default=None): return default if yobj[key] is None: return [] - if isinstance(yobj[key], (list)): - return yobj[key] - return [yobj[key]] + val = yobj[key] + if isinstance(val, (list)): + # Should we ensure they are all strings?? + cval = [str(v) for v in val] + return cval + if not isinstance(val, (str, basestring)): + val = str(val) + return [val] # get a cfg entry by its path array @@ -419,21 +498,21 @@ def runparts(dirp, skip_no_exist=True): if skip_no_exist and not os.path.isdir(dirp): return - failed = 0 - attempted = 0 + failed = [] + attempted = [] for exe_name in sorted(os.listdir(dirp)): exe_path = os.path.join(dirp, exe_name) if os.path.isfile(exe_path) and os.access(exe_path, os.X_OK): - attempted += 1 + attempted.append(exe_path) try: subp([exe_path]) except ProcessExecutionError as e: logexc(LOG, "Failed running %s [%s]", exe_path, e.exit_code) - failed += 1 + failed.append(e) if failed and attempted: raise RuntimeError('Runparts: %s failures in %s attempted commands' - % (failed, attempted)) + % (len(failed), len(attempted))) # read_optional_seed @@ -470,7 +549,7 @@ def load_yaml(blob, default=None, allowed=(dict,)): converted = yaml.load(blob) if not isinstance(converted, allowed): # Yes this will just be caught, but thats ok for now... - raise TypeError("Yaml load allows %s types, but got %s instead" % + raise TypeError("Yaml load allows %s root types, but got %s instead" % (allowed, obj_name(converted))) loaded = converted except (yaml.YAMLError, TypeError, ValueError) as exc: @@ -718,7 +797,8 @@ def close_stdin(): os.dup2(fp.fileno(), sys.stdin.fileno()) -def find_devs_with(criteria=None): +def find_devs_with(criteria=None, oformat='device', + tag=None, no_cache=False, path=None): """ find devices matching given criteria (via blkid) criteria can be *one* of: @@ -726,38 +806,58 @@ def find_devs_with(criteria=None): LABEL=