From 47d15924ea38f249a2c54d33dd24c3c284c4eb72 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 9 Oct 2013 12:22:06 -0700 Subject: Log message around import failure In certain cases import failure is expected and in certain cases it is not expected, in either case it is useful to at least log the failure. --- cloudinit/importer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/importer.py b/cloudinit/importer.py index 71cf2726..a094141a 100644 --- a/cloudinit/importer.py +++ b/cloudinit/importer.py @@ -36,6 +36,7 @@ def find_module(base_name, search_paths, required_attrs=None): found_places = [] if not required_attrs: required_attrs = [] + # NOTE(harlowja): translate the search paths to include the base name. real_paths = [] for path in search_paths: real_path = [] @@ -50,8 +51,9 @@ def find_module(base_name, search_paths, required_attrs=None): mod = None try: mod = import_module(full_path) - except ImportError: - pass + except ImportError as e: + LOG.debug("Failed at attempted import of '%s' due to: %s", + full_path, e) if not mod: continue found_attrs = 0 -- cgit v1.2.3 From 9c762cc3fa13782397a15bd3c68e9c62a3cba689 Mon Sep 17 00:00:00 2001 From: Shraddha Pandhe Date: Fri, 6 Dec 2013 11:16:17 -0800 Subject: This is a new debug module thats supposed to print out some information about the instance being created. The module can be included at any stage of the process - init/config/final LP: #1258619 --- cloudinit/config/cc_debug.py | 82 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 cloudinit/config/cc_debug.py (limited to 'cloudinit') diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py new file mode 100644 index 00000000..86c61d68 --- /dev/null +++ b/cloudinit/config/cc_debug.py @@ -0,0 +1,82 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2013 Yahoo! Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from StringIO import StringIO + +from cloudinit import util + +import copy + +import yaml + + +def _format_yaml(obj): + try: + formatted = yaml.safe_dump(obj, + line_break="\n", + indent=4, + explicit_start=True, + explicit_end=True, + default_flow_style=False) + return formatted.strip() + except: + return "???" + + +def _make_header(text): + header = StringIO() + header.write("-" * 80) + header.write("\n") + header.write(text.center(80, ' ')) + header.write("\n") + header.write("-" * 80) + header.write("\n") + return header.getvalue() + + +def handle(name, cfg, cloud, log, _args): + verbose = util.get_cfg_option_bool(cfg, 'verbose', default=True) + if not verbose: + log.debug(("Skipping module named %s," + " verbose printing disabled"), name) + return + # Clean out some keys that we just don't care about showing... + dump_cfg = copy.deepcopy(cfg) + for k in ['log_cfgs']: + dump_cfg.pop(k, None) + all_keys = list(dump_cfg.keys()) + for k in all_keys: + if k.startswith("_"): + dump_cfg.pop(k, None) + # Now dump it... + to_print = StringIO() + to_print.write(_make_header("Config")) + to_print.write(_format_yaml(dump_cfg)) + to_print.write("\n") + to_print.write(_make_header("MetaData")) + to_print.write(_format_yaml(cloud.datasource.metadata)) + to_print.write("\n") + to_print.write(_make_header("Misc")) + to_print.write("Datasource: %s\n" % (util.obj_name(cloud.datasource))) + to_print.write("Distro: %s\n" % (util.obj_name(cloud.distro))) + to_print.write("Hostname: %s\n" % (cloud.get_hostname(True))) + to_print.write("Instance ID: %s\n" % (cloud.get_instance_id())) + to_print.write("Locale: %s\n" % (cloud.get_locale())) + to_print.write("Launch IDX: %s\n" % (cloud.launch_index)) + contents = to_print.getvalue() + for line in contents.splitlines(): + line = "ci-info: %s\n" % (line) + util.multi_log(line, console=True, stderr=False) -- cgit v1.2.3 From c0d263968d93e44fbaaa98d284690ac93a6ce024 Mon Sep 17 00:00:00 2001 From: Shraddha Pandhe Date: Wed, 11 Dec 2013 14:21:18 -0800 Subject: bug: 1258619 added namespace for config options, output file, commandline argument for output file --- cloudinit/config/cc_debug.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 86c61d68..85dc5c58 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -15,11 +15,9 @@ # along with this program. If not, see . from StringIO import StringIO - from cloudinit import util - +from cloudinit import type_utils import copy - import yaml @@ -47,12 +45,17 @@ def _make_header(text): return header.getvalue() -def handle(name, cfg, cloud, log, _args): - verbose = util.get_cfg_option_bool(cfg, 'verbose', default=True) +def handle(name, cfg, cloud, log, args): + verbose = util.get_cfg_by_path(cfg, ('debug','verbose'), default=True) if not verbose: log.debug(("Skipping module named %s," " verbose printing disabled"), name) return + out_file = None + if args: + out_file = args[0] + else: + out_file = util.get_cfg_by_path(cfg, ('debug','output')) # Clean out some keys that we just don't care about showing... dump_cfg = copy.deepcopy(cfg) for k in ['log_cfgs']: @@ -70,8 +73,8 @@ def handle(name, cfg, cloud, log, _args): to_print.write(_format_yaml(cloud.datasource.metadata)) to_print.write("\n") to_print.write(_make_header("Misc")) - to_print.write("Datasource: %s\n" % (util.obj_name(cloud.datasource))) - to_print.write("Distro: %s\n" % (util.obj_name(cloud.distro))) + to_print.write("Datasource: %s\n" % (type_utils.obj_name(cloud.datasource))) + to_print.write("Distro: %s\n" % (type_utils.obj_name(cloud.distro))) to_print.write("Hostname: %s\n" % (cloud.get_hostname(True))) to_print.write("Instance ID: %s\n" % (cloud.get_instance_id())) to_print.write("Locale: %s\n" % (cloud.get_locale())) @@ -79,4 +82,7 @@ def handle(name, cfg, cloud, log, _args): contents = to_print.getvalue() for line in contents.splitlines(): line = "ci-info: %s\n" % (line) - util.multi_log(line, console=True, stderr=False) + if out_file: + util.write_file(out_file, line, 0644, "a") + else: + util.multi_log(line, console=True, stderr=False) -- cgit v1.2.3 From 7c6f2de4f7f66874d6c9131c04cb84637955e5ce Mon Sep 17 00:00:00 2001 From: Shraddha Pandhe Date: Wed, 11 Dec 2013 15:41:14 -0800 Subject: essage --- cloudinit/config/cc_debug.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 85dc5c58..a3d99da8 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -67,10 +67,10 @@ def handle(name, cfg, cloud, log, args): # Now dump it... to_print = StringIO() to_print.write(_make_header("Config")) - to_print.write(_format_yaml(dump_cfg)) + to_print.write(util.yaml_dumps(dump_cfg)) to_print.write("\n") to_print.write(_make_header("MetaData")) - to_print.write(_format_yaml(cloud.datasource.metadata)) + to_print.write(util.yaml_dumps(cloud.datasource.metadata)) to_print.write("\n") to_print.write(_make_header("Misc")) to_print.write("Datasource: %s\n" % (type_utils.obj_name(cloud.datasource))) @@ -80,9 +80,11 @@ def handle(name, cfg, cloud, log, args): to_print.write("Locale: %s\n" % (cloud.get_locale())) to_print.write("Launch IDX: %s\n" % (cloud.launch_index)) contents = to_print.getvalue() + content_to_file = [] for line in contents.splitlines(): line = "ci-info: %s\n" % (line) - if out_file: - util.write_file(out_file, line, 0644, "a") - else: - util.multi_log(line, console=True, stderr=False) + content_to_file.append(line) + if out_file: + util.write_file(out_file, "".join(content_to_file), 0644, "w") + else: + util.multi_log("".join(content_to_file), console=True, stderr=False) -- cgit v1.2.3 From bce8220f1688af1e155661bfd57e73fe23c42522 Mon Sep 17 00:00:00 2001 From: Shraddha Pandhe Date: Wed, 11 Dec 2013 15:51:40 -0800 Subject: Removed method _format_yaml --- cloudinit/config/cc_debug.py | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index a3d99da8..971af71b 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -21,19 +21,6 @@ import copy import yaml -def _format_yaml(obj): - try: - formatted = yaml.safe_dump(obj, - line_break="\n", - indent=4, - explicit_start=True, - explicit_end=True, - default_flow_style=False) - return formatted.strip() - except: - return "???" - - def _make_header(text): header = StringIO() header.write("-" * 80) -- cgit v1.2.3 From 67ede943d3a02878061be2314300644c636b14f8 Mon Sep 17 00:00:00 2001 From: Shraddha Pandhe Date: Wed, 11 Dec 2013 15:56:08 -0800 Subject: Removed yaml import --- cloudinit/config/cc_debug.py | 1 - 1 file changed, 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 971af71b..c0a447cb 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -18,7 +18,6 @@ from StringIO import StringIO from cloudinit import util from cloudinit import type_utils import copy -import yaml def _make_header(text): -- cgit v1.2.3 From 5414797f1b9286ddbe82c8637dfff74cf352b967 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 12 Dec 2013 11:47:53 -0500 Subject: fix pep8 and pylint warnings This fixes warnings raised by: ./tools/run-pep8 cloudinit/config/cc_debug.py ./tools/run-pylint cloudinit/config/cc_debug.py --- cloudinit/config/cc_debug.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index c0a447cb..55fdf2dd 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -32,7 +32,7 @@ def _make_header(text): def handle(name, cfg, cloud, log, args): - verbose = util.get_cfg_by_path(cfg, ('debug','verbose'), default=True) + verbose = util.get_cfg_by_path(cfg, ('debug', 'verbose'), default=True) if not verbose: log.debug(("Skipping module named %s," " verbose printing disabled"), name) @@ -41,7 +41,7 @@ def handle(name, cfg, cloud, log, args): if args: out_file = args[0] else: - out_file = util.get_cfg_by_path(cfg, ('debug','output')) + out_file = util.get_cfg_by_path(cfg, ('debug', 'output')) # Clean out some keys that we just don't care about showing... dump_cfg = copy.deepcopy(cfg) for k in ['log_cfgs']: @@ -59,7 +59,8 @@ def handle(name, cfg, cloud, log, args): to_print.write(util.yaml_dumps(cloud.datasource.metadata)) to_print.write("\n") to_print.write(_make_header("Misc")) - to_print.write("Datasource: %s\n" % (type_utils.obj_name(cloud.datasource))) + to_print.write("Datasource: %s\n" % + (type_utils.obj_name(cloud.datasource))) to_print.write("Distro: %s\n" % (type_utils.obj_name(cloud.distro))) to_print.write("Hostname: %s\n" % (cloud.get_hostname(True))) to_print.write("Instance ID: %s\n" % (cloud.get_instance_id())) -- cgit v1.2.3 From fd5231ae771cd3b87c26ac2b0839fb672bf0acee Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 12 Dec 2013 11:50:55 -0500 Subject: be verbose explicitly if run from cmdline. Let the command line (or module args) that set outfile explicitly override a config'd value of 'verbose'. Ie, if /etc/cloud/cloud.cfg.d/my.cfg had: debug: verbose: False but the user ran: cloud-init single --frequency=always --name=debug output.txt Then they probably wanted to have the debug in output.txt even though verbose was configured to False. --- cloudinit/config/cc_debug.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 55fdf2dd..cfd31fa1 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -33,15 +33,17 @@ def _make_header(text): def handle(name, cfg, cloud, log, args): verbose = util.get_cfg_by_path(cfg, ('debug', 'verbose'), default=True) - if not verbose: - log.debug(("Skipping module named %s," - " verbose printing disabled"), name) - return - out_file = None if args: + # if args are provided (from cmdline) then explicitly set verbose out_file = args[0] + verbose = True else: out_file = util.get_cfg_by_path(cfg, ('debug', 'output')) + + if not verbose: + log.debug(("Skipping module named %s," + " verbose printing disabled"), name) + return # Clean out some keys that we just don't care about showing... dump_cfg = copy.deepcopy(cfg) for k in ['log_cfgs']: -- cgit v1.2.3 From 66ed882dc3cf24395386869ea3c1d54e8f22c38b Mon Sep 17 00:00:00 2001 From: James Slagle Date: Fri, 13 Dec 2013 10:40:51 -0500 Subject: Cast file path to string. Before passing a path into selinux.matchpathcon, it needs to be casted to a string, since the path could be unicode and selinux.matchpathcon does not support unicode. Closes-Bug: #1260072 LP: #1260072 --- cloudinit/util.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/util.py b/cloudinit/util.py index a8ddb390..a37172dc 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -170,6 +170,8 @@ class SeLinuxGuard(object): def __exit__(self, excp_type, excp_value, excp_traceback): if self.selinux and self.selinux.is_selinux_enabled(): path = os.path.realpath(os.path.expanduser(self.path)) + # path should be a string, not unicode + path = str(path) do_restore = False try: # See if even worth restoring?? -- cgit v1.2.3 From 923a7d0efc3cf3224b891dc783321018d59ce33e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sat, 14 Dec 2013 14:17:31 -0500 Subject: support calling apt with eatmydata, enable by default if available. This allows a general config option to prefix apt-get commands via 'apt_get_wrapper'. By default, the command is set to 'eatmydata', and the mode set to 'auto'. That means if eatmydata is available (via which), it will use it. The 'command' can be either a array or a string. LP: #1236531 --- ChangeLog | 2 ++ cloudinit/distros/debian.py | 28 ++++++++++++++++++++++++++-- doc/examples/cloud-config.txt | 9 +++++++-- 3 files changed, 35 insertions(+), 4 deletions(-) (limited to 'cloudinit') diff --git a/ChangeLog b/ChangeLog index 1f4ca198..1b7948f7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -3,6 +3,8 @@ - Add a debug log message around import failures - add a 'debug' module for easily printing out some information about datasource and cloud-init [Shraddha Pandhe] + - support running apt with 'eatmydata' via configuration token + apt_get_wrapper (LP: #1236531). 0.7.4: - fix issue mounting 'ephemeral0' if ephemeral0 was an alias for a partitioned block device with target filesystem on ephemeral0.1. diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 8fe49cbe..1ae232fd 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -36,6 +36,10 @@ LOG = logging.getLogger(__name__) APT_GET_COMMAND = ('apt-get', '--option=Dpkg::Options::=--force-confold', '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet') +APT_GET_WRAPPER = { + 'command': 'eatmydata', + 'enabled': 'auto', +} class Distro(distros.Distro): @@ -148,7 +152,13 @@ class Distro(distros.Distro): # See: http://tiny.cc/kg91fw # Or: http://tiny.cc/mh91fw e['DEBIAN_FRONTEND'] = 'noninteractive' - cmd = list(self.get_option("apt_get_command", APT_GET_COMMAND)) + + wcfg = self.get_option("apt_get_wrapper", APT_GET_WRAPPER) + cmd = _get_wrapper_prefix( + wcfg.get('command', APT_GET_WRAPPER['command']), + wcfg.get('enabled', APT_GET_WRAPPER['enabled'])) + + cmd.extend(list(self.get_option("apt_get_command", APT_GET_COMMAND))) if args and isinstance(args, str): cmd.append(args) @@ -166,7 +176,9 @@ class Distro(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, env=e, capture=False) + util.log_time(logfunc=LOG.debug, + msg="apt-%s [%s]" % (command, ' '.join(cmd)), func=util.subp, + args=(cmd,), kwargs={'env': e, 'capture': False}) def update_package_sources(self): self._runner.run("update-sources", self.package_command, @@ -175,3 +187,15 @@ class Distro(distros.Distro): def get_primary_arch(self): (arch, _err) = util.subp(['dpkg', '--print-architecture']) return str(arch).strip() + + +def _get_wrapper_prefix(cmd, mode): + if isinstance(cmd, str): + cmd = [str(cmd)] + + if (util.is_true(mode) or + (str(mode).lower() == "auto" and cmd[0] and + util.which(cmd[0]))): + return cmd + else: + return [] diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 8d756c61..61fa6065 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -147,8 +147,13 @@ apt_sources: # '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet'] # # apt_get_upgrade_subcommand: -# Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'. -# This is the subcommand that is invoked if package_upgrade is set to true above. +# Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'. +# This is the subcommand that is invoked if package_upgrade is set to true above. +# +# apt_get_wrapper: +# command: eatmydata +# enabled: [True, False, "auto"] +# # Install additional packages on first boot # -- cgit v1.2.3