diff options
Diffstat (limited to 'cloudinit/cmd')
| -rw-r--r-- | cloudinit/cmd/devel/__init__.py | 0 | ||||
| -rw-r--r-- | cloudinit/cmd/devel/logs.py | 101 | ||||
| -rw-r--r-- | cloudinit/cmd/devel/parser.py | 26 | ||||
| -rw-r--r-- | cloudinit/cmd/devel/tests/__init__.py | 0 | ||||
| -rw-r--r-- | cloudinit/cmd/devel/tests/test_logs.py | 120 | ||||
| -rw-r--r-- | cloudinit/cmd/main.py | 69 |
6 files changed, 282 insertions, 34 deletions
diff --git a/cloudinit/cmd/devel/__init__.py b/cloudinit/cmd/devel/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/cloudinit/cmd/devel/__init__.py diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py new file mode 100644 index 00000000..35ca478f --- /dev/null +++ b/cloudinit/cmd/devel/logs.py @@ -0,0 +1,101 @@ +# Copyright (C) 2017 Canonical Ltd. +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""Define 'collect-logs' utility and handler to include in cloud-init cmd.""" + +import argparse +from cloudinit.util import ( + ProcessExecutionError, chdir, copy, ensure_dir, subp, write_file) +from cloudinit.temp_utils import tempdir +from datetime import datetime +import os +import shutil + + +CLOUDINIT_LOGS = ['/var/log/cloud-init.log', '/var/log/cloud-init-output.log'] +CLOUDINIT_RUN_DIR = '/run/cloud-init' +USER_DATA_FILE = '/var/lib/cloud/instance/user-data.txt' # Optional + + +def get_parser(parser=None): + """Build or extend and arg parser for collect-logs utility. + + @param parser: Optional existing ArgumentParser instance representing the + collect-logs subcommand which will be extended to support the args of + this utility. + + @returns: ArgumentParser with proper argument configuration. + """ + if not parser: + parser = argparse.ArgumentParser( + prog='collect-logs', + description='Collect and tar all cloud-init debug info') + parser.add_argument( + "--tarfile", '-t', default='cloud-init.tar.gz', + help=('The tarfile to create containing all collected logs.' + ' Default: cloud-init.tar.gz')) + parser.add_argument( + "--include-userdata", '-u', default=False, action='store_true', + dest='userdata', help=( + 'Optionally include user-data from {0} which could contain' + ' sensitive information.'.format(USER_DATA_FILE))) + return parser + + +def _write_command_output_to_file(cmd, filename): + """Helper which runs a command and writes output or error to filename.""" + try: + out, _ = subp(cmd) + except ProcessExecutionError as e: + write_file(filename, str(e)) + else: + write_file(filename, out) + + +def collect_logs(tarfile, include_userdata): + """Collect all cloud-init logs and tar them up into the provided tarfile. + + @param tarfile: The path of the tar-gzipped file to create. + @param include_userdata: Boolean, true means include user-data. + """ + tarfile = os.path.abspath(tarfile) + date = datetime.utcnow().date().strftime('%Y-%m-%d') + log_dir = 'cloud-init-logs-{0}'.format(date) + with tempdir(dir='/tmp') as tmp_dir: + log_dir = os.path.join(tmp_dir, log_dir) + _write_command_output_to_file( + ['dpkg-query', '--show', "-f=${Version}\n", 'cloud-init'], + os.path.join(log_dir, 'version')) + _write_command_output_to_file( + ['dmesg'], os.path.join(log_dir, 'dmesg.txt')) + _write_command_output_to_file( + ['journalctl', '-o', 'short-precise'], + os.path.join(log_dir, 'journal.txt')) + for log in CLOUDINIT_LOGS: + copy(log, log_dir) + if include_userdata: + copy(USER_DATA_FILE, log_dir) + run_dir = os.path.join(log_dir, 'run') + ensure_dir(run_dir) + shutil.copytree(CLOUDINIT_RUN_DIR, os.path.join(run_dir, 'cloud-init')) + with chdir(tmp_dir): + subp(['tar', 'czvf', tarfile, log_dir.replace(tmp_dir + '/', '')]) + + +def handle_collect_logs_args(name, args): + """Handle calls to 'cloud-init collect-logs' as a subcommand.""" + collect_logs(args.tarfile, args.userdata) + + +def main(): + """Tool to collect and tar all cloud-init related logs.""" + parser = get_parser() + handle_collect_logs_args('collect-logs', parser.parse_args()) + return 0 + + +if __name__ == '__main__': + main() + +# vi: ts=4 expandtab diff --git a/cloudinit/cmd/devel/parser.py b/cloudinit/cmd/devel/parser.py new file mode 100644 index 00000000..acacc4ed --- /dev/null +++ b/cloudinit/cmd/devel/parser.py @@ -0,0 +1,26 @@ +# Copyright (C) 2017 Canonical Ltd. +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""Define 'devel' subcommand argument parsers to include in cloud-init cmd.""" + +import argparse +from cloudinit.config.schema import ( + get_parser as schema_parser, handle_schema_args) + + +def get_parser(parser=None): + if not parser: + parser = argparse.ArgumentParser( + prog='cloudinit-devel', + description='Run development cloud-init tools') + subparsers = parser.add_subparsers(title='Subcommands', dest='subcommand') + subparsers.required = True + + parser_schema = subparsers.add_parser( + 'schema', help='Validate cloud-config files or document schema') + # Construct schema subcommand parser + schema_parser(parser_schema) + parser_schema.set_defaults(action=('schema', handle_schema_args)) + + return parser diff --git a/cloudinit/cmd/devel/tests/__init__.py b/cloudinit/cmd/devel/tests/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/cloudinit/cmd/devel/tests/__init__.py diff --git a/cloudinit/cmd/devel/tests/test_logs.py b/cloudinit/cmd/devel/tests/test_logs.py new file mode 100644 index 00000000..dc4947cc --- /dev/null +++ b/cloudinit/cmd/devel/tests/test_logs.py @@ -0,0 +1,120 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.cmd.devel import logs +from cloudinit.util import ensure_dir, load_file, subp, write_file +from cloudinit.tests.helpers import FilesystemMockingTestCase, wrap_and_call +from datetime import datetime +import os + + +class TestCollectLogs(FilesystemMockingTestCase): + + def setUp(self): + super(TestCollectLogs, self).setUp() + self.new_root = self.tmp_dir() + self.run_dir = self.tmp_path('run', self.new_root) + + def test_collect_logs_creates_tarfile(self): + """collect-logs creates a tarfile with all related cloud-init info.""" + log1 = self.tmp_path('cloud-init.log', self.new_root) + write_file(log1, 'cloud-init-log') + log2 = self.tmp_path('cloud-init-output.log', self.new_root) + write_file(log2, 'cloud-init-output-log') + ensure_dir(self.run_dir) + write_file(self.tmp_path('results.json', self.run_dir), 'results') + output_tarfile = self.tmp_path('logs.tgz') + + date = datetime.utcnow().date().strftime('%Y-%m-%d') + date_logdir = 'cloud-init-logs-{0}'.format(date) + + expected_subp = { + ('dpkg-query', '--show', "-f=${Version}\n", 'cloud-init'): + '0.7fake\n', + ('dmesg',): 'dmesg-out\n', + ('journalctl', '-o', 'short-precise'): 'journal-out\n', + ('tar', 'czvf', output_tarfile, date_logdir): '' + } + + def fake_subp(cmd): + cmd_tuple = tuple(cmd) + if cmd_tuple not in expected_subp: + raise AssertionError( + 'Unexpected command provided to subp: {0}'.format(cmd)) + if cmd == ['tar', 'czvf', output_tarfile, date_logdir]: + subp(cmd) # Pass through tar cmd so we can check output + return expected_subp[cmd_tuple], '' + + wrap_and_call( + 'cloudinit.cmd.devel.logs', + {'subp': {'side_effect': fake_subp}, + 'CLOUDINIT_LOGS': {'new': [log1, log2]}, + 'CLOUDINIT_RUN_DIR': {'new': self.run_dir}}, + logs.collect_logs, output_tarfile, include_userdata=False) + # unpack the tarfile and check file contents + subp(['tar', 'zxvf', output_tarfile, '-C', self.new_root]) + out_logdir = self.tmp_path(date_logdir, self.new_root) + self.assertEqual( + '0.7fake\n', + load_file(os.path.join(out_logdir, 'version'))) + self.assertEqual( + 'cloud-init-log', + load_file(os.path.join(out_logdir, 'cloud-init.log'))) + self.assertEqual( + 'cloud-init-output-log', + load_file(os.path.join(out_logdir, 'cloud-init-output.log'))) + self.assertEqual( + 'dmesg-out\n', + load_file(os.path.join(out_logdir, 'dmesg.txt'))) + self.assertEqual( + 'journal-out\n', + load_file(os.path.join(out_logdir, 'journal.txt'))) + self.assertEqual( + 'results', + load_file( + os.path.join(out_logdir, 'run', 'cloud-init', 'results.json'))) + + def test_collect_logs_includes_optional_userdata(self): + """collect-logs include userdata when --include-userdata is set.""" + log1 = self.tmp_path('cloud-init.log', self.new_root) + write_file(log1, 'cloud-init-log') + log2 = self.tmp_path('cloud-init-output.log', self.new_root) + write_file(log2, 'cloud-init-output-log') + userdata = self.tmp_path('user-data.txt', self.new_root) + write_file(userdata, 'user-data') + ensure_dir(self.run_dir) + write_file(self.tmp_path('results.json', self.run_dir), 'results') + output_tarfile = self.tmp_path('logs.tgz') + + date = datetime.utcnow().date().strftime('%Y-%m-%d') + date_logdir = 'cloud-init-logs-{0}'.format(date) + + expected_subp = { + ('dpkg-query', '--show', "-f=${Version}\n", 'cloud-init'): + '0.7fake', + ('dmesg',): 'dmesg-out\n', + ('journalctl', '-o', 'short-precise'): 'journal-out\n', + ('tar', 'czvf', output_tarfile, date_logdir): '' + } + + def fake_subp(cmd): + cmd_tuple = tuple(cmd) + if cmd_tuple not in expected_subp: + raise AssertionError( + 'Unexpected command provided to subp: {0}'.format(cmd)) + if cmd == ['tar', 'czvf', output_tarfile, date_logdir]: + subp(cmd) # Pass through tar cmd so we can check output + return expected_subp[cmd_tuple], '' + + wrap_and_call( + 'cloudinit.cmd.devel.logs', + {'subp': {'side_effect': fake_subp}, + 'CLOUDINIT_LOGS': {'new': [log1, log2]}, + 'CLOUDINIT_RUN_DIR': {'new': self.run_dir}, + 'USER_DATA_FILE': {'new': userdata}}, + logs.collect_logs, output_tarfile, include_userdata=True) + # unpack the tarfile and check file contents + subp(['tar', 'zxvf', output_tarfile, '-C', self.new_root]) + out_logdir = self.tmp_path(date_logdir, self.new_root) + self.assertEqual( + 'user-data', + load_file(os.path.join(out_logdir, 'user-data.txt'))) diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 139e03b3..6fb9d9e7 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -50,13 +50,6 @@ WELCOME_MSG_TPL = ("Cloud-init v. {version} running '{action}' at " # Module section template MOD_SECTION_TPL = "cloud_%s_modules" -# Things u can query on -QUERY_DATA_TYPES = [ - 'data', - 'data_raw', - 'instance_id', -] - # Frequency shortname to full name # (so users don't have to remember the full name...) FREQ_SHORT_NAMES = { @@ -510,11 +503,6 @@ def main_modules(action_name, args): return run_module_section(mods, name, name) -def main_query(name, _args): - raise NotImplementedError(("Action '%s' is not" - " currently implemented") % (name)) - - def main_single(name, args): # Cloud-init single stage is broken up into the following sub-stages # 1. Ensure that the init object fetches its config without errors @@ -688,11 +676,10 @@ def main_features(name, args): def main(sysv_args=None): - if sysv_args is not None: - parser = argparse.ArgumentParser(prog=sysv_args[0]) - sysv_args = sysv_args[1:] - else: - parser = argparse.ArgumentParser() + if not sysv_args: + sysv_args = sys.argv + parser = argparse.ArgumentParser(prog=sysv_args[0]) + sysv_args = sysv_args[1:] # Top level args parser.add_argument('--version', '-v', action='version', @@ -713,7 +700,8 @@ def main(sysv_args=None): default=False) parser.set_defaults(reporter=None) - subparsers = parser.add_subparsers() + subparsers = parser.add_subparsers(title='Subcommands', dest='subcommand') + subparsers.required = True # Each action and its sub-options (if any) parser_init = subparsers.add_parser('init', @@ -737,17 +725,6 @@ def main(sysv_args=None): choices=('init', 'config', 'final')) parser_mod.set_defaults(action=('modules', main_modules)) - # These settings are used when you want to query information - # stored in the cloud-init data objects/directories/files - parser_query = subparsers.add_parser('query', - help=('query information stored ' - 'in cloud-init')) - parser_query.add_argument("--name", '-n', action="store", - help="item name to query on", - required=True, - choices=QUERY_DATA_TYPES) - parser_query.set_defaults(action=('query', main_query)) - # This subcommand allows you to run a single module parser_single = subparsers.add_parser('single', help=('run a single module ')) @@ -781,15 +758,39 @@ def main(sysv_args=None): help=('list defined features')) parser_features.set_defaults(action=('features', main_features)) + parser_analyze = subparsers.add_parser( + 'analyze', help='Devel tool: Analyze cloud-init logs and data') + + parser_devel = subparsers.add_parser( + 'devel', help='Run development tools') + + parser_collect_logs = subparsers.add_parser( + 'collect-logs', help='Collect and tar all cloud-init debug info') + + if sysv_args: + # Only load subparsers if subcommand is specified to avoid load cost + if sysv_args[0] == 'analyze': + from cloudinit.analyze.__main__ import get_parser as analyze_parser + # Construct analyze subcommand parser + analyze_parser(parser_analyze) + elif sysv_args[0] == 'devel': + from cloudinit.cmd.devel.parser import get_parser as devel_parser + # Construct devel subcommand parser + devel_parser(parser_devel) + elif sysv_args[0] == 'collect-logs': + from cloudinit.cmd.devel.logs import ( + get_parser as logs_parser, handle_collect_logs_args) + logs_parser(parser_collect_logs) + parser_collect_logs.set_defaults( + action=('collect-logs', handle_collect_logs_args)) + args = parser.parse_args(args=sysv_args) - try: - (name, functor) = args.action - except AttributeError: - parser.error('too few arguments') + # Subparsers.required = True and each subparser sets action=(name, functor) + (name, functor) = args.action # Setup basic logging to start (until reinitialized) - # iff in debug mode... + # iff in debug mode. if args.debug: logging.setupBasicLogging() |