From fa266bf8818a08e37cd32a603d076ba2db300124 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 31 Aug 2017 20:01:57 -0600 Subject: upstart: do not package upstart jobs, drop ubuntu-init-switch module. The ubuntu-init-switch module allowed the use to launch an instance that was booted with upstart and have it switch its init system to systemd and then reboot itself. It was only useful for the time period when Ubuntu was transitioning to systemd but only produced images using upstart. Also, do not run setup with --init-system=upstart. This means that by default, debian packages built with packages/bddeb will not have upstart unit files included. No other removal is done here. --- packages/debian/rules.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/debian/rules.in') diff --git a/packages/debian/rules.in b/packages/debian/rules.in index 053b7649..b87a5e84 100755 --- a/packages/debian/rules.in +++ b/packages/debian/rules.in @@ -1,6 +1,6 @@ ## template:basic #!/usr/bin/make -f -INIT_SYSTEM ?= upstart,systemd +INIT_SYSTEM ?= systemd export PYBUILD_INSTALL_ARGS=--init-system=$(INIT_SYSTEM) PYVER ?= python${pyver} -- cgit v1.2.3 From e626966ee7d339b53d2c8b14a8f2ff8e3fe892ee Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 12 Sep 2017 10:27:07 -0600 Subject: cmdline: add collect-logs subcommand. Add a new collect-logs sub command to the cloud-init CLI. This script will collect all logs pertinent to a cloud-init run and store them in a compressed tar-gzipped file. This tarfile can be attached to any cloud-init bug filed in order to aid in bug triage and resolution. A cloudinit.apport module is also added that allows apport interaction. Here is an example bug filed via ubuntu-bug cloud-init: LP: #1716975. Once the apport launcher is packaged in cloud-init, bugs can be filed against cloud-init with the following command: ubuntu-bug cloud-init LP: #1607345 --- cloudinit/apport.py | 105 +++++++++++++++++++++++++++++ cloudinit/cmd/devel/logs.py | 101 +++++++++++++++++++++++++++ cloudinit/cmd/devel/tests/__init__.py | 0 cloudinit/cmd/devel/tests/test_logs.py | 120 +++++++++++++++++++++++++++++++++ cloudinit/cmd/main.py | 11 ++- packages/debian/rules.in | 1 + tests/unittests/test_cli.py | 22 ++++-- 7 files changed, 354 insertions(+), 6 deletions(-) create mode 100644 cloudinit/apport.py create mode 100644 cloudinit/cmd/devel/logs.py create mode 100644 cloudinit/cmd/devel/tests/__init__.py create mode 100644 cloudinit/cmd/devel/tests/test_logs.py (limited to 'packages/debian/rules.in') diff --git a/cloudinit/apport.py b/cloudinit/apport.py new file mode 100644 index 00000000..221f341c --- /dev/null +++ b/cloudinit/apport.py @@ -0,0 +1,105 @@ +# Copyright (C) 2017 Canonical Ltd. +# +# This file is part of cloud-init. See LICENSE file for license information. + +'''Cloud-init apport interface''' + +try: + from apport.hookutils import ( + attach_file, attach_root_command_outputs, root_command_output) + has_apport = True +except ImportError: + has_apport = False + + +KNOWN_CLOUD_NAMES = [ + 'Amazon - Ec2', 'AliYun', 'AltCloud', 'Azure', 'Bigstep', 'CloudSigma', + 'CloudStack', 'DigitalOcean', 'GCE - Google Compute Engine', 'MAAS', + 'NoCloud', 'OpenNebula', 'OpenStack', 'OVF', 'Scaleway', 'SmartOS', + 'VMware', 'Other'] + +# Potentially clear text collected logs +CLOUDINIT_LOG = '/var/log/cloud-init.log' +CLOUDINIT_OUTPUT_LOG = '/var/log/cloud-init-output.log' +USER_DATA_FILE = '/var/lib/cloud/instance/user-data.txt' # Optional + + +def attach_cloud_init_logs(report, ui=None): + '''Attach cloud-init logs and tarfile from 'cloud-init collect-logs'.''' + attach_root_command_outputs(report, { + 'cloud-init-log-warnings': + 'egrep -i "warn|error" /var/log/cloud-init.log', + 'cloud-init-output.log.txt': 'cat /var/log/cloud-init-output.log'}) + root_command_output( + ['cloud-init', 'collect-logs', '-t', '/tmp/cloud-init-logs.tgz']) + attach_file(report, '/tmp/cloud-init-logs.tgz', 'logs.tgz') + + +def attach_hwinfo(report, ui=None): + '''Optionally attach hardware info from lshw.''' + prompt = ( + 'Your device details (lshw) may be useful to developers when' + ' addressing this bug, but gathering it requires admin privileges.' + ' Would you like to include this info?') + if ui and ui.yesno(prompt): + attach_root_command_outputs(report, {'lshw.txt': 'lshw'}) + + +def attach_cloud_info(report, ui=None): + '''Prompt for cloud details if available.''' + if ui: + prompt = 'Is this machine running in a cloud environment?' + response = ui.yesno(prompt) + if response is None: + raise StopIteration # User cancelled + if response: + prompt = ('Please select the cloud vendor or environment in which' + ' this instance is running') + response = ui.choice(prompt, KNOWN_CLOUD_NAMES) + if response: + report['CloudName'] = KNOWN_CLOUD_NAMES[response[0]] + else: + report['CloudName'] = 'None' + + +def attach_user_data(report, ui=None): + '''Optionally provide user-data if desired.''' + if ui: + prompt = ( + 'Your user-data or cloud-config file can optionally be provided' + ' from {0} and could be useful to developers when addressing this' + ' bug. Do you wish to attach user-data to this bug?'.format( + USER_DATA_FILE)) + response = ui.yesno(prompt) + if response is None: + raise StopIteration # User cancelled + if response: + attach_file(report, USER_DATA_FILE, 'user_data.txt') + + +def add_bug_tags(report): + '''Add any appropriate tags to the bug.''' + if 'JournalErrors' in report.keys(): + errors = report['JournalErrors'] + if 'Breaking ordering cycle' in errors: + report['Tags'] = 'systemd-ordering' + + +def add_info(report, ui): + '''This is an entry point to run cloud-init's apport functionality. + + Distros which want apport support will have a cloud-init package-hook at + /usr/share/apport/package-hooks/cloud-init.py which defines an add_info + function and returns the result of cloudinit.apport.add_info(report, ui). + ''' + if not has_apport: + raise RuntimeError( + 'No apport imports discovered. Apport functionality disabled') + attach_cloud_init_logs(report, ui) + attach_hwinfo(report, ui) + attach_cloud_info(report, ui) + attach_user_data(report, ui) + add_bug_tags(report) + return True + +# vi: ts=4 expandtab diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py new file mode 100644 index 00000000..35ca478f --- /dev/null +++ b/cloudinit/cmd/devel/logs.py @@ -0,0 +1,101 @@ +# Copyright (C) 2017 Canonical Ltd. +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""Define 'collect-logs' utility and handler to include in cloud-init cmd.""" + +import argparse +from cloudinit.util import ( + ProcessExecutionError, chdir, copy, ensure_dir, subp, write_file) +from cloudinit.temp_utils import tempdir +from datetime import datetime +import os +import shutil + + +CLOUDINIT_LOGS = ['/var/log/cloud-init.log', '/var/log/cloud-init-output.log'] +CLOUDINIT_RUN_DIR = '/run/cloud-init' +USER_DATA_FILE = '/var/lib/cloud/instance/user-data.txt' # Optional + + +def get_parser(parser=None): + """Build or extend and arg parser for collect-logs utility. + + @param parser: Optional existing ArgumentParser instance representing the + collect-logs subcommand which will be extended to support the args of + this utility. + + @returns: ArgumentParser with proper argument configuration. + """ + if not parser: + parser = argparse.ArgumentParser( + prog='collect-logs', + description='Collect and tar all cloud-init debug info') + parser.add_argument( + "--tarfile", '-t', default='cloud-init.tar.gz', + help=('The tarfile to create containing all collected logs.' + ' Default: cloud-init.tar.gz')) + parser.add_argument( + "--include-userdata", '-u', default=False, action='store_true', + dest='userdata', help=( + 'Optionally include user-data from {0} which could contain' + ' sensitive information.'.format(USER_DATA_FILE))) + return parser + + +def _write_command_output_to_file(cmd, filename): + """Helper which runs a command and writes output or error to filename.""" + try: + out, _ = subp(cmd) + except ProcessExecutionError as e: + write_file(filename, str(e)) + else: + write_file(filename, out) + + +def collect_logs(tarfile, include_userdata): + """Collect all cloud-init logs and tar them up into the provided tarfile. + + @param tarfile: The path of the tar-gzipped file to create. + @param include_userdata: Boolean, true means include user-data. + """ + tarfile = os.path.abspath(tarfile) + date = datetime.utcnow().date().strftime('%Y-%m-%d') + log_dir = 'cloud-init-logs-{0}'.format(date) + with tempdir(dir='/tmp') as tmp_dir: + log_dir = os.path.join(tmp_dir, log_dir) + _write_command_output_to_file( + ['dpkg-query', '--show', "-f=${Version}\n", 'cloud-init'], + os.path.join(log_dir, 'version')) + _write_command_output_to_file( + ['dmesg'], os.path.join(log_dir, 'dmesg.txt')) + _write_command_output_to_file( + ['journalctl', '-o', 'short-precise'], + os.path.join(log_dir, 'journal.txt')) + for log in CLOUDINIT_LOGS: + copy(log, log_dir) + if include_userdata: + copy(USER_DATA_FILE, log_dir) + run_dir = os.path.join(log_dir, 'run') + ensure_dir(run_dir) + shutil.copytree(CLOUDINIT_RUN_DIR, os.path.join(run_dir, 'cloud-init')) + with chdir(tmp_dir): + subp(['tar', 'czvf', tarfile, log_dir.replace(tmp_dir + '/', '')]) + + +def handle_collect_logs_args(name, args): + """Handle calls to 'cloud-init collect-logs' as a subcommand.""" + collect_logs(args.tarfile, args.userdata) + + +def main(): + """Tool to collect and tar all cloud-init related logs.""" + parser = get_parser() + handle_collect_logs_args('collect-logs', parser.parse_args()) + return 0 + + +if __name__ == '__main__': + main() + +# vi: ts=4 expandtab diff --git a/cloudinit/cmd/devel/tests/__init__.py b/cloudinit/cmd/devel/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloudinit/cmd/devel/tests/test_logs.py b/cloudinit/cmd/devel/tests/test_logs.py new file mode 100644 index 00000000..dc4947cc --- /dev/null +++ b/cloudinit/cmd/devel/tests/test_logs.py @@ -0,0 +1,120 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.cmd.devel import logs +from cloudinit.util import ensure_dir, load_file, subp, write_file +from cloudinit.tests.helpers import FilesystemMockingTestCase, wrap_and_call +from datetime import datetime +import os + + +class TestCollectLogs(FilesystemMockingTestCase): + + def setUp(self): + super(TestCollectLogs, self).setUp() + self.new_root = self.tmp_dir() + self.run_dir = self.tmp_path('run', self.new_root) + + def test_collect_logs_creates_tarfile(self): + """collect-logs creates a tarfile with all related cloud-init info.""" + log1 = self.tmp_path('cloud-init.log', self.new_root) + write_file(log1, 'cloud-init-log') + log2 = self.tmp_path('cloud-init-output.log', self.new_root) + write_file(log2, 'cloud-init-output-log') + ensure_dir(self.run_dir) + write_file(self.tmp_path('results.json', self.run_dir), 'results') + output_tarfile = self.tmp_path('logs.tgz') + + date = datetime.utcnow().date().strftime('%Y-%m-%d') + date_logdir = 'cloud-init-logs-{0}'.format(date) + + expected_subp = { + ('dpkg-query', '--show', "-f=${Version}\n", 'cloud-init'): + '0.7fake\n', + ('dmesg',): 'dmesg-out\n', + ('journalctl', '-o', 'short-precise'): 'journal-out\n', + ('tar', 'czvf', output_tarfile, date_logdir): '' + } + + def fake_subp(cmd): + cmd_tuple = tuple(cmd) + if cmd_tuple not in expected_subp: + raise AssertionError( + 'Unexpected command provided to subp: {0}'.format(cmd)) + if cmd == ['tar', 'czvf', output_tarfile, date_logdir]: + subp(cmd) # Pass through tar cmd so we can check output + return expected_subp[cmd_tuple], '' + + wrap_and_call( + 'cloudinit.cmd.devel.logs', + {'subp': {'side_effect': fake_subp}, + 'CLOUDINIT_LOGS': {'new': [log1, log2]}, + 'CLOUDINIT_RUN_DIR': {'new': self.run_dir}}, + logs.collect_logs, output_tarfile, include_userdata=False) + # unpack the tarfile and check file contents + subp(['tar', 'zxvf', output_tarfile, '-C', self.new_root]) + out_logdir = self.tmp_path(date_logdir, self.new_root) + self.assertEqual( + '0.7fake\n', + load_file(os.path.join(out_logdir, 'version'))) + self.assertEqual( + 'cloud-init-log', + load_file(os.path.join(out_logdir, 'cloud-init.log'))) + self.assertEqual( + 'cloud-init-output-log', + load_file(os.path.join(out_logdir, 'cloud-init-output.log'))) + self.assertEqual( + 'dmesg-out\n', + load_file(os.path.join(out_logdir, 'dmesg.txt'))) + self.assertEqual( + 'journal-out\n', + load_file(os.path.join(out_logdir, 'journal.txt'))) + self.assertEqual( + 'results', + load_file( + os.path.join(out_logdir, 'run', 'cloud-init', 'results.json'))) + + def test_collect_logs_includes_optional_userdata(self): + """collect-logs include userdata when --include-userdata is set.""" + log1 = self.tmp_path('cloud-init.log', self.new_root) + write_file(log1, 'cloud-init-log') + log2 = self.tmp_path('cloud-init-output.log', self.new_root) + write_file(log2, 'cloud-init-output-log') + userdata = self.tmp_path('user-data.txt', self.new_root) + write_file(userdata, 'user-data') + ensure_dir(self.run_dir) + write_file(self.tmp_path('results.json', self.run_dir), 'results') + output_tarfile = self.tmp_path('logs.tgz') + + date = datetime.utcnow().date().strftime('%Y-%m-%d') + date_logdir = 'cloud-init-logs-{0}'.format(date) + + expected_subp = { + ('dpkg-query', '--show', "-f=${Version}\n", 'cloud-init'): + '0.7fake', + ('dmesg',): 'dmesg-out\n', + ('journalctl', '-o', 'short-precise'): 'journal-out\n', + ('tar', 'czvf', output_tarfile, date_logdir): '' + } + + def fake_subp(cmd): + cmd_tuple = tuple(cmd) + if cmd_tuple not in expected_subp: + raise AssertionError( + 'Unexpected command provided to subp: {0}'.format(cmd)) + if cmd == ['tar', 'czvf', output_tarfile, date_logdir]: + subp(cmd) # Pass through tar cmd so we can check output + return expected_subp[cmd_tuple], '' + + wrap_and_call( + 'cloudinit.cmd.devel.logs', + {'subp': {'side_effect': fake_subp}, + 'CLOUDINIT_LOGS': {'new': [log1, log2]}, + 'CLOUDINIT_RUN_DIR': {'new': self.run_dir}, + 'USER_DATA_FILE': {'new': userdata}}, + logs.collect_logs, output_tarfile, include_userdata=True) + # unpack the tarfile and check file contents + subp(['tar', 'zxvf', output_tarfile, '-C', self.new_root]) + out_logdir = self.tmp_path(date_logdir, self.new_root) + self.assertEqual( + 'user-data', + load_file(os.path.join(out_logdir, 'user-data.txt'))) diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 68563e0c..6fb9d9e7 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -764,16 +764,25 @@ def main(sysv_args=None): parser_devel = subparsers.add_parser( 'devel', help='Run development tools') + parser_collect_logs = subparsers.add_parser( + 'collect-logs', help='Collect and tar all cloud-init debug info') + if sysv_args: # Only load subparsers if subcommand is specified to avoid load cost if sysv_args[0] == 'analyze': from cloudinit.analyze.__main__ import get_parser as analyze_parser # Construct analyze subcommand parser analyze_parser(parser_analyze) - if sysv_args[0] == 'devel': + elif sysv_args[0] == 'devel': from cloudinit.cmd.devel.parser import get_parser as devel_parser # Construct devel subcommand parser devel_parser(parser_devel) + elif sysv_args[0] == 'collect-logs': + from cloudinit.cmd.devel.logs import ( + get_parser as logs_parser, handle_collect_logs_args) + logs_parser(parser_collect_logs) + parser_collect_logs.set_defaults( + action=('collect-logs', handle_collect_logs_args)) args = parser.parse_args(args=sysv_args) diff --git a/packages/debian/rules.in b/packages/debian/rules.in index b87a5e84..4aa907e3 100755 --- a/packages/debian/rules.in +++ b/packages/debian/rules.in @@ -10,6 +10,7 @@ PYVER ?= python${pyver} override_dh_install: dh_install install -d debian/cloud-init/etc/rsyslog.d + install -d debian/cloud-init/usr/share/apport/package-hooks cp tools/21-cloudinit.conf debian/cloud-init/etc/rsyslog.d/21-cloudinit.conf install -D ./tools/Z99-cloud-locale-test.sh debian/cloud-init/etc/profile.d/Z99-cloud-locale-test.sh install -D ./tools/Z99-cloudinit-warnings.sh debian/cloud-init/etc/profile.d/Z99-cloudinit-warnings.sh diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 495bdc9f..258a9f08 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -72,18 +72,22 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): def test_conditional_subcommands_from_entry_point_sys_argv(self): """Subcommands from entry-point are properly parsed from sys.argv.""" + stdout = six.StringIO() + self.patchStdoutAndStderr(stdout=stdout) + expected_errors = [ - 'usage: cloud-init analyze', 'usage: cloud-init devel'] - conditional_subcommands = ['analyze', 'devel'] + 'usage: cloud-init analyze', 'usage: cloud-init collect-logs', + 'usage: cloud-init devel'] + conditional_subcommands = ['analyze', 'collect-logs', 'devel'] # The cloud-init entrypoint calls main without passing sys_argv for subcommand in conditional_subcommands: - with mock.patch('sys.argv', ['cloud-init', subcommand]): + with mock.patch('sys.argv', ['cloud-init', subcommand, '-h']): try: cli.main() except SystemExit as e: - self.assertEqual(2, e.code) # exit 2 on proper usage docs + self.assertEqual(0, e.code) # exit 2 on proper -h usage for error_message in expected_errors: - self.assertIn(error_message, self.stderr.getvalue()) + self.assertIn(error_message, stdout.getvalue()) def test_analyze_subcommand_parser(self): """The subcommand cloud-init analyze calls the correct subparser.""" @@ -94,6 +98,14 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): for subcommand in expected_subcommands: self.assertIn(subcommand, error) + def test_collect_logs_subcommand_parser(self): + """The subcommand cloud-init collect-logs calls the subparser.""" + # Provide -h param to collect-logs to avoid having to mock behavior. + stdout = six.StringIO() + self.patchStdoutAndStderr(stdout=stdout) + self._call_main(['cloud-init', 'collect-logs', '-h']) + self.assertIn('usage: cloud-init collect-log', stdout.getvalue()) + def test_devel_subcommand_parser(self): """The subcommand cloud-init devel calls the correct subparser.""" self._call_main(['cloud-init', 'devel']) -- cgit v1.2.3