From a1f678f8ebc080d4737f32275f42947b84ae025a Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 16 Mar 2018 13:43:31 -0600 Subject: cc_snap: Add new module to install and configure snapd and snap packages. Support installing and configuring snaps on ubuntu systems. Now, cloud-config files can provide a list or dictionary of snap:assertions which will be allow configuration of snapd on a system via 'snap ack' calls. The snap:commands configuration option supports arbitrary system commands intended to interact with snappy's cli. This allows users to run arbitrary snappy commands to create users, download, install and configure snap packages and snapd. This branch also deprecates old snappy and snap_config modules leaving warnings in documentation and runtime for consumers of these modules. Deprecated snap* modules will be dropped in cloud-init v.18.2 release. --- cloudinit/config/tests/test_snap.py | 533 ++++++++++++++++++++++++++++++++++++ 1 file changed, 533 insertions(+) create mode 100644 cloudinit/config/tests/test_snap.py (limited to 'cloudinit/config/tests') diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py new file mode 100644 index 00000000..c2dd6afe --- /dev/null +++ b/cloudinit/config/tests/test_snap.py @@ -0,0 +1,533 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import re +from six import StringIO + +from cloudinit.config.cc_snap import ( + ASSERTIONS_FILE, add_assertions, handle, prepend_snap_commands, + maybe_install_squashfuse, run_commands, schema) +from cloudinit.config.schema import validate_cloudconfig_schema +from cloudinit import util +from cloudinit.tests.helpers import CiTestCase, mock, wrap_and_call + + +SYSTEM_USER_ASSERTION = """\ +type: system-user +authority-id: LqvZQdfyfGlYvtep4W6Oj6pFXP9t1Ksp +brand-id: LqvZQdfyfGlYvtep4W6Oj6pFXP9t1Ksp +email: foo@bar.com +password: $6$E5YiAuMIPAwX58jG$miomhVNui/vf7f/3ctB/f0RWSKFxG0YXzrJ9rtJ1ikvzt +series: +- 16 +since: 2016-09-10T16:34:00+03:00 +until: 2017-11-10T16:34:00+03:00 +username: baz +sign-key-sha3-384: RuVvnp4n52GilycjfbbTCI3_L8Y6QlIE75wxMc0KzGV3AUQqVd9GuXoj + +AcLBXAQAAQoABgUCV/UU1wAKCRBKnlMoJQLkZVeLD/9/+hIeVywtzsDA3oxl+P+u9D13y9s6svP +Jd6Wnf4FTw6sq1GjBE4ZA7lrwSaRCUJ9Vcsvf2q9OGPY7mOb2TBxaDe0PbUMjrSrqllSSQwhpNI +zG+NxkkKuxsUmLzFa+k9m6cyojNbw5LFhQZBQCGlr3JYqC0tIREq/UsZxj+90TUC87lDJwkU8GF +s4CR+rejZj4itIcDcVxCSnJH6hv6j2JrJskJmvObqTnoOlcab+JXdamXqbldSP3UIhWoyVjqzkj ++to7mXgx+cCUA9+ngNCcfUG+1huGGTWXPCYkZ78HvErcRlIdeo4d3xwtz1cl/w3vYnq9og1XwsP +Yfetr3boig2qs1Y+j/LpsfYBYncgWjeDfAB9ZZaqQz/oc8n87tIPZDJHrusTlBfop8CqcM4xsKS +d+wnEY8e/F24mdSOYmS1vQCIDiRU3MKb6x138Ud6oHXFlRBbBJqMMctPqWDunWzb5QJ7YR0I39q +BrnEqv5NE0G7w6HOJ1LSPG5Hae3P4T2ea+ATgkb03RPr3KnXnzXg4TtBbW1nytdlgoNc/BafE1H +f3NThcq9gwX4xWZ2PAWnqVPYdDMyCtzW3Ck+o6sIzx+dh4gDLPHIi/6TPe/pUuMop9CBpWwez7V +v1z+1+URx6Xlq3Jq18y5pZ6fY3IDJ6km2nQPMzcm4Q==""" + +ACCOUNT_ASSERTION = """\ +type: account-key +authority-id: canonical +revision: 2 +public-key-sha3-384: BWDEoaqyr25nF5SNCvEv2v7QnM9QsfCc0PBMYD_i2NGSQ32EF2d4D0 +account-id: canonical +name: store +since: 2016-04-01T00:00:00.0Z +body-length: 717 +sign-key-sha3-384: -CvQKAwRQ5h3Ffn10FILJoEZUXOv6km9FwA80-Rcj-f-6jadQ89VRswH + +AcbBTQRWhcGAARAA0KKYYQWuHOrsFVi4p4l7ZzSvX7kLgJFFeFgOkzdWKBTHEnsMKjl5mefFe9j +qe8NlmJdfY7BenP7XeBtwKp700H/t9lLrZbpTNAPHXYxEWFJp5bPqIcJYBZ+29oLVLN1Tc5X482 +vCiDqL8+pPYqBrK2fNlyPlNNSum9wI70rDDL4r6FVvr+osTnGejibdV8JphWX+lrSQDnRSdM8KJ +UM43vTgLGTi9W54oRhsA2OFexRfRksTrnqGoonCjqX5wO3OFSaMDzMsO2MJ/hPfLgDqw53qjzuK +Iec9OL3k5basvu2cj5u9tKwVFDsCKK2GbKUsWWpx2KTpOifmhmiAbzkTHbH9KaoMS7p0kJwhTQG +o9aJ9VMTWHJc/NCBx7eu451u6d46sBPCXS/OMUh2766fQmoRtO1OwCTxsRKG2kkjbMn54UdFULl +VfzvyghMNRKIezsEkmM8wueTqGUGZWa6CEZqZKwhe/PROxOPYzqtDH18XZknbU1n5lNb7vNfem9 +2ai+3+JyFnW9UhfvpVF7gzAgdyCqNli4C6BIN43uwoS8HkykocZS/+Gv52aUQ/NZ8BKOHLw+7an +Q0o8W9ltSLZbEMxFIPSN0stiZlkXAp6DLyvh1Y4wXSynDjUondTpej2fSvSlCz/W5v5V7qA4nIc +vUvV7RjVzv17ut0AEQEAAQ== + +AcLDXAQAAQoABgUCV83k9QAKCRDUpVvql9g3IBT8IACKZ7XpiBZ3W4lqbPssY6On81WmxQLtvsM +WTp6zZpl/wWOSt2vMNUk9pvcmrNq1jG9CuhDfWFLGXEjcrrmVkN3YuCOajMSPFCGrxsIBLSRt/b +nrKykdLAAzMfG8rP1d82bjFFiIieE+urQ0Kcv09Jtdvavq3JT1Tek5mFyyfhHNlQEKOzWqmRWiL +3c3VOZUs1ZD8TSlnuq/x+5T0X0YtOyGjSlVxk7UybbyMNd6MZfNaMpIG4x+mxD3KHFtBAC7O6kL +eX3i6j5nCY5UABfA3DZEAkWP4zlmdBEOvZ9t293NaDdOpzsUHRkoi0Zez/9BHQ/kwx/uNc2WqrY +inCmu16JGNeXqsyinnLl7Ghn2RwhvDMlLxF6RTx8xdx1yk6p3PBTwhZMUvuZGjUtN/AG8BmVJQ1 +rsGSRkkSywvnhVJRB2sudnrMBmNS2goJbzSbmJnOlBrd2WsV0T9SgNMWZBiov3LvU4o2SmAb6b+ +rYwh8H5QHcuuYJuxDjFhPswIp6Wes5T6hUicf3SWtObcDS4HSkVS4ImBjjX9YgCuFy7QdnooOWE +aPvkRw3XCVeYq0K6w9GRsk1YFErD4XmXXZjDYY650MX9v42Sz5MmphHV8jdIY5ssbadwFSe2rCQ +6UX08zy7RsIb19hTndE6ncvSNDChUR9eEnCm73eYaWTWTnq1cxdVP/s52r8uss++OYOkPWqh5nO +haRn7INjH/yZX4qXjNXlTjo0PnHH0q08vNKDwLhxS+D9du+70FeacXFyLIbcWllSbJ7DmbumGpF +yYbtj3FDDPzachFQdIG3lSt+cSUGeyfSs6wVtc3cIPka/2Urx7RprfmoWSI6+a5NcLdj0u2z8O9 +HxeIgxDpg/3gT8ZIuFKePMcLDM19Fh/p0ysCsX+84B9chNWtsMSmIaE57V+959MVtsLu7SLb9gi +skrju0pQCwsu2wHMLTNd1f3PTHmrr49hxetTus07HSQUApMtAGKzQilF5zqFjbyaTd4xgQbd+PK +CjFyzQTDOcUhXpuUGt/IzlqiFfsCsmbj2K4KdSNYMlqIgZ3Azu8KvZLIhsyN7v5vNIZSPfEbjde +ClU9r0VRiJmtYBUjcSghD9LWn+yRLwOxhfQVjm0cBwIt5R/yPF/qC76yIVuWUtM5Y2/zJR1J8OF +qWchvlImHtvDzS9FQeLyzJAOjvZ2CnWp2gILgUz0WQdOk1Dq8ax7KS9BQ42zxw9EZAEPw3PEFqR +IQsRTONp+iVS8YxSmoYZjDlCgRMWUmawez/Fv5b9Fb/XkO5Eq4e+KfrpUujXItaipb+tV8h5v3t +oG3Ie3WOHrVjCLXIdYslpL1O4nadqR6Xv58pHj6k""" + + +class FakeCloud(object): + def __init__(self, distro): + self.distro = distro + + +class TestAddAssertions(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestAddAssertions, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_add_assertions_on_empty_list(self, m_subp): + """When provided with an empty list, add_assertions does nothing.""" + add_assertions([]) + self.assertEqual('', self.logs.getvalue()) + m_subp.assert_not_called() + + def test_add_assertions_on_non_list_or_dict(self): + """When provided an invalid type, add_assertions raises an error.""" + with self.assertRaises(TypeError) as context_manager: + add_assertions(assertions="I'm Not Valid") + self.assertEqual( + "assertion parameter was not a list or dict: I'm Not Valid", + str(context_manager.exception)) + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_add_assertions_adds_assertions_as_list(self, m_subp): + """When provided with a list, add_assertions adds all assertions.""" + self.assertEqual( + ASSERTIONS_FILE, '/var/lib/cloud/instance/snapd.assertions') + assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) + assertions = [SYSTEM_USER_ASSERTION, ACCOUNT_ASSERTION] + wrap_and_call( + 'cloudinit.config.cc_snap', + {'ASSERTIONS_FILE': {'new': assert_file}}, + add_assertions, assertions) + self.assertIn( + 'Importing user-provided snap assertions', self.logs.getvalue()) + self.assertIn( + 'sertions', self.logs.getvalue()) + self.assertEqual( + [mock.call(['snap', 'ack', assert_file], capture=True)], + m_subp.call_args_list) + compare_file = self.tmp_path('comparison', dir=self.tmp) + util.write_file(compare_file, '\n'.join(assertions).encode('utf-8')) + self.assertEqual( + util.load_file(compare_file), util.load_file(assert_file)) + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_add_assertions_adds_assertions_as_dict(self, m_subp): + """When provided with a dict, add_assertions adds all assertions.""" + self.assertEqual( + ASSERTIONS_FILE, '/var/lib/cloud/instance/snapd.assertions') + assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) + assertions = {'00': SYSTEM_USER_ASSERTION, '01': ACCOUNT_ASSERTION} + wrap_and_call( + 'cloudinit.config.cc_snap', + {'ASSERTIONS_FILE': {'new': assert_file}}, + add_assertions, assertions) + self.assertIn( + 'Importing user-provided snap assertions', self.logs.getvalue()) + self.assertIn( + "DEBUG: Snap acking: ['type: system-user', 'authority-id: Lqv", + self.logs.getvalue()) + self.assertIn( + "DEBUG: Snap acking: ['type: account-key', 'authority-id: canonic", + self.logs.getvalue()) + self.assertEqual( + [mock.call(['snap', 'ack', assert_file], capture=True)], + m_subp.call_args_list) + compare_file = self.tmp_path('comparison', dir=self.tmp) + combined = '\n'.join(assertions.values()) + util.write_file(compare_file, combined.encode('utf-8')) + self.assertEqual( + util.load_file(compare_file), util.load_file(assert_file)) + + +class TestPrepentSnapCommands(CiTestCase): + + with_logs = True + + def test_prepend_snap_commands_errors_on_neither_string_nor_list(self): + """Raise an error for each command which is not a string or list.""" + orig_commands = ['ls', 1, {'not': 'gonna work'}, ['snap', 'list']] + with self.assertRaises(TypeError) as context_manager: + prepend_snap_commands(orig_commands) + self.assertEqual( + "Invalid snap config. These commands are not a string or list:\n" + "1\n{'not': 'gonna work'}", + str(context_manager.exception)) + + def test_prepend_snap_commands_warns_on_non_snap_string_commands(self): + """Warn on each non-snap for commands of type string.""" + orig_commands = ['ls', 'snap list', 'touch /blah', 'snap install x'] + fixed_commands = prepend_snap_commands(orig_commands) + self.assertEqual( + 'WARNING: Non-snap commands in snap config:\n' + 'ls\ntouch /blah\n', + self.logs.getvalue()) + self.assertEqual(orig_commands, fixed_commands) + + def test_prepend_snap_commands_prepends_on_non_snap_list_commands(self): + """Prepend 'snap' for each non-snap command of type list.""" + orig_commands = [['ls'], ['snap', 'list'], ['snapa', '/blah'], + ['snap', 'install', 'x']] + expected = [['snap', 'ls'], ['snap', 'list'], + ['snap', 'snapa', '/blah'], + ['snap', 'install', 'x']] + fixed_commands = prepend_snap_commands(orig_commands) + self.assertEqual('', self.logs.getvalue()) + self.assertEqual(expected, fixed_commands) + + def test_prepend_snap_commands_removes_first_item_when_none(self): + """Remove the first element of a non-snap command when it is None.""" + orig_commands = [[None, 'ls'], ['snap', 'list'], + [None, 'touch', '/blah'], + ['snap', 'install', 'x']] + expected = [['ls'], ['snap', 'list'], + ['touch', '/blah'], + ['snap', 'install', 'x']] + fixed_commands = prepend_snap_commands(orig_commands) + self.assertEqual('', self.logs.getvalue()) + self.assertEqual(expected, fixed_commands) + + +class TestRunCommands(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestRunCommands, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_run_commands_on_empty_list(self, m_subp): + """When provided with an empty list, run_commands does nothing.""" + run_commands([]) + self.assertEqual('', self.logs.getvalue()) + m_subp.assert_not_called() + + def test_run_commands_on_non_list_or_dict(self): + """When provided an invalid type, run_commands raises an error.""" + with self.assertRaises(TypeError) as context_manager: + run_commands(commands="I'm Not Valid") + self.assertEqual( + "commands parameter was not a list or dict: I'm Not Valid", + str(context_manager.exception)) + + def test_run_command_logs_commands_and_exit_codes_to_stderr(self): + """All exit codes are logged to stderr.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + + cmd1 = 'echo "HI" >> %s' % outfile + cmd2 = 'bogus command' + cmd3 = 'echo "MOM" >> %s' % outfile + commands = [cmd1, cmd2, cmd3] + + mock_path = 'cloudinit.config.cc_snap.sys.stderr' + with mock.patch(mock_path, new_callable=StringIO) as m_stderr: + with self.assertRaises(RuntimeError) as context_manager: + run_commands(commands=commands) + + self.assertIsNotNone( + re.search(r'bogus: (command )?not found', + str(context_manager.exception)), + msg='Expected bogus command not found') + expected_stderr_log = '\n'.join([ + 'Begin run command: {cmd}'.format(cmd=cmd1), + 'End run command: exit(0)', + 'Begin run command: {cmd}'.format(cmd=cmd2), + 'ERROR: End run command: exit(127)', + 'Begin run command: {cmd}'.format(cmd=cmd3), + 'End run command: exit(0)\n']) + self.assertEqual(expected_stderr_log, m_stderr.getvalue()) + + def test_run_command_as_lists(self): + """When commands are specified as a list, run them in order.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + + cmd1 = 'echo "HI" >> %s' % outfile + cmd2 = 'echo "MOM" >> %s' % outfile + commands = [cmd1, cmd2] + mock_path = 'cloudinit.config.cc_snap.sys.stderr' + with mock.patch(mock_path, new_callable=StringIO): + run_commands(commands=commands) + + self.assertIn( + 'DEBUG: Running user-provided snap commands', + self.logs.getvalue()) + self.assertEqual('HI\nMOM\n', util.load_file(outfile)) + self.assertIn( + 'WARNING: Non-snap commands in snap config:', self.logs.getvalue()) + + def test_run_command_dict_sorted_as_command_script(self): + """When commands are a dict, sort them and run.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + cmd1 = 'echo "HI" >> %s' % outfile + cmd2 = 'echo "MOM" >> %s' % outfile + commands = {'02': cmd1, '01': cmd2} + mock_path = 'cloudinit.config.cc_snap.sys.stderr' + with mock.patch(mock_path, new_callable=StringIO): + run_commands(commands=commands) + + expected_messages = [ + 'DEBUG: Running user-provided snap commands'] + for message in expected_messages: + self.assertIn(message, self.logs.getvalue()) + self.assertEqual('MOM\nHI\n', util.load_file(outfile)) + + +class TestSchema(CiTestCase): + + with_logs = True + + def test_schema_warns_on_snap_not_as_dict(self): + """If the snap configuration is not a dict, emit a warning.""" + validate_cloudconfig_schema({'snap': 'wrong type'}, schema) + self.assertEqual( + "WARNING: Invalid config:\nsnap: 'wrong type' is not of type" + " 'object'\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_schema_disallows_unknown_keys(self, _): + """Unknown keys in the snap configuration emit warnings.""" + validate_cloudconfig_schema( + {'snap': {'commands': ['ls'], 'invalid-key': ''}}, schema) + self.assertIn( + 'WARNING: Invalid config:\nsnap: Additional properties are not' + " allowed ('invalid-key' was unexpected)", + self.logs.getvalue()) + + def test_warn_schema_requires_either_commands_or_assertions(self): + """Warn when snap configuration lacks both commands and assertions.""" + validate_cloudconfig_schema( + {'snap': {}}, schema) + self.assertIn( + 'WARNING: Invalid config:\nsnap: {} does not have enough' + ' properties', + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_warn_schema_commands_is_not_list_or_dict(self, _): + """Warn when snap:commands config is not a list or dict.""" + validate_cloudconfig_schema( + {'snap': {'commands': 'broken'}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nsnap.commands: 'broken' is not of type" + " 'object', 'array'\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_warn_schema_when_commands_is_empty(self, _): + """Emit warnings when snap:commands is an empty list or dict.""" + validate_cloudconfig_schema( + {'snap': {'commands': []}}, schema) + validate_cloudconfig_schema( + {'snap': {'commands': {}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nsnap.commands: [] is too short\n" + "WARNING: Invalid config:\nsnap.commands: {} does not have enough" + " properties\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_schema_when_commands_are_list_or_dict(self, _): + """No warnings when snap:commands are either a list or dict.""" + validate_cloudconfig_schema( + {'snap': {'commands': ['valid']}}, schema) + validate_cloudconfig_schema( + {'snap': {'commands': {'01': 'also valid'}}}, schema) + self.assertEqual('', self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.add_assertions') + def test_warn_schema_assertions_is_not_list_or_dict(self, _): + """Warn when snap:assertions config is not a list or dict.""" + validate_cloudconfig_schema( + {'snap': {'assertions': 'broken'}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nsnap.assertions: 'broken' is not of" + " type 'object', 'array'\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.add_assertions') + def test_warn_schema_when_assertions_is_empty(self, _): + """Emit warnings when snap:assertions is an empty list or dict.""" + validate_cloudconfig_schema( + {'snap': {'assertions': []}}, schema) + validate_cloudconfig_schema( + {'snap': {'assertions': {}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nsnap.assertions: [] is too short\n" + "WARNING: Invalid config:\nsnap.assertions: {} does not have" + " enough properties\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.add_assertions') + def test_schema_when_assertions_are_list_or_dict(self, _): + """No warnings when snap:assertions are a list or dict.""" + validate_cloudconfig_schema( + {'snap': {'assertions': ['valid']}}, schema) + validate_cloudconfig_schema( + {'snap': {'assertions': {'01': 'also valid'}}}, schema) + self.assertEqual('', self.logs.getvalue()) + + +class TestHandle(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestHandle, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('cloudinit.config.cc_snap.run_commands') + @mock.patch('cloudinit.config.cc_snap.add_assertions') + @mock.patch('cloudinit.config.cc_snap.validate_cloudconfig_schema') + def test_handle_no_config(self, m_schema, m_add, m_run): + """When no snap-related configuration is provided, nothing happens.""" + cfg = {} + handle('snap', cfg=cfg, cloud=None, log=self.logger, args=None) + self.assertIn( + "DEBUG: Skipping module named snap, no 'snap' key in config", + self.logs.getvalue()) + m_schema.assert_not_called() + m_add.assert_not_called() + m_run.assert_not_called() + + @mock.patch('cloudinit.config.cc_snap.run_commands') + @mock.patch('cloudinit.config.cc_snap.add_assertions') + @mock.patch('cloudinit.config.cc_snap.maybe_install_squashfuse') + def test_handle_skips_squashfuse_when_unconfigured(self, m_squash, m_add, + m_run): + """When squashfuse_in_container is unset, don't attempt to install.""" + handle( + 'snap', cfg={'snap': {}}, cloud=None, log=self.logger, args=None) + handle( + 'snap', cfg={'snap': {'squashfuse_in_container': None}}, + cloud=None, log=self.logger, args=None) + handle( + 'snap', cfg={'snap': {'squashfuse_in_container': False}}, + cloud=None, log=self.logger, args=None) + self.assertEqual([], m_squash.call_args_list) # No calls + # snap configuration missing assertions and commands will default to [] + self.assertIn(mock.call([]), m_add.call_args_list) + self.assertIn(mock.call([]), m_run.call_args_list) + + @mock.patch('cloudinit.config.cc_snap.maybe_install_squashfuse') + def test_handle_tries_to_install_squashfuse(self, m_squash): + """If squashfuse_in_container is True, try installing squashfuse.""" + cfg = {'snap': {'squashfuse_in_container': True}} + mycloud = FakeCloud(None) + handle('snap', cfg=cfg, cloud=mycloud, log=self.logger, args=None) + self.assertEqual( + [mock.call(mycloud)], m_squash.call_args_list) + + def test_handle_runs_commands_provided(self): + """If commands are specified as a list, run them.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + + cfg = { + 'snap': {'commands': ['echo "HI" >> %s' % outfile, + 'echo "MOM" >> %s' % outfile]}} + handle('snap', cfg=cfg, cloud=None, log=self.logger, args=None) + self.assertEqual('HI\nMOM\n', util.load_file(outfile)) + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_handle_adds_assertions(self, m_subp): + """Any configured snap assertions are provided to add_assertions.""" + assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) + compare_file = self.tmp_path('comparison', dir=self.tmp) + cfg = { + 'snap': {'assertions': [SYSTEM_USER_ASSERTION, ACCOUNT_ASSERTION]}} + wrap_and_call( + 'cloudinit.config.cc_snap', + {'ASSERTIONS_FILE': {'new': assert_file}}, + handle, 'snap', cfg=cfg, cloud=None, log=self.logger, args=None) + content = '\n'.join(cfg['snap']['assertions']) + util.write_file(compare_file, content.encode('utf-8')) + self.assertEqual( + util.load_file(compare_file), util.load_file(assert_file)) + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_handle_validates_schema(self, m_subp): + """Any provided configuration is runs validate_cloudconfig_schema.""" + assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) + cfg = {'snap': {'invalid': ''}} # Generates schema warning + wrap_and_call( + 'cloudinit.config.cc_snap', + {'ASSERTIONS_FILE': {'new': assert_file}}, + handle, 'snap', cfg=cfg, cloud=None, log=self.logger, args=None) + self.assertEqual( + "WARNING: Invalid config:\nsnap: Additional properties are not" + " allowed ('invalid' was unexpected)\n", + self.logs.getvalue()) + + +class TestMaybeInstallSquashFuse(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestMaybeInstallSquashFuse, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('cloudinit.config.cc_snap.util.is_container') + def test_maybe_install_squashfuse_skips_non_containers(self, m_container): + """maybe_install_squashfuse does nothing when not on a container.""" + m_container.return_value = False + maybe_install_squashfuse(cloud=FakeCloud(None)) + self.assertEqual([mock.call()], m_container.call_args_list) + self.assertEqual('', self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.util.is_container') + def test_maybe_install_squashfuse_raises_install_errors(self, m_container): + """maybe_install_squashfuse logs and raises package install errors.""" + m_container.return_value = True + distro = mock.MagicMock() + distro.update_package_sources.side_effect = RuntimeError( + 'Some apt error') + with self.assertRaises(RuntimeError) as context_manager: + maybe_install_squashfuse(cloud=FakeCloud(distro)) + self.assertEqual('Some apt error', str(context_manager.exception)) + self.assertIn('Package update failed\nTraceback', self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.util.is_container') + def test_maybe_install_squashfuse_raises_update_errors(self, m_container): + """maybe_install_squashfuse logs and raises package update errors.""" + m_container.return_value = True + distro = mock.MagicMock() + distro.update_package_sources.side_effect = RuntimeError( + 'Some apt error') + with self.assertRaises(RuntimeError) as context_manager: + maybe_install_squashfuse(cloud=FakeCloud(distro)) + self.assertEqual('Some apt error', str(context_manager.exception)) + self.assertIn('Package update failed\nTraceback', self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.util.is_container') + def test_maybe_install_squashfuse_happy_path(self, m_container): + """maybe_install_squashfuse logs and raises package install errors.""" + m_container.return_value = True + distro = mock.MagicMock() # No errors raised + maybe_install_squashfuse(cloud=FakeCloud(distro)) + self.assertEqual( + [mock.call()], distro.update_package_sources.call_args_list) + self.assertEqual( + [mock.call(['squashfuse'])], + distro.install_packages.call_args_list) + +# vi: ts=4 expandtab -- cgit v1.2.3 From c6f153ee15d2eea350652c0c5111f0ae3bb57561 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 16 Mar 2018 14:27:38 -0600 Subject: tests: patch leaked stderr messages from snap unit tests --- cloudinit/config/tests/test_snap.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'cloudinit/config/tests') diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py index c2dd6afe..3a2b68f3 100644 --- a/cloudinit/config/tests/test_snap.py +++ b/cloudinit/config/tests/test_snap.py @@ -444,7 +444,9 @@ class TestHandle(CiTestCase): cfg = { 'snap': {'commands': ['echo "HI" >> %s' % outfile, 'echo "MOM" >> %s' % outfile]}} - handle('snap', cfg=cfg, cloud=None, log=self.logger, args=None) + mock_path = 'cloudinit.config.cc_snap.sys.stderr' + with mock.patch(mock_path, new_callable=StringIO) as m_stderr: + handle('snap', cfg=cfg, cloud=None, log=self.logger, args=None) self.assertEqual('HI\nMOM\n', util.load_file(outfile)) @mock.patch('cloudinit.config.cc_snap.util.subp') -- cgit v1.2.3 From 7ce839f846de705980839f9c7851bd0fd7353aad Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 16 Mar 2018 15:06:53 -0600 Subject: tests: fix flakes warning for unused variable --- cloudinit/config/tests/test_snap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config/tests') diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py index 3a2b68f3..cb1205e9 100644 --- a/cloudinit/config/tests/test_snap.py +++ b/cloudinit/config/tests/test_snap.py @@ -445,7 +445,7 @@ class TestHandle(CiTestCase): 'snap': {'commands': ['echo "HI" >> %s' % outfile, 'echo "MOM" >> %s' % outfile]}} mock_path = 'cloudinit.config.cc_snap.sys.stderr' - with mock.patch(mock_path, new_callable=StringIO) as m_stderr: + with mock.patch(mock_path, new_callable=StringIO): handle('snap', cfg=cfg, cloud=None, log=self.logger, args=None) self.assertEqual('HI\nMOM\n', util.load_file(outfile)) -- cgit v1.2.3 From 0d51e912146b3031c458ce415b7d4cd6eb17d06e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 22 Mar 2018 21:13:06 -0400 Subject: ubuntu-advantage: Add new config module to support ubuntu-advantage-tools ubuntu-advantage-tools is a package for enabling and disabling extended support services such as Extended Security Maintenance (ESM), Canonical Livepatch and FIPS certified PPAs. Simplify Ubuntu Advantage setup on machines by allowing users to provide a list of ubuntu-advantage commands in cloud-config. --- cloudinit/config/cc_snap.py | 47 +---- cloudinit/config/cc_ubuntu_advantage.py | 173 +++++++++++++++ cloudinit/config/tests/test_snap.py | 52 +---- cloudinit/config/tests/test_ubuntu_advantage.py | 268 ++++++++++++++++++++++++ cloudinit/subp.py | 57 +++++ cloudinit/tests/test_subp.py | 61 ++++++ config/cloud.cfg.tmpl | 3 + doc/rtd/topics/modules.rst | 1 + tests/unittests/test_handler/test_schema.py | 1 + 9 files changed, 568 insertions(+), 95 deletions(-) create mode 100644 cloudinit/config/cc_ubuntu_advantage.py create mode 100644 cloudinit/config/tests/test_ubuntu_advantage.py create mode 100644 cloudinit/subp.py create mode 100644 cloudinit/tests/test_subp.py (limited to 'cloudinit/config/tests') diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py index db965291..34a53fd4 100644 --- a/cloudinit/config/cc_snap.py +++ b/cloudinit/config/cc_snap.py @@ -11,6 +11,7 @@ from cloudinit import log as logging from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_INSTANCE +from cloudinit.subp import prepend_base_command from cloudinit import util @@ -160,50 +161,6 @@ def add_assertions(assertions): util.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) -def prepend_snap_commands(commands): - """Ensure user-provided commands start with SNAP_CMD, warn otherwise. - - Each command is either a list or string. Perform the following: - - When the command is a list, pop the first element if it is None - - When the command is a list, insert SNAP_CMD as the first element if - not present. - - When the command is a string containing a non-snap command, warn. - - Support cut-n-paste snap command sets from public snappy documentation. - Allow flexibility to provide non-snap environment/config setup if needed. - - @commands: List of commands. Each command element is a list or string. - - @return: List of 'fixed up' snap commands. - @raise: TypeError on invalid config item type. - """ - warnings = [] - errors = [] - fixed_commands = [] - for command in commands: - if isinstance(command, list): - if command[0] is None: # Avoid warnings by specifying None - command = command[1:] - elif command[0] != SNAP_CMD: # Automatically prepend SNAP_CMD - command.insert(0, SNAP_CMD) - elif isinstance(command, str): - if not command.startswith('%s ' % SNAP_CMD): - warnings.append(command) - else: - errors.append(str(command)) - continue - fixed_commands.append(command) - - if warnings: - LOG.warning( - 'Non-snap commands in snap config:\n%s', '\n'.join(warnings)) - if errors: - raise TypeError( - 'Invalid snap config.' - ' These commands are not a string or list:\n' + '\n'.join(errors)) - return fixed_commands - - def run_commands(commands): """Run the provided commands provided in snap:commands configuration. @@ -224,7 +181,7 @@ def run_commands(commands): 'commands parameter was not a list or dict: {commands}'.format( commands=commands)) - fixed_snap_commands = prepend_snap_commands(commands) + fixed_snap_commands = prepend_base_command('snap', commands) cmd_failures = [] for command in fixed_snap_commands: diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py new file mode 100644 index 00000000..16b1868b --- /dev/null +++ b/cloudinit/config/cc_ubuntu_advantage.py @@ -0,0 +1,173 @@ +# Copyright (C) 2018 Canonical Ltd. +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""Ubuntu advantage: manage ubuntu-advantage offerings from Canonical.""" + +import sys +from textwrap import dedent + +from cloudinit import log as logging +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) +from cloudinit.settings import PER_INSTANCE +from cloudinit.subp import prepend_base_command +from cloudinit import util + + +distros = ['ubuntu'] +frequency = PER_INSTANCE + +LOG = logging.getLogger(__name__) + +schema = { + 'id': 'cc_ubuntu_advantage', + 'name': 'Ubuntu Advantage', + 'title': 'Install, configure and manage ubuntu-advantage offerings', + 'description': dedent("""\ + This module provides configuration options to setup ubuntu-advantage + subscriptions. + + .. note:: + Both ``commands`` value can be either a dictionary or a list. If + the configuration provided is a dictionary, the keys are only used + to order the execution of the commands and the dictionary is + merged with any vendor-data ubuntu-advantage configuration + provided. If a ``commands`` is provided as a list, any vendor-data + ubuntu-advantage ``commands`` are ignored. + + Ubuntu-advantage ``commands`` is a dictionary or list of + ubuntu-advantage commands to run on the deployed machine. + These commands can be used to enable or disable subscriptions to + various ubuntu-advantage products. See 'man ubuntu-advantage' for more + information on supported subcommands. + + .. note:: + Each command item can be a string or list. If the item is a list, + 'ubuntu-advantage' can be omitted and it will automatically be + inserted as part of the command. + """), + 'distros': distros, + 'examples': [dedent("""\ + # Enable Extended Security Maintenance using your service auth token + ubuntu-advantage: + commands: + 00: ubuntu-advantage enable-esm + """), dedent("""\ + # Enable livepatch by providing your livepatch token + ubuntu-advantage: + commands: + 00: ubuntu-advantage enable-livepatch + + """), dedent("""\ + # Convenience: the ubuntu-advantage command can be omitted when + # specifying commands as a list and 'ubuntu-advantage' will + # automatically be prepended. + # The following commands are equivalent + ubuntu-advantage: + commands: + 00: ['enable-livepatch', 'my-token'] + 01: ['ubuntu-advantage', 'enable-livepatch', 'my-token'] + 02: ubuntu-advantage enable-livepatch my-token + 03: 'ubuntu-advantage enable-livepatch my-token' + """)], + 'frequency': PER_INSTANCE, + 'type': 'object', + 'properties': { + 'ubuntu-advantage': { + 'type': 'object', + 'properties': { + 'commands': { + 'type': ['object', 'array'], # Array of strings or dict + 'items': { + 'oneOf': [ + {'type': 'array', 'items': {'type': 'string'}}, + {'type': 'string'}] + }, + 'additionalItems': False, # Reject non-string & non-list + 'minItems': 1, + 'minProperties': 1, + 'uniqueItems': True + } + }, + 'additionalProperties': False, # Reject keys not in schema + 'required': ['commands'] + } + } +} + +# TODO schema for 'assertions' and 'commands' are too permissive at the moment. +# Once python-jsonschema supports schema draft 6 add support for arbitrary +# object keys with 'patternProperties' constraint to validate string values. + +__doc__ = get_schema_doc(schema) # Supplement python help() + +UA_CMD = "ubuntu-advantage" + + +def run_commands(commands): + """Run the commands provided in ubuntu-advantage:commands config. + + Commands are run individually. Any errors are collected and reported + after attempting all commands. + + @param commands: A list or dict containing commands to run. Keys of a + dict will be used to order the commands provided as dict values. + """ + if not commands: + return + LOG.debug('Running user-provided ubuntu-advantage commands') + if isinstance(commands, dict): + # Sort commands based on dictionary key + commands = [v for _, v in sorted(commands.items())] + elif not isinstance(commands, list): + raise TypeError( + 'commands parameter was not a list or dict: {commands}'.format( + commands=commands)) + + fixed_ua_commands = prepend_base_command('ubuntu-advantage', commands) + + cmd_failures = [] + for command in fixed_ua_commands: + shell = isinstance(command, str) + try: + util.subp(command, shell=shell, status_cb=sys.stderr.write) + except util.ProcessExecutionError as e: + cmd_failures.append(str(e)) + if cmd_failures: + msg = ( + 'Failures running ubuntu-advantage commands:\n' + '{cmd_failures}'.format( + cmd_failures=cmd_failures)) + util.logexc(LOG, msg) + raise RuntimeError(msg) + + +def maybe_install_ua_tools(cloud): + """Install ubuntu-advantage-tools if not present.""" + if util.which('ubuntu-advantage'): + return + try: + cloud.distro.update_package_sources() + except Exception as e: + util.logexc(LOG, "Package update failed") + raise + try: + cloud.distro.install_packages(['ubuntu-advantage-tools']) + except Exception as e: + util.logexc(LOG, "Failed to install ubuntu-advantage-tools") + raise + + +def handle(name, cfg, cloud, log, args): + cfgin = cfg.get('ubuntu-advantage') + if cfgin is None: + LOG.debug(("Skipping module named %s," + " no 'ubuntu-advantage' key in configuration"), name) + return + + validate_cloudconfig_schema(cfg, schema) + maybe_install_ua_tools(cloud) + run_commands(cfgin.get('commands', [])) + +# vi: ts=4 expandtab diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py index cb1205e9..988e7f7c 100644 --- a/cloudinit/config/tests/test_snap.py +++ b/cloudinit/config/tests/test_snap.py @@ -4,8 +4,8 @@ import re from six import StringIO from cloudinit.config.cc_snap import ( - ASSERTIONS_FILE, add_assertions, handle, prepend_snap_commands, - maybe_install_squashfuse, run_commands, schema) + ASSERTIONS_FILE, add_assertions, handle, maybe_install_squashfuse, + run_commands, schema) from cloudinit.config.schema import validate_cloudconfig_schema from cloudinit import util from cloudinit.tests.helpers import CiTestCase, mock, wrap_and_call @@ -158,54 +158,6 @@ class TestAddAssertions(CiTestCase): util.load_file(compare_file), util.load_file(assert_file)) -class TestPrepentSnapCommands(CiTestCase): - - with_logs = True - - def test_prepend_snap_commands_errors_on_neither_string_nor_list(self): - """Raise an error for each command which is not a string or list.""" - orig_commands = ['ls', 1, {'not': 'gonna work'}, ['snap', 'list']] - with self.assertRaises(TypeError) as context_manager: - prepend_snap_commands(orig_commands) - self.assertEqual( - "Invalid snap config. These commands are not a string or list:\n" - "1\n{'not': 'gonna work'}", - str(context_manager.exception)) - - def test_prepend_snap_commands_warns_on_non_snap_string_commands(self): - """Warn on each non-snap for commands of type string.""" - orig_commands = ['ls', 'snap list', 'touch /blah', 'snap install x'] - fixed_commands = prepend_snap_commands(orig_commands) - self.assertEqual( - 'WARNING: Non-snap commands in snap config:\n' - 'ls\ntouch /blah\n', - self.logs.getvalue()) - self.assertEqual(orig_commands, fixed_commands) - - def test_prepend_snap_commands_prepends_on_non_snap_list_commands(self): - """Prepend 'snap' for each non-snap command of type list.""" - orig_commands = [['ls'], ['snap', 'list'], ['snapa', '/blah'], - ['snap', 'install', 'x']] - expected = [['snap', 'ls'], ['snap', 'list'], - ['snap', 'snapa', '/blah'], - ['snap', 'install', 'x']] - fixed_commands = prepend_snap_commands(orig_commands) - self.assertEqual('', self.logs.getvalue()) - self.assertEqual(expected, fixed_commands) - - def test_prepend_snap_commands_removes_first_item_when_none(self): - """Remove the first element of a non-snap command when it is None.""" - orig_commands = [[None, 'ls'], ['snap', 'list'], - [None, 'touch', '/blah'], - ['snap', 'install', 'x']] - expected = [['ls'], ['snap', 'list'], - ['touch', '/blah'], - ['snap', 'install', 'x']] - fixed_commands = prepend_snap_commands(orig_commands) - self.assertEqual('', self.logs.getvalue()) - self.assertEqual(expected, fixed_commands) - - class TestRunCommands(CiTestCase): with_logs = True diff --git a/cloudinit/config/tests/test_ubuntu_advantage.py b/cloudinit/config/tests/test_ubuntu_advantage.py new file mode 100644 index 00000000..0eeadd43 --- /dev/null +++ b/cloudinit/config/tests/test_ubuntu_advantage.py @@ -0,0 +1,268 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import re +from six import StringIO + +from cloudinit.config.cc_ubuntu_advantage import ( + handle, maybe_install_ua_tools, run_commands, schema) +from cloudinit.config.schema import validate_cloudconfig_schema +from cloudinit import util +from cloudinit.tests.helpers import CiTestCase, mock + + +# Module path used in mocks +MPATH = 'cloudinit.config.cc_ubuntu_advantage' + + +class FakeCloud(object): + def __init__(self, distro): + self.distro = distro + + +class TestRunCommands(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestRunCommands, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('%s.util.subp' % MPATH) + def test_run_commands_on_empty_list(self, m_subp): + """When provided with an empty list, run_commands does nothing.""" + run_commands([]) + self.assertEqual('', self.logs.getvalue()) + m_subp.assert_not_called() + + def test_run_commands_on_non_list_or_dict(self): + """When provided an invalid type, run_commands raises an error.""" + with self.assertRaises(TypeError) as context_manager: + run_commands(commands="I'm Not Valid") + self.assertEqual( + "commands parameter was not a list or dict: I'm Not Valid", + str(context_manager.exception)) + + def test_run_command_logs_commands_and_exit_codes_to_stderr(self): + """All exit codes are logged to stderr.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + + cmd1 = 'echo "HI" >> %s' % outfile + cmd2 = 'bogus command' + cmd3 = 'echo "MOM" >> %s' % outfile + commands = [cmd1, cmd2, cmd3] + + mock_path = '%s.sys.stderr' % MPATH + with mock.patch(mock_path, new_callable=StringIO) as m_stderr: + with self.assertRaises(RuntimeError) as context_manager: + run_commands(commands=commands) + + self.assertIsNotNone( + re.search(r'bogus: (command )?not found', + str(context_manager.exception)), + msg='Expected bogus command not found') + expected_stderr_log = '\n'.join([ + 'Begin run command: {cmd}'.format(cmd=cmd1), + 'End run command: exit(0)', + 'Begin run command: {cmd}'.format(cmd=cmd2), + 'ERROR: End run command: exit(127)', + 'Begin run command: {cmd}'.format(cmd=cmd3), + 'End run command: exit(0)\n']) + self.assertEqual(expected_stderr_log, m_stderr.getvalue()) + + def test_run_command_as_lists(self): + """When commands are specified as a list, run them in order.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + + cmd1 = 'echo "HI" >> %s' % outfile + cmd2 = 'echo "MOM" >> %s' % outfile + commands = [cmd1, cmd2] + with mock.patch('%s.sys.stderr' % MPATH, new_callable=StringIO): + run_commands(commands=commands) + + self.assertIn( + 'DEBUG: Running user-provided ubuntu-advantage commands', + self.logs.getvalue()) + self.assertEqual('HI\nMOM\n', util.load_file(outfile)) + self.assertIn( + 'WARNING: Non-ubuntu-advantage commands in ubuntu-advantage' + ' config:', + self.logs.getvalue()) + + def test_run_command_dict_sorted_as_command_script(self): + """When commands are a dict, sort them and run.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + cmd1 = 'echo "HI" >> %s' % outfile + cmd2 = 'echo "MOM" >> %s' % outfile + commands = {'02': cmd1, '01': cmd2} + with mock.patch('%s.sys.stderr' % MPATH, new_callable=StringIO): + run_commands(commands=commands) + + expected_messages = [ + 'DEBUG: Running user-provided ubuntu-advantage commands'] + for message in expected_messages: + self.assertIn(message, self.logs.getvalue()) + self.assertEqual('MOM\nHI\n', util.load_file(outfile)) + + +class TestSchema(CiTestCase): + + with_logs = True + + def test_schema_warns_on_ubuntu_advantage_not_as_dict(self): + """If ubuntu-advantage configuration is not a dict, emit a warning.""" + validate_cloudconfig_schema({'ubuntu-advantage': 'wrong type'}, schema) + self.assertEqual( + "WARNING: Invalid config:\nubuntu-advantage: 'wrong type' is not" + " of type 'object'\n", + self.logs.getvalue()) + + @mock.patch('%s.run_commands' % MPATH) + def test_schema_disallows_unknown_keys(self, _): + """Unknown keys in ubuntu-advantage configuration emit warnings.""" + validate_cloudconfig_schema( + {'ubuntu-advantage': {'commands': ['ls'], 'invalid-key': ''}}, + schema) + self.assertIn( + 'WARNING: Invalid config:\nubuntu-advantage: Additional properties' + " are not allowed ('invalid-key' was unexpected)", + self.logs.getvalue()) + + def test_warn_schema_requires_commands(self): + """Warn when ubuntu-advantage configuration lacks commands.""" + validate_cloudconfig_schema( + {'ubuntu-advantage': {}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nubuntu-advantage: 'commands' is a" + " required property\n", + self.logs.getvalue()) + + @mock.patch('%s.run_commands' % MPATH) + def test_warn_schema_commands_is_not_list_or_dict(self, _): + """Warn when ubuntu-advantage:commands config is not a list or dict.""" + validate_cloudconfig_schema( + {'ubuntu-advantage': {'commands': 'broken'}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nubuntu-advantage.commands: 'broken' is" + " not of type 'object', 'array'\n", + self.logs.getvalue()) + + @mock.patch('%s.run_commands' % MPATH) + def test_warn_schema_when_commands_is_empty(self, _): + """Emit warnings when ubuntu-advantage:commands is empty.""" + validate_cloudconfig_schema( + {'ubuntu-advantage': {'commands': []}}, schema) + validate_cloudconfig_schema( + {'ubuntu-advantage': {'commands': {}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nubuntu-advantage.commands: [] is too" + " short\nWARNING: Invalid config:\nubuntu-advantage.commands: {}" + " does not have enough properties\n", + self.logs.getvalue()) + + @mock.patch('%s.run_commands' % MPATH) + def test_schema_when_commands_are_list_or_dict(self, _): + """No warnings when ubuntu-advantage:commands are a list or dict.""" + validate_cloudconfig_schema( + {'ubuntu-advantage': {'commands': ['valid']}}, schema) + validate_cloudconfig_schema( + {'ubuntu-advantage': {'commands': {'01': 'also valid'}}}, schema) + self.assertEqual('', self.logs.getvalue()) + + +class TestHandle(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestHandle, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('%s.run_commands' % MPATH) + @mock.patch('%s.validate_cloudconfig_schema' % MPATH) + def test_handle_no_config(self, m_schema, m_run): + """When no ua-related configuration is provided, nothing happens.""" + cfg = {} + handle('ua-test', cfg=cfg, cloud=None, log=self.logger, args=None) + self.assertIn( + "DEBUG: Skipping module named ua-test, no 'ubuntu-advantage' key" + " in config", + self.logs.getvalue()) + m_schema.assert_not_called() + m_run.assert_not_called() + + @mock.patch('%s.maybe_install_ua_tools' % MPATH) + def test_handle_tries_to_install_ubuntu_advantage_tools(self, m_install): + """If ubuntu_advantage is provided, try installing ua-tools package.""" + cfg = {'ubuntu-advantage': {}} + mycloud = FakeCloud(None) + handle('nomatter', cfg=cfg, cloud=mycloud, log=self.logger, args=None) + m_install.assert_called_once_with(mycloud) + + @mock.patch('%s.maybe_install_ua_tools' % MPATH) + def test_handle_runs_commands_provided(self, m_install): + """When commands are specified as a list, run them.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + + cfg = { + 'ubuntu-advantage': {'commands': ['echo "HI" >> %s' % outfile, + 'echo "MOM" >> %s' % outfile]}} + mock_path = '%s.sys.stderr' % MPATH + with mock.patch(mock_path, new_callable=StringIO): + handle('nomatter', cfg=cfg, cloud=None, log=self.logger, args=None) + self.assertEqual('HI\nMOM\n', util.load_file(outfile)) + + +class TestMaybeInstallUATools(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestMaybeInstallUATools, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('%s.util.which' % MPATH) + def test_maybe_install_ua_tools_noop_when_ua_tools_present(self, m_which): + """Do nothing if ubuntu-advantage-tools already exists.""" + m_which.return_value = '/usr/bin/ubuntu-advantage' # already installed + distro = mock.MagicMock() + distro.update_package_sources.side_effect = RuntimeError( + 'Some apt error') + maybe_install_ua_tools(cloud=FakeCloud(distro)) # No RuntimeError + + @mock.patch('%s.util.which' % MPATH) + def test_maybe_install_ua_tools_raises_update_errors(self, m_which): + """maybe_install_ua_tools logs and raises apt update errors.""" + m_which.return_value = None + distro = mock.MagicMock() + distro.update_package_sources.side_effect = RuntimeError( + 'Some apt error') + with self.assertRaises(RuntimeError) as context_manager: + maybe_install_ua_tools(cloud=FakeCloud(distro)) + self.assertEqual('Some apt error', str(context_manager.exception)) + self.assertIn('Package update failed\nTraceback', self.logs.getvalue()) + + @mock.patch('%s.util.which' % MPATH) + def test_maybe_install_ua_raises_install_errors(self, m_which): + """maybe_install_ua_tools logs and raises package install errors.""" + m_which.return_value = None + distro = mock.MagicMock() + distro.update_package_sources.return_value = None + distro.install_packages.side_effect = RuntimeError( + 'Some install error') + with self.assertRaises(RuntimeError) as context_manager: + maybe_install_ua_tools(cloud=FakeCloud(distro)) + self.assertEqual('Some install error', str(context_manager.exception)) + self.assertIn( + 'Failed to install ubuntu-advantage-tools\n', self.logs.getvalue()) + + @mock.patch('%s.util.which' % MPATH) + def test_maybe_install_ua_tools_happy_path(self, m_which): + """maybe_install_ua_tools installs ubuntu-advantage-tools.""" + m_which.return_value = None + distro = mock.MagicMock() # No errors raised + maybe_install_ua_tools(cloud=FakeCloud(distro)) + distro.update_package_sources.assert_called_once_with() + distro.install_packages.assert_called_once_with( + ['ubuntu-advantage-tools']) + +# vi: ts=4 expandtab diff --git a/cloudinit/subp.py b/cloudinit/subp.py new file mode 100644 index 00000000..0ad09306 --- /dev/null +++ b/cloudinit/subp.py @@ -0,0 +1,57 @@ +# This file is part of cloud-init. See LICENSE file for license information. +"""Common utility functions for interacting with subprocess.""" + +# TODO move subp shellify and runparts related functions out of util.py + +import logging + +LOG = logging.getLogger(__name__) + + +def prepend_base_command(base_command, commands): + """Ensure user-provided commands start with base_command; warn otherwise. + + Each command is either a list or string. Perform the following: + - If the command is a list, pop the first element if it is None + - If the command is a list, insert base_command as the first element if + not present. + - When the command is a string not starting with 'base-command', warn. + + Allow flexibility to provide non-base-command environment/config setup if + needed. + + @commands: List of commands. Each command element is a list or string. + + @return: List of 'fixed up' commands. + @raise: TypeError on invalid config item type. + """ + warnings = [] + errors = [] + fixed_commands = [] + for command in commands: + if isinstance(command, list): + if command[0] is None: # Avoid warnings by specifying None + command = command[1:] + elif command[0] != base_command: # Automatically prepend + command.insert(0, base_command) + elif isinstance(command, str): + if not command.startswith('%s ' % base_command): + warnings.append(command) + else: + errors.append(str(command)) + continue + fixed_commands.append(command) + + if warnings: + LOG.warning( + 'Non-%s commands in %s config:\n%s', + base_command, base_command, '\n'.join(warnings)) + if errors: + raise TypeError( + 'Invalid {name} config.' + ' These commands are not a string or list:\n{errors}'.format( + name=base_command, errors='\n'.join(errors))) + return fixed_commands + + +# vi: ts=4 expandtab diff --git a/cloudinit/tests/test_subp.py b/cloudinit/tests/test_subp.py new file mode 100644 index 00000000..448097d3 --- /dev/null +++ b/cloudinit/tests/test_subp.py @@ -0,0 +1,61 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Tests for cloudinit.subp utility functions""" + +from cloudinit import subp +from cloudinit.tests.helpers import CiTestCase + + +class TestPrependBaseCommands(CiTestCase): + + with_logs = True + + def test_prepend_base_command_errors_on_neither_string_nor_list(self): + """Raise an error for each command which is not a string or list.""" + orig_commands = ['ls', 1, {'not': 'gonna work'}, ['basecmd', 'list']] + with self.assertRaises(TypeError) as context_manager: + subp.prepend_base_command( + base_command='basecmd', commands=orig_commands) + self.assertEqual( + "Invalid basecmd config. These commands are not a string or" + " list:\n1\n{'not': 'gonna work'}", + str(context_manager.exception)) + + def test_prepend_base_command_warns_on_non_base_string_commands(self): + """Warn on each non-base for commands of type string.""" + orig_commands = [ + 'ls', 'basecmd list', 'touch /blah', 'basecmd install x'] + fixed_commands = subp.prepend_base_command( + base_command='basecmd', commands=orig_commands) + self.assertEqual( + 'WARNING: Non-basecmd commands in basecmd config:\n' + 'ls\ntouch /blah\n', + self.logs.getvalue()) + self.assertEqual(orig_commands, fixed_commands) + + def test_prepend_base_command_prepends_on_non_base_list_commands(self): + """Prepend 'basecmd' for each non-basecmd command of type list.""" + orig_commands = [['ls'], ['basecmd', 'list'], ['basecmda', '/blah'], + ['basecmd', 'install', 'x']] + expected = [['basecmd', 'ls'], ['basecmd', 'list'], + ['basecmd', 'basecmda', '/blah'], + ['basecmd', 'install', 'x']] + fixed_commands = subp.prepend_base_command( + base_command='basecmd', commands=orig_commands) + self.assertEqual('', self.logs.getvalue()) + self.assertEqual(expected, fixed_commands) + + def test_prepend_base_command_removes_first_item_when_none(self): + """Remove the first element of a non-basecmd when it is None.""" + orig_commands = [[None, 'ls'], ['basecmd', 'list'], + [None, 'touch', '/blah'], + ['basecmd', 'install', 'x']] + expected = [['ls'], ['basecmd', 'list'], + ['touch', '/blah'], + ['basecmd', 'install', 'x']] + fixed_commands = subp.prepend_base_command( + base_command='basecmd', commands=orig_commands) + self.assertEqual('', self.logs.getvalue()) + self.assertEqual(expected, fixed_commands) + +# vi: ts=4 expandtab diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 56a34fab..3129d4eb 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -87,6 +87,9 @@ cloud_config_modules: - apt-pipelining - apt-configure {% endif %} +{% if variant in ["ubuntu"] %} + - ubuntu-advantage +{% endif %} {% if variant in ["suse"] %} - zypper-add-repo {% endif %} diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index a0f68129..d9720f6a 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -53,6 +53,7 @@ Modules .. automodule:: cloudinit.config.cc_ssh_authkey_fingerprints .. automodule:: cloudinit.config.cc_ssh_import_id .. automodule:: cloudinit.config.cc_timezone +.. automodule:: cloudinit.config.cc_ubuntu_advantage .. automodule:: cloudinit.config.cc_update_etc_hosts .. automodule:: cloudinit.config.cc_update_hostname .. automodule:: cloudinit.config.cc_users_groups diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 9b50ee79..ac41f124 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -27,6 +27,7 @@ class GetSchemaTest(CiTestCase): 'cc_resizefs', 'cc_runcmd', 'cc_snap', + 'cc_ubuntu_advantage', 'cc_zypper_add_repo' ], [subschema['id'] for subschema in schema['allOf']]) -- cgit v1.2.3 From 0c2f1ea29abc88957d21f56d432649989a8e4dfd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 23 Mar 2018 15:16:55 -0400 Subject: tests: Fix newly added schema unit tests to skip if no jsonschema. The recently added snap and ubuntu_advantage modules had unit tests that exercised jsonschema. Those throw error if jsonschema is not present. Fix to skip in that scenario. --- cloudinit/config/tests/test_snap.py | 5 ++++- cloudinit/config/tests/test_ubuntu_advantage.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'cloudinit/config/tests') diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py index 988e7f7c..c5b4a9de 100644 --- a/cloudinit/config/tests/test_snap.py +++ b/cloudinit/config/tests/test_snap.py @@ -8,7 +8,8 @@ from cloudinit.config.cc_snap import ( run_commands, schema) from cloudinit.config.schema import validate_cloudconfig_schema from cloudinit import util -from cloudinit.tests.helpers import CiTestCase, mock, wrap_and_call +from cloudinit.tests.helpers import ( + CiTestCase, mock, wrap_and_call, skipUnlessJsonSchema) SYSTEM_USER_ASSERTION = """\ @@ -243,6 +244,7 @@ class TestRunCommands(CiTestCase): self.assertEqual('MOM\nHI\n', util.load_file(outfile)) +@skipUnlessJsonSchema() class TestSchema(CiTestCase): with_logs = True @@ -418,6 +420,7 @@ class TestHandle(CiTestCase): util.load_file(compare_file), util.load_file(assert_file)) @mock.patch('cloudinit.config.cc_snap.util.subp') + @skipUnlessJsonSchema() def test_handle_validates_schema(self, m_subp): """Any provided configuration is runs validate_cloudconfig_schema.""" assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) diff --git a/cloudinit/config/tests/test_ubuntu_advantage.py b/cloudinit/config/tests/test_ubuntu_advantage.py index 0eeadd43..f2a59faf 100644 --- a/cloudinit/config/tests/test_ubuntu_advantage.py +++ b/cloudinit/config/tests/test_ubuntu_advantage.py @@ -7,7 +7,7 @@ from cloudinit.config.cc_ubuntu_advantage import ( handle, maybe_install_ua_tools, run_commands, schema) from cloudinit.config.schema import validate_cloudconfig_schema from cloudinit import util -from cloudinit.tests.helpers import CiTestCase, mock +from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJsonSchema # Module path used in mocks @@ -104,6 +104,7 @@ class TestRunCommands(CiTestCase): self.assertEqual('MOM\nHI\n', util.load_file(outfile)) +@skipUnlessJsonSchema() class TestSchema(CiTestCase): with_logs = True -- cgit v1.2.3