diff options
| -rw-r--r-- | cloudinit/config/cc_puppet.py | 8 | ||||
| -rw-r--r-- | cloudinit/config/cc_snap.py | 273 | ||||
| -rw-r--r-- | cloudinit/config/cc_snap_config.py | 7 | ||||
| -rw-r--r-- | cloudinit/config/cc_snappy.py | 8 | ||||
| -rw-r--r-- | cloudinit/config/tests/test_snap.py | 533 | ||||
| -rw-r--r-- | cloudinit/util.py | 13 | ||||
| -rw-r--r-- | config/cloud.cfg.tmpl | 5 | ||||
| -rw-r--r-- | doc/rtd/conf.py | 1 | ||||
| -rw-r--r-- | doc/rtd/topics/modules.rst | 1 | ||||
| -rw-r--r-- | tests/cloud_tests/releases.yaml | 3 | ||||
| -rw-r--r-- | tests/cloud_tests/testcases.yaml | 3 | ||||
| -rw-r--r-- | tests/cloud_tests/testcases/__init__.py | 3 | ||||
| -rw-r--r-- | tests/cloud_tests/testcases/base.py | 173 | ||||
| -rw-r--r-- | tests/cloud_tests/testcases/main/command_output_simple.py | 17 | ||||
| -rw-r--r-- | tests/cloud_tests/testcases/modules/snap.py | 16 | ||||
| -rw-r--r-- | tests/cloud_tests/testcases/modules/snap.yaml | 18 | ||||
| -rw-r--r-- | tests/cloud_tests/testcases/modules/snappy.py | 2 | ||||
| -rw-r--r-- | tests/cloud_tests/verify.py | 11 | ||||
| -rw-r--r-- | tests/unittests/test_handler/test_schema.py | 1 | ||||
| -rw-r--r-- | tests/unittests/test_util.py | 33 |
20 files changed, 1098 insertions, 31 deletions
diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 57a170fb..297e0721 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -140,8 +140,9 @@ def handle(name, cfg, cloud, log, _args): # (TODO(harlowja) is this really needed??) cleaned_lines = [i.lstrip() for i in contents.splitlines()] cleaned_contents = '\n'.join(cleaned_lines) - puppet_config.readfp(StringIO(cleaned_contents), - filename=p_constants.conf_path) + puppet_config.readfp( # pylint: disable=W1505 + StringIO(cleaned_contents), + filename=p_constants.conf_path) for (cfg_name, cfg) in puppet_cfg['conf'].items(): # Cert configuration is a special case # Dump the puppet master ca certificate in the correct place @@ -149,8 +150,7 @@ def handle(name, cfg, cloud, log, _args): # Puppet ssl sub-directory isn't created yet # Create it with the proper permissions and ownership util.ensure_dir(p_constants.ssl_dir, 0o771) - util.chownbyname(p_constants.ssl_dir, 'puppet', 'root') - util.ensure_dir(p_constants.ssl_cert_dir) + util.chownbyname(p_constants.ssl_cert_dir, 'puppet', 'root') util.write_file(p_constants.ssl_cert_path, cfg) util.chownbyname(p_constants.ssl_cert_path, 'puppet', 'root') diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py new file mode 100644 index 00000000..db965291 --- /dev/null +++ b/cloudinit/config/cc_snap.py @@ -0,0 +1,273 @@ +# Copyright (C) 2018 Canonical Ltd. +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""Snap: Install, configure and manage snapd and snap packages.""" + +import sys +from textwrap import dedent + +from cloudinit import log as logging +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) +from cloudinit.settings import PER_INSTANCE +from cloudinit import util + + +distros = ['ubuntu'] +frequency = PER_INSTANCE + +LOG = logging.getLogger(__name__) + +schema = { + 'id': 'cc_snap', + 'name': 'Snap', + 'title': 'Install, configure and manage snapd and snap packages', + 'description': dedent("""\ + This module provides a simple configuration namespace in cloud-init to + both setup snapd and install snaps. + + .. note:: + Both ``assertions`` and ``commands`` values can be either a + dictionary or a list. If these configs are provided as a + dictionary, the keys are only used to order the execution of the + assertions or commands and the dictionary is merged with any + vendor-data snap configuration provided. If a list is provided by + the user instead of a dict, any vendor-data snap configuration is + ignored. + + The ``assertions`` configuration option is a dictionary or list of + properly-signed snap assertions which will run before any snap + ``commands``. They will be added to snapd's assertion database by + invoking ``snap ack <aggregate_assertion_file>``. + + Snap ``commands`` is a dictionary or list of individual snap + commands to run on the target system. These commands can be used to + create snap users, install snaps and provide snap configuration. + + .. note:: + If 'side-loading' private/unpublished snaps on an instance, it is + best to create a snap seed directory and seed.yaml manifest in + **/var/lib/snapd/seed/** which snapd automatically installs on + startup. + + **Development only**: The ``squashfuse_in_container`` boolean can be + set true to install squashfuse package when in a container to enable + snap installs. Default is false. + """), + 'distros': distros, + 'examples': [dedent("""\ + snap: + assertions: + 00: | + signed_assertion_blob_here + 02: | + signed_assertion_blob_here + commands: + 00: snap create-user --sudoer --known <snap-user>@mydomain.com + 01: snap install canonical-livepatch + 02: canonical-livepatch enable <AUTH_TOKEN> + """), dedent("""\ + # LXC-based containers require squashfuse before snaps can be installed + snap: + commands: + 00: apt-get install squashfuse -y + 11: snap install emoj + + """), dedent("""\ + # Convenience: the snap command can be omitted when specifying commands + # as a list and 'snap' will automatically be prepended. + # The following commands are equivalent: + snap: + commands: + 00: ['install', 'vlc'] + 01: ['snap', 'install', 'vlc'] + 02: snap install vlc + 03: 'snap install vlc' + """)], + 'frequency': PER_INSTANCE, + 'type': 'object', + 'properties': { + 'snap': { + 'type': 'object', + 'properties': { + 'assertions': { + 'type': ['object', 'array'], # Array of strings or dict + 'items': {'type': 'string'}, + 'additionalItems': False, # Reject items non-string + 'minItems': 1, + 'minProperties': 1, + 'uniqueItems': True + }, + 'commands': { + 'type': ['object', 'array'], # Array of strings or dict + 'items': { + 'oneOf': [ + {'type': 'array', 'items': {'type': 'string'}}, + {'type': 'string'}] + }, + 'additionalItems': False, # Reject non-string & non-list + 'minItems': 1, + 'minProperties': 1, + 'uniqueItems': True + }, + 'squashfuse_in_container': { + 'type': 'boolean' + } + }, + 'additionalProperties': False, # Reject keys not in schema + 'required': [], + 'minProperties': 1 + } + } +} + +# TODO schema for 'assertions' and 'commands' are too permissive at the moment. +# Once python-jsonschema supports schema draft 6 add support for arbitrary +# object keys with 'patternProperties' constraint to validate string values. + +__doc__ = get_schema_doc(schema) # Supplement python help() + +SNAP_CMD = "snap" +ASSERTIONS_FILE = "/var/lib/cloud/instance/snapd.assertions" + + +def add_assertions(assertions): + """Import list of assertions. + + Import assertions by concatenating each assertion into a + string separated by a '\n'. Write this string to a instance file and + then invoke `snap ack /path/to/file` and check for errors. + If snap exits 0, then all assertions are imported. + """ + if not assertions: + return + LOG.debug('Importing user-provided snap assertions') + if isinstance(assertions, dict): + assertions = assertions.values() + elif not isinstance(assertions, list): + raise TypeError( + 'assertion parameter was not a list or dict: {assertions}'.format( + assertions=assertions)) + + snap_cmd = [SNAP_CMD, 'ack'] + combined = "\n".join(assertions) + + for asrt in assertions: + LOG.debug('Snap acking: %s', asrt.split('\n')[0:2]) + + util.write_file(ASSERTIONS_FILE, combined.encode('utf-8')) + util.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) + + +def prepend_snap_commands(commands): + """Ensure user-provided commands start with SNAP_CMD, warn otherwise. + + Each command is either a list or string. Perform the following: + - When the command is a list, pop the first element if it is None + - When the command is a list, insert SNAP_CMD as the first element if + not present. + - When the command is a string containing a non-snap command, warn. + + Support cut-n-paste snap command sets from public snappy documentation. + Allow flexibility to provide non-snap environment/config setup if needed. + + @commands: List of commands. Each command element is a list or string. + + @return: List of 'fixed up' snap commands. + @raise: TypeError on invalid config item type. + """ + warnings = [] + errors = [] + fixed_commands = [] + for command in commands: + if isinstance(command, list): + if command[0] is None: # Avoid warnings by specifying None + command = command[1:] + elif command[0] != SNAP_CMD: # Automatically prepend SNAP_CMD + command.insert(0, SNAP_CMD) + elif isinstance(command, str): + if not command.startswith('%s ' % SNAP_CMD): + warnings.append(command) + else: + errors.append(str(command)) + continue + fixed_commands.append(command) + + if warnings: + LOG.warning( + 'Non-snap commands in snap config:\n%s', '\n'.join(warnings)) + if errors: + raise TypeError( + 'Invalid snap config.' + ' These commands are not a string or list:\n' + '\n'.join(errors)) + return fixed_commands + + +def run_commands(commands): + """Run the provided commands provided in snap:commands configuration. + + Commands are run individually. Any errors are collected and reported + after attempting all commands. + + @param commands: A list or dict containing commands to run. Keys of a + dict will be used to order the commands provided as dict values. + """ + if not commands: + return + LOG.debug('Running user-provided snap commands') + if isinstance(commands, dict): + # Sort commands based on dictionary key + commands = [v for _, v in sorted(commands.items())] + elif not isinstance(commands, list): + raise TypeError( + 'commands parameter was not a list or dict: {commands}'.format( + commands=commands)) + + fixed_snap_commands = prepend_snap_commands(commands) + + cmd_failures = [] + for command in fixed_snap_commands: + shell = isinstance(command, str) + try: + util.subp(command, shell=shell, status_cb=sys.stderr.write) + except util.ProcessExecutionError as e: + cmd_failures.append(str(e)) + if cmd_failures: + msg = 'Failures running snap commands:\n{cmd_failures}'.format( + cmd_failures=cmd_failures) + util.logexc(LOG, msg) + raise RuntimeError(msg) + + +# RELEASE_BLOCKER: Once LP: #1628289 is released on xenial, drop this function. +def maybe_install_squashfuse(cloud): + """Install squashfuse if we are in a container.""" + if not util.is_container(): + return + try: + cloud.distro.update_package_sources() + except Exception as e: + util.logexc(LOG, "Package update failed") + raise + try: + cloud.distro.install_packages(['squashfuse']) + except Exception as e: + util.logexc(LOG, "Failed to install squashfuse") + raise + + +def handle(name, cfg, cloud, log, args): + cfgin = cfg.get('snap', {}) + if not cfgin: + LOG.debug(("Skipping module named %s," + " no 'snap' key in configuration"), name) + return + + validate_cloudconfig_schema(cfg, schema) + if util.is_true(cfgin.get('squashfuse_in_container', False)): + maybe_install_squashfuse(cloud) + add_assertions(cfgin.get('assertions', [])) + run_commands(cfgin.get('commands', [])) + +# vi: ts=4 expandtab diff --git a/cloudinit/config/cc_snap_config.py b/cloudinit/config/cc_snap_config.py index e82c0811..afe297ee 100644 --- a/cloudinit/config/cc_snap_config.py +++ b/cloudinit/config/cc_snap_config.py @@ -4,11 +4,15 @@ # # This file is part of cloud-init. See LICENSE file for license information. +# RELEASE_BLOCKER: Remove this deprecated module in 18.3 """ Snap Config ----------- **Summary:** snap_config modules allows configuration of snapd. +**Deprecated**: Use :ref:`snap` module instead. This module will not exist +in cloud-init 18.3. + This module uses the same ``snappy`` namespace for configuration but acts only only a subset of the configuration. @@ -154,6 +158,9 @@ def handle(name, cfg, cloud, log, args): LOG.debug('No snappy config provided, skipping') return + log.warning( + 'DEPRECATION: snap_config module will be dropped in 18.3 release.' + ' Use snap module instead') if not(util.system_is_snappy()): LOG.debug("%s: system not snappy", name) return diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index eecb8178..bab80bbe 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -1,10 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. +# RELEASE_BLOCKER: Remove this deprecated module in 18.3 """ Snappy ------ **Summary:** snappy modules allows configuration of snappy. +**Deprecated**: Use :ref:`snap` module instead. This module will not exist +in cloud-init 18.3. + The below example config config would install ``etcd``, and then install ``pkg2.smoser`` with a ``<config-file>`` argument where ``config-file`` has ``config-blob`` inside it. If ``pkgname`` is installed already, then @@ -271,6 +275,10 @@ def handle(name, cfg, cloud, log, args): LOG.debug("%s: 'auto' mode, and system not snappy", name) return + log.warning( + 'DEPRECATION: snappy module will be dropped in 18.3 release.' + ' Use snap module instead') + set_snappy_command() pkg_ops = get_package_ops(packages=mycfg['packages'], diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py new file mode 100644 index 00000000..c2dd6afe --- /dev/null +++ b/cloudinit/config/tests/test_snap.py @@ -0,0 +1,533 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import re +from six import StringIO + +from cloudinit.config.cc_snap import ( + ASSERTIONS_FILE, add_assertions, handle, prepend_snap_commands, + maybe_install_squashfuse, run_commands, schema) +from cloudinit.config.schema import validate_cloudconfig_schema +from cloudinit import util +from cloudinit.tests.helpers import CiTestCase, mock, wrap_and_call + + +SYSTEM_USER_ASSERTION = """\ +type: system-user +authority-id: LqvZQdfyfGlYvtep4W6Oj6pFXP9t1Ksp +brand-id: LqvZQdfyfGlYvtep4W6Oj6pFXP9t1Ksp +email: foo@bar.com +password: $6$E5YiAuMIPAwX58jG$miomhVNui/vf7f/3ctB/f0RWSKFxG0YXzrJ9rtJ1ikvzt +series: +- 16 +since: 2016-09-10T16:34:00+03:00 +until: 2017-11-10T16:34:00+03:00 +username: baz +sign-key-sha3-384: RuVvnp4n52GilycjfbbTCI3_L8Y6QlIE75wxMc0KzGV3AUQqVd9GuXoj + +AcLBXAQAAQoABgUCV/UU1wAKCRBKnlMoJQLkZVeLD/9/+hIeVywtzsDA3oxl+P+u9D13y9s6svP +Jd6Wnf4FTw6sq1GjBE4ZA7lrwSaRCUJ9Vcsvf2q9OGPY7mOb2TBxaDe0PbUMjrSrqllSSQwhpNI +zG+NxkkKuxsUmLzFa+k9m6cyojNbw5LFhQZBQCGlr3JYqC0tIREq/UsZxj+90TUC87lDJwkU8GF +s4CR+rejZj4itIcDcVxCSnJH6hv6j2JrJskJmvObqTnoOlcab+JXdamXqbldSP3UIhWoyVjqzkj ++to7mXgx+cCUA9+ngNCcfUG+1huGGTWXPCYkZ78HvErcRlIdeo4d3xwtz1cl/w3vYnq9og1XwsP +Yfetr3boig2qs1Y+j/LpsfYBYncgWjeDfAB9ZZaqQz/oc8n87tIPZDJHrusTlBfop8CqcM4xsKS +d+wnEY8e/F24mdSOYmS1vQCIDiRU3MKb6x138Ud6oHXFlRBbBJqMMctPqWDunWzb5QJ7YR0I39q +BrnEqv5NE0G7w6HOJ1LSPG5Hae3P4T2ea+ATgkb03RPr3KnXnzXg4TtBbW1nytdlgoNc/BafE1H +f3NThcq9gwX4xWZ2PAWnqVPYdDMyCtzW3Ck+o6sIzx+dh4gDLPHIi/6TPe/pUuMop9CBpWwez7V +v1z+1+URx6Xlq3Jq18y5pZ6fY3IDJ6km2nQPMzcm4Q==""" + +ACCOUNT_ASSERTION = """\ +type: account-key +authority-id: canonical +revision: 2 +public-key-sha3-384: BWDEoaqyr25nF5SNCvEv2v7QnM9QsfCc0PBMYD_i2NGSQ32EF2d4D0 +account-id: canonical +name: store +since: 2016-04-01T00:00:00.0Z +body-length: 717 +sign-key-sha3-384: -CvQKAwRQ5h3Ffn10FILJoEZUXOv6km9FwA80-Rcj-f-6jadQ89VRswH + +AcbBTQRWhcGAARAA0KKYYQWuHOrsFVi4p4l7ZzSvX7kLgJFFeFgOkzdWKBTHEnsMKjl5mefFe9j +qe8NlmJdfY7BenP7XeBtwKp700H/t9lLrZbpTNAPHXYxEWFJp5bPqIcJYBZ+29oLVLN1Tc5X482 +vCiDqL8+pPYqBrK2fNlyPlNNSum9wI70rDDL4r6FVvr+osTnGejibdV8JphWX+lrSQDnRSdM8KJ +UM43vTgLGTi9W54oRhsA2OFexRfRksTrnqGoonCjqX5wO3OFSaMDzMsO2MJ/hPfLgDqw53qjzuK +Iec9OL3k5basvu2cj5u9tKwVFDsCKK2GbKUsWWpx2KTpOifmhmiAbzkTHbH9KaoMS7p0kJwhTQG +o9aJ9VMTWHJc/NCBx7eu451u6d46sBPCXS/OMUh2766fQmoRtO1OwCTxsRKG2kkjbMn54UdFULl +VfzvyghMNRKIezsEkmM8wueTqGUGZWa6CEZqZKwhe/PROxOPYzqtDH18XZknbU1n5lNb7vNfem9 +2ai+3+JyFnW9UhfvpVF7gzAgdyCqNli4C6BIN43uwoS8HkykocZS/+Gv52aUQ/NZ8BKOHLw+7an +Q0o8W9ltSLZbEMxFIPSN0stiZlkXAp6DLyvh1Y4wXSynDjUondTpej2fSvSlCz/W5v5V7qA4nIc +vUvV7RjVzv17ut0AEQEAAQ== + +AcLDXAQAAQoABgUCV83k9QAKCRDUpVvql9g3IBT8IACKZ7XpiBZ3W4lqbPssY6On81WmxQLtvsM +WTp6zZpl/wWOSt2vMNUk9pvcmrNq1jG9CuhDfWFLGXEjcrrmVkN3YuCOajMSPFCGrxsIBLSRt/b +nrKykdLAAzMfG8rP1d82bjFFiIieE+urQ0Kcv09Jtdvavq3JT1Tek5mFyyfhHNlQEKOzWqmRWiL +3c3VOZUs1ZD8TSlnuq/x+5T0X0YtOyGjSlVxk7UybbyMNd6MZfNaMpIG4x+mxD3KHFtBAC7O6kL +eX3i6j5nCY5UABfA3DZEAkWP4zlmdBEOvZ9t293NaDdOpzsUHRkoi0Zez/9BHQ/kwx/uNc2WqrY +inCmu16JGNeXqsyinnLl7Ghn2RwhvDMlLxF6RTx8xdx1yk6p3PBTwhZMUvuZGjUtN/AG8BmVJQ1 +rsGSRkkSywvnhVJRB2sudnrMBmNS2goJbzSbmJnOlBrd2WsV0T9SgNMWZBiov3LvU4o2SmAb6b+ +rYwh8H5QHcuuYJuxDjFhPswIp6Wes5T6hUicf3SWtObcDS4HSkVS4ImBjjX9YgCuFy7QdnooOWE +aPvkRw3XCVeYq0K6w9GRsk1YFErD4XmXXZjDYY650MX9v42Sz5MmphHV8jdIY5ssbadwFSe2rCQ +6UX08zy7RsIb19hTndE6ncvSNDChUR9eEnCm73eYaWTWTnq1cxdVP/s52r8uss++OYOkPWqh5nO +haRn7INjH/yZX4qXjNXlTjo0PnHH0q08vNKDwLhxS+D9du+70FeacXFyLIbcWllSbJ7DmbumGpF +yYbtj3FDDPzachFQdIG3lSt+cSUGeyfSs6wVtc3cIPka/2Urx7RprfmoWSI6+a5NcLdj0u2z8O9 +HxeIgxDpg/3gT8ZIuFKePMcLDM19Fh/p0ysCsX+84B9chNWtsMSmIaE57V+959MVtsLu7SLb9gi +skrju0pQCwsu2wHMLTNd1f3PTHmrr49hxetTus07HSQUApMtAGKzQilF5zqFjbyaTd4xgQbd+PK +CjFyzQTDOcUhXpuUGt/IzlqiFfsCsmbj2K4KdSNYMlqIgZ3Azu8KvZLIhsyN7v5vNIZSPfEbjde +ClU9r0VRiJmtYBUjcSghD9LWn+yRLwOxhfQVjm0cBwIt5R/yPF/qC76yIVuWUtM5Y2/zJR1J8OF +qWchvlImHtvDzS9FQeLyzJAOjvZ2CnWp2gILgUz0WQdOk1Dq8ax7KS9BQ42zxw9EZAEPw3PEFqR +IQsRTONp+iVS8YxSmoYZjDlCgRMWUmawez/Fv5b9Fb/XkO5Eq4e+KfrpUujXItaipb+tV8h5v3t +oG3Ie3WOHrVjCLXIdYslpL1O4nadqR6Xv58pHj6k""" + + +class FakeCloud(object): + def __init__(self, distro): + self.distro = distro + + +class TestAddAssertions(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestAddAssertions, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_add_assertions_on_empty_list(self, m_subp): + """When provided with an empty list, add_assertions does nothing.""" + add_assertions([]) + self.assertEqual('', self.logs.getvalue()) + m_subp.assert_not_called() + + def test_add_assertions_on_non_list_or_dict(self): + """When provided an invalid type, add_assertions raises an error.""" + with self.assertRaises(TypeError) as context_manager: + add_assertions(assertions="I'm Not Valid") + self.assertEqual( + "assertion parameter was not a list or dict: I'm Not Valid", + str(context_manager.exception)) + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_add_assertions_adds_assertions_as_list(self, m_subp): + """When provided with a list, add_assertions adds all assertions.""" + self.assertEqual( + ASSERTIONS_FILE, '/var/lib/cloud/instance/snapd.assertions') + assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) + assertions = [SYSTEM_USER_ASSERTION, ACCOUNT_ASSERTION] + wrap_and_call( + 'cloudinit.config.cc_snap', + {'ASSERTIONS_FILE': {'new': assert_file}}, + add_assertions, assertions) + self.assertIn( + 'Importing user-provided snap assertions', self.logs.getvalue()) + self.assertIn( + 'sertions', self.logs.getvalue()) + self.assertEqual( + [mock.call(['snap', 'ack', assert_file], capture=True)], + m_subp.call_args_list) + compare_file = self.tmp_path('comparison', dir=self.tmp) + util.write_file(compare_file, '\n'.join(assertions).encode('utf-8')) + self.assertEqual( + util.load_file(compare_file), util.load_file(assert_file)) + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_add_assertions_adds_assertions_as_dict(self, m_subp): + """When provided with a dict, add_assertions adds all assertions.""" + self.assertEqual( + ASSERTIONS_FILE, '/var/lib/cloud/instance/snapd.assertions') + assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) + assertions = {'00': SYSTEM_USER_ASSERTION, '01': ACCOUNT_ASSERTION} + wrap_and_call( + 'cloudinit.config.cc_snap', + {'ASSERTIONS_FILE': {'new': assert_file}}, + add_assertions, assertions) + self.assertIn( + 'Importing user-provided snap assertions', self.logs.getvalue()) + self.assertIn( + "DEBUG: Snap acking: ['type: system-user', 'authority-id: Lqv", + self.logs.getvalue()) + self.assertIn( + "DEBUG: Snap acking: ['type: account-key', 'authority-id: canonic", + self.logs.getvalue()) + self.assertEqual( + [mock.call(['snap', 'ack', assert_file], capture=True)], + m_subp.call_args_list) + compare_file = self.tmp_path('comparison', dir=self.tmp) + combined = '\n'.join(assertions.values()) + util.write_file(compare_file, combined.encode('utf-8')) + self.assertEqual( + util.load_file(compare_file), util.load_file(assert_file)) + + +class TestPrepentSnapCommands(CiTestCase): + + with_logs = True + + def test_prepend_snap_commands_errors_on_neither_string_nor_list(self): + """Raise an error for each command which is not a string or list.""" + orig_commands = ['ls', 1, {'not': 'gonna work'}, ['snap', 'list']] + with self.assertRaises(TypeError) as context_manager: + prepend_snap_commands(orig_commands) + self.assertEqual( + "Invalid snap config. These commands are not a string or list:\n" + "1\n{'not': 'gonna work'}", + str(context_manager.exception)) + + def test_prepend_snap_commands_warns_on_non_snap_string_commands(self): + """Warn on each non-snap for commands of type string.""" + orig_commands = ['ls', 'snap list', 'touch /blah', 'snap install x'] + fixed_commands = prepend_snap_commands(orig_commands) + self.assertEqual( + 'WARNING: Non-snap commands in snap config:\n' + 'ls\ntouch /blah\n', + self.logs.getvalue()) + self.assertEqual(orig_commands, fixed_commands) + + def test_prepend_snap_commands_prepends_on_non_snap_list_commands(self): + """Prepend 'snap' for each non-snap command of type list.""" + orig_commands = [['ls'], ['snap', 'list'], ['snapa', '/blah'], + ['snap', 'install', 'x']] + expected = [['snap', 'ls'], ['snap', 'list'], + ['snap', 'snapa', '/blah'], + ['snap', 'install', 'x']] + fixed_commands = prepend_snap_commands(orig_commands) + self.assertEqual('', self.logs.getvalue()) + self.assertEqual(expected, fixed_commands) + + def test_prepend_snap_commands_removes_first_item_when_none(self): + """Remove the first element of a non-snap command when it is None.""" + orig_commands = [[None, 'ls'], ['snap', 'list'], + [None, 'touch', '/blah'], + ['snap', 'install', 'x']] + expected = [['ls'], ['snap', 'list'], + ['touch', '/blah'], + ['snap', 'install', 'x']] + fixed_commands = prepend_snap_commands(orig_commands) + self.assertEqual('', self.logs.getvalue()) + self.assertEqual(expected, fixed_commands) + + +class TestRunCommands(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestRunCommands, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_run_commands_on_empty_list(self, m_subp): + """When provided with an empty list, run_commands does nothing.""" + run_commands([]) + self.assertEqual('', self.logs.getvalue()) + m_subp.assert_not_called() + + def test_run_commands_on_non_list_or_dict(self): + """When provided an invalid type, run_commands raises an error.""" + with self.assertRaises(TypeError) as context_manager: + run_commands(commands="I'm Not Valid") + self.assertEqual( + "commands parameter was not a list or dict: I'm Not Valid", + str(context_manager.exception)) + + def test_run_command_logs_commands_and_exit_codes_to_stderr(self): + """All exit codes are logged to stderr.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + + cmd1 = 'echo "HI" >> %s' % outfile + cmd2 = 'bogus command' + cmd3 = 'echo "MOM" >> %s' % outfile + commands = [cmd1, cmd2, cmd3] + + mock_path = 'cloudinit.config.cc_snap.sys.stderr' + with mock.patch(mock_path, new_callable=StringIO) as m_stderr: + with self.assertRaises(RuntimeError) as context_manager: + run_commands(commands=commands) + + self.assertIsNotNone( + re.search(r'bogus: (command )?not found', + str(context_manager.exception)), + msg='Expected bogus command not found') + expected_stderr_log = '\n'.join([ + 'Begin run command: {cmd}'.format(cmd=cmd1), + 'End run command: exit(0)', + 'Begin run command: {cmd}'.format(cmd=cmd2), + 'ERROR: End run command: exit(127)', + 'Begin run command: {cmd}'.format(cmd=cmd3), + 'End run command: exit(0)\n']) + self.assertEqual(expected_stderr_log, m_stderr.getvalue()) + + def test_run_command_as_lists(self): + """When commands are specified as a list, run them in order.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + + cmd1 = 'echo "HI" >> %s' % outfile + cmd2 = 'echo "MOM" >> %s' % outfile + commands = [cmd1, cmd2] + mock_path = 'cloudinit.config.cc_snap.sys.stderr' + with mock.patch(mock_path, new_callable=StringIO): + run_commands(commands=commands) + + self.assertIn( + 'DEBUG: Running user-provided snap commands', + self.logs.getvalue()) + self.assertEqual('HI\nMOM\n', util.load_file(outfile)) + self.assertIn( + 'WARNING: Non-snap commands in snap config:', self.logs.getvalue()) + + def test_run_command_dict_sorted_as_command_script(self): + """When commands are a dict, sort them and run.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + cmd1 = 'echo "HI" >> %s' % outfile + cmd2 = 'echo "MOM" >> %s' % outfile + commands = {'02': cmd1, '01': cmd2} + mock_path = 'cloudinit.config.cc_snap.sys.stderr' + with mock.patch(mock_path, new_callable=StringIO): + run_commands(commands=commands) + + expected_messages = [ + 'DEBUG: Running user-provided snap commands'] + for message in expected_messages: + self.assertIn(message, self.logs.getvalue()) + self.assertEqual('MOM\nHI\n', util.load_file(outfile)) + + +class TestSchema(CiTestCase): + + with_logs = True + + def test_schema_warns_on_snap_not_as_dict(self): + """If the snap configuration is not a dict, emit a warning.""" + validate_cloudconfig_schema({'snap': 'wrong type'}, schema) + self.assertEqual( + "WARNING: Invalid config:\nsnap: 'wrong type' is not of type" + " 'object'\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_schema_disallows_unknown_keys(self, _): + """Unknown keys in the snap configuration emit warnings.""" + validate_cloudconfig_schema( + {'snap': {'commands': ['ls'], 'invalid-key': ''}}, schema) + self.assertIn( + 'WARNING: Invalid config:\nsnap: Additional properties are not' + " allowed ('invalid-key' was unexpected)", + self.logs.getvalue()) + + def test_warn_schema_requires_either_commands_or_assertions(self): + """Warn when snap configuration lacks both commands and assertions.""" + validate_cloudconfig_schema( + {'snap': {}}, schema) + self.assertIn( + 'WARNING: Invalid config:\nsnap: {} does not have enough' + ' properties', + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_warn_schema_commands_is_not_list_or_dict(self, _): + """Warn when snap:commands config is not a list or dict.""" + validate_cloudconfig_schema( + {'snap': {'commands': 'broken'}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nsnap.commands: 'broken' is not of type" + " 'object', 'array'\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_warn_schema_when_commands_is_empty(self, _): + """Emit warnings when snap:commands is an empty list or dict.""" + validate_cloudconfig_schema( + {'snap': {'commands': []}}, schema) + validate_cloudconfig_schema( + {'snap': {'commands': {}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nsnap.commands: [] is too short\n" + "WARNING: Invalid config:\nsnap.commands: {} does not have enough" + " properties\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_schema_when_commands_are_list_or_dict(self, _): + """No warnings when snap:commands are either a list or dict.""" + validate_cloudconfig_schema( + {'snap': {'commands': ['valid']}}, schema) + validate_cloudconfig_schema( + {'snap': {'commands': {'01': 'also valid'}}}, schema) + self.assertEqual('', self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.add_assertions') + def test_warn_schema_assertions_is_not_list_or_dict(self, _): + """Warn when snap:assertions config is not a list or dict.""" + validate_cloudconfig_schema( + {'snap': {'assertions': 'broken'}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nsnap.assertions: 'broken' is not of" + " type 'object', 'array'\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.add_assertions') + def test_warn_schema_when_assertions_is_empty(self, _): + """Emit warnings when snap:assertions is an empty list or dict.""" + validate_cloudconfig_schema( + {'snap': {'assertions': []}}, schema) + validate_cloudconfig_schema( + {'snap': {'assertions': {}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\nsnap.assertions: [] is too short\n" + "WARNING: Invalid config:\nsnap.assertions: {} does not have" + " enough properties\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.add_assertions') + def test_schema_when_assertions_are_list_or_dict(self, _): + """No warnings when snap:assertions are a list or dict.""" + validate_cloudconfig_schema( + {'snap': {'assertions': ['valid']}}, schema) + validate_cloudconfig_schema( + {'snap': {'assertions': {'01': 'also valid'}}}, schema) + self.assertEqual('', self.logs.getvalue()) + + +class TestHandle(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestHandle, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('cloudinit.config.cc_snap.run_commands') + @mock.patch('cloudinit.config.cc_snap.add_assertions') + @mock.patch('cloudinit.config.cc_snap.validate_cloudconfig_schema') + def test_handle_no_config(self, m_schema, m_add, m_run): + """When no snap-related configuration is provided, nothing happens.""" + cfg = {} + handle('snap', cfg=cfg, cloud=None, log=self.logger, args=None) + self.assertIn( + "DEBUG: Skipping module named snap, no 'snap' key in config", + self.logs.getvalue()) + m_schema.assert_not_called() + m_add.assert_not_called() + m_run.assert_not_called() + + @mock.patch('cloudinit.config.cc_snap.run_commands') + @mock.patch('cloudinit.config.cc_snap.add_assertions') + @mock.patch('cloudinit.config.cc_snap.maybe_install_squashfuse') + def test_handle_skips_squashfuse_when_unconfigured(self, m_squash, m_add, + m_run): + """When squashfuse_in_container is unset, don't attempt to install.""" + handle( + 'snap', cfg={'snap': {}}, cloud=None, log=self.logger, args=None) + handle( + 'snap', cfg={'snap': {'squashfuse_in_container': None}}, + cloud=None, log=self.logger, args=None) + handle( + 'snap', cfg={'snap': {'squashfuse_in_container': False}}, + cloud=None, log=self.logger, args=None) + self.assertEqual([], m_squash.call_args_list) # No calls + # snap configuration missing assertions and commands will default to [] + self.assertIn(mock.call([]), m_add.call_args_list) + self.assertIn(mock.call([]), m_run.call_args_list) + + @mock.patch('cloudinit.config.cc_snap.maybe_install_squashfuse') + def test_handle_tries_to_install_squashfuse(self, m_squash): + """If squashfuse_in_container is True, try installing squashfuse.""" + cfg = {'snap': {'squashfuse_in_container': True}} + mycloud = FakeCloud(None) + handle('snap', cfg=cfg, cloud=mycloud, log=self.logger, args=None) + self.assertEqual( + [mock.call(mycloud)], m_squash.call_args_list) + + def test_handle_runs_commands_provided(self): + """If commands are specified as a list, run them.""" + outfile = self.tmp_path('output.log', dir=self.tmp) + + cfg = { + 'snap': {'commands': ['echo "HI" >> %s' % outfile, + 'echo "MOM" >> %s' % outfile]}} + handle('snap', cfg=cfg, cloud=None, log=self.logger, args=None) + self.assertEqual('HI\nMOM\n', util.load_file(outfile)) + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_handle_adds_assertions(self, m_subp): + """Any configured snap assertions are provided to add_assertions.""" + assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) + compare_file = self.tmp_path('comparison', dir=self.tmp) + cfg = { + 'snap': {'assertions': [SYSTEM_USER_ASSERTION, ACCOUNT_ASSERTION]}} + wrap_and_call( + 'cloudinit.config.cc_snap', + {'ASSERTIONS_FILE': {'new': assert_file}}, + handle, 'snap', cfg=cfg, cloud=None, log=self.logger, args=None) + content = '\n'.join(cfg['snap']['assertions']) + util.write_file(compare_file, content.encode('utf-8')) + self.assertEqual( + util.load_file(compare_file), util.load_file(assert_file)) + + @mock.patch('cloudinit.config.cc_snap.util.subp') + def test_handle_validates_schema(self, m_subp): + """Any provided configuration is runs validate_cloudconfig_schema.""" + assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) + cfg = {'snap': {'invalid': ''}} # Generates schema warning + wrap_and_call( + 'cloudinit.config.cc_snap', + {'ASSERTIONS_FILE': {'new': assert_file}}, + handle, 'snap', cfg=cfg, cloud=None, log=self.logger, args=None) + self.assertEqual( + "WARNING: Invalid config:\nsnap: Additional properties are not" + " allowed ('invalid' was unexpected)\n", + self.logs.getvalue()) + + +class TestMaybeInstallSquashFuse(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestMaybeInstallSquashFuse, self).setUp() + self.tmp = self.tmp_dir() + + @mock.patch('cloudinit.config.cc_snap.util.is_container') + def test_maybe_install_squashfuse_skips_non_containers(self, m_container): + """maybe_install_squashfuse does nothing when not on a container.""" + m_container.return_value = False + maybe_install_squashfuse(cloud=FakeCloud(None)) + self.assertEqual([mock.call()], m_container.call_args_list) + self.assertEqual('', self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.util.is_container') + def test_maybe_install_squashfuse_raises_install_errors(self, m_container): + """maybe_install_squashfuse logs and raises package install errors.""" + m_container.return_value = True + distro = mock.MagicMock() + distro.update_package_sources.side_effect = RuntimeError( + 'Some apt error') + with self.assertRaises(RuntimeError) as context_manager: + maybe_install_squashfuse(cloud=FakeCloud(distro)) + self.assertEqual('Some apt error', str(context_manager.exception)) + self.assertIn('Package update failed\nTraceback', self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.util.is_container') + def test_maybe_install_squashfuse_raises_update_errors(self, m_container): + """maybe_install_squashfuse logs and raises package update errors.""" + m_container.return_value = True + distro = mock.MagicMock() + distro.update_package_sources.side_effect = RuntimeError( + 'Some apt error') + with self.assertRaises(RuntimeError) as context_manager: + maybe_install_squashfuse(cloud=FakeCloud(distro)) + self.assertEqual('Some apt error', str(context_manager.exception)) + self.assertIn('Package update failed\nTraceback', self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.util.is_container') + def test_maybe_install_squashfuse_happy_path(self, m_container): + """maybe_install_squashfuse logs and raises package install errors.""" + m_container.return_value = True + distro = mock.MagicMock() # No errors raised + maybe_install_squashfuse(cloud=FakeCloud(distro)) + self.assertEqual( + [mock.call()], distro.update_package_sources.call_args_list) + self.assertEqual( + [mock.call(['squashfuse'])], + distro.install_packages.call_args_list) + +# vi: ts=4 expandtab diff --git a/cloudinit/util.py b/cloudinit/util.py index 823d80bf..cae8b196 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1827,7 +1827,8 @@ def subp_blob_in_tempfile(blob, *args, **kwargs): def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, - logstring=False, decode="replace", target=None, update_env=None): + logstring=False, decode="replace", target=None, update_env=None, + status_cb=None): # not supported in cloud-init (yet), for now kept in the call signature # to ease maintaining code shared between cloud-init and curtin @@ -1848,6 +1849,9 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, if target_path(target) != "/": args = ['chroot', target] + list(args) + if status_cb: + command = ' '.join(args) if isinstance(args, list) else args + status_cb('Begin run command: {command}\n'.format(command=command)) if not logstring: LOG.debug(("Running command %s with allowed return codes %s" " (shell=%s, capture=%s)"), args, rcs, shell, capture) @@ -1888,6 +1892,8 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, env=env, shell=shell) (out, err) = sp.communicate(data) except OSError as e: + if status_cb: + status_cb('ERROR: End run command: invalid command provided\n') raise ProcessExecutionError( cmd=args, reason=e, errno=e.errno, stdout="-" if decode else b"-", @@ -1912,9 +1918,14 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, rc = sp.returncode if rc not in rcs: + if status_cb: + status_cb( + 'ERROR: End run command: exit({code})\n'.format(code=rc)) raise ProcessExecutionError(stdout=out, stderr=err, exit_code=rc, cmd=args) + if status_cb: + status_cb('End run command: exit({code})\n'.format(code=rc)) return (out, err) diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index cf2e2409..56a34fab 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -72,7 +72,8 @@ cloud_config_modules: # Emit the cloud config ready event # this can be used by upstart jobs for 'start on cloud-config'. - emit_upstart - - snap_config + - snap + - snap_config # DEPRECATED- Drop in version 18.2 {% endif %} - ssh-import-id - locale @@ -102,7 +103,7 @@ cloud_config_modules: # The modules that run in the 'final' stage cloud_final_modules: {% if variant in ["ubuntu", "unknown", "debian"] %} - - snappy + - snappy # DEPRECATED- Drop in version 18.2 {% endif %} - package-update-upgrade-install {% if variant in ["ubuntu", "unknown", "debian"] %} diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py index 0ea3b6bf..50eb05cf 100644 --- a/doc/rtd/conf.py +++ b/doc/rtd/conf.py @@ -29,6 +29,7 @@ project = 'Cloud-Init' extensions = [ 'sphinx.ext.intersphinx', 'sphinx.ext.autodoc', + 'sphinx.ext.autosectionlabel', 'sphinx.ext.viewcode', ] diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index 7b146751..a0f68129 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -45,6 +45,7 @@ Modules .. automodule:: cloudinit.config.cc_seed_random .. automodule:: cloudinit.config.cc_set_hostname .. automodule:: cloudinit.config.cc_set_passwords +.. automodule:: cloudinit.config.cc_snap .. automodule:: cloudinit.config.cc_snappy .. automodule:: cloudinit.config.cc_snap_config .. automodule:: cloudinit.config.cc_spacewalk diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml index d8bc170f..c7dcbe83 100644 --- a/tests/cloud_tests/releases.yaml +++ b/tests/cloud_tests/releases.yaml @@ -30,6 +30,9 @@ default_release_config: mirror_url: https://cloud-images.ubuntu.com/daily mirror_dir: '/srv/citest/images' keyring: /usr/share/keyrings/ubuntu-cloudimage-keyring.gpg + # The OS version formatted as Major.Minor is used to compare releases + version: null # Each release needs to define this, for example 16.04 + ec2: # Choose from: [ebs, instance-store] root-store: ebs diff --git a/tests/cloud_tests/testcases.yaml b/tests/cloud_tests/testcases.yaml index 8e0fb62f..a3e29900 100644 --- a/tests/cloud_tests/testcases.yaml +++ b/tests/cloud_tests/testcases.yaml @@ -15,6 +15,9 @@ base_test_data: instance-id: | #!/bin/sh cat /run/cloud-init/.instance-id + instance-data.json: | + #!/bin/sh + cat /run/cloud-init/instance-data.json result.json: | #!/bin/sh cat /run/cloud-init/result.json diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py index a29a0928..bd548f5a 100644 --- a/tests/cloud_tests/testcases/__init__.py +++ b/tests/cloud_tests/testcases/__init__.py @@ -7,6 +7,8 @@ import inspect import unittest from unittest.util import strclass +from cloudinit.util import read_conf + from tests.cloud_tests import config from tests.cloud_tests.testcases.base import CloudTestCase as base_test @@ -48,6 +50,7 @@ def get_suite(test_name, data, conf): def setUpClass(cls): cls.data = data cls.conf = conf + cls.release_conf = read_conf(config.RELEASES_CONF)['releases'] suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(tmp)) diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 20e95955..324c7c91 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -4,10 +4,14 @@ import crypt import json +import re import unittest + from cloudinit import util as c_util +SkipTest = unittest.SkipTest + class CloudTestCase(unittest.TestCase): """Base test class for verifiers.""" @@ -16,6 +20,43 @@ class CloudTestCase(unittest.TestCase): data = {} conf = None _cloud_config = None + release_conf = {} # The platform's os release configuration + + expected_warnings = () # Subclasses set to ignore expected WARN logs + + @property + def os_cfg(self): + return self.release_conf[self.os_name]['default'] + + def is_distro(self, distro_name): + return self.os_cfg['os'] == distro_name + + def os_version_cmp(self, cmp_version): + """Compare the version of the test to comparison_version. + + @param: cmp_version: Either a float or a string representing + a release os from releases.yaml (e.g. centos66) + + @return: -1 when version < cmp_version, 0 when version=cmp_version and + 1 when version > cmp_version. + """ + version = self.release_conf[self.os_name]['default']['version'] + if isinstance(cmp_version, str): + cmp_version = self.release_conf[cmp_version]['default']['version'] + if version < cmp_version: + return -1 + elif version == cmp_version: + return 0 + else: + return 1 + + @property + def os_name(self): + return self.data.get('os_name', 'UNKNOWN') + + @property + def platform(self): + return self.data.get('platform', 'UNKNOWN') @property def cloud_config(self): @@ -72,12 +113,134 @@ class CloudTestCase(unittest.TestCase): self.assertEqual(len(result['errors']), 0) def test_no_warnings_in_log(self): - """Warnings should not be found in the log.""" + """Unexpected warnings should not be found in the log.""" + warnings = [ + l for l in self.get_data_file('cloud-init.log').splitlines() + if 'WARN' in l] + joined_warnings = '\n'.join(warnings) + for expected_warning in self.expected_warnings: + self.assertIn( + expected_warning, joined_warnings, + msg="Did not find %s in cloud-init.log" % expected_warning) + # Prune expected from discovered warnings + warnings = [w for w in warnings if expected_warning not in w] + self.assertEqual( + [], warnings, msg="'WARN' found inside cloud-init.log") + + def test_instance_data_json_ec2(self): + """Validate instance-data.json content by ec2 platform. + + This content is sourced by snapd when determining snapstore endpoints. + We validate expected values per cloud type to ensure we don't break + snapd. + """ + if self.platform != 'ec2': + raise SkipTest( + 'Skipping ec2 instance-data.json on %s' % self.platform) + out = self.get_data_file('instance-data.json') + if not out: + if self.is_distro('ubuntu') and self.os_version_cmp('bionic') >= 0: + raise AssertionError( + 'No instance-data.json found on %s' % self.os_name) + raise SkipTest( + 'Skipping instance-data.json test.' + ' OS: %s not bionic or newer' % self.os_name) + instance_data = json.loads(out) + self.assertEqual( + ['ds/user-data'], instance_data['base64-encoded-keys']) + ds = instance_data.get('ds', {}) + macs = ds.get('network', {}).get('interfaces', {}).get('macs', {}) + if not macs: + raise AssertionError('No network data from EC2 meta-data') + # Check meta-data items we depend on + expected_net_keys = [ + 'public-ipv4s', 'ipv4-associations', 'local-hostname', + 'public-hostname'] + for mac, mac_data in macs.items(): + for key in expected_net_keys: + self.assertIn(key, mac_data) + self.assertIsNotNone( + ds.get('placement', {}).get('availability-zone'), + 'Could not determine EC2 Availability zone placement') + ds = instance_data.get('ds', {}) + v1_data = instance_data.get('v1', {}) + self.assertIsNotNone( + v1_data['availability-zone'], 'expected ec2 availability-zone') + self.assertEqual('aws', v1_data['cloud-name']) + self.assertIn('i-', v1_data['instance-id']) + self.assertIn('ip-', v1_data['local-hostname']) + self.assertIsNotNone(v1_data['region'], 'expected ec2 region') + + def test_instance_data_json_lxd(self): + """Validate instance-data.json content by lxd platform. + + This content is sourced by snapd when determining snapstore endpoints. + We validate expected values per cloud type to ensure we don't break + snapd. + """ + if self.platform != 'lxd': + raise SkipTest( + 'Skipping lxd instance-data.json on %s' % self.platform) + out = self.get_data_file('instance-data.json') + if not out: + if self.is_distro('ubuntu') and self.os_version_cmp('bionic') >= 0: + raise AssertionError( + 'No instance-data.json found on %s' % self.os_name) + raise SkipTest( + 'Skipping instance-data.json test.' + ' OS: %s not bionic or newer' % self.os_name) + instance_data = json.loads(out) + v1_data = instance_data.get('v1', {}) + self.assertEqual( + ['ds/user-data', 'ds/vendor-data'], + sorted(instance_data['base64-encoded-keys'])) + self.assertEqual('nocloud', v1_data['cloud-name']) + self.assertIsNone( + v1_data['availability-zone'], + 'found unexpected lxd availability-zone %s' % + v1_data['availability-zone']) + self.assertIn('cloud-test', v1_data['instance-id']) + self.assertIn('cloud-test', v1_data['local-hostname']) + self.assertIsNone( + v1_data['region'], + 'found unexpected lxd region %s' % v1_data['region']) + + def test_instance_data_json_kvm(self): + """Validate instance-data.json content by nocloud-kvm platform. + + This content is sourced by snapd when determining snapstore endpoints. + We validate expected values per cloud type to ensure we don't break + snapd. + """ + if self.platform != 'nocloud-kvm': + raise SkipTest( + 'Skipping nocloud-kvm instance-data.json on %s' % + self.platform) + out = self.get_data_file('instance-data.json') + if not out: + if self.is_distro('ubuntu') and self.os_version_cmp('bionic') >= 0: + raise AssertionError( + 'No instance-data.json found on %s' % self.os_name) + raise SkipTest( + 'Skipping instance-data.json test.' + ' OS: %s not bionic or newer' % self.os_name) + instance_data = json.loads(out) + v1_data = instance_data.get('v1', {}) self.assertEqual( - [], - [l for l in self.get_data_file('cloud-init.log').splitlines() - if 'WARN' in l], - msg="'WARN' found inside cloud-init.log") + ['ds/user-data'], instance_data['base64-encoded-keys']) + self.assertEqual('nocloud', v1_data['cloud-name']) + self.assertIsNone( + v1_data['availability-zone'], + 'found unexpected kvm availability-zone %s' % + v1_data['availability-zone']) + self.assertIsNotNone( + re.match('[\da-f]{8}(-[\da-f]{4}){3}-[\da-f]{12}', + v1_data['instance-id']), + 'kvm instance-id is not a UUID: %s' % v1_data['instance-id']) + self.assertIn('ubuntu', v1_data['local-hostname']) + self.assertIsNone( + v1_data['region'], + 'found unexpected lxd region %s' % v1_data['region']) class PasswordListTest(CloudTestCase): diff --git a/tests/cloud_tests/testcases/main/command_output_simple.py b/tests/cloud_tests/testcases/main/command_output_simple.py index 857881cb..80a2c8d7 100644 --- a/tests/cloud_tests/testcases/main/command_output_simple.py +++ b/tests/cloud_tests/testcases/main/command_output_simple.py @@ -7,6 +7,8 @@ from tests.cloud_tests.testcases import base class TestCommandOutputSimple(base.CloudTestCase): """Test functionality of simple output redirection.""" + expected_warnings = ('Stdout, stderr changing to',) + def test_output_file(self): """Ensure that the output file is not empty and has all stages.""" data = self.get_data_file('cloud-init-test-output') @@ -15,20 +17,5 @@ class TestCommandOutputSimple(base.CloudTestCase): data.splitlines()[-1].strip()) # TODO: need to test that all stages redirected here - def test_no_warnings_in_log(self): - """Warnings should not be found in the log. - - This class redirected stderr and stdout, so it expects to find - a warning in cloud-init.log to that effect.""" - redirect_msg = 'Stdout, stderr changing to' - warnings = [ - l for l in self.get_data_file('cloud-init.log').splitlines() - if 'WARN' in l] - self.assertEqual( - [], [w for w in warnings if redirect_msg not in w], - msg="'WARN' found inside cloud-init.log") - self.assertEqual( - 1, len(warnings), - msg="Did not find %s in cloud-init.log" % redirect_msg) # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/snap.py b/tests/cloud_tests/testcases/modules/snap.py new file mode 100644 index 00000000..ff68abbe --- /dev/null +++ b/tests/cloud_tests/testcases/modules/snap.py @@ -0,0 +1,16 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""cloud-init Integration Test Verify Script""" +from tests.cloud_tests.testcases import base + + +class TestSnap(base.CloudTestCase): + """Test snap module""" + + def test_snappy_version(self): + """Expect hello-world and core snaps are installed.""" + out = self.get_data_file('snaplist') + self.assertIn('core', out) + self.assertIn('hello-world', out) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/snap.yaml b/tests/cloud_tests/testcases/modules/snap.yaml new file mode 100644 index 00000000..44043f31 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/snap.yaml @@ -0,0 +1,18 @@ +# +# Install snappy +# +required_features: + - snap +cloud_config: | + #cloud-config + package_update: true + snap: + squashfuse_in_container: true + commands: + - snap install hello-world +collect_scripts: + snaplist: | + #!/bin/bash + snap list + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/snappy.py b/tests/cloud_tests/testcases/modules/snappy.py index b92271c1..7d17fc5b 100644 --- a/tests/cloud_tests/testcases/modules/snappy.py +++ b/tests/cloud_tests/testcases/modules/snappy.py @@ -7,6 +7,8 @@ from tests.cloud_tests.testcases import base class TestSnappy(base.CloudTestCase): """Test snappy module""" + expected_warnings = ('DEPRECATION',) + def test_snappy_version(self): """Test snappy version output""" out = self.get_data_file('snapd') diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py index 2a9fd520..5a68a484 100644 --- a/tests/cloud_tests/verify.py +++ b/tests/cloud_tests/verify.py @@ -8,13 +8,16 @@ import unittest from tests.cloud_tests import (config, LOG, util, testcases) -def verify_data(base_dir, tests): +def verify_data(data_dir, platform, os_name, tests): """Verify test data is correct. - @param base_dir: base directory for data + @param data_dir: top level directory for all tests + @param platform: The platform name we for this test data (e.g. lxd) + @param os_name: The operating system under test (xenial, artful, etc.). @param tests: list of test names @return_value: {<test_name>: {passed: True/False, failures: []}} """ + base_dir = os.sep.join((data_dir, platform, os_name)) runner = unittest.TextTestRunner(verbosity=util.current_verbosity()) res = {} for test_name in tests: @@ -26,7 +29,7 @@ def verify_data(base_dir, tests): cloud_conf = test_conf['cloud_config'] # load script outputs - data = {} + data = {'platform': platform, 'os_name': os_name} test_dir = os.path.join(base_dir, test_name) for script_name in os.listdir(test_dir): with open(os.path.join(test_dir, script_name), 'rb') as fp: @@ -73,7 +76,7 @@ def verify(args): # run test res[platform][os_name] = verify_data( - os.sep.join((args.data_dir, platform, os_name)), + args.data_dir, platform, os_name, tests[platform][os_name]) # handle results diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 1ecb6c68..9b50ee79 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -26,6 +26,7 @@ class GetSchemaTest(CiTestCase): 'cc_ntp', 'cc_resizefs', 'cc_runcmd', + 'cc_snap', 'cc_zypper_add_repo' ], [subschema['id'] for subschema in schema['allOf']]) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 499e7c9f..67d9607d 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -785,6 +785,39 @@ class TestSubp(helpers.CiTestCase): decode=False) self.assertEqual(self.utf8_valid, out) + def test_bogus_command_logs_status_messages(self): + """status_cb gets status messages logs on bogus commands provided.""" + logs = [] + + def status_cb(log): + logs.append(log) + + with self.assertRaises(util.ProcessExecutionError): + util.subp([self.bogus_command], status_cb=status_cb) + + expected = [ + 'Begin run command: {cmd}\n'.format(cmd=self.bogus_command), + 'ERROR: End run command: invalid command provided\n'] + self.assertEqual(expected, logs) + + def test_command_logs_exit_codes_to_status_cb(self): + """status_cb gets status messages containing command exit code.""" + logs = [] + + def status_cb(log): + logs.append(log) + + with self.assertRaises(util.ProcessExecutionError): + util.subp(['ls', '/I/dont/exist'], status_cb=status_cb) + util.subp(['ls'], status_cb=status_cb) + + expected = [ + 'Begin run command: ls /I/dont/exist\n', + 'ERROR: End run command: exit(2)\n', + 'Begin run command: ls\n', + 'End run command: exit(0)\n'] + self.assertEqual(expected, logs) + class TestEncode(helpers.TestCase): """Test the encoding functions""" |