Reorganize unit test locations under tests/unittests (#1126)

This attempts to standardize unit test file location under test/unittests/
such that any source file located at cloudinit/path/to/file.py may have a
corresponding unit test file at test/unittests/path/to/test_file.py.

Noteworthy Comments:
====================
Four different duplicate test files existed:
test_{gpg,util,cc_mounts,cc_resolv_conf}.py
Each of these duplicate file pairs has been merged together. This is a
break in git history for these files.

The test suite appears to have a dependency on test order. Changing test
order causes some tests to fail. This should be rectified, but for now
some tests have been modified in
tests/unittests/config/test_set_passwords.py.

A helper class name starts with "Test" which causes pytest to try
executing it as a test case, which then throws warnings "due to Class
having __init__()".  Silence by changing the name of the class.

# helpers.py is imported in many test files, import paths change
cloudinit/tests/helpers.py -> tests/unittests/helpers.py

# Move directories:
cloudinit/distros/tests -> tests/unittests/distros
cloudinit/cmd/devel/tests -> tests/unittests/cmd/devel
cloudinit/cmd/tests -> tests/unittests/cmd/
cloudinit/sources/helpers/tests -> tests/unittests/sources/helpers
cloudinit/sources/tests -> tests/unittests/sources
cloudinit/net/tests -> tests/unittests/net
cloudinit/config/tests -> tests/unittests/config
cloudinit/analyze/tests/ -> tests/unittests/analyze/

# Standardize tests already in tests/unittests/
test_datasource -> sources
test_distros -> distros
test_vmware -> sources/vmware
test_handler -> config        # this contains cloudconfig module tests
test_runs -> runs

3 files changed, 0 insertions, 311 deletions

diff --git a/cloudinit/cmd/devel/tests/__init__.py b/cloudinit/cmd/devel/tests/__init__.py
deleted file mode 100644
index e69de29b..00000000
--- a/cloudinit/cmd/devel/tests/__init__.py
+++ /dev/null
diff --git a/cloudinit/cmd/devel/tests/test_logs.py b/cloudinit/cmd/devel/tests/test_logs.py
deleted file mode 100644
index ddfd58e1..00000000
--- a/cloudinit/cmd/devel/tests/test_logs.py
+++ /dev/null
@@ -1,167 +0,0 @@
-# This file is part of cloud-init. See LICENSE file for license information.
-
-from datetime import datetime
-import os
-from io import StringIO
-
-from cloudinit.cmd.devel import logs
-from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE
-from cloudinit.tests.helpers import (
-    FilesystemMockingTestCase, mock, wrap_and_call)
-from cloudinit.subp import subp
-from cloudinit.util import ensure_dir, load_file, write_file
-
-
-@mock.patch('cloudinit.cmd.devel.logs.os.getuid')
-class TestCollectLogs(FilesystemMockingTestCase):
-
-    def setUp(self):
-        super(TestCollectLogs, self).setUp()
-        self.new_root = self.tmp_dir()
-        self.run_dir = self.tmp_path('run', self.new_root)
-
-    def test_collect_logs_with_userdata_requires_root_user(self, m_getuid):
-        """collect-logs errors when non-root user collects userdata ."""
-        m_getuid.return_value = 100  # non-root
-        output_tarfile = self.tmp_path('logs.tgz')
-        with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
-            self.assertEqual(
-                1, logs.collect_logs(output_tarfile, include_userdata=True))
-        self.assertEqual(
-            'To include userdata, root user is required.'
-            ' Try sudo cloud-init collect-logs\n',
-            m_stderr.getvalue())
-
-    def test_collect_logs_creates_tarfile(self, m_getuid):
-        """collect-logs creates a tarfile with all related cloud-init info."""
-        m_getuid.return_value = 100
-        log1 = self.tmp_path('cloud-init.log', self.new_root)
-        write_file(log1, 'cloud-init-log')
-        log2 = self.tmp_path('cloud-init-output.log', self.new_root)
-        write_file(log2, 'cloud-init-output-log')
-        ensure_dir(self.run_dir)
-        write_file(self.tmp_path('results.json', self.run_dir), 'results')
-        write_file(self.tmp_path(INSTANCE_JSON_SENSITIVE_FILE, self.run_dir),
-                   'sensitive')
-        output_tarfile = self.tmp_path('logs.tgz')
-
-        date = datetime.utcnow().date().strftime('%Y-%m-%d')
-        date_logdir = 'cloud-init-logs-{0}'.format(date)
-
-        version_out = '/usr/bin/cloud-init 18.2fake\n'
-        expected_subp = {
-            ('dpkg-query', '--show', "-f=${Version}\n", 'cloud-init'):
-                '0.7fake\n',
-            ('cloud-init', '--version'): version_out,
-            ('dmesg',): 'dmesg-out\n',
-            ('journalctl', '--boot=0', '-o', 'short-precise'): 'journal-out\n',
-            ('tar', 'czvf', output_tarfile, date_logdir): ''
-        }
-
-        def fake_subp(cmd):
-            cmd_tuple = tuple(cmd)
-            if cmd_tuple not in expected_subp:
-                raise AssertionError(
-                    'Unexpected command provided to subp: {0}'.format(cmd))
-            if cmd == ['tar', 'czvf', output_tarfile, date_logdir]:
-                subp(cmd)  # Pass through tar cmd so we can check output
-            return expected_subp[cmd_tuple], ''
-
-        fake_stderr = mock.MagicMock()
-
-        wrap_and_call(
-            'cloudinit.cmd.devel.logs',
-            {'subp': {'side_effect': fake_subp},
-             'sys.stderr': {'new': fake_stderr},
-             'CLOUDINIT_LOGS': {'new': [log1, log2]},
-             'CLOUDINIT_RUN_DIR': {'new': self.run_dir}},
-            logs.collect_logs, output_tarfile, include_userdata=False)
-        # unpack the tarfile and check file contents
-        subp(['tar', 'zxvf', output_tarfile, '-C', self.new_root])
-        out_logdir = self.tmp_path(date_logdir, self.new_root)
-        self.assertFalse(
-            os.path.exists(
-                os.path.join(out_logdir, 'run', 'cloud-init',
-                             INSTANCE_JSON_SENSITIVE_FILE)),
-            'Unexpected file found: %s' % INSTANCE_JSON_SENSITIVE_FILE)
-        self.assertEqual(
-            '0.7fake\n',
-            load_file(os.path.join(out_logdir, 'dpkg-version')))
-        self.assertEqual(version_out,
-                         load_file(os.path.join(out_logdir, 'version')))
-        self.assertEqual(
-            'cloud-init-log',
-            load_file(os.path.join(out_logdir, 'cloud-init.log')))
-        self.assertEqual(
-            'cloud-init-output-log',
-            load_file(os.path.join(out_logdir, 'cloud-init-output.log')))
-        self.assertEqual(
-            'dmesg-out\n',
-            load_file(os.path.join(out_logdir, 'dmesg.txt')))
-        self.assertEqual(
-            'journal-out\n',
-            load_file(os.path.join(out_logdir, 'journal.txt')))
-        self.assertEqual(
-            'results',
-            load_file(
-                os.path.join(out_logdir, 'run', 'cloud-init', 'results.json')))
-        fake_stderr.write.assert_any_call('Wrote %s\n' % output_tarfile)
-
-    def test_collect_logs_includes_optional_userdata(self, m_getuid):
-        """collect-logs include userdata when --include-userdata is set."""
-        m_getuid.return_value = 0
-        log1 = self.tmp_path('cloud-init.log', self.new_root)
-        write_file(log1, 'cloud-init-log')
-        log2 = self.tmp_path('cloud-init-output.log', self.new_root)
-        write_file(log2, 'cloud-init-output-log')
-        userdata = self.tmp_path('user-data.txt', self.new_root)
-        write_file(userdata, 'user-data')
-        ensure_dir(self.run_dir)
-        write_file(self.tmp_path('results.json', self.run_dir), 'results')
-        write_file(self.tmp_path(INSTANCE_JSON_SENSITIVE_FILE, self.run_dir),
-                   'sensitive')
-        output_tarfile = self.tmp_path('logs.tgz')
-
-        date = datetime.utcnow().date().strftime('%Y-%m-%d')
-        date_logdir = 'cloud-init-logs-{0}'.format(date)
-
-        version_out = '/usr/bin/cloud-init 18.2fake\n'
-        expected_subp = {
-            ('dpkg-query', '--show', "-f=${Version}\n", 'cloud-init'):
-                '0.7fake',
-            ('cloud-init', '--version'): version_out,
-            ('dmesg',): 'dmesg-out\n',
-            ('journalctl', '--boot=0', '-o', 'short-precise'): 'journal-out\n',
-            ('tar', 'czvf', output_tarfile, date_logdir): ''
-        }
-
-        def fake_subp(cmd):
-            cmd_tuple = tuple(cmd)
-            if cmd_tuple not in expected_subp:
-                raise AssertionError(
-                    'Unexpected command provided to subp: {0}'.format(cmd))
-            if cmd == ['tar', 'czvf', output_tarfile, date_logdir]:
-                subp(cmd)  # Pass through tar cmd so we can check output
-            return expected_subp[cmd_tuple], ''
-
-        fake_stderr = mock.MagicMock()
-
-        wrap_and_call(
-            'cloudinit.cmd.devel.logs',
-            {'subp': {'side_effect': fake_subp},
-             'sys.stderr': {'new': fake_stderr},
-             'CLOUDINIT_LOGS': {'new': [log1, log2]},
-             'CLOUDINIT_RUN_DIR': {'new': self.run_dir},
-             'USER_DATA_FILE': {'new': userdata}},
-            logs.collect_logs, output_tarfile, include_userdata=True)
-        # unpack the tarfile and check file contents
-        subp(['tar', 'zxvf', output_tarfile, '-C', self.new_root])
-        out_logdir = self.tmp_path(date_logdir, self.new_root)
-        self.assertEqual(
-            'user-data',
-            load_file(os.path.join(out_logdir, 'user-data.txt')))
-        self.assertEqual(
-            'sensitive',
-            load_file(os.path.join(out_logdir, 'run', 'cloud-init',
-                                   INSTANCE_JSON_SENSITIVE_FILE)))
-        fake_stderr.write.assert_any_call('Wrote %s\n' % output_tarfile)
diff --git a/cloudinit/cmd/devel/tests/test_render.py b/cloudinit/cmd/devel/tests/test_render.py
deleted file mode 100644
index a7fcf2ce..00000000
--- a/cloudinit/cmd/devel/tests/test_render.py
+++ /dev/null
@@ -1,144 +0,0 @@
-# This file is part of cloud-init. See LICENSE file for license information.
-
-import os
-from io import StringIO
-
-from collections import namedtuple
-from cloudinit.cmd.devel import render
-from cloudinit.helpers import Paths
-from cloudinit.sources import INSTANCE_JSON_FILE, INSTANCE_JSON_SENSITIVE_FILE
-from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJinja
-from cloudinit.util import ensure_dir, write_file
-
-
-class TestRender(CiTestCase):
-
-    with_logs = True
-
-    args = namedtuple('renderargs', 'user_data instance_data debug')
-
-    def setUp(self):
-        super(TestRender, self).setUp()
-        self.tmp = self.tmp_dir()
-
-    def test_handle_args_error_on_missing_user_data(self):
-        """When user_data file path does not exist, log an error."""
-        absent_file = self.tmp_path('user-data', dir=self.tmp)
-        instance_data = self.tmp_path('instance-data', dir=self.tmp)
-        write_file(instance_data, '{}')
-        args = self.args(
-            user_data=absent_file, instance_data=instance_data, debug=False)
-        with mock.patch('sys.stderr', new_callable=StringIO):
-            self.assertEqual(1, render.handle_args('anyname', args))
-        self.assertIn(
-            'Missing user-data file: %s' % absent_file,
-            self.logs.getvalue())
-
-    def test_handle_args_error_on_missing_instance_data(self):
-        """When instance_data file path does not exist, log an error."""
-        user_data = self.tmp_path('user-data', dir=self.tmp)
-        absent_file = self.tmp_path('instance-data', dir=self.tmp)
-        args = self.args(
-            user_data=user_data, instance_data=absent_file, debug=False)
-        with mock.patch('sys.stderr', new_callable=StringIO):
-            self.assertEqual(1, render.handle_args('anyname', args))
-        self.assertIn(
-            'Missing instance-data.json file: %s' % absent_file,
-            self.logs.getvalue())
-
-    def test_handle_args_defaults_instance_data(self):
-        """When no instance_data argument, default to configured run_dir."""
-        user_data = self.tmp_path('user-data', dir=self.tmp)
-        run_dir = self.tmp_path('run_dir', dir=self.tmp)
-        ensure_dir(run_dir)
-        paths = Paths({'run_dir': run_dir})
-        self.add_patch('cloudinit.cmd.devel.render.read_cfg_paths', 'm_paths')
-        self.m_paths.return_value = paths
-        args = self.args(
-            user_data=user_data, instance_data=None, debug=False)
-        with mock.patch('sys.stderr', new_callable=StringIO):
-            self.assertEqual(1, render.handle_args('anyname', args))
-        json_file = os.path.join(run_dir, INSTANCE_JSON_FILE)
-        self.assertIn(
-            'Missing instance-data.json file: %s' % json_file,
-            self.logs.getvalue())
-
-    def test_handle_args_root_fallback_from_sensitive_instance_data(self):
-        """When root user defaults to sensitive.json."""
-        user_data = self.tmp_path('user-data', dir=self.tmp)
-        run_dir = self.tmp_path('run_dir', dir=self.tmp)
-        ensure_dir(run_dir)
-        paths = Paths({'run_dir': run_dir})
-        self.add_patch('cloudinit.cmd.devel.render.read_cfg_paths', 'm_paths')
-        self.m_paths.return_value = paths
-        args = self.args(
-            user_data=user_data, instance_data=None, debug=False)
-        with mock.patch('sys.stderr', new_callable=StringIO):
-            with mock.patch('os.getuid') as m_getuid:
-                m_getuid.return_value = 0
-                self.assertEqual(1, render.handle_args('anyname', args))
-        json_file = os.path.join(run_dir, INSTANCE_JSON_FILE)
-        json_sensitive = os.path.join(run_dir, INSTANCE_JSON_SENSITIVE_FILE)
-        self.assertIn(
-            'WARNING: Missing root-readable %s. Using redacted %s' % (
-                json_sensitive, json_file), self.logs.getvalue())
-        self.assertIn(
-            'ERROR: Missing instance-data.json file: %s' % json_file,
-            self.logs.getvalue())
-
-    def test_handle_args_root_uses_sensitive_instance_data(self):
-        """When root user, and no instance-data arg, use sensitive.json."""
-        user_data = self.tmp_path('user-data', dir=self.tmp)
-        write_file(user_data, '##template: jinja\nrendering: {{ my_var }}')
-        run_dir = self.tmp_path('run_dir', dir=self.tmp)
-        ensure_dir(run_dir)
-        json_sensitive = os.path.join(run_dir, INSTANCE_JSON_SENSITIVE_FILE)
-        write_file(json_sensitive, '{"my-var": "jinja worked"}')
-        paths = Paths({'run_dir': run_dir})
-        self.add_patch('cloudinit.cmd.devel.render.read_cfg_paths', 'm_paths')
-        self.m_paths.return_value = paths
-        args = self.args(
-            user_data=user_data, instance_data=None, debug=False)
-        with mock.patch('sys.stderr', new_callable=StringIO):
-            with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
-                with mock.patch('os.getuid') as m_getuid:
-                    m_getuid.return_value = 0
-                    self.assertEqual(0, render.handle_args('anyname', args))
-        self.assertIn('rendering: jinja worked', m_stdout.getvalue())
-
-    @skipUnlessJinja()
-    def test_handle_args_renders_instance_data_vars_in_template(self):
-        """If user_data file is a jinja template render instance-data vars."""
-        user_data = self.tmp_path('user-data', dir=self.tmp)
-        write_file(user_data, '##template: jinja\nrendering: {{ my_var }}')
-        instance_data = self.tmp_path('instance-data', dir=self.tmp)
-        write_file(instance_data, '{"my-var": "jinja worked"}')
-        args = self.args(
-            user_data=user_data, instance_data=instance_data, debug=True)
-        with mock.patch('sys.stderr', new_callable=StringIO) as m_console_err:
-            with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
-                self.assertEqual(0, render.handle_args('anyname', args))
-        self.assertIn(
-            'DEBUG: Converted jinja variables\n{', self.logs.getvalue())
-        self.assertIn(
-            'DEBUG: Converted jinja variables\n{', m_console_err.getvalue())
-        self.assertEqual('rendering: jinja worked', m_stdout.getvalue())
-
-    @skipUnlessJinja()
-    def test_handle_args_warns_and_gives_up_on_invalid_jinja_operation(self):
-        """If user_data file has invalid jinja operations log warnings."""
-        user_data = self.tmp_path('user-data', dir=self.tmp)
-        write_file(user_data, '##template: jinja\nrendering: {{ my-var }}')
-        instance_data = self.tmp_path('instance-data', dir=self.tmp)
-        write_file(instance_data, '{"my-var": "jinja worked"}')
-        args = self.args(
-            user_data=user_data, instance_data=instance_data, debug=True)
-        with mock.patch('sys.stderr', new_callable=StringIO):
-            self.assertEqual(1, render.handle_args('anyname', args))
-        self.assertIn(
-            'WARNING: Ignoring jinja template for %s: Undefined jinja'
-            ' variable: "my-var". Jinja tried subtraction. Perhaps you meant'
-            ' "my_var"?' % user_data,
-            self.logs.getvalue())
-
-# vi: ts=4 expandtab