cloudinit: drop dependencies on unittest2 and contextlib2 (#322)

These libraries provide backports of Python 3's stdlib components to Python 2. As we only support Python 3, we can simply use the stdlib now. This pull request does the following:

* removes some unneeded compatibility code for the old spelling of `assertRaisesRegex`
* replaces invocations of the Python 2-only `assertItemsEqual` with its new name, `assertCountEqual`
* replaces all usage of `unittest2` with `unittest`
* replaces all usage of `contextlib2` with `contextlib`
* drops `unittest2` and `contextlib2` from requirements files and tox.ini

It also rewrites some `test_azure` helpers to use bare asserts. We were seeing a strange error in xenial builds of this branch which appear to be stemming from the AssertionError that pytest produces being _different_ from the standard AssertionError.  This means that the modified helpers weren't behaving correctly, because they weren't catching AssertionErrors as one would expect. (I believe this is related, in some way, to https://github.com/pytest-dev/pytest/issues/645, but the only version of pytest where we're affected is so far in the past that it's not worth pursuing it any further as we have a workaround.)

12 files changed, 38 insertions, 46 deletions

diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py
index 6bb39f77..e8c371ca 100644
--- a/tests/cloud_tests/testcases/__init__.py
+++ b/tests/cloud_tests/testcases/__init__.py
@@ -4,7 +4,7 @@
 
 import importlib
 import inspect
-import unittest2
+import unittest
 
 from cloudinit.util import read_conf
 
@@ -48,7 +48,7 @@ def get_test_class(test_name, test_data, test_conf):
 
         def __str__(self):
             return "%s (%s)" % (self._testMethodName,
-                                unittest2.util.strclass(self._realclass))
+                                unittest.util.strclass(self._realclass))
 
         @classmethod
         def setUpClass(cls):
@@ -62,9 +62,9 @@ def get_suite(test_name, data, conf):
 
     @return_value: a test suite
     """
-    suite = unittest2.TestSuite()
+    suite = unittest.TestSuite()
     suite.addTest(
-        unittest2.defaultTestLoader.loadTestsFromTestCase(
+        unittest.defaultTestLoader.loadTestsFromTestCase(
             get_test_class(test_name, data, conf)))
     return suite
 
diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py
index b0dfc144..7b67f54e 100644
--- a/tests/cloud_tests/testcases/base.py
+++ b/tests/cloud_tests/testcases/base.py
@@ -5,15 +5,15 @@
 import crypt
 import json
 import re
-import unittest2
+import unittest
 
 
 from cloudinit import util as c_util
 
-SkipTest = unittest2.SkipTest
+SkipTest = unittest.SkipTest
 
 
-class CloudTestCase(unittest2.TestCase):
+class CloudTestCase(unittest.TestCase):
     """Base test class for verifiers."""
 
     # data gets populated in get_suite.setUpClass
@@ -172,7 +172,7 @@ class CloudTestCase(unittest2.TestCase):
                 'Skipping instance-data.json test.'
                 ' OS: %s not bionic or newer' % self.os_name)
         instance_data = json.loads(out)
-        self.assertItemsEqual(['merged_cfg'], instance_data['sensitive_keys'])
+        self.assertCountEqual(['merged_cfg'], instance_data['sensitive_keys'])
         ds = instance_data.get('ds', {})
         v1_data = instance_data.get('v1', {})
         metadata = ds.get('meta-data', {})
@@ -237,7 +237,7 @@ class CloudTestCase(unittest2.TestCase):
                 ' OS: %s not bionic or newer' % self.os_name)
         instance_data = json.loads(out)
         v1_data = instance_data.get('v1', {})
-        self.assertItemsEqual([], sorted(instance_data['base64_encoded_keys']))
+        self.assertCountEqual([], sorted(instance_data['base64_encoded_keys']))
         self.assertEqual('unknown', v1_data['cloud_name'])
         self.assertEqual('lxd', v1_data['platform'])
         self.assertEqual(
@@ -291,7 +291,7 @@ class CloudTestCase(unittest2.TestCase):
                 ' OS: %s not bionic or newer' % self.os_name)
         instance_data = json.loads(out)
         v1_data = instance_data.get('v1', {})
-        self.assertItemsEqual([], instance_data['base64_encoded_keys'])
+        self.assertCountEqual([], instance_data['base64_encoded_keys'])
         self.assertEqual('unknown', v1_data['cloud_name'])
         self.assertEqual('nocloud', v1_data['platform'])
         subplatform = v1_data['subplatform']
diff --git a/tests/cloud_tests/testcases/modules/ntp_chrony.py b/tests/cloud_tests/testcases/modules/ntp_chrony.py
index 0f4c3d08..7d341773 100644
--- a/tests/cloud_tests/testcases/modules/ntp_chrony.py
+++ b/tests/cloud_tests/testcases/modules/ntp_chrony.py
@@ -1,7 +1,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 """cloud-init Integration Test Verify Script."""
-import unittest2
+import unittest
 
 from tests.cloud_tests.testcases import base
 
@@ -13,7 +13,7 @@ class TestNtpChrony(base.CloudTestCase):
         """Skip this suite of tests on lxd and artful or older."""
         if self.platform == 'lxd':
             if self.is_distro('ubuntu') and self.os_version_cmp('artful') <= 0:
-                raise unittest2.SkipTest(
+                raise unittest.SkipTest(
                     'No support for chrony on containers <= artful.'
                     ' LP: #1589780')
         return super(TestNtpChrony, self).setUp()
diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py
index 7018f4d5..0295af40 100644
--- a/tests/cloud_tests/verify.py
+++ b/tests/cloud_tests/verify.py
@@ -3,7 +3,7 @@
 """Verify test results."""
 
 import os
-import unittest2
+import unittest
 
 from tests.cloud_tests import (config, LOG, util, testcases)
 
@@ -18,7 +18,7 @@ def verify_data(data_dir, platform, os_name, tests):
     @return_value: {<test_name>: {passed: True/False, failures: []}}
     """
     base_dir = os.sep.join((data_dir, platform, os_name))
-    runner = unittest2.TextTestRunner(verbosity=util.current_verbosity())
+    runner = unittest.TextTestRunner(verbosity=util.current_verbosity())
     res = {}
     for test_name in tests:
         LOG.debug('verifying test data for %s', test_name)
diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py
index b92ffc79..9045e743 100644
--- a/tests/unittests/test_builtin_handlers.py
+++ b/tests/unittests/test_builtin_handlers.py
@@ -109,7 +109,7 @@ class TestJinjaTemplatePartHandler(CiTestCase):
         cloudconfig_handler = CloudConfigPartHandler(self.paths)
         h = JinjaTemplatePartHandler(
             self.paths, sub_handlers=[script_handler, cloudconfig_handler])
-        self.assertItemsEqual(
+        self.assertCountEqual(
             ['text/cloud-config', 'text/cloud-config-jsonp',
              'text/x-shellscript'],
             h.sub_handlers)
@@ -120,7 +120,7 @@ class TestJinjaTemplatePartHandler(CiTestCase):
         cloudconfig_handler = CloudConfigPartHandler(self.paths)
         h = JinjaTemplatePartHandler(
             self.paths, sub_handlers=[script_handler, cloudconfig_handler])
-        self.assertItemsEqual(
+        self.assertCountEqual(
             ['text/cloud-config', 'text/cloud-config-jsonp',
              'text/x-shellscript'],
             h.sub_handlers)
@@ -302,7 +302,7 @@ class TestConvertJinjaInstanceData(CiTestCase):
         expected_data.update({'v1key1': 'v1.1', 'v2key1': 'v2.1'})
 
         converted_data = convert_jinja_instance_data(data=data)
-        self.assertItemsEqual(
+        self.assertCountEqual(
             ['ds', 'v1', 'v2', 'v1key1', 'v2key1'], converted_data.keys())
         self.assertEqual(
             expected_data,
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 2c6aa44d..2f8561cb 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -528,14 +528,14 @@ scbus-1 on xpt0 bus 0
 
         def tags_exists(x, y):
             for tag in x.keys():
-                self.assertIn(tag, y)
+                assert tag in y
             for tag in y.keys():
-                self.assertIn(tag, x)
+                assert tag in x
 
         def tags_equal(x, y):
             for x_val in x.values():
                 y_val = y.get(x_val.tag)
-                self.assertEqual(x_val.text, y_val.text)
+                assert x_val.text == y_val.text
 
         old_cnt = create_tag_index(oxml)
         new_cnt = create_tag_index(nxml)
@@ -649,7 +649,7 @@ scbus-1 on xpt0 bus 0
 
         crawled_metadata = dsrc.crawl_metadata()
 
-        self.assertItemsEqual(
+        self.assertCountEqual(
             crawled_metadata.keys(),
             ['cfg', 'files', 'metadata', 'userdata_raw'])
         self.assertEqual(crawled_metadata['cfg'], expected_cfg)
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
index 007df09f..6344b0bb 100644
--- a/tests/unittests/test_datasource/test_azure_helper.py
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -1,7 +1,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import os
-import unittest2
+import unittest
 from textwrap import dedent
 
 from cloudinit.sources.helpers import azure as azure_helper
@@ -332,7 +332,7 @@ class TestOpenSSLManagerActions(CiTestCase):
         path = 'tests/data/azure'
         return os.path.join(path, name)
 
-    @unittest2.skip("todo move to cloud_test")
+    @unittest.skip("todo move to cloud_test")
     def test_pubkey_extract(self):
         cert = load_file(self._data_file('pubkey_extract_cert'))
         good_key = load_file(self._data_file('pubkey_extract_ssh_key'))
@@ -344,7 +344,7 @@ class TestOpenSSLManagerActions(CiTestCase):
         fingerprint = sslmgr._get_fingerprint_from_cert(cert)
         self.assertEqual(good_fingerprint, fingerprint)
 
-    @unittest2.skip("todo move to cloud_test")
+    @unittest.skip("todo move to cloud_test")
     @mock.patch.object(azure_helper.OpenSSLManager, '_decrypt_certs_from_xml')
     def test_parse_certificates(self, mock_decrypt_certs):
         """Azure control plane puts private keys as well as certificates
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 62084de5..0f0b42bb 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -22,7 +22,7 @@ import os.path
 import re
 import signal
 import stat
-import unittest2
+import unittest
 import uuid
 
 from cloudinit import serial
@@ -1095,11 +1095,11 @@ class TestNetworkConversion(CiTestCase):
         self.assertEqual(expected, found)
 
 
-@unittest2.skipUnless(get_smartos_environ() == SMARTOS_ENV_KVM,
-                      "Only supported on KVM and bhyve guests under SmartOS")
-@unittest2.skipUnless(os.access(SERIAL_DEVICE, os.W_OK),
-                      "Requires write access to " + SERIAL_DEVICE)
-@unittest2.skipUnless(HAS_PYSERIAL is True, "pyserial not available")
+@unittest.skipUnless(get_smartos_environ() == SMARTOS_ENV_KVM,
+                     "Only supported on KVM and bhyve guests under SmartOS")
+@unittest.skipUnless(os.access(SERIAL_DEVICE, os.W_OK),
+                     "Requires write access to " + SERIAL_DEVICE)
+@unittest.skipUnless(HAS_PYSERIAL is True, "pyserial not available")
 class TestSerialConcurrency(CiTestCase):
     """
        This class tests locking on an actual serial port, and as such can only
diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py
index 5b4105dd..286ef771 100644
--- a/tests/unittests/test_handler/test_handler_ca_certs.py
+++ b/tests/unittests/test_handler/test_handler_ca_certs.py
@@ -11,13 +11,9 @@ import logging
 import shutil
 import tempfile
 import unittest
+from contextlib import ExitStack
 from unittest import mock
 
-try:
-    from contextlib import ExitStack
-except ImportError:
-    from contextlib2 import ExitStack
-
 
 class TestNoConfig(unittest.TestCase):
     def setUp(self):
diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py
index 2dab3a54..513c18b5 100644
--- a/tests/unittests/test_handler/test_handler_chef.py
+++ b/tests/unittests/test_handler/test_handler_chef.py
@@ -65,18 +65,18 @@ class TestInstallChefOmnibus(HttprettyTestCase):
         cc_chef.install_chef_from_omnibus()
         expected_kwargs = {'retries': cc_chef.OMNIBUS_URL_RETRIES,
                            'url': cc_chef.OMNIBUS_URL}
-        self.assertItemsEqual(expected_kwargs, m_rdurl.call_args_list[0][1])
+        self.assertCountEqual(expected_kwargs, m_rdurl.call_args_list[0][1])
         cc_chef.install_chef_from_omnibus(retries=10)
         expected_kwargs = {'retries': 10,
                            'url': cc_chef.OMNIBUS_URL}
-        self.assertItemsEqual(expected_kwargs, m_rdurl.call_args_list[1][1])
+        self.assertCountEqual(expected_kwargs, m_rdurl.call_args_list[1][1])
         expected_subp_kwargs = {
             'args': ['-v', '2.0'],
             'basename': 'chef-omnibus-install',
             'blob': m_rdurl.return_value.contents,
             'capture': False
         }
-        self.assertItemsEqual(
+        self.assertCountEqual(
             expected_subp_kwargs,
             m_subp_blob.call_args_list[0][1])
 
@@ -97,7 +97,7 @@ class TestInstallChefOmnibus(HttprettyTestCase):
             'blob': response,
             'capture': False
         }
-        self.assertItemsEqual(expected_kwargs, called_kwargs)
+        self.assertCountEqual(expected_kwargs, called_kwargs)
 
 
 class TestChef(FilesystemMockingTestCase):
diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py
index 43b53745..501bcca5 100644
--- a/tests/unittests/test_handler/test_handler_growpart.py
+++ b/tests/unittests/test_handler/test_handler_growpart.py
@@ -11,13 +11,9 @@ import logging
 import os
 import re
 import unittest
+from contextlib import ExitStack
 from unittest import mock
 
-try:
-    from contextlib import ExitStack
-except ImportError:
-    from contextlib2 import ExitStack
-
 # growpart:
 #   mode: auto  # off, on, auto, 'growpart'
 #   devices: ['root']
diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
index abde02d3..98628120 100644
--- a/tests/unittests/test_handler/test_schema.py
+++ b/tests/unittests/test_handler/test_schema.py
@@ -20,7 +20,7 @@ class GetSchemaTest(CiTestCase):
     def test_get_schema_coalesces_known_schema(self):
         """Every cloudconfig module with schema is listed in allOf keyword."""
         schema = get_schema()
-        self.assertItemsEqual(
+        self.assertCountEqual(
             [
                 'cc_bootcmd',
                 'cc_ntp',
@@ -38,7 +38,7 @@ class GetSchemaTest(CiTestCase):
             schema['$schema'])
         # FULL_SCHEMA is updated by the get_schema call
         from cloudinit.config.schema import FULL_SCHEMA
-        self.assertItemsEqual(['id', '$schema', 'allOf'], FULL_SCHEMA.keys())
+        self.assertCountEqual(['id', '$schema', 'allOf'], FULL_SCHEMA.keys())
 
     def test_get_schema_returns_global_when_set(self):
         """When FULL_SCHEMA global is already set, get_schema returns it."""