Integration Testing: tox env, pyxld 2.2.3, and revamp framework

Massive update to clean up and greatly enhance the integration testing
framework developed by Wesley Wiedenmeier.

 - Updated tox environment to run integration test 'citest' to utilize
   pylxd 2.2.3
 - Add support for distro feature flags
   - add framework for feature flags to release config with feature groups
     and overrides allowed in any release conf override level
   - add support for feature flags in platform and config handling
   - during collect, skip testcases that require features not supported by
     the image with a warning message
 - Enable additional distros (i.e. centos, debian)
 - Add 'bddeb' command to build a deb from the current working tree
   cleanly in a container, so deps do not have to be installed on host
 - Adds a command line option '--preserve-data' that ensures that
   collected data will be left after tests run. This also allows the
   directory to store collected data in during the run command to be
   specified using '--data-dir'.
 - Updated Read the Docs testing page and doc strings for pep 257
   compliance

12 files changed, 54 insertions, 63 deletions

diff --git a/tests/cloud_tests/testcases/examples/__init__.py b/tests/cloud_tests/testcases/examples/__init__.py
index b3af7f8a..39af88c2 100644
--- a/tests/cloud_tests/testcases/examples/__init__.py
+++ b/tests/cloud_tests/testcases/examples/__init__.py
@@ -1,7 +1,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""
-Test verifiers for cloud-init examples
+"""Test verifiers for cloud-init examples.
+
 See configs/examples/README.md for more information
 """
 
diff --git a/tests/cloud_tests/testcases/examples/add_apt_repositories.py b/tests/cloud_tests/testcases/examples/add_apt_repositories.py
index 15b8f01c..71eede97 100644
--- a/tests/cloud_tests/testcases/examples/add_apt_repositories.py
+++ b/tests/cloud_tests/testcases/examples/add_apt_repositories.py
@@ -1,19 +1,19 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""cloud-init Integration Test Verify Script"""
+"""cloud-init Integration Test Verify Script."""
 from tests.cloud_tests.testcases import base
 
 
 class TestAptconfigurePrimary(base.CloudTestCase):
-    """Example cloud-config test"""
+    """Example cloud-config test."""
 
     def test_ubuntu_sources(self):
-        """Test no default Ubuntu entries exist"""
+        """Test no default Ubuntu entries exist."""
         out = self.get_data_file('ubuntu.sources.list')
         self.assertEqual(0, int(out))
 
     def test_gatech_sources(self):
-        """Test GaTech entires exist"""
+        """Test GaTech entires exist."""
         out = self.get_data_file('gatech.sources.list')
         self.assertEqual(20, int(out))
 
diff --git a/tests/cloud_tests/testcases/examples/alter_completion_message.py b/tests/cloud_tests/testcases/examples/alter_completion_message.py
index b06ad01b..b7b5d5e0 100644
--- a/tests/cloud_tests/testcases/examples/alter_completion_message.py
+++ b/tests/cloud_tests/testcases/examples/alter_completion_message.py
@@ -1,34 +1,27 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""cloud-init Integration Test Verify Script"""
+"""cloud-init Integration Test Verify Script."""
 from tests.cloud_tests.testcases import base
 
 
 class TestFinalMessage(base.CloudTestCase):
-    """
-    test cloud init module `cc_final_message`
-    """
+    """Test cloud init module `cc_final_message`."""
+
     subs_char = '$'
 
     def get_final_message_config(self):
-        """
-        get config for final message
-        """
+        """Get config for final message."""
         self.assertIn('final_message', self.cloud_config)
         return self.cloud_config['final_message']
 
     def get_final_message(self):
-        """
-        get final message from log
-        """
+        """Get final message from log."""
         out = self.get_data_file('cloud-init-output.log')
         lines = len(self.get_final_message_config().splitlines())
         return '\n'.join(out.splitlines()[-1 * lines:])
 
     def test_final_message_string(self):
-        """
-        ensure final handles regular strings
-        """
+        """Ensure final handles regular strings."""
         for actual, config in zip(
                 self.get_final_message().splitlines(),
                 self.get_final_message_config().splitlines()):
@@ -36,9 +29,7 @@ class TestFinalMessage(base.CloudTestCase):
                 self.assertEqual(actual, config)
 
     def test_final_message_subs(self):
-        """
-        test variable substitution in final message
-        """
+        """Test variable substitution in final message."""
         # TODO: add verification of other substitutions
         patterns = {'$datasource': self.get_datasource()}
         for key, expected in patterns.items():
diff --git a/tests/cloud_tests/testcases/examples/configure_instance_trusted_ca_certificates.py b/tests/cloud_tests/testcases/examples/configure_instance_trusted_ca_certificates.py
index 8a4a0db0..38540eb8 100644
--- a/tests/cloud_tests/testcases/examples/configure_instance_trusted_ca_certificates.py
+++ b/tests/cloud_tests/testcases/examples/configure_instance_trusted_ca_certificates.py
@@ -1,24 +1,24 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""cloud-init Integration Test Verify Script"""
+"""cloud-init Integration Test Verify Script."""
 from tests.cloud_tests.testcases import base
 
 
 class TestTrustedCA(base.CloudTestCase):
-    """Example cloud-config test"""
+    """Example cloud-config test."""
 
     def test_cert_count_ca(self):
-        """Test correct count of CAs in .crt"""
+        """Test correct count of CAs in .crt."""
         out = self.get_data_file('cert_count_ca')
         self.assertIn('7 /etc/ssl/certs/ca-certificates.crt', out)
 
     def test_cert_count_cloudinit(self):
-        """Test correct count of CAs in .pem"""
+        """Test correct count of CAs in .pem."""
         out = self.get_data_file('cert_count_cloudinit')
         self.assertIn('7 /etc/ssl/certs/cloud-init-ca-certs.pem', out)
 
     def test_cloudinit_certs(self):
-        """Test text of cert"""
+        """Test text of cert."""
         out = self.get_data_file('cloudinit_certs')
         self.assertIn('-----BEGIN CERTIFICATE-----', out)
         self.assertIn('YOUR-ORGS-TRUSTED-CA-CERT-HERE', out)
diff --git a/tests/cloud_tests/testcases/examples/configure_instances_ssh_keys.py b/tests/cloud_tests/testcases/examples/configure_instances_ssh_keys.py
index 4f651703..691a316b 100644
--- a/tests/cloud_tests/testcases/examples/configure_instances_ssh_keys.py
+++ b/tests/cloud_tests/testcases/examples/configure_instances_ssh_keys.py
@@ -1,29 +1,29 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""cloud-init Integration Test Verify Script"""
+"""cloud-init Integration Test Verify Script."""
 from tests.cloud_tests.testcases import base
 
 
 class TestSSHKeys(base.CloudTestCase):
-    """Example cloud-config test"""
+    """Example cloud-config test."""
 
     def test_cert_count(self):
-        """Test cert count"""
+        """Test cert count."""
         out = self.get_data_file('cert_count')
         self.assertEqual(20, int(out))
 
     def test_dsa_public(self):
-        """Test DSA key has ending"""
+        """Test DSA key has ending."""
         out = self.get_data_file('dsa_public')
         self.assertIn('ZN4XnifuO5krqAybngIy66PMEoQ= smoser@localhost', out)
 
     def test_rsa_public(self):
-        """Test RSA key has specific ending"""
+        """Test RSA key has specific ending."""
         out = self.get_data_file('rsa_public')
         self.assertIn('PemAWthxHO18QJvWPocKJtlsDNi3 smoser@localhost', out)
 
     def test_auth_keys(self):
-        """Test authorized keys has specific ending"""
+        """Test authorized keys has specific ending."""
         out = self.get_data_file('auth_keys')
         self.assertIn('QPOt5Q8zWd9qG7PBl9+eiH5qV7NZ mykey@host', out)
         self.assertIn('Hj29SCmXp5Kt5/82cD/VN3NtHw== smoser@brickies', out)
diff --git a/tests/cloud_tests/testcases/examples/including_user_groups.py b/tests/cloud_tests/testcases/examples/including_user_groups.py
index e5732322..67af527b 100644
--- a/tests/cloud_tests/testcases/examples/including_user_groups.py
+++ b/tests/cloud_tests/testcases/examples/including_user_groups.py
@@ -1,42 +1,42 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""cloud-init Integration Test Verify Script"""
+"""cloud-init Integration Test Verify Script."""
 from tests.cloud_tests.testcases import base
 
 
 class TestUserGroups(base.CloudTestCase):
-    """Example cloud-config test"""
+    """Example cloud-config test."""
 
     def test_group_ubuntu(self):
-        """Test ubuntu group exists"""
+        """Test ubuntu group exists."""
         out = self.get_data_file('group_ubuntu')
         self.assertRegex(out, r'ubuntu:x:[0-9]{4}:')
 
     def test_group_cloud_users(self):
-        """Test cloud users group exists"""
+        """Test cloud users group exists."""
         out = self.get_data_file('group_cloud_users')
         self.assertRegex(out, r'cloud-users:x:[0-9]{4}:barfoo')
 
     def test_user_ubuntu(self):
-        """Test ubuntu user exists"""
+        """Test ubuntu user exists."""
         out = self.get_data_file('user_ubuntu')
         self.assertRegex(
             out, r'ubuntu:x:[0-9]{4}:[0-9]{4}:Ubuntu:/home/ubuntu:/bin/bash')
 
     def test_user_foobar(self):
-        """Test foobar user exists"""
+        """Test foobar user exists."""
         out = self.get_data_file('user_foobar')
         self.assertRegex(
             out, r'foobar:x:[0-9]{4}:[0-9]{4}:Foo B. Bar:/home/foobar:')
 
     def test_user_barfoo(self):
-        """Test barfoo user exists"""
+        """Test barfoo user exists."""
         out = self.get_data_file('user_barfoo')
         self.assertRegex(
             out, r'barfoo:x:[0-9]{4}:[0-9]{4}:Bar B. Foo:/home/barfoo:')
 
     def test_user_cloudy(self):
-        """Test cloudy user exists"""
+        """Test cloudy user exists."""
         out = self.get_data_file('user_cloudy')
         self.assertRegex(out, r'cloudy:x:[0-9]{3,4}:')
 
diff --git a/tests/cloud_tests/testcases/examples/install_arbitrary_packages.py b/tests/cloud_tests/testcases/examples/install_arbitrary_packages.py
index 660d1aa3..df133844 100644
--- a/tests/cloud_tests/testcases/examples/install_arbitrary_packages.py
+++ b/tests/cloud_tests/testcases/examples/install_arbitrary_packages.py
@@ -1,19 +1,19 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""cloud-init Integration Test Verify Script"""
+"""cloud-init Integration Test Verify Script."""
 from tests.cloud_tests.testcases import base
 
 
 class TestInstall(base.CloudTestCase):
-    """Example cloud-config test"""
+    """Example cloud-config test."""
 
     def test_htop(self):
-        """Verify htop installed"""
+        """Verify htop installed."""
         out = self.get_data_file('htop')
         self.assertEqual(1, int(out))
 
     def test_tree(self):
-        """Verify tree installed"""
+        """Verify tree installed."""
         out = self.get_data_file('treeutils')
         self.assertEqual(1, int(out))
 
diff --git a/tests/cloud_tests/testcases/examples/install_run_chef_recipes.py b/tests/cloud_tests/testcases/examples/install_run_chef_recipes.py
index b36486f0..4ec26b8f 100644
--- a/tests/cloud_tests/testcases/examples/install_run_chef_recipes.py
+++ b/tests/cloud_tests/testcases/examples/install_run_chef_recipes.py
@@ -1,14 +1,14 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""cloud-init Integration Test Verify Script"""
+"""cloud-init Integration Test Verify Script."""
 from tests.cloud_tests.testcases import base
 
 
 class TestChefExample(base.CloudTestCase):
-    """Test chef module"""
+    """Test chef module."""
 
     def test_chef_basic(self):
-        """Test chef installed"""
+        """Test chef installed."""
         out = self.get_data_file('chef_installed')
         self.assertIn('install ok', out)
 
diff --git a/tests/cloud_tests/testcases/examples/run_apt_upgrade.py b/tests/cloud_tests/testcases/examples/run_apt_upgrade.py
index 4c04d315..744e49cb 100644
--- a/tests/cloud_tests/testcases/examples/run_apt_upgrade.py
+++ b/tests/cloud_tests/testcases/examples/run_apt_upgrade.py
@@ -1,14 +1,14 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""cloud-init Integration Test Verify Script"""
+"""cloud-init Integration Test Verify Script."""
 from tests.cloud_tests.testcases import base
 
 
 class TestUpgrade(base.CloudTestCase):
-    """Example cloud-config test"""
+    """Example cloud-config test."""
 
     def test_upgrade(self):
-        """Test upgrade exists in apt history"""
+        """Test upgrade exists in apt history."""
         out = self.get_data_file('cloud-init.log')
         self.assertIn(
             '[CLOUDINIT] util.py[DEBUG]: apt-upgrade '
diff --git a/tests/cloud_tests/testcases/examples/run_commands.py b/tests/cloud_tests/testcases/examples/run_commands.py
index 0be21d0f..01d5d4fc 100644
--- a/tests/cloud_tests/testcases/examples/run_commands.py
+++ b/tests/cloud_tests/testcases/examples/run_commands.py
@@ -1,14 +1,14 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""cloud-init Integration Test Verify Script"""
+"""cloud-init Integration Test Verify Script."""
 from tests.cloud_tests.testcases import base
 
 
 class TestRunCmd(base.CloudTestCase):
-    """Example cloud-config test"""
+    """Example cloud-config test."""
 
     def test_run_cmd(self):
-        """Test run command worked"""
+        """Test run command worked."""
         out = self.get_data_file('run_cmd')
         self.assertIn('cloud-init run cmd test', out)
 
diff --git a/tests/cloud_tests/testcases/examples/run_commands_first_boot.py b/tests/cloud_tests/testcases/examples/run_commands_first_boot.py
index baa23130..3f3d8f84 100644
--- a/tests/cloud_tests/testcases/examples/run_commands_first_boot.py
+++ b/tests/cloud_tests/testcases/examples/run_commands_first_boot.py
@@ -1,14 +1,14 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""cloud-init Integration Test Verify Script"""
+"""cloud-init Integration Test Verify Script."""
 from tests.cloud_tests.testcases import base
 
 
 class TestBootCmd(base.CloudTestCase):
-    """Example cloud-config test"""
+    """Example cloud-config test."""
 
     def test_bootcmd_host(self):
-        """Test boot command worked"""
+        """Test boot command worked."""
         out = self.get_data_file('hosts')
         self.assertIn('192.168.1.130 us.archive.ubuntu.com', out)
 
diff --git a/tests/cloud_tests/testcases/examples/writing_out_arbitrary_files.py b/tests/cloud_tests/testcases/examples/writing_out_arbitrary_files.py
index 97dfeec3..7bd520f6 100644
--- a/tests/cloud_tests/testcases/examples/writing_out_arbitrary_files.py
+++ b/tests/cloud_tests/testcases/examples/writing_out_arbitrary_files.py
@@ -1,29 +1,29 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""cloud-init Integration Test Verify Script"""
+"""cloud-init Integration Test Verify Script."""
 from tests.cloud_tests.testcases import base
 
 
 class TestWriteFiles(base.CloudTestCase):
-    """Example cloud-config test"""
+    """Example cloud-config test."""
 
     def test_b64(self):
-        """Test b64 encoded file reads as ascii"""
+        """Test b64 encoded file reads as ascii."""
         out = self.get_data_file('file_b64')
         self.assertIn('ASCII text', out)
 
     def test_binary(self):
-        """Test binary file reads as executable"""
+        """Test binary file reads as executable."""
         out = self.get_data_file('file_binary')
         self.assertIn('ELF 64-bit LSB executable, x86-64, version 1', out)
 
     def test_gzip(self):
-        """Test gzip file shows up as a shell script"""
+        """Test gzip file shows up as a shell script."""
         out = self.get_data_file('file_gzip')
         self.assertIn('POSIX shell script, ASCII text executable', out)
 
     def test_text(self):
-        """Test text shows up as ASCII text"""
+        """Test text shows up as ASCII text."""
         out = self.get_data_file('file_text')
         self.assertIn('ASCII text', out)