Integration Testing: tox env, pyxld 2.2.3, and revamp framework

Massive update to clean up and greatly enhance the integration testing
framework developed by Wesley Wiedenmeier.

 - Updated tox environment to run integration test 'citest' to utilize
   pylxd 2.2.3
 - Add support for distro feature flags
   - add framework for feature flags to release config with feature groups
     and overrides allowed in any release conf override level
   - add support for feature flags in platform and config handling
   - during collect, skip testcases that require features not supported by
     the image with a warning message
 - Enable additional distros (i.e. centos, debian)
 - Add 'bddeb' command to build a deb from the current working tree
   cleanly in a container, so deps do not have to be installed on host
 - Adds a command line option '--preserve-data' that ensures that
   collected data will be left after tests run. This also allows the
   directory to store collected data in during the run command to be
   specified using '--data-dir'.
 - Updated Read the Docs testing page and doc strings for pep 257
   compliance

1 files changed, 115 insertions, 35 deletions

diff --git a/tests/cloud_tests/args.py b/tests/cloud_tests/args.py
index 371b0444..369d60db 100644
--- a/tests/cloud_tests/args.py
+++ b/tests/cloud_tests/args.py
@@ -1,23 +1,43 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
+"""Argparse argument setup and sanitization."""
+
 import os
 
 from tests.cloud_tests import config, util
-from tests.cloud_tests import LOG
+from tests.cloud_tests import LOG, TREE_BASE
 
 ARG_SETS = {
+    'BDDEB': (
+        (('--bddeb-args',),
+         {'help': 'args to pass through to bddeb',
+          'action': 'store', 'default': None, 'required': False}),
+        (('--build-os',),
+         {'help': 'OS to use as build system (default is xenial)',
+          'action': 'store', 'choices': config.ENABLED_DISTROS,
+          'default': 'xenial', 'required': False}),
+        (('--build-platform',),
+         {'help': 'platform to use for build system (default is lxd)',
+          'action': 'store', 'choices': config.ENABLED_PLATFORMS,
+          'default': 'lxd', 'required': False}),
+        (('--cloud-init',),
+         {'help': 'path to base of cloud-init tree', 'metavar': 'DIR',
+          'action': 'store', 'required': False, 'default': TREE_BASE}),),
     'COLLECT': (
         (('-p', '--platform'),
          {'help': 'platform(s) to run tests on', 'metavar': 'PLATFORM',
-          'action': 'append', 'choices': config.list_enabled_platforms(),
+          'action': 'append', 'choices': config.ENABLED_PLATFORMS,
           'default': []}),
         (('-n', '--os-name'),
          {'help': 'the name(s) of the OS(s) to test', 'metavar': 'NAME',
-          'action': 'append', 'choices': config.list_enabled_distros(),
+          'action': 'append', 'choices': config.ENABLED_DISTROS,
           'default': []}),
         (('-t', '--test-config'),
          {'help': 'test config file(s) to use', 'metavar': 'FILE',
-          'action': 'append', 'default': []}),),
+          'action': 'append', 'default': []}),
+        (('--feature-override',),
+         {'help': 'feature flags override(s), <flagname>=<true/false>',
+          'action': 'append', 'default': [], 'required': False}),),
     'CREATE': (
         (('-c', '--config'),
          {'help': 'cloud-config yaml for testcase', 'metavar': 'DATA',
@@ -41,7 +61,15 @@ ARG_SETS = {
     'OUTPUT': (
         (('-d', '--data-dir'),
          {'help': 'directory to store test data in',
-          'action': 'store', 'metavar': 'DIR', 'required': True}),),
+          'action': 'store', 'metavar': 'DIR', 'required': False}),
+        (('--preserve-data',),
+         {'help': 'do not remove collected data after successful run',
+          'action': 'store_true', 'default': False, 'required': False}),),
+    'OUTPUT_DEB': (
+        (('--deb',),
+         {'help': 'path to write output deb to', 'metavar': 'FILE',
+          'action': 'store', 'required': False,
+          'default': 'cloud-init_all.deb'}),),
     'RESULT': (
         (('-r', '--result'),
          {'help': 'file to write results to',
@@ -61,31 +89,54 @@ ARG_SETS = {
          {'help': 'ppa to enable (implies -u)', 'metavar': 'NAME',
           'action': 'store'}),
         (('-u', '--upgrade'),
-         {'help': 'upgrade before starting tests', 'action': 'store_true',
-          'default': False}),),
+         {'help': 'upgrade or install cloud-init from repo',
+          'action': 'store_true', 'default': False}),
+        (('--upgrade-full',),
+         {'help': 'do full system upgrade from repo (implies -u)',
+          'action': 'store_true', 'default': False}),),
+
 }
 
 SUBCMDS = {
+    'bddeb': ('build cloud-init deb from tree',
+              ('BDDEB', 'OUTPUT_DEB', 'INTERFACE')),
     'collect': ('collect test data',
                 ('COLLECT', 'INTERFACE', 'OUTPUT', 'RESULT', 'SETUP')),
     'create': ('create new test case', ('CREATE', 'INTERFACE')),
-    'run': ('run test suite', ('COLLECT', 'INTERFACE', 'RESULT', 'SETUP')),
+    'run': ('run test suite',
+            ('COLLECT', 'INTERFACE', 'RESULT', 'OUTPUT', 'SETUP')),
+    'tree_collect': ('collect using current working tree',
+                     ('BDDEB', 'COLLECT', 'INTERFACE', 'OUTPUT', 'RESULT')),
+    'tree_run': ('run using current working tree',
+                 ('BDDEB', 'COLLECT', 'INTERFACE', 'OUTPUT', 'RESULT')),
     'verify': ('verify test data', ('INTERFACE', 'OUTPUT', 'RESULT')),
 }
 
 
 def _empty_normalizer(args):
+    """Do not normalize arguments."""
+    return args
+
+
+def normalize_bddeb_args(args):
+    """Normalize BDDEB arguments.
+
+    @param args: parsed args
+    @return_value: updated args, or None if errors encountered
     """
-    do not normalize arguments
-    """
+    # make sure cloud-init dir is accessible
+    if not (args.cloud_init and os.path.isdir(args.cloud_init)):
+        LOG.error('invalid cloud-init tree path')
+        return None
+
     return args
 
 
 def normalize_create_args(args):
-    """
-    normalize CREATE arguments
-    args: parsed args
-    return_value: updated args, or None if errors occurred
+    """Normalize CREATE arguments.
+
+    @param args: parsed args
+    @return_value: updated args, or None if errors occurred
     """
     # ensure valid name for new test
     if len(args.name.split('/')) != 2:
@@ -114,22 +165,22 @@ def normalize_create_args(args):
 
 
 def normalize_collect_args(args):
-    """
-    normalize COLLECT arguments
-    args: parsed args
-    return_value: updated args, or None if errors occurred
+    """Normalize COLLECT arguments.
+
+    @param args: parsed args
+    @return_value: updated args, or None if errors occurred
     """
     # platform should default to all supported
     if len(args.platform) == 0:
-        args.platform = config.list_enabled_platforms()
+        args.platform = config.ENABLED_PLATFORMS
     args.platform = util.sorted_unique(args.platform)
 
     # os name should default to all enabled
     # if os name is provided ensure that all provided are supported
     if len(args.os_name) == 0:
-        args.os_name = config.list_enabled_distros()
+        args.os_name = config.ENABLED_DISTROS
     else:
-        supported = config.list_enabled_distros()
+        supported = config.ENABLED_DISTROS
         invalid = [os_name for os_name in args.os_name
                    if os_name not in supported]
         if len(invalid) != 0:
@@ -158,18 +209,33 @@ def normalize_collect_args(args):
             args.test_config = valid
     args.test_config = util.sorted_unique(args.test_config)
 
+    # parse feature flag overrides and ensure all are valid
+    if args.feature_override:
+        overrides = args.feature_override
+        args.feature_override = util.parse_conf_list(
+            overrides, boolean=True, valid=config.list_feature_flags())
+        if not args.feature_override:
+            LOG.error('invalid feature flag override(s): %s', overrides)
+            return None
+    else:
+        args.feature_override = {}
+
     return args
 
 
 def normalize_output_args(args):
+    """Normalize OUTPUT arguments.
+
+    @param args: parsed args
+    @return_value: updated args, or None if errors occurred
     """
-    normalize OUTPUT arguments
-    args: parsed args
-    return_value: updated args, or None if errors occurred
-    """
+    if args.data_dir:
+        args.data_dir = os.path.abspath(args.data_dir)
+        if not os.path.exists(args.data_dir):
+            os.mkdir(args.data_dir)
+
     if not args.data_dir:
-        LOG.error('--data-dir must be specified')
-        return None
+        args.data_dir = None
 
     # ensure clean output dir if collect
     # ensure data exists if verify
@@ -177,19 +243,31 @@ def normalize_output_args(args):
         if not util.is_clean_writable_dir(args.data_dir):
             LOG.error('data_dir must be empty/new and must be writable')
             return None
-    elif args.subcmd == 'verify':
-        if not os.path.exists(args.data_dir):
-            LOG.error('data_dir %s does not exist', args.data_dir)
-            return None
 
     return args
 
 
-def normalize_setup_args(args):
+def normalize_output_deb_args(args):
+    """Normalize OUTPUT_DEB arguments.
+
+    @param args: parsed args
+    @return_value: updated args, or None if erros occurred
     """
-    normalize SETUP arguments
-    args: parsed args
-    return_value: updated_args, or None if errors occurred
+    # make sure to use abspath for deb
+    args.deb = os.path.abspath(args.deb)
+
+    if not args.deb.endswith('.deb'):
+        LOG.error('output filename does not end in ".deb"')
+        return None
+
+    return args
+
+
+def normalize_setup_args(args):
+    """Normalize SETUP arguments.
+
+    @param args: parsed args
+    @return_value: updated_args, or None if errors occurred
     """
     # ensure deb or rpm valid if specified
     for pkg in (args.deb, args.rpm):
@@ -210,10 +288,12 @@ def normalize_setup_args(args):
 
 
 NORMALIZERS = {
+    'BDDEB': normalize_bddeb_args,
     'COLLECT': normalize_collect_args,
     'CREATE': normalize_create_args,
     'INTERFACE': _empty_normalizer,
     'OUTPUT': normalize_output_args,
+    'OUTPUT_DEB': normalize_output_deb_args,
     'RESULT': _empty_normalizer,
     'SETUP': normalize_setup_args,
 }