Integration Testing: tox env, pyxld 2.2.3, and revamp framework

Massive update to clean up and greatly enhance the integration testing
framework developed by Wesley Wiedenmeier.

 - Updated tox environment to run integration test 'citest' to utilize
   pylxd 2.2.3
 - Add support for distro feature flags
   - add framework for feature flags to release config with feature groups
     and overrides allowed in any release conf override level
   - add support for feature flags in platform and config handling
   - during collect, skip testcases that require features not supported by
     the image with a warning message
 - Enable additional distros (i.e. centos, debian)
 - Add 'bddeb' command to build a deb from the current working tree
   cleanly in a container, so deps do not have to be installed on host
 - Adds a command line option '--preserve-data' that ensures that
   collected data will be left after tests run. This also allows the
   directory to store collected data in during the run command to be
   specified using '--data-dir'.
 - Updated Read the Docs testing page and doc strings for pep 257
   compliance

1 files changed, 243 insertions, 76 deletions

diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml
index 183f78c1..45deb58f 100644
--- a/tests/cloud_tests/releases.yaml
+++ b/tests/cloud_tests/releases.yaml
@@ -1,86 +1,253 @@
 # ============================= Release Config ================================
 default_release_config:
-    # all are disabled by default
-    enabled: false
-    # timeout for booting image and running cloud init
-    timeout: 120
-    # platform_ident values for the image, with data to identify the image
-    # on that platform. see platforms.base for more information
-    platform_ident: {}
-    # a script to run after a boot that is used to modify an image, before
-    # making a snapshot of the image. may be useful for removing data left
-    # behind from cloud-init booting, such as logs, to ensure that data from
-    # snapshot.launch() will not include a cloud-init.log from a boot used to
-    # create the snapshot, if cloud-init has not run
-    boot_clean_script: |
-        #!/bin/bash
-        rm -rf /var/log/cloud-init.log /var/log/cloud-init-output.log \
-            /var/lib/cloud/ /run/cloud-init/ /var/log/syslog
+    # global default configuration options
+    default:
+        # all are disabled by default
+        enabled: false
+        # timeout for booting image and running cloud init
+        boot_timeout: 120
+        # a script to run after a boot that is used to modify an image, before
+        # making a snapshot of the image. may be useful for removing data left
+        # behind from cloud-init booting, such as logs, to ensure that data
+        # from snapshot.launch() will not include a cloud-init.log from a boot
+        # used to create the snapshot, if cloud-init has not run
+        boot_clean_script: |
+            #!/bin/bash
+            rm -rf /var/log/cloud-init.log /var/log/cloud-init-output.log \
+                /var/lib/cloud/ /run/cloud-init/ /var/log/syslog
+        # test script to determine if system is booted fully
+        system_ready_script: |
+            # permit running or degraded state as both indicate complete boot
+            [ $(systemctl is-system-running) = 'running' -o
+              $(systemctl is-system-running) = 'degraded' ]
+        # test script to determine if cloud-init has finished
+        cloud_init_ready_script: |
+            [ -f '/run/cloud-init/result.json' ]
+        # currently used features and their uses are:
+        # features groups and additional feature settings
+        feature_groups: []
+        features: {}
+
+    # lxd specific default configuration options
+    lxd:
+        # default sstreams server to use for lxd image retrieval
+        sstreams_server: https://us.images.linuxcontainers.org:8443
+        # keep base image, avoids downloading again next run
+        cache_base_image: true
+        # lxd images from linuxcontainers.org do not have the nocloud seed
+        # templates in place, so the image metadata must be modified
+        override_templates: true
+        # arg overrides to set image up
+        setup_overrides:
+            # lxd images from linuxcontainers.org do not come with
+            # cloud-init, so must pull cloud-init in from repo using
+            # setup_image.upgrade
+            upgrade: true
+
+features:
+    # all currently supported feature flags
+    all:
+        - apt       # image supports apt package manager
+        - byobu     # byobu is available in repositories
+        - landscape # landscape-client available in repos
+        - lxd       # lxd is available in the image
+        - ppa       # image supports ppas
+        - rpm       # image supports rpms
+        - snap      # supports snapd
+        # NOTE: the following feature flags are to work around bugs in the
+        #       images, and can be removed when no longer needed
+        - hostname      # setting system hostname works
+        # NOTE: the following feature flags are to work around issues in the
+        #       testcases, and can be removed when no longer needed
+        - apt_src_cont  # default contents and format of sources.list matches
+                        # ubuntu sources.list
+        - apt_hist_fmt  # apt command history entries use full paths to apt
+                        # executable rather than relative paths
+        - daylight_time # timezones are daylight not standard time
+        - apt_up_out    # 'Calculating upgrade..' present in log output from
+                        # apt-get dist-upgrade output
+        - engb_locale   # locale en_GB.UTF-8 is available
+        - locale_gen    # the /etc/locale.gen file exists
+        - no_ntpdate    # 'ntpdate' is not installed by default
+        - no_file_fmt_e # the 'file' utility does not have a formatting error
+        - ppa_file_name # the name of the source file added to sources.list.d has
+                        # the expected format for newer ubuntu releases
+        - sshd          # requires ssh server to be installed by default
+        - ssh_key_fmt   # ssh auth keys printed to console have expected format
+        - syslog        # test case requires syslog to be written by default
+        - ubuntu_ntp    # expect ubuntu.pool.ntp.org to be used as ntp server
+        - ubuntu_repos  # test case requres ubuntu repositories to be used
+        - ubuntu_user   # test case needs user with the name 'ubuntu' to exist
+        # NOTE: the following feature flags are to work around issues that may
+        #       be considered bugs in cloud-init
+        - lsb_release   # image has lsb_release installed, maybe should install
+                        # if missing by default
+        - sudo          # image has sudo installed, should not be required
+    # feature flag groups
+    groups:
+        base:
+            hostname: true
+            no_file_fmt_e: true
+        ubuntu_specific:
+            apt_src_cont: true
+            apt_hist_fmt: true
+            byobu: true
+            daylight_time: true
+            engb_locale: true
+            landscape: true
+            locale_gen: true
+            lsb_release: true
+            lxd: true
+            ppa: true
+            ppa_file_name: true
+            snap: true
+            sshd: true
+            ssh_key_fmt: true
+            sudo: true
+            syslog: true
+            ubuntu_ntp: true
+            ubuntu_repos: true
+            ubuntu_user: true
+        debian_base:
+            apt: true
+            apt_up_out: true
+            no_ntpdate: true
+        rhel_base:
+            rpm: true
 
 releases:
-    trusty:
-        enabled: true
-        platform_ident:
-            lxd:
-                # if sstreams_server is omitted, default is used, defined in
-                # tests.cloud_tests.platforms.lxd.DEFAULT_SSTREAMS_SERVER as:
-                # sstreams_server: https://us.images.linuxcontainers.org:8443
-                #alias: ubuntu/trusty/default
-                alias: t
-                sstreams_server: https://cloud-images.ubuntu.com/daily
-    xenial:
-        enabled: true
-        platform_ident:
-            lxd:
-                #alias: ubuntu/xenial/default
-                alias: x
-                sstreams_server: https://cloud-images.ubuntu.com/daily
-    yakkety:
-        enabled: true
-        platform_ident:
-            lxd:
-                #alias: ubuntu/yakkety/default
-                alias: y
-                sstreams_server: https://cloud-images.ubuntu.com/daily
-    zesty:
-        enabled: true
-        platform_ident:
-            lxd:
-                #alias: ubuntu/zesty/default
-                alias: z
-                sstreams_server: https://cloud-images.ubuntu.com/daily
+    # UBUNTU =================================================================
     artful:
-        enabled: true
-        platform_ident:
-            lxd:
-                #alias: ubuntu/artful/default
-                alias: a
-                sstreams_server: https://cloud-images.ubuntu.com/daily
-    jessie:
-        platform_ident:
-            lxd:
-                alias: debian/jessie/default
-    sid:
-        platform_ident:
-            lxd:
-                alias: debian/sid/default
+        # EOL: Jul 2018
+        default:
+            enabled: true
+            feature_groups:
+                - base
+                - debian_base
+                - ubuntu_specific
+        lxd:
+            sstreams_server: https://cloud-images.ubuntu.com/daily
+            alias: artful
+            setup_overrides: null
+            override_templates: false
+    zesty:
+        # EOL: Jan 2018
+        default:
+            enabled: true
+            feature_groups:
+                - base
+                - debian_base
+                - ubuntu_specific
+        lxd:
+            sstreams_server: https://cloud-images.ubuntu.com/daily
+            alias: zesty
+            setup_overrides: null
+            override_templates: false
+    yakkety:
+        # EOL: Jul 2017
+        default:
+            enabled: true
+            feature_groups:
+                - base
+                - debian_base
+                - ubuntu_specific
+        lxd:
+            sstreams_server: https://cloud-images.ubuntu.com/daily
+            alias: yakkety
+            setup_overrides: null
+            override_templates: false
+    xenial:
+        # EOL: Apr 2021
+        default:
+            enabled: true
+            feature_groups:
+                - base
+                - debian_base
+                - ubuntu_specific
+        lxd:
+            sstreams_server: https://cloud-images.ubuntu.com/daily
+            alias: xenial
+            setup_overrides: null
+            override_templates: false
+    trusty:
+        # EOL: Apr 2019
+        default:
+            enabled: true
+            feature_groups:
+                - base
+                - debian_base
+                - ubuntu_specific
+            features:
+                apt_up_out: false
+                locale_gen: false
+                lxd: false
+                ppa_file_name: false
+                snap: false
+                ssh_key_fmt: false
+                no_ntpdate: false
+                no_file_fmt_e: false
+            system_ready_script: |
+                #!/bin/bash
+                # upstart based, so use old style runlevels
+                [ $(runlevel | awk '{print $2}') = '2' ]
+        lxd:
+            sstreams_server: https://cloud-images.ubuntu.com/daily
+            alias: trusty
+            setup_overrides: null
+            override_templates: false
+    # DEBIAN =================================================================
     stretch:
-        platform_ident:
-            lxd:
-                alias: debian/stretch/default
-    wheezy:
-        platform_ident:
-            lxd:
-                alias: debian/wheezy/default
+        # EOL: Not yet released
+        default:
+            enabled: true
+            feature_groups:
+                - base
+                - debian_base
+        lxd:
+            alias: debian/stretch/default
+    jessie:
+        # EOL: Jun 2020
+        # NOTE: the cloud-init version shipped with jessie is out of date
+        #       tests work if an up to date deb is used
+        default:
+            enabled: true
+            feature_groups:
+                - base
+                - debian_base
+        lxd:
+            alias: debian/jessie/default
+    # CENTOS =================================================================
     centos70:
-        timeout: 180
-        platform_ident:
-            lxd:
-                alias: centos/7/default
+        # EOL: Jun 2024 (2020 - end of full updates)
+        default:
+            enabled: true
+            feature_groups:
+                - base
+                - rhel_base
+            user_data_overrides:
+                preserve_hostname: true
+        lxd:
+            features:
+                # NOTE: (LP: #1575779)
+                hostname: false
+            alias: centos/7/default
     centos66:
-        timeout: 180
-        platform_ident:
-            lxd:
-                alias: centos/6/default
+        # EOL: Nov 2020
+        default:
+            enabled: true
+            feature_groups:
+                - base
+                - rhel_base
+            # still supported, but only bugfixes after may 2017
+            system_ready_script: |
+                #!/bin/bash
+                [ $(runlevel | awk '{print $2}') = '3' ]
+            user_data_overrides:
+                preserve_hostname: true
+        lxd:
+            features:
+                # NOTE: (LP: #1575779)
+                hostname: false
+            alias: centos/6/default
 
 # vi: ts=4 expandtab