From dc0e70d155b4ff7a3c914ae7aaed3a52571e2107 Mon Sep 17 00:00:00 2001 From: Stephan Telling Date: Thu, 1 Jun 2017 22:08:29 +0200 Subject: fix typos and remove whitespace in various docs --- doc/rtd/topics/datasources.rst | 26 +++++++++++++------------- doc/rtd/topics/datasources/azure.rst | 2 +- doc/rtd/topics/dir_layout.rst | 14 +++++++------- doc/rtd/topics/merging.rst | 4 ++-- doc/rtd/topics/network-config-format-v1.rst | 4 ++-- doc/rtd/topics/network-config.rst | 4 ++-- doc/rtd/topics/tests.rst | 6 +++--- doc/rtd/topics/vendordata.rst | 4 ++-- 8 files changed, 32 insertions(+), 32 deletions(-) (limited to 'doc') diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst index 9acecc53..a60f5eb7 100644 --- a/doc/rtd/topics/datasources.rst +++ b/doc/rtd/topics/datasources.rst @@ -20,7 +20,7 @@ through the typical usage of subclasses. The current interface that a datasource object must provide is the following: .. sourcecode:: python - + # returns a mime multipart message that contains # all the various fully-expanded components that # were found from processing the raw userdata string @@ -28,47 +28,47 @@ The current interface that a datasource object must provide is the following: # this instance id will be returned (or messages with # no instance id) def get_userdata(self, apply_filter=False) - + # returns the raw userdata string (or none) def get_userdata_raw(self) - + # returns a integer (or none) which can be used to identify # this instance in a group of instances which are typically - # created from a single command, thus allowing programatic + # created from a single command, thus allowing programmatic # filtering on this launch index (or other selective actions) @property def launch_index(self) - - # the data sources' config_obj is a cloud-config formated + + # the data sources' config_obj is a cloud-config formatted # object that came to it from ways other than cloud-config # because cloud-config content would be handled elsewhere def get_config_obj(self) - + #returns a list of public ssh keys def get_public_ssh_keys(self) - + # translates a device 'short' name into the actual physical device # fully qualified name (or none if said physical device is not attached # or does not exist) def device_name_to_device(self, name) - + # gets the locale string this instance should be applying # which typically used to adjust the instances locale settings files def get_locale(self) - + @property def availability_zone(self) - + # gets the instance id that was assigned to this instance by the # cloud provider or when said instance id does not exist in the backing # metadata this will return 'iid-datasource' def get_instance_id(self) - + # gets the fully qualified domain name that this host should be using # when configuring network or hostname releated settings, typically # assigned either by the cloud provider or the user creating the vm def get_hostname(self, fqdn=False) - + def get_package_mirror_info(self) diff --git a/doc/rtd/topics/datasources/azure.rst b/doc/rtd/topics/datasources/azure.rst index 4a3735b5..559011ef 100644 --- a/doc/rtd/topics/datasources/azure.rst +++ b/doc/rtd/topics/datasources/azure.rst @@ -8,7 +8,7 @@ This datasource finds metadata and user-data from the Azure cloud platform. Azure Platform -------------- The azure cloud-platform provides initial data to an instance via an attached -CD formated in UDF. That CD contains a 'ovf-env.xml' file that provides some +CD formatted in UDF. That CD contains a 'ovf-env.xml' file that provides some information. Additional information is obtained via interaction with the "endpoint". diff --git a/doc/rtd/topics/dir_layout.rst b/doc/rtd/topics/dir_layout.rst index 3f5aa205..7a6265eb 100644 --- a/doc/rtd/topics/dir_layout.rst +++ b/doc/rtd/topics/dir_layout.rst @@ -41,9 +41,9 @@ Cloudinits's directory structure is somewhat different from a regular applicatio ``data/`` - Contains information releated to instance ids, datasources and hostnames of the previous + Contains information related to instance ids, datasources and hostnames of the previous and current instance if they are different. These can be examined as needed to - determine any information releated to a previous boot (if applicable). + determine any information related to a previous boot (if applicable). ``handlers/`` @@ -59,9 +59,9 @@ Cloudinits's directory structure is somewhat different from a regular applicatio ``instances/`` - All instances that were created using this image end up with instance identifer + All instances that were created using this image end up with instance identifier subdirectories (and corresponding data for each instance). The currently active - instance will be symlinked the the ``instance`` symlink file defined previously. + instance will be symlinked the ``instance`` symlink file defined previously. ``scripts/`` @@ -74,9 +74,9 @@ Cloudinits's directory structure is somewhat different from a regular applicatio ``sem/`` - Cloud-init has a concept of a module sempahore, which basically consists + Cloud-init has a concept of a module semaphore, which basically consists of the module name and its frequency. These files are used to ensure a module - is only ran `per-once`, `per-instance`, `per-always`. This folder contains - sempaphore `files` which are only supposed to run `per-once` (not tied to the instance id). + is only ran `per-once`, `per-instance`, `per-always`. This folder contains + semaphore `files` which are only supposed to run `per-once` (not tied to the instance id). .. vi: textwidth=78 diff --git a/doc/rtd/topics/merging.rst b/doc/rtd/topics/merging.rst index 2f927a47..c75ca59c 100644 --- a/doc/rtd/topics/merging.rst +++ b/doc/rtd/topics/merging.rst @@ -7,7 +7,7 @@ Overview This was implemented because it has been a common feature request that there be a way to specify how cloud-config yaml "dictionaries" provided as user-data are -merged together when there are multiple yamls to merge together (say when +merged together when there are multiple yaml files to merge together (say when performing an #include). Since previously the merging algorithm was very simple and would only overwrite @@ -128,7 +128,7 @@ for your own usage. for, both of which can define the way merging is done (the first header to exist wins). These new headers (in lookup order) are 'Merge-Type' and 'X-Merge-Type'. The value should be a string which will satisfy the new - merging format defintion (see below for this format). + merging format definition (see below for this format). 2. The second way is actually specifying the merge-type in the body of the cloud-config dictionary. There are 2 ways to specify this, either as a diff --git a/doc/rtd/topics/network-config-format-v1.rst b/doc/rtd/topics/network-config-format-v1.rst index 36326b59..ce3a1bde 100644 --- a/doc/rtd/topics/network-config-format-v1.rst +++ b/doc/rtd/topics/network-config-format-v1.rst @@ -246,8 +246,8 @@ Valid keys are: - jumbo0 params: bridge_ageing: 250 - bridge_bridgeprio: 22 - bridge_fd: 1 + bridge_bridgeprio: 22 + bridge_fd: 1 bridge_hello: 1 bridge_maxage: 10 bridge_maxwait: 0 diff --git a/doc/rtd/topics/network-config.rst b/doc/rtd/topics/network-config.rst index 109c86f5..96c1cf59 100644 --- a/doc/rtd/topics/network-config.rst +++ b/doc/rtd/topics/network-config.rst @@ -31,7 +31,7 @@ A ``network:`` entry in /etc/cloud/cloud.cfg.d/* configuration files. ``ip=`` or ``network-config=`` -User-data cannot change an instance's network configuration. In the absense +User-data cannot change an instance's network configuration. In the absence of network configuration in any of the above sources , `Cloud-init`_ will write out a network configuration that will issue a DHCP request on a "first" network interface. @@ -220,7 +220,7 @@ CLI Interface : --output-kind {eni,netplan,sysconfig}, -ok {eni,netplan,sysconfig} -Example output convertion V2 to sysconfig: +Example output converting V2 to sysconfig: .. code-block:: bash diff --git a/doc/rtd/topics/tests.rst b/doc/rtd/topics/tests.rst index 0663811e..60c63bce 100644 --- a/doc/rtd/topics/tests.rst +++ b/doc/rtd/topics/tests.rst @@ -158,7 +158,7 @@ Development Checklist * Named 'your_test_here.py' * Valid unit tests validating output collected * Passes pylint & pep8 checks - * Placed in the appropriate sub-folder in the testcsaes directory + * Placed in the appropriate sub-folder in the testcases directory * Tested by running the test: .. code-block:: bash @@ -222,7 +222,7 @@ collect can be ran by running: $ python3 -m tests.cloud_tests collect -n xenial -d /tmp/collection \ --deb cloud-init_0.7.8~my_patch_all.deb -The above command will run the collection tests on xenial with the +The above command will run the collection tests on Xenial with the provided deb and place all results into `/tmp/collection`. Verify @@ -249,7 +249,7 @@ configuration users can run the integration tests via tox: $ tox -e citest -- run -v -n zesty --deb=cloud-init_all.deb $ tox -e citest -- run -t module/user_groups.yaml -Users need to invoke the citest enviornment and then pass any additional +Users need to invoke the citest environment and then pass any additional arguments. diff --git a/doc/rtd/topics/vendordata.rst b/doc/rtd/topics/vendordata.rst index 2a94318e..cdb552d0 100644 --- a/doc/rtd/topics/vendordata.rst +++ b/doc/rtd/topics/vendordata.rst @@ -22,7 +22,7 @@ caveats: Users providing cloud-config data can use the '#cloud-config-jsonp' method to more finely control their modifications to the vendor supplied cloud-config. -For example, if both vendor and user have provided 'runcnmd' then the default +For example, if both vendor and user have provided 'runcmd' then the default merge handler will cause the user's runcmd to override the one provided by the vendor. To append to 'runcmd', the user could better provide multipart input with a cloud-config-jsonp part like: @@ -31,7 +31,7 @@ with a cloud-config-jsonp part like: #cloud-config-jsonp [{ "op": "add", "path": "/runcmd", "value": ["my", "command", "here"]}] - + Further, we strongly advise vendors to not 'be evil'. By evil, we mean any action that could compromise a system. Since users trust you, please take care to make sure that any vendordata is safe, -- cgit v1.2.3 From 802e7cb2da8e2d0225525160e6edd6b58b275b8c Mon Sep 17 00:00:00 2001 From: Vladimir Pouzanov Date: Tue, 2 May 2017 16:08:34 +0100 Subject: NoCloud: support seed of nocloud from smbios information This allows the user to seed NoCloud in a trivial way from qemu/libvirt, by using a stock image and passing a single command line flag. No custom command line, no filesystem modification, no bootstrap disk image. This is particularly handy now that Ec2 backend is discouraged from use under bug 1660385. LP: #1691772 --- cloudinit/sources/DataSourceNoCloud.py | 12 ++++++++++++ doc/rtd/topics/datasources/nocloud.rst | 22 ++++++++++++++++++++++ tools/ds-identify | 3 +++ 3 files changed, 37 insertions(+) (limited to 'doc') diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index c68f6b8c..e641244d 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -42,6 +42,18 @@ class DataSourceNoCloud(sources.DataSource): mydata = {'meta-data': {}, 'user-data': "", 'vendor-data': "", 'network-config': None} + try: + # Parse the system serial label from dmi. If not empty, try parsing + # like the commandline + md = {} + serial = util.read_dmi_data('system-serial-number') + if serial and load_cmdline_data(md, serial): + found.append("dmi") + mydata = _merge_new_seed(mydata, {'meta-data': md}) + except Exception: + util.logexc(LOG, "Unable to parse dmi data") + return False + try: # Parse the kernel command line, getting data passed in md = {} diff --git a/doc/rtd/topics/datasources/nocloud.rst b/doc/rtd/topics/datasources/nocloud.rst index 0159e853..665057f3 100644 --- a/doc/rtd/topics/datasources/nocloud.rst +++ b/doc/rtd/topics/datasources/nocloud.rst @@ -11,6 +11,28 @@ You can provide meta-data and user-data to a local vm boot via files on a `vfat`_ or `iso9660`_ filesystem. The filesystem volume label must be ``cidata``. +Alternatively, you can provide meta-data via kernel command line or SMBIOS +"serial number" option. The data must be passed in the form of a string: + +:: + + ds=nocloud[;key=val;key=val] + +or + +:: + + ds=nocloud-net[;key=val;key=val] + +e.g. you can pass this option to QEMU: + +:: + + -smbios type=1,serial=ds=nocloud-net;s=http://10.10.0.1:8000/ + +to cause NoCloud to fetch the full meta-data from http://10.10.0.1:8000/meta-data +after the network initialization is complete. + These user-data and meta-data files are expected to be in the following format. :: diff --git a/tools/ds-identify b/tools/ds-identify index 546e0f59..7c8b144b 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -555,6 +555,9 @@ dscheck_NoCloud() { case " ${DI_KERNEL_CMDLINE} " in *\ ds=nocloud*) return ${DS_FOUND};; esac + case " ${DI_DMI_PRODUCT_SERIAL} " in + *\ ds=nocloud*) return ${DS_FOUND};; + esac for d in nocloud nocloud-net; do check_seed_dir "$d" meta-data user-data && return ${DS_FOUND} done -- cgit v1.2.3 From ad2680a689ab78847ccce7766d6591797d99e219 Mon Sep 17 00:00:00 2001 From: JJ Asghar Date: Mon, 5 Jun 2017 20:36:12 -0500 Subject: Chef: Update omnibus url to chef.io, minor doc changes. - Updated to standard chef.io url - Removed the port 4000, due to that has been deprecated - Added Note about the run_list not being required Signed-off-by: JJ Asghar --- cloudinit/config/cc_chef.py | 2 +- doc/examples/cloud-config-chef.txt | 12 ++++++------ .../configs/examples/install_run_chef_recipes.yaml | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) (limited to 'doc') diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 2be2532c..02c70b10 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -92,7 +92,7 @@ REQUIRED_CHEF_DIRS = tuple([ ]) # Used if fetching chef from a omnibus style package -OMNIBUS_URL = "https://www.getchef.com/chef/install.sh" +OMNIBUS_URL = "https://www.chef.io/chef/install.sh" OMNIBUS_URL_RETRIES = 5 CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' diff --git a/doc/examples/cloud-config-chef.txt b/doc/examples/cloud-config-chef.txt index 3cb62006..9d235817 100644 --- a/doc/examples/cloud-config-chef.txt +++ b/doc/examples/cloud-config-chef.txt @@ -1,6 +1,6 @@ #cloud-config # -# This is an example file to automatically install chef-client and run a +# This is an example file to automatically install chef-client and run a # list of recipes when the instance boots for the first time. # Make sure that this file is valid yaml before starting instances. # It should be passed as user-data when starting the instance. @@ -8,7 +8,7 @@ # This example assumes the instance is 16.04 (xenial) -# The default is to install from packages. +# The default is to install from packages. # Key from https://packages.chef.io/chef.asc apt: @@ -60,7 +60,7 @@ chef: force_install: false # Chef settings - server_url: "https://chef.yourorg.com:4000" + server_url: "https://chef.yourorg.com" # Node Name # Defaults to the instance-id if not present @@ -78,8 +78,8 @@ chef: -----BEGIN RSA PRIVATE KEY----- YOUR-ORGS-VALIDATION-KEY-HERE -----END RSA PRIVATE KEY----- - - # A run list for a first boot json + + # A run list for a first boot json, an example (not required) run_list: - "recipe[apache2]" - "role[db]" @@ -92,7 +92,7 @@ chef: keepalive: "off" # if install_type is 'omnibus', change the url to download - omnibus_url: "https://www.opscode.com/chef/install.sh" + omnibus_url: "https://www.chef.io/chef/install.sh" # Capture all subprocess output into a logfile diff --git a/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml b/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml index 0bec305e..66b635a8 100644 --- a/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml +++ b/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml @@ -56,7 +56,7 @@ cloud_config: | force_install: false # Chef settings - server_url: "https://chef.yourorg.com:4000" + server_url: "https://chef.yourorg.com" # Node Name # Defaults to the instance-id if not present @@ -75,7 +75,7 @@ cloud_config: | YOUR-ORGS-VALIDATION-KEY-HERE -----END RSA PRIVATE KEY----- - # A run list for a first boot json + # A run list for a first boot json, this is an example (not required) run_list: - "recipe[apache2]" - "role[db]" @@ -88,7 +88,7 @@ cloud_config: | keepalive: "off" # if install_type is 'omnibus', change the url to download - omnibus_url: "https://www.opscode.com/chef/install.sh" + omnibus_url: "https://www.chef.io/chef/install.sh" # Capture all subprocess output into a logfile -- cgit v1.2.3 From 76d58265e34851b78e952a7f275340863c90a9f5 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Thu, 8 Jun 2017 18:23:31 -0400 Subject: Integration Testing: tox env, pyxld 2.2.3, and revamp framework Massive update to clean up and greatly enhance the integration testing framework developed by Wesley Wiedenmeier. - Updated tox environment to run integration test 'citest' to utilize pylxd 2.2.3 - Add support for distro feature flags - add framework for feature flags to release config with feature groups and overrides allowed in any release conf override level - add support for feature flags in platform and config handling - during collect, skip testcases that require features not supported by the image with a warning message - Enable additional distros (i.e. centos, debian) - Add 'bddeb' command to build a deb from the current working tree cleanly in a container, so deps do not have to be installed on host - Adds a command line option '--preserve-data' that ensures that collected data will be left after tests run. This also allows the directory to store collected data in during the run command to be specified using '--data-dir'. - Updated Read the Docs testing page and doc strings for pep 257 compliance --- doc/rtd/topics/tests.rst | 631 +++++++++++++++++---- tests/cloud_tests/__init__.py | 7 +- tests/cloud_tests/__main__.py | 45 +- tests/cloud_tests/args.py | 150 +++-- tests/cloud_tests/bddeb.py | 118 ++++ tests/cloud_tests/collect.py | 114 ++-- tests/cloud_tests/config.py | 139 +++-- tests/cloud_tests/configs/bugs/lp1628337.yaml | 3 + .../configs/examples/add_apt_repositories.yaml | 2 + .../configs/examples/install_run_chef_recipes.yaml | 6 +- .../configs/modules/apt_configure_conf.yaml | 2 + .../modules/apt_configure_disable_suites.yaml | 3 + .../configs/modules/apt_configure_primary.yaml | 7 + .../configs/modules/apt_configure_proxy.yaml | 2 + .../configs/modules/apt_configure_security.yaml | 3 + .../configs/modules/apt_configure_sources_key.yaml | 3 + .../modules/apt_configure_sources_keyserver.yaml | 3 + .../modules/apt_configure_sources_list.yaml | 3 + .../configs/modules/apt_configure_sources_ppa.yaml | 9 + .../configs/modules/apt_pipelining_disable.yaml | 2 + .../configs/modules/apt_pipelining_os.yaml | 2 + tests/cloud_tests/configs/modules/byobu.yaml | 2 + .../configs/modules/keys_to_console.yaml | 2 + tests/cloud_tests/configs/modules/landscape.yaml | 2 + tests/cloud_tests/configs/modules/locale.yaml | 3 + tests/cloud_tests/configs/modules/lxd_bridge.yaml | 2 + tests/cloud_tests/configs/modules/lxd_dir.yaml | 2 + tests/cloud_tests/configs/modules/ntp.yaml | 11 + tests/cloud_tests/configs/modules/ntp_pools.yaml | 10 + tests/cloud_tests/configs/modules/ntp_servers.yaml | 7 + .../modules/package_update_upgrade_install.yaml | 11 + .../cloud_tests/configs/modules/set_hostname.yaml | 2 + .../configs/modules/set_hostname_fqdn.yaml | 2 + .../cloud_tests/configs/modules/set_password.yaml | 2 + .../configs/modules/set_password_expire.yaml | 2 + tests/cloud_tests/configs/modules/snappy.yaml | 2 + .../modules/ssh_auth_key_fingerprints_disable.yaml | 2 + .../modules/ssh_auth_key_fingerprints_enable.yaml | 5 + .../cloud_tests/configs/modules/ssh_import_id.yaml | 3 + .../configs/modules/ssh_keys_generate.yaml | 2 + .../configs/modules/ssh_keys_provided.yaml | 3 + tests/cloud_tests/configs/modules/timezone.yaml | 2 + tests/cloud_tests/configs/modules/user_groups.yaml | 2 + tests/cloud_tests/configs/modules/write_files.yaml | 4 + tests/cloud_tests/images/__init__.py | 7 +- tests/cloud_tests/images/base.py | 68 +-- tests/cloud_tests/images/lxd.py | 176 ++++-- tests/cloud_tests/instances/__init__.py | 6 +- tests/cloud_tests/instances/base.py | 162 +++--- tests/cloud_tests/instances/lxd.py | 132 +++-- tests/cloud_tests/manage.py | 29 +- tests/cloud_tests/platforms.yaml | 50 +- tests/cloud_tests/platforms/__init__.py | 6 +- tests/cloud_tests/platforms/base.py | 44 +- tests/cloud_tests/platforms/lxd.py | 97 ++-- tests/cloud_tests/releases.yaml | 319 ++++++++--- tests/cloud_tests/run_funcs.py | 75 +++ tests/cloud_tests/setup_image.py | 196 ++++--- tests/cloud_tests/snapshots/__init__.py | 6 +- tests/cloud_tests/snapshots/base.py | 43 +- tests/cloud_tests/snapshots/lxd.py | 51 +- tests/cloud_tests/stage.py | 52 +- tests/cloud_tests/testcases.yaml | 1 + tests/cloud_tests/testcases/__init__.py | 16 +- tests/cloud_tests/testcases/base.py | 51 +- tests/cloud_tests/testcases/bugs/__init__.py | 4 +- tests/cloud_tests/testcases/bugs/lp1511485.py | 6 +- tests/cloud_tests/testcases/bugs/lp1628337.py | 8 +- tests/cloud_tests/testcases/examples/__init__.py | 4 +- .../testcases/examples/add_apt_repositories.py | 8 +- .../testcases/examples/alter_completion_message.py | 23 +- .../configure_instance_trusted_ca_certificates.py | 10 +- .../examples/configure_instances_ssh_keys.py | 12 +- .../testcases/examples/including_user_groups.py | 16 +- .../examples/install_arbitrary_packages.py | 8 +- .../testcases/examples/install_run_chef_recipes.py | 6 +- .../testcases/examples/run_apt_upgrade.py | 6 +- .../cloud_tests/testcases/examples/run_commands.py | 6 +- .../testcases/examples/run_commands_first_boot.py | 6 +- .../examples/writing_out_arbitrary_files.py | 12 +- tests/cloud_tests/testcases/main/__init__.py | 4 +- .../testcases/main/command_output_simple.py | 9 +- tests/cloud_tests/testcases/modules/__init__.py | 4 +- .../testcases/modules/apt_configure_conf.py | 8 +- .../modules/apt_configure_disable_suites.py | 6 +- .../testcases/modules/apt_configure_primary.py | 8 +- .../testcases/modules/apt_configure_proxy.py | 6 +- .../testcases/modules/apt_configure_security.py | 6 +- .../testcases/modules/apt_configure_sources_key.py | 8 +- .../modules/apt_configure_sources_keyserver.py | 8 +- .../modules/apt_configure_sources_list.py | 6 +- .../testcases/modules/apt_configure_sources_ppa.py | 8 +- .../testcases/modules/apt_pipelining_disable.py | 6 +- .../testcases/modules/apt_pipelining_os.py | 6 +- tests/cloud_tests/testcases/modules/bootcmd.py | 6 +- tests/cloud_tests/testcases/modules/byobu.py | 10 +- tests/cloud_tests/testcases/modules/ca_certs.py | 8 +- .../cloud_tests/testcases/modules/debug_disable.py | 6 +- .../cloud_tests/testcases/modules/debug_enable.py | 6 +- .../cloud_tests/testcases/modules/final_message.py | 23 +- .../testcases/modules/keys_to_console.py | 8 +- tests/cloud_tests/testcases/modules/locale.py | 8 +- tests/cloud_tests/testcases/modules/lxd_bridge.py | 10 +- tests/cloud_tests/testcases/modules/lxd_dir.py | 8 +- tests/cloud_tests/testcases/modules/ntp.py | 2 +- tests/cloud_tests/testcases/modules/ntp_pools.py | 4 +- .../modules/package_update_upgrade_install.py | 12 +- tests/cloud_tests/testcases/modules/runcmd.py | 6 +- tests/cloud_tests/testcases/modules/salt_minion.py | 10 +- .../testcases/modules/seed_random_data.py | 6 +- .../cloud_tests/testcases/modules/set_hostname.py | 6 +- .../testcases/modules/set_hostname_fqdn.py | 10 +- .../cloud_tests/testcases/modules/set_password.py | 8 +- .../testcases/modules/set_password_expire.py | 8 +- .../testcases/modules/set_password_list.py | 5 +- .../testcases/modules/set_password_list_string.py | 5 +- .../modules/ssh_auth_key_fingerprints_disable.py | 16 +- .../modules/ssh_auth_key_fingerprints_enable.py | 14 +- .../cloud_tests/testcases/modules/ssh_import_id.py | 6 +- .../testcases/modules/ssh_keys_generate.py | 22 +- .../testcases/modules/ssh_keys_provided.py | 24 +- tests/cloud_tests/testcases/modules/timezone.py | 6 +- tests/cloud_tests/testcases/modules/user_groups.py | 16 +- tests/cloud_tests/testcases/modules/write_files.py | 12 +- tests/cloud_tests/util.py | 235 ++++++-- tests/cloud_tests/verify.py | 22 +- tox.ini | 2 +- 127 files changed, 2511 insertions(+), 1193 deletions(-) create mode 100644 tests/cloud_tests/bddeb.py create mode 100644 tests/cloud_tests/run_funcs.py (limited to 'doc') diff --git a/doc/rtd/topics/tests.rst b/doc/rtd/topics/tests.rst index 60c63bce..d668e3f4 100644 --- a/doc/rtd/topics/tests.rst +++ b/doc/rtd/topics/tests.rst @@ -1,14 +1,186 @@ -**************** -Test Development -**************** - +******************* +Integration Testing +******************* Overview ======== -The purpose of this page is to describe how to write integration tests for -cloud-init. As a test writer you need to develop a test configuration and -a verification file: +This page describes the execution, development, and architecture of the +cloud-init integration tests: + +* Execution explains the options available and running of tests +* Development shows how to write test cases +* Architecture explains the internal processes + +Execution +========= + +Overview +-------- + +In order to avoid the need for dependencies and ease the setup and +configuration users can run the integration tests via tox: + +.. code-block:: bash + + $ git clone https://git.launchpad.net/cloud-init + $ cd cloud-init + $ tox -e citest -- -h + +Everything after the double dash will be passed to the integration tests. +Executing tests has several options: + +* ``run`` an alias to run both ``collect`` and ``verify``. The ``tree_run`` + command does the same thing, except uses a deb built from the current + working tree. + +* ``collect`` deploys on the specified platform and distro, patches with the + requested deb or rpm, and finally collects output of the arbitrary + commands. Similarly, ```tree_collect`` will collect output using a deb + built from the current working tree. + +* ``verify`` given a directory of test data, run the Python unit tests on + it to generate results. + +* ``bddeb`` will build a deb of the current working tree. + +Run +--- + +The first example will provide a complete end-to-end run of data +collection and verification. There are additional examples below +explaining how to run one or the other independently. + +.. code-block:: bash + + $ git clone https://git.launchpad.net/cloud-init + $ cd cloud-init + $ tox -e citest -- run --verbose \ + --os-name stretch --os-name xenial \ + --deb cloud-init_0.7.8~my_patch_all.deb \ + --preserve-data --data-dir ~/collection + +The above command will do the following: + +* ``run`` both collect output and run tests the output + +* ``--verbose`` verbose output + +* ``--os-name stretch`` on the Debian Stretch release + +* ``--os-name xenial`` on the Ubuntu Xenial release + +* ``--deb cloud-init_0.7.8~patch_all.deb`` use this deb as the version of + cloud-init to run with + +* ``--preserve-data`` always preserve collected data, do not remove data + after successful test run + +* ``--data-dir ~/collection`` write collected data into `~/collection`, + rather than using a temporary directory + +For a more detailed explanation of each option see below. + +.. note:: + By default, data collected by the run command will be written into a + temporary directory and deleted after a successful. If you would + like to preserve this data, please use the option ``--preserve-data``. + +Collect +------- + +If developing tests it may be necessary to see if cloud-config works as +expected and the correct files are pulled down. In this case only a +collect can be ran by running: + +.. code-block:: bash + + $ tox -e citest -- collect -n xenial --data-dir /tmp/collection + +The above command will run the collection tests on xenial and place +all results into `/tmp/collection`. + +Verify +------ + +When developing tests it is much easier to simply rerun the verify scripts +without the more lengthy collect process. This can be done by running: + +.. code-block:: bash + + $ tox -e citest -- verify --data-dir /tmp/collection + +The above command will run the verify scripts on the data discovered in +`/tmp/collection`. + +TreeRun and TreeCollect +----------------------- + +If working on a cloud-init feature or resolving a bug, it may be useful to +run the current copy of cloud-init in the integration testing environment. +The integration testing suite can automatically build a deb based on the +current working tree of cloud-init and run the test suite using this deb. + +The ``tree_run`` and ``tree_collect`` commands take the same arguments as +the ``run`` and ``collect`` commands. These commands will build a deb and +write it into a temporary file, then start the test suite and pass that deb +in. To build a deb only, and not run the test suite, the ``bddeb`` command +can be used. + +Note that code in the cloud-init working tree that has not been committed +when the cloud-init deb is built will still be included. To build a +cloud-init deb from or use the ``tree_run`` command using a copy of +cloud-init located in a different directory, use the option ``--cloud-init +/path/to/cloud-init``. + +.. code-block:: bash + + $ tox -e citest -- tree_run --verbose \ + --os-name xenial --os-name stretch \ + --test modules/final_message --test modules/write_files \ + --result /tmp/result.yaml + +Bddeb +----- + +The ``bddeb`` command can be used to generate a deb file. This is used by +the tree_run and tree_collect commands to build a deb of the current +working tree. It can also be used a user to generate a deb for use in other +situations and avoid needing to have all the build and test dependencies +installed locally. + +* ``--bddeb-args``: arguments to pass through to bddeb +* ``--build-os``: distribution to use as build system (default is xenial) +* ``--build-platform``: platform to use for build system (default is lxd) +* ``--cloud-init``: path to base of cloud-init tree (default is '.') +* ``--deb``: path to write output deb to (default is '.') + +Setup Image +----------- + +By default an image that is used will remain unmodified, but certain +scenarios may require image modification. For example, many images may use +a much older cloud-init. As a result tests looking at newer functionality +will fail because a newer version of cloud-init may be required. The +following options can be used for further customization: + +* ``--deb``: install the specified deb into the image +* ``--rpm``: install the specified rpm into the image +* ``--repo``: enable a repository and upgrade cloud-init afterwards +* ``--ppa``: enable a ppa and upgrade cloud-init afterwards +* ``--upgrade``: upgrade cloud-init from repos +* ``--upgrade-full``: run a full system upgrade +* ``--script``: execute a script in the image. This can perform any setup + required that is not covered by the other options + +Test Case Development +===================== + +Overview +-------- + +As a test writer you need to develop a test configuration and a +verification file: * The test configuration specifies a specific cloud-config to be used by cloud-init and a list of arbitrary commands to capture the output of @@ -21,20 +193,28 @@ The names must match, however the extensions will of course be different, yaml vs py. Configuration -============= +------------- The test configuration is a YAML file such as *ntp_server.yaml* below: .. code-block:: yaml # - # NTP config using specific servers (ntp_server.yaml) + # Empty NTP config to setup using defaults # + # NOTE: this should not require apt feature, use 'which' rather than 'dpkg -l' + # NOTE: this should not require no_ntpdate feature, use 'which' to check for + # installation rather than 'dpkg -l', as 'grep ntp' matches 'ntpdate' + # NOTE: the verifier should check for any ntp server not 'ubuntu.pool.ntp.org' cloud_config: | #cloud-config ntp: servers: - pool.ntp.org + required_features: + - apt + - no_ntpdate + - ubuntu_ntp collect_scripts: ntp_installed_servers: | #!/bin/bash @@ -46,21 +226,30 @@ The test configuration is a YAML file such as *ntp_server.yaml* below: #!/bin/bash cat /etc/ntp.conf | grep '^server' - -There are two keys, 1 required and 1 optional, in the YAML file: +There are several keys, 1 required and some optional, in the YAML file: 1. The required key is ``cloud_config``. This should be a string of valid - YAML that is exactly what would normally be placed in a cloud-config file, - including the cloud-config header. This essentially sets up the scenario - under test. + YAML that is exactly what would normally be placed in a cloud-config + file, including the cloud-config header. This essentially sets up the + scenario under test. -2. The optional key is ``collect_scripts``. This key has one or more +2. One optional key is ``collect_scripts``. This key has one or more sub-keys containing strings of arbitrary commands to execute (e.g. ```cat /var/log/cloud-config-output.log```). In the example above the output of dpkg is captured, grep for ntp, and the number of lines reported. The name of the sub-key is important. The sub-key is used by the verification script to recall the output of the commands ran. +3. The optional ``enabled`` key enables or disables the test case. By + default the test case will be enabled. + +4. The optional ``required_features`` key may be used to specify a list + of features flags that an image must have to be able to run the test + case. For example, if a test case relies on an image supporting apt, + then the config for the test case should include ``required_features: + [ apt ]``. + + Default Collect Scripts ----------------------- @@ -75,51 +264,68 @@ no need to specify these items: * ```dpkg-query -W -f='${Version}' cloud-init``` Verification -============ +------------ The verification script is a Python file with unit tests like the one, `ntp_server.py`, below: .. code-block:: python - """cloud-init Integration Test Verify Script (ntp_server.yaml)""" + # This file is part of cloud-init. See LICENSE file for license information. + + """cloud-init Integration Test Verify Script""" from tests.cloud_tests.testcases import base - class TestNtpServers(base.CloudTestCase): + class TestNtp(base.CloudTestCase): """Test ntp module""" def test_ntp_installed(self): """Test ntp installed""" - out = self.get_data_file('ntp_installed_servers') + out = self.get_data_file('ntp_installed_empty') self.assertEqual(1, int(out)) def test_ntp_dist_entries(self): """Test dist config file has one entry""" - out = self.get_data_file('ntp_conf_dist_servers') + out = self.get_data_file('ntp_conf_dist_empty') self.assertEqual(1, int(out)) def test_ntp_entires(self): """Test config entries""" - out = self.get_data_file('ntp_conf_servers') - self.assertIn('server pool.ntp.org iburst', out) + out = self.get_data_file('ntp_conf_empty') + self.assertIn('pool 0.ubuntu.pool.ntp.org iburst', out) + self.assertIn('pool 1.ubuntu.pool.ntp.org iburst', out) + self.assertIn('pool 2.ubuntu.pool.ntp.org iburst', out) + self.assertIn('pool 3.ubuntu.pool.ntp.org iburst', out) + + # vi: ts=4 expandtab Here is a breakdown of the unit test file: * The import statement allows access to the output files. -* The class can be named anything, but must import the ``base.CloudTestCase`` +* The class can be named anything, but must import the + ``base.CloudTestCase``, either directly or via another test class. * There can be 1 to N number of functions with any name, however only - tests starting with ``test_*`` will be executed. + functions starting with ``test_*`` will be executed. + +* There can be 1 to N number of classes in a test module, however only + classes inheriting from ``base.CloudTestCase`` will be loaded. * Output from the commands can be accessed via ``self.get_data_file('key')`` where key is the sub-key of ``collect_scripts`` above. +* The cloud config that the test ran with can be accessed via + ``self.cloud_config``, or any entry from the cloud config can be accessed + via ``self.get_config_entry('key')``. + +* See the base ``CloudTestCase`` for additional helper functions. + Layout -====== +------ Integration tests are located under the `tests/cloud_tests` directory. Test configurations are placed under `configs` and the test verification @@ -144,126 +350,65 @@ The sub-folders of bugs, examples, main, and modules help organize the tests. View the README.md in each to understand in more detail each directory. +Test Creation Helper +-------------------- + +The integration testing suite has a built in helper to aid in test +development. Help can be invoked via ``tox -e citest -- create --help``. It +can create a template test case config file with user data passed in from +the command line, as well as a template test case verifier module. + +The following would create a test case named ``example`` under the +``modules`` category with the given description, and cloud config data read +in from ``/tmp/user_data``. + +.. code-block:: bash + + $ tox -e citest -- create modules/example \ + -d "a simple example test case" -c "$(< /tmp/user_data)" + Development Checklist -===================== +--------------------- * Configuration File - * Named 'your_test_here.yaml' + * Named 'your_test.yaml' * Contains at least a valid cloud-config * Optionally, commands to capture additional output * Valid YAML * Placed in the appropriate sub-folder in the configs directory + * Any image features required for the test are specified * Verification File - * Named 'your_test_here.py' + * Named 'your_test.py' * Valid unit tests validating output collected * Passes pylint & pep8 checks - * Placed in the appropriate sub-folder in the testcases directory + * Placed in the appropriate sub-folder in the test cases directory * Tested by running the test: .. code-block:: bash - $ python3 -m tests.cloud_tests run -v -n \ - --deb \ - -t tests/cloud_tests/configs//your_test_here.yaml - - -Execution -========= - -Executing tests has three options: - -* ``run`` an alias to run both ``collect`` and ``verify`` - -* ``collect`` deploys on the specified platform and os, patches with the - requested deb or rpm, and finally collects output of the arbitrary - commands. - -* ``verify`` given a directory of test data, run the Python unit tests on - it to generate results. - -Run ---- -The first example will provide a complete end-to-end run of data -collection and verification. There are additional examples below -explaining how to run one or the other independently. - -.. code-block:: bash - - $ git clone https://git.launchpad.net/cloud-init - $ cd cloud-init - $ python3 -m tests.cloud_tests run -v -n trusty -n xenial \ - --deb cloud-init_0.7.8~my_patch_all.deb - -The above command will do the following: - -* ``-v`` verbose output - -* ``run`` both collect output and run tests the output - -* ``-n trusty`` on the Ubuntu Trusty release - -* ``-n xenial`` on the Ubuntu Xenial release - -* ``--deb cloud-init_0.7.8~patch_all.deb`` use this deb as the version of - cloud-init to run with - -For a more detailed explanation of each option see below. - -Collect -------- - -If developing tests it may be necessary to see if cloud-config works as -expected and the correct files are pulled down. In this case only a -collect can be ran by running: - -.. code-block:: bash - - $ python3 -m tests.cloud_tests collect -n xenial -d /tmp/collection \ - --deb cloud-init_0.7.8~my_patch_all.deb - -The above command will run the collection tests on Xenial with the -provided deb and place all results into `/tmp/collection`. - -Verify ------- - -When developing tests it is much easier to simply rerun the verify scripts -without the more lengthy collect process. This can be done by running: - -.. code-block:: bash - - $ python3 -m tests.cloud_tests verify -d /tmp/collection - -The above command will run the verify scripts on the data discovered in -`/tmp/collection`. - -Run via tox ------------ -In order to avoid the need for dependencies and ease the setup and -configuration users can run the integration tests via tox: - -.. code-block:: bash - - $ tox -e citest -- run [integration test arguments] - $ tox -e citest -- run -v -n zesty --deb=cloud-init_all.deb - $ tox -e citest -- run -t module/user_groups.yaml - -Users need to invoke the citest environment and then pass any additional -arguments. - + $ tox -e citest -- run -verbose \ + --os-name \ + --test modules/your_test.yaml \ + [--deb ] Architecture ============ -The following outlines the process flow during a complete end-to-end LXD-backed test. +The following section outlines the high-level architecture of the +integration process. + +Overview +-------- +The process flow during a complete end-to-end LXD-backed test. 1. Configuration - * The back end and specific OS releases are verified as supported - * The test or tests that need to be run are determined either by directory or by individual yaml + * The back end and specific distro releases are verified as supported + * The test or tests that need to be run are determined either by + directory or by individual yaml 2. Image Creation - * Acquire the daily LXD image + * Acquire the request LXD image * Install the specified cloud-init package * Clean the image so that it does not appear to have been booted * A snapshot of the image is created and reused by all tests @@ -285,5 +430,247 @@ The following outlines the process flow during a complete end-to-end LXD-backed 5. Results * If any failures were detected the test suite returns a failure + * Results can be dumped in yaml format to a specified file using the + ``-r .yaml`` option + +Configuring the Test Suite +-------------------------- + +Most of the behavior of the test suite is configurable through several yaml +files. These control the behavior of the test suite's platforms, images, and +tests. The main config files for platforms, images and test cases are +``platforms.yaml``, ``releases.yaml`` and ``testcases.yaml``. +Config handling +^^^^^^^^^^^^^^^ + +All configurable parts of the test suite use a defaults + overrides system +for managing config entries. All base config items are dictionaries. + +Merging is done on a key-by-key basis, with all keys in the default and +override represented in the final result. If a key exists both in +the defaults and the overrides, then the behavior depends on the type of data +the key refers to. If it is atomic data or a list, then the overrides will +replace the default. If the data is a dictionary then the value will be the +result of merging that dictionary from the default config and that +dictionary from the overrides. + +Merging is done using the function +``tests.cloud_tests.config.merge_config``, which can be examined for more +detail on config merging behavior. + +The following demonstrates merge behavior: + +.. code-block:: yaml + + defaults: + list_item: + - list_entry_1 + - list_entry_2 + int_item_1: 123 + int_item_2: 234 + dict_item: + subkey_1: 1 + subkey_2: 2 + subkey_dict: + subsubkey_1: a + subsubkey_2: b + + overrides: + list_item: + - overridden_list_entry + int_item_1: 0 + dict_item: + subkey_2: false + subkey_dict: + subsubkey_2: 'new value' + + result: + list_item: + - overridden_list_entry + int_item_1: 0 + int_item_2: 234 + dict_item: + subkey_1: 1 + subkey_2: false + subkey_dict: + subsubkey_1: a + subsubkey_2: 'new value' + + +Image Config +------------ + +Image configuration is handled in ``releases.yaml``. The image configuration +controls how platforms locate and acquire images, how the platforms should +interact with the images, how platforms should detect when an image has +fully booted, any options that are required to set the image up, and +features that the image supports. + +Since settings for locating an image and interacting with it differ from +platform to platform, there are 4 levels of settings available for images on +top of the default image settings. The structure of the image config file +is: + +.. code-block:: yaml + + default_release_config: + default: + ... + : + ... + : + ... + + releases: + : + : + ... + : + ... + : + ... + + +The base config is created from the overall defaults and the overrides for +the platform. The overrides are created from the default config for the +image and the platform specific overrides for the image. + +System Boot +^^^^^^^^^^^ + +The test suite must be able to test if a system has fully booted and if +cloud-init has finished running, so that running collect scripts does not +race against the target image booting. This is done using the +``system_ready_script`` and ``cloud_init_ready_script`` image config keys. + +Each of these keys accepts a small bash test statement as a string that must +return 0 or 1. Since this test statement will be added into a larger bash +statement it must be a single statement using the ``[`` test syntax. + +The default image config provides a system ready script that works for any +systemd based image. If the image is not systemd based, then a different +test statement must be provided. The default config also provides a test +for whether or not cloud-init has finished which checks for the file +``/run/cloud-init/result.json``. This should be sufficient for most systems +as writing this file is one of the last things cloud-init does. + +The setting ``boot_timeout`` controls how long, in seconds, the platform +should wait for an image to boot. If the system ready script has not +indicated that the system is fully booted within this time an error will be +raised. + +Feature Flags +^^^^^^^^^^^^^ + +Not all test cases can work on all images due to features the test case +requires not being present on that image. If a test case requires features +in an image that are not likely to be present across all distros and +platforms that the test suite supports, then the test can be skipped +everywhere it is not supported. + +Feature flags, which are names for features supported on some images, but +not all that may be required by test cases. Configuration for feature flags +is provided in ``releases.yaml`` under the ``features`` top level key. The +features config includes a list of all currently defined feature flags, +their meanings, and a list of feature groups. + +Feature groups are groups of features that many images have in common. For +example, the ``Ubuntu_specific`` feature group includes features that +should be present across most Ubuntu releases, but may or may not be for +other distros. Feature groups are specified for an image as a list under +the key ``feature_groups``. + +An image's feature flags are derived from the features groups that that +image has and any feature overrides provided. Feature overrides can be +specified under the ``features`` key which accepts a dictionary of +``{: true/false}`` mappings. If a feature is omitted from an +image's feature flags or set to false in the overrides then the test suite +will skip any tests that require that feature when using that image. + +Feature flags may be overridden at run time using the ``--feature-override`` +command line argument. It accepts a feature flag and value to set in the +format ``=true/false``. Multiple ``--feature-override`` +flags can be used, and will all be applied to all feature flags for images +used during a test. + +Setup Overrides +^^^^^^^^^^^^^^^ + +If an image requires some of the options for image setup to be used, then it +may specify overrides for the command line arguments passed into setup +image. These may be specified as a dictionary under the ``setup_overrides`` +key. When an image is set up, the arguments that control how it is set up +will be the arguments from the command line, with any entries in +``setup_overrides`` used to override these arguments. + +For example, images that do not come with cloud-init already installed +should have ``setup_overrides: {upgrade: true}`` specified so that in the +event that no additional setup options are given, cloud-init will be +installed from the image's repos before running tests. Note that if other +options such as ``--deb`` are passed in on the command line, these will +still work as expected, since apt's policy for cloud-init would prefer the +locally installed deb over an older version from the repos. + +Platform Specific Options +^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are many platform specific options in image configuration that allow +platforms to locate images and that control additional setup that the +platform may have to do to make the image usable. For information on how +these work, please consult the documentation for that platform in the +integration testing suite and the ``releases.yaml`` file for examples. + +Error Handling +-------------- + +The test suite makes an attempt to run as many tests as possible even in the +event of some failing so that automated runs collect as much data as +possible. In the event that something goes wrong while setting up for or +running a test, the test suite will attempt to continue running any tests +which have not been affected by the error. + +For example, if the test suite was told to run tests on one platform for two +releases and an error occurred setting up the first image, all tests for +that image would be skipped, and the test suite would continue to set up +the second image and run tests on it. Or, if the system does not start +properly for one test case out of many to run on that image, that test case +will be skipped and the next one will be run. + +Note that if any errors occur, the test suite will record the failure and +where it occurred in the result data and write it out to the specified +result file. + +Results +------- +The test suite generates result data that includes how long each stage of +the test suite took and which parts were and were not successful. This data +is dumped to the log after the collect and verify stages, and may also be +written out in yaml format to a file. If part of the setup failed, the +traceback for the failure and the error message will be included in the +result file. If a test verifier finds a problem with the collected data +from a test run, the class, test function and test will be recorded in the +result data. + +Exit Codes +^^^^^^^^^^ + +The test suite counts how many errors occur throughout a run. The exit code +after a run is the number of errors that occurred. If the exit code is +non-zero then something is wrong either with the test suite, the +configuration for an image, a test case, or cloud-init itself. + +Note that the exit code does not always directly correspond to the number +of failed test cases, since in some cases, a single error during image setup +can mean that several test cases are not run. If run is used, then the exit +code will be the sum of the number of errors in the collect and verify +stages. + +Data Dir +^^^^^^^^ + +When using run, the collected data is written into a temporary directory. In +the event that all tests pass, this directory is deleted, but if a test +fails or an error occurs, this data will be left in place, and a message +will be written to the log giving the location of the data. diff --git a/tests/cloud_tests/__init__.py b/tests/cloud_tests/__init__.py index 099c357f..07148c12 100644 --- a/tests/cloud_tests/__init__.py +++ b/tests/cloud_tests/__init__.py @@ -1,17 +1,18 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Main init.""" + import logging import os BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TESTCASES_DIR = os.path.join(BASE_DIR, 'testcases') TEST_CONF_DIR = os.path.join(BASE_DIR, 'configs') +TREE_BASE = os.sep.join(BASE_DIR.split(os.sep)[:-2]) def _initialize_logging(): - """ - configure logging for cloud_tests - """ + """Configure logging for cloud_tests.""" logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) formatter = logging.Formatter( diff --git a/tests/cloud_tests/__main__.py b/tests/cloud_tests/__main__.py index ed654ad3..260ddb3f 100644 --- a/tests/cloud_tests/__main__.py +++ b/tests/cloud_tests/__main__.py @@ -1,19 +1,17 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Main entry point.""" + import argparse import logging -import shutil import sys -import tempfile -from tests.cloud_tests import (args, collect, manage, verify) +from tests.cloud_tests import args, bddeb, collect, manage, run_funcs, verify from tests.cloud_tests import LOG def configure_log(args): - """ - configure logging - """ + """Configure logging.""" level = logging.INFO if args.verbose: level = logging.DEBUG @@ -22,41 +20,15 @@ def configure_log(args): LOG.setLevel(level) -def run(args): - """ - run full test suite - """ - failed = 0 - args.data_dir = tempfile.mkdtemp(prefix='cloud_test_data_') - LOG.debug('using tmpdir %s', args.data_dir) - try: - failed += collect.collect(args) - failed += verify.verify(args) - except Exception: - failed += 1 - raise - finally: - # TODO: make this configurable via environ or cmdline - if failed: - LOG.warning('some tests failed, leaving data in %s', args.data_dir) - else: - shutil.rmtree(args.data_dir) - return failed - - def main(): - """ - entry point for cloud test suite - """ + """Entry point for cloud test suite.""" # configure parser parser = argparse.ArgumentParser(prog='cloud_tests') subparsers = parser.add_subparsers(dest="subcmd") subparsers.required = True def add_subparser(name, description, arg_sets): - """ - add arguments to subparser - """ + """Add arguments to subparser.""" subparser = subparsers.add_parser(name, help=description) for (_args, _kwargs) in (a for arg_set in arg_sets for a in arg_set): subparser.add_argument(*_args, **_kwargs) @@ -80,9 +52,12 @@ def main(): # run handler LOG.debug('running with args: %s\n', parsed) return { + 'bddeb': bddeb.bddeb, 'collect': collect.collect, 'create': manage.create, - 'run': run, + 'run': run_funcs.run, + 'tree_collect': run_funcs.tree_collect, + 'tree_run': run_funcs.tree_run, 'verify': verify.verify, }[parsed.subcmd](parsed) diff --git a/tests/cloud_tests/args.py b/tests/cloud_tests/args.py index 371b0444..369d60db 100644 --- a/tests/cloud_tests/args.py +++ b/tests/cloud_tests/args.py @@ -1,23 +1,43 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Argparse argument setup and sanitization.""" + import os from tests.cloud_tests import config, util -from tests.cloud_tests import LOG +from tests.cloud_tests import LOG, TREE_BASE ARG_SETS = { + 'BDDEB': ( + (('--bddeb-args',), + {'help': 'args to pass through to bddeb', + 'action': 'store', 'default': None, 'required': False}), + (('--build-os',), + {'help': 'OS to use as build system (default is xenial)', + 'action': 'store', 'choices': config.ENABLED_DISTROS, + 'default': 'xenial', 'required': False}), + (('--build-platform',), + {'help': 'platform to use for build system (default is lxd)', + 'action': 'store', 'choices': config.ENABLED_PLATFORMS, + 'default': 'lxd', 'required': False}), + (('--cloud-init',), + {'help': 'path to base of cloud-init tree', 'metavar': 'DIR', + 'action': 'store', 'required': False, 'default': TREE_BASE}),), 'COLLECT': ( (('-p', '--platform'), {'help': 'platform(s) to run tests on', 'metavar': 'PLATFORM', - 'action': 'append', 'choices': config.list_enabled_platforms(), + 'action': 'append', 'choices': config.ENABLED_PLATFORMS, 'default': []}), (('-n', '--os-name'), {'help': 'the name(s) of the OS(s) to test', 'metavar': 'NAME', - 'action': 'append', 'choices': config.list_enabled_distros(), + 'action': 'append', 'choices': config.ENABLED_DISTROS, 'default': []}), (('-t', '--test-config'), {'help': 'test config file(s) to use', 'metavar': 'FILE', - 'action': 'append', 'default': []}),), + 'action': 'append', 'default': []}), + (('--feature-override',), + {'help': 'feature flags override(s), =', + 'action': 'append', 'default': [], 'required': False}),), 'CREATE': ( (('-c', '--config'), {'help': 'cloud-config yaml for testcase', 'metavar': 'DATA', @@ -41,7 +61,15 @@ ARG_SETS = { 'OUTPUT': ( (('-d', '--data-dir'), {'help': 'directory to store test data in', - 'action': 'store', 'metavar': 'DIR', 'required': True}),), + 'action': 'store', 'metavar': 'DIR', 'required': False}), + (('--preserve-data',), + {'help': 'do not remove collected data after successful run', + 'action': 'store_true', 'default': False, 'required': False}),), + 'OUTPUT_DEB': ( + (('--deb',), + {'help': 'path to write output deb to', 'metavar': 'FILE', + 'action': 'store', 'required': False, + 'default': 'cloud-init_all.deb'}),), 'RESULT': ( (('-r', '--result'), {'help': 'file to write results to', @@ -61,31 +89,54 @@ ARG_SETS = { {'help': 'ppa to enable (implies -u)', 'metavar': 'NAME', 'action': 'store'}), (('-u', '--upgrade'), - {'help': 'upgrade before starting tests', 'action': 'store_true', - 'default': False}),), + {'help': 'upgrade or install cloud-init from repo', + 'action': 'store_true', 'default': False}), + (('--upgrade-full',), + {'help': 'do full system upgrade from repo (implies -u)', + 'action': 'store_true', 'default': False}),), + } SUBCMDS = { + 'bddeb': ('build cloud-init deb from tree', + ('BDDEB', 'OUTPUT_DEB', 'INTERFACE')), 'collect': ('collect test data', ('COLLECT', 'INTERFACE', 'OUTPUT', 'RESULT', 'SETUP')), 'create': ('create new test case', ('CREATE', 'INTERFACE')), - 'run': ('run test suite', ('COLLECT', 'INTERFACE', 'RESULT', 'SETUP')), + 'run': ('run test suite', + ('COLLECT', 'INTERFACE', 'RESULT', 'OUTPUT', 'SETUP')), + 'tree_collect': ('collect using current working tree', + ('BDDEB', 'COLLECT', 'INTERFACE', 'OUTPUT', 'RESULT')), + 'tree_run': ('run using current working tree', + ('BDDEB', 'COLLECT', 'INTERFACE', 'OUTPUT', 'RESULT')), 'verify': ('verify test data', ('INTERFACE', 'OUTPUT', 'RESULT')), } def _empty_normalizer(args): + """Do not normalize arguments.""" + return args + + +def normalize_bddeb_args(args): + """Normalize BDDEB arguments. + + @param args: parsed args + @return_value: updated args, or None if errors encountered """ - do not normalize arguments - """ + # make sure cloud-init dir is accessible + if not (args.cloud_init and os.path.isdir(args.cloud_init)): + LOG.error('invalid cloud-init tree path') + return None + return args def normalize_create_args(args): - """ - normalize CREATE arguments - args: parsed args - return_value: updated args, or None if errors occurred + """Normalize CREATE arguments. + + @param args: parsed args + @return_value: updated args, or None if errors occurred """ # ensure valid name for new test if len(args.name.split('/')) != 2: @@ -114,22 +165,22 @@ def normalize_create_args(args): def normalize_collect_args(args): - """ - normalize COLLECT arguments - args: parsed args - return_value: updated args, or None if errors occurred + """Normalize COLLECT arguments. + + @param args: parsed args + @return_value: updated args, or None if errors occurred """ # platform should default to all supported if len(args.platform) == 0: - args.platform = config.list_enabled_platforms() + args.platform = config.ENABLED_PLATFORMS args.platform = util.sorted_unique(args.platform) # os name should default to all enabled # if os name is provided ensure that all provided are supported if len(args.os_name) == 0: - args.os_name = config.list_enabled_distros() + args.os_name = config.ENABLED_DISTROS else: - supported = config.list_enabled_distros() + supported = config.ENABLED_DISTROS invalid = [os_name for os_name in args.os_name if os_name not in supported] if len(invalid) != 0: @@ -158,18 +209,33 @@ def normalize_collect_args(args): args.test_config = valid args.test_config = util.sorted_unique(args.test_config) + # parse feature flag overrides and ensure all are valid + if args.feature_override: + overrides = args.feature_override + args.feature_override = util.parse_conf_list( + overrides, boolean=True, valid=config.list_feature_flags()) + if not args.feature_override: + LOG.error('invalid feature flag override(s): %s', overrides) + return None + else: + args.feature_override = {} + return args def normalize_output_args(args): + """Normalize OUTPUT arguments. + + @param args: parsed args + @return_value: updated args, or None if errors occurred """ - normalize OUTPUT arguments - args: parsed args - return_value: updated args, or None if errors occurred - """ + if args.data_dir: + args.data_dir = os.path.abspath(args.data_dir) + if not os.path.exists(args.data_dir): + os.mkdir(args.data_dir) + if not args.data_dir: - LOG.error('--data-dir must be specified') - return None + args.data_dir = None # ensure clean output dir if collect # ensure data exists if verify @@ -177,19 +243,31 @@ def normalize_output_args(args): if not util.is_clean_writable_dir(args.data_dir): LOG.error('data_dir must be empty/new and must be writable') return None - elif args.subcmd == 'verify': - if not os.path.exists(args.data_dir): - LOG.error('data_dir %s does not exist', args.data_dir) - return None return args -def normalize_setup_args(args): +def normalize_output_deb_args(args): + """Normalize OUTPUT_DEB arguments. + + @param args: parsed args + @return_value: updated args, or None if erros occurred """ - normalize SETUP arguments - args: parsed args - return_value: updated_args, or None if errors occurred + # make sure to use abspath for deb + args.deb = os.path.abspath(args.deb) + + if not args.deb.endswith('.deb'): + LOG.error('output filename does not end in ".deb"') + return None + + return args + + +def normalize_setup_args(args): + """Normalize SETUP arguments. + + @param args: parsed args + @return_value: updated_args, or None if errors occurred """ # ensure deb or rpm valid if specified for pkg in (args.deb, args.rpm): @@ -210,10 +288,12 @@ def normalize_setup_args(args): NORMALIZERS = { + 'BDDEB': normalize_bddeb_args, 'COLLECT': normalize_collect_args, 'CREATE': normalize_create_args, 'INTERFACE': _empty_normalizer, 'OUTPUT': normalize_output_args, + 'OUTPUT_DEB': normalize_output_deb_args, 'RESULT': _empty_normalizer, 'SETUP': normalize_setup_args, } diff --git a/tests/cloud_tests/bddeb.py b/tests/cloud_tests/bddeb.py new file mode 100644 index 00000000..53dbf74e --- /dev/null +++ b/tests/cloud_tests/bddeb.py @@ -0,0 +1,118 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Used to build a deb.""" + +from functools import partial +import os +import tempfile + +from cloudinit import util as c_util +from tests.cloud_tests import (config, LOG) +from tests.cloud_tests import (platforms, images, snapshots, instances) +from tests.cloud_tests.stage import (PlatformComponent, run_stage, run_single) + +build_deps = ['devscripts', 'equivs', 'git', 'tar'] + + +def _out(cmd_res): + """Get clean output from cmd result.""" + return cmd_res[0].strip() + + +def build_deb(args, instance): + """Build deb on system and copy out to location at args.deb. + + @param args: cmdline arguments + @return_value: tuple of results and fail count + """ + # update remote system package list and install build deps + LOG.debug('installing build deps') + pkgs = ' '.join(build_deps) + cmd = 'apt-get update && apt-get install --yes {}'.format(pkgs) + instance.execute(['/bin/sh', '-c', cmd]) + # TODO Remove this call once we have a ci-deps Makefile target + instance.execute(['mk-build-deps', '--install', '-t', + 'apt-get --no-install-recommends --yes', 'cloud-init']) + + # local tmpfile that must be deleted + local_tarball = tempfile.NamedTemporaryFile().name + + # paths to use in remote system + output_link = '/root/cloud-init_all.deb' + remote_tarball = _out(instance.execute(['mktemp'])) + extract_dir = _out(instance.execute(['mktemp', '--directory'])) + bddeb_path = os.path.join(extract_dir, 'packages', 'bddeb') + git_env = {'GIT_DIR': os.path.join(extract_dir, '.git'), + 'GIT_WORK_TREE': extract_dir} + + LOG.debug('creating tarball of cloud-init at: %s', local_tarball) + c_util.subp(['tar', 'cf', local_tarball, '--owner', 'root', + '--group', 'root', '-C', args.cloud_init, '.']) + LOG.debug('copying to remote system at: %s', remote_tarball) + instance.push_file(local_tarball, remote_tarball) + + LOG.debug('extracting tarball in remote system at: %s', extract_dir) + instance.execute(['tar', 'xf', remote_tarball, '-C', extract_dir]) + instance.execute(['git', 'commit', '-a', '-m', 'tmp', '--allow-empty'], + env=git_env) + + LOG.debug('building deb in remote system at: %s', output_link) + bddeb_args = args.bddeb_args.split() if args.bddeb_args else [] + instance.execute([bddeb_path, '-d'] + bddeb_args, env=git_env) + + # copy the deb back to the host system + LOG.debug('copying built deb to host at: %s', args.deb) + instance.pull_file(output_link, args.deb) + + +def setup_build(args): + """Set build system up then run build. + + @param args: cmdline arguments + @return_value: tuple of results and fail count + """ + res = ({}, 1) + + # set up platform + LOG.info('setting up platform: %s', args.build_platform) + platform_config = config.load_platform_config(args.build_platform) + platform_call = partial(platforms.get_platform, args.build_platform, + platform_config) + with PlatformComponent(platform_call) as platform: + + # set up image + LOG.info('acquiring image for os: %s', args.build_os) + img_conf = config.load_os_config(platform.platform_name, args.build_os) + image_call = partial(images.get_image, platform, img_conf) + with PlatformComponent(image_call) as image: + + # set up snapshot + snapshot_call = partial(snapshots.get_snapshot, image) + with PlatformComponent(snapshot_call) as snapshot: + + # create instance with cloud-config to set it up + LOG.info('creating instance to build deb in') + empty_cloud_config = "#cloud-config\n{}" + instance_call = partial( + instances.get_instance, snapshot, empty_cloud_config, + use_desc='build cloud-init deb') + with PlatformComponent(instance_call) as instance: + + # build the deb + res = run_single('build deb on system', + partial(build_deb, args, instance)) + + return res + + +def bddeb(args): + """Entry point for build deb. + + @param args: cmdline arguments + @return_value: fail count + """ + LOG.info('preparing to build cloud-init deb') + (res, failed) = run_stage('build deb', [partial(setup_build, args)]) + return failed + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py index 02fc0e52..b44e8bdd 100644 --- a/tests/cloud_tests/collect.py +++ b/tests/cloud_tests/collect.py @@ -1,34 +1,39 @@ # This file is part of cloud-init. See LICENSE file for license information. -from tests.cloud_tests import (config, LOG, setup_image, util) -from tests.cloud_tests.stage import (PlatformComponent, run_stage, run_single) -from tests.cloud_tests import (platforms, images, snapshots, instances) +"""Used to collect data from platforms during tests.""" from functools import partial import os +from cloudinit import util as c_util +from tests.cloud_tests import (config, LOG, setup_image, util) +from tests.cloud_tests.stage import (PlatformComponent, run_stage, run_single) +from tests.cloud_tests import (platforms, images, snapshots, instances) + def collect_script(instance, base_dir, script, script_name): - """ - collect script data - instance: instance to run script on - base_dir: base directory for output data - script: script contents - script_name: name of script to run - return_value: None, may raise errors + """Collect script data. + + @param instance: instance to run script on + @param base_dir: base directory for output data + @param script: script contents + @param script_name: name of script to run + @return_value: None, may raise errors """ LOG.debug('running collect script: %s', script_name) - util.write_file(os.path.join(base_dir, script_name), - instance.run_script(script)) + (out, err, exit) = instance.run_script( + script, rcs=range(0, 256), + description='collect: {}'.format(script_name)) + c_util.write_file(os.path.join(base_dir, script_name), out) def collect_test_data(args, snapshot, os_name, test_name): - """ - collect data for test case - args: cmdline arguments - snapshot: instantiated snapshot - test_name: name or path of test to run - return_value: tuple of results and fail count + """Collect data for test case. + + @param args: cmdline arguments + @param snapshot: instantiated snapshot + @param test_name: name or path of test to run + @return_value: tuple of results and fail count """ res = ({}, 1) @@ -39,15 +44,27 @@ def collect_test_data(args, snapshot, os_name, test_name): test_scripts = test_config['collect_scripts'] test_output_dir = os.sep.join( (args.data_dir, snapshot.platform_name, os_name, test_name)) - boot_timeout = (test_config.get('boot_timeout') - if isinstance(test_config.get('boot_timeout'), int) else - snapshot.config.get('timeout')) # if test is not enabled, skip and return 0 failures if not test_config.get('enabled', False): LOG.warning('test config %s is not enabled, skipping', test_name) return ({}, 0) + # if testcase requires a feature flag that the image does not support, + # skip the testcase with a warning + req_features = test_config.get('required_features', []) + if any(feature not in snapshot.features for feature in req_features): + LOG.warn('test config %s requires features not supported by image, ' + 'skipping.\nrequired features: %s\nsupported features: %s', + test_name, req_features, snapshot.features) + return ({}, 0) + + # if there are user data overrides required for this test case, apply them + overrides = snapshot.config.get('user_data_overrides', {}) + if overrides: + LOG.debug('updating user data for collect with: %s', overrides) + user_data = util.update_user_data(user_data, overrides) + # create test instance component = PlatformComponent( partial(instances.get_instance, snapshot, user_data, @@ -56,7 +73,7 @@ def collect_test_data(args, snapshot, os_name, test_name): LOG.info('collecting test data for test: %s', test_name) with component as instance: start_call = partial(run_single, 'boot instance', partial( - instance.start, wait=True, wait_time=boot_timeout)) + instance.start, wait=True, wait_for_cloud_init=True)) collect_calls = [partial(run_single, 'script {}'.format(script_name), partial(collect_script, instance, test_output_dir, script, script_name)) @@ -69,11 +86,11 @@ def collect_test_data(args, snapshot, os_name, test_name): def collect_snapshot(args, image, os_name): - """ - collect data for snapshot of image - args: cmdline arguments - image: instantiated image with set up complete - return_value tuple of results and fail count + """Collect data for snapshot of image. + + @param args: cmdline arguments + @param image: instantiated image with set up complete + @return_value tuple of results and fail count """ res = ({}, 1) @@ -91,19 +108,18 @@ def collect_snapshot(args, image, os_name): def collect_image(args, platform, os_name): - """ - collect data for image - args: cmdline arguments - platform: instantiated platform - os_name: name of distro to collect for - return_value: tuple of results and fail count + """Collect data for image. + + @param args: cmdline arguments + @param platform: instantiated platform + @param os_name: name of distro to collect for + @return_value: tuple of results and fail count """ res = ({}, 1) - os_config = config.load_os_config(os_name) - if not os_config.get('enabled'): - raise ValueError('OS {} not enabled'.format(os_name)) - + os_config = config.load_os_config( + platform.platform_name, os_name, require_enabled=True, + feature_overrides=args.feature_override) component = PlatformComponent( partial(images.get_image, platform, os_config)) @@ -118,18 +134,16 @@ def collect_image(args, platform, os_name): def collect_platform(args, platform_name): - """ - collect data for platform - args: cmdline arguments - platform_name: platform to collect for - return_value: tuple of results and fail count + """Collect data for platform. + + @param args: cmdline arguments + @param platform_name: platform to collect for + @return_value: tuple of results and fail count """ res = ({}, 1) - platform_config = config.load_platform_config(platform_name) - if not platform_config.get('enabled'): - raise ValueError('Platform {} not enabled'.format(platform_name)) - + platform_config = config.load_platform_config( + platform_name, require_enabled=True) component = PlatformComponent( partial(platforms.get_platform, platform_name, platform_config)) @@ -143,10 +157,10 @@ def collect_platform(args, platform_name): def collect(args): - """ - entry point for collection - args: cmdline arguments - return_value: fail count + """Entry point for collection. + + @param args: cmdline arguments + @return_value: fail count """ (res, failed) = run_stage( 'collect data', [partial(collect_platform, args, platform_name) diff --git a/tests/cloud_tests/config.py b/tests/cloud_tests/config.py index f3a13c9a..4d5dc801 100644 --- a/tests/cloud_tests/config.py +++ b/tests/cloud_tests/config.py @@ -1,5 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Used to setup test configuration.""" + import glob import os @@ -14,46 +16,44 @@ RELEASES_CONF = os.path.join(BASE_DIR, 'releases.yaml') TESTCASE_CONF = os.path.join(BASE_DIR, 'testcases.yaml') +def get(base, key): + """Get config entry 'key' from base, ensuring is dictionary.""" + return base[key] if key in base and base[key] is not None else {} + + +def enabled(config): + """Test if config item is enabled.""" + return isinstance(config, dict) and config.get('enabled', False) + + def path_to_name(path): - """ - convert abs or rel path to test config to path under configs/ - if already a test name, do nothing - """ + """Convert abs or rel path to test config to path under 'sconfigs/'.""" dir_path, file_name = os.path.split(os.path.normpath(path)) name = os.path.splitext(file_name)[0] return os.sep.join((os.path.basename(dir_path), name)) def name_to_path(name): - """ - convert test config path under configs/ to full config path, - if already a full path, do nothing - """ + """Convert test config path under configs/ to full config path.""" name = os.path.normpath(name) if not name.endswith(CONF_EXT): name = name + CONF_EXT return name if os.path.isabs(name) else os.path.join(TEST_CONF_DIR, name) -def name_sanatize(name): - """ - sanatize test name to be used as a module name - """ +def name_sanitize(name): + """Sanitize test name to be used as a module name.""" return name.replace('-', '_') def name_to_module(name): - """ - convert test name to a loadable module name under testcases/ - """ - name = name_sanatize(path_to_name(name)) + """Convert test name to a loadable module name under 'testcases/'.""" + name = name_sanitize(path_to_name(name)) return name.replace(os.path.sep, '.') def merge_config(base, override): - """ - merge config and base - """ + """Merge config and base.""" res = base.copy() res.update(override) res.update({k: merge_config(base.get(k, {}), v) @@ -61,53 +61,102 @@ def merge_config(base, override): return res -def load_platform_config(platform): +def merge_feature_groups(feature_conf, feature_groups, overrides): + """Combine feature groups and overrides to construct a supported list. + + @param feature_conf: feature config from releases.yaml + @param feature_groups: feature groups the release is a member of + @param overrides: overrides specified by the release's config + @return_value: dict of {feature: true/false} settings """ - load configuration for platform + res = dict().fromkeys(feature_conf['all']) + for group in feature_groups: + res.update(feature_conf['groups'][group]) + res.update(overrides) + return res + + +def load_platform_config(platform_name, require_enabled=False): + """Load configuration for platform. + + @param platform_name: name of platform to retrieve config for + @param require_enabled: if true, raise error if 'enabled' not True + @return_value: config dict """ main_conf = c_util.read_conf(PLATFORM_CONF) - return merge_config(main_conf.get('default_platform_config'), - main_conf.get('platforms')[platform]) + conf = merge_config(main_conf['default_platform_config'], + main_conf['platforms'][platform_name]) + if require_enabled and not enabled(conf): + raise ValueError('Platform is not enabled') + return conf -def load_os_config(os_name): - """ - load configuration for os +def load_os_config(platform_name, os_name, require_enabled=False, + feature_overrides={}): + """Load configuration for os. + + @param platform_name: platform name to load os config for + @param os_name: name of os to retrieve config for + @param require_enabled: if true, raise error if 'enabled' not True + @param feature_overrides: feature flag overrides to merge with features + @return_value: config dict """ main_conf = c_util.read_conf(RELEASES_CONF) - return merge_config(main_conf.get('default_release_config'), - main_conf.get('releases')[os_name]) + default = main_conf['default_release_config'] + image = main_conf['releases'][os_name] + conf = merge_config(merge_config(get(default, 'default'), + get(default, platform_name)), + merge_config(get(image, 'default'), + get(image, platform_name))) + + feature_conf = main_conf['features'] + feature_groups = conf.get('feature_groups', []) + overrides = merge_config(get(conf, 'features'), feature_overrides) + conf['features'] = merge_feature_groups( + feature_conf, feature_groups, overrides) + + if require_enabled and not enabled(conf): + raise ValueError('OS is not enabled') + return conf def load_test_config(path): - """ - load a test config file by either abs path or rel path - """ + """Load a test config file by either abs path or rel path.""" return merge_config(c_util.read_conf(TESTCASE_CONF)['base_test_data'], c_util.read_conf(name_to_path(path))) +def list_feature_flags(): + """List all supported feature flags.""" + feature_conf = get(c_util.read_conf(RELEASES_CONF), 'features') + return feature_conf.get('all', []) + + def list_enabled_platforms(): - """ - list all platforms enabled for testing - """ - platforms = c_util.read_conf(PLATFORM_CONF).get('platforms') - return [k for k, v in platforms.items() if v.get('enabled')] + """List all platforms enabled for testing.""" + platforms = get(c_util.read_conf(PLATFORM_CONF), 'platforms') + return [k for k, v in platforms.items() if enabled(v)] -def list_enabled_distros(): - """ - list all distros enabled for testing - """ - releases = c_util.read_conf(RELEASES_CONF).get('releases') - return [k for k, v in releases.items() if v.get('enabled')] +def list_enabled_distros(platforms): + """List all distros enabled for testing on specified platforms.""" + def platform_has_enabled(config): + """List if platform is enabled.""" + return any(enabled(merge_config(get(config, 'default'), + get(config, platform))) + for platform in platforms) + + releases = get(c_util.read_conf(RELEASES_CONF), 'releases') + return [k for k, v in releases.items() if platform_has_enabled(v)] def list_test_configs(): - """ - list all available test config files by abspath - """ + """List all available test config files by abspath.""" return [os.path.abspath(f) for f in glob.glob(os.sep.join((TEST_CONF_DIR, '*', '*.yaml')))] + +ENABLED_PLATFORMS = sorted(list_enabled_platforms()) +ENABLED_DISTROS = sorted(list_enabled_distros(ENABLED_PLATFORMS)) + # vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/bugs/lp1628337.yaml b/tests/cloud_tests/configs/bugs/lp1628337.yaml index 1d6bf483..e39b3cd8 100644 --- a/tests/cloud_tests/configs/bugs/lp1628337.yaml +++ b/tests/cloud_tests/configs/bugs/lp1628337.yaml @@ -1,6 +1,9 @@ # # LP Bug 1628337: cloud-init tries to install NTP before even configuring the archives # +required_features: + - apt + - lsb_release cloud_config: | #cloud-config ntp: diff --git a/tests/cloud_tests/configs/examples/add_apt_repositories.yaml b/tests/cloud_tests/configs/examples/add_apt_repositories.yaml index b8964357..4b8575f7 100644 --- a/tests/cloud_tests/configs/examples/add_apt_repositories.yaml +++ b/tests/cloud_tests/configs/examples/add_apt_repositories.yaml @@ -4,6 +4,8 @@ # 2016-11-17: Disabled as covered by module based tests # enabled: False +required_features: + - apt cloud_config: | #cloud-config apt: diff --git a/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml b/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml index 66b635a8..0bec305e 100644 --- a/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml +++ b/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml @@ -56,7 +56,7 @@ cloud_config: | force_install: false # Chef settings - server_url: "https://chef.yourorg.com" + server_url: "https://chef.yourorg.com:4000" # Node Name # Defaults to the instance-id if not present @@ -75,7 +75,7 @@ cloud_config: | YOUR-ORGS-VALIDATION-KEY-HERE -----END RSA PRIVATE KEY----- - # A run list for a first boot json, this is an example (not required) + # A run list for a first boot json run_list: - "recipe[apache2]" - "role[db]" @@ -88,7 +88,7 @@ cloud_config: | keepalive: "off" # if install_type is 'omnibus', change the url to download - omnibus_url: "https://www.chef.io/chef/install.sh" + omnibus_url: "https://www.opscode.com/chef/install.sh" # Capture all subprocess output into a logfile diff --git a/tests/cloud_tests/configs/modules/apt_configure_conf.yaml b/tests/cloud_tests/configs/modules/apt_configure_conf.yaml index 163ae3fc..de453000 100644 --- a/tests/cloud_tests/configs/modules/apt_configure_conf.yaml +++ b/tests/cloud_tests/configs/modules/apt_configure_conf.yaml @@ -1,6 +1,8 @@ # # Provide a configuration for APT # +required_features: + - apt cloud_config: | #cloud-config apt: diff --git a/tests/cloud_tests/configs/modules/apt_configure_disable_suites.yaml b/tests/cloud_tests/configs/modules/apt_configure_disable_suites.yaml index 73e4a538..98800673 100644 --- a/tests/cloud_tests/configs/modules/apt_configure_disable_suites.yaml +++ b/tests/cloud_tests/configs/modules/apt_configure_disable_suites.yaml @@ -1,6 +1,9 @@ # # Disables everything in sources.list # +required_features: + - apt + - lsb_release cloud_config: | #cloud-config apt: diff --git a/tests/cloud_tests/configs/modules/apt_configure_primary.yaml b/tests/cloud_tests/configs/modules/apt_configure_primary.yaml index 2ec30ca1..41bcf2fd 100644 --- a/tests/cloud_tests/configs/modules/apt_configure_primary.yaml +++ b/tests/cloud_tests/configs/modules/apt_configure_primary.yaml @@ -1,6 +1,9 @@ # # Setup a custome primary sources.list # +required_features: + - apt + - apt_src_cont cloud_config: | #cloud-config apt: @@ -16,4 +19,8 @@ collect_scripts: #!/bin/bash grep -v '^#' /etc/apt/sources.list | sed '/^\s*$/d' | grep -c gtlib.gatech.edu + sources.list: | + #!/bin/bash + cat /etc/apt/sources.list + # vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/apt_configure_proxy.yaml b/tests/cloud_tests/configs/modules/apt_configure_proxy.yaml index e7371305..be6c6f81 100644 --- a/tests/cloud_tests/configs/modules/apt_configure_proxy.yaml +++ b/tests/cloud_tests/configs/modules/apt_configure_proxy.yaml @@ -1,6 +1,8 @@ # # Set apt proxy # +required_features: + - apt cloud_config: | #cloud-config apt: diff --git a/tests/cloud_tests/configs/modules/apt_configure_security.yaml b/tests/cloud_tests/configs/modules/apt_configure_security.yaml index f6a2c828..83dd51df 100644 --- a/tests/cloud_tests/configs/modules/apt_configure_security.yaml +++ b/tests/cloud_tests/configs/modules/apt_configure_security.yaml @@ -1,6 +1,9 @@ # # Add security to sources.list # +required_features: + - apt + - ubuntu_repos cloud_config: | #cloud-config apt: diff --git a/tests/cloud_tests/configs/modules/apt_configure_sources_key.yaml b/tests/cloud_tests/configs/modules/apt_configure_sources_key.yaml index e7568a6a..bde9398a 100644 --- a/tests/cloud_tests/configs/modules/apt_configure_sources_key.yaml +++ b/tests/cloud_tests/configs/modules/apt_configure_sources_key.yaml @@ -1,6 +1,9 @@ # # Add a sources.list entry with a given key (Debian Jessie) # +required_features: + - apt + - lsb_release cloud_config: | #cloud-config apt: diff --git a/tests/cloud_tests/configs/modules/apt_configure_sources_keyserver.yaml b/tests/cloud_tests/configs/modules/apt_configure_sources_keyserver.yaml index 1a4a238f..11da61ea 100644 --- a/tests/cloud_tests/configs/modules/apt_configure_sources_keyserver.yaml +++ b/tests/cloud_tests/configs/modules/apt_configure_sources_keyserver.yaml @@ -1,6 +1,9 @@ # # Add a sources.list entry with a key from a keyserver # +required_features: + - apt + - lsb_release cloud_config: | #cloud-config apt: diff --git a/tests/cloud_tests/configs/modules/apt_configure_sources_list.yaml b/tests/cloud_tests/configs/modules/apt_configure_sources_list.yaml index 057fc72c..143cb080 100644 --- a/tests/cloud_tests/configs/modules/apt_configure_sources_list.yaml +++ b/tests/cloud_tests/configs/modules/apt_configure_sources_list.yaml @@ -1,6 +1,9 @@ # # Generate a sources.list # +required_features: + - apt + - lsb_release cloud_config: | #cloud-config apt: diff --git a/tests/cloud_tests/configs/modules/apt_configure_sources_ppa.yaml b/tests/cloud_tests/configs/modules/apt_configure_sources_ppa.yaml index dee9dc70..9efdae52 100644 --- a/tests/cloud_tests/configs/modules/apt_configure_sources_ppa.yaml +++ b/tests/cloud_tests/configs/modules/apt_configure_sources_ppa.yaml @@ -1,6 +1,12 @@ # # Add a PPA to source.list # +# NOTE: on older ubuntu releases the sources file added is named +# 'curtin-dev-test-archive-trusty', without 'ubuntu' in the middle +required_features: + - apt + - ppa + - ppa_file_name cloud_config: | #cloud-config apt: @@ -16,5 +22,8 @@ collect_scripts: apt-key: | #!/bin/bash apt-key finger + sources_full: | + #!/bin/bash + cat /etc/apt/sources.list # vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/apt_pipelining_disable.yaml b/tests/cloud_tests/configs/modules/apt_pipelining_disable.yaml index 5fa0cee9..bd9b5d08 100644 --- a/tests/cloud_tests/configs/modules/apt_pipelining_disable.yaml +++ b/tests/cloud_tests/configs/modules/apt_pipelining_disable.yaml @@ -1,6 +1,8 @@ # # Disable apt pipelining value # +required_features: + - apt cloud_config: | #cloud-config apt: diff --git a/tests/cloud_tests/configs/modules/apt_pipelining_os.yaml b/tests/cloud_tests/configs/modules/apt_pipelining_os.yaml index 87d183e7..cbed3ba3 100644 --- a/tests/cloud_tests/configs/modules/apt_pipelining_os.yaml +++ b/tests/cloud_tests/configs/modules/apt_pipelining_os.yaml @@ -1,6 +1,8 @@ # # Set apt pipelining value to OS # +required_features: + - apt cloud_config: | #cloud-config apt: diff --git a/tests/cloud_tests/configs/modules/byobu.yaml b/tests/cloud_tests/configs/modules/byobu.yaml index fd648c77..a9aa1f3f 100644 --- a/tests/cloud_tests/configs/modules/byobu.yaml +++ b/tests/cloud_tests/configs/modules/byobu.yaml @@ -1,6 +1,8 @@ # # Install and enable byobu system wide and default user # +required_features: + - byobu cloud_config: | #cloud-config byobu_by_default: enable diff --git a/tests/cloud_tests/configs/modules/keys_to_console.yaml b/tests/cloud_tests/configs/modules/keys_to_console.yaml index a90e42c1..5d86e739 100644 --- a/tests/cloud_tests/configs/modules/keys_to_console.yaml +++ b/tests/cloud_tests/configs/modules/keys_to_console.yaml @@ -1,6 +1,8 @@ # # Hide printing of ssh key and fingerprints for specific keys # +required_features: + - syslog cloud_config: | #cloud-config ssh_fp_console_blacklist: [ssh-dss, ssh-dsa, ecdsa-sha2-nistp256] diff --git a/tests/cloud_tests/configs/modules/landscape.yaml b/tests/cloud_tests/configs/modules/landscape.yaml index e6f4955a..ed2c37c4 100644 --- a/tests/cloud_tests/configs/modules/landscape.yaml +++ b/tests/cloud_tests/configs/modules/landscape.yaml @@ -4,6 +4,8 @@ # 2016-11-17: Disabled due to this not working # enabled: false +required_features: + - landscape cloud_config: | #cloud-conifg landscape: diff --git a/tests/cloud_tests/configs/modules/locale.yaml b/tests/cloud_tests/configs/modules/locale.yaml index af5ad636..e01518a1 100644 --- a/tests/cloud_tests/configs/modules/locale.yaml +++ b/tests/cloud_tests/configs/modules/locale.yaml @@ -1,6 +1,9 @@ # # Set locale to non-default option and verify # +required_features: + - engb_locale + - locale_gen cloud_config: | #cloud-config locale: en_GB.UTF-8 diff --git a/tests/cloud_tests/configs/modules/lxd_bridge.yaml b/tests/cloud_tests/configs/modules/lxd_bridge.yaml index 568bb700..e6b7e76a 100644 --- a/tests/cloud_tests/configs/modules/lxd_bridge.yaml +++ b/tests/cloud_tests/configs/modules/lxd_bridge.yaml @@ -1,6 +1,8 @@ # # LXD configured with directory backend and IPv4 bridge # +required_features: + - lxd cloud_config: | #cloud-config lxd: diff --git a/tests/cloud_tests/configs/modules/lxd_dir.yaml b/tests/cloud_tests/configs/modules/lxd_dir.yaml index 99b92195..f93a3fa7 100644 --- a/tests/cloud_tests/configs/modules/lxd_dir.yaml +++ b/tests/cloud_tests/configs/modules/lxd_dir.yaml @@ -1,6 +1,8 @@ # # LXD configured with directory backend # +required_features: + - lxd cloud_config: | #cloud-config lxd: diff --git a/tests/cloud_tests/configs/modules/ntp.yaml b/tests/cloud_tests/configs/modules/ntp.yaml index d0941578..0d07ef5a 100644 --- a/tests/cloud_tests/configs/modules/ntp.yaml +++ b/tests/cloud_tests/configs/modules/ntp.yaml @@ -1,6 +1,14 @@ # # Emtpy NTP config to setup using defaults # +# NOTE: this should not require apt feature, use 'which' rather than 'dpkg -l' +# NOTE: this should not require no_ntpdate feature, use 'which' to check for +# installation rather than 'dpkg -l', as 'grep ntp' matches 'ntpdate' +# NOTE: the verifier should check for any ntp server not 'ubuntu.pool.ntp.org' +required_features: + - apt + - no_ntpdate + - ubuntu_ntp cloud_config: | #cloud-config ntp: @@ -16,5 +24,8 @@ collect_scripts: ntp_conf_empty: | #!/bin/bash grep '^pool' /etc/ntp.conf + ntp_installed_list: | + #!/bin/bash + dpkg -l | grep ntp # vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/ntp_pools.yaml b/tests/cloud_tests/configs/modules/ntp_pools.yaml index e040cc32..7561c7f3 100644 --- a/tests/cloud_tests/configs/modules/ntp_pools.yaml +++ b/tests/cloud_tests/configs/modules/ntp_pools.yaml @@ -1,6 +1,16 @@ # # NTP config using specific pools # +# NOTE: this should not require apt feature, use 'which' rather than 'dpkg -l' +# NOTE: this should not require no_ntpdate feature, use 'which' to check for +# installation rather than 'dpkg -l', as 'grep ntp' matches 'ntpdate' +# NOTE: lsb_release listed here because with recent cloud-init deb with +# (LP: 1628337) resolved, cloud-init will attempt to configure archives. +# this fails without lsb_release as UNAVAILABLE is used for $RELEASE +required_features: + - apt + - no_ntpdate + - lsb_release cloud_config: | #cloud-config ntp: diff --git a/tests/cloud_tests/configs/modules/ntp_servers.yaml b/tests/cloud_tests/configs/modules/ntp_servers.yaml index e0564a03..9d1d65ef 100644 --- a/tests/cloud_tests/configs/modules/ntp_servers.yaml +++ b/tests/cloud_tests/configs/modules/ntp_servers.yaml @@ -1,6 +1,13 @@ # # NTP config using specific servers # +# NOTE: this should not require apt feature, use 'which' rather than 'dpkg -l' +# NOTE: this should not require no_ntpdate feature, use 'which' to check for +# installation rather than 'dpkg -l', as 'grep ntp' matches 'ntpdate' +required_features: + - apt + - no_ntpdate + - lsb_release cloud_config: | #cloud-config ntp: diff --git a/tests/cloud_tests/configs/modules/package_update_upgrade_install.yaml b/tests/cloud_tests/configs/modules/package_update_upgrade_install.yaml index d027d540..71d24b83 100644 --- a/tests/cloud_tests/configs/modules/package_update_upgrade_install.yaml +++ b/tests/cloud_tests/configs/modules/package_update_upgrade_install.yaml @@ -1,6 +1,17 @@ # # Update/upgrade via apt and then install a pair of packages # +# NOTE: this should not require apt feature, use 'which' rather than 'dpkg -l' +# NOTE: the testcase for this looks for the command in history.log as +# /usr/bin/apt-get..., which is not how it always appears. it should +# instead look for just apt-get... +# NOTE: this testcase should not require 'apt_up_out', and should look for a +# call to 'apt-get upgrade' or 'apt-get dist-upgrade' in cloud-init.log +# rather than 'Calculating upgrade...' in output +required_features: + - apt + - apt_hist_fmt + - apt_up_out cloud_config: | #cloud-config packages: diff --git a/tests/cloud_tests/configs/modules/set_hostname.yaml b/tests/cloud_tests/configs/modules/set_hostname.yaml index 5aae1506..c96344cf 100644 --- a/tests/cloud_tests/configs/modules/set_hostname.yaml +++ b/tests/cloud_tests/configs/modules/set_hostname.yaml @@ -1,6 +1,8 @@ # # Set the hostname and update /etc/hosts # +required_features: + - hostname cloud_config: | #cloud-config hostname: myhostname diff --git a/tests/cloud_tests/configs/modules/set_hostname_fqdn.yaml b/tests/cloud_tests/configs/modules/set_hostname_fqdn.yaml index 0014c197..daf75931 100644 --- a/tests/cloud_tests/configs/modules/set_hostname_fqdn.yaml +++ b/tests/cloud_tests/configs/modules/set_hostname_fqdn.yaml @@ -1,6 +1,8 @@ # # Set the hostname and update /etc/hosts # +required_features: + - hostname cloud_config: | #cloud-config manage_etc_hosts: true diff --git a/tests/cloud_tests/configs/modules/set_password.yaml b/tests/cloud_tests/configs/modules/set_password.yaml index 8fa46d9f..04d7c58a 100644 --- a/tests/cloud_tests/configs/modules/set_password.yaml +++ b/tests/cloud_tests/configs/modules/set_password.yaml @@ -1,6 +1,8 @@ # # Set password of default user # +required_features: + - ubuntu_user cloud_config: | #cloud-config password: password diff --git a/tests/cloud_tests/configs/modules/set_password_expire.yaml b/tests/cloud_tests/configs/modules/set_password_expire.yaml index 926731f0..789604b0 100644 --- a/tests/cloud_tests/configs/modules/set_password_expire.yaml +++ b/tests/cloud_tests/configs/modules/set_password_expire.yaml @@ -1,6 +1,8 @@ # # Expire password for all users # +required_features: + - sshd cloud_config: | #cloud-config chpasswd: { expire: True } diff --git a/tests/cloud_tests/configs/modules/snappy.yaml b/tests/cloud_tests/configs/modules/snappy.yaml index 0e7dc852..43f93295 100644 --- a/tests/cloud_tests/configs/modules/snappy.yaml +++ b/tests/cloud_tests/configs/modules/snappy.yaml @@ -1,6 +1,8 @@ # # Install snappy # +required_features: + - snap cloud_config: | #cloud-config snappy: diff --git a/tests/cloud_tests/configs/modules/ssh_auth_key_fingerprints_disable.yaml b/tests/cloud_tests/configs/modules/ssh_auth_key_fingerprints_disable.yaml index 33943bdd..746653ec 100644 --- a/tests/cloud_tests/configs/modules/ssh_auth_key_fingerprints_disable.yaml +++ b/tests/cloud_tests/configs/modules/ssh_auth_key_fingerprints_disable.yaml @@ -1,6 +1,8 @@ # # Disable fingerprint printing # +required_features: + - syslog cloud_config: | #cloud-config ssh_genkeytypes: [] diff --git a/tests/cloud_tests/configs/modules/ssh_auth_key_fingerprints_enable.yaml b/tests/cloud_tests/configs/modules/ssh_auth_key_fingerprints_enable.yaml index 4c970778..9f5dc34a 100644 --- a/tests/cloud_tests/configs/modules/ssh_auth_key_fingerprints_enable.yaml +++ b/tests/cloud_tests/configs/modules/ssh_auth_key_fingerprints_enable.yaml @@ -1,6 +1,11 @@ # # Print auth keys with different hash than md5 # +# NOTE: testcase checks for '256 SHA256:.*(ECDSA)' on output line on trusty +# this fails as line in output reads '256:.*(ECDSA)' +required_features: + - syslog + - ssh_key_fmt cloud_config: | #cloud-config ssh_genkeytypes: diff --git a/tests/cloud_tests/configs/modules/ssh_import_id.yaml b/tests/cloud_tests/configs/modules/ssh_import_id.yaml index 6e5a1635..b62d3f69 100644 --- a/tests/cloud_tests/configs/modules/ssh_import_id.yaml +++ b/tests/cloud_tests/configs/modules/ssh_import_id.yaml @@ -1,6 +1,9 @@ # # Import a user's ssh key via gh or lp # +required_features: + - ubuntu_user + - sudo cloud_config: | #cloud-config ssh_import_id: diff --git a/tests/cloud_tests/configs/modules/ssh_keys_generate.yaml b/tests/cloud_tests/configs/modules/ssh_keys_generate.yaml index 637d7835..659fd939 100644 --- a/tests/cloud_tests/configs/modules/ssh_keys_generate.yaml +++ b/tests/cloud_tests/configs/modules/ssh_keys_generate.yaml @@ -1,6 +1,8 @@ # # SSH keys generated using cloud-init # +required_features: + - ubuntu_user cloud_config: | #cloud-config ssh_genkeytypes: diff --git a/tests/cloud_tests/configs/modules/ssh_keys_provided.yaml b/tests/cloud_tests/configs/modules/ssh_keys_provided.yaml index 25df6452..5ceb3623 100644 --- a/tests/cloud_tests/configs/modules/ssh_keys_provided.yaml +++ b/tests/cloud_tests/configs/modules/ssh_keys_provided.yaml @@ -2,6 +2,9 @@ # SSH keys provided via cloud config # enabled: False +required_features: + - ubuntu_user + - sudo cloud_config: | #cloud-config disable_root: false diff --git a/tests/cloud_tests/configs/modules/timezone.yaml b/tests/cloud_tests/configs/modules/timezone.yaml index 8c96ed47..5112aa9f 100644 --- a/tests/cloud_tests/configs/modules/timezone.yaml +++ b/tests/cloud_tests/configs/modules/timezone.yaml @@ -1,6 +1,8 @@ # # Set system timezone # +required_features: + - daylight_time cloud_config: | #cloud-config timezone: US/Aleutian diff --git a/tests/cloud_tests/configs/modules/user_groups.yaml b/tests/cloud_tests/configs/modules/user_groups.yaml index 92655958..71cc9da3 100644 --- a/tests/cloud_tests/configs/modules/user_groups.yaml +++ b/tests/cloud_tests/configs/modules/user_groups.yaml @@ -1,6 +1,8 @@ # # Create groups and users with various options # +required_features: + - ubuntu_user cloud_config: | #cloud-config # Add groups to the system diff --git a/tests/cloud_tests/configs/modules/write_files.yaml b/tests/cloud_tests/configs/modules/write_files.yaml index 4bb2991a..ce936b7b 100644 --- a/tests/cloud_tests/configs/modules/write_files.yaml +++ b/tests/cloud_tests/configs/modules/write_files.yaml @@ -1,6 +1,10 @@ # # Write various file types # +# NOTE: on trusty 'file' has an output formatting error for binary files and +# has 2 spaces in 'LSB executable', which causes a failure here +required_features: + - no_file_fmt_e cloud_config: | #cloud-config write_files: diff --git a/tests/cloud_tests/images/__init__.py b/tests/cloud_tests/images/__init__.py index b27d6931..106c59f3 100644 --- a/tests/cloud_tests/images/__init__.py +++ b/tests/cloud_tests/images/__init__.py @@ -1,11 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Main init.""" + def get_image(platform, config): - """ - get image from platform object using os_name, looking up img_conf in main - config file - """ + """Get image from platform object using os_name.""" return platform.get_image(config) # vi: ts=4 expandtab diff --git a/tests/cloud_tests/images/base.py b/tests/cloud_tests/images/base.py index 394b11ff..0a1e0563 100644 --- a/tests/cloud_tests/images/base.py +++ b/tests/cloud_tests/images/base.py @@ -1,65 +1,69 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Base class for images.""" + class Image(object): - """ - Base class for images - """ + """Base class for images.""" + platform_name = None - def __init__(self, name, config, platform): - """ - setup + def __init__(self, platform, config): + """Set up image. + + @param platform: platform object + @param config: image configuration """ - self.name = name - self.config = config self.platform = platform + self.config = config def __str__(self): - """ - a brief description of the image - """ + """A brief description of the image.""" return '-'.join((self.properties['os'], self.properties['release'])) @property def properties(self): - """ - {} containing: 'arch', 'os', 'version', 'release' - """ + """{} containing: 'arch', 'os', 'version', 'release'.""" raise NotImplementedError - # FIXME: instead of having execute and push_file and other instance methods - # here which pass through to a hidden instance, it might be better - # to expose an instance that the image can be modified through - def execute(self, command, stdin=None, stdout=None, stderr=None, env={}): + @property + def features(self): + """Feature flags supported by this image. + + @return_value: list of feature names """ - execute command in image, modifying image + return [k for k, v in self.config.get('features', {}).items() if v] + + @property + def setup_overrides(self): + """Setup options that need to be overridden for the image. + + @return_value: dictionary to update args with """ + # NOTE: more sophisticated options may be requied at some point + return self.config.get('setup_overrides', {}) + + def execute(self, *args, **kwargs): + """Execute command in image, modifying image.""" raise NotImplementedError def push_file(self, local_path, remote_path): - """ - copy file at 'local_path' to instance at 'remote_path', modifying image - """ + """Copy file at 'local_path' to instance at 'remote_path'.""" raise NotImplementedError - def run_script(self, script): - """ - run script in image, modifying image - return_value: script output + def run_script(self, *args, **kwargs): + """Run script in image, modifying image. + + @return_value: script output """ raise NotImplementedError def snapshot(self): - """ - create snapshot of image, block until done - """ + """Create snapshot of image, block until done.""" raise NotImplementedError def destroy(self): - """ - clean up data associated with image - """ + """Clean up data associated with image.""" pass # vi: ts=4 expandtab diff --git a/tests/cloud_tests/images/lxd.py b/tests/cloud_tests/images/lxd.py index 7a416141..fd4e93c2 100644 --- a/tests/cloud_tests/images/lxd.py +++ b/tests/cloud_tests/images/lxd.py @@ -1,43 +1,67 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""LXD Image Base Class.""" + +import os +import shutil +import tempfile + +from cloudinit import util as c_util from tests.cloud_tests.images import base from tests.cloud_tests.snapshots import lxd as lxd_snapshot +from tests.cloud_tests import util class LXDImage(base.Image): - """ - LXD backed image - """ + """LXD backed image.""" + platform_name = "lxd" - def __init__(self, name, config, platform, pylxd_image): - """ - setup + def __init__(self, platform, config, pylxd_image): + """Set up image. + + @param platform: platform object + @param config: image configuration """ - self.platform = platform - self._pylxd_image = pylxd_image + self.modified = False self._instance = None - super(LXDImage, self).__init__(name, config, platform) + self._pylxd_image = None + self.pylxd_image = pylxd_image + super(LXDImage, self).__init__(platform, config) @property def pylxd_image(self): - self._pylxd_image.sync() + """Property function.""" + if self._pylxd_image: + self._pylxd_image.sync() return self._pylxd_image + @pylxd_image.setter + def pylxd_image(self, pylxd_image): + if self._instance: + self._instance.destroy() + self._instance = None + if (self._pylxd_image and + (self._pylxd_image is not pylxd_image) and + (not self.config.get('cache_base_image') or self.modified)): + self._pylxd_image.delete(wait=True) + self.modified = False + self._pylxd_image = pylxd_image + @property def instance(self): + """Property function.""" if not self._instance: self._instance = self.platform.launch_container( - image=self.pylxd_image.fingerprint, - image_desc=str(self), use_desc='image-modification') - self._instance.start(wait=True, wait_time=self.config.get('timeout')) + self.properties, self.config, self.features, + use_desc='image-modification', image_desc=str(self), + image=self.pylxd_image.fingerprint) + self._instance.start() return self._instance @property def properties(self): - """ - {} containing: 'arch', 'os', 'version', 'release' - """ + """{} containing: 'arch', 'os', 'version', 'release'.""" properties = self.pylxd_image.properties return { 'arch': properties.get('architecture'), @@ -46,47 +70,121 @@ class LXDImage(base.Image): 'release': properties.get('release'), } - def execute(self, *args, **kwargs): + def export_image(self, output_dir): + """Export image from lxd image store to (split) tarball on disk. + + @param output_dir: dir to store tarballs in + @return_value: tuple of path to metadata tarball and rootfs tarball """ - execute command in image, modifying image + # pylxd's image export feature doesn't do split exports, so use cmdline + c_util.subp(['lxc', 'image', 'export', self.pylxd_image.fingerprint, + output_dir], capture=True) + tarballs = [p for p in os.listdir(output_dir) if p.endswith('tar.xz')] + metadata = os.path.join( + output_dir, next(p for p in tarballs if p.startswith('meta-'))) + rootfs = os.path.join( + output_dir, next(p for p in tarballs if not p.startswith('meta-'))) + return (metadata, rootfs) + + def import_image(self, metadata, rootfs): + """Import image to lxd image store from (split) tarball on disk. + + Note, this will replace and delete the current pylxd_image + + @param metadata: metadata tarball + @param rootfs: rootfs tarball + @return_value: imported image fingerprint + """ + alias = util.gen_instance_name( + image_desc=str(self), use_desc='update-metadata') + c_util.subp(['lxc', 'image', 'import', metadata, rootfs, + '--alias', alias], capture=True) + self.pylxd_image = self.platform.query_image_by_alias(alias) + return self.pylxd_image.fingerprint + + def update_templates(self, template_config, template_data): + """Update the image's template configuration. + + Note, this will replace and delete the current pylxd_image + + @param template_config: config overrides for template metadata + @param template_data: template data to place into templates/ """ + # set up tmp files + export_dir = tempfile.mkdtemp(prefix='cloud_test_util_') + extract_dir = tempfile.mkdtemp(prefix='cloud_test_util_') + new_metadata = os.path.join(export_dir, 'new-meta.tar.xz') + metadata_yaml = os.path.join(extract_dir, 'metadata.yaml') + template_dir = os.path.join(extract_dir, 'templates') + + try: + # extract old data + (metadata, rootfs) = self.export_image(export_dir) + shutil.unpack_archive(metadata, extract_dir) + + # update metadata + metadata = c_util.read_conf(metadata_yaml) + templates = metadata.get('templates', {}) + templates.update(template_config) + metadata['templates'] = templates + util.yaml_dump(metadata, metadata_yaml) + + # write out template files + for name, content in template_data.items(): + path = os.path.join(template_dir, name) + c_util.write_file(path, content) + + # store new data, mark new image as modified + util.flat_tar(new_metadata, extract_dir) + self.import_image(new_metadata, rootfs) + self.modified = True + + finally: + # remove tmpfiles + shutil.rmtree(export_dir) + shutil.rmtree(extract_dir) + + def execute(self, *args, **kwargs): + """Execute command in image, modifying image.""" return self.instance.execute(*args, **kwargs) def push_file(self, local_path, remote_path): - """ - copy file at 'local_path' to instance at 'remote_path', modifying image - """ + """Copy file at 'local_path' to instance at 'remote_path'.""" return self.instance.push_file(local_path, remote_path) - def run_script(self, script): - """ - run script in image, modifying image - return_value: script output + def run_script(self, *args, **kwargs): + """Run script in image, modifying image. + + @return_value: script output """ - return self.instance.run_script(script) + return self.instance.run_script(*args, **kwargs) def snapshot(self): - """ - create snapshot of image, block until done - """ - # clone current instance, start and freeze clone + """Create snapshot of image, block until done.""" + # get empty user data to pass in to instance + # if overrides for user data provided, use them + empty_userdata = util.update_user_data( + {}, self.config.get('user_data_overrides', {})) + conf = {'user.user-data': empty_userdata} + # clone current instance instance = self.platform.launch_container( + self.properties, self.config, self.features, container=self.instance.name, image_desc=str(self), - use_desc='snapshot') - instance.start(wait=True, wait_time=self.config.get('timeout')) + use_desc='snapshot', container_config=conf) + # wait for cloud-init before boot_clean_script is run to ensure + # /var/lib/cloud is removed cleanly + instance.start(wait=True, wait_for_cloud_init=True) if self.config.get('boot_clean_script'): instance.run_script(self.config.get('boot_clean_script')) + # freeze current instance and return snapshot instance.freeze() return lxd_snapshot.LXDSnapshot( - self.properties, self.config, self.platform, instance) + self.platform, self.properties, self.config, + self.features, instance) def destroy(self): - """ - clean up data associated with image - """ - if self._instance: - self._instance.destroy() - self.pylxd_image.delete(wait=True) + """Clean up data associated with image.""" + self.pylxd_image = None super(LXDImage, self).destroy() # vi: ts=4 expandtab diff --git a/tests/cloud_tests/instances/__init__.py b/tests/cloud_tests/instances/__init__.py index 85bea99f..fc2e9cbc 100644 --- a/tests/cloud_tests/instances/__init__.py +++ b/tests/cloud_tests/instances/__init__.py @@ -1,10 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Main init.""" + def get_instance(snapshot, *args, **kwargs): - """ - get instance from snapshot - """ + """Get instance from snapshot.""" return snapshot.launch(*args, **kwargs) # vi: ts=4 expandtab diff --git a/tests/cloud_tests/instances/base.py b/tests/cloud_tests/instances/base.py index 9559d286..959e9cce 100644 --- a/tests/cloud_tests/instances/base.py +++ b/tests/cloud_tests/instances/base.py @@ -1,120 +1,148 @@ # This file is part of cloud-init. See LICENSE file for license information. -import os -import uuid +"""Base instance.""" class Instance(object): - """ - Base instance object - """ + """Base instance object.""" + platform_name = None - def __init__(self, name): - """ - setup + def __init__(self, platform, name, properties, config, features): + """Set up instance. + + @param platform: platform object + @param name: hostname of instance + @param properties: image properties + @param config: image config + @param features: supported feature flags """ + self.platform = platform self.name = name + self.properties = properties + self.config = config + self.features = features - def execute(self, command, stdin=None, stdout=None, stderr=None, env={}): - """ - command: the command to execute as root inside the image - stdin, stderr, stdout: file handles - env: environment variables + def execute(self, command, stdout=None, stderr=None, env={}, + rcs=None, description=None): + """Execute command in instance, recording output, error and exit code. - Execute assumes functional networking and execution as root with the + Assumes functional networking and execution as root with the target filesystem being available at /. - return_value: tuple containing stdout data, stderr data, exit code + @param command: the command to execute as root inside the image + @param stdout, stderr: file handles to write output and error to + @param env: environment variables + @param rcs: allowed return codes from command + @param description: purpose of command + @return_value: tuple containing stdout data, stderr data, exit code """ raise NotImplementedError - def read_data(self, remote_path, encode=False): - """ - read_data from instance filesystem - remote_path: path in instance - decode: return as string - return_value: data as str or bytes + def read_data(self, remote_path, decode=False): + """Read data from instance filesystem. + + @param remote_path: path in instance + @param decode: return as string + @return_value: data as str or bytes """ raise NotImplementedError def write_data(self, remote_path, data): - """ - write data to instance filesystem - remote_path: path in instance - data: data to write, either str or bytes + """Write data to instance filesystem. + + @param remote_path: path in instance + @param data: data to write, either str or bytes """ raise NotImplementedError def pull_file(self, remote_path, local_path): - """ - copy file at 'remote_path', from instance to 'local_path' + """Copy file at 'remote_path', from instance to 'local_path'. + + @param remote_path: path on remote instance + @param local_path: path on local instance """ with open(local_path, 'wb') as fp: - fp.write(self.read_data(remote_path), encode=True) + fp.write(self.read_data(remote_path)) def push_file(self, local_path, remote_path): - """ - copy file at 'local_path' to instance at 'remote_path' + """Copy file at 'local_path' to instance at 'remote_path'. + + @param local_path: path on local instance + @param remote_path: path on remote instance """ with open(local_path, 'rb') as fp: self.write_data(remote_path, fp.read()) - def run_script(self, script): + def run_script(self, script, rcs=None, description=None): + """Run script in target and return stdout. + + @param script: script contents + @param rcs: allowed return codes from script + @param description: purpose of script + @return_value: stdout from script """ - run script in target and return stdout + script_path = self.tmpfile() + try: + self.write_data(script_path, script) + return self.execute( + ['/bin/bash', script_path], rcs=rcs, description=description) + finally: + self.execute(['rm', script_path], rcs=rcs) + + def tmpfile(self): + """Get a tmp file in the target. + + @return_value: path to new file in target """ - script_path = os.path.join('/tmp', str(uuid.uuid1())) - self.write_data(script_path, script) - (out, err, exit_code) = self.execute(['/bin/bash', script_path]) - return out + return self.execute(['mktemp'])[0].strip() def console_log(self): - """ - return_value: bytes of this instance’s console + """Instance console. + + @return_value: bytes of this instance’s console """ raise NotImplementedError def reboot(self, wait=True): - """ - reboot instance - """ + """Reboot instance.""" raise NotImplementedError def shutdown(self, wait=True): - """ - shutdown instance - """ + """Shutdown instance.""" raise NotImplementedError - def start(self, wait=True): - """ - start instance - """ + def start(self, wait=True, wait_for_cloud_init=False): + """Start instance.""" raise NotImplementedError def destroy(self): - """ - clean up instance - """ + """Clean up instance.""" pass - def _wait_for_cloud_init(self, wait_time): - """ - wait until system has fully booted and cloud-init has finished + def _wait_for_system(self, wait_for_cloud_init): + """Wait until system has fully booted and cloud-init has finished. + + @param wait_time: maximum time to wait + @return_value: None, may raise OSError if wait_time exceeded """ - if not wait_time: - return - - found_msg = 'found' - cmd = ('for ((i=0;i<{wait};i++)); do [ -f "{file}" ] && ' - '{{ echo "{msg}";break; }} || sleep 1; done').format( - file='/run/cloud-init/result.json', - wait=wait_time, msg=found_msg) - - (out, err, exit) = self.execute(['/bin/bash', '-c', cmd]) - if out.strip() != found_msg: - raise OSError('timeout: after {}s, cloud-init has not started' - .format(wait_time)) + def clean_test(test): + """Clean formatting for system ready test testcase.""" + return ' '.join(l for l in test.strip().splitlines() + if not l.lstrip().startswith('#')) + + time = self.config['boot_timeout'] + tests = [self.config['system_ready_script']] + if wait_for_cloud_init: + tests.append(self.config['cloud_init_ready_script']) + + formatted_tests = ' && '.join(clean_test(t) for t in tests) + test_cmd = ('for ((i=0;i<{time};i++)); do {test} && exit 0; sleep 1; ' + 'done; exit 1;').format(time=time, test=formatted_tests) + cmd = ['/bin/bash', '-c', test_cmd] + + if self.execute(cmd, rcs=(0, 1))[-1] != 0: + raise OSError('timeout: after {}s system not started'.format(time)) + # vi: ts=4 expandtab diff --git a/tests/cloud_tests/instances/lxd.py b/tests/cloud_tests/instances/lxd.py index f0aa1214..b9c2cc6b 100644 --- a/tests/cloud_tests/instances/lxd.py +++ b/tests/cloud_tests/instances/lxd.py @@ -1,115 +1,135 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Base LXD instance.""" + from tests.cloud_tests.instances import base +from tests.cloud_tests import util class LXDInstance(base.Instance): - """ - LXD container backed instance - """ + """LXD container backed instance.""" + platform_name = "lxd" - def __init__(self, name, platform, pylxd_container): - """ - setup + def __init__(self, platform, name, properties, config, features, + pylxd_container): + """Set up instance. + + @param platform: platform object + @param name: hostname of instance + @param properties: image properties + @param config: image config + @param features: supported feature flags """ - self.platform = platform self._pylxd_container = pylxd_container - super(LXDInstance, self).__init__(name) + super(LXDInstance, self).__init__( + platform, name, properties, config, features) @property def pylxd_container(self): + """Property function.""" self._pylxd_container.sync() return self._pylxd_container - def execute(self, command, stdin=None, stdout=None, stderr=None, env={}): - """ - command: the command to execute as root inside the image - stdin, stderr, stdout: file handles - env: environment variables + def execute(self, command, stdout=None, stderr=None, env={}, + rcs=None, description=None): + """Execute command in instance, recording output, error and exit code. - Execute assumes functional networking and execution as root with the + Assumes functional networking and execution as root with the target filesystem being available at /. - return_value: tuple containing stdout data, stderr data, exit code + @param command: the command to execute as root inside the image + @param stdout: file handler to write output + @param stderr: file handler to write error + @param env: environment variables + @param rcs: allowed return codes from command + @param description: purpose of command + @return_value: tuple containing stdout data, stderr data, exit code """ - # TODO: the pylxd api handler for container.execute needs to be - # extended to properly pass in stdin - # TODO: the pylxd api handler for container.execute needs to be - # extended to get the return code, for now just use 0 + # ensure instance is running and execute the command self.start() - if stdin: - raise NotImplementedError res = self.pylxd_container.execute(command, environment=env) - for (f, data) in (i for i in zip((stdout, stderr), res) if i[0]): - f.write(data) - return res + (0,) + + # get out, exit and err from pylxd return + if hasattr(res, 'exit_code'): + # pylxd 2.2 returns ContainerExecuteResult, named tuple of + # (exit_code, out, err) + (exit, out, err) = res + else: + # pylxd 2.1.3 and earlier only return out and err, no exit + # LOG.warning('using pylxd version < 2.2') + (out, err) = res + exit = 0 + + # write data to file descriptors if needed + if stdout: + stdout.write(out) + if stderr: + stderr.write(err) + + # if the command exited with a code not allowed in rcs, then fail + if exit not in (rcs if rcs else (0,)): + error_desc = ('Failed command to: {}'.format(description) + if description else None) + raise util.InTargetExecuteError( + out, err, exit, command, self.name, error_desc) + + return (out, err, exit) def read_data(self, remote_path, decode=False): - """ - read data from instance filesystem - remote_path: path in instance - decode: return as string - return_value: data as str or bytes + """Read data from instance filesystem. + + @param remote_path: path in instance + @param decode: return as string + @return_value: data as str or bytes """ data = self.pylxd_container.files.get(remote_path) return data.decode() if decode and isinstance(data, bytes) else data def write_data(self, remote_path, data): - """ - write data to instance filesystem - remote_path: path in instance - data: data to write, either str or bytes + """Write data to instance filesystem. + + @param remote_path: path in instance + @param data: data to write, either str or bytes """ self.pylxd_container.files.put(remote_path, data) def console_log(self): - """ - return_value: bytes of this instance’s console + """Console log. + + @return_value: bytes of this instance’s console """ raise NotImplementedError def reboot(self, wait=True): - """ - reboot instance - """ + """Reboot instance.""" self.shutdown(wait=wait) self.start(wait=wait) def shutdown(self, wait=True): - """ - shutdown instance - """ + """Shutdown instance.""" if self.pylxd_container.status != 'Stopped': self.pylxd_container.stop(wait=wait) - def start(self, wait=True, wait_time=None): - """ - start instance - """ + def start(self, wait=True, wait_for_cloud_init=False): + """Start instance.""" if self.pylxd_container.status != 'Running': self.pylxd_container.start(wait=wait) - if wait and isinstance(wait_time, int): - self._wait_for_cloud_init(wait_time) + if wait: + self._wait_for_system(wait_for_cloud_init) def freeze(self): - """ - freeze instance - """ + """Freeze instance.""" if self.pylxd_container.status != 'Frozen': self.pylxd_container.freeze(wait=True) def unfreeze(self): - """ - unfreeze instance - """ + """Unfreeze instance.""" if self.pylxd_container.status == 'Frozen': self.pylxd_container.unfreeze(wait=True) def destroy(self): - """ - clean up instance - """ + """Clean up instance.""" self.unfreeze() self.shutdown() self.pylxd_container.delete(wait=True) diff --git a/tests/cloud_tests/manage.py b/tests/cloud_tests/manage.py index 5342612b..5f0cfd23 100644 --- a/tests/cloud_tests/manage.py +++ b/tests/cloud_tests/manage.py @@ -1,11 +1,15 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Create test cases automatically given a user_data script.""" + +import os +import textwrap + +from cloudinit import util as c_util from tests.cloud_tests.config import VERIFY_EXT from tests.cloud_tests import (config, util) from tests.cloud_tests import TESTCASES_DIR -import os -import textwrap _verifier_fmt = textwrap.dedent( """ @@ -35,29 +39,24 @@ _config_fmt = textwrap.dedent( def write_testcase_config(args, fmt_args, testcase_file): - """ - write the testcase config file - """ + """Write the testcase config file.""" testcase_config = {'enabled': args.enable, 'collect_scripts': {}} if args.config: testcase_config['cloud_config'] = args.config fmt_args['config'] = util.yaml_format(testcase_config) - util.write_file(testcase_file, _config_fmt.format(**fmt_args), omode='w') + c_util.write_file(testcase_file, _config_fmt.format(**fmt_args), omode='w') def write_verifier(args, fmt_args, verifier_file): - """ - write the verifier script - """ + """Write the verifier script.""" fmt_args['test_class'] = 'Test{}'.format( - config.name_sanatize(fmt_args['test_name']).title()) - util.write_file(verifier_file, _verifier_fmt.format(**fmt_args), omode='w') + config.name_sanitize(fmt_args['test_name']).title()) + c_util.write_file(verifier_file, + _verifier_fmt.format(**fmt_args), omode='w') def create(args): - """ - create a new testcase - """ + """Create a new testcase.""" (test_category, test_name) = args.name.split('/') fmt_args = {'test_name': test_name, 'test_category': test_category, 'test_description': str(args.description)} @@ -65,7 +64,7 @@ def create(args): testcase_file = config.name_to_path(args.name) verifier_file = os.path.join( TESTCASES_DIR, test_category, - config.name_sanatize(test_name) + VERIFY_EXT) + config.name_sanitize(test_name) + VERIFY_EXT) write_testcase_config(args, fmt_args, testcase_file) write_verifier(args, fmt_args, verifier_file) diff --git a/tests/cloud_tests/platforms.yaml b/tests/cloud_tests/platforms.yaml index 5972b32b..b91834ab 100644 --- a/tests/cloud_tests/platforms.yaml +++ b/tests/cloud_tests/platforms.yaml @@ -10,7 +10,55 @@ default_platform_config: platforms: lxd: enabled: true - get_image_timeout: 600 + # overrides for image templates + template_overrides: + /var/lib/cloud/seed/nocloud-net/meta-data: + when: + - create + - copy + template: cloud-init-meta.tpl + /var/lib/cloud/seed/nocloud-net/network-config: + when: + - create + - copy + template: cloud-init-network.tpl + /var/lib/cloud/seed/nocloud-net/user-data: + when: + - create + - copy + template: cloud-init-user.tpl + properties: + default: | + #cloud-config + {} + /var/lib/cloud/seed/nocloud-net/vendor-data: + when: + - create + - copy + template: cloud-init-vendor.tpl + properties: + default: | + #cloud-config + {} + # overrides image template files + template_files: + cloud-init-meta.tpl: | + #cloud-config + instance-id: {{ container.name }} + local-hostname: {{ container.name }} + {{ config_get("user.meta-data", "") }} + cloud-init-network.tpl: | + {% if config_get("user.network-config", "") == "" %}version: 1 + config: + - type: physical + name: eth0 + subnets: + - type: {% if config_get("user.network_mode", "") == "link-local" %}manual{% else %}dhcp{% endif %} + control: auto{% else %}{{ config_get("user.network-config", "") }}{% endif %} + cloud-init-user.tpl: | + {{ config_get("user.user-data", properties.default) }} + cloud-init-vendor.tpl: | + {{ config_get("user.vendor-data", properties.default) }} ec2: {} azure: {} diff --git a/tests/cloud_tests/platforms/__init__.py b/tests/cloud_tests/platforms/__init__.py index f9f56035..443f6d44 100644 --- a/tests/cloud_tests/platforms/__init__.py +++ b/tests/cloud_tests/platforms/__init__.py @@ -1,5 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Main init.""" + from tests.cloud_tests.platforms import lxd PLATFORMS = { @@ -8,9 +10,7 @@ PLATFORMS = { def get_platform(platform_name, config): - """ - Get the platform object for 'platform_name' and init - """ + """Get the platform object for 'platform_name' and init.""" platform_cls = PLATFORMS.get(platform_name) if not platform_cls: raise ValueError('invalid platform name: {}'.format(platform_name)) diff --git a/tests/cloud_tests/platforms/base.py b/tests/cloud_tests/platforms/base.py index 615e2e06..28975368 100644 --- a/tests/cloud_tests/platforms/base.py +++ b/tests/cloud_tests/platforms/base.py @@ -1,53 +1,27 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Base platform class.""" + class Platform(object): - """ - Base class for platforms - """ + """Base class for platforms.""" + platform_name = None def __init__(self, config): - """ - Set up platform - """ + """Set up platform.""" self.config = config def get_image(self, img_conf): - """ - Get image using 'img_conf', where img_conf is a dict containing all - image configuration parameters - - in this dict there must be a 'platform_ident' key containing - configuration for identifying each image on a per platform basis - - see implementations for get_image() for details about the contents - of the platform's config entry + """Get image using specified image configuration. - note: see 'releases' main_config.yaml for example entries - - img_conf: configuration for image - return_value: cloud_tests.images instance + @param img_conf: configuration for image + @return_value: cloud_tests.images instance """ raise NotImplementedError def destroy(self): - """ - Clean up platform data - """ + """Clean up platform data.""" pass - def _extract_img_platform_config(self, img_conf): - """ - extract platform configuration for current platform from img_conf - """ - platform_ident = img_conf.get('platform_ident') - if not platform_ident: - raise ValueError('invalid img_conf, missing \'platform_ident\'') - ident = platform_ident.get(self.platform_name) - if not ident: - raise ValueError('img_conf: {} missing config for platform {}' - .format(img_conf, self.platform_name)) - return ident - # vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/lxd.py b/tests/cloud_tests/platforms/lxd.py index 847cc549..ead0955b 100644 --- a/tests/cloud_tests/platforms/lxd.py +++ b/tests/cloud_tests/platforms/lxd.py @@ -1,5 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Base LXD platform.""" + from pylxd import (Client, exceptions) from tests.cloud_tests.images import lxd as lxd_image @@ -11,48 +13,49 @@ DEFAULT_SSTREAMS_SERVER = "https://images.linuxcontainers.org:8443" class LXDPlatform(base.Platform): - """ - Lxd test platform - """ + """LXD test platform.""" + platform_name = 'lxd' def __init__(self, config): - """ - Set up platform - """ + """Set up platform.""" super(LXDPlatform, self).__init__(config) # TODO: allow configuration of remote lxd host via env variables # set up lxd connection self.client = Client() def get_image(self, img_conf): + """Get image using specified image configuration. + + @param img_conf: configuration for image + @return_value: cloud_tests.images instance """ - Get image - img_conf: dict containing config for image. platform_ident must have: - alias: alias to use for simplestreams server - sstreams_server: simplestreams server to use, or None for default - return_value: cloud_tests.images instance - """ - lxd_conf = self._extract_img_platform_config(img_conf) - image = self.client.images.create_from_simplestreams( - lxd_conf.get('sstreams_server', DEFAULT_SSTREAMS_SERVER), - lxd_conf['alias']) - return lxd_image.LXDImage( - image.properties['description'], img_conf, self, image) - - def launch_container(self, image=None, container=None, ephemeral=False, - config=None, block=True, - image_desc=None, use_desc=None): - """ - launch a container - image: image fingerprint to launch from - container: container to copy - ephemeral: delete image after first shutdown - config: config options for instance as dict - block: wait until container created - image_desc: description of image being launched - use_desc: description of container's use - return_value: cloud_tests.instances instance + pylxd_image = self.client.images.create_from_simplestreams( + img_conf.get('sstreams_server', DEFAULT_SSTREAMS_SERVER), + img_conf['alias']) + image = lxd_image.LXDImage(self, img_conf, pylxd_image) + if img_conf.get('override_templates', False): + image.update_templates(self.config.get('template_overrides', {}), + self.config.get('template_files', {})) + return image + + def launch_container(self, properties, config, features, + image=None, container=None, ephemeral=False, + container_config=None, block=True, image_desc=None, + use_desc=None): + """Launch a container. + + @param properties: image properties + @param config: image configuration + @param features: image features + @param image: image fingerprint to launch from + @param container: container to copy + @param ephemeral: delete image after first shutdown + @param container_config: config options for instance as dict + @param block: wait until container created + @param image_desc: description of image being launched + @param use_desc: description of container's use + @return_value: cloud_tests.instances instance """ if not (image or container): raise ValueError("either image or container must be specified") @@ -61,16 +64,18 @@ class LXDPlatform(base.Platform): use_desc=use_desc, used_list=self.list_containers()), 'ephemeral': bool(ephemeral), - 'config': config if isinstance(config, dict) else {}, + 'config': (container_config + if isinstance(container_config, dict) else {}), 'source': ({'type': 'image', 'fingerprint': image} if image else {'type': 'copy', 'source': container}) }, wait=block) - return lxd_instance.LXDInstance(container.name, self, container) + return lxd_instance.LXDInstance(self, container.name, properties, + config, features, container) def container_exists(self, container_name): - """ - check if container with name 'container_name' exists - return_value: True if exists else False + """Check if container with name 'container_name' exists. + + @return_value: True if exists else False """ res = True try: @@ -82,16 +87,22 @@ class LXDPlatform(base.Platform): return res def list_containers(self): - """ - list names of all containers - return_value: list of names + """List names of all containers. + + @return_value: list of names """ return [container.name for container in self.client.containers.all()] - def destroy(self): - """ - Clean up platform data + def query_image_by_alias(self, alias): + """Get image by alias in local image store. + + @param alias: alias of image + @return_value: pylxd image (not cloud_tests.images instance) """ + return self.client.images.get_by_alias(alias) + + def destroy(self): + """Clean up platform data.""" super(LXDPlatform, self).destroy() # vi: ts=4 expandtab diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml index 183f78c1..45deb58f 100644 --- a/tests/cloud_tests/releases.yaml +++ b/tests/cloud_tests/releases.yaml @@ -1,86 +1,253 @@ # ============================= Release Config ================================ default_release_config: - # all are disabled by default - enabled: false - # timeout for booting image and running cloud init - timeout: 120 - # platform_ident values for the image, with data to identify the image - # on that platform. see platforms.base for more information - platform_ident: {} - # a script to run after a boot that is used to modify an image, before - # making a snapshot of the image. may be useful for removing data left - # behind from cloud-init booting, such as logs, to ensure that data from - # snapshot.launch() will not include a cloud-init.log from a boot used to - # create the snapshot, if cloud-init has not run - boot_clean_script: | - #!/bin/bash - rm -rf /var/log/cloud-init.log /var/log/cloud-init-output.log \ - /var/lib/cloud/ /run/cloud-init/ /var/log/syslog + # global default configuration options + default: + # all are disabled by default + enabled: false + # timeout for booting image and running cloud init + boot_timeout: 120 + # a script to run after a boot that is used to modify an image, before + # making a snapshot of the image. may be useful for removing data left + # behind from cloud-init booting, such as logs, to ensure that data + # from snapshot.launch() will not include a cloud-init.log from a boot + # used to create the snapshot, if cloud-init has not run + boot_clean_script: | + #!/bin/bash + rm -rf /var/log/cloud-init.log /var/log/cloud-init-output.log \ + /var/lib/cloud/ /run/cloud-init/ /var/log/syslog + # test script to determine if system is booted fully + system_ready_script: | + # permit running or degraded state as both indicate complete boot + [ $(systemctl is-system-running) = 'running' -o + $(systemctl is-system-running) = 'degraded' ] + # test script to determine if cloud-init has finished + cloud_init_ready_script: | + [ -f '/run/cloud-init/result.json' ] + # currently used features and their uses are: + # features groups and additional feature settings + feature_groups: [] + features: {} + + # lxd specific default configuration options + lxd: + # default sstreams server to use for lxd image retrieval + sstreams_server: https://us.images.linuxcontainers.org:8443 + # keep base image, avoids downloading again next run + cache_base_image: true + # lxd images from linuxcontainers.org do not have the nocloud seed + # templates in place, so the image metadata must be modified + override_templates: true + # arg overrides to set image up + setup_overrides: + # lxd images from linuxcontainers.org do not come with + # cloud-init, so must pull cloud-init in from repo using + # setup_image.upgrade + upgrade: true + +features: + # all currently supported feature flags + all: + - apt # image supports apt package manager + - byobu # byobu is available in repositories + - landscape # landscape-client available in repos + - lxd # lxd is available in the image + - ppa # image supports ppas + - rpm # image supports rpms + - snap # supports snapd + # NOTE: the following feature flags are to work around bugs in the + # images, and can be removed when no longer needed + - hostname # setting system hostname works + # NOTE: the following feature flags are to work around issues in the + # testcases, and can be removed when no longer needed + - apt_src_cont # default contents and format of sources.list matches + # ubuntu sources.list + - apt_hist_fmt # apt command history entries use full paths to apt + # executable rather than relative paths + - daylight_time # timezones are daylight not standard time + - apt_up_out # 'Calculating upgrade..' present in log output from + # apt-get dist-upgrade output + - engb_locale # locale en_GB.UTF-8 is available + - locale_gen # the /etc/locale.gen file exists + - no_ntpdate # 'ntpdate' is not installed by default + - no_file_fmt_e # the 'file' utility does not have a formatting error + - ppa_file_name # the name of the source file added to sources.list.d has + # the expected format for newer ubuntu releases + - sshd # requires ssh server to be installed by default + - ssh_key_fmt # ssh auth keys printed to console have expected format + - syslog # test case requires syslog to be written by default + - ubuntu_ntp # expect ubuntu.pool.ntp.org to be used as ntp server + - ubuntu_repos # test case requres ubuntu repositories to be used + - ubuntu_user # test case needs user with the name 'ubuntu' to exist + # NOTE: the following feature flags are to work around issues that may + # be considered bugs in cloud-init + - lsb_release # image has lsb_release installed, maybe should install + # if missing by default + - sudo # image has sudo installed, should not be required + # feature flag groups + groups: + base: + hostname: true + no_file_fmt_e: true + ubuntu_specific: + apt_src_cont: true + apt_hist_fmt: true + byobu: true + daylight_time: true + engb_locale: true + landscape: true + locale_gen: true + lsb_release: true + lxd: true + ppa: true + ppa_file_name: true + snap: true + sshd: true + ssh_key_fmt: true + sudo: true + syslog: true + ubuntu_ntp: true + ubuntu_repos: true + ubuntu_user: true + debian_base: + apt: true + apt_up_out: true + no_ntpdate: true + rhel_base: + rpm: true releases: - trusty: - enabled: true - platform_ident: - lxd: - # if sstreams_server is omitted, default is used, defined in - # tests.cloud_tests.platforms.lxd.DEFAULT_SSTREAMS_SERVER as: - # sstreams_server: https://us.images.linuxcontainers.org:8443 - #alias: ubuntu/trusty/default - alias: t - sstreams_server: https://cloud-images.ubuntu.com/daily - xenial: - enabled: true - platform_ident: - lxd: - #alias: ubuntu/xenial/default - alias: x - sstreams_server: https://cloud-images.ubuntu.com/daily - yakkety: - enabled: true - platform_ident: - lxd: - #alias: ubuntu/yakkety/default - alias: y - sstreams_server: https://cloud-images.ubuntu.com/daily - zesty: - enabled: true - platform_ident: - lxd: - #alias: ubuntu/zesty/default - alias: z - sstreams_server: https://cloud-images.ubuntu.com/daily + # UBUNTU ================================================================= artful: - enabled: true - platform_ident: - lxd: - #alias: ubuntu/artful/default - alias: a - sstreams_server: https://cloud-images.ubuntu.com/daily - jessie: - platform_ident: - lxd: - alias: debian/jessie/default - sid: - platform_ident: - lxd: - alias: debian/sid/default + # EOL: Jul 2018 + default: + enabled: true + feature_groups: + - base + - debian_base + - ubuntu_specific + lxd: + sstreams_server: https://cloud-images.ubuntu.com/daily + alias: artful + setup_overrides: null + override_templates: false + zesty: + # EOL: Jan 2018 + default: + enabled: true + feature_groups: + - base + - debian_base + - ubuntu_specific + lxd: + sstreams_server: https://cloud-images.ubuntu.com/daily + alias: zesty + setup_overrides: null + override_templates: false + yakkety: + # EOL: Jul 2017 + default: + enabled: true + feature_groups: + - base + - debian_base + - ubuntu_specific + lxd: + sstreams_server: https://cloud-images.ubuntu.com/daily + alias: yakkety + setup_overrides: null + override_templates: false + xenial: + # EOL: Apr 2021 + default: + enabled: true + feature_groups: + - base + - debian_base + - ubuntu_specific + lxd: + sstreams_server: https://cloud-images.ubuntu.com/daily + alias: xenial + setup_overrides: null + override_templates: false + trusty: + # EOL: Apr 2019 + default: + enabled: true + feature_groups: + - base + - debian_base + - ubuntu_specific + features: + apt_up_out: false + locale_gen: false + lxd: false + ppa_file_name: false + snap: false + ssh_key_fmt: false + no_ntpdate: false + no_file_fmt_e: false + system_ready_script: | + #!/bin/bash + # upstart based, so use old style runlevels + [ $(runlevel | awk '{print $2}') = '2' ] + lxd: + sstreams_server: https://cloud-images.ubuntu.com/daily + alias: trusty + setup_overrides: null + override_templates: false + # DEBIAN ================================================================= stretch: - platform_ident: - lxd: - alias: debian/stretch/default - wheezy: - platform_ident: - lxd: - alias: debian/wheezy/default + # EOL: Not yet released + default: + enabled: true + feature_groups: + - base + - debian_base + lxd: + alias: debian/stretch/default + jessie: + # EOL: Jun 2020 + # NOTE: the cloud-init version shipped with jessie is out of date + # tests work if an up to date deb is used + default: + enabled: true + feature_groups: + - base + - debian_base + lxd: + alias: debian/jessie/default + # CENTOS ================================================================= centos70: - timeout: 180 - platform_ident: - lxd: - alias: centos/7/default + # EOL: Jun 2024 (2020 - end of full updates) + default: + enabled: true + feature_groups: + - base + - rhel_base + user_data_overrides: + preserve_hostname: true + lxd: + features: + # NOTE: (LP: #1575779) + hostname: false + alias: centos/7/default centos66: - timeout: 180 - platform_ident: - lxd: - alias: centos/6/default + # EOL: Nov 2020 + default: + enabled: true + feature_groups: + - base + - rhel_base + # still supported, but only bugfixes after may 2017 + system_ready_script: | + #!/bin/bash + [ $(runlevel | awk '{print $2}') = '3' ] + user_data_overrides: + preserve_hostname: true + lxd: + features: + # NOTE: (LP: #1575779) + hostname: false + alias: centos/6/default # vi: ts=4 expandtab diff --git a/tests/cloud_tests/run_funcs.py b/tests/cloud_tests/run_funcs.py new file mode 100644 index 00000000..8ae91120 --- /dev/null +++ b/tests/cloud_tests/run_funcs.py @@ -0,0 +1,75 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Run functions.""" + +import os + +from tests.cloud_tests import bddeb, collect, util, verify + + +def tree_collect(args): + """Collect data using deb build from current tree. + + @param args: cmdline args + @return_value: fail count + """ + failed = 0 + tmpdir = util.TempDir(tmpdir=args.data_dir, preserve=args.preserve_data) + + with tmpdir as data_dir: + args.data_dir = data_dir + args.deb = os.path.join(tmpdir.tmpdir, 'cloud-init_all.deb') + try: + failed += bddeb.bddeb(args) + failed += collect.collect(args) + except Exception: + failed += 1 + raise + + return failed + + +def tree_run(args): + """Run test suite using deb build from current tree. + + @param args: cmdline args + @return_value: fail count + """ + failed = 0 + tmpdir = util.TempDir(tmpdir=args.data_dir, preserve=args.preserve_data) + + with tmpdir as data_dir: + args.data_dir = data_dir + args.deb = os.path.join(tmpdir.tmpdir, 'cloud-init_all.deb') + try: + failed += bddeb.bddeb(args) + failed += collect.collect(args) + failed += verify.verify(args) + except Exception: + failed += 1 + raise + + return failed + + +def run(args): + """Run test suite. + + @param args: cmdline args + @return_value: fail count + """ + failed = 0 + tmpdir = util.TempDir(tmpdir=args.data_dir, preserve=args.preserve_data) + + with tmpdir as data_dir: + args.data_dir = data_dir + try: + failed += collect.collect(args) + failed += verify.verify(args) + except Exception: + failed += 1 + raise + + return failed + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/setup_image.py b/tests/cloud_tests/setup_image.py index 5d6c6387..8053a093 100644 --- a/tests/cloud_tests/setup_image.py +++ b/tests/cloud_tests/setup_image.py @@ -1,18 +1,42 @@ # This file is part of cloud-init. See LICENSE file for license information. -from tests.cloud_tests import LOG -from tests.cloud_tests import stage, util +"""Setup image for testing.""" from functools import partial import os +from tests.cloud_tests import LOG +from tests.cloud_tests import stage, util -def install_deb(args, image): + +def installed_package_version(image, package, ensure_installed=True): + """Get installed version of package. + + @param image: cloud_tests.images instance to operate on + @param package: name of package + @param ensure_installed: raise error if not installed + @return_value: cloud-init version string """ - install deb into image - args: cmdline arguments, must contain --deb - image: cloud_tests.images instance to operate on - return_value: None, may raise errors + os_family = util.get_os_family(image.properties['os']) + if os_family == 'debian': + cmd = ['dpkg-query', '-W', "--showformat='${Version}'", package] + elif os_family == 'redhat': + cmd = ['rpm', '-q', '--queryformat', "'%{VERSION}'", package] + else: + raise NotImplementedError + + msg = 'query version for package: {}'.format(package) + (out, err, exit) = image.execute( + cmd, description=msg, rcs=(0,) if ensure_installed else range(0, 256)) + return out.strip() + + +def install_deb(args, image): + """Install deb into image. + + @param args: cmdline arguments, must contain --deb + @param image: cloud_tests.images instance to operate on + @return_value: None, may raise errors """ # ensure system is compatible with package format os_family = util.get_os_family(image.properties['os']) @@ -21,20 +45,18 @@ def install_deb(args, image): 'family: {}'.format(args.deb, os_family)) # install deb - LOG.debug('installing deb: %s into target', args.deb) + msg = 'install deb: "{}" into target'.format(args.deb) + LOG.debug(msg) remote_path = os.path.join('/tmp', os.path.basename(args.deb)) image.push_file(args.deb, remote_path) - (out, err, exit) = image.execute(['dpkg', '-i', remote_path]) - if exit != 0: - raise OSError('failed install deb: {}\n\tstdout: {}\n\tstderr: {}' - .format(args.deb, out, err)) + cmd = 'dpkg -i {} || apt-get install --yes -f'.format(remote_path) + image.execute(['/bin/sh', '-c', cmd], description=msg) # check installed deb version matches package fmt = ['-W', "--showformat='${Version}'"] (out, err, exit) = image.execute(['dpkg-deb'] + fmt + [remote_path]) expected_version = out.strip() - (out, err, exit) = image.execute(['dpkg-query'] + fmt + ['cloud-init']) - found_version = out.strip() + found_version = installed_package_version(image, 'cloud-init') if expected_version != found_version: raise OSError('install deb version "{}" does not match expected "{}"' .format(found_version, expected_version)) @@ -44,32 +66,28 @@ def install_deb(args, image): def install_rpm(args, image): + """Install rpm into image. + + @param args: cmdline arguments, must contain --rpm + @param image: cloud_tests.images instance to operate on + @return_value: None, may raise errors """ - install rpm into image - args: cmdline arguments, must contain --rpm - image: cloud_tests.images instance to operate on - return_value: None, may raise errors - """ - # ensure system is compatible with package format os_family = util.get_os_family(image.properties['os']) - if os_family not in ['redhat', 'sles']: + if os_family != 'redhat': raise NotImplementedError('install rpm: {} not supported on os ' 'family: {}'.format(args.rpm, os_family)) # install rpm - LOG.debug('installing rpm: %s into target', args.rpm) + msg = 'install rpm: "{}" into target'.format(args.rpm) + LOG.debug(msg) remote_path = os.path.join('/tmp', os.path.basename(args.rpm)) image.push_file(args.rpm, remote_path) - (out, err, exit) = image.execute(['rpm', '-U', remote_path]) - if exit != 0: - raise OSError('failed to install rpm: {}\n\tstdout: {}\n\tstderr: {}' - .format(args.rpm, out, err)) + image.execute(['rpm', '-U', remote_path], description=msg) fmt = ['--queryformat', '"%{VERSION}"'] (out, err, exit) = image.execute(['rpm', '-q'] + fmt + [remote_path]) expected_version = out.strip() - (out, err, exit) = image.execute(['rpm', '-q'] + fmt + ['cloud-init']) - found_version = out.strip() + found_version = installed_package_version(image, 'cloud-init') if expected_version != found_version: raise OSError('install rpm version "{}" does not match expected "{}"' .format(found_version, expected_version)) @@ -79,14 +97,32 @@ def install_rpm(args, image): def upgrade(args, image): + """Upgrade or install cloud-init from repo. + + @param args: cmdline arguments + @param image: cloud_tests.images instance to operate on + @return_value: None, may raise errors """ - run the system's upgrade command - args: cmdline arguments - image: cloud_tests.images instance to operate on - return_value: None, may raise errors + os_family = util.get_os_family(image.properties['os']) + if os_family == 'debian': + cmd = 'apt-get update && apt-get install cloud-init --yes' + elif os_family == 'redhat': + cmd = 'sleep 10 && yum install cloud-init --assumeyes' + else: + raise NotImplementedError + + msg = 'upgrading cloud-init' + LOG.debug(msg) + image.execute(['/bin/sh', '-c', cmd], description=msg) + + +def upgrade_full(args, image): + """Run the system's full upgrade command. + + @param args: cmdline arguments + @param image: cloud_tests.images instance to operate on + @return_value: None, may raise errors """ - # determine appropriate upgrade command for os_family - # TODO: maybe use cloudinit.distros for this? os_family = util.get_os_family(image.properties['os']) if os_family == 'debian': cmd = 'apt-get update && apt-get upgrade --yes' @@ -96,53 +132,48 @@ def upgrade(args, image): raise NotImplementedError('upgrade command not configured for distro ' 'from family: {}'.format(os_family)) - # upgrade system - LOG.debug('upgrading system') - (out, err, exit) = image.execute(['/bin/sh', '-c', cmd]) - if exit != 0: - raise OSError('failed to upgrade system\n\tstdout: {}\n\tstderr:{}' - .format(out, err)) + msg = 'full system upgrade' + LOG.debug(msg) + image.execute(['/bin/sh', '-c', cmd], description=msg) def run_script(args, image): + """Run a script in the target image. + + @param args: cmdline arguments, must contain --script + @param image: cloud_tests.images instance to operate on + @return_value: None, may raise errors """ - run a script in the target image - args: cmdline arguments, must contain --script - image: cloud_tests.images instance to operate on - return_value: None, may raise errors - """ - # TODO: get exit status back from script and add error handling here - LOG.debug('running setup image script in target image') - image.run_script(args.script) + msg = 'run setup image script in target image' + LOG.debug(msg) + image.run_script(args.script, description=msg) def enable_ppa(args, image): - """ - enable a ppa in the target image - args: cmdline arguments, must contain --ppa - image: cloud_tests.image instance to operate on - return_value: None, may raise errors + """Enable a ppa in the target image. + + @param args: cmdline arguments, must contain --ppa + @param image: cloud_tests.image instance to operate on + @return_value: None, may raise errors """ # ppa only supported on ubuntu (maybe debian?) - if image.properties['os'] != 'ubuntu': + if image.properties['os'].lower() != 'ubuntu': raise NotImplementedError('enabling a ppa is only available on ubuntu') # add ppa with add-apt-repository and update ppa = 'ppa:{}'.format(args.ppa) - LOG.debug('enabling %s', ppa) + msg = 'enable ppa: "{}" in target'.format(ppa) + LOG.debug(msg) cmd = 'add-apt-repository --yes {} && apt-get update'.format(ppa) - (out, err, exit) = image.execute(['/bin/sh', '-c', cmd]) - if exit != 0: - raise OSError('enable ppa for {} failed\n\tstdout: {}\n\tstderr: {}' - .format(ppa, out, err)) + image.execute(['/bin/sh', '-c', cmd], description=msg) def enable_repo(args, image): - """ - enable a repository in the target image - args: cmdline arguments, must contain --repo - image: cloud_tests.image instance to operate on - return_value: None, may raise errors + """Enable a repository in the target image. + + @param args: cmdline arguments, must contain --repo + @param image: cloud_tests.image instance to operate on + @return_value: None, may raise errors """ # find enable repo command for the distro os_family = util.get_os_family(image.properties['os']) @@ -155,20 +186,23 @@ def enable_repo(args, image): raise NotImplementedError('enable repo command not configured for ' 'distro from family: {}'.format(os_family)) - LOG.debug('enabling repo: "%s"', args.repo) - (out, err, exit) = image.execute(['/bin/sh', '-c', cmd]) - if exit != 0: - raise OSError('enable repo {} failed\n\tstdout: {}\n\tstderr: {}' - .format(args.repo, out, err)) + msg = 'enable repo: "{}" in target'.format(args.repo) + LOG.debug(msg) + image.execute(['/bin/sh', '-c', cmd], description=msg) def setup_image(args, image): + """Set up image as specified in args. + + @param args: cmdline arguments + @param image: cloud_tests.image instance to operate on + @return_value: tuple of results and fail count """ - set up image as specified in args - args: cmdline arguments - image: cloud_tests.image instance to operate on - return_value: tuple of results and fail count - """ + # update the args if necessary for this image + overrides = image.setup_overrides + LOG.debug('updating args for setup with: %s', overrides) + args = util.update_args(args, overrides, preserve_old=True) + # mapping of setup cmdline arg name to setup function # represented as a tuple rather than a dict or odict as lookup by name not # needed, and order is important as --script and --upgrade go at the end @@ -179,17 +213,19 @@ def setup_image(args, image): ('repo', enable_repo, 'setup func for --repo, enable repo'), ('ppa', enable_ppa, 'setup func for --ppa, enable ppa'), ('script', run_script, 'setup func for --script, run script'), - ('upgrade', upgrade, 'setup func for --upgrade, upgrade pkgs'), + ('upgrade', upgrade, 'setup func for --upgrade, upgrade cloud-init'), + ('upgrade-full', upgrade_full, 'setup func for --upgrade-full'), ) # determine which setup functions needed calls = [partial(stage.run_single, desc, partial(func, args, image)) for name, func, desc in handlers if getattr(args, name, None)] - image_name = 'image: distro={}, release={}'.format( - image.properties['os'], image.properties['release']) - LOG.info('setting up %s', image_name) - return stage.run_stage('set up for {}'.format(image_name), calls, - continue_after_error=False) + LOG.info('setting up %s', image) + res = stage.run_stage( + 'set up for {}'.format(image), calls, continue_after_error=False) + LOG.debug('after setup complete, installed cloud-init version is: %s', + installed_package_version(image, 'cloud-init')) + return res # vi: ts=4 expandtab diff --git a/tests/cloud_tests/snapshots/__init__.py b/tests/cloud_tests/snapshots/__init__.py index 2ab654de..93a54f5e 100644 --- a/tests/cloud_tests/snapshots/__init__.py +++ b/tests/cloud_tests/snapshots/__init__.py @@ -1,10 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Main init.""" + def get_snapshot(image): - """ - get snapshot from image - """ + """Get snapshot from image.""" return image.snapshot() # vi: ts=4 expandtab diff --git a/tests/cloud_tests/snapshots/base.py b/tests/cloud_tests/snapshots/base.py index d715f037..94328982 100644 --- a/tests/cloud_tests/snapshots/base.py +++ b/tests/cloud_tests/snapshots/base.py @@ -1,44 +1,45 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Base snapshot.""" + class Snapshot(object): - """ - Base class for snapshots - """ + """Base class for snapshots.""" + platform_name = None - def __init__(self, properties, config): - """ - Set up snapshot + def __init__(self, platform, properties, config, features): + """Set up snapshot. + + @param platform: platform object + @param properties: image properties + @param config: image config + @param features: supported feature flags """ + self.platform = platform self.properties = properties self.config = config + self.features = features def __str__(self): - """ - a brief description of the snapshot - """ + """A brief description of the snapshot.""" return '-'.join((self.properties['os'], self.properties['release'])) def launch(self, user_data, meta_data=None, block=True, start=True, use_desc=None): - """ - launch instance - - user_data: user-data for the instance - instance_id: instance-id for the instance - block: wait until instance is created - start: start instance and wait until fully started - use_desc: description of snapshot instance use + """Launch instance. - return_value: an Instance + @param user_data: user-data for the instance + @param instance_id: instance-id for the instance + @param block: wait until instance is created + @param start: start instance and wait until fully started + @param use_desc: description of snapshot instance use + @return_value: an Instance """ raise NotImplementedError def destroy(self): - """ - Clean up snapshot data - """ + """Clean up snapshot data.""" pass # vi: ts=4 expandtab diff --git a/tests/cloud_tests/snapshots/lxd.py b/tests/cloud_tests/snapshots/lxd.py index eabbce3f..39c55c5e 100644 --- a/tests/cloud_tests/snapshots/lxd.py +++ b/tests/cloud_tests/snapshots/lxd.py @@ -1,49 +1,52 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Base LXD snapshot.""" + from tests.cloud_tests.snapshots import base class LXDSnapshot(base.Snapshot): - """ - LXD image copy backed snapshot - """ + """LXD image copy backed snapshot.""" + platform_name = "lxd" - def __init__(self, properties, config, platform, pylxd_frozen_instance): - """ - Set up snapshot + def __init__(self, platform, properties, config, features, + pylxd_frozen_instance): + """Set up snapshot. + + @param platform: platform object + @param properties: image properties + @param config: image config + @param features: supported feature flags """ - self.platform = platform self.pylxd_frozen_instance = pylxd_frozen_instance - super(LXDSnapshot, self).__init__(properties, config) + super(LXDSnapshot, self).__init__( + platform, properties, config, features) def launch(self, user_data, meta_data=None, block=True, start=True, use_desc=None): - """ - launch instance - - user_data: user-data for the instance - instance_id: instance-id for the instance - block: wait until instance is created - start: start instance and wait until fully started - use_desc: description of snapshot instance use - - return_value: an Instance + """Launch instance. + + @param user_data: user-data for the instance + @param instance_id: instance-id for the instance + @param block: wait until instance is created + @param start: start instance and wait until fully started + @param use_desc: description of snapshot instance use + @return_value: an Instance """ inst_config = {'user.user-data': user_data} if meta_data: inst_config['user.meta-data'] = meta_data instance = self.platform.launch_container( - container=self.pylxd_frozen_instance.name, config=inst_config, - block=block, image_desc=str(self), use_desc=use_desc) + self.properties, self.config, self.features, block=block, + image_desc=str(self), container=self.pylxd_frozen_instance.name, + use_desc=use_desc, container_config=inst_config) if start: - instance.start(wait=True, wait_time=self.config.get('timeout')) + instance.start() return instance def destroy(self): - """ - Clean up snapshot data - """ + """Clean up snapshot data.""" self.pylxd_frozen_instance.destroy() super(LXDSnapshot, self).destroy() diff --git a/tests/cloud_tests/stage.py b/tests/cloud_tests/stage.py index 584cdaee..74a7d46d 100644 --- a/tests/cloud_tests/stage.py +++ b/tests/cloud_tests/stage.py @@ -1,5 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Stage a run.""" + import sys import time import traceback @@ -8,38 +10,29 @@ from tests.cloud_tests import LOG class PlatformComponent(object): - """ - context manager to safely handle platform components, ensuring that - .destroy() is called - """ + """Context manager to safely handle platform components.""" def __init__(self, get_func): - """ - store get_ function as partial taking no args - """ + """Store get_ function as partial with no args.""" self.get_func = get_func def __enter__(self): - """ - create instance of platform component - """ + """Create instance of platform component.""" self.instance = self.get_func() return self.instance def __exit__(self, etype, value, trace): - """ - destroy instance - """ + """Destroy instance.""" if self.instance is not None: self.instance.destroy() def run_single(name, call): - """ - run a single function, keeping track of results and failures and time - name: name of part - call: call to make - return_value: a tuple of result and fail count + """Run a single function, keeping track of results and time. + + @param name: name of part + @param call: call to make + @return_value: a tuple of result and fail count """ res = { 'name': name, @@ -67,17 +60,18 @@ def run_single(name, call): def run_stage(parent_name, calls, continue_after_error=True): - """ - run a stage of collection, keeping track of results and failures - parent_name: name of stage calls are under - calls: list of function call taking no params. must return a tuple - of results and failures. may raise exceptions - continue_after_error: whether or not to proceed to the next call after - catching an exception or recording a failure - return_value: a tuple of results and failures, with result containing - results from the function call under 'stages', and a list - of errors (if any on this level), and elapsed time - running stage, and the name + """Run a stage of collection, keeping track of results and failures. + + @param parent_name: name of stage calls are under + @param calls: list of function call taking no params. must return a tuple + of results and failures. may raise exceptions + @param continue_after_error: whether or not to proceed to the next call + after catching an exception or recording a + failure + @return_value: a tuple of results and failures, with result containing + results from the function call under 'stages', and a list + of errors (if any on this level), and elapsed time + running stage, and the name """ res = { 'name': parent_name, diff --git a/tests/cloud_tests/testcases.yaml b/tests/cloud_tests/testcases.yaml index c22b08ef..7183e017 100644 --- a/tests/cloud_tests/testcases.yaml +++ b/tests/cloud_tests/testcases.yaml @@ -2,6 +2,7 @@ base_test_data: script_timeout: 20 enabled: True + required_features: [] cloud_config: | #cloud-config collect_scripts: diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py index a1d86d45..47217ce6 100644 --- a/tests/cloud_tests/testcases/__init__.py +++ b/tests/cloud_tests/testcases/__init__.py @@ -1,5 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Main init.""" + import importlib import inspect import unittest @@ -9,12 +11,12 @@ from tests.cloud_tests.testcases.base import CloudTestCase as base_test def discover_tests(test_name): - """ - discover tests in test file for 'testname' - return_value: list of test classes + """Discover tests in test file for 'testname'. + + @return_value: list of test classes """ testmod_name = 'tests.cloud_tests.testcases.{}'.format( - config.name_sanatize(test_name)) + config.name_sanitize(test_name)) try: testmod = importlib.import_module(testmod_name) except NameError: @@ -26,9 +28,9 @@ def discover_tests(test_name): def get_suite(test_name, data, conf): - """ - get test suite with all tests for 'testname' - return_value: a test suite + """Get test suite with all tests for 'testname'. + + @return_value: a test suite """ suite = unittest.TestSuite() for test_class in discover_tests(test_name): diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 64d5507a..bb545ab9 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -1,61 +1,55 @@ # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit import util as c_util +"""Base test case module.""" import crypt import json import unittest +from cloudinit import util as c_util + class CloudTestCase(unittest.TestCase): - """ - base test class for verifiers - """ + """Base test class for verifiers.""" + data = None conf = None _cloud_config = None def shortDescription(self): + """Prevent nose from using docstrings.""" return None @property def cloud_config(self): - """ - get the cloud-config used by the test - """ + """Get the cloud-config used by the test.""" if not self._cloud_config: self._cloud_config = c_util.load_yaml(self.conf) return self._cloud_config def get_config_entry(self, name): - """ - get a config entry from cloud-config ensuring that it is present - """ + """Get a config entry from cloud-config ensuring that it is present.""" if name not in self.cloud_config: raise AssertionError('Key "{}" not in cloud config'.format(name)) return self.cloud_config[name] def get_data_file(self, name): - """ - get data file failing test if it is not present - """ + """Get data file failing test if it is not present.""" if name not in self.data: raise AssertionError('File "{}" missing from collect data' .format(name)) return self.data[name] def get_instance_id(self): - """ - get recorded instance id - """ + """Get recorded instance id.""" return self.get_data_file('instance-id').strip() def get_status_data(self, data, version=None): - """ - parse result.json and status.json like data files - data: data to load - version: cloud-init output version, defaults to 'v1' - return_value: dict of data or None if missing + """Parse result.json and status.json like data files. + + @param data: data to load + @param version: cloud-init output version, defaults to 'v1' + @return_value: dict of data or None if missing """ if not version: version = 'v1' @@ -63,16 +57,12 @@ class CloudTestCase(unittest.TestCase): return data.get(version) def get_datasource(self): - """ - get datasource name - """ + """Get datasource name.""" data = self.get_status_data(self.get_data_file('result.json')) return data.get('datasource') def test_no_stages_errors(self): - """ - ensure that there were no errors in any stage - """ + """Ensure that there were no errors in any stage.""" status = self.get_status_data(self.get_data_file('status.json')) for stage in ('init', 'init-local', 'modules-config', 'modules-final'): self.assertIn(stage, status) @@ -84,7 +74,10 @@ class CloudTestCase(unittest.TestCase): class PasswordListTest(CloudTestCase): + """Base password test case class.""" + def test_shadow_passwords(self): + """Test shadow passwords.""" shadow = self.get_data_file('shadow') users = {} dupes = [] @@ -121,7 +114,7 @@ class PasswordListTest(CloudTestCase): self.assertNotEqual(users['harry'], users['dick']) def test_shadow_expected_users(self): - """Test every tom, dick, and harry user in shadow""" + """Test every tom, dick, and harry user in shadow.""" out = self.get_data_file('shadow') self.assertIn('tom:', out) self.assertIn('dick:', out) @@ -130,7 +123,7 @@ class PasswordListTest(CloudTestCase): self.assertIn('mikey:', out) def test_sshd_config(self): - """Test sshd config allows passwords""" + """Test sshd config allows passwords.""" out = self.get_data_file('sshd_config') self.assertIn('PasswordAuthentication yes', out) diff --git a/tests/cloud_tests/testcases/bugs/__init__.py b/tests/cloud_tests/testcases/bugs/__init__.py index 5251d7c1..c6452f9c 100644 --- a/tests/cloud_tests/testcases/bugs/__init__.py +++ b/tests/cloud_tests/testcases/bugs/__init__.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. -""" -Test verifiers for cloud-init bugs +"""Test verifiers for cloud-init bugs. + See configs/bugs/README.md for more information """ diff --git a/tests/cloud_tests/testcases/bugs/lp1511485.py b/tests/cloud_tests/testcases/bugs/lp1511485.py index ac5ccb42..670d3aff 100644 --- a/tests/cloud_tests/testcases/bugs/lp1511485.py +++ b/tests/cloud_tests/testcases/bugs/lp1511485.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestLP1511485(base.CloudTestCase): - """Test LP# 1511485""" + """Test LP# 1511485.""" def test_final_message(self): - """Test final message exists""" + """Test final message exists.""" out = self.get_data_file('cloud-init-output.log') self.assertIn('Final message from cloud-config', out) diff --git a/tests/cloud_tests/testcases/bugs/lp1628337.py b/tests/cloud_tests/testcases/bugs/lp1628337.py index af0ffc75..a2c90481 100644 --- a/tests/cloud_tests/testcases/bugs/lp1628337.py +++ b/tests/cloud_tests/testcases/bugs/lp1628337.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestLP1628337(base.CloudTestCase): - """Test LP# 1511485""" + """Test LP# 1511485.""" def test_fetch_indices(self): - """Verify no apt errors""" + """Verify no apt errors.""" out = self.get_data_file('cloud-init-output.log') self.assertNotIn('W: Failed to fetch', out) self.assertNotIn('W: Some index files failed to download. ' @@ -16,7 +16,7 @@ class TestLP1628337(base.CloudTestCase): out) def test_ntp(self): - """Verify can find ntp and install it""" + """Verify can find ntp and install it.""" out = self.get_data_file('cloud-init-output.log') self.assertNotIn('E: Unable to locate package ntp', out) diff --git a/tests/cloud_tests/testcases/examples/__init__.py b/tests/cloud_tests/testcases/examples/__init__.py index b3af7f8a..39af88c2 100644 --- a/tests/cloud_tests/testcases/examples/__init__.py +++ b/tests/cloud_tests/testcases/examples/__init__.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. -""" -Test verifiers for cloud-init examples +"""Test verifiers for cloud-init examples. + See configs/examples/README.md for more information """ diff --git a/tests/cloud_tests/testcases/examples/add_apt_repositories.py b/tests/cloud_tests/testcases/examples/add_apt_repositories.py index 15b8f01c..71eede97 100644 --- a/tests/cloud_tests/testcases/examples/add_apt_repositories.py +++ b/tests/cloud_tests/testcases/examples/add_apt_repositories.py @@ -1,19 +1,19 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestAptconfigurePrimary(base.CloudTestCase): - """Example cloud-config test""" + """Example cloud-config test.""" def test_ubuntu_sources(self): - """Test no default Ubuntu entries exist""" + """Test no default Ubuntu entries exist.""" out = self.get_data_file('ubuntu.sources.list') self.assertEqual(0, int(out)) def test_gatech_sources(self): - """Test GaTech entires exist""" + """Test GaTech entires exist.""" out = self.get_data_file('gatech.sources.list') self.assertEqual(20, int(out)) diff --git a/tests/cloud_tests/testcases/examples/alter_completion_message.py b/tests/cloud_tests/testcases/examples/alter_completion_message.py index b06ad01b..b7b5d5e0 100644 --- a/tests/cloud_tests/testcases/examples/alter_completion_message.py +++ b/tests/cloud_tests/testcases/examples/alter_completion_message.py @@ -1,34 +1,27 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestFinalMessage(base.CloudTestCase): - """ - test cloud init module `cc_final_message` - """ + """Test cloud init module `cc_final_message`.""" + subs_char = '$' def get_final_message_config(self): - """ - get config for final message - """ + """Get config for final message.""" self.assertIn('final_message', self.cloud_config) return self.cloud_config['final_message'] def get_final_message(self): - """ - get final message from log - """ + """Get final message from log.""" out = self.get_data_file('cloud-init-output.log') lines = len(self.get_final_message_config().splitlines()) return '\n'.join(out.splitlines()[-1 * lines:]) def test_final_message_string(self): - """ - ensure final handles regular strings - """ + """Ensure final handles regular strings.""" for actual, config in zip( self.get_final_message().splitlines(), self.get_final_message_config().splitlines()): @@ -36,9 +29,7 @@ class TestFinalMessage(base.CloudTestCase): self.assertEqual(actual, config) def test_final_message_subs(self): - """ - test variable substitution in final message - """ + """Test variable substitution in final message.""" # TODO: add verification of other substitutions patterns = {'$datasource': self.get_datasource()} for key, expected in patterns.items(): diff --git a/tests/cloud_tests/testcases/examples/configure_instance_trusted_ca_certificates.py b/tests/cloud_tests/testcases/examples/configure_instance_trusted_ca_certificates.py index 8a4a0db0..38540eb8 100644 --- a/tests/cloud_tests/testcases/examples/configure_instance_trusted_ca_certificates.py +++ b/tests/cloud_tests/testcases/examples/configure_instance_trusted_ca_certificates.py @@ -1,24 +1,24 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestTrustedCA(base.CloudTestCase): - """Example cloud-config test""" + """Example cloud-config test.""" def test_cert_count_ca(self): - """Test correct count of CAs in .crt""" + """Test correct count of CAs in .crt.""" out = self.get_data_file('cert_count_ca') self.assertIn('7 /etc/ssl/certs/ca-certificates.crt', out) def test_cert_count_cloudinit(self): - """Test correct count of CAs in .pem""" + """Test correct count of CAs in .pem.""" out = self.get_data_file('cert_count_cloudinit') self.assertIn('7 /etc/ssl/certs/cloud-init-ca-certs.pem', out) def test_cloudinit_certs(self): - """Test text of cert""" + """Test text of cert.""" out = self.get_data_file('cloudinit_certs') self.assertIn('-----BEGIN CERTIFICATE-----', out) self.assertIn('YOUR-ORGS-TRUSTED-CA-CERT-HERE', out) diff --git a/tests/cloud_tests/testcases/examples/configure_instances_ssh_keys.py b/tests/cloud_tests/testcases/examples/configure_instances_ssh_keys.py index 4f651703..691a316b 100644 --- a/tests/cloud_tests/testcases/examples/configure_instances_ssh_keys.py +++ b/tests/cloud_tests/testcases/examples/configure_instances_ssh_keys.py @@ -1,29 +1,29 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestSSHKeys(base.CloudTestCase): - """Example cloud-config test""" + """Example cloud-config test.""" def test_cert_count(self): - """Test cert count""" + """Test cert count.""" out = self.get_data_file('cert_count') self.assertEqual(20, int(out)) def test_dsa_public(self): - """Test DSA key has ending""" + """Test DSA key has ending.""" out = self.get_data_file('dsa_public') self.assertIn('ZN4XnifuO5krqAybngIy66PMEoQ= smoser@localhost', out) def test_rsa_public(self): - """Test RSA key has specific ending""" + """Test RSA key has specific ending.""" out = self.get_data_file('rsa_public') self.assertIn('PemAWthxHO18QJvWPocKJtlsDNi3 smoser@localhost', out) def test_auth_keys(self): - """Test authorized keys has specific ending""" + """Test authorized keys has specific ending.""" out = self.get_data_file('auth_keys') self.assertIn('QPOt5Q8zWd9qG7PBl9+eiH5qV7NZ mykey@host', out) self.assertIn('Hj29SCmXp5Kt5/82cD/VN3NtHw== smoser@brickies', out) diff --git a/tests/cloud_tests/testcases/examples/including_user_groups.py b/tests/cloud_tests/testcases/examples/including_user_groups.py index e5732322..67af527b 100644 --- a/tests/cloud_tests/testcases/examples/including_user_groups.py +++ b/tests/cloud_tests/testcases/examples/including_user_groups.py @@ -1,42 +1,42 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestUserGroups(base.CloudTestCase): - """Example cloud-config test""" + """Example cloud-config test.""" def test_group_ubuntu(self): - """Test ubuntu group exists""" + """Test ubuntu group exists.""" out = self.get_data_file('group_ubuntu') self.assertRegex(out, r'ubuntu:x:[0-9]{4}:') def test_group_cloud_users(self): - """Test cloud users group exists""" + """Test cloud users group exists.""" out = self.get_data_file('group_cloud_users') self.assertRegex(out, r'cloud-users:x:[0-9]{4}:barfoo') def test_user_ubuntu(self): - """Test ubuntu user exists""" + """Test ubuntu user exists.""" out = self.get_data_file('user_ubuntu') self.assertRegex( out, r'ubuntu:x:[0-9]{4}:[0-9]{4}:Ubuntu:/home/ubuntu:/bin/bash') def test_user_foobar(self): - """Test foobar user exists""" + """Test foobar user exists.""" out = self.get_data_file('user_foobar') self.assertRegex( out, r'foobar:x:[0-9]{4}:[0-9]{4}:Foo B. Bar:/home/foobar:') def test_user_barfoo(self): - """Test barfoo user exists""" + """Test barfoo user exists.""" out = self.get_data_file('user_barfoo') self.assertRegex( out, r'barfoo:x:[0-9]{4}:[0-9]{4}:Bar B. Foo:/home/barfoo:') def test_user_cloudy(self): - """Test cloudy user exists""" + """Test cloudy user exists.""" out = self.get_data_file('user_cloudy') self.assertRegex(out, r'cloudy:x:[0-9]{3,4}:') diff --git a/tests/cloud_tests/testcases/examples/install_arbitrary_packages.py b/tests/cloud_tests/testcases/examples/install_arbitrary_packages.py index 660d1aa3..df133844 100644 --- a/tests/cloud_tests/testcases/examples/install_arbitrary_packages.py +++ b/tests/cloud_tests/testcases/examples/install_arbitrary_packages.py @@ -1,19 +1,19 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestInstall(base.CloudTestCase): - """Example cloud-config test""" + """Example cloud-config test.""" def test_htop(self): - """Verify htop installed""" + """Verify htop installed.""" out = self.get_data_file('htop') self.assertEqual(1, int(out)) def test_tree(self): - """Verify tree installed""" + """Verify tree installed.""" out = self.get_data_file('treeutils') self.assertEqual(1, int(out)) diff --git a/tests/cloud_tests/testcases/examples/install_run_chef_recipes.py b/tests/cloud_tests/testcases/examples/install_run_chef_recipes.py index b36486f0..4ec26b8f 100644 --- a/tests/cloud_tests/testcases/examples/install_run_chef_recipes.py +++ b/tests/cloud_tests/testcases/examples/install_run_chef_recipes.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestChefExample(base.CloudTestCase): - """Test chef module""" + """Test chef module.""" def test_chef_basic(self): - """Test chef installed""" + """Test chef installed.""" out = self.get_data_file('chef_installed') self.assertIn('install ok', out) diff --git a/tests/cloud_tests/testcases/examples/run_apt_upgrade.py b/tests/cloud_tests/testcases/examples/run_apt_upgrade.py index 4c04d315..744e49cb 100644 --- a/tests/cloud_tests/testcases/examples/run_apt_upgrade.py +++ b/tests/cloud_tests/testcases/examples/run_apt_upgrade.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestUpgrade(base.CloudTestCase): - """Example cloud-config test""" + """Example cloud-config test.""" def test_upgrade(self): - """Test upgrade exists in apt history""" + """Test upgrade exists in apt history.""" out = self.get_data_file('cloud-init.log') self.assertIn( '[CLOUDINIT] util.py[DEBUG]: apt-upgrade ' diff --git a/tests/cloud_tests/testcases/examples/run_commands.py b/tests/cloud_tests/testcases/examples/run_commands.py index 0be21d0f..01d5d4fc 100644 --- a/tests/cloud_tests/testcases/examples/run_commands.py +++ b/tests/cloud_tests/testcases/examples/run_commands.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestRunCmd(base.CloudTestCase): - """Example cloud-config test""" + """Example cloud-config test.""" def test_run_cmd(self): - """Test run command worked""" + """Test run command worked.""" out = self.get_data_file('run_cmd') self.assertIn('cloud-init run cmd test', out) diff --git a/tests/cloud_tests/testcases/examples/run_commands_first_boot.py b/tests/cloud_tests/testcases/examples/run_commands_first_boot.py index baa23130..3f3d8f84 100644 --- a/tests/cloud_tests/testcases/examples/run_commands_first_boot.py +++ b/tests/cloud_tests/testcases/examples/run_commands_first_boot.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestBootCmd(base.CloudTestCase): - """Example cloud-config test""" + """Example cloud-config test.""" def test_bootcmd_host(self): - """Test boot command worked""" + """Test boot command worked.""" out = self.get_data_file('hosts') self.assertIn('192.168.1.130 us.archive.ubuntu.com', out) diff --git a/tests/cloud_tests/testcases/examples/writing_out_arbitrary_files.py b/tests/cloud_tests/testcases/examples/writing_out_arbitrary_files.py index 97dfeec3..7bd520f6 100644 --- a/tests/cloud_tests/testcases/examples/writing_out_arbitrary_files.py +++ b/tests/cloud_tests/testcases/examples/writing_out_arbitrary_files.py @@ -1,29 +1,29 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestWriteFiles(base.CloudTestCase): - """Example cloud-config test""" + """Example cloud-config test.""" def test_b64(self): - """Test b64 encoded file reads as ascii""" + """Test b64 encoded file reads as ascii.""" out = self.get_data_file('file_b64') self.assertIn('ASCII text', out) def test_binary(self): - """Test binary file reads as executable""" + """Test binary file reads as executable.""" out = self.get_data_file('file_binary') self.assertIn('ELF 64-bit LSB executable, x86-64, version 1', out) def test_gzip(self): - """Test gzip file shows up as a shell script""" + """Test gzip file shows up as a shell script.""" out = self.get_data_file('file_gzip') self.assertIn('POSIX shell script, ASCII text executable', out) def test_text(self): - """Test text shows up as ASCII text""" + """Test text shows up as ASCII text.""" out = self.get_data_file('file_text') self.assertIn('ASCII text', out) diff --git a/tests/cloud_tests/testcases/main/__init__.py b/tests/cloud_tests/testcases/main/__init__.py index 5888990d..0a592637 100644 --- a/tests/cloud_tests/testcases/main/__init__.py +++ b/tests/cloud_tests/testcases/main/__init__.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. -""" -Test verifiers for cloud-init main features +"""Test verifiers for cloud-init main features. + See configs/main/README.md for more information """ diff --git a/tests/cloud_tests/testcases/main/command_output_simple.py b/tests/cloud_tests/testcases/main/command_output_simple.py index c0461a08..fe4c7670 100644 --- a/tests/cloud_tests/testcases/main/command_output_simple.py +++ b/tests/cloud_tests/testcases/main/command_output_simple.py @@ -1,17 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestCommandOutputSimple(base.CloudTestCase): - """ - test functionality of simple output redirection - """ + """Test functionality of simple output redirection.""" def test_output_file(self): - """ - ensure that the output file is not empty and has all stages - """ + """Ensure that the output file is not empty and has all stages.""" data = self.get_data_file('cloud-init-test-output') self.assertNotEqual(len(data), 0, "specified log empty") self.assertEqual(self.get_config_entry('final_message'), diff --git a/tests/cloud_tests/testcases/modules/__init__.py b/tests/cloud_tests/testcases/modules/__init__.py index 9560fb26..6ab8114d 100644 --- a/tests/cloud_tests/testcases/modules/__init__.py +++ b/tests/cloud_tests/testcases/modules/__init__.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. -""" -Test verifiers for cloud-init cc modules +"""Test verifiers for cloud-init cc modules. + See configs/modules/README.md for more information """ diff --git a/tests/cloud_tests/testcases/modules/apt_configure_conf.py b/tests/cloud_tests/testcases/modules/apt_configure_conf.py index 5d96d95c..3bf93447 100644 --- a/tests/cloud_tests/testcases/modules/apt_configure_conf.py +++ b/tests/cloud_tests/testcases/modules/apt_configure_conf.py @@ -1,19 +1,19 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestAptconfigureConf(base.CloudTestCase): - """Test apt-configure module""" + """Test apt-configure module.""" def test_apt_conf_assumeyes(self): - """Test config assumes true""" + """Test config assumes true.""" out = self.get_data_file('94cloud-init-config') self.assertIn('Assume-Yes "true";', out) def test_apt_conf_fixbroken(self): - """Test config fixes broken""" + """Test config fixes broken.""" out = self.get_data_file('94cloud-init-config') self.assertIn('Fix-Broken "true";', out) diff --git a/tests/cloud_tests/testcases/modules/apt_configure_disable_suites.py b/tests/cloud_tests/testcases/modules/apt_configure_disable_suites.py index 0e2dfdeb..eabe4607 100644 --- a/tests/cloud_tests/testcases/modules/apt_configure_disable_suites.py +++ b/tests/cloud_tests/testcases/modules/apt_configure_disable_suites.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestAptconfigureDisableSuites(base.CloudTestCase): - """Test apt-configure module""" + """Test apt-configure module.""" def test_empty_sourcelist(self): - """Test source list is empty""" + """Test source list is empty.""" out = self.get_data_file('sources.list') self.assertEqual('', out) diff --git a/tests/cloud_tests/testcases/modules/apt_configure_primary.py b/tests/cloud_tests/testcases/modules/apt_configure_primary.py index 2918785d..c1c4bbc0 100644 --- a/tests/cloud_tests/testcases/modules/apt_configure_primary.py +++ b/tests/cloud_tests/testcases/modules/apt_configure_primary.py @@ -1,19 +1,19 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestAptconfigurePrimary(base.CloudTestCase): - """Test apt-configure module""" + """Test apt-configure module.""" def test_ubuntu_sources(self): - """Test no default Ubuntu entries exist""" + """Test no default Ubuntu entries exist.""" out = self.get_data_file('ubuntu.sources.list') self.assertEqual(0, int(out)) def test_gatech_sources(self): - """Test GaTech entires exist""" + """Test GaTech entires exist.""" out = self.get_data_file('gatech.sources.list') self.assertEqual(20, int(out)) diff --git a/tests/cloud_tests/testcases/modules/apt_configure_proxy.py b/tests/cloud_tests/testcases/modules/apt_configure_proxy.py index 93ae64c6..0c61b6cc 100644 --- a/tests/cloud_tests/testcases/modules/apt_configure_proxy.py +++ b/tests/cloud_tests/testcases/modules/apt_configure_proxy.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestAptconfigureProxy(base.CloudTestCase): - """Test apt-configure module""" + """Test apt-configure module.""" def test_proxy_config(self): - """Test proxy options added to apt config""" + """Test proxy options added to apt config.""" out = self.get_data_file('90cloud-init-aptproxy') self.assertIn( 'Acquire::http::Proxy "http://squid.internal:3128";', out) diff --git a/tests/cloud_tests/testcases/modules/apt_configure_security.py b/tests/cloud_tests/testcases/modules/apt_configure_security.py index 19c79c64..7d7e2585 100644 --- a/tests/cloud_tests/testcases/modules/apt_configure_security.py +++ b/tests/cloud_tests/testcases/modules/apt_configure_security.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestAptconfigureSecurity(base.CloudTestCase): - """Test apt-configure module""" + """Test apt-configure module.""" def test_security_mirror(self): - """Test security lines added and uncommented in source.list""" + """Test security lines added and uncommented in source.list.""" out = self.get_data_file('sources.list') self.assertEqual(6, int(out)) diff --git a/tests/cloud_tests/testcases/modules/apt_configure_sources_key.py b/tests/cloud_tests/testcases/modules/apt_configure_sources_key.py index d2ee2611..d9061f3c 100644 --- a/tests/cloud_tests/testcases/modules/apt_configure_sources_key.py +++ b/tests/cloud_tests/testcases/modules/apt_configure_sources_key.py @@ -1,21 +1,21 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestAptconfigureSourcesKey(base.CloudTestCase): - """Test apt-configure module""" + """Test apt-configure module.""" def test_apt_key_list(self): - """Test key list updated""" + """Test key list updated.""" out = self.get_data_file('apt_key_list') self.assertIn( '1FF0 D853 5EF7 E719 E5C8 1B9C 083D 06FB E4D3 04DF', out) self.assertIn('Launchpad PPA for cloud init development team', out) def test_source_list(self): - """Test source.list updated""" + """Test source.list updated.""" out = self.get_data_file('sources.list') self.assertIn( 'http://ppa.launchpad.net/cloud-init-dev/test-archive/ubuntu', out) diff --git a/tests/cloud_tests/testcases/modules/apt_configure_sources_keyserver.py b/tests/cloud_tests/testcases/modules/apt_configure_sources_keyserver.py index 3931a92c..2e6b293f 100644 --- a/tests/cloud_tests/testcases/modules/apt_configure_sources_keyserver.py +++ b/tests/cloud_tests/testcases/modules/apt_configure_sources_keyserver.py @@ -1,21 +1,21 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestAptconfigureSourcesKeyserver(base.CloudTestCase): - """Test apt-configure module""" + """Test apt-configure module.""" def test_apt_key_list(self): - """Test specific key added""" + """Test specific key added.""" out = self.get_data_file('apt_key_list') self.assertIn( '1BC3 0F71 5A3B 8612 47A8 1A5E 55FE 7C8C 0165 013E', out) self.assertIn('Launchpad PPA for curtin developers', out) def test_source_list(self): - """Test source.list updated""" + """Test source.list updated.""" out = self.get_data_file('sources.list') self.assertIn( 'http://ppa.launchpad.net/cloud-init-dev/test-archive/ubuntu', out) diff --git a/tests/cloud_tests/testcases/modules/apt_configure_sources_list.py b/tests/cloud_tests/testcases/modules/apt_configure_sources_list.py index a0bb5e6b..129d2264 100644 --- a/tests/cloud_tests/testcases/modules/apt_configure_sources_list.py +++ b/tests/cloud_tests/testcases/modules/apt_configure_sources_list.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestAptconfigureSourcesList(base.CloudTestCase): - """Test apt-configure module""" + """Test apt-configure module.""" def test_sources_list(self): - """Test sources.list includes sources""" + """Test sources.list includes sources.""" out = self.get_data_file('sources.list') self.assertRegex(out, r'deb http:\/\/archive.ubuntu.com\/ubuntu ' '[a-z].* main restricted') diff --git a/tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.py b/tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.py index dcdb3767..d299e9ad 100644 --- a/tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.py +++ b/tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.py @@ -1,20 +1,20 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestAptconfigureSourcesPPA(base.CloudTestCase): - """Test apt-configure module""" + """Test apt-configure module.""" def test_ppa(self): - """test specific ppa added""" + """Test specific ppa added.""" out = self.get_data_file('sources.list') self.assertIn( 'http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu', out) def test_ppa_key(self): - """test ppa key added""" + """Test ppa key added.""" out = self.get_data_file('apt-key') self.assertIn( '1BC3 0F71 5A3B 8612 47A8 1A5E 55FE 7C8C 0165 013E', out) diff --git a/tests/cloud_tests/testcases/modules/apt_pipelining_disable.py b/tests/cloud_tests/testcases/modules/apt_pipelining_disable.py index 446c597d..c98eedef 100644 --- a/tests/cloud_tests/testcases/modules/apt_pipelining_disable.py +++ b/tests/cloud_tests/testcases/modules/apt_pipelining_disable.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestAptPipeliningDisable(base.CloudTestCase): - """Test apt-pipelining module""" + """Test apt-pipelining module.""" def test_disable_pipelining(self): - """Test pipelining disabled""" + """Test pipelining disabled.""" out = self.get_data_file('90cloud-init-pipelining') self.assertIn('Acquire::http::Pipeline-Depth "0";', out) diff --git a/tests/cloud_tests/testcases/modules/apt_pipelining_os.py b/tests/cloud_tests/testcases/modules/apt_pipelining_os.py index ad2a8884..740dc7c0 100644 --- a/tests/cloud_tests/testcases/modules/apt_pipelining_os.py +++ b/tests/cloud_tests/testcases/modules/apt_pipelining_os.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestAptPipeliningOS(base.CloudTestCase): - """Test apt-pipelining module""" + """Test apt-pipelining module.""" def test_os_pipelining(self): - """Test pipelining set to os""" + """Test pipelining set to os.""" out = self.get_data_file('90cloud-init-pipelining') self.assertIn('Acquire::http::Pipeline-Depth "0";', out) diff --git a/tests/cloud_tests/testcases/modules/bootcmd.py b/tests/cloud_tests/testcases/modules/bootcmd.py index 47a51e0a..f5b86b03 100644 --- a/tests/cloud_tests/testcases/modules/bootcmd.py +++ b/tests/cloud_tests/testcases/modules/bootcmd.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestBootCmd(base.CloudTestCase): - """Test bootcmd module""" + """Test bootcmd module.""" def test_bootcmd_host(self): - """Test boot cmd worked""" + """Test boot cmd worked.""" out = self.get_data_file('hosts') self.assertIn('192.168.1.130 us.archive.ubuntu.com', out) diff --git a/tests/cloud_tests/testcases/modules/byobu.py b/tests/cloud_tests/testcases/modules/byobu.py index 204b37b9..005ca014 100644 --- a/tests/cloud_tests/testcases/modules/byobu.py +++ b/tests/cloud_tests/testcases/modules/byobu.py @@ -1,24 +1,24 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestByobu(base.CloudTestCase): - """Test Byobu module""" + """Test Byobu module.""" def test_byobu_installed(self): - """Test byobu installed""" + """Test byobu installed.""" out = self.get_data_file('byobu_installed') self.assertIn('/usr/bin/byobu', out) def test_byobu_profile_enabled(self): - """Test byobu profile.d file exists""" + """Test byobu profile.d file exists.""" out = self.get_data_file('byobu_profile_enabled') self.assertIn('/etc/profile.d/Z97-byobu.sh', out) def test_byobu_launch_exists(self): - """Test byobu-launch exists""" + """Test byobu-launch exists.""" out = self.get_data_file('byobu_launch_exists') self.assertIn('/usr/bin/byobu-launch', out) diff --git a/tests/cloud_tests/testcases/modules/ca_certs.py b/tests/cloud_tests/testcases/modules/ca_certs.py index 7448e480..e75f0413 100644 --- a/tests/cloud_tests/testcases/modules/ca_certs.py +++ b/tests/cloud_tests/testcases/modules/ca_certs.py @@ -1,19 +1,19 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestCaCerts(base.CloudTestCase): - """Test ca certs module""" + """Test ca certs module.""" def test_cert_count(self): - """Test the count is proper""" + """Test the count is proper.""" out = self.get_data_file('cert_count') self.assertEqual(5, int(out)) def test_cert_installed(self): - """Test line from our cert exists""" + """Test line from our cert exists.""" out = self.get_data_file('cert') self.assertIn('a36c744454555024e7f82edc420fd2c8', out) diff --git a/tests/cloud_tests/testcases/modules/debug_disable.py b/tests/cloud_tests/testcases/modules/debug_disable.py index 9899fdfe..e40e4b89 100644 --- a/tests/cloud_tests/testcases/modules/debug_disable.py +++ b/tests/cloud_tests/testcases/modules/debug_disable.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestDebugDisable(base.CloudTestCase): - """Disable debug messages""" + """Disable debug messages.""" def test_debug_disable(self): - """Test verbose output missing from logs""" + """Test verbose output missing from logs.""" out = self.get_data_file('cloud-init.log') self.assertNotIn( out, r'Skipping module named [a-z].* verbose printing disabled') diff --git a/tests/cloud_tests/testcases/modules/debug_enable.py b/tests/cloud_tests/testcases/modules/debug_enable.py index 21c89524..28d26062 100644 --- a/tests/cloud_tests/testcases/modules/debug_enable.py +++ b/tests/cloud_tests/testcases/modules/debug_enable.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestDebugEnable(base.CloudTestCase): - """Test debug messages""" + """Test debug messages.""" def test_debug_enable(self): - """Test debug messages in cloud-init log""" + """Test debug messages in cloud-init log.""" out = self.get_data_file('cloud-init.log') self.assertIn('[DEBUG]', out) diff --git a/tests/cloud_tests/testcases/modules/final_message.py b/tests/cloud_tests/testcases/modules/final_message.py index b06ad01b..b7b5d5e0 100644 --- a/tests/cloud_tests/testcases/modules/final_message.py +++ b/tests/cloud_tests/testcases/modules/final_message.py @@ -1,34 +1,27 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestFinalMessage(base.CloudTestCase): - """ - test cloud init module `cc_final_message` - """ + """Test cloud init module `cc_final_message`.""" + subs_char = '$' def get_final_message_config(self): - """ - get config for final message - """ + """Get config for final message.""" self.assertIn('final_message', self.cloud_config) return self.cloud_config['final_message'] def get_final_message(self): - """ - get final message from log - """ + """Get final message from log.""" out = self.get_data_file('cloud-init-output.log') lines = len(self.get_final_message_config().splitlines()) return '\n'.join(out.splitlines()[-1 * lines:]) def test_final_message_string(self): - """ - ensure final handles regular strings - """ + """Ensure final handles regular strings.""" for actual, config in zip( self.get_final_message().splitlines(), self.get_final_message_config().splitlines()): @@ -36,9 +29,7 @@ class TestFinalMessage(base.CloudTestCase): self.assertEqual(actual, config) def test_final_message_subs(self): - """ - test variable substitution in final message - """ + """Test variable substitution in final message.""" # TODO: add verification of other substitutions patterns = {'$datasource': self.get_datasource()} for key, expected in patterns.items(): diff --git a/tests/cloud_tests/testcases/modules/keys_to_console.py b/tests/cloud_tests/testcases/modules/keys_to_console.py index b36c96cf..88b6812e 100644 --- a/tests/cloud_tests/testcases/modules/keys_to_console.py +++ b/tests/cloud_tests/testcases/modules/keys_to_console.py @@ -1,20 +1,20 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestKeysToConsole(base.CloudTestCase): - """Test proper keys are included and excluded to console""" + """Test proper keys are included and excluded to console.""" def test_excluded_keys(self): - """Test excluded keys missing""" + """Test excluded keys missing.""" out = self.get_data_file('syslog') self.assertNotIn('DSA', out) self.assertNotIn('ECDSA', out) def test_expected_keys(self): - """Test expected keys exist""" + """Test expected keys exist.""" out = self.get_data_file('syslog') self.assertIn('ED25519', out) self.assertIn('RSA', out) diff --git a/tests/cloud_tests/testcases/modules/locale.py b/tests/cloud_tests/testcases/modules/locale.py index bf4e1b07..63e53ff3 100644 --- a/tests/cloud_tests/testcases/modules/locale.py +++ b/tests/cloud_tests/testcases/modules/locale.py @@ -1,19 +1,19 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestLocale(base.CloudTestCase): - """Test locale is set properly""" + """Test locale is set properly.""" def test_locale(self): - """Test locale is set properly""" + """Test locale is set properly.""" out = self.get_data_file('locale_default') self.assertIn('LANG="en_GB.UTF-8"', out) def test_locale_a(self): - """Test locale -a has both options""" + """Test locale -a has both options.""" out = self.get_data_file('locale_a') self.assertIn('en_GB.utf8', out) self.assertIn('en_US.utf8', out) diff --git a/tests/cloud_tests/testcases/modules/lxd_bridge.py b/tests/cloud_tests/testcases/modules/lxd_bridge.py index 4087e2f2..c0262ba3 100644 --- a/tests/cloud_tests/testcases/modules/lxd_bridge.py +++ b/tests/cloud_tests/testcases/modules/lxd_bridge.py @@ -1,24 +1,24 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestLxdBridge(base.CloudTestCase): - """Test LXD module""" + """Test LXD module.""" def test_lxd(self): - """Test lxd installed""" + """Test lxd installed.""" out = self.get_data_file('lxd') self.assertIn('/usr/bin/lxd', out) def test_lxc(self): - """Test lxc installed""" + """Test lxc installed.""" out = self.get_data_file('lxc') self.assertIn('/usr/bin/lxc', out) def test_bridge(self): - """Test bridge config""" + """Test bridge config.""" out = self.get_data_file('lxc-bridge') self.assertIn('lxdbr0', out) self.assertIn('10.100.100.1/24', out) diff --git a/tests/cloud_tests/testcases/modules/lxd_dir.py b/tests/cloud_tests/testcases/modules/lxd_dir.py index 51a9a1f1..1495674e 100644 --- a/tests/cloud_tests/testcases/modules/lxd_dir.py +++ b/tests/cloud_tests/testcases/modules/lxd_dir.py @@ -1,19 +1,19 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestLxdDir(base.CloudTestCase): - """Test LXD module""" + """Test LXD module.""" def test_lxd(self): - """Test lxd installed""" + """Test lxd installed.""" out = self.get_data_file('lxd') self.assertIn('/usr/bin/lxd', out) def test_lxc(self): - """Test lxc installed""" + """Test lxc installed.""" out = self.get_data_file('lxc') self.assertIn('/usr/bin/lxc', out) diff --git a/tests/cloud_tests/testcases/modules/ntp.py b/tests/cloud_tests/testcases/modules/ntp.py index 82d32880..a4b8c3d8 100644 --- a/tests/cloud_tests/testcases/modules/ntp.py +++ b/tests/cloud_tests/testcases/modules/ntp.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base diff --git a/tests/cloud_tests/testcases/modules/ntp_pools.py b/tests/cloud_tests/testcases/modules/ntp_pools.py index ff6d8fa4..336076df 100644 --- a/tests/cloud_tests/testcases/modules/ntp_pools.py +++ b/tests/cloud_tests/testcases/modules/ntp_pools.py @@ -1,11 +1,11 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestNtpPools(base.CloudTestCase): - """Test ntp module""" + """Test ntp module.""" def test_ntp_installed(self): """Test ntp installed""" diff --git a/tests/cloud_tests/testcases/modules/package_update_upgrade_install.py b/tests/cloud_tests/testcases/modules/package_update_upgrade_install.py index 00353ead..a92dec22 100644 --- a/tests/cloud_tests/testcases/modules/package_update_upgrade_install.py +++ b/tests/cloud_tests/testcases/modules/package_update_upgrade_install.py @@ -1,24 +1,24 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestPackageInstallUpdateUpgrade(base.CloudTestCase): - """Test package install update upgrade module""" + """Test package install update upgrade module.""" def test_installed_htop(self): - """Test htop got installed""" + """Test htop got installed.""" out = self.get_data_file('dpkg_htop') self.assertEqual(1, int(out)) def test_installed_tree(self): - """Test tree got installed""" + """Test tree got installed.""" out = self.get_data_file('dpkg_tree') self.assertEqual(1, int(out)) def test_apt_history(self): - """Test apt history for update command""" + """Test apt history for update command.""" out = self.get_data_file('apt_history_cmdline') self.assertIn( 'Commandline: /usr/bin/apt-get --option=Dpkg::Options' @@ -26,7 +26,7 @@ class TestPackageInstallUpdateUpgrade(base.CloudTestCase): '--assume-yes --quiet install htop tree', out) def test_cloud_init_output(self): - """Test cloud-init-output for install & upgrade stuff""" + """Test cloud-init-output for install & upgrade stuff.""" out = self.get_data_file('cloud-init-output.log') self.assertIn('Setting up tree (', out) self.assertIn('Setting up htop (', out) diff --git a/tests/cloud_tests/testcases/modules/runcmd.py b/tests/cloud_tests/testcases/modules/runcmd.py index 780cd186..9fce3062 100644 --- a/tests/cloud_tests/testcases/modules/runcmd.py +++ b/tests/cloud_tests/testcases/modules/runcmd.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestRunCmd(base.CloudTestCase): - """Test runcmd module""" + """Test runcmd module.""" def test_run_cmd(self): - """Test run command worked""" + """Test run command worked.""" out = self.get_data_file('run_cmd') self.assertIn('cloud-init run cmd test', out) diff --git a/tests/cloud_tests/testcases/modules/salt_minion.py b/tests/cloud_tests/testcases/modules/salt_minion.py index 3ef30f7e..c697db2d 100644 --- a/tests/cloud_tests/testcases/modules/salt_minion.py +++ b/tests/cloud_tests/testcases/modules/salt_minion.py @@ -1,26 +1,26 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class Test(base.CloudTestCase): - """Test salt minion module""" + """Test salt minion module.""" def test_minon_master(self): - """Test master value in config""" + """Test master value in config.""" out = self.get_data_file('minion') self.assertIn('master: salt.mydomain.com', out) def test_minion_pem(self): - """Test private key""" + """Test private key.""" out = self.get_data_file('minion.pem') self.assertIn('------BEGIN PRIVATE KEY------', out) self.assertIn('', out) self.assertIn('------END PRIVATE KEY-------', out) def test_minion_pub(self): - """Test public key""" + """Test public key.""" out = self.get_data_file('minion.pub') self.assertIn('------BEGIN PUBLIC KEY-------', out) self.assertIn('', out) diff --git a/tests/cloud_tests/testcases/modules/seed_random_data.py b/tests/cloud_tests/testcases/modules/seed_random_data.py index b2121569..db433d26 100644 --- a/tests/cloud_tests/testcases/modules/seed_random_data.py +++ b/tests/cloud_tests/testcases/modules/seed_random_data.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestSeedRandom(base.CloudTestCase): - """Test seed random module""" + """Test seed random module.""" def test_random_seed_data(self): - """Test random data passed in exists""" + """Test random data passed in exists.""" out = self.get_data_file('seed_data') self.assertIn('MYUb34023nD:LFDK10913jk;dfnk:Df', out) diff --git a/tests/cloud_tests/testcases/modules/set_hostname.py b/tests/cloud_tests/testcases/modules/set_hostname.py index 9501b069..6e96a75c 100644 --- a/tests/cloud_tests/testcases/modules/set_hostname.py +++ b/tests/cloud_tests/testcases/modules/set_hostname.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestHostname(base.CloudTestCase): - """Test hostname module""" + """Test hostname module.""" def test_hostname(self): - """Test hostname command shows correct output""" + """Test hostname command shows correct output.""" out = self.get_data_file('hostname') self.assertIn('myhostname', out) diff --git a/tests/cloud_tests/testcases/modules/set_hostname_fqdn.py b/tests/cloud_tests/testcases/modules/set_hostname_fqdn.py index d89c299d..398f3d40 100644 --- a/tests/cloud_tests/testcases/modules/set_hostname_fqdn.py +++ b/tests/cloud_tests/testcases/modules/set_hostname_fqdn.py @@ -1,24 +1,24 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestHostnameFqdn(base.CloudTestCase): - """Test Hostname module""" + """Test Hostname module.""" def test_hostname(self): - """Test hostname output""" + """Test hostname output.""" out = self.get_data_file('hostname') self.assertIn('myhostname', out) def test_hostname_fqdn(self): - """Test hostname fqdn output""" + """Test hostname fqdn output.""" out = self.get_data_file('fqdn') self.assertIn('host.myorg.com', out) def test_hosts(self): - """Test /etc/hosts file""" + """Test /etc/hosts file.""" out = self.get_data_file('hosts') self.assertIn('127.0.1.1 host.myorg.com myhostname', out) self.assertIn('127.0.0.1 localhost', out) diff --git a/tests/cloud_tests/testcases/modules/set_password.py b/tests/cloud_tests/testcases/modules/set_password.py index 1411a296..a29b2261 100644 --- a/tests/cloud_tests/testcases/modules/set_password.py +++ b/tests/cloud_tests/testcases/modules/set_password.py @@ -1,21 +1,21 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestPassword(base.CloudTestCase): - """Test password module""" + """Test password module.""" # TODO add test to make sure password is actually "password" def test_shadow(self): - """Test ubuntu user in shadow""" + """Test ubuntu user in shadow.""" out = self.get_data_file('shadow') self.assertIn('ubuntu:', out) def test_sshd_config(self): - """Test sshd config allows passwords""" + """Test sshd config allows passwords.""" out = self.get_data_file('sshd_config') self.assertIn('PasswordAuthentication yes', out) diff --git a/tests/cloud_tests/testcases/modules/set_password_expire.py b/tests/cloud_tests/testcases/modules/set_password_expire.py index 1ac9c23f..a1c3aa08 100644 --- a/tests/cloud_tests/testcases/modules/set_password_expire.py +++ b/tests/cloud_tests/testcases/modules/set_password_expire.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestPasswordExpire(base.CloudTestCase): - """Test password module""" + """Test password module.""" def test_shadow(self): - """Test user frozen in shadow""" + """Test user frozen in shadow.""" out = self.get_data_file('shadow') self.assertIn('harry:!:', out) self.assertIn('dick:!:', out) @@ -16,7 +16,7 @@ class TestPasswordExpire(base.CloudTestCase): self.assertIn('harry:!:', out) def test_sshd_config(self): - """Test sshd config allows passwords""" + """Test sshd config allows passwords.""" out = self.get_data_file('sshd_config') self.assertIn('PasswordAuthentication no', out) diff --git a/tests/cloud_tests/testcases/modules/set_password_list.py b/tests/cloud_tests/testcases/modules/set_password_list.py index 6819d259..375cd27d 100644 --- a/tests/cloud_tests/testcases/modules/set_password_list.py +++ b/tests/cloud_tests/testcases/modules/set_password_list.py @@ -1,11 +1,12 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestPasswordList(base.PasswordListTest, base.CloudTestCase): - """Test password setting via list in chpasswd/list""" + """Test password setting via list in chpasswd/list.""" + __test__ = True # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/set_password_list_string.py b/tests/cloud_tests/testcases/modules/set_password_list_string.py index 2c34fada..8c2634c5 100644 --- a/tests/cloud_tests/testcases/modules/set_password_list_string.py +++ b/tests/cloud_tests/testcases/modules/set_password_list_string.py @@ -1,11 +1,12 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestPasswordListString(base.PasswordListTest, base.CloudTestCase): - """Test password setting via string in chpasswd/list""" + """Test password setting via string in chpasswd/list.""" + __test__ = True # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.py b/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.py index a0f8896b..82223217 100644 --- a/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.py +++ b/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.py @@ -1,24 +1,24 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestSshKeyFingerprintsDisable(base.CloudTestCase): - """Test ssh key fingerprints module""" + """Test ssh key fingerprints module.""" def test_cloud_init_log(self): - """Verify disabled""" + """Verify disabled.""" out = self.get_data_file('cloud-init.log') self.assertIn('Skipping module named ssh-authkey-fingerprints, ' 'logging of ssh fingerprints disabled', out) def test_syslog(self): - """Verify output of syslog""" + """Verify output of syslog.""" out = self.get_data_file('syslog') - self.assertNotRegexpMatches(out, r'256 SHA256:.*(ECDSA)') - self.assertNotRegexpMatches(out, r'256 SHA256:.*(ED25519)') - self.assertNotRegexpMatches(out, r'1024 SHA256:.*(DSA)') - self.assertNotRegexpMatches(out, r'2048 SHA256:.*(RSA)') + self.assertNotRegex(out, r'256 SHA256:.*(ECDSA)') + self.assertNotRegex(out, r'256 SHA256:.*(ED25519)') + self.assertNotRegex(out, r'1024 SHA256:.*(DSA)') + self.assertNotRegex(out, r'2048 SHA256:.*(RSA)') # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_enable.py b/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_enable.py index 3c44b0cc..3510e75a 100644 --- a/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_enable.py +++ b/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_enable.py @@ -1,18 +1,18 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestSshKeyFingerprintsEnable(base.CloudTestCase): - """Test ssh key fingerprints module""" + """Test ssh key fingerprints module.""" def test_syslog(self): - """Verify output of syslog""" + """Verify output of syslog.""" out = self.get_data_file('syslog') - self.assertRegexpMatches(out, r'256 SHA256:.*(ECDSA)') - self.assertRegexpMatches(out, r'256 SHA256:.*(ED25519)') - self.assertNotRegexpMatches(out, r'1024 SHA256:.*(DSA)') - self.assertNotRegexpMatches(out, r'2048 SHA256:.*(RSA)') + self.assertRegex(out, r'256 SHA256:.*(ECDSA)') + self.assertRegex(out, r'256 SHA256:.*(ED25519)') + self.assertNotRegex(out, r'1024 SHA256:.*(DSA)') + self.assertNotRegex(out, r'2048 SHA256:.*(RSA)') # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ssh_import_id.py b/tests/cloud_tests/testcases/modules/ssh_import_id.py index 214e710d..055c6a29 100644 --- a/tests/cloud_tests/testcases/modules/ssh_import_id.py +++ b/tests/cloud_tests/testcases/modules/ssh_import_id.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestSshImportId(base.CloudTestCase): - """Test ssh import id module""" + """Test ssh import id module.""" def test_authorized_keys(self): - """Test that ssh keys were imported""" + """Test that ssh keys were imported.""" out = self.get_data_file('auth_keys_ubuntu') # Rather than checking the key fingerprints, you could just check diff --git a/tests/cloud_tests/testcases/modules/ssh_keys_generate.py b/tests/cloud_tests/testcases/modules/ssh_keys_generate.py index 161ace5f..fd6d9ba5 100644 --- a/tests/cloud_tests/testcases/modules/ssh_keys_generate.py +++ b/tests/cloud_tests/testcases/modules/ssh_keys_generate.py @@ -1,56 +1,56 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestSshKeysGenerate(base.CloudTestCase): - """Test ssh keys module""" + """Test ssh keys module.""" # TODO: Check cloud-init-output for the correct keys being generated def test_ubuntu_authorized_keys(self): - """Test passed in key is not in list for ubuntu""" + """Test passed in key is not in list for ubuntu.""" out = self.get_data_file('auth_keys_ubuntu') self.assertEqual('', out) def test_dsa_public(self): - """Test dsa public key not generated""" + """Test dsa public key not generated.""" out = self.get_data_file('dsa_public') self.assertEqual('', out) def test_dsa_private(self): - """Test dsa private key not generated""" + """Test dsa private key not generated.""" out = self.get_data_file('dsa_private') self.assertEqual('', out) def test_rsa_public(self): - """Test rsa public key not generated""" + """Test rsa public key not generated.""" out = self.get_data_file('rsa_public') self.assertEqual('', out) def test_rsa_private(self): - """Test rsa public key not generated""" + """Test rsa public key not generated.""" out = self.get_data_file('rsa_private') self.assertEqual('', out) def test_ecdsa_public(self): - """Test ecdsa public key generated""" + """Test ecdsa public key generated.""" out = self.get_data_file('ecdsa_public') self.assertIsNotNone(out) def test_ecdsa_private(self): - """Test ecdsa public key generated""" + """Test ecdsa public key generated.""" out = self.get_data_file('ecdsa_private') self.assertIsNotNone(out) def test_ed25519_public(self): - """Test ed25519 public key generated""" + """Test ed25519 public key generated.""" out = self.get_data_file('ed25519_public') self.assertIsNotNone(out) def test_ed25519_private(self): - """Test ed25519 public key generated""" + """Test ed25519 public key generated.""" out = self.get_data_file('ed25519_private') self.assertIsNotNone(out) diff --git a/tests/cloud_tests/testcases/modules/ssh_keys_provided.py b/tests/cloud_tests/testcases/modules/ssh_keys_provided.py index 8f18cb94..544649da 100644 --- a/tests/cloud_tests/testcases/modules/ssh_keys_provided.py +++ b/tests/cloud_tests/testcases/modules/ssh_keys_provided.py @@ -1,67 +1,67 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestSshKeysProvided(base.CloudTestCase): - """Test ssh keys module""" + """Test ssh keys module.""" def test_ubuntu_authorized_keys(self): - """Test passed in key is not in list for ubuntu""" + """Test passed in key is not in list for ubuntu.""" out = self.get_data_file('auth_keys_ubuntu') self.assertEqual('', out) def test_root_authorized_keys(self): - """Test passed in key is in authorized list for root""" + """Test passed in key is in authorized list for root.""" out = self.get_data_file('auth_keys_root') self.assertIn('lzrkPqONphoZx0LDV86w7RUz1ksDzAdcm0tvmNRFMN1a0frDs50' '6oA3aWK0oDk4Nmvk8sXGTYYw3iQSkOvDUUlIsqdaO+w==', out) def test_dsa_public(self): - """Test dsa public key passed in""" + """Test dsa public key passed in.""" out = self.get_data_file('dsa_public') self.assertIn('AAAAB3NzaC1kc3MAAACBAPkWy1zbchVIN7qTgM0/yyY8q4RZS8c' 'NM4ZpeuE5UB/Nnr6OSU/nmbO8LuM', out) def test_dsa_private(self): - """Test dsa private key passed in""" + """Test dsa private key passed in.""" out = self.get_data_file('dsa_private') self.assertIn('MIIBuwIBAAKBgQD5Fstc23IVSDe6k4DNP8smPKuEWUvHDTOGaXr' 'hOVAfzZ6+jklP', out) def test_rsa_public(self): - """Test rsa public key passed in""" + """Test rsa public key passed in.""" out = self.get_data_file('rsa_public') self.assertIn('AAAAB3NzaC1yc2EAAAADAQABAAABAQC0/Ho+o3eJISydO2JvIgT' 'LnZOtrxPl+fSvJfKDjoOLY0HB2eOjy2s2/2N6d9X9SGZ4', out) def test_rsa_private(self): - """Test rsa public key passed in""" + """Test rsa public key passed in.""" out = self.get_data_file('rsa_private') self.assertIn('4DOkqNiUGl80Zp1RgZNohHUXlJMtAbrIlAVEk+mTmg7vjfyp2un' 'RQvLZpMRdywBm', out) def test_ecdsa_public(self): - """Test ecdsa public key passed in""" + """Test ecdsa public key passed in.""" out = self.get_data_file('ecdsa_public') self.assertIn('AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAAB' 'BBFsS5Tvky/IC/dXhE/afxxU', out) def test_ecdsa_private(self): - """Test ecdsa public key passed in""" + """Test ecdsa public key passed in.""" out = self.get_data_file('ecdsa_private') self.assertIn('AwEHoUQDQgAEWxLlO+TL8gL91eET9p/HFQbqR1A691AkJgZk3jY' '5mpZqxgX4vcgb', out) def test_ed25519_public(self): - """Test ed25519 public key passed in""" + """Test ed25519 public key passed in.""" out = self.get_data_file('ed25519_public') self.assertIn('AAAAC3NzaC1lZDI1NTE5AAAAINudAZSu4vjZpVWzId5pXmZg1M6' 'G15dqjQ2XkNVOEnb5', out) def test_ed25519_private(self): - """Test ed25519 public key passed in""" + """Test ed25519 public key passed in.""" out = self.get_data_file('ed25519_private') self.assertIn('XAAAAAtzc2gtZWQyNTUxOQAAACDbnQGUruL42aVVsyHeaV5mYNT' 'OhteXao0Nl5DVThJ2+Q', out) diff --git a/tests/cloud_tests/testcases/modules/timezone.py b/tests/cloud_tests/testcases/modules/timezone.py index bf91d490..654fa53d 100644 --- a/tests/cloud_tests/testcases/modules/timezone.py +++ b/tests/cloud_tests/testcases/modules/timezone.py @@ -1,14 +1,14 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestTimezone(base.CloudTestCase): - """Test timezone module""" + """Test timezone module.""" def test_timezone(self): - """Test date prints correct timezone""" + """Test date prints correct timezone.""" out = self.get_data_file('timezone') self.assertEqual('HDT', out.rstrip()) diff --git a/tests/cloud_tests/testcases/modules/user_groups.py b/tests/cloud_tests/testcases/modules/user_groups.py index e5732322..67af527b 100644 --- a/tests/cloud_tests/testcases/modules/user_groups.py +++ b/tests/cloud_tests/testcases/modules/user_groups.py @@ -1,42 +1,42 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestUserGroups(base.CloudTestCase): - """Example cloud-config test""" + """Example cloud-config test.""" def test_group_ubuntu(self): - """Test ubuntu group exists""" + """Test ubuntu group exists.""" out = self.get_data_file('group_ubuntu') self.assertRegex(out, r'ubuntu:x:[0-9]{4}:') def test_group_cloud_users(self): - """Test cloud users group exists""" + """Test cloud users group exists.""" out = self.get_data_file('group_cloud_users') self.assertRegex(out, r'cloud-users:x:[0-9]{4}:barfoo') def test_user_ubuntu(self): - """Test ubuntu user exists""" + """Test ubuntu user exists.""" out = self.get_data_file('user_ubuntu') self.assertRegex( out, r'ubuntu:x:[0-9]{4}:[0-9]{4}:Ubuntu:/home/ubuntu:/bin/bash') def test_user_foobar(self): - """Test foobar user exists""" + """Test foobar user exists.""" out = self.get_data_file('user_foobar') self.assertRegex( out, r'foobar:x:[0-9]{4}:[0-9]{4}:Foo B. Bar:/home/foobar:') def test_user_barfoo(self): - """Test barfoo user exists""" + """Test barfoo user exists.""" out = self.get_data_file('user_barfoo') self.assertRegex( out, r'barfoo:x:[0-9]{4}:[0-9]{4}:Bar B. Foo:/home/barfoo:') def test_user_cloudy(self): - """Test cloudy user exists""" + """Test cloudy user exists.""" out = self.get_data_file('user_cloudy') self.assertRegex(out, r'cloudy:x:[0-9]{3,4}:') diff --git a/tests/cloud_tests/testcases/modules/write_files.py b/tests/cloud_tests/testcases/modules/write_files.py index 97dfeec3..7bd520f6 100644 --- a/tests/cloud_tests/testcases/modules/write_files.py +++ b/tests/cloud_tests/testcases/modules/write_files.py @@ -1,29 +1,29 @@ # This file is part of cloud-init. See LICENSE file for license information. -"""cloud-init Integration Test Verify Script""" +"""cloud-init Integration Test Verify Script.""" from tests.cloud_tests.testcases import base class TestWriteFiles(base.CloudTestCase): - """Example cloud-config test""" + """Example cloud-config test.""" def test_b64(self): - """Test b64 encoded file reads as ascii""" + """Test b64 encoded file reads as ascii.""" out = self.get_data_file('file_b64') self.assertIn('ASCII text', out) def test_binary(self): - """Test binary file reads as executable""" + """Test binary file reads as executable.""" out = self.get_data_file('file_binary') self.assertIn('ELF 64-bit LSB executable, x86-64, version 1', out) def test_gzip(self): - """Test gzip file shows up as a shell script""" + """Test gzip file shows up as a shell script.""" out = self.get_data_file('file_gzip') self.assertIn('POSIX shell script, ASCII text executable', out) def test_text(self): - """Test text shows up as ASCII text""" + """Test text shows up as ASCII text.""" out = self.get_data_file('file_text') self.assertIn('ASCII text', out) diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py index 64a86672..2bbe21c7 100644 --- a/tests/cloud_tests/util.py +++ b/tests/cloud_tests/util.py @@ -1,28 +1,43 @@ # This file is part of cloud-init. See LICENSE file for license information. +"""Utilities for re-use across integration tests.""" + +import copy import glob import os import random +import shutil import string import tempfile import yaml -from cloudinit.distros import OSFAMILIES from cloudinit import util as c_util from tests.cloud_tests import LOG +OS_FAMILY_MAPPING = { + 'debian': ['debian', 'ubuntu'], + 'redhat': ['centos', 'rhel', 'fedora'], + 'gentoo': ['gentoo'], + 'freebsd': ['freebsd'], + 'suse': ['sles'], + 'arch': ['arch'], +} + def list_test_data(data_dir): - """ - find all tests with test data available in data_dir - data_dir should contain /// - return_value: {: {: []}} + """Find all tests with test data available in data_dir. + + @param data_dir: should contain /// + @return_value: {: {: []}} """ if not os.path.isdir(data_dir): raise ValueError("bad data dir") res = {} for platform in os.listdir(data_dir): + if not os.path.isdir(os.path.join(data_dir, platform)): + continue + res[platform] = {} for os_name in os.listdir(os.path.join(data_dir, platform)): res[platform][os_name] = [ @@ -36,39 +51,33 @@ def list_test_data(data_dir): def gen_instance_name(prefix='cloud-test', image_desc=None, use_desc=None, max_len=63, delim='-', max_tries=16, used_list=None, valid=string.ascii_lowercase + string.digits): - """ - generate an unique name for a test instance - prefix: name prefix, defaults to cloud-test, default should be left - image_desc: short string with image desc, will be truncated to 16 chars - use_desc: short string with usage desc, will be truncated to 30 chars - max_len: maximum name length, defaults to 64 chars - delim: delimiter to use between tokens - max_tries: maximum tries to find a unique name before giving up - used_list: already used names, or none to not check - valid: string of valid characters for name - return_value: valid, unused name, may raise StopIteration + """Generate an unique name for a test instance. + + @param prefix: name prefix, defaults to cloud-test, default should be left + @param image_desc: short string (len <= 16) with image desc + @param use_desc: short string (len <= 30) with usage desc + @param max_len: maximum name length, defaults to 64 chars + @param delim: delimiter to use between tokens + @param max_tries: maximum tries to find a unique name before giving up + @param used_list: already used names, or none to not check + @param valid: string of valid characters for name + @return_value: valid, unused name, may raise StopIteration """ unknown = 'unknown' def join(*args): - """ - join args with delim - """ + """Join args with delim.""" return delim.join(args) def fill(*args): - """ - join name elems and fill rest with random data - """ + """Join name elems and fill rest with random data.""" name = join(*args) num = max_len - len(name) - len(delim) return join(name, ''.join(random.choice(valid) for _ in range(num))) def clean(elem, max_len): - """ - filter bad characters out of elem and trim to length - """ - elem = elem[:max_len] if elem else unknown + """Filter bad characters out of elem and trim to length.""" + elem = elem.lower()[:max_len] if elem else unknown return ''.join(c if c in valid else delim for c in elem) return next(name for name in @@ -78,30 +87,39 @@ def gen_instance_name(prefix='cloud-test', image_desc=None, use_desc=None, def sorted_unique(iterable, key=None, reverse=False): - """ - return_value: a sorted list of unique items in iterable + """Create unique sorted list. + + @param iterable: the data structure to sort + @param key: if you have a specific key + @param reverse: to reverse or not + @return_value: a sorted list of unique items in iterable """ return sorted(set(iterable), key=key, reverse=reverse) def get_os_family(os_name): + """Get os family type for os_name. + + @param os_name: name of os + @return_value: family name for os_name """ - get os family type for os_name - """ - return next((k for k, v in OSFAMILIES.items() if os_name in v), None) + return next((k for k, v in OS_FAMILY_MAPPING.items() + if os_name.lower() in v), None) def current_verbosity(): - """ - get verbosity currently in effect from log level - return_value: verbosity, 0-2, 2 = verbose, 0 = quiet + """Get verbosity currently in effect from log level. + + @return_value: verbosity, 0-2, 2=verbose, 0=quiet """ return max(min(3 - int(LOG.level / 10), 2), 0) def is_writable_dir(path): - """ - make sure dir is writable + """Make sure dir is writable. + + @param path: path to determine if writable + @return_value: boolean with result """ try: c_util.ensure_dir(path) @@ -112,9 +130,10 @@ def is_writable_dir(path): def is_clean_writable_dir(path): - """ - make sure dir is empty and writable, creating it if it does not exist - return_value: True/False if successful + """Make sure dir is empty and writable, creating it if it does not exist. + + @param path: path to check + @return_value: True/False if successful """ path = os.path.abspath(path) if not (is_writable_dir(path) and len(os.listdir(path)) == 0): @@ -123,29 +142,31 @@ def is_clean_writable_dir(path): def configure_yaml(): + """Clean yaml.""" yaml.add_representer(str, (lambda dumper, data: dumper.represent_scalar( 'tag:yaml.org,2002:str', data, style='|' if '\n' in data else ''))) -def yaml_format(data): - """ - format data as yaml +def yaml_format(data, content_type=None): + """Format data as yaml. + + @param data: data to dump + @param header: if specified, add a header to the dumped data + @return_value: yaml string """ configure_yaml() - return yaml.dump(data, indent=2, default_flow_style=False) + content_type = ( + '#{}\n'.format(content_type.strip('#\n')) if content_type else '') + return content_type + yaml.dump(data, indent=2, default_flow_style=False) def yaml_dump(data, path): - """ - dump data to path in yaml format - """ - write_file(os.path.abspath(path), yaml_format(data), omode='w') + """Dump data to path in yaml format.""" + c_util.write_file(os.path.abspath(path), yaml_format(data), omode='w') def merge_results(data, path): - """ - handle merging results from collect phase and verify phase - """ + """Handle merging results from collect phase and verify phase.""" current = {} if os.path.exists(path): with open(path, 'r') as fp: @@ -154,10 +175,118 @@ def merge_results(data, path): yaml_dump(current, path) -def write_file(*args, **kwargs): +def rel_files(basedir): + """List of files under directory by relative path, not including dirs. + + @param basedir: directory to search + @return_value: list or relative paths + """ + basedir = os.path.normpath(basedir) + return [path[len(basedir) + 1:] for path in + glob.glob(os.path.join(basedir, '**'), recursive=True) + if not os.path.isdir(path)] + + +def flat_tar(output, basedir, owner='root', group='root'): + """Create a flat tar archive (no leading ./) from basedir. + + @param output: output tar file to write + @param basedir: base directory for archive + @param owner: owner of archive files + @param group: group archive files belong to + @return_value: none + """ + c_util.subp(['tar', 'cf', output, '--owner', owner, '--group', group, + '-C', basedir] + rel_files(basedir), capture=True) + + +def parse_conf_list(entries, valid=None, boolean=False): + """Parse config in a list of strings in key=value format. + + @param entries: list of key=value strings + @param valid: list of valid keys in result, return None if invalid input + @param boolean: if true, then interpret all values as booleans + @return_value: dict of configuration or None if invalid """ - write a file using cloudinit.util.write_file + res = {key: value.lower() == 'true' if boolean else value + for key, value in (i.split('=') for i in entries)} + return res if not valid or all(k in valid for k in res.keys()) else None + + +def update_args(args, updates, preserve_old=True): + """Update cmdline arguments from a dictionary. + + @param args: cmdline arguments + @param updates: dictionary of {arg_name: new_value} mappings + @param preserve_old: if true, create a deep copy of args before updating + @return_value: updated cmdline arguments + """ + args = copy.deepcopy(args) if preserve_old else args + if updates: + vars(args).update(updates) + return args + + +def update_user_data(user_data, updates, dump_to_yaml=True): + """Update user_data from dictionary. + + @param user_data: user data as yaml string or dict + @param updates: dictionary to merge with user data + @param dump_to_yaml: return as yaml dumped string if true + @return_value: updated user data, as yaml string if dump_to_yaml is true """ - c_util.write_file(*args, **kwargs) + user_data = (c_util.load_yaml(user_data) + if isinstance(user_data, str) else copy.deepcopy(user_data)) + user_data.update(updates) + return (yaml_format(user_data, content_type='cloud-config') + if dump_to_yaml else user_data) + + +class InTargetExecuteError(c_util.ProcessExecutionError): + """Error type for in target commands that fail.""" + + default_desc = 'Unexpected error while running command in target instance' + + def __init__(self, stdout, stderr, exit_code, cmd, instance, + description=None): + """Init error and parent error class.""" + if isinstance(cmd, (tuple, list)): + cmd = ' '.join(cmd) + super(InTargetExecuteError, self).__init__( + stdout=stdout, stderr=stderr, exit_code=exit_code, cmd=cmd, + reason="Instance: {}".format(instance), + description=description if description else self.default_desc) + + +class TempDir(object): + """Configurable temporary directory like tempfile.TemporaryDirectory.""" + + def __init__(self, tmpdir=None, preserve=False, prefix='cloud_test_data_'): + """Initialize. + + @param tmpdir: directory to use as tempdir + @param preserve: if true, always preserve data on exit + @param prefix: prefix to use for tempfile name + """ + self.tmpdir = tmpdir + self.preserve = preserve + self.prefix = prefix + + def __enter__(self): + """Create tempdir. + + @return_value: tempdir path + """ + if not self.tmpdir: + self.tmpdir = tempfile.mkdtemp(prefix=self.prefix) + LOG.debug('using tmpdir: %s', self.tmpdir) + return self.tmpdir + + def __exit__(self, etype, value, trace): + """Destroy tempdir if no errors occurred.""" + if etype or self.preserve: + LOG.info('leaving data in %s', self.tmpdir) + else: + shutil.rmtree(self.tmpdir) # vi: ts=4 expandtab diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py index 2a63550e..fc1efcfc 100644 --- a/tests/cloud_tests/verify.py +++ b/tests/cloud_tests/verify.py @@ -1,18 +1,19 @@ # This file is part of cloud-init. See LICENSE file for license information. -from tests.cloud_tests import (config, LOG, util, testcases) +"""Verify test results.""" import os import unittest +from tests.cloud_tests import (config, LOG, util, testcases) + def verify_data(base_dir, tests): - """ - verify test data is correct, - base_dir: base directory for data - test_config: dict of all test config, from util.load_test_config() - tests: list of test names - return_value: {: {passed: True/False, failures: []}} + """Verify test data is correct. + + @param base_dir: base directory for data + @param tests: list of test names + @return_value: {: {passed: True/False, failures: []}} """ runner = unittest.TextTestRunner(verbosity=util.current_verbosity()) res = {} @@ -53,9 +54,10 @@ def verify_data(base_dir, tests): def verify(args): - """ - verify test data - return_value: 0 for success, or number of failed tests + """Verify test data. + + @param args: directory of test data + @return_value: 0 for success, or number of failed tests """ failed = 0 res = {} diff --git a/tox.ini b/tox.ini index 6276662d..89fd3ad4 100644 --- a/tox.ini +++ b/tox.ini @@ -105,4 +105,4 @@ basepython = python3 commands = {envpython} -m tests.cloud_tests {posargs} passenv = HOME deps = - pylxd==2.1.3 + pylxd==2.2.3 -- cgit v1.2.3 From 1025e492412431e95b7ddde46805514e42469db0 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 12 Jun 2017 10:39:20 -0600 Subject: docs: Automatically generate module docs form schema if present. We have started adding jsonschema definitions for cloudconfig modules (cc_ntp). This branch allows us render sphinx docs using the module's shema definition instead of using the module's docstring. This allows us to avoid duplicating schema documentation in the module-level docstring and schema definition. The corresponding module documentation is extended a bit to differentiate between config schema and potential examples. --- doc/rtd/conf.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'doc') diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py index 66b3b654..0ea3b6bf 100644 --- a/doc/rtd/conf.py +++ b/doc/rtd/conf.py @@ -10,6 +10,7 @@ sys.path.insert(0, os.path.abspath('./')) sys.path.insert(0, os.path.abspath('.')) from cloudinit import version +from cloudinit.config.schema import get_schema_doc # Supress warnings for docs that aren't used yet # unused_docs = [ @@ -75,3 +76,12 @@ html_theme_options = { # The name of an image file (relative to this directory) to place at the top # of the sidebar. html_logo = 'static/logo.png' + +def generate_docstring_from_schema(app, what, name, obj, options, lines): + """Override module docs from schema when present.""" + if what == 'module' and hasattr(obj, "schema"): + del lines[:] + lines.extend(get_schema_doc(obj.schema).split('\n')) + +def setup(app): + app.connect('autodoc-process-docstring', generate_docstring_from_schema) -- cgit v1.2.3 From a703c6a8b67bb56cb76bec65a67e62d344923ca8 Mon Sep 17 00:00:00 2001 From: Brian Candler Date: Sat, 8 Jul 2017 16:31:08 +0100 Subject: doc: document the cmdline options to NoCloud Add permitted keys to documentation on seeding NoCloud. --- doc/rtd/topics/datasources/nocloud.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'doc') diff --git a/doc/rtd/topics/datasources/nocloud.rst b/doc/rtd/topics/datasources/nocloud.rst index 665057f3..08578e86 100644 --- a/doc/rtd/topics/datasources/nocloud.rst +++ b/doc/rtd/topics/datasources/nocloud.rst @@ -24,6 +24,16 @@ or ds=nocloud-net[;key=val;key=val] +The permitted keys are: + +- ``h`` or ``local-hostname`` +- ``i`` or ``instance-id`` +- ``s`` or ``seedfrom`` + +With ``ds=nocloud``, the ``seedfrom`` value must start with ``/`` or +``file://``. With ``ds=nocloud-net``, the ``seedfrom`` value must start +with ``http://``, ``https://`` or ``ftp://`` + e.g. you can pass this option to QEMU: :: -- cgit v1.2.3 From fc89390408991021ea90885cc145afd7c1ed6069 Mon Sep 17 00:00:00 2001 From: Sandor Zeestraten Date: Wed, 12 Jul 2017 11:24:01 +0200 Subject: doc: fix disk setup example table_type options This fixes the disk setup example doc which specifies that the only currently supported table_type option is 'mbr' by adding the 'gpt' option which got supported as of 0.7.7. LP: #1703789 --- doc/examples/cloud-config-disk-setup.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'doc') diff --git a/doc/examples/cloud-config-disk-setup.txt b/doc/examples/cloud-config-disk-setup.txt index 38ad0528..dd91477d 100644 --- a/doc/examples/cloud-config-disk-setup.txt +++ b/doc/examples/cloud-config-disk-setup.txt @@ -98,11 +98,11 @@ disk_setup: # # table_type=: Currently the following are supported: # 'mbr': default and setups a MS-DOS partition table +# 'gpt': setups a GPT partition table # -# Note: At this time only 'mbr' partition tables are allowed. -# It is anticipated in the future that we'll have GPT as -# option in the future, or even "RAID" to create a mdadm -# RAID. +# Note: At this time only 'mbr' and 'gpt' partition tables +# are allowed. It is anticipated in the future that +# we'll also have "RAID" to create a mdadm RAID. # # layout={...}: The device layout. This is a list of values, with the # percentage of disk that partition will take. -- cgit v1.2.3