From e7a8f81e6eee390ce6920df053bf7467b5e4dbd7 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 22 Feb 2019 09:41:28 +0000 Subject: tests: integration test failure summary to use traceback if empty error When integration tests verification fails, the object returned contains has 'error' and 'traceback' keys. Each key can contain empty strings. If the simplified 'error' message is empty, fallback and use the more verbose full 'traceback' text in the failure summary. --- tests/cloud_tests/verify.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'tests/cloud_tests') diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py index 9911ecf2..7018f4d5 100644 --- a/tests/cloud_tests/verify.py +++ b/tests/cloud_tests/verify.py @@ -61,12 +61,17 @@ def format_test_failures(test_result): if not test_result['failures']: return '' failure_hdr = ' test failures:' - failure_fmt = ' * {module}.{class}.{function}\n {error}' + failure_fmt = ' * {module}.{class}.{function}\n ' output = [] for failure in test_result['failures']: if not output: output = [failure_hdr] - output.append(failure_fmt.format(**failure)) + msg = failure_fmt.format(**failure) + if failure.get('error'): + msg += failure['error'] + else: + msg += failure.get('traceback', '') + output.append(msg) return '\n'.join(output) -- cgit v1.2.3 From 47c53002ea7a661c674c3e409357db7e8a00297a Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Mon, 1 Apr 2019 14:24:26 +0000 Subject: cloud_tests: fix apt_pipelining test-cases The apt_pipelining test-cases were broken but until cloud-init changed it's default behavior to not disable, these silently passed as both only ever checked if pipelinging was disabled. First, the tests used the 'apt' namespace, which is not for configuring pipelining, rather that requires 'apt_pipelining' as the namespace. Second, the 'os' variant needs to check that cloud-init does not write a configuration file; it was a copy-and-paste error from the disable test-case. This branch fixes the config and collection to validate both scenarios. --- tests/cloud_tests/testcases/modules/apt_pipelining_disable.yaml | 3 +-- tests/cloud_tests/testcases/modules/apt_pipelining_os.py | 6 +++--- tests/cloud_tests/testcases/modules/apt_pipelining_os.yaml | 9 ++++----- 3 files changed, 8 insertions(+), 10 deletions(-) (limited to 'tests/cloud_tests') diff --git a/tests/cloud_tests/testcases/modules/apt_pipelining_disable.yaml b/tests/cloud_tests/testcases/modules/apt_pipelining_disable.yaml index bd9b5d08..22a31dc4 100644 --- a/tests/cloud_tests/testcases/modules/apt_pipelining_disable.yaml +++ b/tests/cloud_tests/testcases/modules/apt_pipelining_disable.yaml @@ -5,8 +5,7 @@ required_features: - apt cloud_config: | #cloud-config - apt: - apt_pipelining: false + apt_pipelining: false collect_scripts: 90cloud-init-pipelining: | #!/bin/bash diff --git a/tests/cloud_tests/testcases/modules/apt_pipelining_os.py b/tests/cloud_tests/testcases/modules/apt_pipelining_os.py index 740dc7c0..2b940a66 100644 --- a/tests/cloud_tests/testcases/modules/apt_pipelining_os.py +++ b/tests/cloud_tests/testcases/modules/apt_pipelining_os.py @@ -8,8 +8,8 @@ class TestAptPipeliningOS(base.CloudTestCase): """Test apt-pipelining module.""" def test_os_pipelining(self): - """Test pipelining set to os.""" - out = self.get_data_file('90cloud-init-pipelining') - self.assertIn('Acquire::http::Pipeline-Depth "0";', out) + """test 'os' settings does not write apt config file.""" + out = self.get_data_file('90cloud-init-pipelining_not_written') + self.assertEqual(0, int(out)) # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/apt_pipelining_os.yaml b/tests/cloud_tests/testcases/modules/apt_pipelining_os.yaml index cbed3ba3..86d5220b 100644 --- a/tests/cloud_tests/testcases/modules/apt_pipelining_os.yaml +++ b/tests/cloud_tests/testcases/modules/apt_pipelining_os.yaml @@ -1,15 +1,14 @@ # -# Set apt pipelining value to OS +# Set apt pipelining value to OS, no conf written # required_features: - apt cloud_config: | #cloud-config - apt: - apt_pipelining: os + apt_pipelining: os collect_scripts: - 90cloud-init-pipelining: | + 90cloud-init-pipelining_not_written: | #!/bin/bash - cat /etc/apt/apt.conf.d/90cloud-init-pipelining + ls /etc/apt/apt.conf.d/90cloud-init-pipelining | wc -l # vi: ts=4 expandtab -- cgit v1.2.3 From ce5fe3a20e86c4745d0310bb9c344d1344d9684c Mon Sep 17 00:00:00 2001 From: Paride Legovini Date: Thu, 9 May 2019 18:05:25 +0000 Subject: tests: add Eoan release --- tests/cloud_tests/releases.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'tests/cloud_tests') diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml index ec5da724..924ad956 100644 --- a/tests/cloud_tests/releases.yaml +++ b/tests/cloud_tests/releases.yaml @@ -129,6 +129,22 @@ features: releases: # UBUNTU ================================================================= + eoan: + # EOL: Jul 2020 + default: + enabled: true + release: eoan + version: 19.10 + os: ubuntu + feature_groups: + - base + - debian_base + - ubuntu_specific + lxd: + sstreams_server: https://cloud-images.ubuntu.com/daily + alias: eoan + setup_overrides: null + override_templates: false disco: # EOL: Jan 2020 default: -- cgit v1.2.3 From a02c0c9aa24a16f1983a81fe5dbfadac3d7e0ad3 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 18 Jul 2019 19:53:50 +0000 Subject: cloud_tests: updates and fixes - Update paramiko and cryptography module versions (2.4.2) to address issues with algo and deprecation warnings. - Modify ssh keypair generation to work with updated paramiko - tools/xkvm sync with newer version from curtin - Update NoCloudKvm instance.py to work with updated xkvm - pass -name to instance, useful for debugging on shared host - Add cache_mode platform config; default to cache=none,aio=native - Switch to yaml.safe_load() in platforms.py --- integration-requirements.txt | 3 +- tests/cloud_tests/platforms.yaml | 1 + tests/cloud_tests/platforms/nocloudkvm/instance.py | 13 +++-- tests/cloud_tests/platforms/platforms.py | 2 +- tests/cloud_tests/setup_image.py | 3 +- tools/xkvm | 61 +++++++++++++++++++--- 6 files changed, 68 insertions(+), 15 deletions(-) (limited to 'tests/cloud_tests') diff --git a/integration-requirements.txt b/integration-requirements.txt index 880d9886..fe5ad45d 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -10,7 +10,8 @@ unittest2 boto3==1.5.9 # ssh communication -paramiko==2.4.1 +paramiko==2.4.2 +cryptography==2.4.2 # lxd backend diff --git a/tests/cloud_tests/platforms.yaml b/tests/cloud_tests/platforms.yaml index 448aa98d..652a7051 100644 --- a/tests/cloud_tests/platforms.yaml +++ b/tests/cloud_tests/platforms.yaml @@ -66,5 +66,6 @@ platforms: {{ config_get("user.vendor-data", properties.default) }} nocloud-kvm: enabled: true + cache_mode: cache=none,aio=native # vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/nocloudkvm/instance.py b/tests/cloud_tests/platforms/nocloudkvm/instance.py index 33ff3f24..96185b75 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/instance.py +++ b/tests/cloud_tests/platforms/nocloudkvm/instance.py @@ -74,6 +74,8 @@ class NoCloudKVMInstance(Instance): self.pid_file = None self.console_file = None self.disk = image_path + self.cache_mode = platform.config.get('cache_mode', + 'cache=none,aio=native') self.meta_data = meta_data def shutdown(self, wait=True): @@ -113,7 +115,10 @@ class NoCloudKVMInstance(Instance): pass if self.pid_file: - os.remove(self.pid_file) + try: + os.remove(self.pid_file) + except Exception: + pass self.pid = None self._ssh_close() @@ -160,13 +165,13 @@ class NoCloudKVMInstance(Instance): self.ssh_port = self.get_free_port() cmd = ['./tools/xkvm', - '--disk', '%s,cache=unsafe' % self.disk, - '--disk', '%s,cache=unsafe' % seed, + '--disk', '%s,%s' % (self.disk, self.cache_mode), + '--disk', '%s' % seed, '--netdev', ','.join(['user', 'hostfwd=tcp::%s-:22' % self.ssh_port, 'dnssearch=%s' % CI_DOMAIN]), '--', '-pidfile', self.pid_file, '-vnc', 'none', - '-m', '2G', '-smp', '2', '-nographic', + '-m', '2G', '-smp', '2', '-nographic', '-name', self.name, '-serial', 'file:' + self.console_file] subprocess.Popen(cmd, close_fds=True, diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py index abbfebba..bebdf1c6 100644 --- a/tests/cloud_tests/platforms/platforms.py +++ b/tests/cloud_tests/platforms/platforms.py @@ -48,7 +48,7 @@ class Platform(object): if os.path.exists(filename): c_util.del_file(filename) - c_util.subp(['ssh-keygen', '-t', 'rsa', '-b', '4096', + c_util.subp(['ssh-keygen', '-m', 'PEM', '-t', 'rsa', '-b', '4096', '-f', filename, '-P', '', '-C', 'ubuntu@cloud_test'], capture=True) diff --git a/tests/cloud_tests/setup_image.py b/tests/cloud_tests/setup_image.py index 39f4517f..a8aaba15 100644 --- a/tests/cloud_tests/setup_image.py +++ b/tests/cloud_tests/setup_image.py @@ -222,7 +222,8 @@ def setup_image(args, image): for name, func, desc in handlers if getattr(args, name, None)] try: - data = yaml.load(image.read_data("/etc/cloud/build.info", decode=True)) + data = yaml.safe_load( + image.read_data("/etc/cloud/build.info", decode=True)) info = ' '.join(["%s=%s" % (k, data.get(k)) for k in ("build_name", "serial") if k in data]) except Exception as e: diff --git a/tools/xkvm b/tools/xkvm index a30ba916..8d44cad7 100755 --- a/tools/xkvm +++ b/tools/xkvm @@ -1,4 +1,6 @@ #!/bin/bash +# This file is part of cloud-init. +# See LICENSE file for copyright and license info. set -f @@ -11,6 +13,8 @@ TAPDEVS=( ) # OVS_CLEANUP gets populated with bridge:devname pairs used with ovs OVS_CLEANUP=( ) MAC_PREFIX="52:54:00:12:34" +# allow this to be set externally. +_QEMU_SUPPORTS_FILE_LOCKING="${_QEMU_SUPPORTS_FILE_LOCKING}" KVM="kvm" declare -A KVM_DEVOPTS @@ -119,6 +123,21 @@ isdevopt() { return 1 } +qemu_supports_file_locking() { + # hackily check if qemu has file.locking in -drive params (LP: #1716028) + if [ -z "$_QEMU_SUPPORTS_FILE_LOCKING" ]; then + # The only way we could find to check presense of file.locking is + # qmp (query-qmp-schema). Simply checking if the virtio-blk driver + # supports 'share-rw' is expected to be equivalent and simpler. + isdevopt virtio-blk share-rw && + _QEMU_SUPPORTS_FILE_LOCKING=true || + _QEMU_SUPPORTS_FILE_LOCKING=false + debug 1 "qemu supports file locking = ${_QEMU_SUPPORTS_FILE_LOCKING}" + fi + [ "$_QEMU_SUPPORTS_FILE_LOCKING" = "true" ] + return +} + padmac() { # return a full mac, given a subset. # assume whatever is input is the last portion to be @@ -367,7 +386,7 @@ main() { [ ${#netdevs[@]} -eq 0 ] && netdevs=( "${DEF_BRIDGE}" ) pt=( "$@" ) - local kvm_pkg="" virtio_scsi_bus="virtio-scsi-pci" + local kvm_pkg="" virtio_scsi_bus="virtio-scsi-pci" virtio_rng_device="virtio-rng-pci" [ -n "$kvm" ] && kvm_pkg="none" case $(uname -m) in i?86) @@ -382,7 +401,10 @@ main() { [ -n "$kvm" ] || { kvm="qemu-system-s390x"; kvm_pkg="qemu-system-misc"; } def_netmodel=${DEF_NETMODEL:-"virtio-net-ccw"} + # disable virtio-scsi-bus virtio_scsi_bus="virtio-scsi-ccw" + virtio_blk_bus="virtio-blk-ccw" + virtio_rng_device="virtio-rng-ccw" ;; ppc64*) [ -n "$kvm" ] || @@ -408,7 +430,7 @@ main() { bios_opts=( "${_RET[@]}" ) local out="" fmt="" bus="" unit="" index="" serial="" driver="" devopts="" - local busorindex="" driveopts="" cur="" val="" file="" + local busorindex="" driveopts="" cur="" val="" file="" wwn="" for((i=0;i<${#diskdevs[@]};i++)); do cur=${diskdevs[$i]} IFS=","; set -- $cur; IFS="$oifs" @@ -420,6 +442,7 @@ main() { unit="" index="" serial="" + wwn="" for tok in "$@"; do [ "${tok#*=}" = "${tok}" -a -f "${tok}" -a -z "$file" ] && file="$tok" val=${tok#*=} @@ -433,6 +456,7 @@ main() { file=*) file=$val;; fmt=*|format=*) fmt=$val;; serial=*) serial=$val;; + wwn=*) wwn=$val;; bus=*) bus=$val;; unit=*) unit=$val;; index=*) index=$val;; @@ -443,14 +467,19 @@ main() { out=$(LANG=C qemu-img info "$file") && fmt=$(echo "$out" | awk '$0 ~ /^file format:/ { print $3 }') || { error "failed to determine format of $file"; return 1; } - else + elif [ -z "$fmt" ]; then fmt=raw fi if [ -z "$driver" ]; then driver="$def_disk_driver" fi if [ -z "$serial" ]; then - serial="${file##*/}" + # use filename as serial if not provided a wwn + if [ -n "$wwn" ]; then + serial="$wwn" + else + serial="${file##*/}" + fi fi # make sure we add either bus= or index= @@ -470,11 +499,21 @@ main() { id=*|if=*|driver=*|$file|file=*) continue;; fmt=*|format=*) continue;; serial=*|bus=*|unit=*|index=*) continue;; + file.locking=*) + qemu_supports_file_locking || { + debug 2 "qemu has no file locking." \ + "Dropping '$tok' from: $cur" + continue + };; esac isdevopt "$driver" "$tok" && devopts="${devopts},$tok" || diskopts="${diskopts},${tok}" done - + case $driver in + virtio-blk-ccw) + # disable scsi when using virtio-blk-ccw + devopts="${devopts},scsi=off";; + esac diskargs=( "${diskargs[@]}" -drive "$diskopts" -device "$devopts" ) done @@ -623,10 +662,16 @@ main() { done local bus_devices - bus_devices=( -device "$virtio_scsi_bus,id=virtio-scsi-xkvm" ) - cmd=( "${kvmcmd[@]}" "${archopts[@]}" + if [ -n "${virtio_scsi_bus}" ]; then + bus_devices=( -device "$virtio_scsi_bus,id=virtio-scsi-xkvm" ) + fi + local rng_devices + rng_devices=( -object "rng-random,filename=/dev/urandom,id=objrng0" + -device "$virtio_rng_device,rng=objrng0,id=rng0" ) + cmd=( "${kvmcmd[@]}" "${archopts[@]}" "${bios_opts[@]}" "${bus_devices[@]}" + "${rng_devices[@]}" "${netargs[@]}" "${diskargs[@]}" "${pt[@]}" ) local pcmd=$(quote_cmd "${cmd[@]}") @@ -661,4 +706,4 @@ else main "$@" fi -# vi: ts=4 expandtab +# vi: ts=4 expandtab syntax=sh -- cgit v1.2.3 From 823708ea031290c864e3aef67f60f6eb495f281d Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Fri, 11 Oct 2019 16:52:13 +0000 Subject: cloud_test/lxd: Retry container delete a few times LXD integration tests fail sometimes due to failure to delete the container, usually related to ZFS backend. This is a transient issue unrelated to the test itself. Teach LXD platform to retry this a few times before returning an error. --- tests/cloud_tests/platforms/lxd/instance.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'tests/cloud_tests') diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py index 83c97ab4..2b804a62 100644 --- a/tests/cloud_tests/platforms/lxd/instance.py +++ b/tests/cloud_tests/platforms/lxd/instance.py @@ -4,6 +4,7 @@ import os import shutil +import time from tempfile import mkdtemp from cloudinit.util import load_yaml, subp, ProcessExecutionError, which @@ -224,7 +225,18 @@ class LXDInstance(Instance): LOG.debug("%s: deleting container.", self) self.unfreeze() self.shutdown() - self.pylxd_container.delete(wait=True) + retries = [1] * 5 + for attempt, wait in enumerate(retries): + try: + self.pylxd_container.delete(wait=True) + break + except Exception: + if attempt + 1 >= len(retries): + raise + LOG.debug('Failed to delete container %s (%s/%s) retrying...', + self, attempt + 1, len(retries)) + time.sleep(wait) + self._pylxd_container = None if self.platform.container_exists(self.name): -- cgit v1.2.3 From aa3e4961ceae5a5c5b5cf13221b5f6721991fe75 Mon Sep 17 00:00:00 2001 From: ahosmanmsft Date: Tue, 26 Nov 2019 11:36:00 -0700 Subject: cloud_tests: add azure platform support to integration tests Added Azure to cloud tests supporting upstream integration testing. Implement the inherited platform classes, Azure configurations to release/platform, and docs on how to run Azure CI. --- .pylintrc | 2 +- doc/rtd/topics/tests.rst | 52 +++++ integration-requirements.txt | 9 + tests/cloud_tests/platforms.yaml | 6 + tests/cloud_tests/platforms/__init__.py | 2 + tests/cloud_tests/platforms/azurecloud/__init__.py | 0 tests/cloud_tests/platforms/azurecloud/image.py | 108 +++++++++ tests/cloud_tests/platforms/azurecloud/instance.py | 243 +++++++++++++++++++++ tests/cloud_tests/platforms/azurecloud/platform.py | 232 ++++++++++++++++++++ .../cloud_tests/platforms/azurecloud/regions.json | 42 ++++ tests/cloud_tests/platforms/azurecloud/snapshot.py | 58 +++++ tests/cloud_tests/platforms/ec2/image.py | 1 + tests/cloud_tests/platforms/ec2/platform.py | 3 +- tests/cloud_tests/releases.yaml | 2 + tox.ini | 2 + 15 files changed, 760 insertions(+), 2 deletions(-) create mode 100644 tests/cloud_tests/platforms/azurecloud/__init__.py create mode 100644 tests/cloud_tests/platforms/azurecloud/image.py create mode 100644 tests/cloud_tests/platforms/azurecloud/instance.py create mode 100644 tests/cloud_tests/platforms/azurecloud/platform.py create mode 100644 tests/cloud_tests/platforms/azurecloud/regions.json create mode 100644 tests/cloud_tests/platforms/azurecloud/snapshot.py (limited to 'tests/cloud_tests') diff --git a/.pylintrc b/.pylintrc index 365c8c8b..c83546a6 100644 --- a/.pylintrc +++ b/.pylintrc @@ -62,7 +62,7 @@ ignored-modules= # for classes with dynamically set attributes). This supports the use of # qualified names. # argparse.Namespace from https://github.com/PyCQA/pylint/issues/2413 -ignored-classes=argparse.Namespace,optparse.Values,thread._local +ignored-classes=argparse.Namespace,optparse.Values,thread._local,ImageManager,ContainerManager # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular diff --git a/doc/rtd/topics/tests.rst b/doc/rtd/topics/tests.rst index a2c703a5..3b27f805 100644 --- a/doc/rtd/topics/tests.rst +++ b/doc/rtd/topics/tests.rst @@ -423,6 +423,58 @@ generated when running ``aws configure``: region = us-west-2 +Azure Cloud +----------- + +To run on Azure Cloud platform users login with Service Principal and export +credentials file. Region is defaulted and can be set in ``tests/cloud_tests/platforms.yaml``. +The Service Principal credentials are the standard authentication for Azure SDK +to interact with Azure Services: + +Create Service Principal account or login + +.. code-block:: shell-session + + $ az ad sp create-for-rbac --name "APP_ID" --password "STRONG-SECRET-PASSWORD" + +.. code-block:: shell-session + + $ az login --service-principal --username "APP_ID" --password "STRONG-SECRET-PASSWORD" + +Export credentials + +.. code-block:: shell-session + + $ az ad sp create-for-rbac --sdk-auth > $HOME/.azure/credentials.json + +.. code-block:: json + + { + "clientId": "", + "clientSecret": "", + "subscriptionId": "", + "tenantId": "", + "activeDirectoryEndpointUrl": "https://login.microsoftonline.com", + "resourceManagerEndpointUrl": "https://management.azure.com/", + "activeDirectoryGraphResourceId": "https://graph.windows.net/", + "sqlManagementEndpointUrl": "https://management.core.windows.net:8443/", + "galleryEndpointUrl": "https://gallery.azure.com/", + "managementEndpointUrl": "https://management.core.windows.net/" + } + +Set region in platforms.yaml + +.. code-block:: yaml + :emphasize-lines: 3 + + azurecloud: + enabled: true + region: West US 2 + vm_size: Standard_DS1_v2 + storage_sku: standard_lrs + tag: ci + + Architecture ============ diff --git a/integration-requirements.txt b/integration-requirements.txt index fe5ad45d..897d6110 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -20,3 +20,12 @@ git+https://github.com/lxc/pylxd.git@4b8ab1802f9aee4eb29cf7b119dae0aa47150779 # finds latest image information git+https://git.launchpad.net/simplestreams + +# azure backend +azure-storage==0.36.0 +msrestazure==0.6.1 +azure-common==1.1.23 +azure-mgmt-compute==7.0.0 +azure-mgmt-network==5.0.0 +azure-mgmt-resource==4.0.0 +azure-mgmt-storage==6.0.0 diff --git a/tests/cloud_tests/platforms.yaml b/tests/cloud_tests/platforms.yaml index 652a7051..eaaa0a71 100644 --- a/tests/cloud_tests/platforms.yaml +++ b/tests/cloud_tests/platforms.yaml @@ -67,5 +67,11 @@ platforms: nocloud-kvm: enabled: true cache_mode: cache=none,aio=native + azurecloud: + enabled: true + region: West US 2 + vm_size: Standard_DS1_v2 + storage_sku: standard_lrs + tag: ci # vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/__init__.py b/tests/cloud_tests/platforms/__init__.py index a01e51ac..6a410b84 100644 --- a/tests/cloud_tests/platforms/__init__.py +++ b/tests/cloud_tests/platforms/__init__.py @@ -5,11 +5,13 @@ from .ec2 import platform as ec2 from .lxd import platform as lxd from .nocloudkvm import platform as nocloudkvm +from .azurecloud import platform as azurecloud PLATFORMS = { 'ec2': ec2.EC2Platform, 'nocloud-kvm': nocloudkvm.NoCloudKVMPlatform, 'lxd': lxd.LXDPlatform, + 'azurecloud': azurecloud.AzureCloudPlatform, } diff --git a/tests/cloud_tests/platforms/azurecloud/__init__.py b/tests/cloud_tests/platforms/azurecloud/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cloud_tests/platforms/azurecloud/image.py b/tests/cloud_tests/platforms/azurecloud/image.py new file mode 100644 index 00000000..96a946f3 --- /dev/null +++ b/tests/cloud_tests/platforms/azurecloud/image.py @@ -0,0 +1,108 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Azure Cloud image Base class.""" + +from tests.cloud_tests import LOG + +from ..images import Image +from .snapshot import AzureCloudSnapshot + + +class AzureCloudImage(Image): + """Azure Cloud backed image.""" + + platform_name = 'azurecloud' + + def __init__(self, platform, config, image_id): + """Set up image. + + @param platform: platform object + @param config: image configuration + @param image_id: image id used to boot instance + """ + super(AzureCloudImage, self).__init__(platform, config) + self.image_id = image_id + self._img_instance = None + + @property + def _instance(self): + """Internal use only, returns a running instance""" + LOG.debug('creating instance') + if not self._img_instance: + self._img_instance = self.platform.create_instance( + self.properties, self.config, self.features, + self.image_id, user_data=None) + return self._img_instance + + def destroy(self): + """Delete the instance used to create a custom image.""" + LOG.debug('deleting VM that was used to create image') + if self._img_instance: + LOG.debug('Deleting instance %s', self._img_instance.name) + delete_vm = self.platform.compute_client.virtual_machines.delete( + self.platform.resource_group.name, self.image_id) + delete_vm.wait() + + super(AzureCloudImage, self).destroy() + + def _execute(self, *args, **kwargs): + """Execute command in image, modifying image.""" + LOG.debug('executing commands on image') + self._instance.start() + return self._instance._execute(*args, **kwargs) + + def push_file(self, local_path, remote_path): + """Copy file at 'local_path' to instance at 'remote_path'.""" + LOG.debug('pushing file to image') + return self._instance.push_file(local_path, remote_path) + + def run_script(self, *args, **kwargs): + """Run script in image, modifying image. + + @return_value: script output + """ + LOG.debug('running script on image') + self._instance.start() + return self._instance.run_script(*args, **kwargs) + + def snapshot(self): + """ Create snapshot (image) of instance, wait until done. + + If no instance has been booted, base image is returned. + Otherwise runs the clean script, deallocates, generalizes + and creates custom image from instance. + """ + LOG.debug('creating image from VM') + if not self._img_instance: + return AzureCloudSnapshot(self.platform, self.properties, + self.config, self.features, + self.image_id, delete_on_destroy=False) + + if self.config.get('boot_clean_script'): + self._img_instance.run_script(self.config.get('boot_clean_script')) + + deallocate = self.platform.compute_client.virtual_machines.deallocate( + self.platform.resource_group.name, self.image_id) + deallocate.wait() + + self.platform.compute_client.virtual_machines.generalize( + self.platform.resource_group.name, self.image_id) + + image_params = { + "location": self.platform.location, + "properties": { + "sourceVirtualMachine": { + "id": self._img_instance.instance.id + } + } + } + self.platform.compute_client.images.create_or_update( + self.platform.resource_group.name, self.image_id, + image_params) + + self.destroy() + + return AzureCloudSnapshot(self.platform, self.properties, self.config, + self.features, self.image_id) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py new file mode 100644 index 00000000..3d77a1a7 --- /dev/null +++ b/tests/cloud_tests/platforms/azurecloud/instance.py @@ -0,0 +1,243 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base Azure Cloud instance.""" + +from datetime import datetime, timedelta +from urllib.parse import urlparse +from time import sleep +import traceback +import os + + +# pylint: disable=no-name-in-module +from azure.storage.blob import BlockBlobService, BlobPermissions +from msrestazure.azure_exceptions import CloudError + +from tests.cloud_tests import LOG + +from ..instances import Instance + + +class AzureCloudInstance(Instance): + """Azure Cloud backed instance.""" + + platform_name = 'azurecloud' + + def __init__(self, platform, properties, config, + features, image_id, user_data=None): + """Set up instance. + + @param platform: platform object + @param properties: dictionary of properties + @param config: dictionary of configuration values + @param features: dictionary of supported feature flags + @param image_id: image to find and/or use + @param user_data: test user-data to pass to instance + """ + super(AzureCloudInstance, self).__init__( + platform, image_id, properties, config, features) + + self.ssh_port = 22 + self.ssh_ip = None + self.instance = None + self.image_id = image_id + self.user_data = user_data + self.ssh_key_file = os.path.join( + platform.config['data_dir'], platform.config['private_key']) + self.ssh_pubkey_file = os.path.join( + platform.config['data_dir'], platform.config['public_key']) + self.blob_client, self.container, self.blob = None, None, None + + def start(self, wait=True, wait_for_cloud_init=False): + """Start instance with the platforms NIC.""" + if self.instance: + return + data = self.image_id.split('-') + release, support = data[2].replace('_', '.'), data[3] + sku = '%s-%s' % (release, support) if support == 'LTS' else release + image_resource_id = '/subscriptions/%s' \ + '/resourceGroups/%s' \ + '/providers/Microsoft.Compute/images/%s' % ( + self.platform.subscription_id, + self.platform.resource_group.name, + self.image_id) + storage_uri = "http://%s.blob.core.windows.net" \ + % self.platform.storage.name + with open(self.ssh_pubkey_file, 'r') as key: + ssh_pub_keydata = key.read() + + image_exists = False + try: + LOG.debug('finding image in resource group using image_id') + self.platform.compute_client.images.get( + self.platform.resource_group.name, + self.image_id + ) + image_exists = True + LOG.debug('image found, launching instance') + except CloudError: + LOG.debug( + 'image not found, launching instance with base image') + pass + + vm_params = { + 'location': self.platform.location, + 'os_profile': { + 'computer_name': 'CI', + 'admin_username': self.ssh_username, + "customData": self.user_data, + "linuxConfiguration": { + "disable_password_authentication": True, + "ssh": { + "public_keys": [{ + "path": "/home/%s/.ssh/authorized_keys" % + self.ssh_username, + "keyData": ssh_pub_keydata + }] + } + } + }, + "diagnosticsProfile": { + "bootDiagnostics": { + "storageUri": storage_uri, + "enabled": True + } + }, + 'hardware_profile': { + 'vm_size': self.platform.vm_size + }, + 'storage_profile': { + 'image_reference': { + 'id': image_resource_id + } if image_exists else { + 'publisher': 'Canonical', + 'offer': 'UbuntuServer', + 'sku': sku, + 'version': 'latest' + } + }, + 'network_profile': { + 'network_interfaces': [{ + 'id': self.platform.nic.id + }] + }, + 'tags': { + 'Name': self.platform.tag, + } + } + + try: + self.instance = self.platform.compute_client.virtual_machines.\ + create_or_update(self.platform.resource_group.name, + self.image_id, vm_params) + except CloudError: + raise RuntimeError('failed creating instance:\n{}'.format( + traceback.format_exc())) + + if wait: + self.instance.wait() + self.ssh_ip = self.platform.network_client.\ + public_ip_addresses.get( + self.platform.resource_group.name, + self.platform.public_ip.name + ).ip_address + self._wait_for_system(wait_for_cloud_init) + + self.instance = self.instance.result() + self.blob_client, self.container, self.blob =\ + self._get_blob_client() + + def shutdown(self, wait=True): + """Finds console log then stopping/deallocates VM""" + LOG.debug('waiting on console log before stopping') + attempts, exists = 5, False + while not exists and attempts: + try: + attempts -= 1 + exists = self.blob_client.get_blob_to_bytes( + self.container, self.blob) + LOG.debug('found console log') + except Exception as e: + if attempts: + LOG.debug('Unable to find console log, ' + '%s attempts remaining', attempts) + sleep(15) + else: + LOG.warning('Could not find console log: %s', e) + pass + + LOG.debug('stopping instance %s', self.image_id) + vm_deallocate = \ + self.platform.compute_client.virtual_machines.deallocate( + self.platform.resource_group.name, self.image_id) + if wait: + vm_deallocate.wait() + + def destroy(self): + """Delete VM and close all connections""" + if self.instance: + LOG.debug('destroying instance: %s', self.image_id) + vm_delete = self.platform.compute_client.virtual_machines.delete( + self.platform.resource_group.name, self.image_id) + vm_delete.wait() + + self._ssh_close() + + super(AzureCloudInstance, self).destroy() + + def _execute(self, command, stdin=None, env=None): + """Execute command on instance.""" + env_args = [] + if env: + env_args = ['env'] + ["%s=%s" for k, v in env.items()] + + return self._ssh(['sudo'] + env_args + list(command), stdin=stdin) + + def _get_blob_client(self): + """ + Use VM details to retrieve container and blob name. + Then Create blob service client for sas token to + retrieve console log. + + :return: blob service, container name, blob name + """ + LOG.debug('creating blob service for console log') + storage = self.platform.storage_client.storage_accounts.get_properties( + self.platform.resource_group.name, self.platform.storage.name) + + keys = self.platform.storage_client.storage_accounts.list_keys( + self.platform.resource_group.name, self.platform.storage.name + ).keys[0].value + + virtual_machine = self.platform.compute_client.virtual_machines.get( + self.platform.resource_group.name, self.instance.name, + expand='instanceView') + + blob_uri = virtual_machine.instance_view.boot_diagnostics.\ + serial_console_log_blob_uri + + container, blob = urlparse(blob_uri).path.split('/')[-2:] + + blob_client = BlockBlobService( + account_name=storage.name, + account_key=keys) + + sas = blob_client.generate_blob_shared_access_signature( + container_name=container, blob_name=blob, protocol='https', + expiry=datetime.utcnow() + timedelta(hours=1), + permission=BlobPermissions.READ) + + blob_client = BlockBlobService( + account_name=storage.name, + sas_token=sas) + + return blob_client, container, blob + + def console_log(self): + """Instance console. + + @return_value: bytes of this instance’s console + """ + boot_diagnostics = self.blob_client.get_blob_to_bytes( + self.container, self.blob) + return boot_diagnostics.content diff --git a/tests/cloud_tests/platforms/azurecloud/platform.py b/tests/cloud_tests/platforms/azurecloud/platform.py new file mode 100644 index 00000000..77f159eb --- /dev/null +++ b/tests/cloud_tests/platforms/azurecloud/platform.py @@ -0,0 +1,232 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base Azure Cloud class.""" + +import os +import base64 +import traceback +from datetime import datetime +from tests.cloud_tests import LOG + +# pylint: disable=no-name-in-module +from azure.common.credentials import ServicePrincipalCredentials +# pylint: disable=no-name-in-module +from azure.mgmt.resource import ResourceManagementClient +# pylint: disable=no-name-in-module +from azure.mgmt.network import NetworkManagementClient +# pylint: disable=no-name-in-module +from azure.mgmt.compute import ComputeManagementClient +# pylint: disable=no-name-in-module +from azure.mgmt.storage import StorageManagementClient +from msrestazure.azure_exceptions import CloudError + +from .image import AzureCloudImage +from .instance import AzureCloudInstance +from ..platforms import Platform + +from cloudinit import util as c_util + + +class AzureCloudPlatform(Platform): + """Azure Cloud test platforms.""" + + platform_name = 'azurecloud' + + def __init__(self, config): + """Set up platform.""" + super(AzureCloudPlatform, self).__init__(config) + self.tag = '%s-%s' % ( + config['tag'], datetime.now().strftime('%Y%m%d%H%M%S')) + self.storage_sku = config['storage_sku'] + self.vm_size = config['vm_size'] + self.location = config['region'] + + try: + self.credentials, self.subscription_id = self._get_credentials() + + self.resource_client = ResourceManagementClient( + self.credentials, self.subscription_id) + self.compute_client = ComputeManagementClient( + self.credentials, self.subscription_id) + self.network_client = NetworkManagementClient( + self.credentials, self.subscription_id) + self.storage_client = StorageManagementClient( + self.credentials, self.subscription_id) + + self.resource_group = self._create_resource_group() + self.public_ip = self._create_public_ip_address() + self.storage = self._create_storage_account(config) + self.vnet = self._create_vnet() + self.subnet = self._create_subnet() + self.nic = self._create_nic() + except CloudError: + raise RuntimeError('failed creating a resource:\n{}'.format( + traceback.format_exc())) + + def create_instance(self, properties, config, features, + image_id, user_data=None): + """Create an instance + + @param properties: image properties + @param config: image configuration + @param features: image features + @param image_id: string of image id + @param user_data: test user-data to pass to instance + @return_value: cloud_tests.instances instance + """ + user_data = str(base64.b64encode( + user_data.encode('utf-8')), 'utf-8') + + return AzureCloudInstance(self, properties, config, features, + image_id, user_data) + + def get_image(self, img_conf): + """Get image using specified image configuration. + + @param img_conf: configuration for image + @return_value: cloud_tests.images instance + """ + ss_region = self.azure_location_to_simplestreams_region() + + filters = [ + 'arch=%s' % 'amd64', + 'endpoint=https://management.core.windows.net/', + 'region=%s' % ss_region, + 'release=%s' % img_conf['release'] + ] + + LOG.debug('finding image using streams') + image = self._query_streams(img_conf, filters) + + try: + image_id = image['id'] + LOG.debug('found image: %s', image_id) + if image_id.find('__') > 0: + image_id = image_id.split('__')[1] + LOG.debug('image_id shortened to %s', image_id) + except KeyError: + raise RuntimeError('no images found for %s' % img_conf['release']) + + return AzureCloudImage(self, img_conf, image_id) + + def destroy(self): + """Delete all resources in resource group.""" + LOG.debug("Deleting resource group: %s", self.resource_group.name) + delete = self.resource_client.resource_groups.delete( + self.resource_group.name) + delete.wait() + + def azure_location_to_simplestreams_region(self): + """Convert location to simplestreams region""" + location = self.location.lower().replace(' ', '') + LOG.debug('finding location %s using simple streams', location) + regions_file = os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'regions.json') + region_simplestreams_map = c_util.load_json( + c_util.load_file(regions_file)) + return region_simplestreams_map.get(location, location) + + def _get_credentials(self): + """Get credentials from environment""" + LOG.debug('getting credentials from environment') + cred_file = os.path.expanduser('~/.azure/credentials.json') + try: + azure_creds = c_util.load_json( + c_util.load_file(cred_file)) + subscription_id = azure_creds['subscriptionId'] + credentials = ServicePrincipalCredentials( + client_id=azure_creds['clientId'], + secret=azure_creds['clientSecret'], + tenant=azure_creds['tenantId']) + return credentials, subscription_id + except KeyError: + raise RuntimeError('Please configure Azure service principal' + ' credentials in %s' % cred_file) + + def _create_resource_group(self): + """Create resource group""" + LOG.debug('creating resource group') + resource_group_name = self.tag + resource_group_params = { + 'location': self.location + } + resource_group = self.resource_client.resource_groups.create_or_update( + resource_group_name, resource_group_params) + return resource_group + + def _create_storage_account(self, config): + LOG.debug('creating storage account') + storage_account_name = 'storage%s' % datetime.now().\ + strftime('%Y%m%d%H%M%S') + storage_params = { + 'sku': { + 'name': config['storage_sku'] + }, + 'kind': "Storage", + 'location': self.location + } + storage_account = self.storage_client.storage_accounts.create( + self.resource_group.name, storage_account_name, storage_params) + return storage_account.result() + + def _create_public_ip_address(self): + """Create public ip address""" + LOG.debug('creating public ip address') + public_ip_name = '%s-ip' % self.resource_group.name + public_ip_params = { + 'location': self.location, + 'public_ip_allocation_method': 'Dynamic' + } + ip = self.network_client.public_ip_addresses.create_or_update( + self.resource_group.name, public_ip_name, public_ip_params) + return ip.result() + + def _create_vnet(self): + """create virtual network""" + LOG.debug('creating vnet') + vnet_name = '%s-vnet' % self.resource_group.name + vnet_params = { + 'location': self.location, + 'address_space': { + 'address_prefixes': ['10.0.0.0/16'] + } + } + vnet = self.network_client.virtual_networks.create_or_update( + self.resource_group.name, vnet_name, vnet_params) + return vnet.result() + + def _create_subnet(self): + """create sub-network""" + LOG.debug('creating subnet') + subnet_name = '%s-subnet' % self.resource_group.name + subnet_params = { + 'address_prefix': '10.0.0.0/24' + } + subnet = self.network_client.subnets.create_or_update( + self.resource_group.name, self.vnet.name, + subnet_name, subnet_params) + return subnet.result() + + def _create_nic(self): + """Create network interface controller""" + LOG.debug('creating nic') + nic_name = '%s-nic' % self.resource_group.name + nic_params = { + 'location': self.location, + 'ip_configurations': [{ + 'name': 'ipconfig', + 'subnet': { + 'id': self.subnet.id + }, + 'publicIpAddress': { + 'id': "/subscriptions/%s" + "/resourceGroups/%s/providers/Microsoft.Network" + "/publicIPAddresses/%s" % ( + self.subscription_id, self.resource_group.name, + self.public_ip.name), + } + }] + } + nic = self.network_client.network_interfaces.create_or_update( + self.resource_group.name, nic_name, nic_params) + return nic.result() diff --git a/tests/cloud_tests/platforms/azurecloud/regions.json b/tests/cloud_tests/platforms/azurecloud/regions.json new file mode 100644 index 00000000..c1b4da20 --- /dev/null +++ b/tests/cloud_tests/platforms/azurecloud/regions.json @@ -0,0 +1,42 @@ +{ + "eastasia": "East Asia", + "southeastasia": "Southeast Asia", + "centralus": "Central US", + "eastus": "East US", + "eastus2": "East US 2", + "westus": "West US", + "northcentralus": "North Central US", + "southcentralus": "South Central US", + "northeurope": "North Europe", + "westeurope": "West Europe", + "japanwest": "Japan West", + "japaneast": "Japan East", + "brazilsouth": "Brazil South", + "australiaeast": "Australia East", + "australiasoutheast": "Australia Southeast", + "southindia": "South India", + "centralindia": "Central India", + "westindia": "West India", + "canadacentral": "Canada Central", + "canadaeast": "Canada East", + "uksouth": "UK South", + "ukwest": "UK West", + "westcentralus": "West Central US", + "westus2": "West US 2", + "koreacentral": "Korea Central", + "koreasouth": "Korea South", + "francecentral": "France Central", + "francesouth": "France South", + "australiacentral": "Australia Central", + "australiacentral2": "Australia Central 2", + "uaecentral": "UAE Central", + "uaenorth": "UAE North", + "southafricanorth": "South Africa North", + "southafricawest": "South Africa West", + "switzerlandnorth": "Switzerland North", + "switzerlandwest": "Switzerland West", + "germanynorth": "Germany North", + "germanywestcentral": "Germany West Central", + "norwaywest": "Norway West", + "norwayeast": "Norway East" +} diff --git a/tests/cloud_tests/platforms/azurecloud/snapshot.py b/tests/cloud_tests/platforms/azurecloud/snapshot.py new file mode 100644 index 00000000..580cc596 --- /dev/null +++ b/tests/cloud_tests/platforms/azurecloud/snapshot.py @@ -0,0 +1,58 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base Azure Cloud snapshot.""" + +from ..snapshots import Snapshot + +from tests.cloud_tests import LOG + + +class AzureCloudSnapshot(Snapshot): + """Azure Cloud image copy backed snapshot.""" + + platform_name = 'azurecloud' + + def __init__(self, platform, properties, config, features, image_id, + delete_on_destroy=True): + """Set up snapshot. + + @param platform: platform object + @param properties: image properties + @param config: image config + @param features: supported feature flags + """ + super(AzureCloudSnapshot, self).__init__( + platform, properties, config, features) + + self.image_id = image_id + self.delete_on_destroy = delete_on_destroy + + def launch(self, user_data, meta_data=None, block=True, start=True, + use_desc=None): + """Launch instance. + + @param user_data: user-data for the instance + @param meta_data: meta_data for the instance + @param block: wait until instance is created + @param start: start instance and wait until fully started + @param use_desc: description of snapshot instance use + @return_value: an Instance + """ + if meta_data is not None: + raise ValueError("metadata not supported on Azure Cloud tests") + + instance = self.platform.create_instance( + self.properties, self.config, self.features, + self.image_id, user_data) + + return instance + + def destroy(self): + """Clean up snapshot data.""" + LOG.debug('destroying image %s', self.image_id) + if self.delete_on_destroy: + self.platform.compute_client.images.delete( + self.platform.resource_group.name, + self.image_id) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/ec2/image.py b/tests/cloud_tests/platforms/ec2/image.py index 7bedf59d..d7b2c908 100644 --- a/tests/cloud_tests/platforms/ec2/image.py +++ b/tests/cloud_tests/platforms/ec2/image.py @@ -4,6 +4,7 @@ from ..images import Image from .snapshot import EC2Snapshot + from tests.cloud_tests import LOG diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py index f188c27b..7a3d0fe0 100644 --- a/tests/cloud_tests/platforms/ec2/platform.py +++ b/tests/cloud_tests/platforms/ec2/platform.py @@ -135,6 +135,7 @@ class EC2Platform(Platform): def _create_internet_gateway(self): """Create Internet Gateway and assign to VPC.""" LOG.debug('creating internet gateway') + # pylint: disable=no-member internet_gateway = self.ec2_resource.create_internet_gateway() internet_gateway.attach_to_vpc(VpcId=self.vpc.id) self._tag_resource(internet_gateway) @@ -190,7 +191,7 @@ class EC2Platform(Platform): """Setup AWS EC2 VPC or return existing VPC.""" LOG.debug('creating new vpc') try: - vpc = self.ec2_resource.create_vpc( + vpc = self.ec2_resource.create_vpc( # pylint: disable=no-member CidrBlock=self.ipv4_cidr, AmazonProvidedIpv6CidrBlock=True) except botocore.exceptions.ClientError as e: diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml index 924ad956..7ddc5b85 100644 --- a/tests/cloud_tests/releases.yaml +++ b/tests/cloud_tests/releases.yaml @@ -55,6 +55,8 @@ default_release_config: # cloud-init, so must pull cloud-init in from repo using # setup_image.upgrade upgrade: true + azurecloud: + boot_timeout: 300 features: # all currently supported feature flags diff --git a/tox.ini b/tox.ini index f5baf328..042346bb 100644 --- a/tox.ini +++ b/tox.ini @@ -24,6 +24,7 @@ deps = pylint==2.3.1 # test-requirements because unit tests are now present in cloudinit tree -r{toxinidir}/test-requirements.txt + -r{toxinidir}/integration-requirements.txt commands = {envpython} -m pylint {posargs:cloudinit tests tools} [testenv:py3] @@ -135,6 +136,7 @@ deps = pylint # test-requirements -r{toxinidir}/test-requirements.txt + -r{toxinidir}/integration-requirements.txt [testenv:citest] basepython = python3 -- cgit v1.2.3 From 87f2cb0acc7e802f93fa71ff3432dfd6708717ca Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 19 Dec 2019 19:00:27 -0500 Subject: cc_snappy: remove deprecated module (#127) * cc_snappy: remove deprecated module * cloud_tests: remove cc_snappy tests (and references) This module was deprecated in favor of cc_snap in cloud-init v.18.2 --- cloudinit/config/cc_snappy.py | 322 ----------- config/cloud.cfg.tmpl | 3 - doc/rtd/topics/modules.rst | 1 - tests/cloud_tests/testcases/modules/TODO.md | 3 - tests/cloud_tests/testcases/modules/snappy.py | 17 - tests/cloud_tests/testcases/modules/snappy.yaml | 18 - .../unittests/test_handler/test_handler_snappy.py | 601 --------------------- 7 files changed, 965 deletions(-) delete mode 100644 cloudinit/config/cc_snappy.py delete mode 100644 tests/cloud_tests/testcases/modules/snappy.py delete mode 100644 tests/cloud_tests/testcases/modules/snappy.yaml delete mode 100644 tests/unittests/test_handler/test_handler_snappy.py (limited to 'tests/cloud_tests') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py deleted file mode 100644 index b94cd04e..00000000 --- a/cloudinit/config/cc_snappy.py +++ /dev/null @@ -1,322 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -# RELEASE_BLOCKER: Remove this deprecated module in 18.3 -""" -Snappy ------- -**Summary:** snappy modules allows configuration of snappy. - -**Deprecated**: Use :ref:`snap` module instead. This module will not exist -in cloud-init 18.3. - -The below example config config would install ``etcd``, and then install -``pkg2.smoser`` with a ```` argument where ``config-file`` has -``config-blob`` inside it. If ``pkgname`` is installed already, then -``snappy config pkgname `` -will be called where ``file`` has ``pkgname-config-blob`` as its content. - -Entries in ``config`` can be namespaced or non-namespaced for a package. -In either case, the config provided to snappy command is non-namespaced. -The package name is provided as it appears. - -If ``packages_dir`` has files in it that end in ``.snap``, then they are -installed. Given 3 files: - - - /foo.snap - - /foo.config - - /bar.snap - -cloud-init will invoke: - - - snappy install /foo.snap /foo.config - - snappy install /bar.snap - -.. note:: - that if provided a ``config`` entry for ``ubuntu-core``, then - cloud-init will invoke: snappy config ubuntu-core - Allowing you to configure ubuntu-core in this way. - -The ``ssh_enabled`` key controls the system's ssh service. The default value -is ``auto``. Options are: - - - **True:** enable ssh service - - **False:** disable ssh service - - **auto:** enable ssh service if either ssh keys have been provided - or user has requested password authentication (ssh_pwauth). - -**Internal name:** ``cc_snappy`` - -**Module frequency:** per instance - -**Supported distros:** ubuntu - -**Config keys**:: - - #cloud-config - snappy: - system_snappy: auto - ssh_enabled: auto - packages: [etcd, pkg2.smoser] - config: - pkgname: - key2: value2 - pkg2: - key1: value1 - packages_dir: '/writable/user-data/cloud-init/snaps' -""" - -from cloudinit import log as logging -from cloudinit.settings import PER_INSTANCE -from cloudinit import temp_utils -from cloudinit import safeyaml -from cloudinit import util - -import glob -import os - -LOG = logging.getLogger(__name__) - -frequency = PER_INSTANCE -SNAPPY_CMD = "snappy" -NAMESPACE_DELIM = '.' - -BUILTIN_CFG = { - 'packages': [], - 'packages_dir': '/writable/user-data/cloud-init/snaps', - 'ssh_enabled': "auto", - 'system_snappy': "auto", - 'config': {}, -} - -distros = ['ubuntu'] - - -def parse_filename(fname): - fname = os.path.basename(fname) - fname_noext = fname.rpartition(".")[0] - name = fname_noext.partition("_")[0] - shortname = name.partition(".")[0] - return(name, shortname, fname_noext) - - -def get_fs_package_ops(fspath): - if not fspath: - return [] - ops = [] - for snapfile in sorted(glob.glob(os.path.sep.join([fspath, '*.snap']))): - (name, shortname, fname_noext) = parse_filename(snapfile) - cfg = None - for cand in (fname_noext, name, shortname): - fpcand = os.path.sep.join([fspath, cand]) + ".config" - if os.path.isfile(fpcand): - cfg = fpcand - break - ops.append(makeop('install', name, config=None, - path=snapfile, cfgfile=cfg)) - return ops - - -def makeop(op, name, config=None, path=None, cfgfile=None): - return({'op': op, 'name': name, 'config': config, 'path': path, - 'cfgfile': cfgfile}) - - -def get_package_config(configs, name): - # load the package's config from the configs dict. - # prefer full-name entry (config-example.canonical) - # over short name entry (config-example) - if name in configs: - return configs[name] - return configs.get(name.partition(NAMESPACE_DELIM)[0]) - - -def get_package_ops(packages, configs, installed=None, fspath=None): - # get the install an config operations that should be done - if installed is None: - installed = read_installed_packages() - short_installed = [p.partition(NAMESPACE_DELIM)[0] for p in installed] - - if not packages: - packages = [] - if not configs: - configs = {} - - ops = [] - ops += get_fs_package_ops(fspath) - - for name in packages: - ops.append(makeop('install', name, get_package_config(configs, name))) - - to_install = [f['name'] for f in ops] - short_to_install = [f['name'].partition(NAMESPACE_DELIM)[0] for f in ops] - - for name in configs: - if name in to_install: - continue - shortname = name.partition(NAMESPACE_DELIM)[0] - if shortname in short_to_install: - continue - if name in installed or shortname in short_installed: - ops.append(makeop('config', name, - config=get_package_config(configs, name))) - - # prefer config entries to filepath entries - for op in ops: - if op['op'] != 'install' or not op['cfgfile']: - continue - name = op['name'] - fromcfg = get_package_config(configs, op['name']) - if fromcfg: - LOG.debug("preferring configs[%(name)s] over '%(cfgfile)s'", op) - op['cfgfile'] = None - op['config'] = fromcfg - - return ops - - -def render_snap_op(op, name, path=None, cfgfile=None, config=None): - if op not in ('install', 'config'): - raise ValueError("cannot render op '%s'" % op) - - shortname = name.partition(NAMESPACE_DELIM)[0] - try: - cfg_tmpf = None - if config is not None: - # input to 'snappy config packagename' must have nested data. odd. - # config: - # packagename: - # config - # Note, however, we do not touch config files on disk. - nested_cfg = {'config': {shortname: config}} - (fd, cfg_tmpf) = temp_utils.mkstemp() - os.write(fd, safeyaml.dumps(nested_cfg).encode()) - os.close(fd) - cfgfile = cfg_tmpf - - cmd = [SNAPPY_CMD, op] - if op == 'install': - if path: - cmd.append("--allow-unauthenticated") - cmd.append(path) - else: - cmd.append(name) - if cfgfile: - cmd.append(cfgfile) - elif op == 'config': - cmd += [name, cfgfile] - - util.subp(cmd) - - finally: - if cfg_tmpf: - os.unlink(cfg_tmpf) - - -def read_installed_packages(): - ret = [] - for (name, _date, _version, dev) in read_pkg_data(): - if dev: - ret.append(NAMESPACE_DELIM.join([name, dev])) - else: - ret.append(name) - return ret - - -def read_pkg_data(): - out, _err = util.subp([SNAPPY_CMD, "list"]) - pkg_data = [] - for line in out.splitlines()[1:]: - toks = line.split(sep=None, maxsplit=3) - if len(toks) == 3: - (name, date, version) = toks - dev = None - else: - (name, date, version, dev) = toks - pkg_data.append((name, date, version, dev,)) - return pkg_data - - -def disable_enable_ssh(enabled): - LOG.debug("setting enablement of ssh to: %s", enabled) - # do something here that would enable or disable - not_to_be_run = "/etc/ssh/sshd_not_to_be_run" - if enabled: - util.del_file(not_to_be_run) - # this is an indempotent operation - util.subp(["systemctl", "start", "ssh"]) - else: - # this is an indempotent operation - util.subp(["systemctl", "stop", "ssh"]) - util.write_file(not_to_be_run, "cloud-init\n") - - -def set_snappy_command(): - global SNAPPY_CMD - if util.which("snappy-go"): - SNAPPY_CMD = "snappy-go" - elif util.which("snappy"): - SNAPPY_CMD = "snappy" - else: - SNAPPY_CMD = "snap" - LOG.debug("snappy command is '%s'", SNAPPY_CMD) - - -def handle(name, cfg, cloud, log, args): - cfgin = cfg.get('snappy') - if not cfgin: - cfgin = {} - mycfg = util.mergemanydict([cfgin, BUILTIN_CFG]) - - sys_snappy = str(mycfg.get("system_snappy", "auto")) - if util.is_false(sys_snappy): - LOG.debug("%s: System is not snappy. disabling", name) - return - - if sys_snappy.lower() == "auto" and not(util.system_is_snappy()): - LOG.debug("%s: 'auto' mode, and system not snappy", name) - return - - log.warning( - 'DEPRECATION: snappy module will be dropped in 18.3 release.' - ' Use snap module instead') - - set_snappy_command() - - pkg_ops = get_package_ops(packages=mycfg['packages'], - configs=mycfg['config'], - fspath=mycfg['packages_dir']) - - fails = [] - for pkg_op in pkg_ops: - try: - render_snap_op(**pkg_op) - except Exception as e: - fails.append((pkg_op, e,)) - LOG.warning("'%s' failed for '%s': %s", - pkg_op['op'], pkg_op['name'], e) - - # Default to disabling SSH - ssh_enabled = mycfg.get('ssh_enabled', "auto") - - # If the user has not explicitly enabled or disabled SSH, then enable it - # when password SSH authentication is requested or there are SSH keys - if ssh_enabled == "auto": - user_ssh_keys = cloud.get_public_ssh_keys() or None - password_auth_enabled = cfg.get('ssh_pwauth', False) - if user_ssh_keys: - LOG.debug("Enabling SSH, ssh keys found in datasource") - ssh_enabled = True - elif cfg.get('ssh_authorized_keys'): - LOG.debug("Enabling SSH, ssh keys found in config") - elif password_auth_enabled: - LOG.debug("Enabling SSH, password authentication requested") - ssh_enabled = True - elif ssh_enabled not in (True, False): - LOG.warning("Unknown value '%s' in ssh_enabled", ssh_enabled) - - disable_enable_ssh(ssh_enabled) - - if fails: - raise Exception("failed to install/configure snaps") - -# vi: ts=4 expandtab diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 87c37ba0..7aab265e 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -103,9 +103,6 @@ cloud_config_modules: # The modules that run in the 'final' stage cloud_final_modules: -{% if variant in ["ubuntu", "unknown", "debian"] %} - - snappy # DEPRECATED- Drop in version 18.2 -{% endif %} - package-update-upgrade-install {% if variant in ["ubuntu", "unknown", "debian"] %} - fan diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index 3dcdd3bc..1aa2f88d 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -46,7 +46,6 @@ Modules .. automodule:: cloudinit.config.cc_set_hostname .. automodule:: cloudinit.config.cc_set_passwords .. automodule:: cloudinit.config.cc_snap -.. automodule:: cloudinit.config.cc_snappy .. automodule:: cloudinit.config.cc_snap_config .. automodule:: cloudinit.config.cc_spacewalk .. automodule:: cloudinit.config.cc_ssh diff --git a/tests/cloud_tests/testcases/modules/TODO.md b/tests/cloud_tests/testcases/modules/TODO.md index 0b933b3b..ee7e9213 100644 --- a/tests/cloud_tests/testcases/modules/TODO.md +++ b/tests/cloud_tests/testcases/modules/TODO.md @@ -78,9 +78,6 @@ Not applicable to write a test for this as it specifies when something should be ## scripts vendor Not applicable to write a test for this as it specifies when something should be run. -## snappy -2016-11-17: Need test to install snaps from store - ## snap-config 2016-11-17: Need to investigate diff --git a/tests/cloud_tests/testcases/modules/snappy.py b/tests/cloud_tests/testcases/modules/snappy.py deleted file mode 100644 index 7d17fc5b..00000000 --- a/tests/cloud_tests/testcases/modules/snappy.py +++ /dev/null @@ -1,17 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""cloud-init Integration Test Verify Script""" -from tests.cloud_tests.testcases import base - - -class TestSnappy(base.CloudTestCase): - """Test snappy module""" - - expected_warnings = ('DEPRECATION',) - - def test_snappy_version(self): - """Test snappy version output""" - out = self.get_data_file('snapd') - self.assertIn('Status: install ok installed', out) - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/snappy.yaml b/tests/cloud_tests/testcases/modules/snappy.yaml deleted file mode 100644 index 8ac322ae..00000000 --- a/tests/cloud_tests/testcases/modules/snappy.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# -# Install snappy -# -# Aug 17, 2018: Disabled due to requiring a proxy for testing -# tests do not handle the proxy well at this time. -enabled: False -required_features: - - snap -cloud_config: | - #cloud-config - snappy: - system_snappy: auto -collect_scripts: - snapd: | - #!/bin/bash - dpkg -s snapd - -# vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py deleted file mode 100644 index 76b79c29..00000000 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ /dev/null @@ -1,601 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -from cloudinit.config.cc_snappy import ( - makeop, get_package_ops, render_snap_op) -from cloudinit.config.cc_snap_config import ( - add_assertions, add_snap_user, ASSERTIONS_FILE) -from cloudinit import (distros, helpers, cloud, util) -from cloudinit.config.cc_snap_config import handle as snap_handle -from cloudinit.sources import DataSourceNone -from cloudinit.tests.helpers import FilesystemMockingTestCase, mock - -from cloudinit.tests import helpers as t_help - -import logging -import os -import shutil -import tempfile -import textwrap -import yaml - -LOG = logging.getLogger(__name__) -ALLOWED = (dict, list, int, str) - - -class TestInstallPackages(t_help.TestCase): - def setUp(self): - super(TestInstallPackages, self).setUp() - self.unapply = [] - - # by default 'which' has nothing in its path - self.apply_patches([(util, 'subp', self._subp)]) - self.subp_called = [] - self.snapcmds = [] - self.tmp = tempfile.mkdtemp(prefix="TestInstallPackages") - - def tearDown(self): - apply_patches([i for i in reversed(self.unapply)]) - shutil.rmtree(self.tmp) - - def apply_patches(self, patches): - ret = apply_patches(patches) - self.unapply += ret - - def populate_tmp(self, files): - return t_help.populate_dir(self.tmp, files) - - def _subp(self, *args, **kwargs): - # supports subp calling with cmd as args or kwargs - if 'args' not in kwargs: - kwargs['args'] = args[0] - self.subp_called.append(kwargs) - args = kwargs['args'] - # here we basically parse the snappy command invoked - # and append to snapcmds a list of (mode, pkg, config) - if args[0:2] == ['snappy', 'config']: - if args[3] == "-": - config = kwargs.get('data', '') - else: - with open(args[3], "rb") as fp: - config = yaml.safe_load(fp.read()) - self.snapcmds.append(['config', args[2], config]) - elif args[0:2] == ['snappy', 'install']: - config = None - pkg = None - for arg in args[2:]: - if arg.startswith("-"): - continue - if not pkg: - pkg = arg - elif not config: - cfgfile = arg - if cfgfile == "-": - config = kwargs.get('data', '') - elif cfgfile: - with open(cfgfile, "rb") as fp: - config = yaml.safe_load(fp.read()) - self.snapcmds.append(['install', pkg, config]) - - def test_package_ops_1(self): - ret = get_package_ops( - packages=['pkg1', 'pkg2', 'pkg3'], - configs={'pkg2': b'mycfg2'}, installed=[]) - self.assertEqual( - ret, [makeop('install', 'pkg1', None, None), - makeop('install', 'pkg2', b'mycfg2', None), - makeop('install', 'pkg3', None, None)]) - - def test_package_ops_config_only(self): - ret = get_package_ops( - packages=None, - configs={'pkg2': b'mycfg2'}, installed=['pkg1', 'pkg2']) - self.assertEqual( - ret, [makeop('config', 'pkg2', b'mycfg2')]) - - def test_package_ops_install_and_config(self): - ret = get_package_ops( - packages=['pkg3', 'pkg2'], - configs={'pkg2': b'mycfg2', 'xinstalled': b'xcfg'}, - installed=['xinstalled']) - self.assertEqual( - ret, [makeop('install', 'pkg3'), - makeop('install', 'pkg2', b'mycfg2'), - makeop('config', 'xinstalled', b'xcfg')]) - - def test_package_ops_install_long_config_short(self): - # a package can be installed by full name, but have config by short - cfg = {'k1': 'k2'} - ret = get_package_ops( - packages=['config-example.canonical'], - configs={'config-example': cfg}, installed=[]) - self.assertEqual( - ret, [makeop('install', 'config-example.canonical', cfg)]) - - def test_package_ops_with_file(self): - self.populate_tmp( - {"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg", - "snapf2.snap": b"foo2", "foo.bar": "ignored"}) - ret = get_package_ops( - packages=['pkg1'], configs={}, installed=[], fspath=self.tmp) - self.assertEqual( - ret, - [makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap", - cfgfile="snapf1.config"), - makeop_tmpd(self.tmp, 'install', 'snapf2', path="snapf2.snap"), - makeop('install', 'pkg1')]) - - def test_package_ops_common_filename(self): - # fish package name from filename - # package names likely look like: pkgname.namespace_version_arch.snap - - # find filenames - self.populate_tmp( - {"pkg-ws.smoser_0.3.4_all.snap": "pkg-ws-snapdata", - "pkg-ws.config": "pkg-ws-config", - "pkg1.smoser_1.2.3_all.snap": "pkg1.snapdata", - "pkg1.smoser.config": "pkg1.smoser.config-data", - "pkg1.config": "pkg1.config-data", - "pkg2.smoser_0.0_amd64.snap": "pkg2-snapdata", - "pkg2.smoser_0.0_amd64.config": "pkg2.config"}) - - ret = get_package_ops( - packages=[], configs={}, installed=[], fspath=self.tmp) - self.assertEqual( - ret, - [makeop_tmpd(self.tmp, 'install', 'pkg-ws.smoser', - path="pkg-ws.smoser_0.3.4_all.snap", - cfgfile="pkg-ws.config"), - makeop_tmpd(self.tmp, 'install', 'pkg1.smoser', - path="pkg1.smoser_1.2.3_all.snap", - cfgfile="pkg1.smoser.config"), - makeop_tmpd(self.tmp, 'install', 'pkg2.smoser', - path="pkg2.smoser_0.0_amd64.snap", - cfgfile="pkg2.smoser_0.0_amd64.config"), - ]) - - def test_package_ops_config_overrides_file(self): - # config data overrides local file .config - self.populate_tmp( - {"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg"}) - ret = get_package_ops( - packages=[], configs={'snapf1': 'snapf1cfg-config'}, - installed=[], fspath=self.tmp) - self.assertEqual( - ret, [makeop_tmpd(self.tmp, 'install', 'snapf1', - path="snapf1.snap", config="snapf1cfg-config")]) - - def test_package_ops_namespacing(self): - cfgs = { - 'config-example': {'k1': 'v1'}, - 'pkg1': {'p1': 'p2'}, - 'ubuntu-core': {'c1': 'c2'}, - 'notinstalled.smoser': {'s1': 's2'}, - } - ret = get_package_ops( - packages=['config-example.canonical'], configs=cfgs, - installed=['config-example.smoser', 'pkg1.canonical', - 'ubuntu-core']) - - expected_configs = [ - makeop('config', 'pkg1', config=cfgs['pkg1']), - makeop('config', 'ubuntu-core', config=cfgs['ubuntu-core'])] - expected_installs = [ - makeop('install', 'config-example.canonical', - config=cfgs['config-example'])] - - installs = [i for i in ret if i['op'] == 'install'] - configs = [c for c in ret if c['op'] == 'config'] - - self.assertEqual(installs, expected_installs) - # configs are not ordered - self.assertEqual(len(configs), len(expected_configs)) - self.assertTrue(all(found in expected_configs for found in configs)) - - def test_render_op_localsnap(self): - self.populate_tmp({"snapf1.snap": b"foo1"}) - op = makeop_tmpd(self.tmp, 'install', 'snapf1', - path='snapf1.snap') - render_snap_op(**op) - self.assertEqual( - self.snapcmds, [['install', op['path'], None]]) - - def test_render_op_localsnap_localconfig(self): - self.populate_tmp( - {"snapf1.snap": b"foo1", 'snapf1.config': b'snapf1cfg'}) - op = makeop_tmpd(self.tmp, 'install', 'snapf1', - path='snapf1.snap', cfgfile='snapf1.config') - render_snap_op(**op) - self.assertEqual( - self.snapcmds, [['install', op['path'], 'snapf1cfg']]) - - def test_render_op_snap(self): - op = makeop('install', 'snapf1') - render_snap_op(**op) - self.assertEqual( - self.snapcmds, [['install', 'snapf1', None]]) - - def test_render_op_snap_config(self): - mycfg = {'key1': 'value1'} - name = "snapf1" - op = makeop('install', name, config=mycfg) - render_snap_op(**op) - self.assertEqual( - self.snapcmds, [['install', name, {'config': {name: mycfg}}]]) - - def test_render_op_config_bytes(self): - name = "snapf1" - mycfg = b'myconfig' - op = makeop('config', name, config=mycfg) - render_snap_op(**op) - self.assertEqual( - self.snapcmds, [['config', 'snapf1', {'config': {name: mycfg}}]]) - - def test_render_op_config_string(self): - name = 'snapf1' - mycfg = 'myconfig: foo\nhisconfig: bar\n' - op = makeop('config', name, config=mycfg) - render_snap_op(**op) - self.assertEqual( - self.snapcmds, [['config', 'snapf1', {'config': {name: mycfg}}]]) - - def test_render_op_config_dict(self): - # config entry for package can be a dict, not a string blob - mycfg = {'foo': 'bar'} - name = 'snapf1' - op = makeop('config', name, config=mycfg) - render_snap_op(**op) - # snapcmds is a list of 3-entry lists. data_found will be the - # blob of data in the file in 'snappy install --config=' - data_found = self.snapcmds[0][2] - self.assertEqual(mycfg, data_found['config'][name]) - - def test_render_op_config_list(self): - # config entry for package can be a list, not a string blob - mycfg = ['foo', 'bar', 'wark', {'f1': 'b1'}] - name = "snapf1" - op = makeop('config', name, config=mycfg) - render_snap_op(**op) - data_found = self.snapcmds[0][2] - self.assertEqual(mycfg, data_found['config'][name]) - - def test_render_op_config_int(self): - # config entry for package can be a list, not a string blob - mycfg = 1 - name = 'snapf1' - op = makeop('config', name, config=mycfg) - render_snap_op(**op) - data_found = self.snapcmds[0][2] - self.assertEqual(mycfg, data_found['config'][name]) - - def test_render_long_configs_short(self): - # install a namespaced package should have un-namespaced config - mycfg = {'k1': 'k2'} - name = 'snapf1' - op = makeop('install', name + ".smoser", config=mycfg) - render_snap_op(**op) - data_found = self.snapcmds[0][2] - self.assertEqual(mycfg, data_found['config'][name]) - - def test_render_does_not_pad_cfgfile(self): - # package_ops with cfgfile should not modify --file= content. - mydata = "foo1: bar1\nk: [l1, l2, l3]\n" - self.populate_tmp( - {"snapf1.snap": b"foo1", "snapf1.config": mydata.encode()}) - ret = get_package_ops( - packages=[], configs={}, installed=[], fspath=self.tmp) - self.assertEqual( - ret, - [makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap", - cfgfile="snapf1.config")]) - - # now the op was ok, but test that render didn't mess it up. - render_snap_op(**ret[0]) - data_found = self.snapcmds[0][2] - # the data found gets loaded in the snapcmd interpretation - # so this comparison is a bit lossy, but input to snappy config - # is expected to be yaml loadable, so it should be OK. - self.assertEqual(yaml.safe_load(mydata), data_found) - - -class TestSnapConfig(FilesystemMockingTestCase): - - SYSTEM_USER_ASSERTION = textwrap.dedent(""" - type: system-user - authority-id: LqvZQdfyfGlYvtep4W6Oj6pFXP9t1Ksp - brand-id: LqvZQdfyfGlYvtep4W6Oj6pFXP9t1Ksp - email: foo@bar.com - password: $6$E5YiAuMIPAwX58jG$miomhVNui/vf7f/3ctB/f0RWSKFxG0YXzrJ9rtJ1ikvzt - series: - - 16 - since: 2016-09-10T16:34:00+03:00 - until: 2017-11-10T16:34:00+03:00 - username: baz - sign-key-sha3-384: RuVvnp4n52GilycjfbbTCI3_L8Y6QlIE75wxMc0KzGV3AUQqVd9GuXoj - - AcLBXAQAAQoABgUCV/UU1wAKCRBKnlMoJQLkZVeLD/9/+hIeVywtzsDA3oxl+P+u9D13y9s6svP - Jd6Wnf4FTw6sq1GjBE4ZA7lrwSaRCUJ9Vcsvf2q9OGPY7mOb2TBxaDe0PbUMjrSrqllSSQwhpNI - zG+NxkkKuxsUmLzFa+k9m6cyojNbw5LFhQZBQCGlr3JYqC0tIREq/UsZxj+90TUC87lDJwkU8GF - s4CR+rejZj4itIcDcVxCSnJH6hv6j2JrJskJmvObqTnoOlcab+JXdamXqbldSP3UIhWoyVjqzkj - +to7mXgx+cCUA9+ngNCcfUG+1huGGTWXPCYkZ78HvErcRlIdeo4d3xwtz1cl/w3vYnq9og1XwsP - Yfetr3boig2qs1Y+j/LpsfYBYncgWjeDfAB9ZZaqQz/oc8n87tIPZDJHrusTlBfop8CqcM4xsKS - d+wnEY8e/F24mdSOYmS1vQCIDiRU3MKb6x138Ud6oHXFlRBbBJqMMctPqWDunWzb5QJ7YR0I39q - BrnEqv5NE0G7w6HOJ1LSPG5Hae3P4T2ea+ATgkb03RPr3KnXnzXg4TtBbW1nytdlgoNc/BafE1H - f3NThcq9gwX4xWZ2PAWnqVPYdDMyCtzW3Ck+o6sIzx+dh4gDLPHIi/6TPe/pUuMop9CBpWwez7V - v1z+1+URx6Xlq3Jq18y5pZ6fY3IDJ6km2nQPMzcm4Q==""") - - ACCOUNT_ASSERTION = textwrap.dedent(""" - type: account-key - authority-id: canonical - revision: 2 - public-key-sha3-384: BWDEoaqyr25nF5SNCvEv2v7QnM9QsfCc0PBMYD_i2NGSQ32EF2d4D0 - account-id: canonical - name: store - since: 2016-04-01T00:00:00.0Z - body-length: 717 - sign-key-sha3-384: -CvQKAwRQ5h3Ffn10FILJoEZUXOv6km9FwA80-Rcj-f-6jadQ89VRswH - - AcbBTQRWhcGAARAA0KKYYQWuHOrsFVi4p4l7ZzSvX7kLgJFFeFgOkzdWKBTHEnsMKjl5mefFe9j - qe8NlmJdfY7BenP7XeBtwKp700H/t9lLrZbpTNAPHXYxEWFJp5bPqIcJYBZ+29oLVLN1Tc5X482 - vCiDqL8+pPYqBrK2fNlyPlNNSum9wI70rDDL4r6FVvr+osTnGejibdV8JphWX+lrSQDnRSdM8KJ - UM43vTgLGTi9W54oRhsA2OFexRfRksTrnqGoonCjqX5wO3OFSaMDzMsO2MJ/hPfLgDqw53qjzuK - Iec9OL3k5basvu2cj5u9tKwVFDsCKK2GbKUsWWpx2KTpOifmhmiAbzkTHbH9KaoMS7p0kJwhTQG - o9aJ9VMTWHJc/NCBx7eu451u6d46sBPCXS/OMUh2766fQmoRtO1OwCTxsRKG2kkjbMn54UdFULl - VfzvyghMNRKIezsEkmM8wueTqGUGZWa6CEZqZKwhe/PROxOPYzqtDH18XZknbU1n5lNb7vNfem9 - 2ai+3+JyFnW9UhfvpVF7gzAgdyCqNli4C6BIN43uwoS8HkykocZS/+Gv52aUQ/NZ8BKOHLw+7an - Q0o8W9ltSLZbEMxFIPSN0stiZlkXAp6DLyvh1Y4wXSynDjUondTpej2fSvSlCz/W5v5V7qA4nIc - vUvV7RjVzv17ut0AEQEAAQ== - - AcLDXAQAAQoABgUCV83k9QAKCRDUpVvql9g3IBT8IACKZ7XpiBZ3W4lqbPssY6On81WmxQLtvsM - WTp6zZpl/wWOSt2vMNUk9pvcmrNq1jG9CuhDfWFLGXEjcrrmVkN3YuCOajMSPFCGrxsIBLSRt/b - nrKykdLAAzMfG8rP1d82bjFFiIieE+urQ0Kcv09Jtdvavq3JT1Tek5mFyyfhHNlQEKOzWqmRWiL - 3c3VOZUs1ZD8TSlnuq/x+5T0X0YtOyGjSlVxk7UybbyMNd6MZfNaMpIG4x+mxD3KHFtBAC7O6kL - eX3i6j5nCY5UABfA3DZEAkWP4zlmdBEOvZ9t293NaDdOpzsUHRkoi0Zez/9BHQ/kwx/uNc2WqrY - inCmu16JGNeXqsyinnLl7Ghn2RwhvDMlLxF6RTx8xdx1yk6p3PBTwhZMUvuZGjUtN/AG8BmVJQ1 - rsGSRkkSywvnhVJRB2sudnrMBmNS2goJbzSbmJnOlBrd2WsV0T9SgNMWZBiov3LvU4o2SmAb6b+ - rYwh8H5QHcuuYJuxDjFhPswIp6Wes5T6hUicf3SWtObcDS4HSkVS4ImBjjX9YgCuFy7QdnooOWE - aPvkRw3XCVeYq0K6w9GRsk1YFErD4XmXXZjDYY650MX9v42Sz5MmphHV8jdIY5ssbadwFSe2rCQ - 6UX08zy7RsIb19hTndE6ncvSNDChUR9eEnCm73eYaWTWTnq1cxdVP/s52r8uss++OYOkPWqh5nO - haRn7INjH/yZX4qXjNXlTjo0PnHH0q08vNKDwLhxS+D9du+70FeacXFyLIbcWllSbJ7DmbumGpF - yYbtj3FDDPzachFQdIG3lSt+cSUGeyfSs6wVtc3cIPka/2Urx7RprfmoWSI6+a5NcLdj0u2z8O9 - HxeIgxDpg/3gT8ZIuFKePMcLDM19Fh/p0ysCsX+84B9chNWtsMSmIaE57V+959MVtsLu7SLb9gi - skrju0pQCwsu2wHMLTNd1f3PTHmrr49hxetTus07HSQUApMtAGKzQilF5zqFjbyaTd4xgQbd+PK - CjFyzQTDOcUhXpuUGt/IzlqiFfsCsmbj2K4KdSNYMlqIgZ3Azu8KvZLIhsyN7v5vNIZSPfEbjde - ClU9r0VRiJmtYBUjcSghD9LWn+yRLwOxhfQVjm0cBwIt5R/yPF/qC76yIVuWUtM5Y2/zJR1J8OF - qWchvlImHtvDzS9FQeLyzJAOjvZ2CnWp2gILgUz0WQdOk1Dq8ax7KS9BQ42zxw9EZAEPw3PEFqR - IQsRTONp+iVS8YxSmoYZjDlCgRMWUmawez/Fv5b9Fb/XkO5Eq4e+KfrpUujXItaipb+tV8h5v3t - oG3Ie3WOHrVjCLXIdYslpL1O4nadqR6Xv58pHj6k""") - - test_assertions = [ACCOUNT_ASSERTION, SYSTEM_USER_ASSERTION] - - def setUp(self): - super(TestSnapConfig, self).setUp() - self.subp = util.subp - self.new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.new_root) - - def _get_cloud(self, distro, metadata=None): - self.patchUtils(self.new_root) - paths = helpers.Paths({}) - cls = distros.fetch(distro) - mydist = cls(distro, {}, paths) - myds = DataSourceNone.DataSourceNone({}, mydist, paths) - if metadata: - myds.metadata.update(metadata) - return cloud.Cloud(myds, paths, {}, mydist, None) - - @mock.patch('cloudinit.util.write_file') - @mock.patch('cloudinit.util.subp') - def test_snap_config_add_assertions(self, msubp, mwrite): - add_assertions(self.test_assertions) - - combined = "\n".join(self.test_assertions) - mwrite.assert_any_call(ASSERTIONS_FILE, combined.encode('utf-8')) - msubp.assert_called_with(['snap', 'ack', ASSERTIONS_FILE], - capture=True) - - def test_snap_config_add_assertions_empty(self): - self.assertRaises(ValueError, add_assertions, []) - - def test_add_assertions_nonlist(self): - self.assertRaises(ValueError, add_assertions, {}) - - @mock.patch('cloudinit.util.write_file') - @mock.patch('cloudinit.util.subp') - def test_snap_config_add_assertions_ack_fails(self, msubp, mwrite): - msubp.side_effect = [util.ProcessExecutionError("Invalid assertion")] - self.assertRaises(util.ProcessExecutionError, add_assertions, - self.test_assertions) - - @mock.patch('cloudinit.config.cc_snap_config.add_assertions') - @mock.patch('cloudinit.config.cc_snap_config.util') - def test_snap_config_handle_no_config(self, mock_util, mock_add): - cfg = {} - cc = self._get_cloud('ubuntu') - cc.distro = mock.MagicMock() - cc.distro.name = 'ubuntu' - mock_util.which.return_value = None - snap_handle('snap_config', cfg, cc, LOG, None) - mock_add.assert_not_called() - - def test_snap_config_add_snap_user_no_config(self): - usercfg = add_snap_user(cfg=None) - self.assertIsNone(usercfg) - - def test_snap_config_add_snap_user_not_dict(self): - cfg = ['foobar'] - self.assertRaises(ValueError, add_snap_user, cfg) - - def test_snap_config_add_snap_user_no_email(self): - cfg = {'assertions': [], 'known': True} - usercfg = add_snap_user(cfg=cfg) - self.assertIsNone(usercfg) - - @mock.patch('cloudinit.config.cc_snap_config.util') - def test_snap_config_add_snap_user_email_only(self, mock_util): - email = 'janet@planetjanet.org' - cfg = {'email': email} - mock_util.which.return_value = None - mock_util.system_is_snappy.return_value = True - mock_util.subp.side_effect = [ - ("false\n", ""), # snap managed - ] - - usercfg = add_snap_user(cfg=cfg) - - self.assertEqual(usercfg, {'snapuser': email, 'known': False}) - - @mock.patch('cloudinit.config.cc_snap_config.util') - def test_snap_config_add_snap_user_email_known(self, mock_util): - email = 'janet@planetjanet.org' - known = True - cfg = {'email': email, 'known': known} - mock_util.which.return_value = None - mock_util.system_is_snappy.return_value = True - mock_util.subp.side_effect = [ - ("false\n", ""), # snap managed - (self.SYSTEM_USER_ASSERTION, ""), # snap known system-user - ] - - usercfg = add_snap_user(cfg=cfg) - - self.assertEqual(usercfg, {'snapuser': email, 'known': known}) - - @mock.patch('cloudinit.config.cc_snap_config.add_assertions') - @mock.patch('cloudinit.config.cc_snap_config.util') - def test_snap_config_handle_system_not_snappy(self, mock_util, mock_add): - cfg = {'snappy': {'assertions': self.test_assertions}} - cc = self._get_cloud('ubuntu') - cc.distro = mock.MagicMock() - cc.distro.name = 'ubuntu' - mock_util.which.return_value = None - mock_util.system_is_snappy.return_value = False - - snap_handle('snap_config', cfg, cc, LOG, None) - - mock_add.assert_not_called() - - @mock.patch('cloudinit.config.cc_snap_config.add_assertions') - @mock.patch('cloudinit.config.cc_snap_config.util') - def test_snap_config_handle_snapuser(self, mock_util, mock_add): - email = 'janet@planetjanet.org' - cfg = { - 'snappy': { - 'assertions': self.test_assertions, - 'email': email, - } - } - cc = self._get_cloud('ubuntu') - cc.distro = mock.MagicMock() - cc.distro.name = 'ubuntu' - mock_util.which.return_value = None - mock_util.system_is_snappy.return_value = True - mock_util.subp.side_effect = [ - ("false\n", ""), # snap managed - ] - - snap_handle('snap_config', cfg, cc, LOG, None) - - mock_add.assert_called_with(self.test_assertions) - usercfg = {'snapuser': email, 'known': False} - cc.distro.create_user.assert_called_with(email, **usercfg) - - @mock.patch('cloudinit.config.cc_snap_config.add_assertions') - @mock.patch('cloudinit.config.cc_snap_config.util') - def test_snap_config_handle_snapuser_known(self, mock_util, mock_add): - email = 'janet@planetjanet.org' - cfg = { - 'snappy': { - 'assertions': self.test_assertions, - 'email': email, - 'known': True, - } - } - cc = self._get_cloud('ubuntu') - cc.distro = mock.MagicMock() - cc.distro.name = 'ubuntu' - mock_util.which.return_value = None - mock_util.system_is_snappy.return_value = True - mock_util.subp.side_effect = [ - ("false\n", ""), # snap managed - (self.SYSTEM_USER_ASSERTION, ""), # snap known system-user - ] - - snap_handle('snap_config', cfg, cc, LOG, None) - - mock_add.assert_called_with(self.test_assertions) - usercfg = {'snapuser': email, 'known': True} - cc.distro.create_user.assert_called_with(email, **usercfg) - - @mock.patch('cloudinit.config.cc_snap_config.add_assertions') - @mock.patch('cloudinit.config.cc_snap_config.util') - def test_snap_config_handle_snapuser_known_managed(self, mock_util, - mock_add): - email = 'janet@planetjanet.org' - cfg = { - 'snappy': { - 'assertions': self.test_assertions, - 'email': email, - 'known': True, - } - } - cc = self._get_cloud('ubuntu') - cc.distro = mock.MagicMock() - cc.distro.name = 'ubuntu' - mock_util.which.return_value = None - mock_util.system_is_snappy.return_value = True - mock_util.subp.side_effect = [ - ("true\n", ""), # snap managed - ] - - snap_handle('snap_config', cfg, cc, LOG, None) - - mock_add.assert_called_with(self.test_assertions) - cc.distro.create_user.assert_not_called() - - @mock.patch('cloudinit.config.cc_snap_config.add_assertions') - @mock.patch('cloudinit.config.cc_snap_config.util') - def test_snap_config_handle_snapuser_known_no_assertion(self, mock_util, - mock_add): - email = 'janet@planetjanet.org' - cfg = { - 'snappy': { - 'assertions': [self.ACCOUNT_ASSERTION], - 'email': email, - 'known': True, - } - } - cc = self._get_cloud('ubuntu') - cc.distro = mock.MagicMock() - cc.distro.name = 'ubuntu' - mock_util.which.return_value = None - mock_util.system_is_snappy.return_value = True - mock_util.subp.side_effect = [ - ("true\n", ""), # snap managed - ("", ""), # snap known system-user - ] - - snap_handle('snap_config', cfg, cc, LOG, None) - - mock_add.assert_called_with([self.ACCOUNT_ASSERTION]) - cc.distro.create_user.assert_not_called() - - -def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): - if cfgfile: - cfgfile = os.path.sep.join([tmpd, cfgfile]) - if path: - path = os.path.sep.join([tmpd, path]) - return(makeop(op=op, name=name, config=config, path=path, cfgfile=cfgfile)) - - -def apply_patches(patches): - ret = [] - for (ref, name, replace) in patches: - if replace is None: - continue - orig = getattr(ref, name) - setattr(ref, name, replace) - ret.append((ref, name, orig)) - return ret - -# vi: ts=4 expandtab -- cgit v1.2.3 From c7dca877c1fa3dc8b63486bb88022ee23a31bebe Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 20 Dec 2019 13:34:30 -0700 Subject: modules: drop cc_snap_config config module (#134) cloud-init has moved to cc_snap module and a top-level config key 'snap'. cc_snap_config was deprecated in cloud-init version 18.2 Co-authored-by: Daniel Watkins --- cloudinit/config/cc_snap_config.py | 184 ---------------------------- config/cloud.cfg.tmpl | 1 - doc/rtd/topics/modules.rst | 1 - tests/cloud_tests/testcases/modules/TODO.md | 4 +- 4 files changed, 2 insertions(+), 188 deletions(-) delete mode 100644 cloudinit/config/cc_snap_config.py (limited to 'tests/cloud_tests') diff --git a/cloudinit/config/cc_snap_config.py b/cloudinit/config/cc_snap_config.py deleted file mode 100644 index afe297ee..00000000 --- a/cloudinit/config/cc_snap_config.py +++ /dev/null @@ -1,184 +0,0 @@ -# Copyright (C) 2016 Canonical Ltd. -# -# Author: Ryan Harper -# -# This file is part of cloud-init. See LICENSE file for license information. - -# RELEASE_BLOCKER: Remove this deprecated module in 18.3 -""" -Snap Config ------------ -**Summary:** snap_config modules allows configuration of snapd. - -**Deprecated**: Use :ref:`snap` module instead. This module will not exist -in cloud-init 18.3. - -This module uses the same ``snappy`` namespace for configuration but -acts only only a subset of the configuration. - -If ``assertions`` is set and the user has included a list of assertions -then cloud-init will collect the assertions into a single assertion file -and invoke ``snap ack `` which will attempt -to load the provided assertions into the snapd assertion database. - -If ``email`` is set, this value is used to create an authorized user for -contacting and installing snaps from the Ubuntu Store. This is done by -calling ``snap create-user`` command. - -If ``known`` is set to True, then it is expected the user also included -an assertion of type ``system-user``. When ``snap create-user`` is called -cloud-init will append '--known' flag which instructs snapd to look for -a system-user assertion with the details. If ``known`` is not set, then -``snap create-user`` will contact the Ubuntu SSO for validating and importing -a system-user for the instance. - -.. note:: - If the system is already managed, then cloud-init will not attempt to - create a system-user. - -**Internal name:** ``cc_snap_config`` - -**Module frequency:** per instance - -**Supported distros:** any with 'snapd' available - -**Config keys**:: - - #cloud-config - snappy: - assertions: - - | - - - | - - email: user@user.org - known: true - -""" - -from cloudinit import log as logging -from cloudinit.settings import PER_INSTANCE -from cloudinit import util - -LOG = logging.getLogger(__name__) - -frequency = PER_INSTANCE -SNAPPY_CMD = "snap" -ASSERTIONS_FILE = "/var/lib/cloud/instance/snapd.assertions" - - -""" -snappy: - assertions: - - | - - - | - - email: foo@foo.io - known: true -""" - - -def add_assertions(assertions=None): - """Import list of assertions. - - Import assertions by concatenating each assertion into a - string separated by a '\n'. Write this string to a instance file and - then invoke `snap ack /path/to/file` and check for errors. - If snap exits 0, then all assertions are imported. - """ - if not assertions: - assertions = [] - - if not isinstance(assertions, list): - raise ValueError( - 'assertion parameter was not a list: {assertions}'.format( - assertions=assertions)) - - snap_cmd = [SNAPPY_CMD, 'ack'] - combined = "\n".join(assertions) - if len(combined) == 0: - raise ValueError("Assertion list is empty") - - for asrt in assertions: - LOG.debug('Acking: %s', asrt.split('\n')[0:2]) - - util.write_file(ASSERTIONS_FILE, combined.encode('utf-8')) - util.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) - - -def add_snap_user(cfg=None): - """Add a snap system-user if provided with email under snappy config. - - - Check that system is not already managed. - - Check that if using a system-user assertion, that it's - imported into snapd. - - Returns a dictionary to be passed to Distro.create_user - """ - - if not cfg: - cfg = {} - - if not isinstance(cfg, dict): - raise ValueError( - 'configuration parameter was not a dict: {cfg}'.format(cfg=cfg)) - - snapuser = cfg.get('email', None) - if not snapuser: - return - - usercfg = { - 'snapuser': snapuser, - 'known': cfg.get('known', False), - } - - # query if we're already registered - out, _ = util.subp([SNAPPY_CMD, 'managed'], capture=True) - if out.strip() == "true": - LOG.warning('This device is already managed. ' - 'Skipping system-user creation') - return - - if usercfg.get('known'): - # Check that we imported a system-user assertion - out, _ = util.subp([SNAPPY_CMD, 'known', 'system-user'], - capture=True) - if len(out) == 0: - LOG.error('Missing "system-user" assertion. ' - 'Check "snappy" user-data assertions.') - return - - return usercfg - - -def handle(name, cfg, cloud, log, args): - cfgin = cfg.get('snappy') - if not cfgin: - LOG.debug('No snappy config provided, skipping') - return - - log.warning( - 'DEPRECATION: snap_config module will be dropped in 18.3 release.' - ' Use snap module instead') - if not(util.system_is_snappy()): - LOG.debug("%s: system not snappy", name) - return - - assertions = cfgin.get('assertions', []) - if len(assertions) > 0: - LOG.debug('Importing user-provided snap assertions') - add_assertions(assertions) - - # Create a snap user if requested. - # Snap systems contact the store with a user's email - # and extract information needed to create a local user. - # A user may provide a 'system-user' assertion which includes - # the required information. Using such an assertion to create - # a local user requires specifying 'known: true' in the supplied - # user-data. - usercfg = add_snap_user(cfg=cfgin) - if usercfg: - cloud.distro.create_user(usercfg.get('snapuser'), **usercfg) - -# vi: ts=4 expandtab diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 7aab265e..18ab0ac5 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -71,7 +71,6 @@ cloud_config_modules: # this can be used by upstart jobs for 'start on cloud-config'. - emit_upstart - snap - - snap_config # DEPRECATED- Drop in version 18.2 {% endif %} - ssh-import-id - locale diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index 1aa2f88d..4117da60 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -46,7 +46,6 @@ Modules .. automodule:: cloudinit.config.cc_set_hostname .. automodule:: cloudinit.config.cc_set_passwords .. automodule:: cloudinit.config.cc_snap -.. automodule:: cloudinit.config.cc_snap_config .. automodule:: cloudinit.config.cc_spacewalk .. automodule:: cloudinit.config.cc_ssh .. automodule:: cloudinit.config.cc_ssh_authkey_fingerprints diff --git a/tests/cloud_tests/testcases/modules/TODO.md b/tests/cloud_tests/testcases/modules/TODO.md index ee7e9213..9513cb2d 100644 --- a/tests/cloud_tests/testcases/modules/TODO.md +++ b/tests/cloud_tests/testcases/modules/TODO.md @@ -78,8 +78,8 @@ Not applicable to write a test for this as it specifies when something should be ## scripts vendor Not applicable to write a test for this as it specifies when something should be run. -## snap-config -2016-11-17: Need to investigate +## snap +2019-12-19: Need to investigate ## spacewalk -- cgit v1.2.3 From 3f6192b39894761e8ddecefa3736ea0abbc35b90 Mon Sep 17 00:00:00 2001 From: Paride Legovini Date: Fri, 10 Jan 2020 15:56:03 +0100 Subject: ssh_auth_key_fingerprints_disable test: fix capitalization (#165) Adapt the test to the new capitalization introduced in 8116493950e7c47af0ce66fc1bb5d799ce5e477a. --- .../cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/cloud_tests') diff --git a/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.py b/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.py index e7329d48..02935447 100644 --- a/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.py +++ b/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.py @@ -11,6 +11,6 @@ class TestSshKeyFingerprintsDisable(base.CloudTestCase): """Verify disabled.""" out = self.get_data_file('cloud-init.log') self.assertIn('Skipping module named ssh-authkey-fingerprints, ' - 'logging of ssh fingerprints disabled', out) + 'logging of SSH fingerprints disabled', out) # vi: ts=4 expandtab -- cgit v1.2.3 From 651e24066ba4b9464c7843553f60d17a459cf06e Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 16 Jan 2020 13:30:12 -0500 Subject: util: rename get_architecture to get_dpkg_architecture (#173) This makes it clearer that we should only use this in code paths that will definitely have dpkg available to them. - Rename get_architecture -> get_dpkg_architecture - Add docstring to get_dpkg_architecture --- cloudinit/config/cc_apt_configure.py | 6 +++--- cloudinit/distros/debian.py | 2 +- cloudinit/util.py | 7 ++++++- tests/cloud_tests/config.py | 2 +- tests/cloud_tests/platforms/nocloudkvm/platform.py | 10 +++++++--- .../test_handler_apt_configure_sources_list_v1.py | 2 +- .../test_handler_apt_configure_sources_list_v3.py | 2 +- .../test_handler/test_handler_apt_source_v1.py | 2 +- .../test_handler/test_handler_apt_source_v3.py | 20 +++++++++++--------- 9 files changed, 32 insertions(+), 21 deletions(-) (limited to 'tests/cloud_tests') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 4a3aed36..c44dec45 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -253,7 +253,7 @@ def get_default_mirrors(arch=None, target=None): architecture, for more see: https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports""" if arch is None: - arch = util.get_architecture(target) + arch = util.get_dpkg_architecture(target) if arch in PRIMARY_ARCHES: return PRIMARY_ARCH_MIRRORS.copy() if arch in PORTS_ARCHES: @@ -303,7 +303,7 @@ def apply_apt(cfg, cloud, target): LOG.debug("handling apt config: %s", cfg) release = util.lsb_release(target=target)['codename'] - arch = util.get_architecture(target) + arch = util.get_dpkg_architecture(target) mirrors = find_apt_mirror_info(cfg, cloud, arch=arch) LOG.debug("Apt Mirror info: %s", mirrors) @@ -896,7 +896,7 @@ def find_apt_mirror_info(cfg, cloud, arch=None): """ if arch is None: - arch = util.get_architecture() + arch = util.get_dpkg_architecture() LOG.debug("got arch for mirror selection: %s", arch) pmirror = get_mirror(cfg, "primary", arch, cloud) LOG.debug("got primary mirror: %s", pmirror) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 79268371..128bb523 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -205,7 +205,7 @@ class Distro(distros.Distro): ["update"], freq=PER_INSTANCE) def get_primary_arch(self): - return util.get_architecture() + return util.get_dpkg_architecture() def _get_wrapper_prefix(cmd, mode): diff --git a/cloudinit/util.py b/cloudinit/util.py index 87480767..d99e82fa 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -79,7 +79,12 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], @lru_cache() -def get_architecture(target=None): +def get_dpkg_architecture(target=None): + """Return the sanitized string output by `dpkg --print-architecture`. + + N.B. This function is wrapped in functools.lru_cache, so repeated calls + won't shell out every time. + """ out, _ = subp(['dpkg', '--print-architecture'], capture=True, target=target) return out.strip() diff --git a/tests/cloud_tests/config.py b/tests/cloud_tests/config.py index 8bd569fd..06536edc 100644 --- a/tests/cloud_tests/config.py +++ b/tests/cloud_tests/config.py @@ -114,7 +114,7 @@ def load_os_config(platform_name, os_name, require_enabled=False, feature_conf = main_conf['features'] feature_groups = conf.get('feature_groups', []) overrides = merge_config(get(conf, 'features'), feature_overrides) - conf['arch'] = c_util.get_architecture() + conf['arch'] = c_util.get_dpkg_architecture() conf['features'] = merge_feature_groups( feature_conf, feature_groups, overrides) diff --git a/tests/cloud_tests/platforms/nocloudkvm/platform.py b/tests/cloud_tests/platforms/nocloudkvm/platform.py index 85933463..2d1480f5 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/platform.py +++ b/tests/cloud_tests/platforms/nocloudkvm/platform.py @@ -29,9 +29,13 @@ class NoCloudKVMPlatform(Platform): """ (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) - filter = filters.get_filters(['arch=%s' % c_util.get_architecture(), - 'release=%s' % img_conf['release'], - 'ftype=disk1.img']) + filter = filters.get_filters( + [ + 'arch=%s' % c_util.get_dpkg_architecture(), + 'release=%s' % img_conf['release'], + 'ftype=disk1.img', + ] + ) mirror_config = {'filters': filter, 'keep_items': False, 'max_items': 1, diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py index 23bd6e10..7c17a264 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py @@ -78,7 +78,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): get_rel = rpatcher.start() get_rel.return_value = {'codename': "fakerelease"} self.addCleanup(rpatcher.stop) - apatcher = mock.patch("cloudinit.util.get_architecture") + apatcher = mock.patch("cloudinit.util.get_dpkg_architecture") get_arch = apatcher.start() get_arch.return_value = 'amd64' self.addCleanup(apatcher.stop) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py index f7608c28..0a68cb8f 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py @@ -106,7 +106,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): get_rel = rpatcher.start() get_rel.return_value = {'codename': "fakerel"} self.addCleanup(rpatcher.stop) - apatcher = mock.patch("cloudinit.util.get_architecture") + apatcher = mock.patch("cloudinit.util.get_dpkg_architecture") get_arch = apatcher.start() get_arch.return_value = 'amd64' self.addCleanup(apatcher.stop) diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py index a3132fbd..652d97ab 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py @@ -77,7 +77,7 @@ class TestAptSourceConfig(TestCase): get_rel = rpatcher.start() get_rel.return_value = {'codename': self.release} self.addCleanup(rpatcher.stop) - apatcher = mock.patch("cloudinit.util.get_architecture") + apatcher = mock.patch("cloudinit.util.get_dpkg_architecture") get_arch = apatcher.start() get_arch.return_value = 'amd64' self.addCleanup(apatcher.stop) diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py index dcffbe13..c5cf6785 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py @@ -453,14 +453,14 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): self.assertFalse(os.path.isfile(self.aptlistfile2)) self.assertFalse(os.path.isfile(self.aptlistfile3)) - @mock.patch("cloudinit.config.cc_apt_configure.util.get_architecture") - def test_apt_v3_list_rename(self, m_get_architecture): + @mock.patch("cloudinit.config.cc_apt_configure.util.get_dpkg_architecture") + def test_apt_v3_list_rename(self, m_get_dpkg_architecture): """test_apt_v3_list_rename - Test find mirror and apt list renaming""" pre = "/var/lib/apt/lists" # filenames are archive dependent arch = 's390x' - m_get_architecture.return_value = arch + m_get_dpkg_architecture.return_value = arch component = "ubuntu-ports" archive = "ports.ubuntu.com" @@ -491,13 +491,13 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): mockren.assert_any_call(fromfn, tofn) - @mock.patch("cloudinit.config.cc_apt_configure.util.get_architecture") - def test_apt_v3_list_rename_non_slash(self, m_get_architecture): + @mock.patch("cloudinit.config.cc_apt_configure.util.get_dpkg_architecture") + def test_apt_v3_list_rename_non_slash(self, m_get_dpkg_architecture): target = os.path.join(self.tmp, "rename_non_slash") apt_lists_d = os.path.join(target, "./" + cc_apt_configure.APT_LISTS) arch = 'amd64' - m_get_architecture.return_value = arch + m_get_dpkg_architecture.return_value = arch mirror_path = "some/random/path/" primary = "http://test.ubuntu.com/" + mirror_path @@ -626,10 +626,12 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): self.assertEqual(mirrors['SECURITY'], smir) - @mock.patch("cloudinit.config.cc_apt_configure.util.get_architecture") - def test_apt_v3_get_def_mir_non_intel_no_arch(self, m_get_architecture): + @mock.patch("cloudinit.config.cc_apt_configure.util.get_dpkg_architecture") + def test_apt_v3_get_def_mir_non_intel_no_arch( + self, m_get_dpkg_architecture + ): arch = 'ppc64el' - m_get_architecture.return_value = arch + m_get_dpkg_architecture.return_value = arch expected = {'PRIMARY': 'http://ports.ubuntu.com/ubuntu-ports', 'SECURITY': 'http://ports.ubuntu.com/ubuntu-ports'} self.assertEqual(expected, cc_apt_configure.get_default_mirrors()) -- cgit v1.2.3 From ecffd25df840277ab1fa7d5372659abe833cacbe Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 13 Feb 2020 14:11:17 -0600 Subject: azurecloud: fix issues with instances not starting (#205) The azurecloud platform did not always start instances during collect runs. This was a result of two issues. First the image class _instance method did not invoke the start() method which then allowed collect stage to attempt to run scripts without an endpoint. Second, azurecloud used the image_id as both an instance handle (which is typically vmName in azure api) as well as an image handle (for image capture). Resolve this by adding a .vm_name property to the AzureCloudInstance and reference this property in AzureCloudImage. Also in this branch - Fix error encoding user-data when value is None - Add additional logging in AzureCloud platform - Update logging format to print pathname,funcName and line number This greatly eases debugging. LP: #1861921 --- tests/cloud_tests/__init__.py | 3 +- tests/cloud_tests/platforms/azurecloud/image.py | 32 ++++++++++++++-------- tests/cloud_tests/platforms/azurecloud/instance.py | 15 ++++++---- tests/cloud_tests/platforms/azurecloud/platform.py | 5 ++-- tests/cloud_tests/setup_image.py | 2 +- 5 files changed, 36 insertions(+), 21 deletions(-) (limited to 'tests/cloud_tests') diff --git a/tests/cloud_tests/__init__.py b/tests/cloud_tests/__init__.py index dd436989..6c632f99 100644 --- a/tests/cloud_tests/__init__.py +++ b/tests/cloud_tests/__init__.py @@ -22,7 +22,8 @@ def _initialize_logging(): logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + '%(asctime)s - %(pathname)s:%(funcName)s:%(lineno)s ' + '[%(levelname)s]: %(message)s') console = logging.StreamHandler() console.setLevel(logging.DEBUG) diff --git a/tests/cloud_tests/platforms/azurecloud/image.py b/tests/cloud_tests/platforms/azurecloud/image.py index 96a946f3..aad2bca1 100644 --- a/tests/cloud_tests/platforms/azurecloud/image.py +++ b/tests/cloud_tests/platforms/azurecloud/image.py @@ -21,26 +21,26 @@ class AzureCloudImage(Image): @param image_id: image id used to boot instance """ super(AzureCloudImage, self).__init__(platform, config) - self.image_id = image_id self._img_instance = None + self.image_id = image_id @property def _instance(self): """Internal use only, returns a running instance""" - LOG.debug('creating instance') if not self._img_instance: self._img_instance = self.platform.create_instance( self.properties, self.config, self.features, self.image_id, user_data=None) + self._img_instance.start(wait=True, wait_for_cloud_init=True) return self._img_instance def destroy(self): """Delete the instance used to create a custom image.""" - LOG.debug('deleting VM that was used to create image') if self._img_instance: - LOG.debug('Deleting instance %s', self._img_instance.name) + LOG.debug('Deleting backing instance %s', + self._img_instance.vm_name) delete_vm = self.platform.compute_client.virtual_machines.delete( - self.platform.resource_group.name, self.image_id) + self.platform.resource_group.name, self._img_instance.vm_name) delete_vm.wait() super(AzureCloudImage, self).destroy() @@ -48,7 +48,7 @@ class AzureCloudImage(Image): def _execute(self, *args, **kwargs): """Execute command in image, modifying image.""" LOG.debug('executing commands on image') - self._instance.start() + self._instance.start(wait=True) return self._instance._execute(*args, **kwargs) def push_file(self, local_path, remote_path): @@ -72,21 +72,26 @@ class AzureCloudImage(Image): Otherwise runs the clean script, deallocates, generalizes and creates custom image from instance. """ - LOG.debug('creating image from VM') + LOG.debug('creating snapshot of image') if not self._img_instance: + LOG.debug('No existing image, snapshotting base image') return AzureCloudSnapshot(self.platform, self.properties, self.config, self.features, - self.image_id, delete_on_destroy=False) + self._instance.vm_name, + delete_on_destroy=False) + LOG.debug('creating snapshot from instance: %s', self._img_instance) if self.config.get('boot_clean_script'): self._img_instance.run_script(self.config.get('boot_clean_script')) + LOG.debug('deallocating instance %s', self._instance.vm_name) deallocate = self.platform.compute_client.virtual_machines.deallocate( - self.platform.resource_group.name, self.image_id) + self.platform.resource_group.name, self._instance.vm_name) deallocate.wait() + LOG.debug('generalizing instance %s', self._instance.vm_name) self.platform.compute_client.virtual_machines.generalize( - self.platform.resource_group.name, self.image_id) + self.platform.resource_group.name, self._instance.vm_name) image_params = { "location": self.platform.location, @@ -96,13 +101,16 @@ class AzureCloudImage(Image): } } } + LOG.debug('updating resource group image %s', self._instance.vm_name) self.platform.compute_client.images.create_or_update( - self.platform.resource_group.name, self.image_id, + self.platform.resource_group.name, self._instance.vm_name, image_params) + LOG.debug('destroying self') self.destroy() + LOG.debug('snapshot complete') return AzureCloudSnapshot(self.platform, self.properties, self.config, - self.features, self.image_id) + self.features, self._instance.vm_name) # vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py index 3d77a1a7..f1e28a96 100644 --- a/tests/cloud_tests/platforms/azurecloud/instance.py +++ b/tests/cloud_tests/platforms/azurecloud/instance.py @@ -41,6 +41,7 @@ class AzureCloudInstance(Instance): self.ssh_ip = None self.instance = None self.image_id = image_id + self.vm_name = 'ci-azure-i-%s' % self.platform.tag self.user_data = user_data self.ssh_key_file = os.path.join( platform.config['data_dir'], platform.config['private_key']) @@ -74,16 +75,18 @@ class AzureCloudInstance(Instance): self.image_id ) image_exists = True - LOG.debug('image found, launching instance') + LOG.debug('image found, launching instance, image_id=%s', + self.image_id) except CloudError: - LOG.debug( - 'image not found, launching instance with base image') + LOG.debug(('image not found, launching instance with base image, ' + 'image_id=%s'), self.image_id) pass vm_params = { + 'name': self.vm_name, 'location': self.platform.location, 'os_profile': { - 'computer_name': 'CI', + 'computer_name': 'CI-%s' % self.platform.tag, 'admin_username': self.ssh_username, "customData": self.user_data, "linuxConfiguration": { @@ -129,7 +132,9 @@ class AzureCloudInstance(Instance): try: self.instance = self.platform.compute_client.virtual_machines.\ create_or_update(self.platform.resource_group.name, - self.image_id, vm_params) + self.vm_name, vm_params) + LOG.debug('creating instance %s from image_id=%s', self.vm_name, + self.image_id) except CloudError: raise RuntimeError('failed creating instance:\n{}'.format( traceback.format_exc())) diff --git a/tests/cloud_tests/platforms/azurecloud/platform.py b/tests/cloud_tests/platforms/azurecloud/platform.py index 77f159eb..cb62a74b 100644 --- a/tests/cloud_tests/platforms/azurecloud/platform.py +++ b/tests/cloud_tests/platforms/azurecloud/platform.py @@ -74,8 +74,9 @@ class AzureCloudPlatform(Platform): @param user_data: test user-data to pass to instance @return_value: cloud_tests.instances instance """ - user_data = str(base64.b64encode( - user_data.encode('utf-8')), 'utf-8') + if user_data is not None: + user_data = str(base64.b64encode( + user_data.encode('utf-8')), 'utf-8') return AzureCloudInstance(self, properties, config, features, image_id, user_data) diff --git a/tests/cloud_tests/setup_image.py b/tests/cloud_tests/setup_image.py index a8aaba15..69e66e3f 100644 --- a/tests/cloud_tests/setup_image.py +++ b/tests/cloud_tests/setup_image.py @@ -229,7 +229,7 @@ def setup_image(args, image): except Exception as e: info = "N/A (%s)" % e - LOG.info('setting up %s (%s)', image, info) + LOG.info('setting up image %s (info %s)', image, info) res = stage.run_stage( 'set up for {}'.format(image), calls, continue_after_error=False) return res -- cgit v1.2.3