From aa3e4961ceae5a5c5b5cf13221b5f6721991fe75 Mon Sep 17 00:00:00 2001 From: ahosmanmsft Date: Tue, 26 Nov 2019 11:36:00 -0700 Subject: cloud_tests: add azure platform support to integration tests Added Azure to cloud tests supporting upstream integration testing. Implement the inherited platform classes, Azure configurations to release/platform, and docs on how to run Azure CI. --- tests/cloud_tests/platforms/azurecloud/instance.py | 243 +++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 tests/cloud_tests/platforms/azurecloud/instance.py (limited to 'tests/cloud_tests/platforms/azurecloud/instance.py') diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py new file mode 100644 index 00000000..3d77a1a7 --- /dev/null +++ b/tests/cloud_tests/platforms/azurecloud/instance.py @@ -0,0 +1,243 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base Azure Cloud instance.""" + +from datetime import datetime, timedelta +from urllib.parse import urlparse +from time import sleep +import traceback +import os + + +# pylint: disable=no-name-in-module +from azure.storage.blob import BlockBlobService, BlobPermissions +from msrestazure.azure_exceptions import CloudError + +from tests.cloud_tests import LOG + +from ..instances import Instance + + +class AzureCloudInstance(Instance): + """Azure Cloud backed instance.""" + + platform_name = 'azurecloud' + + def __init__(self, platform, properties, config, + features, image_id, user_data=None): + """Set up instance. + + @param platform: platform object + @param properties: dictionary of properties + @param config: dictionary of configuration values + @param features: dictionary of supported feature flags + @param image_id: image to find and/or use + @param user_data: test user-data to pass to instance + """ + super(AzureCloudInstance, self).__init__( + platform, image_id, properties, config, features) + + self.ssh_port = 22 + self.ssh_ip = None + self.instance = None + self.image_id = image_id + self.user_data = user_data + self.ssh_key_file = os.path.join( + platform.config['data_dir'], platform.config['private_key']) + self.ssh_pubkey_file = os.path.join( + platform.config['data_dir'], platform.config['public_key']) + self.blob_client, self.container, self.blob = None, None, None + + def start(self, wait=True, wait_for_cloud_init=False): + """Start instance with the platforms NIC.""" + if self.instance: + return + data = self.image_id.split('-') + release, support = data[2].replace('_', '.'), data[3] + sku = '%s-%s' % (release, support) if support == 'LTS' else release + image_resource_id = '/subscriptions/%s' \ + '/resourceGroups/%s' \ + '/providers/Microsoft.Compute/images/%s' % ( + self.platform.subscription_id, + self.platform.resource_group.name, + self.image_id) + storage_uri = "http://%s.blob.core.windows.net" \ + % self.platform.storage.name + with open(self.ssh_pubkey_file, 'r') as key: + ssh_pub_keydata = key.read() + + image_exists = False + try: + LOG.debug('finding image in resource group using image_id') + self.platform.compute_client.images.get( + self.platform.resource_group.name, + self.image_id + ) + image_exists = True + LOG.debug('image found, launching instance') + except CloudError: + LOG.debug( + 'image not found, launching instance with base image') + pass + + vm_params = { + 'location': self.platform.location, + 'os_profile': { + 'computer_name': 'CI', + 'admin_username': self.ssh_username, + "customData": self.user_data, + "linuxConfiguration": { + "disable_password_authentication": True, + "ssh": { + "public_keys": [{ + "path": "/home/%s/.ssh/authorized_keys" % + self.ssh_username, + "keyData": ssh_pub_keydata + }] + } + } + }, + "diagnosticsProfile": { + "bootDiagnostics": { + "storageUri": storage_uri, + "enabled": True + } + }, + 'hardware_profile': { + 'vm_size': self.platform.vm_size + }, + 'storage_profile': { + 'image_reference': { + 'id': image_resource_id + } if image_exists else { + 'publisher': 'Canonical', + 'offer': 'UbuntuServer', + 'sku': sku, + 'version': 'latest' + } + }, + 'network_profile': { + 'network_interfaces': [{ + 'id': self.platform.nic.id + }] + }, + 'tags': { + 'Name': self.platform.tag, + } + } + + try: + self.instance = self.platform.compute_client.virtual_machines.\ + create_or_update(self.platform.resource_group.name, + self.image_id, vm_params) + except CloudError: + raise RuntimeError('failed creating instance:\n{}'.format( + traceback.format_exc())) + + if wait: + self.instance.wait() + self.ssh_ip = self.platform.network_client.\ + public_ip_addresses.get( + self.platform.resource_group.name, + self.platform.public_ip.name + ).ip_address + self._wait_for_system(wait_for_cloud_init) + + self.instance = self.instance.result() + self.blob_client, self.container, self.blob =\ + self._get_blob_client() + + def shutdown(self, wait=True): + """Finds console log then stopping/deallocates VM""" + LOG.debug('waiting on console log before stopping') + attempts, exists = 5, False + while not exists and attempts: + try: + attempts -= 1 + exists = self.blob_client.get_blob_to_bytes( + self.container, self.blob) + LOG.debug('found console log') + except Exception as e: + if attempts: + LOG.debug('Unable to find console log, ' + '%s attempts remaining', attempts) + sleep(15) + else: + LOG.warning('Could not find console log: %s', e) + pass + + LOG.debug('stopping instance %s', self.image_id) + vm_deallocate = \ + self.platform.compute_client.virtual_machines.deallocate( + self.platform.resource_group.name, self.image_id) + if wait: + vm_deallocate.wait() + + def destroy(self): + """Delete VM and close all connections""" + if self.instance: + LOG.debug('destroying instance: %s', self.image_id) + vm_delete = self.platform.compute_client.virtual_machines.delete( + self.platform.resource_group.name, self.image_id) + vm_delete.wait() + + self._ssh_close() + + super(AzureCloudInstance, self).destroy() + + def _execute(self, command, stdin=None, env=None): + """Execute command on instance.""" + env_args = [] + if env: + env_args = ['env'] + ["%s=%s" for k, v in env.items()] + + return self._ssh(['sudo'] + env_args + list(command), stdin=stdin) + + def _get_blob_client(self): + """ + Use VM details to retrieve container and blob name. + Then Create blob service client for sas token to + retrieve console log. + + :return: blob service, container name, blob name + """ + LOG.debug('creating blob service for console log') + storage = self.platform.storage_client.storage_accounts.get_properties( + self.platform.resource_group.name, self.platform.storage.name) + + keys = self.platform.storage_client.storage_accounts.list_keys( + self.platform.resource_group.name, self.platform.storage.name + ).keys[0].value + + virtual_machine = self.platform.compute_client.virtual_machines.get( + self.platform.resource_group.name, self.instance.name, + expand='instanceView') + + blob_uri = virtual_machine.instance_view.boot_diagnostics.\ + serial_console_log_blob_uri + + container, blob = urlparse(blob_uri).path.split('/')[-2:] + + blob_client = BlockBlobService( + account_name=storage.name, + account_key=keys) + + sas = blob_client.generate_blob_shared_access_signature( + container_name=container, blob_name=blob, protocol='https', + expiry=datetime.utcnow() + timedelta(hours=1), + permission=BlobPermissions.READ) + + blob_client = BlockBlobService( + account_name=storage.name, + sas_token=sas) + + return blob_client, container, blob + + def console_log(self): + """Instance console. + + @return_value: bytes of this instance’s console + """ + boot_diagnostics = self.blob_client.get_blob_to_bytes( + self.container, self.blob) + return boot_diagnostics.content -- cgit v1.2.3 From ecffd25df840277ab1fa7d5372659abe833cacbe Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 13 Feb 2020 14:11:17 -0600 Subject: azurecloud: fix issues with instances not starting (#205) The azurecloud platform did not always start instances during collect runs. This was a result of two issues. First the image class _instance method did not invoke the start() method which then allowed collect stage to attempt to run scripts without an endpoint. Second, azurecloud used the image_id as both an instance handle (which is typically vmName in azure api) as well as an image handle (for image capture). Resolve this by adding a .vm_name property to the AzureCloudInstance and reference this property in AzureCloudImage. Also in this branch - Fix error encoding user-data when value is None - Add additional logging in AzureCloud platform - Update logging format to print pathname,funcName and line number This greatly eases debugging. LP: #1861921 --- tests/cloud_tests/__init__.py | 3 +- tests/cloud_tests/platforms/azurecloud/image.py | 32 ++++++++++++++-------- tests/cloud_tests/platforms/azurecloud/instance.py | 15 ++++++---- tests/cloud_tests/platforms/azurecloud/platform.py | 5 ++-- tests/cloud_tests/setup_image.py | 2 +- 5 files changed, 36 insertions(+), 21 deletions(-) (limited to 'tests/cloud_tests/platforms/azurecloud/instance.py') diff --git a/tests/cloud_tests/__init__.py b/tests/cloud_tests/__init__.py index dd436989..6c632f99 100644 --- a/tests/cloud_tests/__init__.py +++ b/tests/cloud_tests/__init__.py @@ -22,7 +22,8 @@ def _initialize_logging(): logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + '%(asctime)s - %(pathname)s:%(funcName)s:%(lineno)s ' + '[%(levelname)s]: %(message)s') console = logging.StreamHandler() console.setLevel(logging.DEBUG) diff --git a/tests/cloud_tests/platforms/azurecloud/image.py b/tests/cloud_tests/platforms/azurecloud/image.py index 96a946f3..aad2bca1 100644 --- a/tests/cloud_tests/platforms/azurecloud/image.py +++ b/tests/cloud_tests/platforms/azurecloud/image.py @@ -21,26 +21,26 @@ class AzureCloudImage(Image): @param image_id: image id used to boot instance """ super(AzureCloudImage, self).__init__(platform, config) - self.image_id = image_id self._img_instance = None + self.image_id = image_id @property def _instance(self): """Internal use only, returns a running instance""" - LOG.debug('creating instance') if not self._img_instance: self._img_instance = self.platform.create_instance( self.properties, self.config, self.features, self.image_id, user_data=None) + self._img_instance.start(wait=True, wait_for_cloud_init=True) return self._img_instance def destroy(self): """Delete the instance used to create a custom image.""" - LOG.debug('deleting VM that was used to create image') if self._img_instance: - LOG.debug('Deleting instance %s', self._img_instance.name) + LOG.debug('Deleting backing instance %s', + self._img_instance.vm_name) delete_vm = self.platform.compute_client.virtual_machines.delete( - self.platform.resource_group.name, self.image_id) + self.platform.resource_group.name, self._img_instance.vm_name) delete_vm.wait() super(AzureCloudImage, self).destroy() @@ -48,7 +48,7 @@ class AzureCloudImage(Image): def _execute(self, *args, **kwargs): """Execute command in image, modifying image.""" LOG.debug('executing commands on image') - self._instance.start() + self._instance.start(wait=True) return self._instance._execute(*args, **kwargs) def push_file(self, local_path, remote_path): @@ -72,21 +72,26 @@ class AzureCloudImage(Image): Otherwise runs the clean script, deallocates, generalizes and creates custom image from instance. """ - LOG.debug('creating image from VM') + LOG.debug('creating snapshot of image') if not self._img_instance: + LOG.debug('No existing image, snapshotting base image') return AzureCloudSnapshot(self.platform, self.properties, self.config, self.features, - self.image_id, delete_on_destroy=False) + self._instance.vm_name, + delete_on_destroy=False) + LOG.debug('creating snapshot from instance: %s', self._img_instance) if self.config.get('boot_clean_script'): self._img_instance.run_script(self.config.get('boot_clean_script')) + LOG.debug('deallocating instance %s', self._instance.vm_name) deallocate = self.platform.compute_client.virtual_machines.deallocate( - self.platform.resource_group.name, self.image_id) + self.platform.resource_group.name, self._instance.vm_name) deallocate.wait() + LOG.debug('generalizing instance %s', self._instance.vm_name) self.platform.compute_client.virtual_machines.generalize( - self.platform.resource_group.name, self.image_id) + self.platform.resource_group.name, self._instance.vm_name) image_params = { "location": self.platform.location, @@ -96,13 +101,16 @@ class AzureCloudImage(Image): } } } + LOG.debug('updating resource group image %s', self._instance.vm_name) self.platform.compute_client.images.create_or_update( - self.platform.resource_group.name, self.image_id, + self.platform.resource_group.name, self._instance.vm_name, image_params) + LOG.debug('destroying self') self.destroy() + LOG.debug('snapshot complete') return AzureCloudSnapshot(self.platform, self.properties, self.config, - self.features, self.image_id) + self.features, self._instance.vm_name) # vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py index 3d77a1a7..f1e28a96 100644 --- a/tests/cloud_tests/platforms/azurecloud/instance.py +++ b/tests/cloud_tests/platforms/azurecloud/instance.py @@ -41,6 +41,7 @@ class AzureCloudInstance(Instance): self.ssh_ip = None self.instance = None self.image_id = image_id + self.vm_name = 'ci-azure-i-%s' % self.platform.tag self.user_data = user_data self.ssh_key_file = os.path.join( platform.config['data_dir'], platform.config['private_key']) @@ -74,16 +75,18 @@ class AzureCloudInstance(Instance): self.image_id ) image_exists = True - LOG.debug('image found, launching instance') + LOG.debug('image found, launching instance, image_id=%s', + self.image_id) except CloudError: - LOG.debug( - 'image not found, launching instance with base image') + LOG.debug(('image not found, launching instance with base image, ' + 'image_id=%s'), self.image_id) pass vm_params = { + 'name': self.vm_name, 'location': self.platform.location, 'os_profile': { - 'computer_name': 'CI', + 'computer_name': 'CI-%s' % self.platform.tag, 'admin_username': self.ssh_username, "customData": self.user_data, "linuxConfiguration": { @@ -129,7 +132,9 @@ class AzureCloudInstance(Instance): try: self.instance = self.platform.compute_client.virtual_machines.\ create_or_update(self.platform.resource_group.name, - self.image_id, vm_params) + self.vm_name, vm_params) + LOG.debug('creating instance %s from image_id=%s', self.vm_name, + self.image_id) except CloudError: raise RuntimeError('failed creating instance:\n{}'.format( traceback.format_exc())) diff --git a/tests/cloud_tests/platforms/azurecloud/platform.py b/tests/cloud_tests/platforms/azurecloud/platform.py index 77f159eb..cb62a74b 100644 --- a/tests/cloud_tests/platforms/azurecloud/platform.py +++ b/tests/cloud_tests/platforms/azurecloud/platform.py @@ -74,8 +74,9 @@ class AzureCloudPlatform(Platform): @param user_data: test user-data to pass to instance @return_value: cloud_tests.instances instance """ - user_data = str(base64.b64encode( - user_data.encode('utf-8')), 'utf-8') + if user_data is not None: + user_data = str(base64.b64encode( + user_data.encode('utf-8')), 'utf-8') return AzureCloudInstance(self, properties, config, features, image_id, user_data) diff --git a/tests/cloud_tests/setup_image.py b/tests/cloud_tests/setup_image.py index a8aaba15..69e66e3f 100644 --- a/tests/cloud_tests/setup_image.py +++ b/tests/cloud_tests/setup_image.py @@ -229,7 +229,7 @@ def setup_image(args, image): except Exception as e: info = "N/A (%s)" % e - LOG.info('setting up %s (%s)', image, info) + LOG.info('setting up image %s (info %s)', image, info) res = stage.run_stage( 'set up for {}'.format(image), calls, continue_after_error=False) return res -- cgit v1.2.3