diff options
Diffstat (limited to 'tests/cloud_tests/platforms')
26 files changed, 0 insertions, 2756 deletions
diff --git a/tests/cloud_tests/platforms/__init__.py b/tests/cloud_tests/platforms/__init__.py deleted file mode 100644 index e506baa0..00000000 --- a/tests/cloud_tests/platforms/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Main init.""" - -from .ec2 import platform as ec2 -from .lxd import platform as lxd -from .nocloudkvm import platform as nocloudkvm -from .azurecloud import platform as azurecloud -from ..util import emit_dots_on_travis - -PLATFORMS = { - 'ec2': ec2.EC2Platform, - 'nocloud-kvm': nocloudkvm.NoCloudKVMPlatform, - 'lxd': lxd.LXDPlatform, - 'azurecloud': azurecloud.AzureCloudPlatform, -} - - -def get_image(platform, config): - """Get image from platform object using os_name.""" - with emit_dots_on_travis(): - return platform.get_image(config) - - -def get_instance(snapshot, *args, **kwargs): - """Get instance from snapshot.""" - return snapshot.launch(*args, **kwargs) - - -def get_platform(platform_name, config): - """Get the platform object for 'platform_name' and init.""" - platform_cls = PLATFORMS.get(platform_name) - if not platform_cls: - raise ValueError('invalid platform name: {}'.format(platform_name)) - return platform_cls(config) - - -def get_snapshot(image): - """Get snapshot from image.""" - return image.snapshot() - - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/azurecloud/__init__.py b/tests/cloud_tests/platforms/azurecloud/__init__.py deleted file mode 100644 index e69de29b..00000000 --- a/tests/cloud_tests/platforms/azurecloud/__init__.py +++ /dev/null diff --git a/tests/cloud_tests/platforms/azurecloud/image.py b/tests/cloud_tests/platforms/azurecloud/image.py deleted file mode 100644 index aad2bca1..00000000 --- a/tests/cloud_tests/platforms/azurecloud/image.py +++ /dev/null @@ -1,116 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Azure Cloud image Base class.""" - -from tests.cloud_tests import LOG - -from ..images import Image -from .snapshot import AzureCloudSnapshot - - -class AzureCloudImage(Image): - """Azure Cloud backed image.""" - - platform_name = 'azurecloud' - - def __init__(self, platform, config, image_id): - """Set up image. - - @param platform: platform object - @param config: image configuration - @param image_id: image id used to boot instance - """ - super(AzureCloudImage, self).__init__(platform, config) - self._img_instance = None - self.image_id = image_id - - @property - def _instance(self): - """Internal use only, returns a running instance""" - if not self._img_instance: - self._img_instance = self.platform.create_instance( - self.properties, self.config, self.features, - self.image_id, user_data=None) - self._img_instance.start(wait=True, wait_for_cloud_init=True) - return self._img_instance - - def destroy(self): - """Delete the instance used to create a custom image.""" - if self._img_instance: - LOG.debug('Deleting backing instance %s', - self._img_instance.vm_name) - delete_vm = self.platform.compute_client.virtual_machines.delete( - self.platform.resource_group.name, self._img_instance.vm_name) - delete_vm.wait() - - super(AzureCloudImage, self).destroy() - - def _execute(self, *args, **kwargs): - """Execute command in image, modifying image.""" - LOG.debug('executing commands on image') - self._instance.start(wait=True) - return self._instance._execute(*args, **kwargs) - - def push_file(self, local_path, remote_path): - """Copy file at 'local_path' to instance at 'remote_path'.""" - LOG.debug('pushing file to image') - return self._instance.push_file(local_path, remote_path) - - def run_script(self, *args, **kwargs): - """Run script in image, modifying image. - - @return_value: script output - """ - LOG.debug('running script on image') - self._instance.start() - return self._instance.run_script(*args, **kwargs) - - def snapshot(self): - """ Create snapshot (image) of instance, wait until done. - - If no instance has been booted, base image is returned. - Otherwise runs the clean script, deallocates, generalizes - and creates custom image from instance. - """ - LOG.debug('creating snapshot of image') - if not self._img_instance: - LOG.debug('No existing image, snapshotting base image') - return AzureCloudSnapshot(self.platform, self.properties, - self.config, self.features, - self._instance.vm_name, - delete_on_destroy=False) - - LOG.debug('creating snapshot from instance: %s', self._img_instance) - if self.config.get('boot_clean_script'): - self._img_instance.run_script(self.config.get('boot_clean_script')) - - LOG.debug('deallocating instance %s', self._instance.vm_name) - deallocate = self.platform.compute_client.virtual_machines.deallocate( - self.platform.resource_group.name, self._instance.vm_name) - deallocate.wait() - - LOG.debug('generalizing instance %s', self._instance.vm_name) - self.platform.compute_client.virtual_machines.generalize( - self.platform.resource_group.name, self._instance.vm_name) - - image_params = { - "location": self.platform.location, - "properties": { - "sourceVirtualMachine": { - "id": self._img_instance.instance.id - } - } - } - LOG.debug('updating resource group image %s', self._instance.vm_name) - self.platform.compute_client.images.create_or_update( - self.platform.resource_group.name, self._instance.vm_name, - image_params) - - LOG.debug('destroying self') - self.destroy() - - LOG.debug('snapshot complete') - return AzureCloudSnapshot(self.platform, self.properties, self.config, - self.features, self._instance.vm_name) - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py deleted file mode 100644 index eedbaae8..00000000 --- a/tests/cloud_tests/platforms/azurecloud/instance.py +++ /dev/null @@ -1,247 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base Azure Cloud instance.""" - -from datetime import datetime, timedelta -from urllib.parse import urlparse -from time import sleep -import traceback -import os - - -# pylint: disable=no-name-in-module -from azure.storage.blob import BlockBlobService, BlobPermissions -from msrestazure.azure_exceptions import CloudError - -from tests.cloud_tests import LOG - -from ..instances import Instance - - -class AzureCloudInstance(Instance): - """Azure Cloud backed instance.""" - - platform_name = 'azurecloud' - - def __init__(self, platform, properties, config, - features, image_id, user_data=None): - """Set up instance. - - @param platform: platform object - @param properties: dictionary of properties - @param config: dictionary of configuration values - @param features: dictionary of supported feature flags - @param image_id: image to find and/or use - @param user_data: test user-data to pass to instance - """ - super(AzureCloudInstance, self).__init__( - platform, image_id, properties, config, features) - - self.ssh_port = 22 - self.ssh_ip = None - self.instance = None - self.image_id = image_id - self.vm_name = 'ci-azure-i-%s' % self.platform.tag - self.user_data = user_data - self.ssh_key_file = os.path.join( - platform.config['data_dir'], platform.config['private_key']) - self.ssh_pubkey_file = os.path.join( - platform.config['data_dir'], platform.config['public_key']) - self.blob_client, self.container, self.blob = None, None, None - - def start(self, wait=True, wait_for_cloud_init=False): - """Start instance with the platforms NIC.""" - if self.instance: - return - data = self.image_id.split('-') - release, support = data[2].replace('_', '.'), data[3] - sku = '%s-%s' % (release, support) if support == 'LTS' else release - image_resource_id = '/subscriptions/%s' \ - '/resourceGroups/%s' \ - '/providers/Microsoft.Compute/images/%s' % ( - self.platform.subscription_id, - self.platform.resource_group.name, - self.image_id) - storage_uri = "http://%s.blob.core.windows.net" \ - % self.platform.storage.name - with open(self.ssh_pubkey_file, 'r') as key: - ssh_pub_keydata = key.read() - - image_exists = False - try: - LOG.debug('finding image in resource group using image_id') - self.platform.compute_client.images.get( - self.platform.resource_group.name, - self.image_id - ) - image_exists = True - LOG.debug('image found, launching instance, image_id=%s', - self.image_id) - except CloudError: - LOG.debug(('image not found, launching instance with base image, ' - 'image_id=%s'), self.image_id) - - vm_params = { - 'name': self.vm_name, - 'location': self.platform.location, - 'os_profile': { - 'computer_name': 'CI-%s' % self.platform.tag, - 'admin_username': self.ssh_username, - "customData": self.user_data, - "linuxConfiguration": { - "disable_password_authentication": True, - "ssh": { - "public_keys": [{ - "path": "/home/%s/.ssh/authorized_keys" % - self.ssh_username, - "keyData": ssh_pub_keydata - }] - } - } - }, - "diagnosticsProfile": { - "bootDiagnostics": { - "storageUri": storage_uri, - "enabled": True - } - }, - 'hardware_profile': { - 'vm_size': self.platform.vm_size - }, - 'storage_profile': { - 'image_reference': { - 'id': image_resource_id - } if image_exists else { - 'publisher': 'Canonical', - 'offer': 'UbuntuServer', - 'sku': sku, - 'version': 'latest' - } - }, - 'network_profile': { - 'network_interfaces': [{ - 'id': self.platform.nic.id - }] - }, - 'tags': { - 'Name': self.platform.tag, - } - } - - try: - self.instance = self.platform.compute_client.virtual_machines.\ - create_or_update(self.platform.resource_group.name, - self.vm_name, vm_params) - LOG.debug('creating instance %s from image_id=%s', self.vm_name, - self.image_id) - except CloudError as e: - raise RuntimeError( - 'failed creating instance:\n{}'.format(traceback.format_exc()) - ) from e - - if wait: - self.instance.wait() - self.ssh_ip = self.platform.network_client.\ - public_ip_addresses.get( - self.platform.resource_group.name, - self.platform.public_ip.name - ).ip_address - self._wait_for_system(wait_for_cloud_init) - - self.instance = self.instance.result() - self.blob_client, self.container, self.blob =\ - self._get_blob_client() - - def shutdown(self, wait=True): - """Finds console log then stopping/deallocates VM""" - LOG.debug('waiting on console log before stopping') - attempts, exists = 5, False - while not exists and attempts: - try: - attempts -= 1 - exists = self.blob_client.get_blob_to_bytes( - self.container, self.blob) - LOG.debug('found console log') - except Exception as e: - if attempts: - LOG.debug('Unable to find console log, ' - '%s attempts remaining', attempts) - sleep(15) - else: - LOG.warning('Could not find console log: %s', e) - - LOG.debug('stopping instance %s', self.image_id) - vm_deallocate = \ - self.platform.compute_client.virtual_machines.deallocate( - self.platform.resource_group.name, self.image_id) - if wait: - vm_deallocate.wait() - - def destroy(self): - """Delete VM and close all connections""" - if self.instance: - LOG.debug('destroying instance: %s', self.image_id) - vm_delete = self.platform.compute_client.virtual_machines.delete( - self.platform.resource_group.name, self.image_id) - vm_delete.wait() - - self._ssh_close() - - super(AzureCloudInstance, self).destroy() - - def _execute(self, command, stdin=None, env=None): - """Execute command on instance.""" - env_args = [] - if env: - env_args = ['env'] + ["%s=%s" for k, v in env.items()] - - return self._ssh(['sudo'] + env_args + list(command), stdin=stdin) - - def _get_blob_client(self): - """ - Use VM details to retrieve container and blob name. - Then Create blob service client for sas token to - retrieve console log. - - :return: blob service, container name, blob name - """ - LOG.debug('creating blob service for console log') - storage = self.platform.storage_client.storage_accounts.get_properties( - self.platform.resource_group.name, self.platform.storage.name) - - keys = self.platform.storage_client.storage_accounts.list_keys( - self.platform.resource_group.name, self.platform.storage.name - ).keys[0].value - - virtual_machine = self.platform.compute_client.virtual_machines.get( - self.platform.resource_group.name, self.instance.name, - expand='instanceView') - - blob_uri = virtual_machine.instance_view.boot_diagnostics.\ - serial_console_log_blob_uri - - container, blob = urlparse(blob_uri).path.split('/')[-2:] - - blob_client = BlockBlobService( - account_name=storage.name, - account_key=keys) - - sas = blob_client.generate_blob_shared_access_signature( - container_name=container, blob_name=blob, protocol='https', - expiry=datetime.utcnow() + timedelta(hours=1), - permission=BlobPermissions.READ) - - blob_client = BlockBlobService( - account_name=storage.name, - sas_token=sas) - - return blob_client, container, blob - - def console_log(self): - """Instance console. - - @return_value: bytes of this instance’s console - """ - boot_diagnostics = self.blob_client.get_blob_to_bytes( - self.container, self.blob) - return boot_diagnostics.content diff --git a/tests/cloud_tests/platforms/azurecloud/platform.py b/tests/cloud_tests/platforms/azurecloud/platform.py deleted file mode 100644 index a664f612..00000000 --- a/tests/cloud_tests/platforms/azurecloud/platform.py +++ /dev/null @@ -1,240 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base Azure Cloud class.""" - -import os -import base64 -import traceback -from datetime import datetime -from tests.cloud_tests import LOG - -# pylint: disable=no-name-in-module -from azure.common.credentials import ServicePrincipalCredentials -# pylint: disable=no-name-in-module -from azure.mgmt.resource import ResourceManagementClient -# pylint: disable=no-name-in-module -from azure.mgmt.network import NetworkManagementClient -# pylint: disable=no-name-in-module -from azure.mgmt.compute import ComputeManagementClient -# pylint: disable=no-name-in-module -from azure.mgmt.storage import StorageManagementClient -from msrestazure.azure_exceptions import CloudError - -from .image import AzureCloudImage -from .instance import AzureCloudInstance -from ..platforms import Platform - -from cloudinit import util as c_util - - -class AzureCloudPlatform(Platform): - """Azure Cloud test platforms.""" - - platform_name = 'azurecloud' - - def __init__(self, config): - """Set up platform.""" - super(AzureCloudPlatform, self).__init__(config) - self.tag = '%s-%s' % ( - config['tag'], datetime.now().strftime('%Y%m%d%H%M%S')) - self.storage_sku = config['storage_sku'] - self.vm_size = config['vm_size'] - self.location = config['region'] - - try: - self.credentials, self.subscription_id = self._get_credentials() - - self.resource_client = ResourceManagementClient( - self.credentials, self.subscription_id) - self.compute_client = ComputeManagementClient( - self.credentials, self.subscription_id) - self.network_client = NetworkManagementClient( - self.credentials, self.subscription_id) - self.storage_client = StorageManagementClient( - self.credentials, self.subscription_id) - - self.resource_group = self._create_resource_group() - self.public_ip = self._create_public_ip_address() - self.storage = self._create_storage_account(config) - self.vnet = self._create_vnet() - self.subnet = self._create_subnet() - self.nic = self._create_nic() - except CloudError as e: - raise RuntimeError( - 'failed creating a resource:\n{}'.format( - traceback.format_exc() - ) - ) from e - - def create_instance(self, properties, config, features, - image_id, user_data=None): - """Create an instance - - @param properties: image properties - @param config: image configuration - @param features: image features - @param image_id: string of image id - @param user_data: test user-data to pass to instance - @return_value: cloud_tests.instances instance - """ - if user_data is not None: - user_data = str(base64.b64encode( - user_data.encode('utf-8')), 'utf-8') - - return AzureCloudInstance(self, properties, config, features, - image_id, user_data) - - def get_image(self, img_conf): - """Get image using specified image configuration. - - @param img_conf: configuration for image - @return_value: cloud_tests.images instance - """ - ss_region = self.azure_location_to_simplestreams_region() - - filters = [ - 'arch=%s' % 'amd64', - 'endpoint=https://management.core.windows.net/', - 'region=%s' % ss_region, - 'release=%s' % img_conf['release'] - ] - - LOG.debug('finding image using streams') - image = self._query_streams(img_conf, filters) - - try: - image_id = image['id'] - LOG.debug('found image: %s', image_id) - if image_id.find('__') > 0: - image_id = image_id.split('__')[1] - LOG.debug('image_id shortened to %s', image_id) - except KeyError as e: - raise RuntimeError( - 'no images found for %s' % img_conf['release'] - ) from e - - return AzureCloudImage(self, img_conf, image_id) - - def destroy(self): - """Delete all resources in resource group.""" - LOG.debug("Deleting resource group: %s", self.resource_group.name) - delete = self.resource_client.resource_groups.delete( - self.resource_group.name) - delete.wait() - - def azure_location_to_simplestreams_region(self): - """Convert location to simplestreams region""" - location = self.location.lower().replace(' ', '') - LOG.debug('finding location %s using simple streams', location) - regions_file = os.path.join( - os.path.dirname(os.path.abspath(__file__)), 'regions.json') - region_simplestreams_map = c_util.load_json( - c_util.load_file(regions_file)) - return region_simplestreams_map.get(location, location) - - def _get_credentials(self): - """Get credentials from environment""" - LOG.debug('getting credentials from environment') - cred_file = os.path.expanduser('~/.azure/credentials.json') - try: - azure_creds = c_util.load_json( - c_util.load_file(cred_file)) - subscription_id = azure_creds['subscriptionId'] - credentials = ServicePrincipalCredentials( - client_id=azure_creds['clientId'], - secret=azure_creds['clientSecret'], - tenant=azure_creds['tenantId']) - return credentials, subscription_id - except KeyError as e: - raise RuntimeError( - 'Please configure Azure service principal' - ' credentials in %s' % cred_file - ) from e - - def _create_resource_group(self): - """Create resource group""" - LOG.debug('creating resource group') - resource_group_name = self.tag - resource_group_params = { - 'location': self.location - } - resource_group = self.resource_client.resource_groups.create_or_update( - resource_group_name, resource_group_params) - return resource_group - - def _create_storage_account(self, config): - LOG.debug('creating storage account') - storage_account_name = 'storage%s' % datetime.now().\ - strftime('%Y%m%d%H%M%S') - storage_params = { - 'sku': { - 'name': config['storage_sku'] - }, - 'kind': "Storage", - 'location': self.location - } - storage_account = self.storage_client.storage_accounts.create( - self.resource_group.name, storage_account_name, storage_params) - return storage_account.result() - - def _create_public_ip_address(self): - """Create public ip address""" - LOG.debug('creating public ip address') - public_ip_name = '%s-ip' % self.resource_group.name - public_ip_params = { - 'location': self.location, - 'public_ip_allocation_method': 'Dynamic' - } - ip = self.network_client.public_ip_addresses.create_or_update( - self.resource_group.name, public_ip_name, public_ip_params) - return ip.result() - - def _create_vnet(self): - """create virtual network""" - LOG.debug('creating vnet') - vnet_name = '%s-vnet' % self.resource_group.name - vnet_params = { - 'location': self.location, - 'address_space': { - 'address_prefixes': ['10.0.0.0/16'] - } - } - vnet = self.network_client.virtual_networks.create_or_update( - self.resource_group.name, vnet_name, vnet_params) - return vnet.result() - - def _create_subnet(self): - """create sub-network""" - LOG.debug('creating subnet') - subnet_name = '%s-subnet' % self.resource_group.name - subnet_params = { - 'address_prefix': '10.0.0.0/24' - } - subnet = self.network_client.subnets.create_or_update( - self.resource_group.name, self.vnet.name, - subnet_name, subnet_params) - return subnet.result() - - def _create_nic(self): - """Create network interface controller""" - LOG.debug('creating nic') - nic_name = '%s-nic' % self.resource_group.name - nic_params = { - 'location': self.location, - 'ip_configurations': [{ - 'name': 'ipconfig', - 'subnet': { - 'id': self.subnet.id - }, - 'publicIpAddress': { - 'id': "/subscriptions/%s" - "/resourceGroups/%s/providers/Microsoft.Network" - "/publicIPAddresses/%s" % ( - self.subscription_id, self.resource_group.name, - self.public_ip.name), - } - }] - } - nic = self.network_client.network_interfaces.create_or_update( - self.resource_group.name, nic_name, nic_params) - return nic.result() diff --git a/tests/cloud_tests/platforms/azurecloud/regions.json b/tests/cloud_tests/platforms/azurecloud/regions.json deleted file mode 100644 index c1b4da20..00000000 --- a/tests/cloud_tests/platforms/azurecloud/regions.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "eastasia": "East Asia", - "southeastasia": "Southeast Asia", - "centralus": "Central US", - "eastus": "East US", - "eastus2": "East US 2", - "westus": "West US", - "northcentralus": "North Central US", - "southcentralus": "South Central US", - "northeurope": "North Europe", - "westeurope": "West Europe", - "japanwest": "Japan West", - "japaneast": "Japan East", - "brazilsouth": "Brazil South", - "australiaeast": "Australia East", - "australiasoutheast": "Australia Southeast", - "southindia": "South India", - "centralindia": "Central India", - "westindia": "West India", - "canadacentral": "Canada Central", - "canadaeast": "Canada East", - "uksouth": "UK South", - "ukwest": "UK West", - "westcentralus": "West Central US", - "westus2": "West US 2", - "koreacentral": "Korea Central", - "koreasouth": "Korea South", - "francecentral": "France Central", - "francesouth": "France South", - "australiacentral": "Australia Central", - "australiacentral2": "Australia Central 2", - "uaecentral": "UAE Central", - "uaenorth": "UAE North", - "southafricanorth": "South Africa North", - "southafricawest": "South Africa West", - "switzerlandnorth": "Switzerland North", - "switzerlandwest": "Switzerland West", - "germanynorth": "Germany North", - "germanywestcentral": "Germany West Central", - "norwaywest": "Norway West", - "norwayeast": "Norway East" -} diff --git a/tests/cloud_tests/platforms/azurecloud/snapshot.py b/tests/cloud_tests/platforms/azurecloud/snapshot.py deleted file mode 100644 index 580cc596..00000000 --- a/tests/cloud_tests/platforms/azurecloud/snapshot.py +++ /dev/null @@ -1,58 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base Azure Cloud snapshot.""" - -from ..snapshots import Snapshot - -from tests.cloud_tests import LOG - - -class AzureCloudSnapshot(Snapshot): - """Azure Cloud image copy backed snapshot.""" - - platform_name = 'azurecloud' - - def __init__(self, platform, properties, config, features, image_id, - delete_on_destroy=True): - """Set up snapshot. - - @param platform: platform object - @param properties: image properties - @param config: image config - @param features: supported feature flags - """ - super(AzureCloudSnapshot, self).__init__( - platform, properties, config, features) - - self.image_id = image_id - self.delete_on_destroy = delete_on_destroy - - def launch(self, user_data, meta_data=None, block=True, start=True, - use_desc=None): - """Launch instance. - - @param user_data: user-data for the instance - @param meta_data: meta_data for the instance - @param block: wait until instance is created - @param start: start instance and wait until fully started - @param use_desc: description of snapshot instance use - @return_value: an Instance - """ - if meta_data is not None: - raise ValueError("metadata not supported on Azure Cloud tests") - - instance = self.platform.create_instance( - self.properties, self.config, self.features, - self.image_id, user_data) - - return instance - - def destroy(self): - """Clean up snapshot data.""" - LOG.debug('destroying image %s', self.image_id) - if self.delete_on_destroy: - self.platform.compute_client.images.delete( - self.platform.resource_group.name, - self.image_id) - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/ec2/__init__.py b/tests/cloud_tests/platforms/ec2/__init__.py deleted file mode 100644 index e69de29b..00000000 --- a/tests/cloud_tests/platforms/ec2/__init__.py +++ /dev/null diff --git a/tests/cloud_tests/platforms/ec2/image.py b/tests/cloud_tests/platforms/ec2/image.py deleted file mode 100644 index d7b2c908..00000000 --- a/tests/cloud_tests/platforms/ec2/image.py +++ /dev/null @@ -1,100 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""EC2 Image Base Class.""" - -from ..images import Image -from .snapshot import EC2Snapshot - -from tests.cloud_tests import LOG - - -class EC2Image(Image): - """EC2 backed image.""" - - platform_name = 'ec2' - - def __init__(self, platform, config, image_ami): - """Set up image. - - @param platform: platform object - @param config: image configuration - @param image_ami: string of image ami ID - """ - super(EC2Image, self).__init__(platform, config) - self._img_instance = None - self.image_ami = image_ami - - @property - def _instance(self): - """Internal use only, returns a running instance""" - if not self._img_instance: - self._img_instance = self.platform.create_instance( - self.properties, self.config, self.features, - self.image_ami, user_data=None) - self._img_instance.start(wait=True, wait_for_cloud_init=True) - return self._img_instance - - def destroy(self): - """Delete the instance used to create a custom image.""" - if self._img_instance: - LOG.debug('terminating backing instance %s', - self._img_instance.instance.instance_id) - self._img_instance.instance.terminate() - self._img_instance.instance.wait_until_terminated() - - super(EC2Image, self).destroy() - - def _execute(self, *args, **kwargs): - """Execute command in image, modifying image.""" - self._instance.start(wait=True) - return self._instance._execute(*args, **kwargs) - - def push_file(self, local_path, remote_path): - """Copy file at 'local_path' to instance at 'remote_path'.""" - self._instance.start(wait=True) - return self._instance.push_file(local_path, remote_path) - - def run_script(self, *args, **kwargs): - """Run script in image, modifying image. - - @return_value: script output - """ - self._instance.start(wait=True) - return self._instance.run_script(*args, **kwargs) - - def snapshot(self): - """Create snapshot of image, block until done. - - Will return base image_ami if no instance has been booted, otherwise - will run the clean script, shutdown the instance, create a custom - AMI, and use that AMI once available. - """ - if not self._img_instance: - return EC2Snapshot(self.platform, self.properties, self.config, - self.features, self.image_ami, - delete_on_destroy=False) - - if self.config.get('boot_clean_script'): - self._img_instance.run_script(self.config.get('boot_clean_script')) - - self._img_instance.shutdown(wait=True) - - LOG.debug('creating custom ami from instance %s', - self._img_instance.instance.instance_id) - response = self.platform.ec2_client.create_image( - Name='%s-%s' % (self.platform.tag, self.image_ami), - InstanceId=self._img_instance.instance.instance_id - ) - image_ami_edited = response['ImageId'] - - # Create image and wait until it is in the 'available' state - image = self.platform.ec2_resource.Image(image_ami_edited) - image.wait_until_exists() - waiter = self.platform.ec2_client.get_waiter('image_available') - waiter.wait(ImageIds=[image.id]) - image.reload() - - return EC2Snapshot(self.platform, self.properties, self.config, - self.features, image_ami_edited) - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/ec2/instance.py b/tests/cloud_tests/platforms/ec2/instance.py deleted file mode 100644 index d2e84047..00000000 --- a/tests/cloud_tests/platforms/ec2/instance.py +++ /dev/null @@ -1,132 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base EC2 instance.""" -import os - -import botocore - -from ..instances import Instance -from tests.cloud_tests import LOG, util - - -class EC2Instance(Instance): - """EC2 backed instance.""" - - platform_name = "ec2" - _ssh_client = None - - def __init__(self, platform, properties, config, features, - image_ami, user_data=None): - """Set up instance. - - @param platform: platform object - @param properties: dictionary of properties - @param config: dictionary of configuration values - @param features: dictionary of supported feature flags - @param image_ami: AWS AMI ID for image to use - @param user_data: test user-data to pass to instance - """ - super(EC2Instance, self).__init__( - platform, image_ami, properties, config, features) - - self.image_ami = image_ami - self.instance = None - self.user_data = user_data - self.ssh_ip = None - self.ssh_port = 22 - self.ssh_key_file = os.path.join( - platform.config['data_dir'], platform.config['private_key']) - self.ssh_pubkey_file = os.path.join( - platform.config['data_dir'], platform.config['public_key']) - - def console_log(self): - """Collect console log from instance. - - The console log is buffered and not always present, therefore - may return empty string. - """ - try: - # OutputBytes comes from platform._decode_console_output_as_bytes - response = self.instance.console_output() - return response['OutputBytes'] - except KeyError as e: - if 'Output' in response: - msg = ("'OutputBytes' did not exist in console_output() but " - "'Output' did: %s..." % response['Output'][0:128]) - raise util.PlatformError('console_log', msg) from e - return ('No Console Output [%s]' % self.instance).encode() - - def destroy(self): - """Clean up instance.""" - if self.instance: - LOG.debug('destroying instance %s', self.instance.id) - self.instance.terminate() - self.instance.wait_until_terminated() - - self._ssh_close() - - super(EC2Instance, self).destroy() - - def _execute(self, command, stdin=None, env=None): - """Execute command on instance.""" - env_args = [] - if env: - env_args = ['env'] + ["%s=%s" for k, v in env.items()] - - return self._ssh(['sudo'] + env_args + list(command), stdin=stdin) - - def start(self, wait=True, wait_for_cloud_init=False): - """Start instance on EC2 with the platfrom's VPC.""" - if self.instance: - if self.instance.state['Name'] == 'running': - return - - LOG.debug('starting instance %s', self.instance.id) - self.instance.start() - else: - LOG.debug('launching instance') - - args = { - 'ImageId': self.image_ami, - 'InstanceType': self.platform.instance_type, - 'KeyName': self.platform.key_name, - 'MaxCount': 1, - 'MinCount': 1, - 'SecurityGroupIds': [self.platform.security_group.id], - 'SubnetId': self.platform.subnet.id, - 'TagSpecifications': [{ - 'ResourceType': 'instance', - 'Tags': [{ - 'Key': 'Name', 'Value': self.platform.tag - }] - }], - } - - if self.user_data: - args['UserData'] = self.user_data - - try: - instances = self.platform.ec2_resource.create_instances(**args) - except botocore.exceptions.ClientError as error: - error_msg = error.response['Error']['Message'] - raise util.PlatformError('start', error_msg) - - self.instance = instances[0] - - LOG.debug('instance id: %s', self.instance.id) - if wait: - self.instance.wait_until_running() - self.instance.reload() - self.ssh_ip = self.instance.public_ip_address - self._wait_for_system(wait_for_cloud_init) - - def shutdown(self, wait=True): - """Shutdown instance.""" - LOG.debug('stopping instance %s', self.instance.id) - self.instance.stop() - - if wait: - self.instance.wait_until_stopped() - self.instance.reload() - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py deleted file mode 100644 index b61a2ffb..00000000 --- a/tests/cloud_tests/platforms/ec2/platform.py +++ /dev/null @@ -1,263 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base EC2 platform.""" -from datetime import datetime -import os - -import boto3 -import botocore -from botocore import session, handlers -import base64 - -from ..platforms import Platform -from .image import EC2Image -from .instance import EC2Instance -from tests.cloud_tests import LOG - - -class EC2Platform(Platform): - """EC2 test platform.""" - - platform_name = 'ec2' - ipv4_cidr = '192.168.1.0/20' - - def __init__(self, config): - """Set up platform.""" - super(EC2Platform, self).__init__(config) - # Used for unique VPC, SSH key, and custom AMI generation naming - self.tag = '%s-%s' % ( - config['tag'], datetime.now().strftime('%Y%m%d%H%M%S')) - self.instance_type = config['instance-type'] - - try: - b3session = get_session() - self.ec2_client = b3session.client('ec2') - self.ec2_resource = b3session.resource('ec2') - self.ec2_region = b3session.region_name - self.key_name = self._upload_public_key(config) - except botocore.exceptions.NoRegionError as e: - raise RuntimeError( - 'Please configure default region in $HOME/.aws/config' - ) from e - except botocore.exceptions.NoCredentialsError as e: - raise RuntimeError( - 'Please configure ec2 credentials in $HOME/.aws/credentials' - ) from e - - self.vpc = self._create_vpc() - self.internet_gateway = self._create_internet_gateway() - self.subnet = self._create_subnet() - self.routing_table = self._create_routing_table() - self.security_group = self._create_security_group() - - def create_instance(self, properties, config, features, - image_ami, user_data=None): - """Create an instance - - @param src_img_path: image path to launch from - @param properties: image properties - @param config: image configuration - @param features: image features - @param image_ami: string of image ami ID - @param user_data: test user-data to pass to instance - @return_value: cloud_tests.instances instance - """ - return EC2Instance(self, properties, config, features, - image_ami, user_data) - - def destroy(self): - """Delete SSH keys, terminate all instances, and delete VPC.""" - for instance in self.vpc.instances.all(): - LOG.debug('waiting for instance %s termination', instance.id) - instance.terminate() - instance.wait_until_terminated() - - if self.key_name: - LOG.debug('deleting SSH key %s', self.key_name) - self.ec2_client.delete_key_pair(KeyName=self.key_name) - - if self.security_group: - LOG.debug('deleting security group %s', self.security_group.id) - self.security_group.delete() - - if self.subnet: - LOG.debug('deleting subnet %s', self.subnet.id) - self.subnet.delete() - - if self.routing_table: - LOG.debug('deleting routing table %s', self.routing_table.id) - self.routing_table.delete() - - if self.internet_gateway: - LOG.debug('deleting internet gateway %s', self.internet_gateway.id) - self.internet_gateway.detach_from_vpc(VpcId=self.vpc.id) - self.internet_gateway.delete() - - if self.vpc: - LOG.debug('deleting vpc %s', self.vpc.id) - self.vpc.delete() - - def get_image(self, img_conf): - """Get image using specified image configuration. - - Hard coded for 'amd64' based images. - - @param img_conf: configuration for image - @return_value: cloud_tests.images instance - """ - if img_conf['root-store'] == 'ebs': - root_store = 'ssd' - elif img_conf['root-store'] == 'instance-store': - root_store = 'instance' - else: - raise RuntimeError('Unknown root-store type: %s' % - (img_conf['root-store'])) - - filters = [ - 'arch=%s' % 'amd64', - 'endpoint=https://ec2.%s.amazonaws.com' % self.ec2_region, - 'region=%s' % self.ec2_region, - 'release=%s' % img_conf['release'], - 'root_store=%s' % root_store, - 'virt=hvm', - ] - - LOG.debug('finding image using streams') - image = self._query_streams(img_conf, filters) - - try: - image_ami = image['id'] - except KeyError as e: - raise RuntimeError( - 'No images found for %s!' % img_conf['release'] - ) from e - - LOG.debug('found image: %s', image_ami) - image = EC2Image(self, img_conf, image_ami) - return image - - def _create_internet_gateway(self): - """Create Internet Gateway and assign to VPC.""" - LOG.debug('creating internet gateway') - # pylint: disable=no-member - internet_gateway = self.ec2_resource.create_internet_gateway() - internet_gateway.attach_to_vpc(VpcId=self.vpc.id) - self._tag_resource(internet_gateway) - - return internet_gateway - - def _create_routing_table(self): - """Update default routing table with internet gateway. - - This sets up internet access between the VPC via the internet gateway - by configuring routing tables for IPv4 and IPv6. - """ - LOG.debug('creating routing table') - route_table = self.vpc.create_route_table() - route_table.create_route(DestinationCidrBlock='0.0.0.0/0', - GatewayId=self.internet_gateway.id) - route_table.create_route(DestinationIpv6CidrBlock='::/0', - GatewayId=self.internet_gateway.id) - route_table.associate_with_subnet(SubnetId=self.subnet.id) - self._tag_resource(route_table) - - return route_table - - def _create_security_group(self): - """Enables ingress to default VPC security group.""" - LOG.debug('creating security group') - security_group = self.vpc.create_security_group( - GroupName=self.tag, Description='integration test security group') - security_group.authorize_ingress( - IpProtocol='-1', FromPort=-1, ToPort=-1, CidrIp='0.0.0.0/0') - self._tag_resource(security_group) - - return security_group - - def _create_subnet(self): - """Generate IPv4 and IPv6 subnets for use.""" - ipv6_cidr = self.vpc.ipv6_cidr_block_association_set[0][ - 'Ipv6CidrBlock'][:-2] + '64' - - LOG.debug('creating subnet with following ranges:') - LOG.debug('ipv4: %s', self.ipv4_cidr) - LOG.debug('ipv6: %s', ipv6_cidr) - subnet = self.vpc.create_subnet(CidrBlock=self.ipv4_cidr, - Ipv6CidrBlock=ipv6_cidr) - modify_subnet = subnet.meta.client.modify_subnet_attribute - modify_subnet(SubnetId=subnet.id, - MapPublicIpOnLaunch={'Value': True}) - self._tag_resource(subnet) - - return subnet - - def _create_vpc(self): - """Setup AWS EC2 VPC or return existing VPC.""" - LOG.debug('creating new vpc') - try: - vpc = self.ec2_resource.create_vpc( # pylint: disable=no-member - CidrBlock=self.ipv4_cidr, - AmazonProvidedIpv6CidrBlock=True) - except botocore.exceptions.ClientError as e: - raise RuntimeError(e) from e - - vpc.wait_until_available() - self._tag_resource(vpc) - - return vpc - - def _tag_resource(self, resource): - """Tag a resource with the specified tag. - - This makes finding and deleting resources specific to this testing - much easier to find. - - @param resource: resource to tag - """ - tag = { - 'Key': 'Name', - 'Value': self.tag - } - resource.create_tags(Tags=[tag]) - - def _upload_public_key(self, config): - """Generate random name and upload SSH key with that name. - - @param config: platform config - @return: string of ssh key name - """ - key_file = os.path.join(config['data_dir'], config['public_key']) - with open(key_file, 'r') as file: - public_key = file.read().strip('\n') - - LOG.debug('uploading SSH key %s', self.tag) - self.ec2_client.import_key_pair(KeyName=self.tag, - PublicKeyMaterial=public_key) - - return self.tag - - -def _decode_console_output_as_bytes(parsed, **kwargs): - """Provide console output as bytes in OutputBytes. - - For this to be useful, the session has to have had the - decode_console_output handler unregistered already. - - https://github.com/boto/botocore/issues/1351 .""" - if 'Output' not in parsed: - return - orig = parsed['Output'] - handlers.decode_console_output(parsed, **kwargs) - parsed['OutputBytes'] = base64.b64decode(orig) - - -def get_session(): - mysess = session.get_session() - mysess.unregister('after-call.ec2.GetConsoleOutput', - handlers.decode_console_output) - mysess.register('after-call.ec2.GetConsoleOutput', - _decode_console_output_as_bytes) - return boto3.Session(botocore_session=mysess) - - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/ec2/snapshot.py b/tests/cloud_tests/platforms/ec2/snapshot.py deleted file mode 100644 index 2c48cb54..00000000 --- a/tests/cloud_tests/platforms/ec2/snapshot.py +++ /dev/null @@ -1,66 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base EC2 snapshot.""" - -from ..snapshots import Snapshot -from tests.cloud_tests import LOG - - -class EC2Snapshot(Snapshot): - """EC2 image copy backed snapshot.""" - - platform_name = 'ec2' - - def __init__(self, platform, properties, config, features, image_ami, - delete_on_destroy=True): - """Set up snapshot. - - @param platform: platform object - @param properties: image properties - @param config: image config - @param features: supported feature flags - @param image_ami: string of image ami ID - @param delete_on_destroy: boolean to delete on destroy - """ - super(EC2Snapshot, self).__init__( - platform, properties, config, features) - - self.image_ami = image_ami - self.delete_on_destroy = delete_on_destroy - - def destroy(self): - """Deregister the backing AMI.""" - if self.delete_on_destroy: - image = self.platform.ec2_resource.Image(self.image_ami) - snapshot_id = image.block_device_mappings[0]['Ebs']['SnapshotId'] - - LOG.debug('removing custom ami %s', self.image_ami) - self.platform.ec2_client.deregister_image(ImageId=self.image_ami) - - LOG.debug('removing custom snapshot %s', snapshot_id) - self.platform.ec2_client.delete_snapshot(SnapshotId=snapshot_id) - - def launch(self, user_data, meta_data=None, block=True, start=True, - use_desc=None): - """Launch instance. - - @param user_data: user-data for the instance - @param meta_data: meta_data for the instance - @param block: wait until instance is created - @param start: start instance and wait until fully started - @param use_desc: string of test name - @return_value: an Instance - """ - if meta_data is not None: - raise ValueError("metadata not supported on Ec2") - - instance = self.platform.create_instance( - self.properties, self.config, self.features, - self.image_ami, user_data) - - if start: - instance.start() - - return instance - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/images.py b/tests/cloud_tests/platforms/images.py deleted file mode 100644 index f047de2e..00000000 --- a/tests/cloud_tests/platforms/images.py +++ /dev/null @@ -1,56 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base class for images.""" - -from ..util import TargetBase - - -class Image(TargetBase): - """Base class for images.""" - - platform_name = None - - def __init__(self, platform, config): - """Set up image. - - @param platform: platform object - @param config: image configuration - """ - self.platform = platform - self.config = config - - def __str__(self): - """A brief description of the image.""" - return '-'.join((self.properties['os'], self.properties['release'])) - - @property - def properties(self): - """{} containing: 'arch', 'os', 'version', 'release'.""" - return {k: self.config[k] - for k in ('arch', 'os', 'release', 'version')} - - @property - def features(self): - """Feature flags supported by this image. - - @return_value: list of feature names - """ - return [k for k, v in self.config.get('features', {}).items() if v] - - @property - def setup_overrides(self): - """Setup options that need to be overridden for the image. - - @return_value: dictionary to update args with - """ - # NOTE: more sophisticated options may be requied at some point - return self.config.get('setup_overrides', {}) - - def snapshot(self): - """Create snapshot of image, block until done.""" - raise NotImplementedError - - def destroy(self): - """Clean up data associated with image.""" - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/instances.py b/tests/cloud_tests/platforms/instances.py deleted file mode 100644 index efc35c7f..00000000 --- a/tests/cloud_tests/platforms/instances.py +++ /dev/null @@ -1,165 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base instance.""" -import time - -import paramiko -from paramiko.ssh_exception import ( - BadHostKeyException, AuthenticationException, SSHException) - -from ..util import TargetBase -from tests.cloud_tests import LOG, util - - -class Instance(TargetBase): - """Base instance object.""" - - platform_name = None - _ssh_client = None - - def __init__(self, platform, name, properties, config, features): - """Set up instance. - - @param platform: platform object - @param name: hostname of instance - @param properties: image properties - @param config: image config - @param features: supported feature flags - """ - self.platform = platform - self.name = name - self.properties = properties - self.config = config - self.features = features - self._tmp_count = 0 - - self.ssh_ip = None - self.ssh_port = None - self.ssh_key_file = None - self.ssh_username = 'ubuntu' - - def console_log(self): - """Instance console. - - @return_value: bytes of this instance’s console - """ - raise NotImplementedError - - def reboot(self, wait=True): - """Reboot instance.""" - raise NotImplementedError - - def shutdown(self, wait=True): - """Shutdown instance.""" - raise NotImplementedError - - def start(self, wait=True, wait_for_cloud_init=False): - """Start instance.""" - raise NotImplementedError - - def destroy(self): - """Clean up instance.""" - self._ssh_close() - - def _ssh(self, command, stdin=None): - """Run a command via SSH.""" - client = self._ssh_connect() - - cmd = util.shell_pack(command) - fp_in, fp_out, fp_err = client.exec_command(cmd) - channel = fp_in.channel - - if stdin is not None: - fp_in.write(stdin) - fp_in.close() - - channel.shutdown_write() - rc = channel.recv_exit_status() - - return (fp_out.read(), fp_err.read(), rc) - - def _ssh_close(self): - if self._ssh_client: - try: - self._ssh_client.close() - except SSHException: - LOG.warning('Failed to close SSH connection.') - self._ssh_client = None - - def _ssh_connect(self): - """Connect via SSH. - - Attempt to SSH to the client on the specific IP and port. If it - fails in some manner, then retry 2 more times for a total of 3 - attempts; sleeping a few seconds between attempts. - """ - if self._ssh_client: - return self._ssh_client - - if not self.ssh_ip or not self.ssh_port: - raise ValueError("Cannot ssh_connect, ssh_ip=%s ssh_port=%s" % - (self.ssh_ip, self.ssh_port)) - - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - private_key = paramiko.RSAKey.from_private_key_file(self.ssh_key_file) - - retries = 3 - while retries: - try: - client.connect(username=self.ssh_username, - hostname=self.ssh_ip, port=self.ssh_port, - pkey=private_key) - self._ssh_client = client - return client - except (ConnectionRefusedError, AuthenticationException, - BadHostKeyException, ConnectionResetError, SSHException, - OSError): - retries -= 1 - LOG.debug('Retrying ssh connection on connect failure') - time.sleep(3) - - ssh_cmd = 'Failed ssh connection to %s@%s:%s after 3 retries' % ( - self.ssh_username, self.ssh_ip, self.ssh_port - ) - raise util.InTargetExecuteError(b'', b'', 1, ssh_cmd, 'ssh') - - def _wait_for_system(self, wait_for_cloud_init): - """Wait until system has fully booted and cloud-init has finished. - - @param wait_time: maximum time to wait - @return_value: None, may raise OSError if wait_time exceeded - """ - def clean_test(test): - """Clean formatting for system ready test testcase.""" - return ' '.join(line for line in test.strip().splitlines() - if not line.lstrip().startswith('#')) - - boot_timeout = self.config['boot_timeout'] - tests = [self.config['system_ready_script']] - if wait_for_cloud_init: - tests.append(self.config['cloud_init_ready_script']) - - formatted_tests = ' && '.join(clean_test(t) for t in tests) - cmd = ('i=0; while [ $i -lt {time} ] && i=$(($i+1)); do {test} && ' - 'exit 0; sleep 1; done; exit 1').format(time=boot_timeout, - test=formatted_tests) - - end_time = time.time() + boot_timeout - while True: - try: - return_code = self.execute( - cmd, rcs=(0, 1), description='wait for instance start' - )[-1] - if return_code == 0: - break - except util.InTargetExecuteError: - LOG.warning("failed to connect via SSH") - - if time.time() < end_time: - time.sleep(3) - else: - raise util.PlatformError('ssh', 'after %ss instance is not ' - 'reachable' % boot_timeout) - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/lxd/__init__.py b/tests/cloud_tests/platforms/lxd/__init__.py deleted file mode 100644 index e69de29b..00000000 --- a/tests/cloud_tests/platforms/lxd/__init__.py +++ /dev/null diff --git a/tests/cloud_tests/platforms/lxd/image.py b/tests/cloud_tests/platforms/lxd/image.py deleted file mode 100644 index a88b47f3..00000000 --- a/tests/cloud_tests/platforms/lxd/image.py +++ /dev/null @@ -1,211 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""LXD Image Base Class.""" - -import os -import shutil -import tempfile - -from ..images import Image -from .snapshot import LXDSnapshot -from cloudinit import subp -from cloudinit import util as c_util -from tests.cloud_tests import util - - -class LXDImage(Image): - """LXD backed image.""" - - platform_name = "lxd" - - def __init__(self, platform, config, pylxd_image): - """Set up image. - - @param platform: platform object - @param config: image configuration - """ - self.modified = False - self._img_instance = None - self._pylxd_image = None - self.pylxd_image = pylxd_image - super(LXDImage, self).__init__(platform, config) - - @property - def pylxd_image(self): - """Property function.""" - if self._pylxd_image: - self._pylxd_image.sync() - return self._pylxd_image - - @pylxd_image.setter - def pylxd_image(self, pylxd_image): - if self._img_instance: - self._instance.destroy() - self._img_instance = None - if (self._pylxd_image and - (self._pylxd_image is not pylxd_image) and - (not self.config.get('cache_base_image') or self.modified)): - self._pylxd_image.delete(wait=True) - self.modified = False - self._pylxd_image = pylxd_image - - @property - def _instance(self): - """Internal use only, returns a instance - - This starts an lxc instance from the image, so it is "dirty". - Better would be some way to modify this "at rest". - lxc-pstart would be an option.""" - if not self._img_instance: - self._img_instance = self.platform.launch_container( - self.properties, self.config, self.features, - use_desc='image-modification', image_desc=str(self), - image=self.pylxd_image.fingerprint) - self._img_instance.start() - return self._img_instance - - @property - def properties(self): - """{} containing: 'arch', 'os', 'version', 'release'.""" - properties = self.pylxd_image.properties - return { - 'arch': properties.get('architecture'), - 'os': properties.get('os'), - 'version': properties.get('version'), - 'release': properties.get('release'), - } - - def export_image(self, output_dir): - """Export image from lxd image store to disk. - - @param output_dir: dir to store the exported image in - @return_value: tuple of path to metadata tarball and rootfs - - Only the "split" image format with separate rootfs and metadata - files is supported, e.g: - - 71f171df[...]cd31.squashfs (could also be: .tar.xz or .tar.gz) - meta-71f171df[...]cd31.tar.xz - - Combined images made by a single tarball are not supported. - """ - # pylxd's image export feature doesn't do split exports, so use cmdline - fp = self.pylxd_image.fingerprint - subp.subp(['lxc', 'image', 'export', fp, output_dir], capture=True) - image_files = [p for p in os.listdir(output_dir) if fp in p] - - if len(image_files) != 2: - raise NotImplementedError( - "Image %s has unsupported format. " - "Expected 2 files, found %d: %s." - % (fp, len(image_files), ', '.join(image_files))) - - metadata = os.path.join( - output_dir, - next(p for p in image_files if p.startswith('meta-'))) - rootfs = os.path.join( - output_dir, - next(p for p in image_files if not p.startswith('meta-'))) - return (metadata, rootfs) - - def import_image(self, metadata, rootfs): - """Import image to lxd image store from (split) tarball on disk. - - Note, this will replace and delete the current pylxd_image - - @param metadata: metadata tarball - @param rootfs: rootfs tarball - @return_value: imported image fingerprint - """ - alias = util.gen_instance_name( - image_desc=str(self), use_desc='update-metadata') - subp.subp(['lxc', 'image', 'import', metadata, rootfs, - '--alias', alias], capture=True) - self.pylxd_image = self.platform.query_image_by_alias(alias) - return self.pylxd_image.fingerprint - - def update_templates(self, template_config, template_data): - """Update the image's template configuration. - - Note, this will replace and delete the current pylxd_image - - @param template_config: config overrides for template metadata - @param template_data: template data to place into templates/ - """ - # set up tmp files - export_dir = tempfile.mkdtemp(prefix='cloud_test_util_') - extract_dir = tempfile.mkdtemp(prefix='cloud_test_util_') - new_metadata = os.path.join(export_dir, 'new-meta.tar.xz') - metadata_yaml = os.path.join(extract_dir, 'metadata.yaml') - template_dir = os.path.join(extract_dir, 'templates') - - try: - # extract old data - (metadata, rootfs) = self.export_image(export_dir) - shutil.unpack_archive(metadata, extract_dir) - - # update metadata - metadata = c_util.read_conf(metadata_yaml) - templates = metadata.get('templates', {}) - templates.update(template_config) - metadata['templates'] = templates - util.yaml_dump(metadata, metadata_yaml) - - # write out template files - for name, content in template_data.items(): - path = os.path.join(template_dir, name) - c_util.write_file(path, content) - - # store new data, mark new image as modified - util.flat_tar(new_metadata, extract_dir) - self.import_image(new_metadata, rootfs) - self.modified = True - - finally: - # remove tmpfiles - shutil.rmtree(export_dir) - shutil.rmtree(extract_dir) - - def _execute(self, *args, **kwargs): - """Execute command in image, modifying image.""" - return self._instance._execute(*args, **kwargs) - - def push_file(self, local_path, remote_path): - """Copy file at 'local_path' to instance at 'remote_path'.""" - return self._instance.push_file(local_path, remote_path) - - def run_script(self, *args, **kwargs): - """Run script in image, modifying image. - - @return_value: script output - """ - return self._instance.run_script(*args, **kwargs) - - def snapshot(self): - """Create snapshot of image, block until done.""" - # get empty user data to pass in to instance - # if overrides for user data provided, use them - empty_userdata = util.update_user_data( - {}, self.config.get('user_data_overrides', {})) - conf = {'user.user-data': empty_userdata} - # clone current instance - instance = self.platform.launch_container( - self.properties, self.config, self.features, - container=self._instance.name, image_desc=str(self), - use_desc='snapshot', container_config=conf) - # wait for cloud-init before boot_clean_script is run to ensure - # /var/lib/cloud is removed cleanly - instance.start(wait=True, wait_for_cloud_init=True) - if self.config.get('boot_clean_script'): - instance.run_script(self.config.get('boot_clean_script')) - # freeze current instance and return snapshot - instance.freeze() - return LXDSnapshot(self.platform, self.properties, self.config, - self.features, instance) - - def destroy(self): - """Clean up data associated with image.""" - self.pylxd_image = None - super(LXDImage, self).destroy() - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py deleted file mode 100644 index 2b973a08..00000000 --- a/tests/cloud_tests/platforms/lxd/instance.py +++ /dev/null @@ -1,278 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base LXD instance.""" - -import os -import shutil -import time -from tempfile import mkdtemp - -from cloudinit.subp import subp, ProcessExecutionError, which -from cloudinit.util import load_yaml -from tests.cloud_tests import LOG -from tests.cloud_tests.util import PlatformError - -from ..instances import Instance - -from pylxd import exceptions as pylxd_exc - - -class LXDInstance(Instance): - """LXD container backed instance.""" - - platform_name = "lxd" - _console_log_method = None - _console_log_file = None - - def __init__(self, platform, name, properties, config, features, - pylxd_container): - """Set up instance. - - @param platform: platform object - @param name: hostname of instance - @param properties: image properties - @param config: image config - @param features: supported feature flags - """ - if not pylxd_container: - raise ValueError("Invalid value pylxd_container: %s" % - pylxd_container) - self._pylxd_container = pylxd_container - super(LXDInstance, self).__init__( - platform, name, properties, config, features) - self.tmpd = mkdtemp(prefix="%s-%s" % (type(self).__name__, name)) - self.name = name - self._setup_console_log() - - @property - def pylxd_container(self): - """Property function.""" - if self._pylxd_container is None: - raise RuntimeError( - "%s: Attempted use of pylxd_container after deletion." % self) - self._pylxd_container.sync() - return self._pylxd_container - - def __str__(self): - return ( - '%s(name=%s) status=%s' % - (self.__class__.__name__, self.name, - ("deleted" if self._pylxd_container is None else - self.pylxd_container.status))) - - def _execute(self, command, stdin=None, env=None): - if env is None: - env = {} - - env_args = [] - if env: - env_args = ['env'] + ["%s=%s" for k, v in env.items()] - - # ensure instance is running and execute the command - self.start() - - # Use cmdline client due to https://github.com/lxc/pylxd/issues/268 - exit_code = 0 - try: - stdout, stderr = subp( - ['lxc', 'exec', self.name, '--'] + env_args + list(command), - data=stdin, decode=False) - except ProcessExecutionError as e: - exit_code = e.exit_code - stdout = e.stdout - stderr = e.stderr - - return stdout, stderr, exit_code - - def read_data(self, remote_path, decode=False): - """Read data from instance filesystem. - - @param remote_path: path in instance - @param decode: decode data before returning. - @return_value: content of remote_path as bytes if 'decode' is False, - and as string if 'decode' is True. - """ - data = self.pylxd_container.files.get(remote_path) - return data.decode() if decode else data - - def write_data(self, remote_path, data): - """Write data to instance filesystem. - - @param remote_path: path in instance - @param data: data to write in bytes - """ - self.pylxd_container.files.put(remote_path, data) - - @property - def console_log_method(self): - if self._console_log_method is not None: - return self._console_log_method - - client = which('lxc') - if not client: - raise PlatformError("No 'lxc' client.") - - elif _has_proper_console_support(): - self._console_log_method = 'show-log' - elif client.startswith("/snap"): - self._console_log_method = 'logfile-snap' - else: - self._console_log_method = 'logfile-tmp' - - LOG.debug("Set console log method to %s", self._console_log_method) - return self._console_log_method - - def _setup_console_log(self): - method = self.console_log_method - if not method.startswith("logfile-"): - return - - if method == "logfile-snap": - log_dir = "/var/snap/lxd/common/consoles" - if not os.path.exists(log_dir): - raise PlatformError( - "Unable to log with snap lxc. Please run:\n" - " sudo mkdir --mode=1777 -p %s" % log_dir) - elif method == "logfile-tmp": - log_dir = "/tmp" - else: - raise PlatformError( - "Unexpected value for console method: %s" % method) - - # doing this ensures we can read it. Otherwise it ends up root:root. - log_file = os.path.join(log_dir, self.name) - with open(log_file, "w") as fp: - fp.write("# %s\n" % self.name) - - cfg = "lxc.console.logfile=%s" % log_file - orig = self._pylxd_container.config.get('raw.lxc', "") - if orig: - orig += "\n" - self._pylxd_container.config['raw.lxc'] = orig + cfg - self._pylxd_container.save() - self._console_log_file = log_file - - def console_log(self): - """Console log. - - @return_value: bytes of this instance's console - """ - - if self._console_log_file: - if not os.path.exists(self._console_log_file): - raise NotImplementedError( - "Console log '%s' does not exist. If this is a remote " - "lxc, then this is really NotImplementedError. If it is " - "A local lxc, then this is a RuntimeError." - "https://github.com/lxc/lxd/issues/1129") - with open(self._console_log_file, "rb") as fp: - return fp.read() - - try: - return subp(['lxc', 'console', '--show-log', self.name], - decode=False)[0] - except ProcessExecutionError as e: - raise PlatformError( - "console log", - "Console log failed [%d]: stdout=%s stderr=%s" % ( - e.exit_code, e.stdout, e.stderr) - ) from e - - def reboot(self, wait=True): - """Reboot instance.""" - self.shutdown(wait=wait) - self.start(wait=wait) - - def shutdown(self, wait=True, retry=1): - """Shutdown instance.""" - if self.pylxd_container.status == 'Stopped': - return - - try: - LOG.debug("%s: shutting down (wait=%s)", self, wait) - self.pylxd_container.stop(wait=wait) - except (pylxd_exc.LXDAPIException, pylxd_exc.NotFound) as e: - # An exception happens here sometimes (LP: #1783198) - # LOG it, and try again. - LOG.warning( - ("%s: shutdown(retry=%d) caught %s in shutdown " - "(response=%s): %s"), - self, retry, e.__class__.__name__, e.response, e) - if isinstance(e, pylxd_exc.NotFound): - LOG.debug("container_exists(%s) == %s", - self.name, self.platform.container_exists(self.name)) - if retry == 0: - raise e - return self.shutdown(wait=wait, retry=retry - 1) - - def start(self, wait=True, wait_for_cloud_init=False): - """Start instance.""" - if self.pylxd_container.status != 'Running': - self.pylxd_container.start(wait=wait) - if wait: - self._wait_for_system(wait_for_cloud_init) - - def freeze(self): - """Freeze instance.""" - if self.pylxd_container.status != 'Frozen': - self.pylxd_container.freeze(wait=True) - - def unfreeze(self): - """Unfreeze instance.""" - if self.pylxd_container.status == 'Frozen': - self.pylxd_container.unfreeze(wait=True) - - def destroy(self): - """Clean up instance.""" - LOG.debug("%s: deleting container.", self) - self.unfreeze() - self.shutdown() - retries = [1] * 5 - for attempt, wait in enumerate(retries): - try: - self.pylxd_container.delete(wait=True) - break - except Exception: - if attempt + 1 >= len(retries): - raise - LOG.debug('Failed to delete container %s (%s/%s) retrying...', - self, attempt + 1, len(retries)) - time.sleep(wait) - - self._pylxd_container = None - - if self.platform.container_exists(self.name): - raise OSError('%s: container was not properly removed' % self) - if self._console_log_file and os.path.exists(self._console_log_file): - os.unlink(self._console_log_file) - shutil.rmtree(self.tmpd) - super(LXDInstance, self).destroy() - - -def _has_proper_console_support(): - stdout, _ = subp(['lxc', 'info']) - info = load_yaml(stdout) - reason = None - if 'console' not in info.get('api_extensions', []): - reason = "LXD server does not support console api extension" - else: - dver = str(info.get('environment', {}).get('driver_version', "")) - if dver.startswith("2.") or dver.startswith("1."): - reason = "LXD Driver version not 3.x+ (%s)" % dver - else: - try: - stdout = subp(['lxc', 'console', '--help'], decode=False)[0] - if not (b'console' in stdout and b'log' in stdout): - reason = "no '--log' in lxc console --help" - except ProcessExecutionError: - reason = "no 'console' command in lxc client" - - if reason: - LOG.debug("no console-support: %s", reason) - return False - else: - LOG.debug("console-support looks good") - return True - - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/lxd/platform.py b/tests/cloud_tests/platforms/lxd/platform.py deleted file mode 100644 index f7251a07..00000000 --- a/tests/cloud_tests/platforms/lxd/platform.py +++ /dev/null @@ -1,104 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base LXD platform.""" - -from pylxd import (Client, exceptions) - -from ..platforms import Platform -from .image import LXDImage -from .instance import LXDInstance -from tests.cloud_tests import util - -DEFAULT_SSTREAMS_SERVER = "https://images.linuxcontainers.org:8443" - - -class LXDPlatform(Platform): - """LXD test platform.""" - - platform_name = 'lxd' - - def __init__(self, config): - """Set up platform.""" - super(LXDPlatform, self).__init__(config) - # TODO: allow configuration of remote lxd host via env variables - # set up lxd connection - self.client = Client() - - def get_image(self, img_conf): - """Get image using specified image configuration. - - @param img_conf: configuration for image - @return_value: cloud_tests.images instance - """ - pylxd_image = self.client.images.create_from_simplestreams( - img_conf.get('sstreams_server', DEFAULT_SSTREAMS_SERVER), - img_conf['alias']) - image = LXDImage(self, img_conf, pylxd_image) - if img_conf.get('override_templates', False): - image.update_templates(self.config.get('template_overrides', {}), - self.config.get('template_files', {})) - return image - - def launch_container(self, properties, config, features, - image=None, container=None, ephemeral=False, - container_config=None, block=True, image_desc=None, - use_desc=None): - """Launch a container. - - @param properties: image properties - @param config: image configuration - @param features: image features - @param image: image fingerprint to launch from - @param container: container to copy - @param ephemeral: delete image after first shutdown - @param container_config: config options for instance as dict - @param block: wait until container created - @param image_desc: description of image being launched - @param use_desc: description of container's use - @return_value: cloud_tests.instances instance - """ - if not (image or container): - raise ValueError("either image or container must be specified") - container = self.client.containers.create({ - 'name': util.gen_instance_name(image_desc=image_desc, - use_desc=use_desc, - used_list=self.list_containers()), - 'ephemeral': bool(ephemeral), - 'config': (container_config - if isinstance(container_config, dict) else {}), - 'source': ({'type': 'image', 'fingerprint': image} if image else - {'type': 'copy', 'source': container}) - }, wait=block) - return LXDInstance(self, container.name, properties, config, features, - container) - - def container_exists(self, container_name): - """Check if container with name 'container_name' exists. - - @return_value: True if exists else False - """ - res = True - try: - self.client.containers.get(container_name) - except exceptions.LXDAPIException as e: - res = False - if e.response.status_code != 404: - raise - return res - - def list_containers(self): - """List names of all containers. - - @return_value: list of names - """ - return [container.name for container in self.client.containers.all()] - - def query_image_by_alias(self, alias): - """Get image by alias in local image store. - - @param alias: alias of image - @return_value: pylxd image (not cloud_tests.images instance) - """ - return self.client.images.get_by_alias(alias) - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/lxd/snapshot.py b/tests/cloud_tests/platforms/lxd/snapshot.py deleted file mode 100644 index b524644f..00000000 --- a/tests/cloud_tests/platforms/lxd/snapshot.py +++ /dev/null @@ -1,53 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base LXD snapshot.""" - -from ..snapshots import Snapshot - - -class LXDSnapshot(Snapshot): - """LXD image copy backed snapshot.""" - - platform_name = "lxd" - - def __init__(self, platform, properties, config, features, - pylxd_frozen_instance): - """Set up snapshot. - - @param platform: platform object - @param properties: image properties - @param config: image config - @param features: supported feature flags - """ - self.pylxd_frozen_instance = pylxd_frozen_instance - super(LXDSnapshot, self).__init__( - platform, properties, config, features) - - def launch(self, user_data, meta_data=None, block=True, start=True, - use_desc=None): - """Launch instance. - - @param user_data: user-data for the instance - @param instance_id: instance-id for the instance - @param block: wait until instance is created - @param start: start instance and wait until fully started - @param use_desc: description of snapshot instance use - @return_value: an Instance - """ - inst_config = {'user.user-data': user_data} - if meta_data: - inst_config['user.meta-data'] = meta_data - instance = self.platform.launch_container( - self.properties, self.config, self.features, block=block, - image_desc=str(self), container=self.pylxd_frozen_instance.name, - use_desc=use_desc, container_config=inst_config) - if start: - instance.start() - return instance - - def destroy(self): - """Clean up snapshot data.""" - self.pylxd_frozen_instance.destroy() - super(LXDSnapshot, self).destroy() - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/nocloudkvm/__init__.py b/tests/cloud_tests/platforms/nocloudkvm/__init__.py deleted file mode 100644 index e69de29b..00000000 --- a/tests/cloud_tests/platforms/nocloudkvm/__init__.py +++ /dev/null diff --git a/tests/cloud_tests/platforms/nocloudkvm/image.py b/tests/cloud_tests/platforms/nocloudkvm/image.py deleted file mode 100644 index ff5b6ad7..00000000 --- a/tests/cloud_tests/platforms/nocloudkvm/image.py +++ /dev/null @@ -1,79 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""NoCloud KVM Image Base Class.""" - -from cloudinit import subp - -import os -import shutil -import tempfile - -from ..images import Image -from .snapshot import NoCloudKVMSnapshot - - -class NoCloudKVMImage(Image): - """NoCloud KVM backed image.""" - - platform_name = "nocloud-kvm" - - def __init__(self, platform, config, orig_img_path): - """Set up image. - - @param platform: platform object - @param config: image configuration - @param img_path: path to the image - """ - self.modified = False - self._workd = tempfile.mkdtemp(prefix='NoCloudKVMImage') - self._orig_img_path = orig_img_path - self._img_path = os.path.join(self._workd, - os.path.basename(self._orig_img_path)) - - subp.subp(['qemu-img', 'create', '-f', 'qcow2', - '-b', orig_img_path, self._img_path]) - - super(NoCloudKVMImage, self).__init__(platform, config) - - def _execute(self, command, stdin=None, env=None): - """Execute command in image, modifying image.""" - return self.mount_image_callback(command, stdin=stdin, env=env) - - def mount_image_callback(self, command, stdin=None, env=None): - """Run mount-image-callback.""" - - env_args = [] - if env: - env_args = ['env'] + ["%s=%s" for k, v in env.items()] - - mic_chroot = ['sudo', 'mount-image-callback', '--system-mounts', - '--system-resolvconf', self._img_path, - '--', 'chroot', '_MOUNTPOINT_'] - try: - out, err = subp.subp(mic_chroot + env_args + list(command), - data=stdin, decode=False) - return (out, err, 0) - except subp.ProcessExecutionError as e: - return (e.stdout, e.stderr, e.exit_code) - - def snapshot(self): - """Create snapshot of image, block until done.""" - if not self._img_path: - raise RuntimeError() - - return NoCloudKVMSnapshot(self.platform, self.properties, self.config, - self.features, self._img_path) - - def destroy(self): - """Unset path to signal image is no longer used. - - The removal of the images and all other items is handled by the - framework. In some cases we want to keep the images, so let the - framework decide whether to keep or destroy everything. - """ - self._img_path = None - shutil.rmtree(self._workd) - - super(NoCloudKVMImage, self).destroy() - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/nocloudkvm/instance.py b/tests/cloud_tests/platforms/nocloudkvm/instance.py deleted file mode 100644 index 5140a11c..00000000 --- a/tests/cloud_tests/platforms/nocloudkvm/instance.py +++ /dev/null @@ -1,197 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base NoCloud KVM instance.""" - -import copy -import os -import socket -import subprocess -import time -import uuid - -from ..instances import Instance -from cloudinit.atomic_helper import write_json -from cloudinit import subp -from tests.cloud_tests import LOG, util - -# This domain contains reverse lookups for hostnames that are used. -# The primary reason is so sudo will return quickly when it attempts -# to look up the hostname. i9n is just short for 'integration'. -# see also bug 1730744 for why we had to do this. -CI_DOMAIN = "i9n.cloud-init.io" - - -class NoCloudKVMInstance(Instance): - """NoCloud KVM backed instance.""" - - platform_name = "nocloud-kvm" - - def __init__(self, platform, name, image_path, properties, config, - features, user_data, meta_data): - """Set up instance. - - @param platform: platform object - @param name: image path - @param image_path: path to disk image to boot. - @param properties: dictionary of properties - @param config: dictionary of configuration values - @param features: dictionary of supported feature flags - """ - super(NoCloudKVMInstance, self).__init__( - platform, name, properties, config, features - ) - - self.user_data = user_data - if meta_data: - meta_data = copy.deepcopy(meta_data) - else: - meta_data = {} - - if 'instance-id' in meta_data: - iid = meta_data['instance-id'] - else: - iid = str(uuid.uuid1()) - meta_data['instance-id'] = iid - - self.instance_id = iid - self.ssh_key_file = os.path.join( - platform.config['data_dir'], platform.config['private_key']) - self.ssh_pubkey_file = os.path.join( - platform.config['data_dir'], platform.config['public_key']) - - self.ssh_pubkey = None - if self.ssh_pubkey_file: - with open(self.ssh_pubkey_file, "r") as fp: - self.ssh_pubkey = fp.read().rstrip('\n') - - if not meta_data.get('public-keys'): - meta_data['public-keys'] = [] - meta_data['public-keys'].append(self.ssh_pubkey) - - self.ssh_ip = '127.0.0.1' - self.ssh_port = None - self.pid = None - self.pid_file = None - self.console_file = None - self.disk = image_path - self.cache_mode = platform.config.get('cache_mode', - 'cache=none,aio=native') - self.meta_data = meta_data - - def shutdown(self, wait=True): - """Shutdown instance.""" - - if self.pid: - # This relies on _execute which uses sudo over ssh. The ssh - # connection would get killed before sudo exited, so ignore errors. - cmd = ['shutdown', 'now'] - try: - self._execute(cmd) - except util.InTargetExecuteError: - pass - self._ssh_close() - - if wait: - LOG.debug("Executed shutdown. waiting on pid %s to end", - self.pid) - time_for_shutdown = 120 - give_up_at = time.time() + time_for_shutdown - pid_file_path = '/proc/%s' % self.pid - msg = ("pid %s did not exit in %s seconds after shutdown." % - (self.pid, time_for_shutdown)) - while True: - if not os.path.exists(pid_file_path): - break - if time.time() > give_up_at: - raise util.PlatformError("shutdown", msg) - self.pid = None - - def destroy(self): - """Clean up instance.""" - if self.pid: - try: - subp.subp(['kill', '-9', self.pid]) - except subp.ProcessExecutionError: - pass - - if self.pid_file: - try: - os.remove(self.pid_file) - except Exception: - pass - - self.pid = None - self._ssh_close() - - super(NoCloudKVMInstance, self).destroy() - - def _execute(self, command, stdin=None, env=None): - env_args = [] - if env: - env_args = ['env'] + ["%s=%s" for k, v in env.items()] - - return self._ssh(['sudo'] + env_args + list(command), stdin=stdin) - - def generate_seed(self, tmpdir): - """Generate nocloud seed from user-data""" - seed_file = os.path.join(tmpdir, '%s_seed.img' % self.name) - user_data_file = os.path.join(tmpdir, '%s_user_data' % self.name) - meta_data_file = os.path.join(tmpdir, '%s_meta_data' % self.name) - - with open(user_data_file, "w") as ud_file: - ud_file.write(self.user_data) - - # meta-data can be yaml, but more easily pretty printed with json - write_json(meta_data_file, self.meta_data) - subp.subp(['cloud-localds', seed_file, user_data_file, - meta_data_file]) - - return seed_file - - def get_free_port(self): - """Get a free port assigned by the kernel.""" - s = socket.socket() - s.bind(('', 0)) - num = s.getsockname()[1] - s.close() - return num - - def start(self, wait=True, wait_for_cloud_init=False): - """Start instance.""" - tmpdir = self.platform.config['data_dir'] - seed = self.generate_seed(tmpdir) - self.pid_file = os.path.join(tmpdir, '%s.pid' % self.name) - self.console_file = os.path.join(tmpdir, '%s-console.log' % self.name) - self.ssh_port = self.get_free_port() - - cmd = ['./tools/xkvm', - '--disk', '%s,%s' % (self.disk, self.cache_mode), - '--disk', '%s' % seed, - '--netdev', ','.join(['user', - 'hostfwd=tcp::%s-:22' % self.ssh_port, - 'dnssearch=%s' % CI_DOMAIN]), - '--', '-pidfile', self.pid_file, '-vnc', 'none', - '-m', '2G', '-smp', '2', '-nographic', '-name', self.name, - '-serial', 'file:' + self.console_file] - subprocess.Popen(cmd, - close_fds=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - while not os.path.exists(self.pid_file): - time.sleep(1) - - with open(self.pid_file, 'r') as pid_f: - self.pid = pid_f.readlines()[0].strip() - - if wait: - self._wait_for_system(wait_for_cloud_init) - - def console_log(self): - if not self.console_file: - return b'' - with open(self.console_file, "rb") as fp: - return fp.read() - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/nocloudkvm/platform.py b/tests/cloud_tests/platforms/nocloudkvm/platform.py deleted file mode 100644 index 53c8ebf2..00000000 --- a/tests/cloud_tests/platforms/nocloudkvm/platform.py +++ /dev/null @@ -1,94 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base NoCloud KVM platform.""" -import glob -import os - -from simplestreams import filters -from simplestreams import mirrors -from simplestreams import objectstores -from simplestreams import util as s_util - -from ..platforms import Platform -from .image import NoCloudKVMImage -from .instance import NoCloudKVMInstance -from cloudinit import subp -from cloudinit import util as c_util -from tests.cloud_tests import util - - -class NoCloudKVMPlatform(Platform): - """NoCloud KVM test platform.""" - - platform_name = 'nocloud-kvm' - - def get_image(self, img_conf): - """Get image using specified image configuration. - - @param img_conf: configuration for image - @return_value: cloud_tests.images instance - """ - (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) - - filter = filters.get_filters( - [ - 'arch=%s' % c_util.get_dpkg_architecture(), - 'release=%s' % img_conf['release'], - 'ftype=disk1.img', - ] - ) - mirror_config = {'filters': filter, - 'keep_items': False, - 'max_items': 1, - 'checksumming_reader': True, - 'item_download': True - } - - def policy(content, path): - return s_util.read_signed(content, keyring=img_conf['keyring']) - - smirror = mirrors.UrlMirrorReader(url, policy=policy) - tstore = objectstores.FileStore(img_conf['mirror_dir']) - tmirror = mirrors.ObjectFilterMirror(config=mirror_config, - objectstore=tstore) - tmirror.sync(smirror, path) - - search_d = os.path.join(img_conf['mirror_dir'], '**', - img_conf['release'], '**', '*.img') - - images = [] - for fname in glob.iglob(search_d, recursive=True): - images.append(fname) - - if len(images) < 1: - raise RuntimeError("No images found under '%s'" % search_d) - if len(images) > 1: - raise RuntimeError( - "Multiple images found in '%s': %s" % (search_d, - ' '.join(images))) - - image = NoCloudKVMImage(self, img_conf, images[0]) - return image - - def create_instance(self, properties, config, features, - src_img_path, image_desc=None, use_desc=None, - user_data=None, meta_data=None): - """Create an instance - - @param src_img_path: image path to launch from - @param properties: image properties - @param config: image configuration - @param features: image features - @param image_desc: description of image being launched - @param use_desc: description of container's use - @return_value: cloud_tests.instances instance - """ - name = util.gen_instance_name(image_desc=image_desc, use_desc=use_desc) - img_path = os.path.join(self.config['data_dir'], name + '.qcow2') - subp.subp(['qemu-img', 'create', '-f', 'qcow2', - '-b', src_img_path, img_path]) - - return NoCloudKVMInstance(self, name, img_path, properties, config, - features, user_data, meta_data) - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/nocloudkvm/snapshot.py b/tests/cloud_tests/platforms/nocloudkvm/snapshot.py deleted file mode 100644 index 2dae3590..00000000 --- a/tests/cloud_tests/platforms/nocloudkvm/snapshot.py +++ /dev/null @@ -1,59 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base NoCloud KVM snapshot.""" -import os -import shutil -import tempfile - -from ..snapshots import Snapshot - - -class NoCloudKVMSnapshot(Snapshot): - """NoCloud KVM image copy backed snapshot.""" - - platform_name = "nocloud-kvm" - - def __init__(self, platform, properties, config, features, image_path): - """Set up snapshot. - - @param platform: platform object - @param properties: image properties - @param config: image config - @param features: supported feature flags - @param image_path: image file to snapshot. - """ - self._workd = tempfile.mkdtemp(prefix='NoCloudKVMSnapshot') - snapshot = os.path.join(self._workd, 'snapshot') - shutil.copyfile(image_path, snapshot) - self._image_path = snapshot - - super(NoCloudKVMSnapshot, self).__init__( - platform, properties, config, features) - - def launch(self, user_data, meta_data=None, block=True, start=True, - use_desc=None): - """Launch instance. - - @param user_data: user-data for the instance - @param instance_id: instance-id for the instance - @param block: wait until instance is created - @param start: start instance and wait until fully started - @param use_desc: description of snapshot instance use - @return_value: an Instance - """ - instance = self.platform.create_instance( - self.properties, self.config, self.features, - self._image_path, image_desc=str(self), use_desc=use_desc, - user_data=user_data, meta_data=meta_data) - - if start: - instance.start() - - return instance - - def destroy(self): - """Clean up snapshot data.""" - shutil.rmtree(self._workd) - super(NoCloudKVMSnapshot, self).destroy() - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py deleted file mode 100644 index ac3b6563..00000000 --- a/tests/cloud_tests/platforms/platforms.py +++ /dev/null @@ -1,109 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base platform class.""" -import os -import shutil - -from simplestreams import filters, mirrors -from simplestreams import util as s_util - -from cloudinit import subp -from cloudinit import util as c_util - -from tests.cloud_tests import util - - -class Platform(object): - """Base class for platforms.""" - - platform_name = None - - def __init__(self, config): - """Set up platform.""" - self.config = config - self.tmpdir = util.mkdtemp() - if 'data_dir' in config: - self.data_dir = config['data_dir'] - else: - self.data_dir = os.path.join(self.tmpdir, "data_dir") - os.mkdir(self.data_dir) - - self._generate_ssh_keys(self.data_dir) - - def get_image(self, img_conf): - """Get image using specified image configuration. - - @param img_conf: configuration for image - @return_value: cloud_tests.images instance - """ - raise NotImplementedError - - def destroy(self): - """Clean up platform data.""" - shutil.rmtree(self.tmpdir) - - def _generate_ssh_keys(self, data_dir): - """Generate SSH keys to be used with image.""" - filename = os.path.join(data_dir, self.config['private_key']) - - if os.path.exists(filename): - c_util.del_file(filename) - - subp.subp(['ssh-keygen', '-m', 'PEM', '-t', 'rsa', '-b', '4096', - '-f', filename, '-P', '', - '-C', 'ubuntu@cloud_test'], - capture=True) - - @staticmethod - def _query_streams(img_conf, img_filter): - """Query streams for latest image given a specific filter. - - @param img_conf: configuration for image - @param filters: array of filters as strings format 'key=value' - @return: dictionary with latest image information or empty - """ - def policy(content, path): - return s_util.read_signed(content, keyring=img_conf['keyring']) - - (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) - smirror = mirrors.UrlMirrorReader(url, policy=policy) - - config = {'max_items': 1, 'filters': filters.get_filters(img_filter)} - tmirror = FilterMirror(config) - tmirror.sync(smirror, path) - - try: - return tmirror.json_entries[0] - except IndexError as e: - raise RuntimeError( - 'no images found with filter: %s' % img_filter - ) from e - - -class FilterMirror(mirrors.BasicMirrorWriter): - """Taken from sstream-query to return query result as json array.""" - - def __init__(self, config=None): - super(FilterMirror, self).__init__(config=config) - if config is None: - config = {} - self.config = config - self.filters = config.get('filters', []) - self.json_entries = [] - - def load_products(self, path=None, content_id=None): - return {'content_id': content_id, 'products': {}} - - def filter_item(self, data, src, target, pedigree): - return filters.filter_item(self.filters, data, src, pedigree) - - def insert_item(self, data, src, target, pedigree, contentsource): - # src and target are top level products:1.0 - # data is src['products'][ped[0]]['versions'][ped[1]]['items'][ped[2]] - # contentsource is a ContentSource if 'path' exists in data or None - data = s_util.products_exdata(src, pedigree) - if 'path' in data: - data.update({'item_url': contentsource.url}) - self.json_entries.append(data) - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/snapshots.py b/tests/cloud_tests/platforms/snapshots.py deleted file mode 100644 index 0f5f8bb6..00000000 --- a/tests/cloud_tests/platforms/snapshots.py +++ /dev/null @@ -1,44 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base snapshot.""" - - -class Snapshot(object): - """Base class for snapshots.""" - - platform_name = None - - def __init__(self, platform, properties, config, features): - """Set up snapshot. - - @param platform: platform object - @param properties: image properties - @param config: image config - @param features: supported feature flags - """ - self.platform = platform - self.properties = properties - self.config = config - self.features = features - - def __str__(self): - """A brief description of the snapshot.""" - return '-'.join((self.properties['os'], self.properties['release'])) - - def launch(self, user_data, meta_data=None, block=True, start=True, - use_desc=None): - """Launch instance. - - @param user_data: user-data for the instance - @param instance_id: instance-id for the instance - @param block: wait until instance is created - @param start: start instance and wait until fully started - @param use_desc: description of snapshot instance use - @return_value: an Instance - """ - raise NotImplementedError - - def destroy(self): - """Clean up snapshot data.""" - -# vi: ts=4 expandtab |