diff options
Diffstat (limited to 'tests/cloud_tests/platforms')
18 files changed, 1664 insertions, 45 deletions
diff --git a/tests/cloud_tests/platforms/__init__.py b/tests/cloud_tests/platforms/__init__.py index 3490fe87..a01e51ac 100644 --- a/tests/cloud_tests/platforms/__init__.py +++ b/tests/cloud_tests/platforms/__init__.py @@ -2,15 +2,27 @@ """Main init.""" -from tests.cloud_tests.platforms import lxd -from tests.cloud_tests.platforms import nocloudkvm +from .ec2 import platform as ec2 +from .lxd import platform as lxd +from .nocloudkvm import platform as nocloudkvm PLATFORMS = { + 'ec2': ec2.EC2Platform, 'nocloud-kvm': nocloudkvm.NoCloudKVMPlatform, 'lxd': lxd.LXDPlatform, } +def get_image(platform, config): + """Get image from platform object using os_name.""" + return platform.get_image(config) + + +def get_instance(snapshot, *args, **kwargs): + """Get instance from snapshot.""" + return snapshot.launch(*args, **kwargs) + + def get_platform(platform_name, config): """Get the platform object for 'platform_name' and init.""" platform_cls = PLATFORMS.get(platform_name) @@ -18,4 +30,10 @@ def get_platform(platform_name, config): raise ValueError('invalid platform name: {}'.format(platform_name)) return platform_cls(config) + +def get_snapshot(image): + """Get snapshot from image.""" + return image.snapshot() + + # vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/base.py b/tests/cloud_tests/platforms/base.py deleted file mode 100644 index 28975368..00000000 --- a/tests/cloud_tests/platforms/base.py +++ /dev/null @@ -1,27 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Base platform class.""" - - -class Platform(object): - """Base class for platforms.""" - - platform_name = None - - def __init__(self, config): - """Set up platform.""" - self.config = config - - def get_image(self, img_conf): - """Get image using specified image configuration. - - @param img_conf: configuration for image - @return_value: cloud_tests.images instance - """ - raise NotImplementedError - - def destroy(self): - """Clean up platform data.""" - pass - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/ec2/image.py b/tests/cloud_tests/platforms/ec2/image.py new file mode 100644 index 00000000..7bedf59d --- /dev/null +++ b/tests/cloud_tests/platforms/ec2/image.py @@ -0,0 +1,99 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""EC2 Image Base Class.""" + +from ..images import Image +from .snapshot import EC2Snapshot +from tests.cloud_tests import LOG + + +class EC2Image(Image): + """EC2 backed image.""" + + platform_name = 'ec2' + + def __init__(self, platform, config, image_ami): + """Set up image. + + @param platform: platform object + @param config: image configuration + @param image_ami: string of image ami ID + """ + super(EC2Image, self).__init__(platform, config) + self._img_instance = None + self.image_ami = image_ami + + @property + def _instance(self): + """Internal use only, returns a running instance""" + if not self._img_instance: + self._img_instance = self.platform.create_instance( + self.properties, self.config, self.features, + self.image_ami, user_data=None) + self._img_instance.start(wait=True, wait_for_cloud_init=True) + return self._img_instance + + def destroy(self): + """Delete the instance used to create a custom image.""" + if self._img_instance: + LOG.debug('terminating backing instance %s', + self._img_instance.instance.instance_id) + self._img_instance.instance.terminate() + self._img_instance.instance.wait_until_terminated() + + super(EC2Image, self).destroy() + + def _execute(self, *args, **kwargs): + """Execute command in image, modifying image.""" + self._instance.start(wait=True) + return self._instance._execute(*args, **kwargs) + + def push_file(self, local_path, remote_path): + """Copy file at 'local_path' to instance at 'remote_path'.""" + self._instance.start(wait=True) + return self._instance.push_file(local_path, remote_path) + + def run_script(self, *args, **kwargs): + """Run script in image, modifying image. + + @return_value: script output + """ + self._instance.start(wait=True) + return self._instance.run_script(*args, **kwargs) + + def snapshot(self): + """Create snapshot of image, block until done. + + Will return base image_ami if no instance has been booted, otherwise + will run the clean script, shutdown the instance, create a custom + AMI, and use that AMI once available. + """ + if not self._img_instance: + return EC2Snapshot(self.platform, self.properties, self.config, + self.features, self.image_ami, + delete_on_destroy=False) + + if self.config.get('boot_clean_script'): + self._img_instance.run_script(self.config.get('boot_clean_script')) + + self._img_instance.shutdown(wait=True) + + LOG.debug('creating custom ami from instance %s', + self._img_instance.instance.instance_id) + response = self.platform.ec2_client.create_image( + Name='%s-%s' % (self.platform.tag, self.image_ami), + InstanceId=self._img_instance.instance.instance_id + ) + image_ami_edited = response['ImageId'] + + # Create image and wait until it is in the 'available' state + image = self.platform.ec2_resource.Image(image_ami_edited) + image.wait_until_exists() + waiter = self.platform.ec2_client.get_waiter('image_available') + waiter.wait(ImageIds=[image.id]) + image.reload() + + return EC2Snapshot(self.platform, self.properties, self.config, + self.features, image_ami_edited) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/ec2/instance.py b/tests/cloud_tests/platforms/ec2/instance.py new file mode 100644 index 00000000..ab6037b1 --- /dev/null +++ b/tests/cloud_tests/platforms/ec2/instance.py @@ -0,0 +1,132 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base EC2 instance.""" +import os + +import botocore + +from ..instances import Instance +from tests.cloud_tests import LOG, util + + +class EC2Instance(Instance): + """EC2 backed instance.""" + + platform_name = "ec2" + _ssh_client = None + + def __init__(self, platform, properties, config, features, + image_ami, user_data=None): + """Set up instance. + + @param platform: platform object + @param properties: dictionary of properties + @param config: dictionary of configuration values + @param features: dictionary of supported feature flags + @param image_ami: AWS AMI ID for image to use + @param user_data: test user-data to pass to instance + """ + super(EC2Instance, self).__init__( + platform, image_ami, properties, config, features) + + self.image_ami = image_ami + self.instance = None + self.user_data = user_data + self.ssh_ip = None + self.ssh_port = 22 + self.ssh_key_file = os.path.join( + platform.config['data_dir'], platform.config['private_key']) + self.ssh_pubkey_file = os.path.join( + platform.config['data_dir'], platform.config['public_key']) + + def console_log(self): + """Collect console log from instance. + + The console log is buffered and not always present, therefore + may return empty string. + """ + try: + # OutputBytes comes from platform._decode_console_output_as_bytes + response = self.instance.console_output() + return response['OutputBytes'] + except KeyError: + if 'Output' in response: + msg = ("'OutputBytes' did not exist in console_output() but " + "'Output' did: %s..." % response['Output'][0:128]) + raise util.PlatformError('console_log', msg) + return ('No Console Output [%s]' % self.instance).encode() + + def destroy(self): + """Clean up instance.""" + if self.instance: + LOG.debug('destroying instance %s', self.instance.id) + self.instance.terminate() + self.instance.wait_until_terminated() + + self._ssh_close() + + super(EC2Instance, self).destroy() + + def _execute(self, command, stdin=None, env=None): + """Execute command on instance.""" + env_args = [] + if env: + env_args = ['env'] + ["%s=%s" for k, v in env.items()] + + return self._ssh(['sudo'] + env_args + list(command), stdin=stdin) + + def start(self, wait=True, wait_for_cloud_init=False): + """Start instance on EC2 with the platfrom's VPC.""" + if self.instance: + if self.instance.state['Name'] == 'running': + return + + LOG.debug('starting instance %s', self.instance.id) + self.instance.start() + else: + LOG.debug('launching instance') + + args = { + 'ImageId': self.image_ami, + 'InstanceType': self.platform.instance_type, + 'KeyName': self.platform.key_name, + 'MaxCount': 1, + 'MinCount': 1, + 'SecurityGroupIds': [self.platform.security_group.id], + 'SubnetId': self.platform.subnet.id, + 'TagSpecifications': [{ + 'ResourceType': 'instance', + 'Tags': [{ + 'Key': 'Name', 'Value': self.platform.tag + }] + }], + } + + if self.user_data: + args['UserData'] = self.user_data + + try: + instances = self.platform.ec2_resource.create_instances(**args) + except botocore.exceptions.ClientError as error: + error_msg = error.response['Error']['Message'] + raise util.PlatformError('start', error_msg) + + self.instance = instances[0] + + LOG.debug('instance id: %s', self.instance.id) + if wait: + self.instance.wait_until_running() + self.instance.reload() + self.ssh_ip = self.instance.public_ip_address + self._wait_for_system(wait_for_cloud_init) + + def shutdown(self, wait=True): + """Shutdown instance.""" + LOG.debug('stopping instance %s', self.instance.id) + self.instance.stop() + + if wait: + self.instance.wait_until_stopped() + self.instance.reload() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py new file mode 100644 index 00000000..f188c27b --- /dev/null +++ b/tests/cloud_tests/platforms/ec2/platform.py @@ -0,0 +1,258 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base EC2 platform.""" +from datetime import datetime +import os + +import boto3 +import botocore +from botocore import session, handlers +import base64 + +from ..platforms import Platform +from .image import EC2Image +from .instance import EC2Instance +from tests.cloud_tests import LOG + + +class EC2Platform(Platform): + """EC2 test platform.""" + + platform_name = 'ec2' + ipv4_cidr = '192.168.1.0/20' + + def __init__(self, config): + """Set up platform.""" + super(EC2Platform, self).__init__(config) + # Used for unique VPC, SSH key, and custom AMI generation naming + self.tag = '%s-%s' % ( + config['tag'], datetime.now().strftime('%Y%m%d%H%M%S')) + self.instance_type = config['instance-type'] + + try: + b3session = get_session() + self.ec2_client = b3session.client('ec2') + self.ec2_resource = b3session.resource('ec2') + self.ec2_region = b3session.region_name + self.key_name = self._upload_public_key(config) + except botocore.exceptions.NoRegionError: + raise RuntimeError( + 'Please configure default region in $HOME/.aws/config') + except botocore.exceptions.NoCredentialsError: + raise RuntimeError( + 'Please configure ec2 credentials in $HOME/.aws/credentials') + + self.vpc = self._create_vpc() + self.internet_gateway = self._create_internet_gateway() + self.subnet = self._create_subnet() + self.routing_table = self._create_routing_table() + self.security_group = self._create_security_group() + + def create_instance(self, properties, config, features, + image_ami, user_data=None): + """Create an instance + + @param src_img_path: image path to launch from + @param properties: image properties + @param config: image configuration + @param features: image features + @param image_ami: string of image ami ID + @param user_data: test user-data to pass to instance + @return_value: cloud_tests.instances instance + """ + return EC2Instance(self, properties, config, features, + image_ami, user_data) + + def destroy(self): + """Delete SSH keys, terminate all instances, and delete VPC.""" + for instance in self.vpc.instances.all(): + LOG.debug('waiting for instance %s termination', instance.id) + instance.terminate() + instance.wait_until_terminated() + + if self.key_name: + LOG.debug('deleting SSH key %s', self.key_name) + self.ec2_client.delete_key_pair(KeyName=self.key_name) + + if self.security_group: + LOG.debug('deleting security group %s', self.security_group.id) + self.security_group.delete() + + if self.subnet: + LOG.debug('deleting subnet %s', self.subnet.id) + self.subnet.delete() + + if self.routing_table: + LOG.debug('deleting routing table %s', self.routing_table.id) + self.routing_table.delete() + + if self.internet_gateway: + LOG.debug('deleting internet gateway %s', self.internet_gateway.id) + self.internet_gateway.detach_from_vpc(VpcId=self.vpc.id) + self.internet_gateway.delete() + + if self.vpc: + LOG.debug('deleting vpc %s', self.vpc.id) + self.vpc.delete() + + def get_image(self, img_conf): + """Get image using specified image configuration. + + Hard coded for 'amd64' based images. + + @param img_conf: configuration for image + @return_value: cloud_tests.images instance + """ + if img_conf['root-store'] == 'ebs': + root_store = 'ssd' + elif img_conf['root-store'] == 'instance-store': + root_store = 'instance' + else: + raise RuntimeError('Unknown root-store type: %s' % + (img_conf['root-store'])) + + filters = [ + 'arch=%s' % 'amd64', + 'endpoint=https://ec2.%s.amazonaws.com' % self.ec2_region, + 'region=%s' % self.ec2_region, + 'release=%s' % img_conf['release'], + 'root_store=%s' % root_store, + 'virt=hvm', + ] + + LOG.debug('finding image using streams') + image = self._query_streams(img_conf, filters) + + try: + image_ami = image['id'] + except KeyError: + raise RuntimeError('No images found for %s!' % img_conf['release']) + + LOG.debug('found image: %s', image_ami) + image = EC2Image(self, img_conf, image_ami) + return image + + def _create_internet_gateway(self): + """Create Internet Gateway and assign to VPC.""" + LOG.debug('creating internet gateway') + internet_gateway = self.ec2_resource.create_internet_gateway() + internet_gateway.attach_to_vpc(VpcId=self.vpc.id) + self._tag_resource(internet_gateway) + + return internet_gateway + + def _create_routing_table(self): + """Update default routing table with internet gateway. + + This sets up internet access between the VPC via the internet gateway + by configuring routing tables for IPv4 and IPv6. + """ + LOG.debug('creating routing table') + route_table = self.vpc.create_route_table() + route_table.create_route(DestinationCidrBlock='0.0.0.0/0', + GatewayId=self.internet_gateway.id) + route_table.create_route(DestinationIpv6CidrBlock='::/0', + GatewayId=self.internet_gateway.id) + route_table.associate_with_subnet(SubnetId=self.subnet.id) + self._tag_resource(route_table) + + return route_table + + def _create_security_group(self): + """Enables ingress to default VPC security group.""" + LOG.debug('creating security group') + security_group = self.vpc.create_security_group( + GroupName=self.tag, Description='integration test security group') + security_group.authorize_ingress( + IpProtocol='-1', FromPort=-1, ToPort=-1, CidrIp='0.0.0.0/0') + self._tag_resource(security_group) + + return security_group + + def _create_subnet(self): + """Generate IPv4 and IPv6 subnets for use.""" + ipv6_cidr = self.vpc.ipv6_cidr_block_association_set[0][ + 'Ipv6CidrBlock'][:-2] + '64' + + LOG.debug('creating subnet with following ranges:') + LOG.debug('ipv4: %s', self.ipv4_cidr) + LOG.debug('ipv6: %s', ipv6_cidr) + subnet = self.vpc.create_subnet(CidrBlock=self.ipv4_cidr, + Ipv6CidrBlock=ipv6_cidr) + modify_subnet = subnet.meta.client.modify_subnet_attribute + modify_subnet(SubnetId=subnet.id, + MapPublicIpOnLaunch={'Value': True}) + self._tag_resource(subnet) + + return subnet + + def _create_vpc(self): + """Setup AWS EC2 VPC or return existing VPC.""" + LOG.debug('creating new vpc') + try: + vpc = self.ec2_resource.create_vpc( + CidrBlock=self.ipv4_cidr, + AmazonProvidedIpv6CidrBlock=True) + except botocore.exceptions.ClientError as e: + raise RuntimeError(e) + + vpc.wait_until_available() + self._tag_resource(vpc) + + return vpc + + def _tag_resource(self, resource): + """Tag a resource with the specified tag. + + This makes finding and deleting resources specific to this testing + much easier to find. + + @param resource: resource to tag + """ + tag = { + 'Key': 'Name', + 'Value': self.tag + } + resource.create_tags(Tags=[tag]) + + def _upload_public_key(self, config): + """Generate random name and upload SSH key with that name. + + @param config: platform config + @return: string of ssh key name + """ + key_file = os.path.join(config['data_dir'], config['public_key']) + with open(key_file, 'r') as file: + public_key = file.read().strip('\n') + + LOG.debug('uploading SSH key %s', self.tag) + self.ec2_client.import_key_pair(KeyName=self.tag, + PublicKeyMaterial=public_key) + + return self.tag + + +def _decode_console_output_as_bytes(parsed, **kwargs): + """Provide console output as bytes in OutputBytes. + + For this to be useful, the session has to have had the + decode_console_output handler unregistered already. + + https://github.com/boto/botocore/issues/1351 .""" + if 'Output' not in parsed: + return + orig = parsed['Output'] + handlers.decode_console_output(parsed, **kwargs) + parsed['OutputBytes'] = base64.b64decode(orig) + + +def get_session(): + mysess = session.get_session() + mysess.unregister('after-call.ec2.GetConsoleOutput', + handlers.decode_console_output) + mysess.register('after-call.ec2.GetConsoleOutput', + _decode_console_output_as_bytes) + return boto3.Session(botocore_session=mysess) + + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/ec2/snapshot.py b/tests/cloud_tests/platforms/ec2/snapshot.py new file mode 100644 index 00000000..2c48cb54 --- /dev/null +++ b/tests/cloud_tests/platforms/ec2/snapshot.py @@ -0,0 +1,66 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base EC2 snapshot.""" + +from ..snapshots import Snapshot +from tests.cloud_tests import LOG + + +class EC2Snapshot(Snapshot): + """EC2 image copy backed snapshot.""" + + platform_name = 'ec2' + + def __init__(self, platform, properties, config, features, image_ami, + delete_on_destroy=True): + """Set up snapshot. + + @param platform: platform object + @param properties: image properties + @param config: image config + @param features: supported feature flags + @param image_ami: string of image ami ID + @param delete_on_destroy: boolean to delete on destroy + """ + super(EC2Snapshot, self).__init__( + platform, properties, config, features) + + self.image_ami = image_ami + self.delete_on_destroy = delete_on_destroy + + def destroy(self): + """Deregister the backing AMI.""" + if self.delete_on_destroy: + image = self.platform.ec2_resource.Image(self.image_ami) + snapshot_id = image.block_device_mappings[0]['Ebs']['SnapshotId'] + + LOG.debug('removing custom ami %s', self.image_ami) + self.platform.ec2_client.deregister_image(ImageId=self.image_ami) + + LOG.debug('removing custom snapshot %s', snapshot_id) + self.platform.ec2_client.delete_snapshot(SnapshotId=snapshot_id) + + def launch(self, user_data, meta_data=None, block=True, start=True, + use_desc=None): + """Launch instance. + + @param user_data: user-data for the instance + @param meta_data: meta_data for the instance + @param block: wait until instance is created + @param start: start instance and wait until fully started + @param use_desc: string of test name + @return_value: an Instance + """ + if meta_data is not None: + raise ValueError("metadata not supported on Ec2") + + instance = self.platform.create_instance( + self.properties, self.config, self.features, + self.image_ami, user_data) + + if start: + instance.start() + + return instance + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/images.py b/tests/cloud_tests/platforms/images.py new file mode 100644 index 00000000..557a5cf6 --- /dev/null +++ b/tests/cloud_tests/platforms/images.py @@ -0,0 +1,57 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base class for images.""" + +from ..util import TargetBase + + +class Image(TargetBase): + """Base class for images.""" + + platform_name = None + + def __init__(self, platform, config): + """Set up image. + + @param platform: platform object + @param config: image configuration + """ + self.platform = platform + self.config = config + + def __str__(self): + """A brief description of the image.""" + return '-'.join((self.properties['os'], self.properties['release'])) + + @property + def properties(self): + """{} containing: 'arch', 'os', 'version', 'release'.""" + return {k: self.config[k] + for k in ('arch', 'os', 'release', 'version')} + + @property + def features(self): + """Feature flags supported by this image. + + @return_value: list of feature names + """ + return [k for k, v in self.config.get('features', {}).items() if v] + + @property + def setup_overrides(self): + """Setup options that need to be overridden for the image. + + @return_value: dictionary to update args with + """ + # NOTE: more sophisticated options may be requied at some point + return self.config.get('setup_overrides', {}) + + def snapshot(self): + """Create snapshot of image, block until done.""" + raise NotImplementedError + + def destroy(self): + """Clean up data associated with image.""" + pass + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/instances.py b/tests/cloud_tests/platforms/instances.py new file mode 100644 index 00000000..3bad021f --- /dev/null +++ b/tests/cloud_tests/platforms/instances.py @@ -0,0 +1,145 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base instance.""" +import time + +import paramiko +from paramiko.ssh_exception import ( + BadHostKeyException, AuthenticationException, SSHException) + +from ..util import TargetBase +from tests.cloud_tests import LOG, util + + +class Instance(TargetBase): + """Base instance object.""" + + platform_name = None + _ssh_client = None + + def __init__(self, platform, name, properties, config, features): + """Set up instance. + + @param platform: platform object + @param name: hostname of instance + @param properties: image properties + @param config: image config + @param features: supported feature flags + """ + self.platform = platform + self.name = name + self.properties = properties + self.config = config + self.features = features + self._tmp_count = 0 + + self.ssh_ip = None + self.ssh_port = None + self.ssh_key_file = None + self.ssh_username = 'ubuntu' + + def console_log(self): + """Instance console. + + @return_value: bytes of this instance’s console + """ + raise NotImplementedError + + def reboot(self, wait=True): + """Reboot instance.""" + raise NotImplementedError + + def shutdown(self, wait=True): + """Shutdown instance.""" + raise NotImplementedError + + def start(self, wait=True, wait_for_cloud_init=False): + """Start instance.""" + raise NotImplementedError + + def destroy(self): + """Clean up instance.""" + self._ssh_close() + + def _ssh(self, command, stdin=None): + """Run a command via SSH.""" + client = self._ssh_connect() + + cmd = util.shell_pack(command) + fp_in, fp_out, fp_err = client.exec_command(cmd) + channel = fp_in.channel + + if stdin is not None: + fp_in.write(stdin) + fp_in.close() + + channel.shutdown_write() + rc = channel.recv_exit_status() + + return (fp_out.read(), fp_err.read(), rc) + + def _ssh_close(self): + if self._ssh_client: + try: + self._ssh_client.close() + except SSHException: + LOG.warning('Failed to close SSH connection.') + self._ssh_client = None + + def _ssh_connect(self): + """Connect via SSH.""" + if self._ssh_client: + return self._ssh_client + + if not self.ssh_ip or not self.ssh_port: + raise ValueError + + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + private_key = paramiko.RSAKey.from_private_key_file(self.ssh_key_file) + + retries = 30 + while retries: + try: + client.connect(username=self.ssh_username, + hostname=self.ssh_ip, port=self.ssh_port, + pkey=private_key, banner_timeout=30) + self._ssh_client = client + return client + except (ConnectionRefusedError, AuthenticationException, + BadHostKeyException, ConnectionResetError, SSHException, + OSError) as e: + retries -= 1 + time.sleep(10) + + ssh_cmd = 'Failed ssh connection to %s@%s:%s after 300 seconds' % ( + self.ssh_username, self.ssh_ip, self.ssh_port + ) + raise util.InTargetExecuteError(b'', b'', 1, ssh_cmd, 'ssh') + + def _wait_for_system(self, wait_for_cloud_init): + """Wait until system has fully booted and cloud-init has finished. + + @param wait_time: maximum time to wait + @return_value: None, may raise OSError if wait_time exceeded + """ + def clean_test(test): + """Clean formatting for system ready test testcase.""" + return ' '.join(l for l in test.strip().splitlines() + if not l.lstrip().startswith('#')) + + time = self.config['boot_timeout'] + tests = [self.config['system_ready_script']] + if wait_for_cloud_init: + tests.append(self.config['cloud_init_ready_script']) + + formatted_tests = ' && '.join(clean_test(t) for t in tests) + cmd = ('i=0; while [ $i -lt {time} ] && i=$(($i+1)); do {test} && ' + 'exit 0; sleep 1; done; exit 1').format(time=time, + test=formatted_tests) + + if self.execute(cmd, rcs=(0, 1))[-1] != 0: + raise OSError('timeout: after {}s system not started'.format(time)) + + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/lxd/image.py b/tests/cloud_tests/platforms/lxd/image.py new file mode 100644 index 00000000..b5de1f52 --- /dev/null +++ b/tests/cloud_tests/platforms/lxd/image.py @@ -0,0 +1,193 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""LXD Image Base Class.""" + +import os +import shutil +import tempfile + +from ..images import Image +from .snapshot import LXDSnapshot +from cloudinit import util as c_util +from tests.cloud_tests import util + + +class LXDImage(Image): + """LXD backed image.""" + + platform_name = "lxd" + + def __init__(self, platform, config, pylxd_image): + """Set up image. + + @param platform: platform object + @param config: image configuration + """ + self.modified = False + self._img_instance = None + self._pylxd_image = None + self.pylxd_image = pylxd_image + super(LXDImage, self).__init__(platform, config) + + @property + def pylxd_image(self): + """Property function.""" + if self._pylxd_image: + self._pylxd_image.sync() + return self._pylxd_image + + @pylxd_image.setter + def pylxd_image(self, pylxd_image): + if self._img_instance: + self._instance.destroy() + self._img_instance = None + if (self._pylxd_image and + (self._pylxd_image is not pylxd_image) and + (not self.config.get('cache_base_image') or self.modified)): + self._pylxd_image.delete(wait=True) + self.modified = False + self._pylxd_image = pylxd_image + + @property + def _instance(self): + """Internal use only, returns a instance + + This starts an lxc instance from the image, so it is "dirty". + Better would be some way to modify this "at rest". + lxc-pstart would be an option.""" + if not self._img_instance: + self._img_instance = self.platform.launch_container( + self.properties, self.config, self.features, + use_desc='image-modification', image_desc=str(self), + image=self.pylxd_image.fingerprint) + self._img_instance.start() + return self._img_instance + + @property + def properties(self): + """{} containing: 'arch', 'os', 'version', 'release'.""" + properties = self.pylxd_image.properties + return { + 'arch': properties.get('architecture'), + 'os': properties.get('os'), + 'version': properties.get('version'), + 'release': properties.get('release'), + } + + def export_image(self, output_dir): + """Export image from lxd image store to (split) tarball on disk. + + @param output_dir: dir to store tarballs in + @return_value: tuple of path to metadata tarball and rootfs tarball + """ + # pylxd's image export feature doesn't do split exports, so use cmdline + c_util.subp(['lxc', 'image', 'export', self.pylxd_image.fingerprint, + output_dir], capture=True) + tarballs = [p for p in os.listdir(output_dir) if p.endswith('tar.xz')] + metadata = os.path.join( + output_dir, next(p for p in tarballs if p.startswith('meta-'))) + rootfs = os.path.join( + output_dir, next(p for p in tarballs if not p.startswith('meta-'))) + return (metadata, rootfs) + + def import_image(self, metadata, rootfs): + """Import image to lxd image store from (split) tarball on disk. + + Note, this will replace and delete the current pylxd_image + + @param metadata: metadata tarball + @param rootfs: rootfs tarball + @return_value: imported image fingerprint + """ + alias = util.gen_instance_name( + image_desc=str(self), use_desc='update-metadata') + c_util.subp(['lxc', 'image', 'import', metadata, rootfs, + '--alias', alias], capture=True) + self.pylxd_image = self.platform.query_image_by_alias(alias) + return self.pylxd_image.fingerprint + + def update_templates(self, template_config, template_data): + """Update the image's template configuration. + + Note, this will replace and delete the current pylxd_image + + @param template_config: config overrides for template metadata + @param template_data: template data to place into templates/ + """ + # set up tmp files + export_dir = tempfile.mkdtemp(prefix='cloud_test_util_') + extract_dir = tempfile.mkdtemp(prefix='cloud_test_util_') + new_metadata = os.path.join(export_dir, 'new-meta.tar.xz') + metadata_yaml = os.path.join(extract_dir, 'metadata.yaml') + template_dir = os.path.join(extract_dir, 'templates') + + try: + # extract old data + (metadata, rootfs) = self.export_image(export_dir) + shutil.unpack_archive(metadata, extract_dir) + + # update metadata + metadata = c_util.read_conf(metadata_yaml) + templates = metadata.get('templates', {}) + templates.update(template_config) + metadata['templates'] = templates + util.yaml_dump(metadata, metadata_yaml) + + # write out template files + for name, content in template_data.items(): + path = os.path.join(template_dir, name) + c_util.write_file(path, content) + + # store new data, mark new image as modified + util.flat_tar(new_metadata, extract_dir) + self.import_image(new_metadata, rootfs) + self.modified = True + + finally: + # remove tmpfiles + shutil.rmtree(export_dir) + shutil.rmtree(extract_dir) + + def _execute(self, *args, **kwargs): + """Execute command in image, modifying image.""" + return self._instance._execute(*args, **kwargs) + + def push_file(self, local_path, remote_path): + """Copy file at 'local_path' to instance at 'remote_path'.""" + return self._instance.push_file(local_path, remote_path) + + def run_script(self, *args, **kwargs): + """Run script in image, modifying image. + + @return_value: script output + """ + return self._instance.run_script(*args, **kwargs) + + def snapshot(self): + """Create snapshot of image, block until done.""" + # get empty user data to pass in to instance + # if overrides for user data provided, use them + empty_userdata = util.update_user_data( + {}, self.config.get('user_data_overrides', {})) + conf = {'user.user-data': empty_userdata} + # clone current instance + instance = self.platform.launch_container( + self.properties, self.config, self.features, + container=self._instance.name, image_desc=str(self), + use_desc='snapshot', container_config=conf) + # wait for cloud-init before boot_clean_script is run to ensure + # /var/lib/cloud is removed cleanly + instance.start(wait=True, wait_for_cloud_init=True) + if self.config.get('boot_clean_script'): + instance.run_script(self.config.get('boot_clean_script')) + # freeze current instance and return snapshot + instance.freeze() + return LXDSnapshot(self.platform, self.properties, self.config, + self.features, instance) + + def destroy(self): + """Clean up data associated with image.""" + self.pylxd_image = None + super(LXDImage, self).destroy() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py new file mode 100644 index 00000000..d2d2a1fd --- /dev/null +++ b/tests/cloud_tests/platforms/lxd/instance.py @@ -0,0 +1,152 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base LXD instance.""" + +import os +import shutil +from tempfile import mkdtemp + +from cloudinit.util import subp, ProcessExecutionError + +from ..instances import Instance + + +class LXDInstance(Instance): + """LXD container backed instance.""" + + platform_name = "lxd" + + def __init__(self, platform, name, properties, config, features, + pylxd_container): + """Set up instance. + + @param platform: platform object + @param name: hostname of instance + @param properties: image properties + @param config: image config + @param features: supported feature flags + """ + self._pylxd_container = pylxd_container + super(LXDInstance, self).__init__( + platform, name, properties, config, features) + self.tmpd = mkdtemp(prefix="%s-%s" % (type(self).__name__, name)) + self._setup_console_log() + self.name = name + + @property + def pylxd_container(self): + """Property function.""" + self._pylxd_container.sync() + return self._pylxd_container + + def _setup_console_log(self): + logf = os.path.join(self.tmpd, "console.log") + + # doing this ensures we can read it. Otherwise it ends up root:root. + with open(logf, "w") as fp: + fp.write("# %s\n" % self.name) + + cfg = "lxc.console.logfile=%s" % logf + orig = self._pylxd_container.config.get('raw.lxc', "") + if orig: + orig += "\n" + self._pylxd_container.config['raw.lxc'] = orig + cfg + self._pylxd_container.save() + self._console_log_file = logf + + def _execute(self, command, stdin=None, env=None): + if env is None: + env = {} + + env_args = [] + if env: + env_args = ['env'] + ["%s=%s" for k, v in env.items()] + + # ensure instance is running and execute the command + self.start() + + # Use cmdline client due to https://github.com/lxc/pylxd/issues/268 + exit_code = 0 + try: + stdout, stderr = subp( + ['lxc', 'exec', self.name, '--'] + env_args + list(command), + data=stdin, decode=False) + except ProcessExecutionError as e: + exit_code = e.exit_code + stdout = e.stdout + stderr = e.stderr + + return stdout, stderr, exit_code + + def read_data(self, remote_path, decode=False): + """Read data from instance filesystem. + + @param remote_path: path in instance + @param decode: decode data before returning. + @return_value: content of remote_path as bytes if 'decode' is False, + and as string if 'decode' is True. + """ + data = self.pylxd_container.files.get(remote_path) + return data.decode() if decode else data + + def write_data(self, remote_path, data): + """Write data to instance filesystem. + + @param remote_path: path in instance + @param data: data to write in bytes + """ + self.pylxd_container.files.put(remote_path, data) + + def console_log(self): + """Console log. + + @return_value: bytes of this instance’s console + """ + if not os.path.exists(self._console_log_file): + raise NotImplementedError( + "Console log '%s' does not exist. If this is a remote " + "lxc, then this is really NotImplementedError. If it is " + "A local lxc, then this is a RuntimeError." + "https://github.com/lxc/lxd/issues/1129") + with open(self._console_log_file, "rb") as fp: + return fp.read() + + def reboot(self, wait=True): + """Reboot instance.""" + self.shutdown(wait=wait) + self.start(wait=wait) + + def shutdown(self, wait=True): + """Shutdown instance.""" + if self.pylxd_container.status != 'Stopped': + self.pylxd_container.stop(wait=wait) + + def start(self, wait=True, wait_for_cloud_init=False): + """Start instance.""" + if self.pylxd_container.status != 'Running': + self.pylxd_container.start(wait=wait) + if wait: + self._wait_for_system(wait_for_cloud_init) + + def freeze(self): + """Freeze instance.""" + if self.pylxd_container.status != 'Frozen': + self.pylxd_container.freeze(wait=True) + + def unfreeze(self): + """Unfreeze instance.""" + if self.pylxd_container.status == 'Frozen': + self.pylxd_container.unfreeze(wait=True) + + def destroy(self): + """Clean up instance.""" + self.unfreeze() + self.shutdown() + self.pylxd_container.delete(wait=True) + if self.platform.container_exists(self.name): + raise OSError('container {} was not properly removed' + .format(self.name)) + shutil.rmtree(self.tmpd) + super(LXDInstance, self).destroy() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/lxd.py b/tests/cloud_tests/platforms/lxd/platform.py index ead0955b..6a016929 100644 --- a/tests/cloud_tests/platforms/lxd.py +++ b/tests/cloud_tests/platforms/lxd/platform.py @@ -4,15 +4,15 @@ from pylxd import (Client, exceptions) -from tests.cloud_tests.images import lxd as lxd_image -from tests.cloud_tests.instances import lxd as lxd_instance -from tests.cloud_tests.platforms import base +from ..platforms import Platform +from .image import LXDImage +from .instance import LXDInstance from tests.cloud_tests import util DEFAULT_SSTREAMS_SERVER = "https://images.linuxcontainers.org:8443" -class LXDPlatform(base.Platform): +class LXDPlatform(Platform): """LXD test platform.""" platform_name = 'lxd' @@ -33,7 +33,7 @@ class LXDPlatform(base.Platform): pylxd_image = self.client.images.create_from_simplestreams( img_conf.get('sstreams_server', DEFAULT_SSTREAMS_SERVER), img_conf['alias']) - image = lxd_image.LXDImage(self, img_conf, pylxd_image) + image = LXDImage(self, img_conf, pylxd_image) if img_conf.get('override_templates', False): image.update_templates(self.config.get('template_overrides', {}), self.config.get('template_files', {})) @@ -69,8 +69,8 @@ class LXDPlatform(base.Platform): 'source': ({'type': 'image', 'fingerprint': image} if image else {'type': 'copy', 'source': container}) }, wait=block) - return lxd_instance.LXDInstance(self, container.name, properties, - config, features, container) + return LXDInstance(self, container.name, properties, config, features, + container) def container_exists(self, container_name): """Check if container with name 'container_name' exists. diff --git a/tests/cloud_tests/platforms/lxd/snapshot.py b/tests/cloud_tests/platforms/lxd/snapshot.py new file mode 100644 index 00000000..b524644f --- /dev/null +++ b/tests/cloud_tests/platforms/lxd/snapshot.py @@ -0,0 +1,53 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base LXD snapshot.""" + +from ..snapshots import Snapshot + + +class LXDSnapshot(Snapshot): + """LXD image copy backed snapshot.""" + + platform_name = "lxd" + + def __init__(self, platform, properties, config, features, + pylxd_frozen_instance): + """Set up snapshot. + + @param platform: platform object + @param properties: image properties + @param config: image config + @param features: supported feature flags + """ + self.pylxd_frozen_instance = pylxd_frozen_instance + super(LXDSnapshot, self).__init__( + platform, properties, config, features) + + def launch(self, user_data, meta_data=None, block=True, start=True, + use_desc=None): + """Launch instance. + + @param user_data: user-data for the instance + @param instance_id: instance-id for the instance + @param block: wait until instance is created + @param start: start instance and wait until fully started + @param use_desc: description of snapshot instance use + @return_value: an Instance + """ + inst_config = {'user.user-data': user_data} + if meta_data: + inst_config['user.meta-data'] = meta_data + instance = self.platform.launch_container( + self.properties, self.config, self.features, block=block, + image_desc=str(self), container=self.pylxd_frozen_instance.name, + use_desc=use_desc, container_config=inst_config) + if start: + instance.start() + return instance + + def destroy(self): + """Clean up snapshot data.""" + self.pylxd_frozen_instance.destroy() + super(LXDSnapshot, self).destroy() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/nocloudkvm/image.py b/tests/cloud_tests/platforms/nocloudkvm/image.py new file mode 100644 index 00000000..bc2b6e75 --- /dev/null +++ b/tests/cloud_tests/platforms/nocloudkvm/image.py @@ -0,0 +1,79 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""NoCloud KVM Image Base Class.""" + +from cloudinit import util as c_util + +import os +import shutil +import tempfile + +from ..images import Image +from .snapshot import NoCloudKVMSnapshot + + +class NoCloudKVMImage(Image): + """NoCloud KVM backed image.""" + + platform_name = "nocloud-kvm" + + def __init__(self, platform, config, orig_img_path): + """Set up image. + + @param platform: platform object + @param config: image configuration + @param img_path: path to the image + """ + self.modified = False + self._workd = tempfile.mkdtemp(prefix='NoCloudKVMImage') + self._orig_img_path = orig_img_path + self._img_path = os.path.join(self._workd, + os.path.basename(self._orig_img_path)) + + c_util.subp(['qemu-img', 'create', '-f', 'qcow2', + '-b', orig_img_path, self._img_path]) + + super(NoCloudKVMImage, self).__init__(platform, config) + + def _execute(self, command, stdin=None, env=None): + """Execute command in image, modifying image.""" + return self.mount_image_callback(command, stdin=stdin, env=env) + + def mount_image_callback(self, command, stdin=None, env=None): + """Run mount-image-callback.""" + + env_args = [] + if env: + env_args = ['env'] + ["%s=%s" for k, v in env.items()] + + mic_chroot = ['sudo', 'mount-image-callback', '--system-mounts', + '--system-resolvconf', self._img_path, + '--', 'chroot', '_MOUNTPOINT_'] + try: + out, err = c_util.subp(mic_chroot + env_args + list(command), + data=stdin, decode=False) + return (out, err, 0) + except c_util.ProcessExecutionError as e: + return (e.stdout, e.stderr, e.exit_code) + + def snapshot(self): + """Create snapshot of image, block until done.""" + if not self._img_path: + raise RuntimeError() + + return NoCloudKVMSnapshot(self.platform, self.properties, self.config, + self.features, self._img_path) + + def destroy(self): + """Unset path to signal image is no longer used. + + The removal of the images and all other items is handled by the + framework. In some cases we want to keep the images, so let the + framework decide whether to keep or destroy everything. + """ + self._img_path = None + shutil.rmtree(self._workd) + + super(NoCloudKVMImage, self).destroy() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/nocloudkvm/instance.py b/tests/cloud_tests/platforms/nocloudkvm/instance.py new file mode 100644 index 00000000..932dc0fa --- /dev/null +++ b/tests/cloud_tests/platforms/nocloudkvm/instance.py @@ -0,0 +1,192 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base NoCloud KVM instance.""" + +import copy +import os +import socket +import subprocess +import time +import uuid + +from ..instances import Instance +from cloudinit.atomic_helper import write_json +from cloudinit import util as c_util +from tests.cloud_tests import LOG, util + +# This domain contains reverse lookups for hostnames that are used. +# The primary reason is so sudo will return quickly when it attempts +# to look up the hostname. i9n is just short for 'integration'. +# see also bug 1730744 for why we had to do this. +CI_DOMAIN = "i9n.cloud-init.io" + + +class NoCloudKVMInstance(Instance): + """NoCloud KVM backed instance.""" + + platform_name = "nocloud-kvm" + + def __init__(self, platform, name, image_path, properties, config, + features, user_data, meta_data): + """Set up instance. + + @param platform: platform object + @param name: image path + @param image_path: path to disk image to boot. + @param properties: dictionary of properties + @param config: dictionary of configuration values + @param features: dictionary of supported feature flags + """ + super(NoCloudKVMInstance, self).__init__( + platform, name, properties, config, features + ) + + self.user_data = user_data + if meta_data: + meta_data = copy.deepcopy(meta_data) + else: + meta_data = {} + + if 'instance-id' in meta_data: + iid = meta_data['instance-id'] + else: + iid = str(uuid.uuid1()) + meta_data['instance-id'] = iid + + self.instance_id = iid + self.ssh_key_file = os.path.join( + platform.config['data_dir'], platform.config['private_key']) + self.ssh_pubkey_file = os.path.join( + platform.config['data_dir'], platform.config['public_key']) + + self.ssh_pubkey = None + if self.ssh_pubkey_file: + with open(self.ssh_pubkey_file, "r") as fp: + self.ssh_pubkey = fp.read().rstrip('\n') + + if not meta_data.get('public-keys'): + meta_data['public-keys'] = [] + meta_data['public-keys'].append(self.ssh_pubkey) + + self.ssh_ip = '127.0.0.1' + self.ssh_port = None + self.pid = None + self.pid_file = None + self.console_file = None + self.disk = image_path + self.meta_data = meta_data + + def shutdown(self, wait=True): + """Shutdown instance.""" + + if self.pid: + # This relies on _execute which uses sudo over ssh. The ssh + # connection would get killed before sudo exited, so ignore errors. + cmd = ['shutdown', 'now'] + try: + self._execute(cmd) + except util.InTargetExecuteError: + pass + self._ssh_close() + + if wait: + LOG.debug("Executed shutdown. waiting on pid %s to end", + self.pid) + time_for_shutdown = 120 + give_up_at = time.time() + time_for_shutdown + pid_file_path = '/proc/%s' % self.pid + msg = ("pid %s did not exit in %s seconds after shutdown." % + (self.pid, time_for_shutdown)) + while True: + if not os.path.exists(pid_file_path): + break + if time.time() > give_up_at: + raise util.PlatformError("shutdown", msg) + self.pid = None + + def destroy(self): + """Clean up instance.""" + if self.pid: + try: + c_util.subp(['kill', '-9', self.pid]) + except util.ProcessExectuionError: + pass + + if self.pid_file: + os.remove(self.pid_file) + + self.pid = None + self._ssh_close() + + super(NoCloudKVMInstance, self).destroy() + + def _execute(self, command, stdin=None, env=None): + env_args = [] + if env: + env_args = ['env'] + ["%s=%s" for k, v in env.items()] + + return self._ssh(['sudo'] + env_args + list(command), stdin=stdin) + + def generate_seed(self, tmpdir): + """Generate nocloud seed from user-data""" + seed_file = os.path.join(tmpdir, '%s_seed.img' % self.name) + user_data_file = os.path.join(tmpdir, '%s_user_data' % self.name) + meta_data_file = os.path.join(tmpdir, '%s_meta_data' % self.name) + + with open(user_data_file, "w") as ud_file: + ud_file.write(self.user_data) + + # meta-data can be yaml, but more easily pretty printed with json + write_json(meta_data_file, self.meta_data) + c_util.subp(['cloud-localds', seed_file, user_data_file, + meta_data_file]) + + return seed_file + + def get_free_port(self): + """Get a free port assigned by the kernel.""" + s = socket.socket() + s.bind(('', 0)) + num = s.getsockname()[1] + s.close() + return num + + def start(self, wait=True, wait_for_cloud_init=False): + """Start instance.""" + tmpdir = self.platform.config['data_dir'] + seed = self.generate_seed(tmpdir) + self.pid_file = os.path.join(tmpdir, '%s.pid' % self.name) + self.console_file = os.path.join(tmpdir, '%s-console.log' % self.name) + self.ssh_port = self.get_free_port() + + cmd = ['./tools/xkvm', + '--disk', '%s,cache=unsafe' % self.disk, + '--disk', '%s,cache=unsafe' % seed, + '--netdev', ','.join(['user', + 'hostfwd=tcp::%s-:22' % self.ssh_port, + 'dnssearch=%s' % CI_DOMAIN]), + '--', '-pidfile', self.pid_file, '-vnc', 'none', + '-m', '2G', '-smp', '2', '-nographic', + '-serial', 'file:' + self.console_file] + subprocess.Popen(cmd, + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + while not os.path.exists(self.pid_file): + time.sleep(1) + + with open(self.pid_file, 'r') as pid_f: + self.pid = pid_f.readlines()[0].strip() + + if wait: + self._wait_for_system(wait_for_cloud_init) + + def console_log(self): + if not self.console_file: + return b'' + with open(self.console_file, "rb") as fp: + return fp.read() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/nocloudkvm.py b/tests/cloud_tests/platforms/nocloudkvm/platform.py index 76cd83ad..a7e6f5de 100644 --- a/tests/cloud_tests/platforms/nocloudkvm.py +++ b/tests/cloud_tests/platforms/nocloudkvm/platform.py @@ -9,18 +9,22 @@ from simplestreams import mirrors from simplestreams import objectstores from simplestreams import util as s_util +from ..platforms import Platform +from .image import NoCloudKVMImage +from .instance import NoCloudKVMInstance from cloudinit import util as c_util -from tests.cloud_tests.images import nocloudkvm as nocloud_kvm_image -from tests.cloud_tests.instances import nocloudkvm as nocloud_kvm_instance -from tests.cloud_tests.platforms import base from tests.cloud_tests import util -class NoCloudKVMPlatform(base.Platform): +class NoCloudKVMPlatform(Platform): """NoCloud KVM test platform.""" platform_name = 'nocloud-kvm' + def __init__(self, config): + """Set up platform.""" + super(NoCloudKVMPlatform, self).__init__(config) + def get_image(self, img_conf): """Get image using specified image configuration. @@ -62,7 +66,7 @@ class NoCloudKVMPlatform(base.Platform): "Multiple images found in '%s': %s" % (search_d, ' '.join(images))) - image = nocloud_kvm_image.NoCloudKVMImage(self, img_conf, images[0]) + image = NoCloudKVMImage(self, img_conf, images[0]) return image def create_instance(self, properties, config, features, @@ -83,9 +87,7 @@ class NoCloudKVMPlatform(base.Platform): c_util.subp(['qemu-img', 'create', '-f', 'qcow2', '-b', src_img_path, img_path]) - return nocloud_kvm_instance.NoCloudKVMInstance(self, name, img_path, - properties, config, - features, user_data, - meta_data) + return NoCloudKVMInstance(self, name, img_path, properties, config, + features, user_data, meta_data) # vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/nocloudkvm/snapshot.py b/tests/cloud_tests/platforms/nocloudkvm/snapshot.py new file mode 100644 index 00000000..2dae3590 --- /dev/null +++ b/tests/cloud_tests/platforms/nocloudkvm/snapshot.py @@ -0,0 +1,59 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base NoCloud KVM snapshot.""" +import os +import shutil +import tempfile + +from ..snapshots import Snapshot + + +class NoCloudKVMSnapshot(Snapshot): + """NoCloud KVM image copy backed snapshot.""" + + platform_name = "nocloud-kvm" + + def __init__(self, platform, properties, config, features, image_path): + """Set up snapshot. + + @param platform: platform object + @param properties: image properties + @param config: image config + @param features: supported feature flags + @param image_path: image file to snapshot. + """ + self._workd = tempfile.mkdtemp(prefix='NoCloudKVMSnapshot') + snapshot = os.path.join(self._workd, 'snapshot') + shutil.copyfile(image_path, snapshot) + self._image_path = snapshot + + super(NoCloudKVMSnapshot, self).__init__( + platform, properties, config, features) + + def launch(self, user_data, meta_data=None, block=True, start=True, + use_desc=None): + """Launch instance. + + @param user_data: user-data for the instance + @param instance_id: instance-id for the instance + @param block: wait until instance is created + @param start: start instance and wait until fully started + @param use_desc: description of snapshot instance use + @return_value: an Instance + """ + instance = self.platform.create_instance( + self.properties, self.config, self.features, + self._image_path, image_desc=str(self), use_desc=use_desc, + user_data=user_data, meta_data=meta_data) + + if start: + instance.start() + + return instance + + def destroy(self): + """Clean up snapshot data.""" + shutil.rmtree(self._workd) + super(NoCloudKVMSnapshot, self).destroy() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py new file mode 100644 index 00000000..1542b3be --- /dev/null +++ b/tests/cloud_tests/platforms/platforms.py @@ -0,0 +1,96 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base platform class.""" +import os + +from simplestreams import filters, mirrors +from simplestreams import util as s_util + +from cloudinit import util as c_util + + +class Platform(object): + """Base class for platforms.""" + + platform_name = None + + def __init__(self, config): + """Set up platform.""" + self.config = config + self._generate_ssh_keys(config['data_dir']) + + def get_image(self, img_conf): + """Get image using specified image configuration. + + @param img_conf: configuration for image + @return_value: cloud_tests.images instance + """ + raise NotImplementedError + + def destroy(self): + """Clean up platform data.""" + pass + + def _generate_ssh_keys(self, data_dir): + """Generate SSH keys to be used with image.""" + filename = os.path.join(data_dir, self.config['private_key']) + + if os.path.exists(filename): + c_util.del_file(filename) + + c_util.subp(['ssh-keygen', '-t', 'rsa', '-b', '4096', + '-f', filename, '-P', '', + '-C', 'ubuntu@cloud_test'], + capture=True) + + @staticmethod + def _query_streams(img_conf, img_filter): + """Query streams for latest image given a specific filter. + + @param img_conf: configuration for image + @param filters: array of filters as strings format 'key=value' + @return: dictionary with latest image information or empty + """ + def policy(content, path): + return s_util.read_signed(content, keyring=img_conf['keyring']) + + (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) + smirror = mirrors.UrlMirrorReader(url, policy=policy) + + config = {'max_items': 1, 'filters': filters.get_filters(img_filter)} + tmirror = FilterMirror(config) + tmirror.sync(smirror, path) + + try: + return tmirror.json_entries[0] + except IndexError: + raise RuntimeError('no images found with filter: %s' % img_filter) + + +class FilterMirror(mirrors.BasicMirrorWriter): + """Taken from sstream-query to return query result as json array.""" + + def __init__(self, config=None): + super(FilterMirror, self).__init__(config=config) + if config is None: + config = {} + self.config = config + self.filters = config.get('filters', []) + self.json_entries = [] + + def load_products(self, path=None, content_id=None): + return {'content_id': content_id, 'products': {}} + + def filter_item(self, data, src, target, pedigree): + return filters.filter_item(self.filters, data, src, pedigree) + + def insert_item(self, data, src, target, pedigree, contentsource): + # src and target are top level products:1.0 + # data is src['products'][ped[0]]['versions'][ped[1]]['items'][ped[2]] + # contentsource is a ContentSource if 'path' exists in data or None + data = s_util.products_exdata(src, pedigree) + if 'path' in data: + data.update({'item_url': contentsource.url}) + self.json_entries.append(data) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/snapshots.py b/tests/cloud_tests/platforms/snapshots.py new file mode 100644 index 00000000..94328982 --- /dev/null +++ b/tests/cloud_tests/platforms/snapshots.py @@ -0,0 +1,45 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base snapshot.""" + + +class Snapshot(object): + """Base class for snapshots.""" + + platform_name = None + + def __init__(self, platform, properties, config, features): + """Set up snapshot. + + @param platform: platform object + @param properties: image properties + @param config: image config + @param features: supported feature flags + """ + self.platform = platform + self.properties = properties + self.config = config + self.features = features + + def __str__(self): + """A brief description of the snapshot.""" + return '-'.join((self.properties['os'], self.properties['release'])) + + def launch(self, user_data, meta_data=None, block=True, start=True, + use_desc=None): + """Launch instance. + + @param user_data: user-data for the instance + @param instance_id: instance-id for the instance + @param block: wait until instance is created + @param start: start instance and wait until fully started + @param use_desc: description of snapshot instance use + @return_value: an Instance + """ + raise NotImplementedError + + def destroy(self): + """Clean up snapshot data.""" + pass + +# vi: ts=4 expandtab |